thomas/jobbi-bewerbung
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a job-offer blacklist with URL-based de-duplication

Browse Source 
Offers are now de-duplicated by normalized URL (tracking params stripped)
in addition to (quelle, external_id), so the same posting never lands
twice — even re-scraped under a new id. Deleting an offer (web or API)
auto-blacklists it, so it can never reappear.

lib/blacklist.js provides shared normalization + matching. Manual entries
can block a URL, a whole domain, a company, or a company+title posting
(gender-marker tolerant). New /blacklist page lists and manages entries.

REST API: GET/POST /joboffers/blacklist, DELETE /joboffers/blacklist/{id};
POST /joboffers returns 409 when blacklisted; DELETE /joboffers/{id}
auto-blacklists (opt out with ?blacklist=false). Swagger updated with the
new paths and schemas.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
thomas
2026-07-04 13:33:12 +02:00
co-authored by Claude Opus 4.8
parent 1655427b4c
commit dae077486f
7 changed files with 799 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server.js
+109 -2
View File
@@ -31,6 +31,7 @@ const { generateApplicationDocuments, generateEmailReply } = require('./lib/docu
const mailer = require('./lib/mailer');
const { createExternalApi } = require('./lib/api');
const { buildOpenApiSpec } = require('./lib/openapi');
const blacklist = require('./lib/blacklist');
const app = express();
const PORT = process.env.PORT || 3000;
@@ -244,6 +245,30 @@ function dbRun(sql, params = []) {
});
}
// ---------------------------------------------------------------------------
// Job-offer blacklist helpers (shared shape with lib/api.js)
// ---------------------------------------------------------------------------
// Insert one prepared blacklist entry (see lib/blacklist.buildManual/AutoEntry).
async function insertBlacklistEntry(entry) {
const cols = blacklist.COLUMNS;
const placeholders = cols.map(() => '?').join(', ');
const values = cols.map((c) => (entry[c] === undefined ? null : entry[c]));
return dbRun(
`INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${placeholders})`,
values
);
}
// Auto-blacklist an offer row so it can never be ingested again, then it is safe
// to delete. Skips silently if the offer is already covered by an entry.
async function autoBlacklistOffer(offer, grund) {
if (!offer) return;
const rows = await dbAll('SELECT * FROM jobangebote_blacklist');
if (blacklist.matchBlacklist(rows, offer)) return; // already blocked
await insertBlacklistEntry(blacklist.buildAutoEntry(offer, grund));
}
// ---------------------------------------------------------------------------
// E-Mail correspondence: IMAP polling, storing & matching incoming replies
// ---------------------------------------------------------------------------
@@ -632,6 +657,39 @@ function initializeDatabase() {
// Migration: add columns to pre-existing jobangebote tables.
db.run('ALTER TABLE jobangebote ADD COLUMN anzeige_datum DATE', () => {});
db.run('ALTER TABLE jobangebote ADD COLUMN kontakt_email TEXT', () => {});
// Normalized URL for URL-based de-duplication of offers (see lib/blacklist).
db.run('ALTER TABLE jobangebote ADD COLUMN url_norm TEXT', () => {
// Backfill url_norm for rows ingested before this column existed.
db.all('SELECT id, quelle_url FROM jobangebote WHERE url_norm IS NULL AND quelle_url IS NOT NULL AND quelle_url != ""', (err, rows) => {
if (err || !rows) return;
rows.forEach((r) => {
const norm = blacklist.normalizeUrl(r.quelle_url);
if (norm) db.run('UPDATE jobangebote SET url_norm = ? WHERE id = ?', [norm, r.id], () => {});
});
});
});
db.run('CREATE INDEX IF NOT EXISTS idx_jobangebote_url_norm ON jobangebote(url_norm)', () => {});
// Blacklist of job offers that must never (re)appear in the list. A
// deleted offer is auto-blacklisted; manual entries can block a URL,
// a whole domain, a company, or a specific company+title posting.
db.run(`
CREATE TABLE IF NOT EXISTS jobangebote_blacklist (
id INTEGER PRIMARY KEY AUTOINCREMENT,
typ TEXT NOT NULL DEFAULT 'auto',
url_norm TEXT,
domain TEXT,
quelle TEXT,
external_id TEXT,
firma_norm TEXT,
stelle_norm TEXT,
ort_norm TEXT,
firma TEXT,
stelle TEXT,
quelle_url TEXT,
grund TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`, () => {});
// Remember the last recipient address per application (prefill).
db.run('ALTER TABLE bewerbungen ADD COLUMN email_empfaenger TEXT', () => {});
@@ -1676,10 +1734,16 @@ initializeDatabase().then(() => {
}
});
// Delete a job offer (cascades nothing — verknuepfte_bewerbung_id is SET NULL).
// Delete a job offer. Deleting always blacklists it first, so the same offer
// can never be ingested/listed again (the requirement: never appears twice,
// even after deletion). The entry can be removed later on /blacklist.
app.post('/jobangebote/:id/delete', async (req, res) => {
try {
await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]);
const offer = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]);
if (offer) {
await autoBlacklistOffer(offer, 'Jobangebot gelöscht (Web-UI)');
await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]);
}
res.redirect('/jobangebote');
} catch (error) {
console.error('Error deleting job offer:', error);
@@ -1687,6 +1751,49 @@ initializeDatabase().then(() => {
}
});
// Blacklist management page — see what is blocked and remove entries.
app.get('/blacklist', async (req, res) => {
try {
const eintraege = await dbAll(
'SELECT * FROM jobangebote_blacklist ORDER BY created_at DESC, id DESC'
);
res.render('blacklist', { eintraege, blacklistTypen: blacklist.TYPES, hideSettings: false });
} catch (error) {
console.error('Error listing blacklist:', error);
res.status(500).send('Serverfehler');
}
});
// Add a manual blacklist entry (URL / domain / company / company+title).
app.post('/blacklist', async (req, res) => {
try {
const entry = blacklist.buildManualEntry({
typ: req.body.typ,
wert: sanitizeInput(req.body.wert || ''),
firma: sanitizeInput(req.body.firma || ''),
stelle: sanitizeInput(req.body.stelle || ''),
ort: sanitizeInput(req.body.ort || ''),
grund: sanitizeInput(req.body.grund || ''),
});
if (entry) await insertBlacklistEntry(entry);
res.redirect('/blacklist');
} catch (error) {
console.error('Error adding blacklist entry:', error);
res.status(500).send('Serverfehler');
}
});
// Remove a blacklist entry (offer may then be ingested again).
app.post('/blacklist/:id/delete', async (req, res) => {
try {
await dbRun('DELETE FROM jobangebote_blacklist WHERE id = ?', [req.params.id]);
res.redirect('/blacklist');
} catch (error) {
console.error('Error deleting blacklist entry:', error);
res.status(500).send('Serverfehler');
}
});
// ----- Third-party REST API (/api/v1) + OpenAPI/Swagger -----
// API key for third-party software. When unset, the API responds 503 on
// every endpoint except /health — it never silently exposes data.