thomas/jobbi-bewerbung
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a job-offer blacklist with URL-based de-duplication

Browse Source 
Offers are now de-duplicated by normalized URL (tracking params stripped)
in addition to (quelle, external_id), so the same posting never lands
twice — even re-scraped under a new id. Deleting an offer (web or API)
auto-blacklists it, so it can never reappear.

lib/blacklist.js provides shared normalization + matching. Manual entries
can block a URL, a whole domain, a company, or a company+title posting
(gender-marker tolerant). New /blacklist page lists and manages entries.

REST API: GET/POST /joboffers/blacklist, DELETE /joboffers/blacklist/{id};
POST /joboffers returns 409 when blacklisted; DELETE /joboffers/{id}
auto-blacklists (opt out with ?blacklist=false). Swagger updated with the
new paths and schemas.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
thomas
2026-07-04 13:33:12 +02:00
co-authored by Claude Opus 4.8
parent 1655427b4c
commit dae077486f
7 changed files with 799 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/api.js
+115 -16
View File
@@ -8,6 +8,7 @@
const express = require('express');
const path = require('path');
const fs = require('fs');
const blacklist = require('./blacklist');
const STATUS_OPTIONS = [
'Entwurf', 'Gesendet', 'Eingangsbestätigung', 'Vorstellungsgespräch',
@@ -517,6 +518,62 @@ function createExternalApi(deps) {
}
});
// --- Blacklist (must be registered BEFORE /joboffers/:id) ----------
// Blocks offers from ever (re)appearing. See lib/blacklist for the match
// rules. A deleted offer is auto-blacklisted so it can never return.
router.get('/joboffers/blacklist', async (req, res) => {
try {
const rows = await dbAll(
'SELECT * FROM jobangebote_blacklist ORDER BY created_at DESC, id DESC'
);
res.json(rows);
} catch (error) {
console.error('API list blacklist error:', error);
res.status(500).json({ error: 'Serverfehler' });
}
});
router.post('/joboffers/blacklist', async (req, res) => {
try {
const b = req.body || {};
const entry = blacklist.buildManualEntry({
typ: b.typ,
wert: b.wert != null ? sanitizeInput(String(b.wert)) : '',
firma: sanitizeInput(b.firma || ''),
stelle: sanitizeInput(b.stelle || ''),
ort: sanitizeInput(b.ort || ''),
grund: sanitizeInput(b.grund || ''),
});
if (!entry) {
return res.status(400).json({
error: `Ungültiger Blacklist-Eintrag. typ muss eine von ${blacklist.TYPES.join(', ')} sein und einen passenden Wert haben.`,
});
}
const cols = blacklist.COLUMNS;
const result = await dbRun(
`INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`,
cols.map((c) => (entry[c] === undefined ? null : entry[c]))
);
const row = await dbGet('SELECT * FROM jobangebote_blacklist WHERE id = ?', [result.lastID]);
res.status(201).json({ success: true, entry: row });
} catch (error) {
console.error('API create blacklist entry error:', error);
res.status(500).json({ error: 'Serverfehler' });
}
});
router.delete('/joboffers/blacklist/:id', async (req, res) => {
try {
const row = await dbGet('SELECT id FROM jobangebote_blacklist WHERE id = ?', [req.params.id]);
if (!row) return res.status(404).json({ error: 'Blacklist-Eintrag nicht gefunden' });
await dbRun('DELETE FROM jobangebote_blacklist WHERE id = ?', [req.params.id]);
res.json({ success: true });
} catch (error) {
console.error('API delete blacklist entry error:', error);
res.status(500).json({ error: 'Serverfehler' });
}
});
router.get('/joboffers/:id', async (req, res) => {
try {
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]);
@@ -534,10 +591,25 @@ function createExternalApi(deps) {
if (!b.firma || !b.stelle) {
return res.status(400).json({ error: 'firma und stelle sind erforderlich.' });
}
// Reject blacklisted offers outright — they must never reappear.
const blacklistRows = await dbAll('SELECT * FROM jobangebote_blacklist');
const blocked = blacklist.matchBlacklist(blacklistRows, b);
if (blocked) {
return res.status(409).json({
blacklisted: true,
matched_by: blocked.typ,
blacklist_entry: blocked,
error: 'Dieses Jobangebot steht auf der Blacklist und wird nicht (erneut) aufgenommen.',
});
}
const quelle = sanitizeInput(b.quelle || 'drittanbieter');
const externalId = b.external_id != null ? sanitizeInput(String(b.external_id)) : null;
const urlNorm = blacklist.normalizeUrl(b.quelle_url || '') || null;
const anzeigeDatum = sanitizeInput(b.anzeige_datum || '');
const kontaktEmail = sanitizeInput(b.kontakt_email || '');
// Column order matches the UPDATE/INSERT statements below (url_norm last).
const fields = [
sanitizeInput(b.firma),
sanitizeInput(b.stelle),
@@ -549,30 +621,40 @@ function createExternalApi(deps) {
anzeigeDatum,
kontaktEmail,
sanitizeInput(b.status || 'offen'),
urlNorm,
];
// Upsert: a matching (quelle, external_id) row is updated, else inserted.
// De-dup: prefer a (quelle, external_id) match, else the same normalized
// URL — so the same posting never lands twice, even with a new id.
let existing = null;
if (externalId) {
const existing = await dbGet(
existing = await dbGet(
'SELECT id FROM jobangebote WHERE quelle = ? AND external_id = ?',
[quelle, externalId]
);
if (existing) {
await dbRun(
`UPDATE jobangebote SET firma = ?, stelle = ?, ort = ?, gehalt = ?,
beschreibung = ?, quelle_url = ?, art = ?, anzeige_datum = ?, kontakt_email = ?, status = ?,
updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
[...fields, existing.id]
);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [existing.id]);
return res.json({ success: true, action: 'updated', joboffer: row });
}
}
if (!existing && urlNorm) {
existing = await dbGet(
'SELECT id FROM jobangebote WHERE url_norm = ? ORDER BY id ASC LIMIT 1',
[urlNorm]
);
}
if (existing) {
await dbRun(
`UPDATE jobangebote SET firma = ?, stelle = ?, ort = ?, gehalt = ?,
beschreibung = ?, quelle_url = ?, art = ?, anzeige_datum = ?, kontakt_email = ?, status = ?,
url_norm = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
[...fields, existing.id]
);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [existing.id]);
return res.json({ success: true, action: 'updated', joboffer: row });
}
const result = await dbRun(
`INSERT INTO jobangebote
(external_id, quelle, firma, stelle, ort, gehalt, beschreibung, quelle_url, art, anzeige_datum, kontakt_email, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
(external_id, quelle, firma, stelle, ort, gehalt, beschreibung, quelle_url, art, anzeige_datum, kontakt_email, status, url_norm)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[externalId, quelle, ...fields]
);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [result.lastID]);
@@ -583,12 +665,29 @@ function createExternalApi(deps) {
}
});
// Delete a job offer. By default it is auto-blacklisted first so it can never
// be ingested again; pass ?blacklist=false to hard-delete without blocking.
router.delete('/joboffers/:id', async (req, res) => {
try {
const row = await dbGet('SELECT id FROM jobangebote WHERE id = ?', [req.params.id]);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]);
if (!row) return res.status(404).json({ error: 'Jobangebot nicht gefunden' });
const skipBlacklist = req.query.blacklist === 'false' || req.query.blacklist === '0';
let blacklisted = false;
if (!skipBlacklist) {
const rows = await dbAll('SELECT * FROM jobangebote_blacklist');
if (!blacklist.matchBlacklist(rows, row)) {
const cols = blacklist.COLUMNS;
const entry = blacklist.buildAutoEntry(row, 'Jobangebot gelöscht (REST-API)');
await dbRun(
`INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`,
cols.map((c) => (entry[c] === undefined ? null : entry[c]))
);
}
blacklisted = true;
}
await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]);
res.json({ success: true });
res.json({ success: true, blacklisted });
} catch (error) {
console.error('API delete job offer error:', error);
res.status(500).json({ error: 'Serverfehler' });