From dae077486faf1edea7a9061f065132e8d087fc84 Mon Sep 17 00:00:00 2001 From: Thomas Hackner Date: Sat, 4 Jul 2026 13:33:12 +0200 Subject: [PATCH] Add a job-offer blacklist with URL-based de-duplication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Offers are now de-duplicated by normalized URL (tracking params stripped) in addition to (quelle, external_id), so the same posting never lands twice — even re-scraped under a new id. Deleting an offer (web or API) auto-blacklists it, so it can never reappear. lib/blacklist.js provides shared normalization + matching. Manual entries can block a URL, a whole domain, a company, or a company+title posting (gender-marker tolerant). New /blacklist page lists and manages entries. REST API: GET/POST /joboffers/blacklist, DELETE /joboffers/blacklist/{id}; POST /joboffers returns 409 when blacklisted; DELETE /joboffers/{id} auto-blacklists (opt out with ?blacklist=false). Swagger updated with the new paths and schemas. Co-Authored-By: Claude Opus 4.8 --- lib/api.js | 131 ++++++++++++++++++++--- lib/blacklist.js | 218 ++++++++++++++++++++++++++++++++++++++ lib/openapi.js | 145 ++++++++++++++++++++++++- server.js | 111 ++++++++++++++++++- views/blacklist.ejs | 196 ++++++++++++++++++++++++++++++++++ views/jobangebote.ejs | 16 ++- views/partials/header.ejs | 10 ++ 7 files changed, 799 insertions(+), 28 deletions(-) create mode 100644 lib/blacklist.js create mode 100644 views/blacklist.ejs diff --git a/lib/api.js b/lib/api.js index 4249bd7..1a619b2 100644 --- a/lib/api.js +++ b/lib/api.js @@ -8,6 +8,7 @@ const express = require('express'); const path = require('path'); const fs = require('fs'); +const blacklist = require('./blacklist'); const STATUS_OPTIONS = [ 'Entwurf', 'Gesendet', 'Eingangsbestätigung', 'Vorstellungsgespräch', @@ -517,6 +518,62 @@ function createExternalApi(deps) { } }); + // --- Blacklist (must be registered BEFORE /joboffers/:id) ---------- + // Blocks offers from ever (re)appearing. See lib/blacklist for the match + // rules. A deleted offer is auto-blacklisted so it can never return. + router.get('/joboffers/blacklist', async (req, res) => { + try { + const rows = await dbAll( + 'SELECT * FROM jobangebote_blacklist ORDER BY created_at DESC, id DESC' + ); + res.json(rows); + } catch (error) { + console.error('API list blacklist error:', error); + res.status(500).json({ error: 'Serverfehler' }); + } + }); + + router.post('/joboffers/blacklist', async (req, res) => { + try { + const b = req.body || {}; + const entry = blacklist.buildManualEntry({ + typ: b.typ, + wert: b.wert != null ? sanitizeInput(String(b.wert)) : '', + firma: sanitizeInput(b.firma || ''), + stelle: sanitizeInput(b.stelle || ''), + ort: sanitizeInput(b.ort || ''), + grund: sanitizeInput(b.grund || ''), + }); + if (!entry) { + return res.status(400).json({ + error: `Ungültiger Blacklist-Eintrag. typ muss eine von ${blacklist.TYPES.join(', ')} sein und einen passenden Wert haben.`, + }); + } + const cols = blacklist.COLUMNS; + const result = await dbRun( + `INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`, + cols.map((c) => (entry[c] === undefined ? null : entry[c])) + ); + const row = await dbGet('SELECT * FROM jobangebote_blacklist WHERE id = ?', [result.lastID]); + res.status(201).json({ success: true, entry: row }); + } catch (error) { + console.error('API create blacklist entry error:', error); + res.status(500).json({ error: 'Serverfehler' }); + } + }); + + router.delete('/joboffers/blacklist/:id', async (req, res) => { + try { + const row = await dbGet('SELECT id FROM jobangebote_blacklist WHERE id = ?', [req.params.id]); + if (!row) return res.status(404).json({ error: 'Blacklist-Eintrag nicht gefunden' }); + await dbRun('DELETE FROM jobangebote_blacklist WHERE id = ?', [req.params.id]); + res.json({ success: true }); + } catch (error) { + console.error('API delete blacklist entry error:', error); + res.status(500).json({ error: 'Serverfehler' }); + } + }); + router.get('/joboffers/:id', async (req, res) => { try { const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]); @@ -534,10 +591,25 @@ function createExternalApi(deps) { if (!b.firma || !b.stelle) { return res.status(400).json({ error: 'firma und stelle sind erforderlich.' }); } + + // Reject blacklisted offers outright — they must never reappear. + const blacklistRows = await dbAll('SELECT * FROM jobangebote_blacklist'); + const blocked = blacklist.matchBlacklist(blacklistRows, b); + if (blocked) { + return res.status(409).json({ + blacklisted: true, + matched_by: blocked.typ, + blacklist_entry: blocked, + error: 'Dieses Jobangebot steht auf der Blacklist und wird nicht (erneut) aufgenommen.', + }); + } + const quelle = sanitizeInput(b.quelle || 'drittanbieter'); const externalId = b.external_id != null ? sanitizeInput(String(b.external_id)) : null; + const urlNorm = blacklist.normalizeUrl(b.quelle_url || '') || null; const anzeigeDatum = sanitizeInput(b.anzeige_datum || ''); const kontaktEmail = sanitizeInput(b.kontakt_email || ''); + // Column order matches the UPDATE/INSERT statements below (url_norm last). const fields = [ sanitizeInput(b.firma), sanitizeInput(b.stelle), @@ -549,30 +621,40 @@ function createExternalApi(deps) { anzeigeDatum, kontaktEmail, sanitizeInput(b.status || 'offen'), + urlNorm, ]; - // Upsert: a matching (quelle, external_id) row is updated, else inserted. + // De-dup: prefer a (quelle, external_id) match, else the same normalized + // URL — so the same posting never lands twice, even with a new id. + let existing = null; if (externalId) { - const existing = await dbGet( + existing = await dbGet( 'SELECT id FROM jobangebote WHERE quelle = ? AND external_id = ?', [quelle, externalId] ); - if (existing) { - await dbRun( - `UPDATE jobangebote SET firma = ?, stelle = ?, ort = ?, gehalt = ?, - beschreibung = ?, quelle_url = ?, art = ?, anzeige_datum = ?, kontakt_email = ?, status = ?, - updated_at = CURRENT_TIMESTAMP WHERE id = ?`, - [...fields, existing.id] - ); - const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [existing.id]); - return res.json({ success: true, action: 'updated', joboffer: row }); - } + } + if (!existing && urlNorm) { + existing = await dbGet( + 'SELECT id FROM jobangebote WHERE url_norm = ? ORDER BY id ASC LIMIT 1', + [urlNorm] + ); + } + + if (existing) { + await dbRun( + `UPDATE jobangebote SET firma = ?, stelle = ?, ort = ?, gehalt = ?, + beschreibung = ?, quelle_url = ?, art = ?, anzeige_datum = ?, kontakt_email = ?, status = ?, + url_norm = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`, + [...fields, existing.id] + ); + const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [existing.id]); + return res.json({ success: true, action: 'updated', joboffer: row }); } const result = await dbRun( `INSERT INTO jobangebote - (external_id, quelle, firma, stelle, ort, gehalt, beschreibung, quelle_url, art, anzeige_datum, kontakt_email, status) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (external_id, quelle, firma, stelle, ort, gehalt, beschreibung, quelle_url, art, anzeige_datum, kontakt_email, status, url_norm) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [externalId, quelle, ...fields] ); const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [result.lastID]); @@ -583,12 +665,29 @@ function createExternalApi(deps) { } }); + // Delete a job offer. By default it is auto-blacklisted first so it can never + // be ingested again; pass ?blacklist=false to hard-delete without blocking. router.delete('/joboffers/:id', async (req, res) => { try { - const row = await dbGet('SELECT id FROM jobangebote WHERE id = ?', [req.params.id]); + const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]); if (!row) return res.status(404).json({ error: 'Jobangebot nicht gefunden' }); + + const skipBlacklist = req.query.blacklist === 'false' || req.query.blacklist === '0'; + let blacklisted = false; + if (!skipBlacklist) { + const rows = await dbAll('SELECT * FROM jobangebote_blacklist'); + if (!blacklist.matchBlacklist(rows, row)) { + const cols = blacklist.COLUMNS; + const entry = blacklist.buildAutoEntry(row, 'Jobangebot gelöscht (REST-API)'); + await dbRun( + `INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`, + cols.map((c) => (entry[c] === undefined ? null : entry[c])) + ); + } + blacklisted = true; + } await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]); - res.json({ success: true }); + res.json({ success: true, blacklisted }); } catch (error) { console.error('API delete job offer error:', error); res.status(500).json({ error: 'Serverfehler' }); diff --git a/lib/blacklist.js b/lib/blacklist.js new file mode 100644 index 0000000..e47874d --- /dev/null +++ b/lib/blacklist.js @@ -0,0 +1,218 @@ +// Job-offer blacklist: normalization + matching helpers. +// +// Shared by the web routes (server.js) and the REST API (lib/api.js) so both +// use identical rules. A blacklisted offer is never ingested again, and every +// deleted offer is auto-blacklisted so it can never reappear in the list. +// +// A blacklist row has a `typ` and a set of normalized match keys; only the keys +// relevant to its typ are set: +// 'url' -> url_norm (exact normalized URL) +// 'domain' -> domain (whole host, e.g. a job board) +// 'firma' -> firma_norm (block a company entirely) +// 'firma_stelle' -> firma_norm + stelle_norm (+ optional ort_norm) +// 'auto' -> url_norm / quelle+external_id / firma_norm+stelle_norm +// (created on delete; matches on ANY stored signature) + +const TYPES = ['url', 'domain', 'firma', 'firma_stelle', 'auto']; + +// Query params that carry no identity — stripped before comparing URLs so the +// same posting with different tracking/referrer params still matches. +const TRACKING_PARAMS = new Set([ + 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id', + 'gclid', 'fbclid', 'msclkid', 'mc_cid', 'mc_eid', 'yclid', 'igshid', 'gh_src', + 'ref', 'referrer', 'source', 'src', 'from', 'campaignid', 'cmpid', 'cmp', + 'vjk', 'tk', 'rgtk', 'hl', 'spa', 'sp', 'advn', 'adid', 'trk', 'trkid', +]); + +// Reverse the web layer's HTML sanitisation (sanitizeInput encodes < > " ') +// so a value stored via the UI normalises to the same thing as the raw input. +function unescapeInput(s) { + return String(s == null ? '' : s) + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + +const HAS_SCHEME = /^[a-z][a-z0-9+.-]*:\/\//i; + +function parseUrl(raw) { + const s = unescapeInput(raw).trim(); + if (!s) return null; + try { + return new URL(HAS_SCHEME.test(s) ? s : 'https://' + s); + } catch (e) { + return null; + } +} + +// Host without a leading "www.", lowercased. +function urlDomain(raw) { + const u = parseUrl(raw); + return u ? u.hostname.toLowerCase().replace(/^www\./, '') : ''; +} + +// A stable identity string for a URL: lowercased host (no www), path without a +// trailing slash, and the meaningful query params sorted (tracking dropped). +// Path/query case is preserved because job IDs can be case-sensitive. +function normalizeUrl(raw) { + const u = parseUrl(raw); + if (!u) return unescapeInput(raw).trim().toLowerCase(); + const host = u.hostname.toLowerCase().replace(/^www\./, ''); + const params = []; + for (const [k, v] of u.searchParams.entries()) { + if (TRACKING_PARAMS.has(k.toLowerCase())) continue; + params.push([k, v]); + } + params.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)); + const pathname = u.pathname.replace(/\/+$/, ''); + const qs = params.length + ? '?' + params.map(([k, v]) => (v === '' ? k : `${k}=${v}`)).join('&') + : ''; + return host + pathname + qs; +} + +// Normalise a company / job-title string for fuzzy identity matching: +// unescape, lowercase, drop gender markers like "(m/w/d)", collapse anything +// non-alphanumeric to single spaces. +function normText(s) { + return unescapeInput(s) + .toLowerCase() + .replace(/\((?:[mwdfax](?:\s*\/\s*[mwdfax])*)\)/gi, ' ') + .replace(/\b[mwfd](?:\s*\/\s*[mwfd]){1,2}\b/gi, ' ') + .replace(/[^\p{L}\p{N}]+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +// Normalised signature of an offer (raw or DB row), used for matching. +function offerSignature(offer) { + const o = offer || {}; + return { + url_norm: normalizeUrl(o.quelle_url), + domain: urlDomain(o.quelle_url), + quelle: (o.quelle && String(o.quelle).trim()) || 'drittanbieter', + external_id: o.external_id != null && String(o.external_id).trim() !== '' + ? String(o.external_id).trim() : null, + firma_norm: normText(o.firma), + stelle_norm: normText(o.stelle), + ort_norm: normText(o.ort), + }; +} + +// Does one blacklist row block the given signature? +function rowMatches(row, sig) { + switch (row.typ) { + case 'url': + return !!row.url_norm && row.url_norm === sig.url_norm; + case 'domain': + return !!row.domain && row.domain === sig.domain; + case 'firma': + return !!row.firma_norm && row.firma_norm === sig.firma_norm; + case 'firma_stelle': + return !!row.firma_norm && row.firma_norm === sig.firma_norm + && !!row.stelle_norm && row.stelle_norm === sig.stelle_norm + && (!row.ort_norm || row.ort_norm === sig.ort_norm); + case 'auto': + default: + if (row.url_norm && sig.url_norm && row.url_norm === sig.url_norm) return true; + if (row.quelle && row.external_id && sig.external_id + && row.quelle === sig.quelle && row.external_id === sig.external_id) return true; + if (row.firma_norm && row.stelle_norm + && row.firma_norm === sig.firma_norm && row.stelle_norm === sig.stelle_norm + && (!row.ort_norm || row.ort_norm === sig.ort_norm)) return true; + return false; + } +} + +// First blacklist row that blocks this offer, or null. +function matchBlacklist(rows, offer) { + const sig = offerSignature(offer); + for (const row of rows || []) { + if (rowMatches(row, sig)) return row; + } + return null; +} + +// Column set for INSERT — keeps server.js and api.js in lock-step. +const COLUMNS = [ + 'typ', 'url_norm', 'domain', 'quelle', 'external_id', + 'firma_norm', 'stelle_norm', 'ort_norm', 'firma', 'stelle', 'quelle_url', 'grund', +]; + +function emptyEntry() { + const e = {}; + COLUMNS.forEach((c) => { e[c] = null; }); + return e; +} + +// Build a blacklist row from a manual UI/API request. Returns null if the +// chosen typ has no usable value. +function buildManualEntry(input) { + const inp = input || {}; + const typ = TYPES.includes(inp.typ) ? inp.typ : 'url'; + const e = emptyEntry(); + e.typ = typ; + e.grund = inp.grund ? String(inp.grund) : null; + const wert = inp.wert != null ? String(inp.wert) : ''; + + if (typ === 'url') { + const w = wert || inp.quelle_url || ''; + e.url_norm = normalizeUrl(w); + e.domain = urlDomain(w) || null; + e.quelle_url = w || null; + if (!e.url_norm) return null; + } else if (typ === 'domain') { + const w = wert || inp.domain || ''; + e.domain = urlDomain(w) || normText(w).replace(/\s+/g, '') || null; + if (!e.domain) return null; + } else if (typ === 'firma') { + const w = wert || inp.firma || ''; + e.firma = w || null; + e.firma_norm = normText(w); + if (!e.firma_norm) return null; + } else if (typ === 'firma_stelle') { + const f = inp.firma || ''; + const s = inp.stelle || wert || ''; + e.firma = f || null; + e.stelle = s || null; + e.firma_norm = normText(f); + e.stelle_norm = normText(s); + e.ort_norm = inp.ort ? normText(inp.ort) : null; + if (!e.firma_norm || !e.stelle_norm) return null; + } + return e; +} + +// Build the automatic blacklist row created when an offer is deleted: it stores +// every available signature so the offer can never be ingested again. +function buildAutoEntry(offer, grund) { + const sig = offerSignature(offer); + const e = emptyEntry(); + e.typ = 'auto'; + e.grund = grund || null; + e.url_norm = sig.url_norm || null; + e.domain = sig.domain || null; + e.quelle = sig.external_id ? sig.quelle : null; + e.external_id = sig.external_id || null; + e.firma_norm = sig.firma_norm || null; + e.stelle_norm = sig.stelle_norm || null; + e.ort_norm = sig.ort_norm || null; + e.firma = offer && offer.firma ? offer.firma : null; + e.stelle = offer && offer.stelle ? offer.stelle : null; + e.quelle_url = offer && offer.quelle_url ? offer.quelle_url : null; + return e; +} + +module.exports = { + TYPES, + COLUMNS, + normalizeUrl, + urlDomain, + normText, + offerSignature, + rowMatches, + matchBlacklist, + buildManualEntry, + buildAutoEntry, +}; diff --git a/lib/openapi.js b/lib/openapi.js index a39bcef..1992e2c 100644 --- a/lib/openapi.js +++ b/lib/openapi.js @@ -486,9 +486,11 @@ function buildOpenApiSpec(baseUrl = '') { summary: 'Jobangebot einspielen (Upsert)', description: 'Spielt ein Jobangebot ein. Wird von Drittanbietersoftware genutzt, um ' + - 'Stellen in den Tracker zu übernehmen. Upsert über (quelle, external_id): ' + - 'wiederholtes Senden desselben Angebots aktualisiert es statt ein Duplikat ' + - 'anzulegen. Wird 201 (created) oder 200 (updated) zurückgegeben.', + 'Stellen in den Tracker zu übernehmen. De-Dup: Upsert über (quelle, ' + + 'external_id) oder – falls keine external_id passt – über die normalisierte ' + + 'URL (quelle_url). Dieselbe Stelle landet damit nie doppelt. Steht das ' + + 'Angebot auf der Blacklist (siehe /joboffers/blacklist), wird es mit 409 ' + + 'abgelehnt und NICHT aufgenommen. Rückgabe 201 (created) oder 200 (updated).', requestBody: { required: true, content: { 'application/json': { schema: { $ref: '#/components/schemas/JobOfferCreate' } } }, @@ -504,6 +506,66 @@ function buildOpenApiSpec(baseUrl = '') { }, '400': errorResponse, '401': errorResponse, + '409': { + description: 'Angebot steht auf der Blacklist und wurde nicht aufgenommen', + content: { 'application/json': { schema: { $ref: '#/components/schemas/BlacklistConflict' } } }, + }, + '500': errorResponse, + }, + }, + }, + + // NOTE: registered before /joboffers/{id} so "blacklist" is not read as an id. + '/joboffers/blacklist': { + get: { + tags: ['JobOffers'], + summary: 'Blacklist auflisten', + description: 'Alle Blacklist-Einträge (neueste zuerst). Blockierte Angebote werden nie eingespielt.', + responses: { + '200': { + description: 'Blacklist-Einträge', + content: { + 'application/json': { + schema: { type: 'array', items: { $ref: '#/components/schemas/BlacklistEntry' } }, + }, + }, + }, + '401': errorResponse, + '500': errorResponse, + }, + }, + post: { + tags: ['JobOffers'], + summary: 'Blacklist-Eintrag anlegen', + description: + 'Blockiert Angebote anhand einer URL, einer ganzen Domain, einer Firma ' + + 'oder einer Firma + Stelle. Passende Angebote werden danach nicht mehr eingespielt.', + requestBody: { + required: true, + content: { 'application/json': { schema: { $ref: '#/components/schemas/BlacklistEntryCreate' } } }, + }, + responses: { + '201': { + description: 'Eintrag angelegt', + content: { 'application/json': { schema: { $ref: '#/components/schemas/BlacklistEntryResult' } } }, + }, + '400': errorResponse, + '401': errorResponse, + '500': errorResponse, + }, + }, + }, + + '/joboffers/blacklist/{id}': { + delete: { + tags: ['JobOffers'], + summary: 'Blacklist-Eintrag entfernen', + description: 'Entfernt einen Block. Betroffene Angebote können danach wieder eingespielt werden.', + parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'integer' } }], + responses: { + '200': { description: 'Entfernt', content: { 'application/json': { schema: { $ref: '#/components/schemas/Ok' } } } }, + '401': errorResponse, + '404': errorResponse, '500': errorResponse, }, }, @@ -527,9 +589,22 @@ function buildOpenApiSpec(baseUrl = '') { delete: { tags: ['JobOffers'], summary: 'Jobangebot löschen', - parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'integer' } }], + description: + 'Löscht ein Jobangebot. Standardmäßig wird es zuvor automatisch auf die ' + + 'Blacklist gesetzt, damit dieselbe Stelle nie erneut eingespielt wird. Mit ' + + '?blacklist=false wird ohne Blockierung hart gelöscht.', + parameters: [ + { name: 'id', in: 'path', required: true, schema: { type: 'integer' } }, + { + name: 'blacklist', + in: 'query', + required: false, + schema: { type: 'boolean', default: true }, + description: 'false = löschen ohne Blacklisting (Standard: true).', + }, + ], responses: { - '200': { description: 'Gelöscht', content: { 'application/json': { schema: { $ref: '#/components/schemas/Ok' } } } }, + '200': { description: 'Gelöscht', content: { 'application/json': { schema: { $ref: '#/components/schemas/JobOfferDeleteResult' } } } }, '401': errorResponse, '404': errorResponse, '500': errorResponse, @@ -788,6 +863,7 @@ function buildOpenApiSpec(baseUrl = '') { kontakt_email: { type: 'string', format: 'email', nullable: true, description: 'Kontakt-E-Mail-Adresse der Stelle' }, status: { type: 'string', enum: ['offen', 'uebernommen', 'abgelehnt'] }, verknuepfte_bewerbung_id: { type: 'integer', nullable: true }, + url_norm: { type: 'string', nullable: true, description: 'Normalisierte URL für die De-Duplizierung (serverseitig gesetzt)' }, created_at: { type: 'string', format: 'date-time' }, updated_at: { type: 'string', format: 'date-time' }, }, @@ -818,6 +894,65 @@ function buildOpenApiSpec(baseUrl = '') { joboffer: { $ref: '#/components/schemas/JobOffer' }, }, }, + JobOfferDeleteResult: { + type: 'object', + properties: { + success: { type: 'boolean', example: true }, + blacklisted: { type: 'boolean', description: 'true, wenn das Angebot beim Löschen auf die Blacklist gesetzt wurde' }, + }, + }, + BlacklistEntry: { + type: 'object', + description: 'Ein Blacklist-Eintrag. Nur die zum typ passenden Felder sind gesetzt.', + properties: { + id: { type: 'integer' }, + typ: { type: 'string', enum: ['url', 'domain', 'firma', 'firma_stelle', 'auto'] }, + url_norm: { type: 'string', nullable: true, description: 'Normalisierte URL (Tracking-Parameter entfernt)' }, + domain: { type: 'string', nullable: true }, + quelle: { type: 'string', nullable: true }, + external_id: { type: 'string', nullable: true }, + firma_norm: { type: 'string', nullable: true }, + stelle_norm: { type: 'string', nullable: true }, + ort_norm: { type: 'string', nullable: true }, + firma: { type: 'string', nullable: true }, + stelle: { type: 'string', nullable: true }, + quelle_url: { type: 'string', nullable: true }, + grund: { type: 'string', nullable: true }, + created_at: { type: 'string', format: 'date-time' }, + }, + }, + BlacklistEntryCreate: { + type: 'object', + required: ['typ'], + properties: { + typ: { + type: 'string', + enum: ['url', 'domain', 'firma', 'firma_stelle'], + description: 'url/domain/firma nutzen "wert"; firma_stelle nutzt "firma"+"stelle" (+optional "ort").', + }, + wert: { type: 'string', description: 'URL (typ=url), Domain (typ=domain) oder Firmenname (typ=firma).' }, + firma: { type: 'string', description: 'Nur bei typ=firma_stelle.' }, + stelle: { type: 'string', description: 'Nur bei typ=firma_stelle.' }, + ort: { type: 'string', description: 'Optional bei typ=firma_stelle.' }, + grund: { type: 'string', description: 'Optionaler Freitext-Grund.' }, + }, + }, + BlacklistEntryResult: { + type: 'object', + properties: { + success: { type: 'boolean', example: true }, + entry: { $ref: '#/components/schemas/BlacklistEntry' }, + }, + }, + BlacklistConflict: { + type: 'object', + properties: { + blacklisted: { type: 'boolean', example: true }, + matched_by: { type: 'string', description: 'typ des greifenden Blacklist-Eintrags' }, + blacklist_entry: { $ref: '#/components/schemas/BlacklistEntry' }, + error: { type: 'string' }, + }, + }, }, }, security: [{ ApiKeyAuth: [] }], diff --git a/server.js b/server.js index 93f5e60..7d11477 100644 --- a/server.js +++ b/server.js @@ -31,6 +31,7 @@ const { generateApplicationDocuments, generateEmailReply } = require('./lib/docu const mailer = require('./lib/mailer'); const { createExternalApi } = require('./lib/api'); const { buildOpenApiSpec } = require('./lib/openapi'); +const blacklist = require('./lib/blacklist'); const app = express(); const PORT = process.env.PORT || 3000; @@ -244,6 +245,30 @@ function dbRun(sql, params = []) { }); } +// --------------------------------------------------------------------------- +// Job-offer blacklist helpers (shared shape with lib/api.js) +// --------------------------------------------------------------------------- + +// Insert one prepared blacklist entry (see lib/blacklist.buildManual/AutoEntry). +async function insertBlacklistEntry(entry) { + const cols = blacklist.COLUMNS; + const placeholders = cols.map(() => '?').join(', '); + const values = cols.map((c) => (entry[c] === undefined ? null : entry[c])); + return dbRun( + `INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${placeholders})`, + values + ); +} + +// Auto-blacklist an offer row so it can never be ingested again, then it is safe +// to delete. Skips silently if the offer is already covered by an entry. +async function autoBlacklistOffer(offer, grund) { + if (!offer) return; + const rows = await dbAll('SELECT * FROM jobangebote_blacklist'); + if (blacklist.matchBlacklist(rows, offer)) return; // already blocked + await insertBlacklistEntry(blacklist.buildAutoEntry(offer, grund)); +} + // --------------------------------------------------------------------------- // E-Mail correspondence: IMAP polling, storing & matching incoming replies // --------------------------------------------------------------------------- @@ -632,6 +657,39 @@ function initializeDatabase() { // Migration: add columns to pre-existing jobangebote tables. db.run('ALTER TABLE jobangebote ADD COLUMN anzeige_datum DATE', () => {}); db.run('ALTER TABLE jobangebote ADD COLUMN kontakt_email TEXT', () => {}); + // Normalized URL for URL-based de-duplication of offers (see lib/blacklist). + db.run('ALTER TABLE jobangebote ADD COLUMN url_norm TEXT', () => { + // Backfill url_norm for rows ingested before this column existed. + db.all('SELECT id, quelle_url FROM jobangebote WHERE url_norm IS NULL AND quelle_url IS NOT NULL AND quelle_url != ""', (err, rows) => { + if (err || !rows) return; + rows.forEach((r) => { + const norm = blacklist.normalizeUrl(r.quelle_url); + if (norm) db.run('UPDATE jobangebote SET url_norm = ? WHERE id = ?', [norm, r.id], () => {}); + }); + }); + }); + db.run('CREATE INDEX IF NOT EXISTS idx_jobangebote_url_norm ON jobangebote(url_norm)', () => {}); + // Blacklist of job offers that must never (re)appear in the list. A + // deleted offer is auto-blacklisted; manual entries can block a URL, + // a whole domain, a company, or a specific company+title posting. + db.run(` + CREATE TABLE IF NOT EXISTS jobangebote_blacklist ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + typ TEXT NOT NULL DEFAULT 'auto', + url_norm TEXT, + domain TEXT, + quelle TEXT, + external_id TEXT, + firma_norm TEXT, + stelle_norm TEXT, + ort_norm TEXT, + firma TEXT, + stelle TEXT, + quelle_url TEXT, + grund TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + `, () => {}); // Remember the last recipient address per application (prefill). db.run('ALTER TABLE bewerbungen ADD COLUMN email_empfaenger TEXT', () => {}); @@ -1676,10 +1734,16 @@ initializeDatabase().then(() => { } }); - // Delete a job offer (cascades nothing — verknuepfte_bewerbung_id is SET NULL). + // Delete a job offer. Deleting always blacklists it first, so the same offer + // can never be ingested/listed again (the requirement: never appears twice, + // even after deletion). The entry can be removed later on /blacklist. app.post('/jobangebote/:id/delete', async (req, res) => { try { - await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]); + const offer = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]); + if (offer) { + await autoBlacklistOffer(offer, 'Jobangebot gelöscht (Web-UI)'); + await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]); + } res.redirect('/jobangebote'); } catch (error) { console.error('Error deleting job offer:', error); @@ -1687,6 +1751,49 @@ initializeDatabase().then(() => { } }); + // Blacklist management page — see what is blocked and remove entries. + app.get('/blacklist', async (req, res) => { + try { + const eintraege = await dbAll( + 'SELECT * FROM jobangebote_blacklist ORDER BY created_at DESC, id DESC' + ); + res.render('blacklist', { eintraege, blacklistTypen: blacklist.TYPES, hideSettings: false }); + } catch (error) { + console.error('Error listing blacklist:', error); + res.status(500).send('Serverfehler'); + } + }); + + // Add a manual blacklist entry (URL / domain / company / company+title). + app.post('/blacklist', async (req, res) => { + try { + const entry = blacklist.buildManualEntry({ + typ: req.body.typ, + wert: sanitizeInput(req.body.wert || ''), + firma: sanitizeInput(req.body.firma || ''), + stelle: sanitizeInput(req.body.stelle || ''), + ort: sanitizeInput(req.body.ort || ''), + grund: sanitizeInput(req.body.grund || ''), + }); + if (entry) await insertBlacklistEntry(entry); + res.redirect('/blacklist'); + } catch (error) { + console.error('Error adding blacklist entry:', error); + res.status(500).send('Serverfehler'); + } + }); + + // Remove a blacklist entry (offer may then be ingested again). + app.post('/blacklist/:id/delete', async (req, res) => { + try { + await dbRun('DELETE FROM jobangebote_blacklist WHERE id = ?', [req.params.id]); + res.redirect('/blacklist'); + } catch (error) { + console.error('Error deleting blacklist entry:', error); + res.status(500).send('Serverfehler'); + } + }); + // ----- Third-party REST API (/api/v1) + OpenAPI/Swagger ----- // API key for third-party software. When unset, the API responds 503 on // every endpoint except /health — it never silently exposes data. diff --git a/views/blacklist.ejs b/views/blacklist.ejs new file mode 100644 index 0000000..0b89b75 --- /dev/null +++ b/views/blacklist.ejs @@ -0,0 +1,196 @@ + + + + <%- include('partials/head') %> + + + <%- include('partials/header') %> + +
+ + + + + + Zurück zu den Jobangeboten + + +
+
+
+

Blacklist

+

+ Angebote auf der Blacklist werden nie (erneut) in die Jobangebote + aufgenommen. Ein gelöschtes Jobangebot landet automatisch + hier, damit dieselbe Stelle nicht doppelt auftaucht. Du kannst auch manuell + eine URL, eine ganze Domain, eine Firma oder eine konkrete Firma + Stelle + blockieren. Einträge lassen sich jederzeit wieder entfernen. +

+
+ + + <%= eintraege.length %> Eintrag/Einträge + +
+ + <% if (eintraege && eintraege.length) { %> +
    + <% eintraege.forEach(function(e){ + var typLabel = { url: 'URL', domain: 'Domain', firma: 'Firma', firma_stelle: 'Firma + Stelle', auto: 'Automatisch (gelöscht)' }[e.typ] || e.typ; + var typClass = e.typ === 'auto' + ? 'bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-200' + : 'bg-purple-100 text-purple-800 dark:bg-purple-900/50 dark:text-purple-200'; + var wert = ''; + if (e.typ === 'url') wert = e.quelle_url || e.url_norm || ''; + else if (e.typ === 'domain') wert = e.domain || ''; + else if (e.typ === 'firma') wert = e.firma || e.firma_norm || ''; + else if (e.typ === 'firma_stelle') wert = [e.firma, e.stelle].filter(Boolean).join(' · '); + else wert = [e.firma, e.stelle].filter(Boolean).join(' · ') || e.quelle_url || e.url_norm || e.domain || ''; + %> +
  • +
    +
    +
    + <%= typLabel %> + <% if (e.typ === 'auto' && e.quelle_url) { %> + URL erfasst + <% } %> + <% if (e.external_id) { %> + ext. ID + <% } %> +
    +

    <%= wert || '(kein Wert)' %>

    + <% if (e.typ === 'auto' && e.quelle_url && wert !== e.quelle_url) { %> +

    <%= e.quelle_url %>

    + <% } %> + <% if (e.grund) { %> +

    Grund: <%= e.grund %>

    + <% } %> +

    + Hinzugefügt: <%= e.created_at ? new Date(e.created_at + 'Z').toLocaleString('de-DE') : '' %> +

    +
    +
    + +
    +
    +
    • + <% }); %> +
+ <% } else { %> +
+ + + +

Die Blacklist ist leer.

+
+ <% } %> +
+ + +
+

Eintrag manuell hinzufügen

+
+
+ + +
+ +
+ + +
+ + + +
+ + +
+ + +
+
+
+ + <%- include('partials/footer') %> + + + + diff --git a/views/jobangebote.ejs b/views/jobangebote.ejs index 2e1109f..6b32698 100644 --- a/views/jobangebote.ejs +++ b/views/jobangebote.ejs @@ -26,10 +26,16 @@ als Bewerbung-Entwurf übernehmen.

- - - <%= jobangebote.length %> Angebot(e) - +
+ + + Blacklist + + + + <%= jobangebote.length %> Angebot(e) + +
<% if (jobangebote && jobangebote.length) { %> @@ -124,7 +130,7 @@ <% } %>
+ onsubmit="return confirm('Jobangebot „<%= j.firma %> · <%= j.stelle %>“ löschen? Es kommt auf die Blacklist und wird nicht erneut eingespielt.');">