Add a job-offer blacklist with URL-based de-duplication

Offers are now de-duplicated by normalized URL (tracking params stripped)
in addition to (quelle, external_id), so the same posting never lands
twice — even re-scraped under a new id. Deleting an offer (web or API)
auto-blacklists it, so it can never reappear.

lib/blacklist.js provides shared normalization + matching. Manual entries
can block a URL, a whole domain, a company, or a company+title posting
(gender-marker tolerant). New /blacklist page lists and manages entries.

REST API: GET/POST /joboffers/blacklist, DELETE /joboffers/blacklist/{id};
POST /joboffers returns 409 when blacklisted; DELETE /joboffers/{id}
auto-blacklists (opt out with ?blacklist=false). Swagger updated with the
new paths and schemas.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-04 13:33:12 +02:00
co-authored by Claude Opus 4.8
parent 1655427b4c
commit dae077486f
7 changed files with 799 additions and 28 deletions
+115 -16
View File
@@ -8,6 +8,7 @@
const express = require('express');
const path = require('path');
const fs = require('fs');
const blacklist = require('./blacklist');
const STATUS_OPTIONS = [
'Entwurf', 'Gesendet', 'Eingangsbestätigung', 'Vorstellungsgespräch',
@@ -517,6 +518,62 @@ function createExternalApi(deps) {
}
});
// --- Blacklist (must be registered BEFORE /joboffers/:id) ----------
// Blocks offers from ever (re)appearing. See lib/blacklist for the match
// rules. A deleted offer is auto-blacklisted so it can never return.
router.get('/joboffers/blacklist', async (req, res) => {
try {
const rows = await dbAll(
'SELECT * FROM jobangebote_blacklist ORDER BY created_at DESC, id DESC'
);
res.json(rows);
} catch (error) {
console.error('API list blacklist error:', error);
res.status(500).json({ error: 'Serverfehler' });
}
});
router.post('/joboffers/blacklist', async (req, res) => {
try {
const b = req.body || {};
const entry = blacklist.buildManualEntry({
typ: b.typ,
wert: b.wert != null ? sanitizeInput(String(b.wert)) : '',
firma: sanitizeInput(b.firma || ''),
stelle: sanitizeInput(b.stelle || ''),
ort: sanitizeInput(b.ort || ''),
grund: sanitizeInput(b.grund || ''),
});
if (!entry) {
return res.status(400).json({
error: `Ungültiger Blacklist-Eintrag. typ muss eine von ${blacklist.TYPES.join(', ')} sein und einen passenden Wert haben.`,
});
}
const cols = blacklist.COLUMNS;
const result = await dbRun(
`INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`,
cols.map((c) => (entry[c] === undefined ? null : entry[c]))
);
const row = await dbGet('SELECT * FROM jobangebote_blacklist WHERE id = ?', [result.lastID]);
res.status(201).json({ success: true, entry: row });
} catch (error) {
console.error('API create blacklist entry error:', error);
res.status(500).json({ error: 'Serverfehler' });
}
});
router.delete('/joboffers/blacklist/:id', async (req, res) => {
try {
const row = await dbGet('SELECT id FROM jobangebote_blacklist WHERE id = ?', [req.params.id]);
if (!row) return res.status(404).json({ error: 'Blacklist-Eintrag nicht gefunden' });
await dbRun('DELETE FROM jobangebote_blacklist WHERE id = ?', [req.params.id]);
res.json({ success: true });
} catch (error) {
console.error('API delete blacklist entry error:', error);
res.status(500).json({ error: 'Serverfehler' });
}
});
router.get('/joboffers/:id', async (req, res) => {
try {
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]);
@@ -534,10 +591,25 @@ function createExternalApi(deps) {
if (!b.firma || !b.stelle) {
return res.status(400).json({ error: 'firma und stelle sind erforderlich.' });
}
// Reject blacklisted offers outright — they must never reappear.
const blacklistRows = await dbAll('SELECT * FROM jobangebote_blacklist');
const blocked = blacklist.matchBlacklist(blacklistRows, b);
if (blocked) {
return res.status(409).json({
blacklisted: true,
matched_by: blocked.typ,
blacklist_entry: blocked,
error: 'Dieses Jobangebot steht auf der Blacklist und wird nicht (erneut) aufgenommen.',
});
}
const quelle = sanitizeInput(b.quelle || 'drittanbieter');
const externalId = b.external_id != null ? sanitizeInput(String(b.external_id)) : null;
const urlNorm = blacklist.normalizeUrl(b.quelle_url || '') || null;
const anzeigeDatum = sanitizeInput(b.anzeige_datum || '');
const kontaktEmail = sanitizeInput(b.kontakt_email || '');
// Column order matches the UPDATE/INSERT statements below (url_norm last).
const fields = [
sanitizeInput(b.firma),
sanitizeInput(b.stelle),
@@ -549,30 +621,40 @@ function createExternalApi(deps) {
anzeigeDatum,
kontaktEmail,
sanitizeInput(b.status || 'offen'),
urlNorm,
];
// Upsert: a matching (quelle, external_id) row is updated, else inserted.
// De-dup: prefer a (quelle, external_id) match, else the same normalized
// URL — so the same posting never lands twice, even with a new id.
let existing = null;
if (externalId) {
const existing = await dbGet(
existing = await dbGet(
'SELECT id FROM jobangebote WHERE quelle = ? AND external_id = ?',
[quelle, externalId]
);
if (existing) {
await dbRun(
`UPDATE jobangebote SET firma = ?, stelle = ?, ort = ?, gehalt = ?,
beschreibung = ?, quelle_url = ?, art = ?, anzeige_datum = ?, kontakt_email = ?, status = ?,
updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
[...fields, existing.id]
);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [existing.id]);
return res.json({ success: true, action: 'updated', joboffer: row });
}
}
if (!existing && urlNorm) {
existing = await dbGet(
'SELECT id FROM jobangebote WHERE url_norm = ? ORDER BY id ASC LIMIT 1',
[urlNorm]
);
}
if (existing) {
await dbRun(
`UPDATE jobangebote SET firma = ?, stelle = ?, ort = ?, gehalt = ?,
beschreibung = ?, quelle_url = ?, art = ?, anzeige_datum = ?, kontakt_email = ?, status = ?,
url_norm = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
[...fields, existing.id]
);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [existing.id]);
return res.json({ success: true, action: 'updated', joboffer: row });
}
const result = await dbRun(
`INSERT INTO jobangebote
(external_id, quelle, firma, stelle, ort, gehalt, beschreibung, quelle_url, art, anzeige_datum, kontakt_email, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
(external_id, quelle, firma, stelle, ort, gehalt, beschreibung, quelle_url, art, anzeige_datum, kontakt_email, status, url_norm)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[externalId, quelle, ...fields]
);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [result.lastID]);
@@ -583,12 +665,29 @@ function createExternalApi(deps) {
}
});
// Delete a job offer. By default it is auto-blacklisted first so it can never
// be ingested again; pass ?blacklist=false to hard-delete without blocking.
router.delete('/joboffers/:id', async (req, res) => {
try {
const row = await dbGet('SELECT id FROM jobangebote WHERE id = ?', [req.params.id]);
const row = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]);
if (!row) return res.status(404).json({ error: 'Jobangebot nicht gefunden' });
const skipBlacklist = req.query.blacklist === 'false' || req.query.blacklist === '0';
let blacklisted = false;
if (!skipBlacklist) {
const rows = await dbAll('SELECT * FROM jobangebote_blacklist');
if (!blacklist.matchBlacklist(rows, row)) {
const cols = blacklist.COLUMNS;
const entry = blacklist.buildAutoEntry(row, 'Jobangebot gelöscht (REST-API)');
await dbRun(
`INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`,
cols.map((c) => (entry[c] === undefined ? null : entry[c]))
);
}
blacklisted = true;
}
await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]);
res.json({ success: true });
res.json({ success: true, blacklisted });
} catch (error) {
console.error('API delete job offer error:', error);
res.status(500).json({ error: 'Serverfehler' });
+218
View File
@@ -0,0 +1,218 @@
// Job-offer blacklist: normalization + matching helpers.
//
// Shared by the web routes (server.js) and the REST API (lib/api.js) so both
// use identical rules. A blacklisted offer is never ingested again, and every
// deleted offer is auto-blacklisted so it can never reappear in the list.
//
// A blacklist row has a `typ` and a set of normalized match keys; only the keys
// relevant to its typ are set:
// 'url' -> url_norm (exact normalized URL)
// 'domain' -> domain (whole host, e.g. a job board)
// 'firma' -> firma_norm (block a company entirely)
// 'firma_stelle' -> firma_norm + stelle_norm (+ optional ort_norm)
// 'auto' -> url_norm / quelle+external_id / firma_norm+stelle_norm
// (created on delete; matches on ANY stored signature)
const TYPES = ['url', 'domain', 'firma', 'firma_stelle', 'auto'];
// Query params that carry no identity — stripped before comparing URLs so the
// same posting with different tracking/referrer params still matches.
const TRACKING_PARAMS = new Set([
'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id',
'gclid', 'fbclid', 'msclkid', 'mc_cid', 'mc_eid', 'yclid', 'igshid', 'gh_src',
'ref', 'referrer', 'source', 'src', 'from', 'campaignid', 'cmpid', 'cmp',
'vjk', 'tk', 'rgtk', 'hl', 'spa', 'sp', 'advn', 'adid', 'trk', 'trkid',
]);
// Reverse the web layer's HTML sanitisation (sanitizeInput encodes < > " ')
// so a value stored via the UI normalises to the same thing as the raw input.
function unescapeInput(s) {
return String(s == null ? '' : s)
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'");
}
const HAS_SCHEME = /^[a-z][a-z0-9+.-]*:\/\//i;
function parseUrl(raw) {
const s = unescapeInput(raw).trim();
if (!s) return null;
try {
return new URL(HAS_SCHEME.test(s) ? s : 'https://' + s);
} catch (e) {
return null;
}
}
// Host without a leading "www.", lowercased.
function urlDomain(raw) {
const u = parseUrl(raw);
return u ? u.hostname.toLowerCase().replace(/^www\./, '') : '';
}
// A stable identity string for a URL: lowercased host (no www), path without a
// trailing slash, and the meaningful query params sorted (tracking dropped).
// Path/query case is preserved because job IDs can be case-sensitive.
function normalizeUrl(raw) {
const u = parseUrl(raw);
if (!u) return unescapeInput(raw).trim().toLowerCase();
const host = u.hostname.toLowerCase().replace(/^www\./, '');
const params = [];
for (const [k, v] of u.searchParams.entries()) {
if (TRACKING_PARAMS.has(k.toLowerCase())) continue;
params.push([k, v]);
}
params.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
const pathname = u.pathname.replace(/\/+$/, '');
const qs = params.length
? '?' + params.map(([k, v]) => (v === '' ? k : `${k}=${v}`)).join('&')
: '';
return host + pathname + qs;
}
// Normalise a company / job-title string for fuzzy identity matching:
// unescape, lowercase, drop gender markers like "(m/w/d)", collapse anything
// non-alphanumeric to single spaces.
function normText(s) {
return unescapeInput(s)
.toLowerCase()
.replace(/\((?:[mwdfax](?:\s*\/\s*[mwdfax])*)\)/gi, ' ')
.replace(/\b[mwfd](?:\s*\/\s*[mwfd]){1,2}\b/gi, ' ')
.replace(/[^\p{L}\p{N}]+/gu, ' ')
.replace(/\s+/g, ' ')
.trim();
}
// Normalised signature of an offer (raw or DB row), used for matching.
function offerSignature(offer) {
const o = offer || {};
return {
url_norm: normalizeUrl(o.quelle_url),
domain: urlDomain(o.quelle_url),
quelle: (o.quelle && String(o.quelle).trim()) || 'drittanbieter',
external_id: o.external_id != null && String(o.external_id).trim() !== ''
? String(o.external_id).trim() : null,
firma_norm: normText(o.firma),
stelle_norm: normText(o.stelle),
ort_norm: normText(o.ort),
};
}
// Does one blacklist row block the given signature?
function rowMatches(row, sig) {
switch (row.typ) {
case 'url':
return !!row.url_norm && row.url_norm === sig.url_norm;
case 'domain':
return !!row.domain && row.domain === sig.domain;
case 'firma':
return !!row.firma_norm && row.firma_norm === sig.firma_norm;
case 'firma_stelle':
return !!row.firma_norm && row.firma_norm === sig.firma_norm
&& !!row.stelle_norm && row.stelle_norm === sig.stelle_norm
&& (!row.ort_norm || row.ort_norm === sig.ort_norm);
case 'auto':
default:
if (row.url_norm && sig.url_norm && row.url_norm === sig.url_norm) return true;
if (row.quelle && row.external_id && sig.external_id
&& row.quelle === sig.quelle && row.external_id === sig.external_id) return true;
if (row.firma_norm && row.stelle_norm
&& row.firma_norm === sig.firma_norm && row.stelle_norm === sig.stelle_norm
&& (!row.ort_norm || row.ort_norm === sig.ort_norm)) return true;
return false;
}
}
// First blacklist row that blocks this offer, or null.
function matchBlacklist(rows, offer) {
const sig = offerSignature(offer);
for (const row of rows || []) {
if (rowMatches(row, sig)) return row;
}
return null;
}
// Column set for INSERT — keeps server.js and api.js in lock-step.
const COLUMNS = [
'typ', 'url_norm', 'domain', 'quelle', 'external_id',
'firma_norm', 'stelle_norm', 'ort_norm', 'firma', 'stelle', 'quelle_url', 'grund',
];
function emptyEntry() {
const e = {};
COLUMNS.forEach((c) => { e[c] = null; });
return e;
}
// Build a blacklist row from a manual UI/API request. Returns null if the
// chosen typ has no usable value.
function buildManualEntry(input) {
const inp = input || {};
const typ = TYPES.includes(inp.typ) ? inp.typ : 'url';
const e = emptyEntry();
e.typ = typ;
e.grund = inp.grund ? String(inp.grund) : null;
const wert = inp.wert != null ? String(inp.wert) : '';
if (typ === 'url') {
const w = wert || inp.quelle_url || '';
e.url_norm = normalizeUrl(w);
e.domain = urlDomain(w) || null;
e.quelle_url = w || null;
if (!e.url_norm) return null;
} else if (typ === 'domain') {
const w = wert || inp.domain || '';
e.domain = urlDomain(w) || normText(w).replace(/\s+/g, '') || null;
if (!e.domain) return null;
} else if (typ === 'firma') {
const w = wert || inp.firma || '';
e.firma = w || null;
e.firma_norm = normText(w);
if (!e.firma_norm) return null;
} else if (typ === 'firma_stelle') {
const f = inp.firma || '';
const s = inp.stelle || wert || '';
e.firma = f || null;
e.stelle = s || null;
e.firma_norm = normText(f);
e.stelle_norm = normText(s);
e.ort_norm = inp.ort ? normText(inp.ort) : null;
if (!e.firma_norm || !e.stelle_norm) return null;
}
return e;
}
// Build the automatic blacklist row created when an offer is deleted: it stores
// every available signature so the offer can never be ingested again.
function buildAutoEntry(offer, grund) {
const sig = offerSignature(offer);
const e = emptyEntry();
e.typ = 'auto';
e.grund = grund || null;
e.url_norm = sig.url_norm || null;
e.domain = sig.domain || null;
e.quelle = sig.external_id ? sig.quelle : null;
e.external_id = sig.external_id || null;
e.firma_norm = sig.firma_norm || null;
e.stelle_norm = sig.stelle_norm || null;
e.ort_norm = sig.ort_norm || null;
e.firma = offer && offer.firma ? offer.firma : null;
e.stelle = offer && offer.stelle ? offer.stelle : null;
e.quelle_url = offer && offer.quelle_url ? offer.quelle_url : null;
return e;
}
module.exports = {
TYPES,
COLUMNS,
normalizeUrl,
urlDomain,
normText,
offerSignature,
rowMatches,
matchBlacklist,
buildManualEntry,
buildAutoEntry,
};
+140 -5
View File
@@ -486,9 +486,11 @@ function buildOpenApiSpec(baseUrl = '') {
summary: 'Jobangebot einspielen (Upsert)',
description:
'Spielt ein Jobangebot ein. Wird von Drittanbietersoftware genutzt, um ' +
'Stellen in den Tracker zu übernehmen. Upsert über (quelle, external_id): ' +
'wiederholtes Senden desselben Angebots aktualisiert es statt ein Duplikat ' +
'anzulegen. Wird 201 (created) oder 200 (updated) zurückgegeben.',
'Stellen in den Tracker zu übernehmen. De-Dup: Upsert über (quelle, ' +
'external_id) oder falls keine external_id passt über die normalisierte ' +
'URL (quelle_url). Dieselbe Stelle landet damit nie doppelt. Steht das ' +
'Angebot auf der Blacklist (siehe /joboffers/blacklist), wird es mit 409 ' +
'abgelehnt und NICHT aufgenommen. Rückgabe 201 (created) oder 200 (updated).',
requestBody: {
required: true,
content: { 'application/json': { schema: { $ref: '#/components/schemas/JobOfferCreate' } } },
@@ -504,6 +506,66 @@ function buildOpenApiSpec(baseUrl = '') {
},
'400': errorResponse,
'401': errorResponse,
'409': {
description: 'Angebot steht auf der Blacklist und wurde nicht aufgenommen',
content: { 'application/json': { schema: { $ref: '#/components/schemas/BlacklistConflict' } } },
},
'500': errorResponse,
},
},
},
// NOTE: registered before /joboffers/{id} so "blacklist" is not read as an id.
'/joboffers/blacklist': {
get: {
tags: ['JobOffers'],
summary: 'Blacklist auflisten',
description: 'Alle Blacklist-Einträge (neueste zuerst). Blockierte Angebote werden nie eingespielt.',
responses: {
'200': {
description: 'Blacklist-Einträge',
content: {
'application/json': {
schema: { type: 'array', items: { $ref: '#/components/schemas/BlacklistEntry' } },
},
},
},
'401': errorResponse,
'500': errorResponse,
},
},
post: {
tags: ['JobOffers'],
summary: 'Blacklist-Eintrag anlegen',
description:
'Blockiert Angebote anhand einer URL, einer ganzen Domain, einer Firma ' +
'oder einer Firma + Stelle. Passende Angebote werden danach nicht mehr eingespielt.',
requestBody: {
required: true,
content: { 'application/json': { schema: { $ref: '#/components/schemas/BlacklistEntryCreate' } } },
},
responses: {
'201': {
description: 'Eintrag angelegt',
content: { 'application/json': { schema: { $ref: '#/components/schemas/BlacklistEntryResult' } } },
},
'400': errorResponse,
'401': errorResponse,
'500': errorResponse,
},
},
},
'/joboffers/blacklist/{id}': {
delete: {
tags: ['JobOffers'],
summary: 'Blacklist-Eintrag entfernen',
description: 'Entfernt einen Block. Betroffene Angebote können danach wieder eingespielt werden.',
parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'integer' } }],
responses: {
'200': { description: 'Entfernt', content: { 'application/json': { schema: { $ref: '#/components/schemas/Ok' } } } },
'401': errorResponse,
'404': errorResponse,
'500': errorResponse,
},
},
@@ -527,9 +589,22 @@ function buildOpenApiSpec(baseUrl = '') {
delete: {
tags: ['JobOffers'],
summary: 'Jobangebot löschen',
parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'integer' } }],
description:
'Löscht ein Jobangebot. Standardmäßig wird es zuvor automatisch auf die ' +
'Blacklist gesetzt, damit dieselbe Stelle nie erneut eingespielt wird. Mit ' +
'?blacklist=false wird ohne Blockierung hart gelöscht.',
parameters: [
{ name: 'id', in: 'path', required: true, schema: { type: 'integer' } },
{
name: 'blacklist',
in: 'query',
required: false,
schema: { type: 'boolean', default: true },
description: 'false = löschen ohne Blacklisting (Standard: true).',
},
],
responses: {
'200': { description: 'Gelöscht', content: { 'application/json': { schema: { $ref: '#/components/schemas/Ok' } } } },
'200': { description: 'Gelöscht', content: { 'application/json': { schema: { $ref: '#/components/schemas/JobOfferDeleteResult' } } } },
'401': errorResponse,
'404': errorResponse,
'500': errorResponse,
@@ -788,6 +863,7 @@ function buildOpenApiSpec(baseUrl = '') {
kontakt_email: { type: 'string', format: 'email', nullable: true, description: 'Kontakt-E-Mail-Adresse der Stelle' },
status: { type: 'string', enum: ['offen', 'uebernommen', 'abgelehnt'] },
verknuepfte_bewerbung_id: { type: 'integer', nullable: true },
url_norm: { type: 'string', nullable: true, description: 'Normalisierte URL für die De-Duplizierung (serverseitig gesetzt)' },
created_at: { type: 'string', format: 'date-time' },
updated_at: { type: 'string', format: 'date-time' },
},
@@ -818,6 +894,65 @@ function buildOpenApiSpec(baseUrl = '') {
joboffer: { $ref: '#/components/schemas/JobOffer' },
},
},
JobOfferDeleteResult: {
type: 'object',
properties: {
success: { type: 'boolean', example: true },
blacklisted: { type: 'boolean', description: 'true, wenn das Angebot beim Löschen auf die Blacklist gesetzt wurde' },
},
},
BlacklistEntry: {
type: 'object',
description: 'Ein Blacklist-Eintrag. Nur die zum typ passenden Felder sind gesetzt.',
properties: {
id: { type: 'integer' },
typ: { type: 'string', enum: ['url', 'domain', 'firma', 'firma_stelle', 'auto'] },
url_norm: { type: 'string', nullable: true, description: 'Normalisierte URL (Tracking-Parameter entfernt)' },
domain: { type: 'string', nullable: true },
quelle: { type: 'string', nullable: true },
external_id: { type: 'string', nullable: true },
firma_norm: { type: 'string', nullable: true },
stelle_norm: { type: 'string', nullable: true },
ort_norm: { type: 'string', nullable: true },
firma: { type: 'string', nullable: true },
stelle: { type: 'string', nullable: true },
quelle_url: { type: 'string', nullable: true },
grund: { type: 'string', nullable: true },
created_at: { type: 'string', format: 'date-time' },
},
},
BlacklistEntryCreate: {
type: 'object',
required: ['typ'],
properties: {
typ: {
type: 'string',
enum: ['url', 'domain', 'firma', 'firma_stelle'],
description: 'url/domain/firma nutzen "wert"; firma_stelle nutzt "firma"+"stelle" (+optional "ort").',
},
wert: { type: 'string', description: 'URL (typ=url), Domain (typ=domain) oder Firmenname (typ=firma).' },
firma: { type: 'string', description: 'Nur bei typ=firma_stelle.' },
stelle: { type: 'string', description: 'Nur bei typ=firma_stelle.' },
ort: { type: 'string', description: 'Optional bei typ=firma_stelle.' },
grund: { type: 'string', description: 'Optionaler Freitext-Grund.' },
},
},
BlacklistEntryResult: {
type: 'object',
properties: {
success: { type: 'boolean', example: true },
entry: { $ref: '#/components/schemas/BlacklistEntry' },
},
},
BlacklistConflict: {
type: 'object',
properties: {
blacklisted: { type: 'boolean', example: true },
matched_by: { type: 'string', description: 'typ des greifenden Blacklist-Eintrags' },
blacklist_entry: { $ref: '#/components/schemas/BlacklistEntry' },
error: { type: 'string' },
},
},
},
},
security: [{ ApiKeyAuth: [] }],
+109 -2
View File
@@ -31,6 +31,7 @@ const { generateApplicationDocuments, generateEmailReply } = require('./lib/docu
const mailer = require('./lib/mailer');
const { createExternalApi } = require('./lib/api');
const { buildOpenApiSpec } = require('./lib/openapi');
const blacklist = require('./lib/blacklist');
const app = express();
const PORT = process.env.PORT || 3000;
@@ -244,6 +245,30 @@ function dbRun(sql, params = []) {
});
}
// ---------------------------------------------------------------------------
// Job-offer blacklist helpers (shared shape with lib/api.js)
// ---------------------------------------------------------------------------
// Insert one prepared blacklist entry (see lib/blacklist.buildManual/AutoEntry).
async function insertBlacklistEntry(entry) {
const cols = blacklist.COLUMNS;
const placeholders = cols.map(() => '?').join(', ');
const values = cols.map((c) => (entry[c] === undefined ? null : entry[c]));
return dbRun(
`INSERT INTO jobangebote_blacklist (${cols.join(', ')}) VALUES (${placeholders})`,
values
);
}
// Auto-blacklist an offer row so it can never be ingested again, then it is safe
// to delete. Skips silently if the offer is already covered by an entry.
async function autoBlacklistOffer(offer, grund) {
if (!offer) return;
const rows = await dbAll('SELECT * FROM jobangebote_blacklist');
if (blacklist.matchBlacklist(rows, offer)) return; // already blocked
await insertBlacklistEntry(blacklist.buildAutoEntry(offer, grund));
}
// ---------------------------------------------------------------------------
// E-Mail correspondence: IMAP polling, storing & matching incoming replies
// ---------------------------------------------------------------------------
@@ -632,6 +657,39 @@ function initializeDatabase() {
// Migration: add columns to pre-existing jobangebote tables.
db.run('ALTER TABLE jobangebote ADD COLUMN anzeige_datum DATE', () => {});
db.run('ALTER TABLE jobangebote ADD COLUMN kontakt_email TEXT', () => {});
// Normalized URL for URL-based de-duplication of offers (see lib/blacklist).
db.run('ALTER TABLE jobangebote ADD COLUMN url_norm TEXT', () => {
// Backfill url_norm for rows ingested before this column existed.
db.all('SELECT id, quelle_url FROM jobangebote WHERE url_norm IS NULL AND quelle_url IS NOT NULL AND quelle_url != ""', (err, rows) => {
if (err || !rows) return;
rows.forEach((r) => {
const norm = blacklist.normalizeUrl(r.quelle_url);
if (norm) db.run('UPDATE jobangebote SET url_norm = ? WHERE id = ?', [norm, r.id], () => {});
});
});
});
db.run('CREATE INDEX IF NOT EXISTS idx_jobangebote_url_norm ON jobangebote(url_norm)', () => {});
// Blacklist of job offers that must never (re)appear in the list. A
// deleted offer is auto-blacklisted; manual entries can block a URL,
// a whole domain, a company, or a specific company+title posting.
db.run(`
CREATE TABLE IF NOT EXISTS jobangebote_blacklist (
id INTEGER PRIMARY KEY AUTOINCREMENT,
typ TEXT NOT NULL DEFAULT 'auto',
url_norm TEXT,
domain TEXT,
quelle TEXT,
external_id TEXT,
firma_norm TEXT,
stelle_norm TEXT,
ort_norm TEXT,
firma TEXT,
stelle TEXT,
quelle_url TEXT,
grund TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`, () => {});
// Remember the last recipient address per application (prefill).
db.run('ALTER TABLE bewerbungen ADD COLUMN email_empfaenger TEXT', () => {});
@@ -1676,10 +1734,16 @@ initializeDatabase().then(() => {
}
});
// Delete a job offer (cascades nothing — verknuepfte_bewerbung_id is SET NULL).
// Delete a job offer. Deleting always blacklists it first, so the same offer
// can never be ingested/listed again (the requirement: never appears twice,
// even after deletion). The entry can be removed later on /blacklist.
app.post('/jobangebote/:id/delete', async (req, res) => {
try {
await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]);
const offer = await dbGet('SELECT * FROM jobangebote WHERE id = ?', [req.params.id]);
if (offer) {
await autoBlacklistOffer(offer, 'Jobangebot gelöscht (Web-UI)');
await dbRun('DELETE FROM jobangebote WHERE id = ?', [req.params.id]);
}
res.redirect('/jobangebote');
} catch (error) {
console.error('Error deleting job offer:', error);
@@ -1687,6 +1751,49 @@ initializeDatabase().then(() => {
}
});
// Blacklist management page — see what is blocked and remove entries.
app.get('/blacklist', async (req, res) => {
try {
const eintraege = await dbAll(
'SELECT * FROM jobangebote_blacklist ORDER BY created_at DESC, id DESC'
);
res.render('blacklist', { eintraege, blacklistTypen: blacklist.TYPES, hideSettings: false });
} catch (error) {
console.error('Error listing blacklist:', error);
res.status(500).send('Serverfehler');
}
});
// Add a manual blacklist entry (URL / domain / company / company+title).
app.post('/blacklist', async (req, res) => {
try {
const entry = blacklist.buildManualEntry({
typ: req.body.typ,
wert: sanitizeInput(req.body.wert || ''),
firma: sanitizeInput(req.body.firma || ''),
stelle: sanitizeInput(req.body.stelle || ''),
ort: sanitizeInput(req.body.ort || ''),
grund: sanitizeInput(req.body.grund || ''),
});
if (entry) await insertBlacklistEntry(entry);
res.redirect('/blacklist');
} catch (error) {
console.error('Error adding blacklist entry:', error);
res.status(500).send('Serverfehler');
}
});
// Remove a blacklist entry (offer may then be ingested again).
app.post('/blacklist/:id/delete', async (req, res) => {
try {
await dbRun('DELETE FROM jobangebote_blacklist WHERE id = ?', [req.params.id]);
res.redirect('/blacklist');
} catch (error) {
console.error('Error deleting blacklist entry:', error);
res.status(500).send('Serverfehler');
}
});
// ----- Third-party REST API (/api/v1) + OpenAPI/Swagger -----
// API key for third-party software. When unset, the API responds 503 on
// every endpoint except /health — it never silently exposes data.
+196
View File
@@ -0,0 +1,196 @@
<!DOCTYPE html>
<html lang="de">
<head>
<%- include('partials/head') %>
</head>
<body class="min-h-screen transition-colors duration-300 bg-gray-50 dark:bg-gray-900 text-gray-900 dark:text-gray-100" id="body">
<%- include('partials/header') %>
<main class="container mx-auto px-4 py-8 max-w-5xl">
<!-- Back link -->
<a href="/jobangebote" class="inline-flex items-center gap-2 text-sm text-blue-600 dark:text-blue-400 hover:underline mb-6">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 19l-7-7m0 0l7-7m-7 7h18"></path>
</svg>
Zurück zu den Jobangeboten
</a>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-md p-6 mb-8">
<div class="flex flex-wrap items-start justify-between gap-3 mb-2">
<div>
<h2 class="text-lg font-semibold text-gray-800 dark:text-white">Blacklist</h2>
<p class="text-sm text-gray-500 dark:text-gray-400 mt-1 max-w-2xl">
Angebote auf der Blacklist werden nie (erneut) in die Jobangebote
aufgenommen. Ein <strong>gelöschtes</strong> Jobangebot landet automatisch
hier, damit dieselbe Stelle nicht doppelt auftaucht. Du kannst auch manuell
eine URL, eine ganze Domain, eine Firma oder eine konkrete Firma + Stelle
blockieren. Einträge lassen sich jederzeit wieder entfernen.
</p>
</div>
<span class="shrink-0 inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-xs font-medium bg-red-50 dark:bg-red-900/40 text-red-700 dark:text-red-300 border border-red-200 dark:border-red-800">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M18.364 18.364A9 9 0 005.636 5.636m12.728 12.728A9 9 0 015.636 5.636m12.728 12.728L5.636 5.636"></path></svg>
<%= eintraege.length %> Eintrag/Einträge
</span>
</div>
<% if (eintraege && eintraege.length) { %>
<ul class="mt-4 space-y-3">
<% eintraege.forEach(function(e){
var typLabel = { url: 'URL', domain: 'Domain', firma: 'Firma', firma_stelle: 'Firma + Stelle', auto: 'Automatisch (gelöscht)' }[e.typ] || e.typ;
var typClass = e.typ === 'auto'
? 'bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-200'
: 'bg-purple-100 text-purple-800 dark:bg-purple-900/50 dark:text-purple-200';
var wert = '';
if (e.typ === 'url') wert = e.quelle_url || e.url_norm || '';
else if (e.typ === 'domain') wert = e.domain || '';
else if (e.typ === 'firma') wert = e.firma || e.firma_norm || '';
else if (e.typ === 'firma_stelle') wert = [e.firma, e.stelle].filter(Boolean).join(' · ');
else wert = [e.firma, e.stelle].filter(Boolean).join(' · ') || e.quelle_url || e.url_norm || e.domain || '';
%>
<li class="rounded-lg border border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-700/40 px-4 py-3">
<div class="flex flex-wrap items-start justify-between gap-3">
<div class="min-w-0 flex-1">
<div class="flex flex-wrap items-center gap-2 mb-1">
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium <%= typClass %>"><%= typLabel %></span>
<% if (e.typ === 'auto' && e.quelle_url) { %>
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-gray-200 text-gray-600 dark:bg-gray-700 dark:text-gray-300">URL erfasst</span>
<% } %>
<% if (e.external_id) { %>
<span class="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-gray-200 text-gray-600 dark:bg-gray-700 dark:text-gray-300">ext. ID</span>
<% } %>
</div>
<p class="text-sm font-medium text-gray-800 dark:text-gray-100 break-all"><%= wert || '(kein Wert)' %></p>
<% if (e.typ === 'auto' && e.quelle_url && wert !== e.quelle_url) { %>
<p class="text-xs text-gray-500 dark:text-gray-400 break-all mt-0.5"><%= e.quelle_url %></p>
<% } %>
<% if (e.grund) { %>
<p class="text-xs text-gray-500 dark:text-gray-400 mt-1">Grund: <%= e.grund %></p>
<% } %>
<p class="text-xs text-gray-400 dark:text-gray-500 mt-1">
Hinzugefügt: <%= e.created_at ? new Date(e.created_at + 'Z').toLocaleString('de-DE') : '' %>
</p>
</div>
<form action="/blacklist/<%= e.id %>/delete" method="POST"
onsubmit="return confirm('Blacklist-Eintrag entfernen? Das Angebot kann danach wieder eingespielt werden.');">
<button type="submit"
class="inline-flex items-center justify-center gap-1.5 px-3 py-1.5 text-xs border border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-100 dark:hover:bg-gray-700 rounded-md transition-colors">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path></svg>
Entfernen
</button>
</form>
</div>
</li>
<% }); %>
</ul>
<% } else { %>
<div class="text-center py-10">
<svg class="w-14 h-14 mx-auto text-gray-300 dark:text-gray-600 mb-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z"></path>
</svg>
<p class="text-sm text-gray-500 dark:text-gray-400">Die Blacklist ist leer.</p>
</div>
<% } %>
</div>
<!-- Add a manual blacklist entry -->
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-md p-6">
<h3 class="text-base font-semibold text-gray-800 dark:text-white mb-4">Eintrag manuell hinzufügen</h3>
<form action="/blacklist" method="POST" class="space-y-4">
<div>
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1" for="bl-typ">Typ</label>
<select id="bl-typ" name="typ"
class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-3 py-2 text-sm">
<option value="url">URL blockieren</option>
<option value="domain">Ganze Domain blockieren</option>
<option value="firma">Firma komplett blockieren</option>
<option value="firma_stelle">Firma + Stelle blockieren</option>
</select>
</div>
<div data-when="url domain firma">
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1" for="bl-wert">
<span data-label="url">URL</span>
<span data-label="domain" class="hidden">Domain (z. B. beispiel.de)</span>
<span data-label="firma" class="hidden">Firmenname</span>
</label>
<input id="bl-wert" name="wert" type="text" autocomplete="off"
class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-3 py-2 text-sm"
placeholder="https://…">
</div>
<div data-when="firma_stelle" class="hidden grid gap-4 sm:grid-cols-3">
<div>
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1" for="bl-firma">Firma</label>
<input id="bl-firma" name="firma" type="text"
class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-3 py-2 text-sm">
</div>
<div>
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1" for="bl-stelle">Stelle</label>
<input id="bl-stelle" name="stelle" type="text"
class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-3 py-2 text-sm">
</div>
<div>
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1" for="bl-ort">Ort <span class="text-gray-400">(optional)</span></label>
<input id="bl-ort" name="ort" type="text"
class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-3 py-2 text-sm">
</div>
</div>
<div>
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1" for="bl-grund">Grund <span class="text-gray-400">(optional)</span></label>
<input id="bl-grund" name="grund" type="text"
class="w-full rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-3 py-2 text-sm"
placeholder="z. B. Zeitarbeit, unseriös, kein Interesse …">
</div>
<button type="submit"
class="inline-flex items-center gap-2 px-4 py-2 text-sm bg-blue-600 hover:bg-blue-700 text-white rounded-md transition-colors">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 4v16m8-8H4"></path></svg>
Zur Blacklist hinzufügen
</button>
</form>
</div>
</main>
<%- include('partials/footer') %>
<script>
(function () {
const root = document.documentElement;
const dm = localStorage.getItem('darkMode');
if (dm === 'enabled' || (!dm && window.matchMedia('(prefers-color-scheme: dark)').matches)) {
root.classList.add('dark');
}
const toggle = document.getElementById('darkModeToggle');
const sun = document.getElementById('sunIcon');
const moon = document.getElementById('moonIcon');
function sync() {
const d = root.classList.contains('dark');
if (sun) sun.classList.toggle('hidden', d);
if (moon) moon.classList.toggle('hidden', !d);
}
sync();
if (toggle) toggle.addEventListener('click', () => {
root.classList.toggle('dark');
localStorage.setItem('darkMode', root.classList.contains('dark') ? 'enabled' : 'disabled');
sync();
});
const cy = document.getElementById('currentYear');
if (cy) cy.textContent = new Date().getFullYear();
// Show only the fields relevant to the chosen blacklist type.
const typ = document.getElementById('bl-typ');
function applyTyp() {
const v = typ.value;
document.querySelectorAll('[data-when]').forEach(function (el) {
el.classList.toggle('hidden', el.getAttribute('data-when').split(' ').indexOf(v) === -1);
});
document.querySelectorAll('[data-label]').forEach(function (el) {
el.classList.toggle('hidden', el.getAttribute('data-label') !== v);
});
}
if (typ) { typ.addEventListener('change', applyTyp); applyTyp(); }
})();
</script>
</body>
</html>
+11 -5
View File
@@ -26,10 +26,16 @@
als Bewerbung-Entwurf übernehmen.
</p>
</div>
<span class="shrink-0 inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-xs font-medium bg-blue-50 dark:bg-blue-900/40 text-blue-700 dark:text-blue-300 border border-blue-200 dark:border-blue-800">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"></path></svg>
<%= jobangebote.length %> Angebot(e)
</span>
<div class="shrink-0 flex items-center gap-2">
<a href="/blacklist" class="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-xs font-medium bg-red-50 dark:bg-red-900/40 text-red-700 dark:text-red-300 border border-red-200 dark:border-red-800 hover:bg-red-100 dark:hover:bg-red-900/60 transition-colors" title="Geblockte Angebote verwalten">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M18.364 18.364A9 9 0 005.636 5.636m12.728 12.728A9 9 0 015.636 5.636m12.728 12.728L5.636 5.636"></path></svg>
Blacklist
</a>
<span class="inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-xs font-medium bg-blue-50 dark:bg-blue-900/40 text-blue-700 dark:text-blue-300 border border-blue-200 dark:border-blue-800">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"></path></svg>
<%= jobangebote.length %> Angebot(e)
</span>
</div>
</div>
<% if (jobangebote && jobangebote.length) { %>
@@ -124,7 +130,7 @@
</form>
<% } %>
<form action="/jobangebote/<%= j.id %>/delete" method="POST"
onsubmit="return confirm('Jobangebot „<%= j.firma %> · <%= j.stelle %>“ löschen?');">
onsubmit="return confirm('Jobangebot „<%= j.firma %> · <%= j.stelle %>“ löschen? Es kommt auf die Blacklist und wird nicht erneut eingespielt.');">
<button type="submit"
class="inline-flex items-center justify-center gap-1.5 px-3 py-1.5 text-xs border border-gray-300 dark:border-gray-600 text-gray-600 dark:text-gray-300 hover:bg-gray-100 dark:hover:bg-gray-700 rounded-md transition-colors w-full">
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"></path></svg>
+10
View File
@@ -32,6 +32,16 @@
<span id="jobangeboteBadge" class="hidden absolute -top-1 -right-1 min-w-[18px] h-[18px] px-1 flex items-center justify-center rounded-full bg-green-500 text-white text-[10px] font-bold ring-2 ring-blue-800 dark:ring-gray-900">0</span>
</a>
<!-- Blacklist (blocked job offers) link -->
<a href="/blacklist"
class="flex items-center gap-1.5 px-3 py-2 rounded-md bg-white/20 hover:bg-white/30 transition-colors text-white text-sm font-medium"
title="Geblockte Jobangebote (Blacklist) verwalten">
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M18.364 18.364A9 9 0 005.636 5.636m12.728 12.728A9 9 0 015.636 5.636m12.728 12.728L5.636 5.636"></path>
</svg>
<span class="hidden sm:inline">Blacklist</span>
</a>
<!-- Vorlagen (base documents) link -->
<a href="/vorlagen"
class="flex items-center gap-1.5 px-3 py-2 rounded-md bg-white/20 hover:bg-white/30 transition-colors text-white text-sm font-medium"