From f935e00b53ef45598f6c615d1ac7684621e9b90a Mon Sep 17 00:00:00 2001 From: Thomas Hackner Date: Mon, 6 Jul 2026 22:25:59 +0200 Subject: [PATCH] Add robust company slug for job-offer blacklist matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Company matching relied on normText(), which kept legal-form suffixes, spacing and umlaut spelling — so "Bosch GmbH", "Bosch AG" and "bosch gmbh" did not match the same entry, making the blacklist unreliable. Introduce firmaSlug(): transliterate German umlauts, strip diacritics and trailing legal-form tokens (GmbH/AG/SE/KG/…), then kebab-join. Add a firma_slug field to jobangebote_blacklist (schema + ALTER/backfill migration) and match on it for typ firma/firma_stelle/auto, falling back to firma_norm for legacy rows. POST /joboffers rejects blacklisted offers via matchBlacklist automatically since offerSignature now carries the slug. Co-Authored-By: Claude Opus 4.8 --- lib/blacklist.js | 63 ++++++++++++++++++++++++++++++++++++++++++------ lib/openapi.js | 1 + server.js | 20 +++++++++++++++ 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/lib/blacklist.js b/lib/blacklist.js index e47874d..1bf56ac 100644 --- a/lib/blacklist.js +++ b/lib/blacklist.js @@ -8,9 +8,10 @@ // relevant to its typ are set: // 'url' -> url_norm (exact normalized URL) // 'domain' -> domain (whole host, e.g. a job board) -// 'firma' -> firma_norm (block a company entirely) -// 'firma_stelle' -> firma_norm + stelle_norm (+ optional ort_norm) -// 'auto' -> url_norm / quelle+external_id / firma_norm+stelle_norm +// 'firma' -> firma_slug (block a company entirely; slug is a +// legal-form/umlaut-robust key, falls back to firma_norm) +// 'firma_stelle' -> firma_slug + stelle_norm (+ optional ort_norm) +// 'auto' -> url_norm / quelle+external_id / firma_slug+stelle_norm // (created on delete; matches on ANY stored signature) const TYPES = ['url', 'domain', 'firma', 'firma_stelle', 'auto']; @@ -72,6 +73,40 @@ function normalizeUrl(raw) { return host + pathname + qs; } +// Legal-form / entity suffixes stripped from a company name before slugging, +// so "Bosch GmbH", "Bosch AG", "Bosch GmbH & Co. KG" and "Bosch" all collapse +// to the same slug. Matched as whole space-separated tokens at the end. +const LEGAL_FORM_TOKENS = new Set([ + 'gmbh', 'ggmbh', 'mbh', 'ug', 'haftungsbeschraenkt', 'ag', 'kg', 'kgaa', + 'ohg', 'gbr', 'se', 'ek', 'eg', 'ev', 'partg', 'partmbb', 'co', 'cie', + 'inc', 'incorporated', 'llc', 'ltd', 'limited', 'plc', 'corp', 'corporation', + 'company', 'sa', 'sarl', 'sas', 'bv', 'nv', 'oy', 'ab', 'as', 'aps', 'srl', 'spa', +]); + +// A robust matching key for a company name. Beyond normText it transliterates +// German umlauts (ä→ae …), strips diacritics, drops trailing legal-form tokens +// and joins the rest with hyphens. This is what the blacklist compares on so +// that entity form, spacing and umlaut spelling no longer cause misses. +function firmaSlug(s) { + let t = unescapeInput(s) + .toLowerCase() + .replace(/ä/g, 'ae').replace(/ö/g, 'oe').replace(/ü/g, 'ue').replace(/ß/g, 'ss') + .normalize("NFKD").replace(/[\u0300-\u036f]/g, "") + .replace(/\((?:[mwdfax](?:\s*\/\s*[mwdfax])*)\)/gi, ' ') + .replace(/[^a-z0-9]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!t) return ''; + const words = t.split(' '); + const kept = words.slice(); + // Drop legal-form tokens from the end (e.g. "gmbh & co kg" → "gmbh co kg"), + // but never strip everything away. + while (kept.length > 1 && LEGAL_FORM_TOKENS.has(kept[kept.length - 1])) { + kept.pop(); + } + return (kept.length ? kept : words).join('-'); +} + // Normalise a company / job-title string for fuzzy identity matching: // unescape, lowercase, drop gender markers like "(m/w/d)", collapse anything // non-alphanumeric to single spaces. @@ -95,11 +130,19 @@ function offerSignature(offer) { external_id: o.external_id != null && String(o.external_id).trim() !== '' ? String(o.external_id).trim() : null, firma_norm: normText(o.firma), + firma_slug: firmaSlug(o.firma), stelle_norm: normText(o.stelle), ort_norm: normText(o.ort), }; } +// Company identity match: prefer the robust slug, fall back to firma_norm for +// legacy rows written before firma_slug existed (or not yet backfilled). +function firmaMatch(row, sig) { + if (row.firma_slug && sig.firma_slug) return row.firma_slug === sig.firma_slug; + return !!row.firma_norm && row.firma_norm === sig.firma_norm; +} + // Does one blacklist row block the given signature? function rowMatches(row, sig) { switch (row.typ) { @@ -108,9 +151,9 @@ function rowMatches(row, sig) { case 'domain': return !!row.domain && row.domain === sig.domain; case 'firma': - return !!row.firma_norm && row.firma_norm === sig.firma_norm; + return firmaMatch(row, sig); case 'firma_stelle': - return !!row.firma_norm && row.firma_norm === sig.firma_norm + return firmaMatch(row, sig) && !!row.stelle_norm && row.stelle_norm === sig.stelle_norm && (!row.ort_norm || row.ort_norm === sig.ort_norm); case 'auto': @@ -118,8 +161,8 @@ function rowMatches(row, sig) { if (row.url_norm && sig.url_norm && row.url_norm === sig.url_norm) return true; if (row.quelle && row.external_id && sig.external_id && row.quelle === sig.quelle && row.external_id === sig.external_id) return true; - if (row.firma_norm && row.stelle_norm - && row.firma_norm === sig.firma_norm && row.stelle_norm === sig.stelle_norm + if ((row.firma_slug || row.firma_norm) && row.stelle_norm + && firmaMatch(row, sig) && row.stelle_norm === sig.stelle_norm && (!row.ort_norm || row.ort_norm === sig.ort_norm)) return true; return false; } @@ -137,7 +180,7 @@ function matchBlacklist(rows, offer) { // Column set for INSERT — keeps server.js and api.js in lock-step. const COLUMNS = [ 'typ', 'url_norm', 'domain', 'quelle', 'external_id', - 'firma_norm', 'stelle_norm', 'ort_norm', 'firma', 'stelle', 'quelle_url', 'grund', + 'firma_norm', 'firma_slug', 'stelle_norm', 'ort_norm', 'firma', 'stelle', 'quelle_url', 'grund', ]; function emptyEntry() { @@ -170,6 +213,7 @@ function buildManualEntry(input) { const w = wert || inp.firma || ''; e.firma = w || null; e.firma_norm = normText(w); + e.firma_slug = firmaSlug(w) || null; if (!e.firma_norm) return null; } else if (typ === 'firma_stelle') { const f = inp.firma || ''; @@ -177,6 +221,7 @@ function buildManualEntry(input) { e.firma = f || null; e.stelle = s || null; e.firma_norm = normText(f); + e.firma_slug = firmaSlug(f) || null; e.stelle_norm = normText(s); e.ort_norm = inp.ort ? normText(inp.ort) : null; if (!e.firma_norm || !e.stelle_norm) return null; @@ -196,6 +241,7 @@ function buildAutoEntry(offer, grund) { e.quelle = sig.external_id ? sig.quelle : null; e.external_id = sig.external_id || null; e.firma_norm = sig.firma_norm || null; + e.firma_slug = sig.firma_slug || null; e.stelle_norm = sig.stelle_norm || null; e.ort_norm = sig.ort_norm || null; e.firma = offer && offer.firma ? offer.firma : null; @@ -210,6 +256,7 @@ module.exports = { normalizeUrl, urlDomain, normText, + firmaSlug, offerSignature, rowMatches, matchBlacklist, diff --git a/lib/openapi.js b/lib/openapi.js index c33ef54..c1ca45e 100644 --- a/lib/openapi.js +++ b/lib/openapi.js @@ -938,6 +938,7 @@ function buildOpenApiSpec(baseUrl = '') { quelle: { type: 'string', nullable: true }, external_id: { type: 'string', nullable: true }, firma_norm: { type: 'string', nullable: true }, + firma_slug: { type: 'string', nullable: true, description: 'Rechtsform-/Umlaut-robuster Firmen-Slug; primärer Abgleich für typ=firma/firma_stelle/auto.' }, stelle_norm: { type: 'string', nullable: true }, ort_norm: { type: 'string', nullable: true }, firma: { type: 'string', nullable: true }, diff --git a/server.js b/server.js index fd0a30b..43c2353 100644 --- a/server.js +++ b/server.js @@ -815,6 +815,7 @@ function initializeDatabase() { quelle TEXT, external_id TEXT, firma_norm TEXT, + firma_slug TEXT, stelle_norm TEXT, ort_norm TEXT, firma TEXT, @@ -824,6 +825,25 @@ function initializeDatabase() { created_at DATETIME DEFAULT CURRENT_TIMESTAMP ) `, () => {}); + // Add + backfill firma_slug on pre-existing installs. The slug is a + // legal-form/umlaut-robust company key that the blacklist now matches + // on; backfill from the stored original company name so old company + // entries start blocking reliably too. + db.run('ALTER TABLE jobangebote_blacklist ADD COLUMN firma_slug TEXT', () => { + db.all( + `SELECT id, firma, firma_norm FROM jobangebote_blacklist + WHERE firma_slug IS NULL AND (firma IS NOT NULL OR firma_norm IS NOT NULL)`, + (err, rows) => { + if (err || !rows) return; + for (const r of rows) { + const slug = blacklist.firmaSlug(r.firma || r.firma_norm || ''); + if (slug) { + db.run('UPDATE jobangebote_blacklist SET firma_slug = ? WHERE id = ?', [slug, r.id], () => {}); + } + } + } + ); + }); // Remember the last recipient address per application (prefill). db.run('ALTER TABLE bewerbungen ADD COLUMN email_empfaenger TEXT', () => {});