jobbi-bewerbung/lib/blacklist.js

// Job-offer blacklist: normalization + matching helpers.
//
// Shared by the web routes (server.js) and the REST API (lib/api.js) so both
// use identical rules. A blacklisted offer is never ingested again, and every
// deleted offer is auto-blacklisted so it can never reappear in the list.
//
// A blacklist row has a `typ` and a set of normalized match keys; only the keys
// relevant to its typ are set:
//   'url'          -> url_norm            (exact normalized URL)
//   'domain'       -> domain              (whole host, e.g. a job board)
//   'firma'        -> firma_slug          (block a company entirely; slug is a
//                     legal-form/umlaut-robust key, falls back to firma_norm)
//   'firma_stelle' -> firma_slug + stelle_norm (+ optional ort_norm)
//   'auto'         -> url_norm / quelle+external_id / firma_slug+stelle_norm
//                     (created on delete; matches on ANY stored signature)

const TYPES = ['url', 'domain', 'firma', 'firma_stelle', 'auto'];

// Query params that carry no identity — stripped before comparing URLs so the
// same posting with different tracking/referrer params still matches.
const TRACKING_PARAMS = new Set([
  'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id',
  'gclid', 'fbclid', 'msclkid', 'mc_cid', 'mc_eid', 'yclid', 'igshid', 'gh_src',
  'ref', 'referrer', 'source', 'src', 'from', 'campaignid', 'cmpid', 'cmp',
  'vjk', 'tk', 'rgtk', 'hl', 'spa', 'sp', 'advn', 'adid', 'trk', 'trkid',
]);

// Reverse the web layer's HTML sanitisation (sanitizeInput encodes < > " ')
// so a value stored via the UI normalises to the same thing as the raw input.
function unescapeInput(s) {
  return String(s == null ? '' : s)
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'");
}

const HAS_SCHEME = /^[a-z][a-z0-9+.-]*:\/\//i;

function parseUrl(raw) {
  const s = unescapeInput(raw).trim();
  if (!s) return null;
  try {
    return new URL(HAS_SCHEME.test(s) ? s : 'https://' + s);
  } catch (e) {
    return null;
  }
}

// Host without a leading "www.", lowercased.
function urlDomain(raw) {
  const u = parseUrl(raw);
  return u ? u.hostname.toLowerCase().replace(/^www\./, '') : '';
}

// A stable identity string for a URL: lowercased host (no www), path without a
// trailing slash, and the meaningful query params sorted (tracking dropped).
// Path/query case is preserved because job IDs can be case-sensitive.
function normalizeUrl(raw) {
  const u = parseUrl(raw);
  if (!u) return unescapeInput(raw).trim().toLowerCase();
  const host = u.hostname.toLowerCase().replace(/^www\./, '');
  const params = [];
  for (const [k, v] of u.searchParams.entries()) {
    if (TRACKING_PARAMS.has(k.toLowerCase())) continue;
    params.push([k, v]);
  }
  params.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
  const pathname = u.pathname.replace(/\/+$/, '');
  const qs = params.length
    ? '?' + params.map(([k, v]) => (v === '' ? k : `${k}=${v}`)).join('&')
    : '';
  return host + pathname + qs;
}

// Legal-form / entity suffixes stripped from a company name before slugging,
// so "Bosch GmbH", "Bosch AG", "Bosch GmbH & Co. KG" and "Bosch" all collapse
// to the same slug. Matched as whole space-separated tokens at the end.
const LEGAL_FORM_TOKENS = new Set([
  'gmbh', 'ggmbh', 'mbh', 'ug', 'haftungsbeschraenkt', 'ag', 'kg', 'kgaa',
  'ohg', 'gbr', 'se', 'ek', 'eg', 'ev', 'partg', 'partmbb', 'co', 'cie',
  'inc', 'incorporated', 'llc', 'ltd', 'limited', 'plc', 'corp', 'corporation',
  'company', 'sa', 'sarl', 'sas', 'bv', 'nv', 'oy', 'ab', 'as', 'aps', 'srl', 'spa',
]);

// A robust matching key for a company name. Beyond normText it transliterates
// German umlauts (ä→ae …), strips diacritics, drops trailing legal-form tokens
// and joins the rest with hyphens. This is what the blacklist compares on so
// that entity form, spacing and umlaut spelling no longer cause misses.
function firmaSlug(s) {
  let t = unescapeInput(s)
    .toLowerCase()
    .replace(/ä/g, 'ae').replace(/ö/g, 'oe').replace(/ü/g, 'ue').replace(/ß/g, 'ss')
    .normalize("NFKD").replace(/[\u0300-\u036f]/g, "")
    .replace(/\((?:[mwdfax](?:\s*\/\s*[mwdfax])*)\)/gi, ' ')
    .replace(/[^a-z0-9]+/g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
  if (!t) return '';
  const words = t.split(' ');
  const kept = words.slice();
  // Drop legal-form tokens from the end (e.g. "gmbh & co kg" → "gmbh co kg"),
  // but never strip everything away.
  while (kept.length > 1 && LEGAL_FORM_TOKENS.has(kept[kept.length - 1])) {
    kept.pop();
  }
  return (kept.length ? kept : words).join('-');
}

// Do two company slugs refer to the SAME employer? True when they are equal, or
// when the shorter one's hyphen tokens are a leading prefix of the longer one's.
// The prefix rule collapses "short name" vs "full legal name" spellings of one
// firm — e.g. "it-problemloeser" vs "it-problemloeser-verwaltungs-und-handels",
// or "stadtwerke-berlin" vs "stadtwerke-berlin-netz". To avoid merging distinct
// firms that merely share a first word ("meyer-it" vs "meyer-logistik"), a pure
// prefix match requires the shorter slug to carry at least two tokens; a
// single-token slug only matches an identical one.
function sameCompany(a, b) {
  if (!a || !b) return false;
  if (a === b) return true;
  const ta = String(a).split('-').filter(Boolean);
  const tb = String(b).split('-').filter(Boolean);
  const [short, long] = ta.length <= tb.length ? [ta, tb] : [tb, ta];
  if (short.length < 2) return false; // single distinctive token → exact only
  for (let i = 0; i < short.length; i++) {
    if (short[i] !== long[i]) return false;
  }
  return true;
}

// Normalise a company / job-title string for fuzzy identity matching:
// unescape, lowercase, drop gender markers like "(m/w/d)", collapse anything
// non-alphanumeric to single spaces.
function normText(s) {
  return unescapeInput(s)
    .toLowerCase()
    .replace(/\((?:[mwdfax](?:\s*\/\s*[mwdfax])*)\)/gi, ' ')
    .replace(/\b[mwfd](?:\s*\/\s*[mwfd]){1,2}\b/gi, ' ')
    .replace(/[^\p{L}\p{N}]+/gu, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

// Normalised signature of an offer (raw or DB row), used for matching.
function offerSignature(offer) {
  const o = offer || {};
  return {
    url_norm: normalizeUrl(o.quelle_url),
    domain: urlDomain(o.quelle_url),
    quelle: (o.quelle && String(o.quelle).trim()) || 'drittanbieter',
    external_id: o.external_id != null && String(o.external_id).trim() !== ''
      ? String(o.external_id).trim() : null,
    firma_norm: normText(o.firma),
    // Prefer a client-supplied slug (offers carry firma_slug); else derive it.
    firma_slug: o.firma_slug != null && String(o.firma_slug).trim() !== ''
      ? String(o.firma_slug).trim() : firmaSlug(o.firma),
    stelle_norm: normText(o.stelle),
    ort_norm: normText(o.ort),
  };
}

// Company identity match: prefer the robust slug (equal or prefix — see
// sameCompany, which absorbs spelling / legal-form / umlaut variants of one
// employer), fall back to firma_norm for legacy rows written before firma_slug
// existed (or not yet backfilled).
function firmaMatch(row, sig) {
  if (row.firma_slug && sig.firma_slug) return sameCompany(row.firma_slug, sig.firma_slug);
  return !!row.firma_norm && row.firma_norm === sig.firma_norm;
}

// Does one blacklist row block the given signature?
function rowMatches(row, sig) {
  switch (row.typ) {
    case 'url':
      return !!row.url_norm && row.url_norm === sig.url_norm;
    case 'domain':
      return !!row.domain && row.domain === sig.domain;
    case 'firma':
      return firmaMatch(row, sig);
    case 'firma_stelle':
      return firmaMatch(row, sig)
        && !!row.stelle_norm && row.stelle_norm === sig.stelle_norm
        && (!row.ort_norm || row.ort_norm === sig.ort_norm);
    case 'auto':
    default:
      if (row.url_norm && sig.url_norm && row.url_norm === sig.url_norm) return true;
      if (row.quelle && row.external_id && sig.external_id
        && row.quelle === sig.quelle && row.external_id === sig.external_id) return true;
      if ((row.firma_slug || row.firma_norm) && row.stelle_norm
        && firmaMatch(row, sig) && row.stelle_norm === sig.stelle_norm
        && (!row.ort_norm || row.ort_norm === sig.ort_norm)) return true;
      return false;
  }
}

// First blacklist row that blocks this offer, or null.
function matchBlacklist(rows, offer) {
  const sig = offerSignature(offer);
  for (const row of rows || []) {
    if (rowMatches(row, sig)) return row;
  }
  return null;
}

// Column set for INSERT — keeps server.js and api.js in lock-step.
const COLUMNS = [
  'typ', 'url_norm', 'domain', 'quelle', 'external_id',
  'firma_norm', 'firma_slug', 'stelle_norm', 'ort_norm', 'firma', 'stelle', 'quelle_url', 'grund',
];

function emptyEntry() {
  const e = {};
  COLUMNS.forEach((c) => { e[c] = null; });
  return e;
}

// Build a blacklist row from a manual UI/API request. Returns null if the
// chosen typ has no usable value.
function buildManualEntry(input) {
  const inp = input || {};
  const typ = TYPES.includes(inp.typ) ? inp.typ : 'url';
  const e = emptyEntry();
  e.typ = typ;
  e.grund = inp.grund ? String(inp.grund) : null;
  const wert = inp.wert != null ? String(inp.wert) : '';

  if (typ === 'url') {
    const w = wert || inp.quelle_url || '';
    e.url_norm = normalizeUrl(w);
    e.domain = urlDomain(w) || null;
    e.quelle_url = w || null;
    if (!e.url_norm) return null;
  } else if (typ === 'domain') {
    const w = wert || inp.domain || '';
    e.domain = urlDomain(w) || normText(w).replace(/\s+/g, '') || null;
    if (!e.domain) return null;
  } else if (typ === 'firma') {
    const w = wert || inp.firma || '';
    e.firma = w || null;
    e.firma_norm = normText(w);
    e.firma_slug = firmaSlug(w) || null;
    if (!e.firma_norm) return null;
  } else if (typ === 'firma_stelle') {
    const f = inp.firma || '';
    const s = inp.stelle || wert || '';
    e.firma = f || null;
    e.stelle = s || null;
    e.firma_norm = normText(f);
    e.firma_slug = firmaSlug(f) || null;
    e.stelle_norm = normText(s);
    e.ort_norm = inp.ort ? normText(inp.ort) : null;
    if (!e.firma_norm || !e.stelle_norm) return null;
  }
  return e;
}

// Build the automatic blacklist row created when an offer is deleted: it stores
// every available signature so the offer can never be ingested again.
function buildAutoEntry(offer, grund) {
  const sig = offerSignature(offer);
  const e = emptyEntry();
  e.typ = 'auto';
  e.grund = grund || null;
  e.url_norm = sig.url_norm || null;
  e.domain = sig.domain || null;
  e.quelle = sig.external_id ? sig.quelle : null;
  e.external_id = sig.external_id || null;
  e.firma_norm = sig.firma_norm || null;
  e.firma_slug = sig.firma_slug || null;
  e.stelle_norm = sig.stelle_norm || null;
  e.ort_norm = sig.ort_norm || null;
  e.firma = offer && offer.firma ? offer.firma : null;
  e.stelle = offer && offer.stelle ? offer.stelle : null;
  e.quelle_url = offer && offer.quelle_url ? offer.quelle_url : null;
  return e;
}

module.exports = {
  TYPES,
  COLUMNS,
  normalizeUrl,
  urlDomain,
  normText,
  firmaSlug,
  sameCompany,
  offerSignature,
  rowMatches,
  matchBlacklist,
  buildManualEntry,
  buildAutoEntry,
};