fix(260907-efh): Fingerabdruck auf Vergabestelle/Titel/Frist verengen

Die dritte Dedup-Stufe hashte bisher fuenf Segmente, darunter CPV-Division und geschaetzten Wert — beide werden von Nicht-DOE-Quellen systematisch nie geliefert, wodurch dieselbe Ausschreibung aus DOE und aus einem Scraper nie denselben Fingerabdruck ergab (WINDOWS.md #11). - tenderFingerprint() nimmt nur noch buyerName, title, deadlineAt entgegen - Versionsmarke FINGERPRINT_VERSION_PREFIX ("v2:") vor dem Hash, damit Task 3 veraltete Zeilen erkennen kann - valueBucketsContradict() als eigene, exportierte Funktion fuer den Wert-Veto in Task 2 — kein Hash-Bestandteil mehr - backfill-tender-source.ts an die neue Signatur angepasst - Testsuite auf das neue Verhalten umgeschrieben inkl. Goldwert-Test Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K5jtbGzC5Sf9npJ3JCjKhq
2026-09-07 10:36:21 +02:00
parent 4502664fb2
commit cb23e3bc78
3 changed files with 156 additions and 82 deletions
@@ -16,8 +16,15 @@
 *
 * Idempotent: recomputes and overwrites fingerprint for every row it reads,
 * so re-running is safe (same deterministic output for unchanged inputs).
+ *
+ * This script remains the manual, force-recompute-everything path. As of
+ * Task 3 of the WINDOWS.md #11 fix, the automatic path for ordinary
+ * startup is TenderFingerprintBackfillService, which recomputes only rows
+ * whose stored fingerprint doesn't yet carry FINGERPRINT_VERSION_PREFIX —
+ * this script stays useful for a deliberate full recompute (e.g. after a
+ * formula change) or ad-hoc verification.
 */
-import { PrismaClient, Prisma } from '@prisma/client';
+import { PrismaClient } from '@prisma/client';
 import { tenderFingerprint } from './tender-fingerprint';

 async function main(): Promise<void> {
@@ -28,9 +35,7 @@ async function main(): Promise<void> {
        id: true,
        title: true,
        buyerName: true,
-        cpvDivisions: true,
        deadlineAt: true,
-        estimatedValue: true,
      },
    });

@@ -39,10 +44,7 @@ async function main(): Promise<void> {
      const fingerprint = tenderFingerprint({
        buyerName: t.buyerName,
        title: t.title,
-        cpvDivisions: t.cpvDivisions,
        deadlineAt: t.deadlineAt,
-        // estimatedValue is Decimal? — convert to number|null before bucketing (T-13-01-03).
-        estimatedValue: decimalToNumberOrNull(t.estimatedValue),
      });
      await prisma.tender.update({
        where: { id: t.id },
@@ -57,10 +59,6 @@ async function main(): Promise<void> {
  }
 }

-function decimalToNumberOrNull(v: Prisma.Decimal | null): number | null {
-  return v === null ? null : Number(v);
-}
-
 main().catch((err) => {
  console.error('backfill-tender-source failed:', err);
  process.exitCode = 1;
@@ -1,66 +1,108 @@
 import { describe, expect, it } from 'vitest';
-import { tenderFingerprint } from './tender-fingerprint';
+import { tenderFingerprint, valueBucketsContradict } from './tender-fingerprint';

 /**
- * SCHEMA-03 (D-04) — pure fingerprint function. NULL-tolerant because the
- * live DB is 92.4% NULL estimatedValue / 15.7% NULL deadlineAt (13-RESEARCH
- * Pattern 4) — a fingerprint that requires those fields would never match
- * the overwhelming majority of real DÖE tenders.
+ * SCHEMA-03 (D-04) — pure fingerprint function, v2 formula (WINDOWS.md #11).
+ * v2 hashes only [buyerName, title, deadlineAt]. CPV and estimatedValue are
+ * NOT hash inputs anymore — see the file comment on tender-fingerprint.ts
+ * for why and for the three limits this plan deliberately leaves in place.
 */
 describe('tenderFingerprint', () => {
-  const base = {
+  it('is the Kernregression this plan exists for: a DOE-shaped record and a scraper-shaped record of the same tender collide', () => {
+    const doeShaped = {
      buyerName: 'Stadt München',
      title: 'Neubau einer Kindertagesstätte',
-    cpvDivisions: ['45', '71'],
-    deadlineAt: null as Date | null,
+      cpvDivisions: ['45'],
+      deadlineAt: new Date('2026-06-01T00:00:00Z'),
+      estimatedValue: 387291,
+    };
+    const scraperShaped = {
+      buyerName: 'Stadt München',
+      title: 'Neubau einer Kindertagesstätte',
+      cpvDivisions: [] as string[],
+      deadlineAt: new Date('2026-06-01T00:00:00Z'),
      estimatedValue: null as number | null,
    };
-
-  it('matches when title+buyer+cpv are equal and value+deadline are both NULL', () => {
-    const a = tenderFingerprint({ ...base });
-    const b = tenderFingerprint({ ...base });
-    expect(a).toBe(b);
+    // Extra fields (cpvDivisions, estimatedValue) prove they can't reach
+    // the hash: they're passed straight through as typed variables, the
+    // way the resolver calls this function with its wider normalized
+    // record — TS excess-property checking only fires on object literals.
+    expect(tenderFingerprint(doeShaped)).toBe(tenderFingerprint(scraperShaped));
  });

  it('differs for a different title (collision guard, Pitfall 1)', () => {
+    const base = { buyerName: 'Stadt München', title: 'Neubau einer Kindertagesstätte', deadlineAt: null };
    const a = tenderFingerprint(base);
    const b = tenderFingerprint({ ...base, title: 'Sanierung eines Kindergartens' });
    expect(a).not.toBe(b);
  });

  it('normalizes umlauts so "München" and "Muenchen" contribute the same text', () => {
+    const base = { title: 'Neubau einer Kindertagesstätte', deadlineAt: null };
    const a = tenderFingerprint({ ...base, buyerName: 'Stadt München' });
    const b = tenderFingerprint({ ...base, buyerName: 'Stadt Muenchen' });
    expect(a).toBe(b);
  });

-  it('is order-independent and deduplicated for cpvDivisions', () => {
-    const a = tenderFingerprint({ ...base, cpvDivisions: ['45', '71'] });
-    const b = tenderFingerprint({ ...base, cpvDivisions: ['71', '45', '71'] });
-    expect(a).toBe(b);
-  });
-
-  it('buckets estimatedValue by order of magnitude — 12000 and 15000 collide', () => {
-    const a = tenderFingerprint({ ...base, estimatedValue: 12000 });
-    const b = tenderFingerprint({ ...base, estimatedValue: 15000 });
-    expect(a).toBe(b);
-  });
-
-  it('separates estimatedValue across a magnitude boundary', () => {
-    const a = tenderFingerprint({ ...base, estimatedValue: 9000 });
-    const b = tenderFingerprint({ ...base, estimatedValue: 90000 });
-    expect(a).not.toBe(b);
-  });
-
-  it('returns a stable sha256 hex string (deterministic across runs)', () => {
-    const result = tenderFingerprint(base);
-    expect(result).toMatch(/^[a-f0-9]{64}$/);
+  it('collapses two deadlines on the same day to the same fingerprint, and separates two different days', () => {
+    const base = { buyerName: 'Stadt München', title: 'Neubau einer Kindertagesstätte' };
+    const sameDayMorning = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-01T08:00:00Z') });
+    const sameDayEvening = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-01T20:00:00Z') });
+    const otherDay = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-02T08:00:00Z') });
+    expect(sameDayMorning).toBe(sameDayEvening);
+    expect(sameDayMorning).not.toBe(otherDay);
  });

  it('treats a null buyerName as an empty segment, not a crash', () => {
-    expect(() => tenderFingerprint({ ...base, buyerName: null })).not.toThrow();
-    const a = tenderFingerprint({ ...base, buyerName: null });
-    const b = tenderFingerprint({ ...base, buyerName: null });
+    const args = { buyerName: null as string | null, title: 'Neubau einer Kindertagesstätte', deadlineAt: null };
+    expect(() => tenderFingerprint(args)).not.toThrow();
+    const a = tenderFingerprint(args);
+    const b = tenderFingerprint(args);
    expect(a).toBe(b);
  });
+
+  it('returns "v2:" followed by 64 hex characters', () => {
+    const result = tenderFingerprint({
+      buyerName: 'Stadt München',
+      title: 'Neubau einer Kindertagesstätte',
+      deadlineAt: null,
+    });
+    expect(result).toMatch(/^v2:[a-f0-9]{64}$/);
+  });
+
+  it('gold value: nails the segment count and order — a fourth (even empty) segment or a reordering turns this red', () => {
+    // Hand-computed independently of production code from the canonical
+    // string "stadt musterstadt|neubau turnhalle|2026-03-04" — not built
+    // with normText/deadlineKey.
+    const result = tenderFingerprint({
+      buyerName: 'Stadt Musterstadt',
+      title: 'Neubau Turnhalle',
+      deadlineAt: new Date('2026-03-04T09:00:00Z'),
+    });
+    expect(result).toBe(
+      'v2:6104874e683082d4e656e5585947629dcb8c667eb77888c456020ce86993be34',
+    );
+  });
+});
+
+describe('valueBucketsContradict', () => {
+  it('is false when the left side is null', () => {
+    expect(valueBucketsContradict(null, 5000)).toBe(false);
+  });
+
+  it('is false when the right side is null', () => {
+    expect(valueBucketsContradict(5000, null)).toBe(false);
+  });
+
+  it('is false when both sides are null', () => {
+    expect(valueBucketsContradict(null, null)).toBe(false);
+  });
+
+  it('is false for two values of the same order of magnitude', () => {
+    expect(valueBucketsContradict(12000, 15000)).toBe(false);
+  });
+
+  it('is true for two values across a magnitude boundary', () => {
+    expect(valueBucketsContradict(9000, 90000)).toBe(true);
+  });
 });
@@ -2,8 +2,43 @@ import { createHash } from 'crypto';

 /**
 * tenderFingerprint — pure, deterministic cross-source dedup key (SCHEMA-03,
- * D-04). Used by the Task-2 backfill script and (Plan 13-03) the dedup
- * resolver's third match tier (OCID -> source:noticeId -> fingerprint).
+ * D-04). Used by the backfill script, the automatic startup backfill
+ * (TenderFingerprintBackfillService), and the dedup resolver's third match
+ * tier (OCID -> source:noticeId -> fingerprint).
+ *
+ * v2 formula (2026-09-07, WINDOWS.md #11): the v1 formula hashed FIVE
+ * segments — buyer, title, CPV division, deadline, value bucket — and two
+ * of them, CPV and value, are systematically absent from exactly the
+ * records this stage exists to match. Measured at the live DB: non-DÖE
+ * sources never carry either field, and 21 measured groups of genuine
+ * cross-source duplicates were kept apart purely by a differing CPV
+ * division — typically because one row carries the substantive division
+ * and the other carries 50 (repair/maintenance). Hashing those two fields
+ * did not add precision; it silently disabled the stage for its entire
+ * intended use case. v2 hashes only [buyerName, title, deadlineAt]. CPV is
+ * gone from the hash with no replacement — it is not compared anywhere in
+ * this file anymore.
+ *
+ * The estimated value does NOT move into the hash's replacement gap.
+ * Instead it returns as a veto at the match site: see
+ * `valueBucketsContradict` below, applied by TenderDedupService AFTER a
+ * fingerprint hit, never as a hash input. At the live DB this rescues 2 of
+ * 3 measured lot-pair groups that a value-free hash would otherwise merge;
+ * the third pair (117.647 vs 386.554, same order of magnitude) is still
+ * merged incorrectly — a deliberately accepted false merge, because a
+ * wrong merge is visible and reversible, while 21 missed merges were the
+ * stage's silent total failure. The comparison stays an EXACT
+ * order-of-magnitude match, never a similarity threshold — see the
+ * "Deliberately a deterministic hash" paragraph below, which still applies
+ * unchanged to both the hash and the veto.
+ *
+ * The remaining known limit: records with neither a buyer name nor a
+ * deadline (RSS, email-alert) match on title alone. In the ordinary case
+ * this means they still never match a DÖE row (DÖE almost always carries
+ * both). But if a DÖE row itself is missing both buyer and deadline, title
+ * alone is enough to merge. Exactly one such pair is measured in the live
+ * DB today (title "LSA242", once via service-bund, once via
+ * doe-opendata) and will be merged together after this change.
 *
 * NULL-tolerant by design: the live DB is 92.4% NULL estimatedValue and
 * 15.7% NULL deadlineAt (13-RESEARCH Pattern 4) — a fingerprint requiring
@@ -11,34 +46,39 @@ import { createHash } from 'crypto';
 * DÖE tenders. NULL fields contribute an empty canonical segment instead
 * of excluding the record.
 *
- * Title + buyer carry the dominant weight (almost always present). CPV
- * uses the 2-digit division (not the full code) — more robust across
- * differently-formatted source CPV codes (Pitfall 2 pattern). Value is
- * bucketed to an order-of-magnitude (sources round differently, and it's
- * usually NULL anyway). Deadline is truncated to day-grain (timezone/time
- * differences between sources).
- *
 * Deliberately a deterministic hash, NOT a similarity threshold: O(1)
- * lookup via a DB `@unique`/indexed column, unit-testable, no O(n) scan
- * against the 2851+ row backlog per ingest (13-RESEARCH "Don't Hand-Roll").
- * If real-world collisions appear, increase field granularity (e.g. full
- * CPV code instead of division) — do NOT introduce threshold matching.
+ * lookup via a DB indexed column, unit-testable, no O(n) scan against the
+ * growing tender backlog per ingest (13-RESEARCH "Don't Hand-Roll"). If
+ * real-world collisions appear, increase field granularity — do NOT
+ * introduce threshold matching. `valueBucketsContradict` below is exact
+ * order-of-magnitude equality, not a distance measure, for the same
+ * reason.
 */
+export const FINGERPRINT_VERSION_PREFIX = 'v2:';
+
 export function tenderFingerprint(f: {
  buyerName: string | null;
  title: string;
-  cpvDivisions: string[];
  deadlineAt: Date | null;
-  estimatedValue: number | null;
 }): string {
-  const canonical = [
-    normText(f.buyerName),
-    normText(f.title),
-    cpvDivisionKey(f.cpvDivisions),
-    deadlineKey(f.deadlineAt),
-    valueBucket(f.estimatedValue),
-  ].join('|');
-  return createHash('sha256').update(canonical).digest('hex');
+  const canonical = [normText(f.buyerName), normText(f.title), deadlineKey(f.deadlineAt)].join(
+    '|',
+  );
+  return FINGERPRINT_VERSION_PREFIX + createHash('sha256').update(canonical).digest('hex');
+}
+
+/**
+ * Exact order-of-magnitude contradiction check, used as a veto AFTER a
+ * fingerprint hit (TenderDedupService Stufe 3) — never as a hash input.
+ * Returns true ONLY when BOTH values are set and their magnitude buckets
+ * differ. Either side null means the value can't say anything, so the
+ * function returns false (does not block the merge). This is an exact
+ * comparison of two magnitude buckets, not a similarity/distance measure —
+ * see the file comment above.
+ */
+export function valueBucketsContradict(a: number | null, b: number | null): boolean {
+  if (a === null || b === null) return false;
+  return valueBucket(a) !== valueBucket(b);
 }

 /** lowercase, German umlaut expansion, non-alphanumeric -> space, trim/collapse whitespace. */
@@ -55,14 +95,8 @@ function normText(s: string | null): string {
    .replace(/\s+/g, ' ');
 }

-/** Order-independent, deduplicated CPV division key. */
-function cpvDivisionKey(cpvDivisions: string[]): string {
-  return [...new Set(cpvDivisions)].sort().join(',');
-}
-
-/** NULL -> empty segment (92.4% NULL live); otherwise order-of-magnitude bucket. */
-function valueBucket(v: number | null): string {
-  if (v === null) return '';
+/** Order-of-magnitude bucket (private — valueBucketsContradict is its only caller). */
+function valueBucket(v: number): string {
  return String(Math.floor(Math.log10(Math.max(v, 1))));
 }