fix(260907-efh): Fingerabdruck auf Vergabestelle/Titel/Frist verengen

Die dritte Dedup-Stufe hashte bisher fuenf Segmente, darunter CPV-Division und geschaetzten Wert — beide werden von Nicht-DOE-Quellen systematisch nie geliefert, wodurch dieselbe Ausschreibung aus DOE und aus einem Scraper nie denselben Fingerabdruck ergab (WINDOWS.md #11). - tenderFingerprint() nimmt nur noch buyerName, title, deadlineAt entgegen - Versionsmarke FINGERPRINT_VERSION_PREFIX ("v2:") vor dem Hash, damit Task 3 veraltete Zeilen erkennen kann - valueBucketsContradict() als eigene, exportierte Funktion fuer den Wert-Veto in Task 2 — kein Hash-Bestandteil mehr - backfill-tender-source.ts an die neue Signatur angepasst - Testsuite auf das neue Verhalten umgeschrieben inkl. Goldwert-Test Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K5jtbGzC5Sf9npJ3JCjKhq
2026-09-07 10:36:21 +02:00
parent 4502664fb2
commit cb23e3bc78
3 changed files with 156 additions and 82 deletions
@@ -1,66 +1,108 @@
 import { describe, expect, it } from 'vitest';
-import { tenderFingerprint } from './tender-fingerprint';
+import { tenderFingerprint, valueBucketsContradict } from './tender-fingerprint';

 /**
- * SCHEMA-03 (D-04) — pure fingerprint function. NULL-tolerant because the
- * live DB is 92.4% NULL estimatedValue / 15.7% NULL deadlineAt (13-RESEARCH
- * Pattern 4) — a fingerprint that requires those fields would never match
- * the overwhelming majority of real DÖE tenders.
+ * SCHEMA-03 (D-04) — pure fingerprint function, v2 formula (WINDOWS.md #11).
+ * v2 hashes only [buyerName, title, deadlineAt]. CPV and estimatedValue are
+ * NOT hash inputs anymore — see the file comment on tender-fingerprint.ts
+ * for why and for the three limits this plan deliberately leaves in place.
 */
 describe('tenderFingerprint', () => {
-  const base = {
-    buyerName: 'Stadt München',
-    title: 'Neubau einer Kindertagesstätte',
-    cpvDivisions: ['45', '71'],
-    deadlineAt: null as Date | null,
-    estimatedValue: null as number | null,
-  };
-
-  it('matches when title+buyer+cpv are equal and value+deadline are both NULL', () => {
-    const a = tenderFingerprint({ ...base });
-    const b = tenderFingerprint({ ...base });
-    expect(a).toBe(b);
+  it('is the Kernregression this plan exists for: a DOE-shaped record and a scraper-shaped record of the same tender collide', () => {
+    const doeShaped = {
+      buyerName: 'Stadt München',
+      title: 'Neubau einer Kindertagesstätte',
+      cpvDivisions: ['45'],
+      deadlineAt: new Date('2026-06-01T00:00:00Z'),
+      estimatedValue: 387291,
+    };
+    const scraperShaped = {
+      buyerName: 'Stadt München',
+      title: 'Neubau einer Kindertagesstätte',
+      cpvDivisions: [] as string[],
+      deadlineAt: new Date('2026-06-01T00:00:00Z'),
+      estimatedValue: null as number | null,
+    };
+    // Extra fields (cpvDivisions, estimatedValue) prove they can't reach
+    // the hash: they're passed straight through as typed variables, the
+    // way the resolver calls this function with its wider normalized
+    // record — TS excess-property checking only fires on object literals.
+    expect(tenderFingerprint(doeShaped)).toBe(tenderFingerprint(scraperShaped));
  });

  it('differs for a different title (collision guard, Pitfall 1)', () => {
+    const base = { buyerName: 'Stadt München', title: 'Neubau einer Kindertagesstätte', deadlineAt: null };
    const a = tenderFingerprint(base);
    const b = tenderFingerprint({ ...base, title: 'Sanierung eines Kindergartens' });
    expect(a).not.toBe(b);
  });

  it('normalizes umlauts so "München" and "Muenchen" contribute the same text', () => {
+    const base = { title: 'Neubau einer Kindertagesstätte', deadlineAt: null };
    const a = tenderFingerprint({ ...base, buyerName: 'Stadt München' });
    const b = tenderFingerprint({ ...base, buyerName: 'Stadt Muenchen' });
    expect(a).toBe(b);
  });

-  it('is order-independent and deduplicated for cpvDivisions', () => {
-    const a = tenderFingerprint({ ...base, cpvDivisions: ['45', '71'] });
-    const b = tenderFingerprint({ ...base, cpvDivisions: ['71', '45', '71'] });
-    expect(a).toBe(b);
-  });
-
-  it('buckets estimatedValue by order of magnitude — 12000 and 15000 collide', () => {
-    const a = tenderFingerprint({ ...base, estimatedValue: 12000 });
-    const b = tenderFingerprint({ ...base, estimatedValue: 15000 });
-    expect(a).toBe(b);
-  });
-
-  it('separates estimatedValue across a magnitude boundary', () => {
-    const a = tenderFingerprint({ ...base, estimatedValue: 9000 });
-    const b = tenderFingerprint({ ...base, estimatedValue: 90000 });
-    expect(a).not.toBe(b);
-  });
-
-  it('returns a stable sha256 hex string (deterministic across runs)', () => {
-    const result = tenderFingerprint(base);
-    expect(result).toMatch(/^[a-f0-9]{64}$/);
+  it('collapses two deadlines on the same day to the same fingerprint, and separates two different days', () => {
+    const base = { buyerName: 'Stadt München', title: 'Neubau einer Kindertagesstätte' };
+    const sameDayMorning = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-01T08:00:00Z') });
+    const sameDayEvening = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-01T20:00:00Z') });
+    const otherDay = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-02T08:00:00Z') });
+    expect(sameDayMorning).toBe(sameDayEvening);
+    expect(sameDayMorning).not.toBe(otherDay);
  });

  it('treats a null buyerName as an empty segment, not a crash', () => {
-    expect(() => tenderFingerprint({ ...base, buyerName: null })).not.toThrow();
-    const a = tenderFingerprint({ ...base, buyerName: null });
-    const b = tenderFingerprint({ ...base, buyerName: null });
+    const args = { buyerName: null as string | null, title: 'Neubau einer Kindertagesstätte', deadlineAt: null };
+    expect(() => tenderFingerprint(args)).not.toThrow();
+    const a = tenderFingerprint(args);
+    const b = tenderFingerprint(args);
    expect(a).toBe(b);
  });
+
+  it('returns "v2:" followed by 64 hex characters', () => {
+    const result = tenderFingerprint({
+      buyerName: 'Stadt München',
+      title: 'Neubau einer Kindertagesstätte',
+      deadlineAt: null,
+    });
+    expect(result).toMatch(/^v2:[a-f0-9]{64}$/);
+  });
+
+  it('gold value: nails the segment count and order — a fourth (even empty) segment or a reordering turns this red', () => {
+    // Hand-computed independently of production code from the canonical
+    // string "stadt musterstadt|neubau turnhalle|2026-03-04" — not built
+    // with normText/deadlineKey.
+    const result = tenderFingerprint({
+      buyerName: 'Stadt Musterstadt',
+      title: 'Neubau Turnhalle',
+      deadlineAt: new Date('2026-03-04T09:00:00Z'),
+    });
+    expect(result).toBe(
+      'v2:6104874e683082d4e656e5585947629dcb8c667eb77888c456020ce86993be34',
+    );
+  });
+});
+
+describe('valueBucketsContradict', () => {
+  it('is false when the left side is null', () => {
+    expect(valueBucketsContradict(null, 5000)).toBe(false);
+  });
+
+  it('is false when the right side is null', () => {
+    expect(valueBucketsContradict(5000, null)).toBe(false);
+  });
+
+  it('is false when both sides are null', () => {
+    expect(valueBucketsContradict(null, null)).toBe(false);
+  });
+
+  it('is false for two values of the same order of magnitude', () => {
+    expect(valueBucketsContradict(12000, 15000)).toBe(false);
+  });
+
+  it('is true for two values across a magnitude boundary', () => {
+    expect(valueBucketsContradict(9000, 90000)).toBe(true);
+  });
 });