diff --git a/apps/api/src/tenders/backfill-tender-source.ts b/apps/api/src/tenders/backfill-tender-source.ts index fe236cf..8b17884 100644 --- a/apps/api/src/tenders/backfill-tender-source.ts +++ b/apps/api/src/tenders/backfill-tender-source.ts @@ -16,8 +16,15 @@ * * Idempotent: recomputes and overwrites fingerprint for every row it reads, * so re-running is safe (same deterministic output for unchanged inputs). + * + * This script remains the manual, force-recompute-everything path. As of + * Task 3 of the WINDOWS.md #11 fix, the automatic path for ordinary + * startup is TenderFingerprintBackfillService, which recomputes only rows + * whose stored fingerprint doesn't yet carry FINGERPRINT_VERSION_PREFIX — + * this script stays useful for a deliberate full recompute (e.g. after a + * formula change) or ad-hoc verification. */ -import { PrismaClient, Prisma } from '@prisma/client'; +import { PrismaClient } from '@prisma/client'; import { tenderFingerprint } from './tender-fingerprint'; async function main(): Promise { @@ -28,9 +35,7 @@ async function main(): Promise { id: true, title: true, buyerName: true, - cpvDivisions: true, deadlineAt: true, - estimatedValue: true, }, }); @@ -39,10 +44,7 @@ async function main(): Promise { const fingerprint = tenderFingerprint({ buyerName: t.buyerName, title: t.title, - cpvDivisions: t.cpvDivisions, deadlineAt: t.deadlineAt, - // estimatedValue is Decimal? — convert to number|null before bucketing (T-13-01-03). - estimatedValue: decimalToNumberOrNull(t.estimatedValue), }); await prisma.tender.update({ where: { id: t.id }, @@ -57,10 +59,6 @@ async function main(): Promise { } } -function decimalToNumberOrNull(v: Prisma.Decimal | null): number | null { - return v === null ? null : Number(v); -} - main().catch((err) => { console.error('backfill-tender-source failed:', err); process.exitCode = 1; diff --git a/apps/api/src/tenders/tender-fingerprint.spec.ts b/apps/api/src/tenders/tender-fingerprint.spec.ts index 3c78e76..257e582 100644 --- a/apps/api/src/tenders/tender-fingerprint.spec.ts +++ b/apps/api/src/tenders/tender-fingerprint.spec.ts @@ -1,66 +1,108 @@ import { describe, expect, it } from 'vitest'; -import { tenderFingerprint } from './tender-fingerprint'; +import { tenderFingerprint, valueBucketsContradict } from './tender-fingerprint'; /** - * SCHEMA-03 (D-04) — pure fingerprint function. NULL-tolerant because the - * live DB is 92.4% NULL estimatedValue / 15.7% NULL deadlineAt (13-RESEARCH - * Pattern 4) — a fingerprint that requires those fields would never match - * the overwhelming majority of real DÖE tenders. + * SCHEMA-03 (D-04) — pure fingerprint function, v2 formula (WINDOWS.md #11). + * v2 hashes only [buyerName, title, deadlineAt]. CPV and estimatedValue are + * NOT hash inputs anymore — see the file comment on tender-fingerprint.ts + * for why and for the three limits this plan deliberately leaves in place. */ describe('tenderFingerprint', () => { - const base = { - buyerName: 'Stadt München', - title: 'Neubau einer Kindertagesstätte', - cpvDivisions: ['45', '71'], - deadlineAt: null as Date | null, - estimatedValue: null as number | null, - }; - - it('matches when title+buyer+cpv are equal and value+deadline are both NULL', () => { - const a = tenderFingerprint({ ...base }); - const b = tenderFingerprint({ ...base }); - expect(a).toBe(b); + it('is the Kernregression this plan exists for: a DOE-shaped record and a scraper-shaped record of the same tender collide', () => { + const doeShaped = { + buyerName: 'Stadt München', + title: 'Neubau einer Kindertagesstätte', + cpvDivisions: ['45'], + deadlineAt: new Date('2026-06-01T00:00:00Z'), + estimatedValue: 387291, + }; + const scraperShaped = { + buyerName: 'Stadt München', + title: 'Neubau einer Kindertagesstätte', + cpvDivisions: [] as string[], + deadlineAt: new Date('2026-06-01T00:00:00Z'), + estimatedValue: null as number | null, + }; + // Extra fields (cpvDivisions, estimatedValue) prove they can't reach + // the hash: they're passed straight through as typed variables, the + // way the resolver calls this function with its wider normalized + // record — TS excess-property checking only fires on object literals. + expect(tenderFingerprint(doeShaped)).toBe(tenderFingerprint(scraperShaped)); }); it('differs for a different title (collision guard, Pitfall 1)', () => { + const base = { buyerName: 'Stadt München', title: 'Neubau einer Kindertagesstätte', deadlineAt: null }; const a = tenderFingerprint(base); const b = tenderFingerprint({ ...base, title: 'Sanierung eines Kindergartens' }); expect(a).not.toBe(b); }); it('normalizes umlauts so "München" and "Muenchen" contribute the same text', () => { + const base = { title: 'Neubau einer Kindertagesstätte', deadlineAt: null }; const a = tenderFingerprint({ ...base, buyerName: 'Stadt München' }); const b = tenderFingerprint({ ...base, buyerName: 'Stadt Muenchen' }); expect(a).toBe(b); }); - it('is order-independent and deduplicated for cpvDivisions', () => { - const a = tenderFingerprint({ ...base, cpvDivisions: ['45', '71'] }); - const b = tenderFingerprint({ ...base, cpvDivisions: ['71', '45', '71'] }); - expect(a).toBe(b); - }); - - it('buckets estimatedValue by order of magnitude — 12000 and 15000 collide', () => { - const a = tenderFingerprint({ ...base, estimatedValue: 12000 }); - const b = tenderFingerprint({ ...base, estimatedValue: 15000 }); - expect(a).toBe(b); - }); - - it('separates estimatedValue across a magnitude boundary', () => { - const a = tenderFingerprint({ ...base, estimatedValue: 9000 }); - const b = tenderFingerprint({ ...base, estimatedValue: 90000 }); - expect(a).not.toBe(b); - }); - - it('returns a stable sha256 hex string (deterministic across runs)', () => { - const result = tenderFingerprint(base); - expect(result).toMatch(/^[a-f0-9]{64}$/); + it('collapses two deadlines on the same day to the same fingerprint, and separates two different days', () => { + const base = { buyerName: 'Stadt München', title: 'Neubau einer Kindertagesstätte' }; + const sameDayMorning = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-01T08:00:00Z') }); + const sameDayEvening = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-01T20:00:00Z') }); + const otherDay = tenderFingerprint({ ...base, deadlineAt: new Date('2026-06-02T08:00:00Z') }); + expect(sameDayMorning).toBe(sameDayEvening); + expect(sameDayMorning).not.toBe(otherDay); }); it('treats a null buyerName as an empty segment, not a crash', () => { - expect(() => tenderFingerprint({ ...base, buyerName: null })).not.toThrow(); - const a = tenderFingerprint({ ...base, buyerName: null }); - const b = tenderFingerprint({ ...base, buyerName: null }); + const args = { buyerName: null as string | null, title: 'Neubau einer Kindertagesstätte', deadlineAt: null }; + expect(() => tenderFingerprint(args)).not.toThrow(); + const a = tenderFingerprint(args); + const b = tenderFingerprint(args); expect(a).toBe(b); }); + + it('returns "v2:" followed by 64 hex characters', () => { + const result = tenderFingerprint({ + buyerName: 'Stadt München', + title: 'Neubau einer Kindertagesstätte', + deadlineAt: null, + }); + expect(result).toMatch(/^v2:[a-f0-9]{64}$/); + }); + + it('gold value: nails the segment count and order — a fourth (even empty) segment or a reordering turns this red', () => { + // Hand-computed independently of production code from the canonical + // string "stadt musterstadt|neubau turnhalle|2026-03-04" — not built + // with normText/deadlineKey. + const result = tenderFingerprint({ + buyerName: 'Stadt Musterstadt', + title: 'Neubau Turnhalle', + deadlineAt: new Date('2026-03-04T09:00:00Z'), + }); + expect(result).toBe( + 'v2:6104874e683082d4e656e5585947629dcb8c667eb77888c456020ce86993be34', + ); + }); +}); + +describe('valueBucketsContradict', () => { + it('is false when the left side is null', () => { + expect(valueBucketsContradict(null, 5000)).toBe(false); + }); + + it('is false when the right side is null', () => { + expect(valueBucketsContradict(5000, null)).toBe(false); + }); + + it('is false when both sides are null', () => { + expect(valueBucketsContradict(null, null)).toBe(false); + }); + + it('is false for two values of the same order of magnitude', () => { + expect(valueBucketsContradict(12000, 15000)).toBe(false); + }); + + it('is true for two values across a magnitude boundary', () => { + expect(valueBucketsContradict(9000, 90000)).toBe(true); + }); }); diff --git a/apps/api/src/tenders/tender-fingerprint.ts b/apps/api/src/tenders/tender-fingerprint.ts index 57abb54..ceeb8dc 100644 --- a/apps/api/src/tenders/tender-fingerprint.ts +++ b/apps/api/src/tenders/tender-fingerprint.ts @@ -2,8 +2,43 @@ import { createHash } from 'crypto'; /** * tenderFingerprint — pure, deterministic cross-source dedup key (SCHEMA-03, - * D-04). Used by the Task-2 backfill script and (Plan 13-03) the dedup - * resolver's third match tier (OCID -> source:noticeId -> fingerprint). + * D-04). Used by the backfill script, the automatic startup backfill + * (TenderFingerprintBackfillService), and the dedup resolver's third match + * tier (OCID -> source:noticeId -> fingerprint). + * + * v2 formula (2026-09-07, WINDOWS.md #11): the v1 formula hashed FIVE + * segments — buyer, title, CPV division, deadline, value bucket — and two + * of them, CPV and value, are systematically absent from exactly the + * records this stage exists to match. Measured at the live DB: non-DÖE + * sources never carry either field, and 21 measured groups of genuine + * cross-source duplicates were kept apart purely by a differing CPV + * division — typically because one row carries the substantive division + * and the other carries 50 (repair/maintenance). Hashing those two fields + * did not add precision; it silently disabled the stage for its entire + * intended use case. v2 hashes only [buyerName, title, deadlineAt]. CPV is + * gone from the hash with no replacement — it is not compared anywhere in + * this file anymore. + * + * The estimated value does NOT move into the hash's replacement gap. + * Instead it returns as a veto at the match site: see + * `valueBucketsContradict` below, applied by TenderDedupService AFTER a + * fingerprint hit, never as a hash input. At the live DB this rescues 2 of + * 3 measured lot-pair groups that a value-free hash would otherwise merge; + * the third pair (117.647 vs 386.554, same order of magnitude) is still + * merged incorrectly — a deliberately accepted false merge, because a + * wrong merge is visible and reversible, while 21 missed merges were the + * stage's silent total failure. The comparison stays an EXACT + * order-of-magnitude match, never a similarity threshold — see the + * "Deliberately a deterministic hash" paragraph below, which still applies + * unchanged to both the hash and the veto. + * + * The remaining known limit: records with neither a buyer name nor a + * deadline (RSS, email-alert) match on title alone. In the ordinary case + * this means they still never match a DÖE row (DÖE almost always carries + * both). But if a DÖE row itself is missing both buyer and deadline, title + * alone is enough to merge. Exactly one such pair is measured in the live + * DB today (title "LSA242", once via service-bund, once via + * doe-opendata) and will be merged together after this change. * * NULL-tolerant by design: the live DB is 92.4% NULL estimatedValue and * 15.7% NULL deadlineAt (13-RESEARCH Pattern 4) — a fingerprint requiring @@ -11,34 +46,39 @@ import { createHash } from 'crypto'; * DÖE tenders. NULL fields contribute an empty canonical segment instead * of excluding the record. * - * Title + buyer carry the dominant weight (almost always present). CPV - * uses the 2-digit division (not the full code) — more robust across - * differently-formatted source CPV codes (Pitfall 2 pattern). Value is - * bucketed to an order-of-magnitude (sources round differently, and it's - * usually NULL anyway). Deadline is truncated to day-grain (timezone/time - * differences between sources). - * * Deliberately a deterministic hash, NOT a similarity threshold: O(1) - * lookup via a DB `@unique`/indexed column, unit-testable, no O(n) scan - * against the 2851+ row backlog per ingest (13-RESEARCH "Don't Hand-Roll"). - * If real-world collisions appear, increase field granularity (e.g. full - * CPV code instead of division) — do NOT introduce threshold matching. + * lookup via a DB indexed column, unit-testable, no O(n) scan against the + * growing tender backlog per ingest (13-RESEARCH "Don't Hand-Roll"). If + * real-world collisions appear, increase field granularity — do NOT + * introduce threshold matching. `valueBucketsContradict` below is exact + * order-of-magnitude equality, not a distance measure, for the same + * reason. */ +export const FINGERPRINT_VERSION_PREFIX = 'v2:'; + export function tenderFingerprint(f: { buyerName: string | null; title: string; - cpvDivisions: string[]; deadlineAt: Date | null; - estimatedValue: number | null; }): string { - const canonical = [ - normText(f.buyerName), - normText(f.title), - cpvDivisionKey(f.cpvDivisions), - deadlineKey(f.deadlineAt), - valueBucket(f.estimatedValue), - ].join('|'); - return createHash('sha256').update(canonical).digest('hex'); + const canonical = [normText(f.buyerName), normText(f.title), deadlineKey(f.deadlineAt)].join( + '|', + ); + return FINGERPRINT_VERSION_PREFIX + createHash('sha256').update(canonical).digest('hex'); +} + +/** + * Exact order-of-magnitude contradiction check, used as a veto AFTER a + * fingerprint hit (TenderDedupService Stufe 3) — never as a hash input. + * Returns true ONLY when BOTH values are set and their magnitude buckets + * differ. Either side null means the value can't say anything, so the + * function returns false (does not block the merge). This is an exact + * comparison of two magnitude buckets, not a similarity/distance measure — + * see the file comment above. + */ +export function valueBucketsContradict(a: number | null, b: number | null): boolean { + if (a === null || b === null) return false; + return valueBucket(a) !== valueBucket(b); } /** lowercase, German umlaut expansion, non-alphanumeric -> space, trim/collapse whitespace. */ @@ -55,14 +95,8 @@ function normText(s: string | null): string { .replace(/\s+/g, ' '); } -/** Order-independent, deduplicated CPV division key. */ -function cpvDivisionKey(cpvDivisions: string[]): string { - return [...new Set(cpvDivisions)].sort().join(','); -} - -/** NULL -> empty segment (92.4% NULL live); otherwise order-of-magnitude bucket. */ -function valueBucket(v: number | null): string { - if (v === null) return ''; +/** Order-of-magnitude bucket (private — valueBucketsContradict is its only caller). */ +function valueBucket(v: number): string { return String(Math.floor(Math.log10(Math.max(v, 1)))); }