feat(13-01): implement tenderFingerprint pure NULL-tolerant dedup key

GREEN: title+buyer dominant, CPV division (order-independent, dedup'd), value bucketed by order-of-magnitude, deadline truncated to day-grain. sha256 hex, deterministic, no I/O — foundation for the Task-2 backfill and the Plan 13-03 dedup resolver's fingerprint tier. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-23 08:33:40 +02:00
parent 063ba5b180
commit c6cac696ff
1 changed files with 72 additions and 0 deletions
@@ -0,0 +1,72 @@
+import { createHash } from 'crypto';
+
+/**
+ * tenderFingerprint — pure, deterministic cross-source dedup key (SCHEMA-03,
+ * D-04). Used by the Task-2 backfill script and (Plan 13-03) the dedup
+ * resolver's third match tier (OCID -> source:noticeId -> fingerprint).
+ *
+ * NULL-tolerant by design: the live DB is 92.4% NULL estimatedValue and
+ * 15.7% NULL deadlineAt (13-RESEARCH Pattern 4) — a fingerprint requiring
+ * those fields would never match the overwhelming majority of real
+ * DÖE tenders. NULL fields contribute an empty canonical segment instead
+ * of excluding the record.
+ *
+ * Title + buyer carry the dominant weight (almost always present). CPV
+ * uses the 2-digit division (not the full code) — more robust across
+ * differently-formatted source CPV codes (Pitfall 2 pattern). Value is
+ * bucketed to an order-of-magnitude (sources round differently, and it's
+ * usually NULL anyway). Deadline is truncated to day-grain (timezone/time
+ * differences between sources).
+ *
+ * Deliberately a deterministic hash, NOT a similarity threshold: O(1)
+ * lookup via a DB `@unique`/indexed column, unit-testable, no O(n) scan
+ * against the 2851+ row backlog per ingest (13-RESEARCH "Don't Hand-Roll").
+ * If real-world collisions appear, increase field granularity (e.g. full
+ * CPV code instead of division) — do NOT introduce threshold matching.
+ */
+export function tenderFingerprint(f: {
+  buyerName: string | null;
+  title: string;
+  cpvDivisions: string[];
+  deadlineAt: Date | null;
+  estimatedValue: number | null;
+}): string {
+  const canonical = [
+    normText(f.buyerName),
+    normText(f.title),
+    cpvDivisionKey(f.cpvDivisions),
+    deadlineKey(f.deadlineAt),
+    valueBucket(f.estimatedValue),
+  ].join('|');
+  return createHash('sha256').update(canonical).digest('hex');
+}
+
+/** lowercase, German umlaut expansion, non-alphanumeric -> space, trim/collapse whitespace. */
+function normText(s: string | null): string {
+  if (!s) return '';
+  return s
+    .toLowerCase()
+    .replace(/ä/g, 'ae')
+    .replace(/ö/g, 'oe')
+    .replace(/ü/g, 'ue')
+    .replace(/ß/g, 'ss')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .replace(/\s+/g, ' ');
+}
+
+/** Order-independent, deduplicated CPV division key. */
+function cpvDivisionKey(cpvDivisions: string[]): string {
+  return [...new Set(cpvDivisions)].sort().join(',');
+}
+
+/** NULL -> empty segment (92.4% NULL live); otherwise order-of-magnitude bucket. */
+function valueBucket(v: number | null): string {
+  if (v === null) return '';
+  return String(Math.floor(Math.log10(Math.max(v, 1))));
+}
+
+/** NULL -> empty segment (15.7% NULL live); otherwise date-only (day grain). */
+function deadlineKey(d: Date | null): string {
+  return d ? d.toISOString().slice(0, 10) : '';
+}