From c6cac696ff61a358707f227ed9243833fc18d4ae Mon Sep 17 00:00:00 2001
From: Schalli <kschaller@ctl.de>
Date: Thu, 23 Jul 2026 08:33:40 +0200
Subject: [PATCH] feat(13-01): implement tenderFingerprint pure NULL-tolerant
 dedup key
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

GREEN: title+buyer dominant, CPV division (order-independent, dedup'd),
value bucketed by order-of-magnitude, deadline truncated to day-grain.
sha256 hex, deterministic, no I/O — foundation for the Task-2 backfill
and the Plan 13-03 dedup resolver's fingerprint tier.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/api/src/tenders/tender-fingerprint.ts | 72 ++++++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 apps/api/src/tenders/tender-fingerprint.ts

diff --git a/apps/api/src/tenders/tender-fingerprint.ts b/apps/api/src/tenders/tender-fingerprint.ts
new file mode 100644
index 0000000..57abb54
--- /dev/null
+++ b/apps/api/src/tenders/tender-fingerprint.ts
@@ -0,0 +1,72 @@
+import { createHash } from 'crypto';
+
+/**
+ * tenderFingerprint — pure, deterministic cross-source dedup key (SCHEMA-03,
+ * D-04). Used by the Task-2 backfill script and (Plan 13-03) the dedup
+ * resolver's third match tier (OCID -> source:noticeId -> fingerprint).
+ *
+ * NULL-tolerant by design: the live DB is 92.4% NULL estimatedValue and
+ * 15.7% NULL deadlineAt (13-RESEARCH Pattern 4) — a fingerprint requiring
+ * those fields would never match the overwhelming majority of real
+ * DÖE tenders. NULL fields contribute an empty canonical segment instead
+ * of excluding the record.
+ *
+ * Title + buyer carry the dominant weight (almost always present). CPV
+ * uses the 2-digit division (not the full code) — more robust across
+ * differently-formatted source CPV codes (Pitfall 2 pattern). Value is
+ * bucketed to an order-of-magnitude (sources round differently, and it's
+ * usually NULL anyway). Deadline is truncated to day-grain (timezone/time
+ * differences between sources).
+ *
+ * Deliberately a deterministic hash, NOT a similarity threshold: O(1)
+ * lookup via a DB `@unique`/indexed column, unit-testable, no O(n) scan
+ * against the 2851+ row backlog per ingest (13-RESEARCH "Don't Hand-Roll").
+ * If real-world collisions appear, increase field granularity (e.g. full
+ * CPV code instead of division) — do NOT introduce threshold matching.
+ */
+export function tenderFingerprint(f: {
+  buyerName: string | null;
+  title: string;
+  cpvDivisions: string[];
+  deadlineAt: Date | null;
+  estimatedValue: number | null;
+}): string {
+  const canonical = [
+    normText(f.buyerName),
+    normText(f.title),
+    cpvDivisionKey(f.cpvDivisions),
+    deadlineKey(f.deadlineAt),
+    valueBucket(f.estimatedValue),
+  ].join('|');
+  return createHash('sha256').update(canonical).digest('hex');
+}
+
+/** lowercase, German umlaut expansion, non-alphanumeric -> space, trim/collapse whitespace. */
+function normText(s: string | null): string {
+  if (!s) return '';
+  return s
+    .toLowerCase()
+    .replace(/ä/g, 'ae')
+    .replace(/ö/g, 'oe')
+    .replace(/ü/g, 'ue')
+    .replace(/ß/g, 'ss')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .replace(/\s+/g, ' ');
+}
+
+/** Order-independent, deduplicated CPV division key. */
+function cpvDivisionKey(cpvDivisions: string[]): string {
+  return [...new Set(cpvDivisions)].sort().join(',');
+}
+
+/** NULL -> empty segment (92.4% NULL live); otherwise order-of-magnitude bucket. */
+function valueBucket(v: number | null): string {
+  if (v === null) return '';
+  return String(Math.floor(Math.log10(Math.max(v, 1))));
+}
+
+/** NULL -> empty segment (15.7% NULL live); otherwise date-only (day grain). */
+function deadlineKey(d: Date | null): string {
+  return d ? d.toISOString().slice(0, 10) : '';
+}