feat(13-01): add TenderSource model + Tender.fingerprint, backfill 2851 rows

Additive schema change (SCHEMA-03/D-03/D-04): new model TenderSource (1:n Tender, @@unique[sourcePortal, sourceNoticeId], onDelete Cascade) and a nullable Tender.fingerprint column + index. dedupKey stays unchanged as the SCHEMA-02 upsert target. Migration 20260723120000_add_tender_source applies in strict order (Pitfall 5): table+column create, then one TenderSource row per pre-existing Tender via SQL INSERT/SELECT, then the unique constraint. Applied locally against the tessera dev DB (container IP, no host port) — verified via psql: TenderSource count == Tender count == 2851. backfill-tender-source.ts is a one-time script that computes Tender.fingerprint via the Task-1 tenderFingerprint() function (Decimal->number conversion for estimatedValue, T-13-01-03) — run via the compiled dist/ output (source uses standard extensionless TS imports for tsc compatibility). Confirmed: 2851/2851 rows backfilled, idempotent re-run verified. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-23 08:38:26 +02:00
parent c6cac696ff
commit 447fb74e0c
3 changed files with 136 additions and 0 deletions
@@ -0,0 +1,44 @@
+-- SCHEMA-03 (D-03/D-04) — TenderSource 1:n model + Tender.fingerprint.
+--
+-- Two-step ordering (Pitfall 5, 13-RESEARCH.md): (1) create the table and
+-- add the additive Tender.fingerprint column BEFORE (2) backfilling
+-- TenderSource rows from the pre-existing ~2851 DÖE Tender rows, and only
+-- THEN activating the unique constraint. DÖE sourceNoticeIds are live-unique
+-- (0% NULL OCID) so the backfill insert itself will not violate the
+-- constraint, but the ordering is kept strict regardless.
+--
+-- fingerprint backfill (Tender.fingerprint) is NOT done here in SQL — the
+-- umlaut/CPV-division normalization it needs lives in code
+-- (tender-fingerprint.ts). It is populated by the one-time TS script
+-- `backfill-tender-source.ts`, run once after this migration is applied.
+
+-- Step 1a: additive nullable column on Tender.
+ALTER TABLE "Tender" ADD COLUMN "fingerprint" TEXT;
+CREATE INDEX "Tender_fingerprint_idx" ON "Tender"("fingerprint");
+
+-- Step 1b: TenderSource table.
+CREATE TABLE "TenderSource" (
+    "id" TEXT NOT NULL,
+    "tenderId" TEXT NOT NULL,
+    "sourcePortal" TEXT NOT NULL,
+    "sourceNoticeId" TEXT NOT NULL,
+    "ocid" TEXT,
+    "sourceUrl" TEXT,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "TenderSource_pkey" PRIMARY KEY ("id")
+);
+
+CREATE INDEX "TenderSource_tenderId_idx" ON "TenderSource"("tenderId");
+
+ALTER TABLE "TenderSource" ADD CONSTRAINT "TenderSource_tenderId_fkey"
+    FOREIGN KEY ("tenderId") REFERENCES "Tender"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Step 2: backfill — one TenderSource row per pre-existing Tender, from its
+-- current sourcePortal/sourceNoticeId/sourceUrl/ocid (D-05).
+INSERT INTO "TenderSource" (id, "tenderId", "sourcePortal", "sourceNoticeId", ocid, "sourceUrl", "createdAt")
+SELECT gen_random_uuid(), t.id, t."sourcePortal", t."sourceNoticeId", t.ocid, t."sourceUrl", now()
+FROM "Tender" t;
+
+-- Step 3: unique constraint, applied AFTER the backfill insert succeeds.
+CREATE UNIQUE INDEX "TenderSource_sourcePortal_sourceNoticeId_key" ON "TenderSource"("sourcePortal", "sourceNoticeId");
@@ -281,6 +281,10 @@ model Tender {
  sourceUrl      String?
  contentHash    String // SCHEMA-02 change detection
  rawPayload     Json? // debugging / re-normalization
+  // SCHEMA-03 (D-04) — Fuzzy-Dedup lookup key, additive + nullable. Backfilled
+  // for pre-existing rows by backfill-tender-source.ts (Plan 13-01 Task 2).
+  // dedupKey above stays the SCHEMA-02 upsert target — NOT replaced here.
+  fingerprint    String?
  publishedAt    DateTime
  createdAt      DateTime  @default(now())
  updatedAt      DateTime  @updatedAt
@@ -290,9 +294,30 @@ model Tender {
  @@index([publishedAt])
  @@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance
  @@index([cpvDivisions], type: Gin) // FILTER-03: post-backfill CPV-Divisions-Filter-Performance (hasSome)
+  @@index([fingerprint]) // SCHEMA-03: dedup resolver fingerprint-tier lookup
  // Deliberately NO tenant column and NO tenant index — this is global data (D-03)
  triage  TenderTriage[]
  matches TenderMatch[]
+  sources TenderSource[] // SCHEMA-03/D-03 — all source portals this tender was seen on
+}
+
+// SCHEMA-03 (D-03) — Quell-Ebene einer Ausschreibung. 1:n zu Tender: ein
+// Tender ist EIN Trefferlisten-Eintrag; mehrere TenderSource-Zeilen sind die
+// Liste der Quell-Portale/Links, über die er gefunden wurde (Cross-Source
+// Dedup, "ein Eintrag + Liste ALLER Quell-Links", nicht "Primärquelle
+// gewinnt"). Backfilled 1:1 for pre-existing DÖE tenders (D-05).
+model TenderSource {
+  id             String   @id @default(uuid())
+  tenderId       String
+  sourcePortal   String // 'doe-opendata' | 'tender24' | 'lhs-vpbw' | 'vergabe.landbw' | 'cosinex-dtvp'
+  sourceNoticeId String
+  ocid           String?
+  sourceUrl      String?
+  createdAt      DateTime @default(now())
+  tender         Tender   @relation(fields: [tenderId], references: [id], onDelete: Cascade)
+
+  @@unique([sourcePortal, sourceNoticeId]) // eine Quell-Notiz gehört zu genau einem Tender
+  @@index([tenderId])
 }

 // UI-03/04 — per-user Triage-Zustand pro Tender (gelesen/ungelesen, Favorit).
@@ -0,0 +1,67 @@
+/**
+ * One-time backfill script (SCHEMA-03, D-05) — computes and persists
+ * `Tender.fingerprint` for every pre-existing row.
+ *
+ * The TenderSource row backfill itself lives in the migration SQL
+ * (20260723120000_add_tender_source/migration.sql) because it's a pure
+ * column copy. The fingerprint backfill does NOT belong in SQL: it needs
+ * the umlaut/CPV-normalization logic in tender-fingerprint.ts, which is
+ * code, not SQL (13-RESEARCH.md "Backfill-Migration").
+ *
+ * Run once, manually, after the migration has been applied (compiled first
+ * — this file uses standard extensionless TS imports, so it must run as
+ * built CommonJS, not as a raw .ts file):
+ *   pnpm --filter @tessera/api build
+ *   DATABASE_URL=... node dist/tenders/backfill-tender-source.js
+ *
+ * Idempotent: recomputes and overwrites fingerprint for every row it reads,
+ * so re-running is safe (same deterministic output for unchanged inputs).
+ */
+import { PrismaClient, Prisma } from '@prisma/client';
+import { tenderFingerprint } from './tender-fingerprint';
+
+async function main(): Promise<void> {
+  const prisma = new PrismaClient();
+  try {
+    const tenders = await prisma.tender.findMany({
+      select: {
+        id: true,
+        title: true,
+        buyerName: true,
+        cpvDivisions: true,
+        deadlineAt: true,
+        estimatedValue: true,
+      },
+    });
+
+    let updated = 0;
+    for (const t of tenders) {
+      const fingerprint = tenderFingerprint({
+        buyerName: t.buyerName,
+        title: t.title,
+        cpvDivisions: t.cpvDivisions,
+        deadlineAt: t.deadlineAt,
+        // estimatedValue is Decimal? — convert to number|null before bucketing (T-13-01-03).
+        estimatedValue: decimalToNumberOrNull(t.estimatedValue),
+      });
+      await prisma.tender.update({
+        where: { id: t.id },
+        data: { fingerprint },
+      });
+      updated += 1;
+    }
+
+    console.log(`backfill-tender-source: fingerprint set on ${updated}/${tenders.length} tenders`);
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+
+function decimalToNumberOrNull(v: Prisma.Decimal | null): number | null {
+  return v === null ? null : Number(v);
+}
+
+main().catch((err) => {
+  console.error('backfill-tender-source failed:', err);
+  process.exitCode = 1;
+});