diff --git a/apps/api/prisma/migrations/20260723120000_add_tender_source/migration.sql b/apps/api/prisma/migrations/20260723120000_add_tender_source/migration.sql new file mode 100644 index 0000000..70c5c61 --- /dev/null +++ b/apps/api/prisma/migrations/20260723120000_add_tender_source/migration.sql @@ -0,0 +1,44 @@ +-- SCHEMA-03 (D-03/D-04) — TenderSource 1:n model + Tender.fingerprint. +-- +-- Two-step ordering (Pitfall 5, 13-RESEARCH.md): (1) create the table and +-- add the additive Tender.fingerprint column BEFORE (2) backfilling +-- TenderSource rows from the pre-existing ~2851 DÖE Tender rows, and only +-- THEN activating the unique constraint. DÖE sourceNoticeIds are live-unique +-- (0% NULL OCID) so the backfill insert itself will not violate the +-- constraint, but the ordering is kept strict regardless. +-- +-- fingerprint backfill (Tender.fingerprint) is NOT done here in SQL — the +-- umlaut/CPV-division normalization it needs lives in code +-- (tender-fingerprint.ts). It is populated by the one-time TS script +-- `backfill-tender-source.ts`, run once after this migration is applied. + +-- Step 1a: additive nullable column on Tender. +ALTER TABLE "Tender" ADD COLUMN "fingerprint" TEXT; +CREATE INDEX "Tender_fingerprint_idx" ON "Tender"("fingerprint"); + +-- Step 1b: TenderSource table. +CREATE TABLE "TenderSource" ( + "id" TEXT NOT NULL, + "tenderId" TEXT NOT NULL, + "sourcePortal" TEXT NOT NULL, + "sourceNoticeId" TEXT NOT NULL, + "ocid" TEXT, + "sourceUrl" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "TenderSource_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX "TenderSource_tenderId_idx" ON "TenderSource"("tenderId"); + +ALTER TABLE "TenderSource" ADD CONSTRAINT "TenderSource_tenderId_fkey" + FOREIGN KEY ("tenderId") REFERENCES "Tender"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Step 2: backfill — one TenderSource row per pre-existing Tender, from its +-- current sourcePortal/sourceNoticeId/sourceUrl/ocid (D-05). +INSERT INTO "TenderSource" (id, "tenderId", "sourcePortal", "sourceNoticeId", ocid, "sourceUrl", "createdAt") +SELECT gen_random_uuid(), t.id, t."sourcePortal", t."sourceNoticeId", t.ocid, t."sourceUrl", now() +FROM "Tender" t; + +-- Step 3: unique constraint, applied AFTER the backfill insert succeeds. +CREATE UNIQUE INDEX "TenderSource_sourcePortal_sourceNoticeId_key" ON "TenderSource"("sourcePortal", "sourceNoticeId"); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index cdf4764..54ef5f7 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -281,6 +281,10 @@ model Tender { sourceUrl String? contentHash String // SCHEMA-02 change detection rawPayload Json? // debugging / re-normalization + // SCHEMA-03 (D-04) — Fuzzy-Dedup lookup key, additive + nullable. Backfilled + // for pre-existing rows by backfill-tender-source.ts (Plan 13-01 Task 2). + // dedupKey above stays the SCHEMA-02 upsert target — NOT replaced here. + fingerprint String? publishedAt DateTime createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -290,9 +294,30 @@ model Tender { @@index([publishedAt]) @@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance @@index([cpvDivisions], type: Gin) // FILTER-03: post-backfill CPV-Divisions-Filter-Performance (hasSome) + @@index([fingerprint]) // SCHEMA-03: dedup resolver fingerprint-tier lookup // Deliberately NO tenant column and NO tenant index — this is global data (D-03) triage TenderTriage[] matches TenderMatch[] + sources TenderSource[] // SCHEMA-03/D-03 — all source portals this tender was seen on +} + +// SCHEMA-03 (D-03) — Quell-Ebene einer Ausschreibung. 1:n zu Tender: ein +// Tender ist EIN Trefferlisten-Eintrag; mehrere TenderSource-Zeilen sind die +// Liste der Quell-Portale/Links, über die er gefunden wurde (Cross-Source +// Dedup, "ein Eintrag + Liste ALLER Quell-Links", nicht "Primärquelle +// gewinnt"). Backfilled 1:1 for pre-existing DÖE tenders (D-05). +model TenderSource { + id String @id @default(uuid()) + tenderId String + sourcePortal String // 'doe-opendata' | 'tender24' | 'lhs-vpbw' | 'vergabe.landbw' | 'cosinex-dtvp' + sourceNoticeId String + ocid String? + sourceUrl String? + createdAt DateTime @default(now()) + tender Tender @relation(fields: [tenderId], references: [id], onDelete: Cascade) + + @@unique([sourcePortal, sourceNoticeId]) // eine Quell-Notiz gehört zu genau einem Tender + @@index([tenderId]) } // UI-03/04 — per-user Triage-Zustand pro Tender (gelesen/ungelesen, Favorit). diff --git a/apps/api/src/tenders/backfill-tender-source.ts b/apps/api/src/tenders/backfill-tender-source.ts new file mode 100644 index 0000000..fe236cf --- /dev/null +++ b/apps/api/src/tenders/backfill-tender-source.ts @@ -0,0 +1,67 @@ +/** + * One-time backfill script (SCHEMA-03, D-05) — computes and persists + * `Tender.fingerprint` for every pre-existing row. + * + * The TenderSource row backfill itself lives in the migration SQL + * (20260723120000_add_tender_source/migration.sql) because it's a pure + * column copy. The fingerprint backfill does NOT belong in SQL: it needs + * the umlaut/CPV-normalization logic in tender-fingerprint.ts, which is + * code, not SQL (13-RESEARCH.md "Backfill-Migration"). + * + * Run once, manually, after the migration has been applied (compiled first + * — this file uses standard extensionless TS imports, so it must run as + * built CommonJS, not as a raw .ts file): + * pnpm --filter @tessera/api build + * DATABASE_URL=... node dist/tenders/backfill-tender-source.js + * + * Idempotent: recomputes and overwrites fingerprint for every row it reads, + * so re-running is safe (same deterministic output for unchanged inputs). + */ +import { PrismaClient, Prisma } from '@prisma/client'; +import { tenderFingerprint } from './tender-fingerprint'; + +async function main(): Promise { + const prisma = new PrismaClient(); + try { + const tenders = await prisma.tender.findMany({ + select: { + id: true, + title: true, + buyerName: true, + cpvDivisions: true, + deadlineAt: true, + estimatedValue: true, + }, + }); + + let updated = 0; + for (const t of tenders) { + const fingerprint = tenderFingerprint({ + buyerName: t.buyerName, + title: t.title, + cpvDivisions: t.cpvDivisions, + deadlineAt: t.deadlineAt, + // estimatedValue is Decimal? — convert to number|null before bucketing (T-13-01-03). + estimatedValue: decimalToNumberOrNull(t.estimatedValue), + }); + await prisma.tender.update({ + where: { id: t.id }, + data: { fingerprint }, + }); + updated += 1; + } + + console.log(`backfill-tender-source: fingerprint set on ${updated}/${tenders.length} tenders`); + } finally { + await prisma.$disconnect(); + } +} + +function decimalToNumberOrNull(v: Prisma.Decimal | null): number | null { + return v === null ? null : Number(v); +} + +main().catch((err) => { + console.error('backfill-tender-source failed:', err); + process.exitCode = 1; +});