diff --git a/apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql b/apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql new file mode 100644 index 0000000..ee52bdb --- /dev/null +++ b/apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql @@ -0,0 +1,39 @@ +-- FILTER-02 (Pitfall 1): Tender.bundesland is 100% NULL in the live DB +-- (0/1671 rows, verified) because the Phase-10 normalizer deferred +-- NUTS->Bundesland mapping to Phase 11. This migration: +-- 1. Adds the CREATE INDEX for bundesland (filter performance, matches +-- schema.prisma @@index([bundesland])). +-- 2. Backfills the ~1671 pre-existing rows by deriving bundesland from +-- the region column's NUTS-1 prefix (DE1..DEG -> the 16 Bundesländer), +-- mirroring geo/nuts-bundesland.ts's NUTS1_BUNDESLAND map exactly. +-- +-- Idempotent: only touches rows where bundesland IS NULL AND region IS NOT +-- NULL — safe to run more than once, and never overwrites a value already +-- set by the (now-fixed) normalizer for newer ingests. +-- No user input is interpolated (T-11-05) — pure derived-field UPDATE. + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "Tender_bundesland_idx" ON "Tender"("bundesland"); + +-- Backfill +UPDATE "Tender" +SET "bundesland" = CASE upper(left("region", 3)) + WHEN 'DE1' THEN 'Baden-Württemberg' + WHEN 'DE2' THEN 'Bayern' + WHEN 'DE3' THEN 'Berlin' + WHEN 'DE4' THEN 'Brandenburg' + WHEN 'DE5' THEN 'Bremen' + WHEN 'DE6' THEN 'Hamburg' + WHEN 'DE7' THEN 'Hessen' + WHEN 'DE8' THEN 'Mecklenburg-Vorpommern' + WHEN 'DE9' THEN 'Niedersachsen' + WHEN 'DEA' THEN 'Nordrhein-Westfalen' + WHEN 'DEB' THEN 'Rheinland-Pfalz' + WHEN 'DEC' THEN 'Saarland' + WHEN 'DED' THEN 'Sachsen' + WHEN 'DEE' THEN 'Sachsen-Anhalt' + WHEN 'DEF' THEN 'Schleswig-Holstein' + WHEN 'DEG' THEN 'Thüringen' + ELSE NULL +END +WHERE "bundesland" IS NULL AND "region" IS NOT NULL; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 48c9a53..70549da 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -286,6 +286,7 @@ model Tender { @@index([status]) @@index([deadlineAt]) @@index([publishedAt]) + @@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance // Deliberately NO tenant column and NO tenant index — this is global data (D-03) } diff --git a/apps/api/src/tenders/tender-normalizer.service.spec.ts b/apps/api/src/tenders/tender-normalizer.service.spec.ts index a26ed7f..c853d87 100644 --- a/apps/api/src/tenders/tender-normalizer.service.spec.ts +++ b/apps/api/src/tenders/tender-normalizer.service.spec.ts @@ -89,6 +89,37 @@ describe('TenderNormalizerService', () => { expect(normalized.title).toBeTruthy(); }); + it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + + const normalized = service.normalize(raw); + + if (normalized.region) { + expect(normalized.bundesland).not.toBeNull(); + } else { + expect(normalized.bundesland).toBeNull(); + } + }); + + it('bundesland is null when region is null/absent — never crashes', () => { + const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); + const rawWithoutRegion: RawTenderRecord = { + ...raw, + ocdsPayload: { + ...(raw.ocdsPayload as Record), + buyer: { + ...(raw.ocdsPayload as { buyer?: Record })?.buyer, + address: { postalCode: '12345' }, + }, + }, + }; + + const normalized = service.normalize(rawWithoutRegion); + + expect(normalized.region).toBeNull(); + expect(normalized.bundesland).toBeNull(); + }); + it('dedupKey = ocid when present', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts index 51c1cfe..da5e579 100644 --- a/apps/api/src/tenders/tender-normalizer.service.ts +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; import { createHash } from 'crypto'; +import { bundeslandFromRegion } from './geo/nuts-bundesland'; import type { NormalizedTenderFields, RawTenderRecord } from './tender.types'; /** @@ -33,9 +34,11 @@ export class TenderNormalizerService { const cpvCodes = extractCpvCodes(ocdsTender); const region = textValue(ocdsBuyer?.address?.region); const plz = textValue(ocdsBuyer?.address?.postalCode); - // NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not - // required for this phase's ingestion core. - const bundesland: string | null = null; + // Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's + // NUTS-1 prefix so new ingests are filterable immediately — the + // ~1671 pre-existing rows are backfilled by a one-time migration + // (20260721140000_tender_bundesland_backfill). + const bundesland = bundeslandFromRegion(region); const deadlineAt = extractEformsDeadline(eformsRoot) ?? diff --git a/apps/api/src/tenders/tender.types.ts b/apps/api/src/tenders/tender.types.ts index 70e8b6a..a6ae9c2 100644 --- a/apps/api/src/tenders/tender.types.ts +++ b/apps/api/src/tenders/tender.types.ts @@ -55,10 +55,10 @@ export interface NormalizedTenderFields { title: string; buyerName: string | null; cpvCodes: string[]; - /** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */ + /** NUTS region code (e.g. "DEA41") from the buyer address. */ region: string | null; plz: string | null; - /** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */ + /** Derived from region's NUTS-1 prefix via bundeslandFromRegion() (Phase 11, FILTER-02). Null when region is null/unrecognized. */ bundesland: string | null; /** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */ deadlineAt: Date | null;