From e4db602597b156381fb56983c45251cd833dbc82 Mon Sep 17 00:00:00 2001 From: Schalli Date: Tue, 21 Jul 2026 16:01:32 +0200 Subject: [PATCH] feat(11-02): normalizer derives bundesland + backfill migration for existing rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the Phase-10-deferred `bundesland = null` assignment with bundeslandFromRegion(region) so new ingests are Bundesland-filterable immediately. Add @@index([bundesland]) for filter performance. New handwritten migration 20260721140000_tender_bundesland_backfill backfills the ~1671 pre-existing rows (idempotent UPDATE, only where bundesland IS NULL AND region IS NOT NULL) — applied locally via `docker exec tessera-ctl-db-1 psql`, resolved as applied in _prisma_migrations, and prisma generate re-run. Verified on the local dev DB: 933/1671 rows now have bundesland set across all 16 Länder (738 remain NULL where region itself is NULL). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../migration.sql | 39 +++++++++++++++++++ apps/api/prisma/schema.prisma | 1 + .../tenders/tender-normalizer.service.spec.ts | 31 +++++++++++++++ .../src/tenders/tender-normalizer.service.ts | 9 +++-- apps/api/src/tenders/tender.types.ts | 4 +- 5 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql diff --git a/apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql b/apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql new file mode 100644 index 0000000..ee52bdb --- /dev/null +++ b/apps/api/prisma/migrations/20260721140000_tender_bundesland_backfill/migration.sql @@ -0,0 +1,39 @@ +-- FILTER-02 (Pitfall 1): Tender.bundesland is 100% NULL in the live DB +-- (0/1671 rows, verified) because the Phase-10 normalizer deferred +-- NUTS->Bundesland mapping to Phase 11. This migration: +-- 1. Adds the CREATE INDEX for bundesland (filter performance, matches +-- schema.prisma @@index([bundesland])). +-- 2. Backfills the ~1671 pre-existing rows by deriving bundesland from +-- the region column's NUTS-1 prefix (DE1..DEG -> the 16 Bundesländer), +-- mirroring geo/nuts-bundesland.ts's NUTS1_BUNDESLAND map exactly. +-- +-- Idempotent: only touches rows where bundesland IS NULL AND region IS NOT +-- NULL — safe to run more than once, and never overwrites a value already +-- set by the (now-fixed) normalizer for newer ingests. +-- No user input is interpolated (T-11-05) — pure derived-field UPDATE. + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "Tender_bundesland_idx" ON "Tender"("bundesland"); + +-- Backfill +UPDATE "Tender" +SET "bundesland" = CASE upper(left("region", 3)) + WHEN 'DE1' THEN 'Baden-Württemberg' + WHEN 'DE2' THEN 'Bayern' + WHEN 'DE3' THEN 'Berlin' + WHEN 'DE4' THEN 'Brandenburg' + WHEN 'DE5' THEN 'Bremen' + WHEN 'DE6' THEN 'Hamburg' + WHEN 'DE7' THEN 'Hessen' + WHEN 'DE8' THEN 'Mecklenburg-Vorpommern' + WHEN 'DE9' THEN 'Niedersachsen' + WHEN 'DEA' THEN 'Nordrhein-Westfalen' + WHEN 'DEB' THEN 'Rheinland-Pfalz' + WHEN 'DEC' THEN 'Saarland' + WHEN 'DED' THEN 'Sachsen' + WHEN 'DEE' THEN 'Sachsen-Anhalt' + WHEN 'DEF' THEN 'Schleswig-Holstein' + WHEN 'DEG' THEN 'Thüringen' + ELSE NULL +END +WHERE "bundesland" IS NULL AND "region" IS NOT NULL; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 48c9a53..70549da 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -286,6 +286,7 @@ model Tender { @@index([status]) @@index([deadlineAt]) @@index([publishedAt]) + @@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance // Deliberately NO tenant column and NO tenant index — this is global data (D-03) } diff --git a/apps/api/src/tenders/tender-normalizer.service.spec.ts b/apps/api/src/tenders/tender-normalizer.service.spec.ts index a26ed7f..c853d87 100644 --- a/apps/api/src/tenders/tender-normalizer.service.spec.ts +++ b/apps/api/src/tenders/tender-normalizer.service.spec.ts @@ -89,6 +89,37 @@ describe('TenderNormalizerService', () => { expect(normalized.title).toBeTruthy(); }); + it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + + const normalized = service.normalize(raw); + + if (normalized.region) { + expect(normalized.bundesland).not.toBeNull(); + } else { + expect(normalized.bundesland).toBeNull(); + } + }); + + it('bundesland is null when region is null/absent — never crashes', () => { + const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); + const rawWithoutRegion: RawTenderRecord = { + ...raw, + ocdsPayload: { + ...(raw.ocdsPayload as Record), + buyer: { + ...(raw.ocdsPayload as { buyer?: Record })?.buyer, + address: { postalCode: '12345' }, + }, + }, + }; + + const normalized = service.normalize(rawWithoutRegion); + + expect(normalized.region).toBeNull(); + expect(normalized.bundesland).toBeNull(); + }); + it('dedupKey = ocid when present', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts index 51c1cfe..da5e579 100644 --- a/apps/api/src/tenders/tender-normalizer.service.ts +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; import { createHash } from 'crypto'; +import { bundeslandFromRegion } from './geo/nuts-bundesland'; import type { NormalizedTenderFields, RawTenderRecord } from './tender.types'; /** @@ -33,9 +34,11 @@ export class TenderNormalizerService { const cpvCodes = extractCpvCodes(ocdsTender); const region = textValue(ocdsBuyer?.address?.region); const plz = textValue(ocdsBuyer?.address?.postalCode); - // NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not - // required for this phase's ingestion core. - const bundesland: string | null = null; + // Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's + // NUTS-1 prefix so new ingests are filterable immediately — the + // ~1671 pre-existing rows are backfilled by a one-time migration + // (20260721140000_tender_bundesland_backfill). + const bundesland = bundeslandFromRegion(region); const deadlineAt = extractEformsDeadline(eformsRoot) ?? diff --git a/apps/api/src/tenders/tender.types.ts b/apps/api/src/tenders/tender.types.ts index 70e8b6a..a6ae9c2 100644 --- a/apps/api/src/tenders/tender.types.ts +++ b/apps/api/src/tenders/tender.types.ts @@ -55,10 +55,10 @@ export interface NormalizedTenderFields { title: string; buyerName: string | null; cpvCodes: string[]; - /** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */ + /** NUTS region code (e.g. "DEA41") from the buyer address. */ region: string | null; plz: string | null; - /** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */ + /** Derived from region's NUTS-1 prefix via bundeslandFromRegion() (Phase 11, FILTER-02). Null when region is null/unrecognized. */ bundesland: string | null; /** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */ deadlineAt: Date | null;