feat(11-02): normalizer derives bundesland + backfill migration for existing rows

Replace the Phase-10-deferred `bundesland = null` assignment with
bundeslandFromRegion(region) so new ingests are Bundesland-filterable
immediately. Add @@index([bundesland]) for filter performance. New
handwritten migration 20260721140000_tender_bundesland_backfill backfills
the ~1671 pre-existing rows (idempotent UPDATE, only where bundesland IS
NULL AND region IS NOT NULL) — applied locally via
`docker exec tessera-ctl-db-1 psql`, resolved as applied in
_prisma_migrations, and prisma generate re-run.

Verified on the local dev DB: 933/1671 rows now have bundesland set
across all 16 Länder (738 remain NULL where region itself is NULL).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-21 16:01:32 +02:00
parent d12e1c9aa9
commit e4db602597
5 changed files with 79 additions and 5 deletions
@@ -0,0 +1,39 @@
-- FILTER-02 (Pitfall 1): Tender.bundesland is 100% NULL in the live DB
-- (0/1671 rows, verified) because the Phase-10 normalizer deferred
-- NUTS->Bundesland mapping to Phase 11. This migration:
-- 1. Adds the CREATE INDEX for bundesland (filter performance, matches
-- schema.prisma @@index([bundesland])).
-- 2. Backfills the ~1671 pre-existing rows by deriving bundesland from
-- the region column's NUTS-1 prefix (DE1..DEG -> the 16 Bundesländer),
-- mirroring geo/nuts-bundesland.ts's NUTS1_BUNDESLAND map exactly.
--
-- Idempotent: only touches rows where bundesland IS NULL AND region IS NOT
-- NULL — safe to run more than once, and never overwrites a value already
-- set by the (now-fixed) normalizer for newer ingests.
-- No user input is interpolated (T-11-05) — pure derived-field UPDATE.
-- CreateIndex
CREATE INDEX IF NOT EXISTS "Tender_bundesland_idx" ON "Tender"("bundesland");
-- Backfill
UPDATE "Tender"
SET "bundesland" = CASE upper(left("region", 3))
WHEN 'DE1' THEN 'Baden-Württemberg'
WHEN 'DE2' THEN 'Bayern'
WHEN 'DE3' THEN 'Berlin'
WHEN 'DE4' THEN 'Brandenburg'
WHEN 'DE5' THEN 'Bremen'
WHEN 'DE6' THEN 'Hamburg'
WHEN 'DE7' THEN 'Hessen'
WHEN 'DE8' THEN 'Mecklenburg-Vorpommern'
WHEN 'DE9' THEN 'Niedersachsen'
WHEN 'DEA' THEN 'Nordrhein-Westfalen'
WHEN 'DEB' THEN 'Rheinland-Pfalz'
WHEN 'DEC' THEN 'Saarland'
WHEN 'DED' THEN 'Sachsen'
WHEN 'DEE' THEN 'Sachsen-Anhalt'
WHEN 'DEF' THEN 'Schleswig-Holstein'
WHEN 'DEG' THEN 'Thüringen'
ELSE NULL
END
WHERE "bundesland" IS NULL AND "region" IS NOT NULL;
+1
View File
@@ -286,6 +286,7 @@ model Tender {
@@index([status])
@@index([deadlineAt])
@@index([publishedAt])
@@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance
// Deliberately NO tenant column and NO tenant index — this is global data (D-03)
}
@@ -89,6 +89,37 @@ describe('TenderNormalizerService', () => {
expect(normalized.title).toBeTruthy();
});
it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => {
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
const normalized = service.normalize(raw);
if (normalized.region) {
expect(normalized.bundesland).not.toBeNull();
} else {
expect(normalized.bundesland).toBeNull();
}
});
it('bundesland is null when region is null/absent — never crashes', () => {
const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
const rawWithoutRegion: RawTenderRecord = {
...raw,
ocdsPayload: {
...(raw.ocdsPayload as Record<string, unknown>),
buyer: {
...(raw.ocdsPayload as { buyer?: Record<string, unknown> })?.buyer,
address: { postalCode: '12345' },
},
},
};
const normalized = service.normalize(rawWithoutRegion);
expect(normalized.region).toBeNull();
expect(normalized.bundesland).toBeNull();
});
it('dedupKey = ocid when present', () => {
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import { createHash } from 'crypto';
import { bundeslandFromRegion } from './geo/nuts-bundesland';
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
/**
@@ -33,9 +34,11 @@ export class TenderNormalizerService {
const cpvCodes = extractCpvCodes(ocdsTender);
const region = textValue(ocdsBuyer?.address?.region);
const plz = textValue(ocdsBuyer?.address?.postalCode);
// NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not
// required for this phase's ingestion core.
const bundesland: string | null = null;
// Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
// NUTS-1 prefix so new ingests are filterable immediately — the
// ~1671 pre-existing rows are backfilled by a one-time migration
// (20260721140000_tender_bundesland_backfill).
const bundesland = bundeslandFromRegion(region);
const deadlineAt =
extractEformsDeadline(eformsRoot) ??
+2 -2
View File
@@ -55,10 +55,10 @@ export interface NormalizedTenderFields {
title: string;
buyerName: string | null;
cpvCodes: string[];
/** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */
/** NUTS region code (e.g. "DEA41") from the buyer address. */
region: string | null;
plz: string | null;
/** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */
/** Derived from region's NUTS-1 prefix via bundeslandFromRegion() (Phase 11, FILTER-02). Null when region is null/unrecognized. */
bundesland: string | null;
/** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */
deadlineAt: Date | null;