feat(11-02): normalizer derives bundesland + backfill migration for existing rows
Replace the Phase-10-deferred `bundesland = null` assignment with bundeslandFromRegion(region) so new ingests are Bundesland-filterable immediately. Add @@index([bundesland]) for filter performance. New handwritten migration 20260721140000_tender_bundesland_backfill backfills the ~1671 pre-existing rows (idempotent UPDATE, only where bundesland IS NULL AND region IS NOT NULL) — applied locally via `docker exec tessera-ctl-db-1 psql`, resolved as applied in _prisma_migrations, and prisma generate re-run. Verified on the local dev DB: 933/1671 rows now have bundesland set across all 16 Länder (738 remain NULL where region itself is NULL). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
-- FILTER-02 (Pitfall 1): Tender.bundesland is 100% NULL in the live DB
|
||||
-- (0/1671 rows, verified) because the Phase-10 normalizer deferred
|
||||
-- NUTS->Bundesland mapping to Phase 11. This migration:
|
||||
-- 1. Adds the CREATE INDEX for bundesland (filter performance, matches
|
||||
-- schema.prisma @@index([bundesland])).
|
||||
-- 2. Backfills the ~1671 pre-existing rows by deriving bundesland from
|
||||
-- the region column's NUTS-1 prefix (DE1..DEG -> the 16 Bundesländer),
|
||||
-- mirroring geo/nuts-bundesland.ts's NUTS1_BUNDESLAND map exactly.
|
||||
--
|
||||
-- Idempotent: only touches rows where bundesland IS NULL AND region IS NOT
|
||||
-- NULL — safe to run more than once, and never overwrites a value already
|
||||
-- set by the (now-fixed) normalizer for newer ingests.
|
||||
-- No user input is interpolated (T-11-05) — pure derived-field UPDATE.
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX IF NOT EXISTS "Tender_bundesland_idx" ON "Tender"("bundesland");
|
||||
|
||||
-- Backfill
|
||||
UPDATE "Tender"
|
||||
SET "bundesland" = CASE upper(left("region", 3))
|
||||
WHEN 'DE1' THEN 'Baden-Württemberg'
|
||||
WHEN 'DE2' THEN 'Bayern'
|
||||
WHEN 'DE3' THEN 'Berlin'
|
||||
WHEN 'DE4' THEN 'Brandenburg'
|
||||
WHEN 'DE5' THEN 'Bremen'
|
||||
WHEN 'DE6' THEN 'Hamburg'
|
||||
WHEN 'DE7' THEN 'Hessen'
|
||||
WHEN 'DE8' THEN 'Mecklenburg-Vorpommern'
|
||||
WHEN 'DE9' THEN 'Niedersachsen'
|
||||
WHEN 'DEA' THEN 'Nordrhein-Westfalen'
|
||||
WHEN 'DEB' THEN 'Rheinland-Pfalz'
|
||||
WHEN 'DEC' THEN 'Saarland'
|
||||
WHEN 'DED' THEN 'Sachsen'
|
||||
WHEN 'DEE' THEN 'Sachsen-Anhalt'
|
||||
WHEN 'DEF' THEN 'Schleswig-Holstein'
|
||||
WHEN 'DEG' THEN 'Thüringen'
|
||||
ELSE NULL
|
||||
END
|
||||
WHERE "bundesland" IS NULL AND "region" IS NOT NULL;
|
||||
@@ -286,6 +286,7 @@ model Tender {
|
||||
@@index([status])
|
||||
@@index([deadlineAt])
|
||||
@@index([publishedAt])
|
||||
@@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance
|
||||
// Deliberately NO tenant column and NO tenant index — this is global data (D-03)
|
||||
}
|
||||
|
||||
|
||||
@@ -89,6 +89,37 @@ describe('TenderNormalizerService', () => {
|
||||
expect(normalized.title).toBeTruthy();
|
||||
});
|
||||
|
||||
it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => {
|
||||
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
|
||||
|
||||
const normalized = service.normalize(raw);
|
||||
|
||||
if (normalized.region) {
|
||||
expect(normalized.bundesland).not.toBeNull();
|
||||
} else {
|
||||
expect(normalized.bundesland).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it('bundesland is null when region is null/absent — never crashes', () => {
|
||||
const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
|
||||
const rawWithoutRegion: RawTenderRecord = {
|
||||
...raw,
|
||||
ocdsPayload: {
|
||||
...(raw.ocdsPayload as Record<string, unknown>),
|
||||
buyer: {
|
||||
...(raw.ocdsPayload as { buyer?: Record<string, unknown> })?.buyer,
|
||||
address: { postalCode: '12345' },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const normalized = service.normalize(rawWithoutRegion);
|
||||
|
||||
expect(normalized.region).toBeNull();
|
||||
expect(normalized.bundesland).toBeNull();
|
||||
});
|
||||
|
||||
it('dedupKey = ocid when present', () => {
|
||||
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { createHash } from 'crypto';
|
||||
import { bundeslandFromRegion } from './geo/nuts-bundesland';
|
||||
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
|
||||
|
||||
/**
|
||||
@@ -33,9 +34,11 @@ export class TenderNormalizerService {
|
||||
const cpvCodes = extractCpvCodes(ocdsTender);
|
||||
const region = textValue(ocdsBuyer?.address?.region);
|
||||
const plz = textValue(ocdsBuyer?.address?.postalCode);
|
||||
// NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not
|
||||
// required for this phase's ingestion core.
|
||||
const bundesland: string | null = null;
|
||||
// Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
|
||||
// NUTS-1 prefix so new ingests are filterable immediately — the
|
||||
// ~1671 pre-existing rows are backfilled by a one-time migration
|
||||
// (20260721140000_tender_bundesland_backfill).
|
||||
const bundesland = bundeslandFromRegion(region);
|
||||
|
||||
const deadlineAt =
|
||||
extractEformsDeadline(eformsRoot) ??
|
||||
|
||||
@@ -55,10 +55,10 @@ export interface NormalizedTenderFields {
|
||||
title: string;
|
||||
buyerName: string | null;
|
||||
cpvCodes: string[];
|
||||
/** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */
|
||||
/** NUTS region code (e.g. "DEA41") from the buyer address. */
|
||||
region: string | null;
|
||||
plz: string | null;
|
||||
/** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */
|
||||
/** Derived from region's NUTS-1 prefix via bundeslandFromRegion() (Phase 11, FILTER-02). Null when region is null/unrecognized. */
|
||||
bundesland: string | null;
|
||||
/** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */
|
||||
deadlineAt: Date | null;
|
||||
|
||||
Reference in New Issue
Block a user