feat(11-02): normalizer derives bundesland + backfill migration for existing rows
Replace the Phase-10-deferred `bundesland = null` assignment with bundeslandFromRegion(region) so new ingests are Bundesland-filterable immediately. Add @@index([bundesland]) for filter performance. New handwritten migration 20260721140000_tender_bundesland_backfill backfills the ~1671 pre-existing rows (idempotent UPDATE, only where bundesland IS NULL AND region IS NOT NULL) — applied locally via `docker exec tessera-ctl-db-1 psql`, resolved as applied in _prisma_migrations, and prisma generate re-run. Verified on the local dev DB: 933/1671 rows now have bundesland set across all 16 Länder (738 remain NULL where region itself is NULL). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,39 @@
|
|||||||
|
-- FILTER-02 (Pitfall 1): Tender.bundesland is 100% NULL in the live DB
|
||||||
|
-- (0/1671 rows, verified) because the Phase-10 normalizer deferred
|
||||||
|
-- NUTS->Bundesland mapping to Phase 11. This migration:
|
||||||
|
-- 1. Adds the CREATE INDEX for bundesland (filter performance, matches
|
||||||
|
-- schema.prisma @@index([bundesland])).
|
||||||
|
-- 2. Backfills the ~1671 pre-existing rows by deriving bundesland from
|
||||||
|
-- the region column's NUTS-1 prefix (DE1..DEG -> the 16 Bundesländer),
|
||||||
|
-- mirroring geo/nuts-bundesland.ts's NUTS1_BUNDESLAND map exactly.
|
||||||
|
--
|
||||||
|
-- Idempotent: only touches rows where bundesland IS NULL AND region IS NOT
|
||||||
|
-- NULL — safe to run more than once, and never overwrites a value already
|
||||||
|
-- set by the (now-fixed) normalizer for newer ingests.
|
||||||
|
-- No user input is interpolated (T-11-05) — pure derived-field UPDATE.
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX IF NOT EXISTS "Tender_bundesland_idx" ON "Tender"("bundesland");
|
||||||
|
|
||||||
|
-- Backfill
|
||||||
|
UPDATE "Tender"
|
||||||
|
SET "bundesland" = CASE upper(left("region", 3))
|
||||||
|
WHEN 'DE1' THEN 'Baden-Württemberg'
|
||||||
|
WHEN 'DE2' THEN 'Bayern'
|
||||||
|
WHEN 'DE3' THEN 'Berlin'
|
||||||
|
WHEN 'DE4' THEN 'Brandenburg'
|
||||||
|
WHEN 'DE5' THEN 'Bremen'
|
||||||
|
WHEN 'DE6' THEN 'Hamburg'
|
||||||
|
WHEN 'DE7' THEN 'Hessen'
|
||||||
|
WHEN 'DE8' THEN 'Mecklenburg-Vorpommern'
|
||||||
|
WHEN 'DE9' THEN 'Niedersachsen'
|
||||||
|
WHEN 'DEA' THEN 'Nordrhein-Westfalen'
|
||||||
|
WHEN 'DEB' THEN 'Rheinland-Pfalz'
|
||||||
|
WHEN 'DEC' THEN 'Saarland'
|
||||||
|
WHEN 'DED' THEN 'Sachsen'
|
||||||
|
WHEN 'DEE' THEN 'Sachsen-Anhalt'
|
||||||
|
WHEN 'DEF' THEN 'Schleswig-Holstein'
|
||||||
|
WHEN 'DEG' THEN 'Thüringen'
|
||||||
|
ELSE NULL
|
||||||
|
END
|
||||||
|
WHERE "bundesland" IS NULL AND "region" IS NOT NULL;
|
||||||
@@ -286,6 +286,7 @@ model Tender {
|
|||||||
@@index([status])
|
@@index([status])
|
||||||
@@index([deadlineAt])
|
@@index([deadlineAt])
|
||||||
@@index([publishedAt])
|
@@index([publishedAt])
|
||||||
|
@@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance
|
||||||
// Deliberately NO tenant column and NO tenant index — this is global data (D-03)
|
// Deliberately NO tenant column and NO tenant index — this is global data (D-03)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -89,6 +89,37 @@ describe('TenderNormalizerService', () => {
|
|||||||
expect(normalized.title).toBeTruthy();
|
expect(normalized.title).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => {
|
||||||
|
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
|
||||||
|
|
||||||
|
const normalized = service.normalize(raw);
|
||||||
|
|
||||||
|
if (normalized.region) {
|
||||||
|
expect(normalized.bundesland).not.toBeNull();
|
||||||
|
} else {
|
||||||
|
expect(normalized.bundesland).toBeNull();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('bundesland is null when region is null/absent — never crashes', () => {
|
||||||
|
const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
|
||||||
|
const rawWithoutRegion: RawTenderRecord = {
|
||||||
|
...raw,
|
||||||
|
ocdsPayload: {
|
||||||
|
...(raw.ocdsPayload as Record<string, unknown>),
|
||||||
|
buyer: {
|
||||||
|
...(raw.ocdsPayload as { buyer?: Record<string, unknown> })?.buyer,
|
||||||
|
address: { postalCode: '12345' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalized = service.normalize(rawWithoutRegion);
|
||||||
|
|
||||||
|
expect(normalized.region).toBeNull();
|
||||||
|
expect(normalized.bundesland).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it('dedupKey = ocid when present', () => {
|
it('dedupKey = ocid when present', () => {
|
||||||
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
|
const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
|
import { bundeslandFromRegion } from './geo/nuts-bundesland';
|
||||||
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
|
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -33,9 +34,11 @@ export class TenderNormalizerService {
|
|||||||
const cpvCodes = extractCpvCodes(ocdsTender);
|
const cpvCodes = extractCpvCodes(ocdsTender);
|
||||||
const region = textValue(ocdsBuyer?.address?.region);
|
const region = textValue(ocdsBuyer?.address?.region);
|
||||||
const plz = textValue(ocdsBuyer?.address?.postalCode);
|
const plz = textValue(ocdsBuyer?.address?.postalCode);
|
||||||
// NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not
|
// Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
|
||||||
// required for this phase's ingestion core.
|
// NUTS-1 prefix so new ingests are filterable immediately — the
|
||||||
const bundesland: string | null = null;
|
// ~1671 pre-existing rows are backfilled by a one-time migration
|
||||||
|
// (20260721140000_tender_bundesland_backfill).
|
||||||
|
const bundesland = bundeslandFromRegion(region);
|
||||||
|
|
||||||
const deadlineAt =
|
const deadlineAt =
|
||||||
extractEformsDeadline(eformsRoot) ??
|
extractEformsDeadline(eformsRoot) ??
|
||||||
|
|||||||
@@ -55,10 +55,10 @@ export interface NormalizedTenderFields {
|
|||||||
title: string;
|
title: string;
|
||||||
buyerName: string | null;
|
buyerName: string | null;
|
||||||
cpvCodes: string[];
|
cpvCodes: string[];
|
||||||
/** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */
|
/** NUTS region code (e.g. "DEA41") from the buyer address. */
|
||||||
region: string | null;
|
region: string | null;
|
||||||
plz: string | null;
|
plz: string | null;
|
||||||
/** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */
|
/** Derived from region's NUTS-1 prefix via bundeslandFromRegion() (Phase 11, FILTER-02). Null when region is null/unrecognized. */
|
||||||
bundesland: string | null;
|
bundesland: string | null;
|
||||||
/** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */
|
/** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */
|
||||||
deadlineAt: Date | null;
|
deadlineAt: Date | null;
|
||||||
|
|||||||
Reference in New Issue
Block a user