feat(11-03): cpvDivisions column — normalizer + backfill migration

Adds Tender.cpvDivisions String[] (@@index Gin) derived at ingestion time
via cpv-catalog.ts's divisionOf() — replaces exact-match cpvCodes
comparison with a typesafe, GIN-indexable hasSome target (FILTER-03,
Pitfall 2). Backfill migration 20260721150000_tender_cpv_divisions_backfill
applied locally: 1612/1671 rows populated across all observed divisions
(741 rows carry division '45' — Bauarbeiten); idempotent (second run:
UPDATE 0, ADD COLUMN IF NOT EXISTS / CREATE INDEX IF NOT EXISTS both skip
cleanly). Applied via docker exec psql + `prisma migrate resolve
--applied` + `prisma generate` against the local dev DB only — no
Docker deploy on the test server.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-21 16:11:40 +02:00
parent 7651ab6813
commit f9591dc491
6 changed files with 110 additions and 0 deletions
@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import { createHash } from 'crypto';
import { divisionOf } from './cpv/cpv-catalog';
import { bundeslandFromRegion } from './geo/nuts-bundesland';
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
@@ -32,6 +33,12 @@ export class TenderNormalizerService {
'Unbenannte Ausschreibung';
const buyerName = textValue(ocdsBuyer?.name);
const cpvCodes = extractCpvCodes(ocdsTender);
// Phase 11 (FILTER-03, Pitfall 2): derive distinct CPV divisions from
// the raw, inconsistently-formatted cpvCodes ("45", "45000000",
// "45000000-7") so new ingests are hasSome-filterable immediately —
// the ~1671 pre-existing rows are backfilled by a one-time migration
// (20260721150000_tender_cpv_divisions_backfill).
const cpvDivisions = deriveCpvDivisions(cpvCodes);
const region = textValue(ocdsBuyer?.address?.region);
const plz = textValue(ocdsBuyer?.address?.postalCode);
// Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
@@ -69,6 +76,7 @@ export class TenderNormalizerService {
title,
buyerName,
cpvCodes,
cpvDivisions,
region,
plz,
bundesland,
@@ -221,6 +229,18 @@ function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] {
return Array.from(new Set(codes));
}
/**
* Distinct CPV division codes (leading 2 digits) derived from the raw
* cpvCodes array via cpv-catalog.ts's divisionOf() — handles mixed formats
* ("45", "45000000", "45000000-7") uniformly (Pitfall 2). Never throws on
* an empty/malformed code; divisionOf('') === '' is filtered out below so
* cpvDivisions never contains an empty-string entry.
*/
function deriveCpvDivisions(cpvCodes: string[]): string[] {
const divisions = cpvCodes.map((code) => divisionOf(code)).filter(Boolean);
return Array.from(new Set(divisions));
}
function parseOcdsDate(value: unknown): Date | null {
if (typeof value !== 'string' || !value) return null;
const parsed = new Date(value);