feat(11-03): cpvDivisions column — normalizer + backfill migration
Adds Tender.cpvDivisions String[] (@@index Gin) derived at ingestion time via cpv-catalog.ts's divisionOf() — replaces exact-match cpvCodes comparison with a typesafe, GIN-indexable hasSome target (FILTER-03, Pitfall 2). Backfill migration 20260721150000_tender_cpv_divisions_backfill applied locally: 1612/1671 rows populated across all observed divisions (741 rows carry division '45' — Bauarbeiten); idempotent (second run: UPDATE 0, ADD COLUMN IF NOT EXISTS / CREATE INDEX IF NOT EXISTS both skip cleanly). Applied via docker exec psql + `prisma migrate resolve --applied` + `prisma generate` against the local dev DB only — no Docker deploy on the test server. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { createHash } from 'crypto';
|
||||
import { divisionOf } from './cpv/cpv-catalog';
|
||||
import { bundeslandFromRegion } from './geo/nuts-bundesland';
|
||||
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
|
||||
|
||||
@@ -32,6 +33,12 @@ export class TenderNormalizerService {
|
||||
'Unbenannte Ausschreibung';
|
||||
const buyerName = textValue(ocdsBuyer?.name);
|
||||
const cpvCodes = extractCpvCodes(ocdsTender);
|
||||
// Phase 11 (FILTER-03, Pitfall 2): derive distinct CPV divisions from
|
||||
// the raw, inconsistently-formatted cpvCodes ("45", "45000000",
|
||||
// "45000000-7") so new ingests are hasSome-filterable immediately —
|
||||
// the ~1671 pre-existing rows are backfilled by a one-time migration
|
||||
// (20260721150000_tender_cpv_divisions_backfill).
|
||||
const cpvDivisions = deriveCpvDivisions(cpvCodes);
|
||||
const region = textValue(ocdsBuyer?.address?.region);
|
||||
const plz = textValue(ocdsBuyer?.address?.postalCode);
|
||||
// Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
|
||||
@@ -69,6 +76,7 @@ export class TenderNormalizerService {
|
||||
title,
|
||||
buyerName,
|
||||
cpvCodes,
|
||||
cpvDivisions,
|
||||
region,
|
||||
plz,
|
||||
bundesland,
|
||||
@@ -221,6 +229,18 @@ function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] {
|
||||
return Array.from(new Set(codes));
|
||||
}
|
||||
|
||||
/**
|
||||
* Distinct CPV division codes (leading 2 digits) derived from the raw
|
||||
* cpvCodes array via cpv-catalog.ts's divisionOf() — handles mixed formats
|
||||
* ("45", "45000000", "45000000-7") uniformly (Pitfall 2). Never throws on
|
||||
* an empty/malformed code; divisionOf('') === '' is filtered out below so
|
||||
* cpvDivisions never contains an empty-string entry.
|
||||
*/
|
||||
function deriveCpvDivisions(cpvCodes: string[]): string[] {
|
||||
const divisions = cpvCodes.map((code) => divisionOf(code)).filter(Boolean);
|
||||
return Array.from(new Set(divisions));
|
||||
}
|
||||
|
||||
function parseOcdsDate(value: unknown): Date | null {
|
||||
if (typeof value !== 'string' || !value) return null;
|
||||
const parsed = new Date(value);
|
||||
|
||||
Reference in New Issue
Block a user