From f9591dc4916f64a52a0462a184eacbc4ebaf967e Mon Sep 17 00:00:00 2001 From: Schalli Date: Tue, 21 Jul 2026 16:11:40 +0200 Subject: [PATCH] =?UTF-8?q?feat(11-03):=20cpvDivisions=20column=20?= =?UTF-8?q?=E2=80=94=20normalizer=20+=20backfill=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Tender.cpvDivisions String[] (@@index Gin) derived at ingestion time via cpv-catalog.ts's divisionOf() — replaces exact-match cpvCodes comparison with a typesafe, GIN-indexable hasSome target (FILTER-03, Pitfall 2). Backfill migration 20260721150000_tender_cpv_divisions_backfill applied locally: 1612/1671 rows populated across all observed divisions (741 rows carry division '45' — Bauarbeiten); idempotent (second run: UPDATE 0, ADD COLUMN IF NOT EXISTS / CREATE INDEX IF NOT EXISTS both skip cleanly). Applied via docker exec psql + `prisma migrate resolve --applied` + `prisma generate` against the local dev DB only — no Docker deploy on the test server. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../migration.sql | 41 +++++++++++++++++++ apps/api/prisma/schema.prisma | 2 + .../src/tenders/tender-ingestion.service.ts | 2 + .../tenders/tender-normalizer.service.spec.ts | 36 ++++++++++++++++ .../src/tenders/tender-normalizer.service.ts | 20 +++++++++ apps/api/src/tenders/tender.types.ts | 9 ++++ 6 files changed, 110 insertions(+) create mode 100644 apps/api/prisma/migrations/20260721150000_tender_cpv_divisions_backfill/migration.sql diff --git a/apps/api/prisma/migrations/20260721150000_tender_cpv_divisions_backfill/migration.sql b/apps/api/prisma/migrations/20260721150000_tender_cpv_divisions_backfill/migration.sql new file mode 100644 index 0000000..a70a49e --- /dev/null +++ b/apps/api/prisma/migrations/20260721150000_tender_cpv_divisions_backfill/migration.sql @@ -0,0 +1,41 @@ +-- FILTER-03 (Pitfall 2): Tender.cpvCodes is stored in inconsistent formats +-- in the live DB (verified sample: "45", "45000000", "45000000-7" all +-- referring to the same CPV division "45" = Bauarbeiten; 638 distinct raw +-- values). An exact-equality `cpvCodes has '45000000'` filter would miss +-- rows stored in a different format. This migration: +-- 1. Adds the `cpvDivisions` String[] column (typesafe, GIN-indexable +-- hasSome-match target — replaces raw-SQL per-element LIKE matching, +-- RESEARCH Pattern 3 Option B / Don't Hand-Roll). +-- 2. Adds the matching GIN index (matches schema.prisma +-- @@index([cpvDivisions], type: Gin)). +-- 3. Backfills the ~1671 pre-existing rows by deriving the distinct +-- 2-digit CPV division from every element of cpvCodes, mirroring +-- cpv/cpv-catalog.ts's normalizeCpv()+divisionOf() logic exactly +-- (strip non-digit characters, take the leading 2 digits). +-- +-- Idempotent: the backfill UPDATE only touches rows whose cpvDivisions is +-- still the column default ('{}') — safe to run more than once, and never +-- overwrites a value already set by the (now-fixed) normalizer for newer +-- ingests. No user input is interpolated (T-11-06/07) — pure derived-field +-- UPDATE over stored data. + +-- AlterTable +ALTER TABLE "Tender" ADD COLUMN IF NOT EXISTS "cpvDivisions" TEXT[] NOT NULL DEFAULT '{}'; + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "Tender_cpvDivisions_idx" ON "Tender" USING GIN ("cpvDivisions"); + +-- Backfill: distinct leading-2-digit division per row, derived from every +-- element of cpvCodes (implicit LATERAL unnest — a set-returning function +-- in FROM may refer to a preceding FROM item without the LATERAL keyword). +UPDATE "Tender" t +SET "cpvDivisions" = sub.divisions +FROM ( + SELECT "Tender".id AS id, + array_agg(DISTINCT left(regexp_replace(code, '\D', '', 'g'), 2)) AS divisions + FROM "Tender", unnest("Tender"."cpvCodes") AS code + WHERE length(regexp_replace(code, '\D', '', 'g')) > 0 + GROUP BY "Tender".id +) AS sub +WHERE t.id = sub.id + AND (t."cpvDivisions" IS NULL OR t."cpvDivisions" = '{}'); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 70549da..f02e441 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -269,6 +269,7 @@ model Tender { title String buyerName String? cpvCodes String[] @default([]) + cpvDivisions String[] @default([]) // FILTER-03: normalized 2-digit CPV divisions (hasSome-filterable, Pitfall 2) region String? plz String? bundesland String? @@ -287,6 +288,7 @@ model Tender { @@index([deadlineAt]) @@index([publishedAt]) @@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance + @@index([cpvDivisions], type: Gin) // FILTER-03: post-backfill CPV-Divisions-Filter-Performance (hasSome) // Deliberately NO tenant column and NO tenant index — this is global data (D-03) } diff --git a/apps/api/src/tenders/tender-ingestion.service.ts b/apps/api/src/tenders/tender-ingestion.service.ts index bbf8611..fce97bc 100644 --- a/apps/api/src/tenders/tender-ingestion.service.ts +++ b/apps/api/src/tenders/tender-ingestion.service.ts @@ -104,6 +104,7 @@ export class TenderIngestionService { title: tender.title, buyerName: tender.buyerName, cpvCodes: tender.cpvCodes, + cpvDivisions: tender.cpvDivisions, region: tender.region, plz: tender.plz, bundesland: tender.bundesland, @@ -121,6 +122,7 @@ export class TenderIngestionService { title: tender.title, buyerName: tender.buyerName, cpvCodes: tender.cpvCodes, + cpvDivisions: tender.cpvDivisions, region: tender.region, plz: tender.plz, bundesland: tender.bundesland, diff --git a/apps/api/src/tenders/tender-normalizer.service.spec.ts b/apps/api/src/tenders/tender-normalizer.service.spec.ts index c853d87..99acd93 100644 --- a/apps/api/src/tenders/tender-normalizer.service.spec.ts +++ b/apps/api/src/tenders/tender-normalizer.service.spec.ts @@ -120,6 +120,42 @@ describe('TenderNormalizerService', () => { expect(normalized.bundesland).toBeNull(); }); + it('derives distinct cpvDivisions (leading 2 digits) from cpvCodes (FILTER-03, Pitfall 2)', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + + const normalized = service.normalize(raw); + + expect(Array.isArray(normalized.cpvDivisions)).toBe(true); + for (const division of normalized.cpvDivisions) { + expect(division).toMatch(/^\d{2}$/); + } + // Every cpvCodes entry must normalize into one of the derived divisions. + for (const code of normalized.cpvCodes) { + const digits = code.replace(/\D/g, ''); + if (digits.length >= 2) { + expect(normalized.cpvDivisions).toContain(digits.slice(0, 2)); + } + } + }); + + it('cpvDivisions is an empty array when cpvCodes is empty — never crashes', () => { + const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); + const rawWithoutCpv: RawTenderRecord = { + ...raw, + ocdsPayload: { + ...(raw.ocdsPayload as Record), + tender: { + ...(raw.ocdsPayload as { tender?: Record })?.tender, + items: [], + }, + }, + }; + + const normalized = service.normalize(rawWithoutCpv); + + expect(normalized.cpvDivisions).toEqual([]); + }); + it('dedupKey = ocid when present', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts index da5e579..99d66cd 100644 --- a/apps/api/src/tenders/tender-normalizer.service.ts +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; import { createHash } from 'crypto'; +import { divisionOf } from './cpv/cpv-catalog'; import { bundeslandFromRegion } from './geo/nuts-bundesland'; import type { NormalizedTenderFields, RawTenderRecord } from './tender.types'; @@ -32,6 +33,12 @@ export class TenderNormalizerService { 'Unbenannte Ausschreibung'; const buyerName = textValue(ocdsBuyer?.name); const cpvCodes = extractCpvCodes(ocdsTender); + // Phase 11 (FILTER-03, Pitfall 2): derive distinct CPV divisions from + // the raw, inconsistently-formatted cpvCodes ("45", "45000000", + // "45000000-7") so new ingests are hasSome-filterable immediately — + // the ~1671 pre-existing rows are backfilled by a one-time migration + // (20260721150000_tender_cpv_divisions_backfill). + const cpvDivisions = deriveCpvDivisions(cpvCodes); const region = textValue(ocdsBuyer?.address?.region); const plz = textValue(ocdsBuyer?.address?.postalCode); // Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's @@ -69,6 +76,7 @@ export class TenderNormalizerService { title, buyerName, cpvCodes, + cpvDivisions, region, plz, bundesland, @@ -221,6 +229,18 @@ function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] { return Array.from(new Set(codes)); } +/** + * Distinct CPV division codes (leading 2 digits) derived from the raw + * cpvCodes array via cpv-catalog.ts's divisionOf() — handles mixed formats + * ("45", "45000000", "45000000-7") uniformly (Pitfall 2). Never throws on + * an empty/malformed code; divisionOf('') === '' is filtered out below so + * cpvDivisions never contains an empty-string entry. + */ +function deriveCpvDivisions(cpvCodes: string[]): string[] { + const divisions = cpvCodes.map((code) => divisionOf(code)).filter(Boolean); + return Array.from(new Set(divisions)); +} + function parseOcdsDate(value: unknown): Date | null { if (typeof value !== 'string' || !value) return null; const parsed = new Date(value); diff --git a/apps/api/src/tenders/tender.types.ts b/apps/api/src/tenders/tender.types.ts index a6ae9c2..52025a9 100644 --- a/apps/api/src/tenders/tender.types.ts +++ b/apps/api/src/tenders/tender.types.ts @@ -55,6 +55,15 @@ export interface NormalizedTenderFields { title: string; buyerName: string | null; cpvCodes: string[]; + /** + * Distinct CPV division codes (leading 2 digits) derived from cpvCodes + * via cpv-catalog.ts's normalizeCpv/divisionOf (Phase 11, FILTER-03, + * Pitfall 2). Precomputed at normalization time so the query layer can + * use a typesafe, GIN-indexable `hasSome` match instead of raw-SQL + * per-element prefix matching against the inconsistently-formatted + * cpvCodes array. + */ + cpvDivisions: string[]; /** NUTS region code (e.g. "DEA41") from the buyer address. */ region: string | null; plz: string | null;