feat(11-03): cpvDivisions column — normalizer + backfill migration

Adds Tender.cpvDivisions String[] (@@index Gin) derived at ingestion time
via cpv-catalog.ts's divisionOf() — replaces exact-match cpvCodes
comparison with a typesafe, GIN-indexable hasSome target (FILTER-03,
Pitfall 2). Backfill migration 20260721150000_tender_cpv_divisions_backfill
applied locally: 1612/1671 rows populated across all observed divisions
(741 rows carry division '45' — Bauarbeiten); idempotent (second run:
UPDATE 0, ADD COLUMN IF NOT EXISTS / CREATE INDEX IF NOT EXISTS both skip
cleanly). Applied via docker exec psql + `prisma migrate resolve
--applied` + `prisma generate` against the local dev DB only — no
Docker deploy on the test server.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/api/prisma/migrations/20260721150000_tender_cpv_divisions_backfill/migration.sql

@@ -0,0 +1,41 @@
+-- FILTER-03 (Pitfall 2): Tender.cpvCodes is stored in inconsistent formats
+-- in the live DB (verified sample: "45", "45000000", "45000000-7" all
+-- referring to the same CPV division "45" = Bauarbeiten; 638 distinct raw
+-- values). An exact-equality `cpvCodes has '45000000'` filter would miss
+-- rows stored in a different format. This migration:
+--   1. Adds the `cpvDivisions` String[] column (typesafe, GIN-indexable
+--      hasSome-match target — replaces raw-SQL per-element LIKE matching,
+--      RESEARCH Pattern 3 Option B / Don't Hand-Roll).
+--   2. Adds the matching GIN index (matches schema.prisma
+--      @@index([cpvDivisions], type: Gin)).
+--   3. Backfills the ~1671 pre-existing rows by deriving the distinct
+--      2-digit CPV division from every element of cpvCodes, mirroring
+--      cpv/cpv-catalog.ts's normalizeCpv()+divisionOf() logic exactly
+--      (strip non-digit characters, take the leading 2 digits).
+--
+-- Idempotent: the backfill UPDATE only touches rows whose cpvDivisions is
+-- still the column default ('{}') — safe to run more than once, and never
+-- overwrites a value already set by the (now-fixed) normalizer for newer
+-- ingests. No user input is interpolated (T-11-06/07) — pure derived-field
+-- UPDATE over stored data.
+
+-- AlterTable
+ALTER TABLE "Tender" ADD COLUMN IF NOT EXISTS "cpvDivisions" TEXT[] NOT NULL DEFAULT '{}';
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "Tender_cpvDivisions_idx" ON "Tender" USING GIN ("cpvDivisions");
+
+-- Backfill: distinct leading-2-digit division per row, derived from every
+-- element of cpvCodes (implicit LATERAL unnest — a set-returning function
+-- in FROM may refer to a preceding FROM item without the LATERAL keyword).
+UPDATE "Tender" t
+SET "cpvDivisions" = sub.divisions
+FROM (
+  SELECT "Tender".id AS id,
+         array_agg(DISTINCT left(regexp_replace(code, '\D', '', 'g'), 2)) AS divisions
+  FROM "Tender", unnest("Tender"."cpvCodes") AS code
+  WHERE length(regexp_replace(code, '\D', '', 'g')) > 0
+  GROUP BY "Tender".id
+) AS sub
+WHERE t.id = sub.id
+  AND (t."cpvDivisions" IS NULL OR t."cpvDivisions" = '{}');

apps/api/prisma/schema.prisma

@@ -269,6 +269,7 @@ model Tender {
   title          String
   buyerName      String?
   cpvCodes       String[]  @default([])
+  cpvDivisions   String[]  @default([]) // FILTER-03: normalized 2-digit CPV divisions (hasSome-filterable, Pitfall 2)
   region         String?
   plz            String?
   bundesland     String?
@@ -287,6 +288,7 @@ model Tender {
   @@index([deadlineAt])
   @@index([publishedAt])
   @@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance
+  @@index([cpvDivisions], type: Gin) // FILTER-03: post-backfill CPV-Divisions-Filter-Performance (hasSome)
   // Deliberately NO tenant column and NO tenant index — this is global data (D-03)
 }
 

apps/api/src/tenders/tender-ingestion.service.ts

@@ -104,6 +104,7 @@ export class TenderIngestionService {
               title: tender.title,
               buyerName: tender.buyerName,
               cpvCodes: tender.cpvCodes,
+              cpvDivisions: tender.cpvDivisions,
               region: tender.region,
               plz: tender.plz,
               bundesland: tender.bundesland,
@@ -121,6 +122,7 @@ export class TenderIngestionService {
               title: tender.title,
               buyerName: tender.buyerName,
               cpvCodes: tender.cpvCodes,
+              cpvDivisions: tender.cpvDivisions,
               region: tender.region,
               plz: tender.plz,
               bundesland: tender.bundesland,

apps/api/src/tenders/tender-normalizer.service.spec.ts

@@ -120,6 +120,42 @@ describe('TenderNormalizerService', () => {
     expect(normalized.bundesland).toBeNull();
   });
 
+  it('derives distinct cpvDivisions (leading 2 digits) from cpvCodes (FILTER-03, Pitfall 2)', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
+
+    const normalized = service.normalize(raw);
+
+    expect(Array.isArray(normalized.cpvDivisions)).toBe(true);
+    for (const division of normalized.cpvDivisions) {
+      expect(division).toMatch(/^\d{2}$/);
+    }
+    // Every cpvCodes entry must normalize into one of the derived divisions.
+    for (const code of normalized.cpvCodes) {
+      const digits = code.replace(/\D/g, '');
+      if (digits.length >= 2) {
+        expect(normalized.cpvDivisions).toContain(digits.slice(0, 2));
+      }
+    }
+  });
+
+  it('cpvDivisions is an empty array when cpvCodes is empty — never crashes', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
+    const rawWithoutCpv: RawTenderRecord = {
+      ...raw,
+      ocdsPayload: {
+        ...(raw.ocdsPayload as Record<string, unknown>),
+        tender: {
+          ...(raw.ocdsPayload as { tender?: Record<string, unknown> })?.tender,
+          items: [],
+        },
+      },
+    };
+
+    const normalized = service.normalize(rawWithoutCpv);
+
+    expect(normalized.cpvDivisions).toEqual([]);
+  });
+
   it('dedupKey = ocid when present', () => {
     const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
 

apps/api/src/tenders/tender-normalizer.service.ts

@@ -1,5 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { createHash } from 'crypto';
+import { divisionOf } from './cpv/cpv-catalog';
 import { bundeslandFromRegion } from './geo/nuts-bundesland';
 import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
 
@@ -32,6 +33,12 @@ export class TenderNormalizerService {
       'Unbenannte Ausschreibung';
     const buyerName = textValue(ocdsBuyer?.name);
     const cpvCodes = extractCpvCodes(ocdsTender);
+    // Phase 11 (FILTER-03, Pitfall 2): derive distinct CPV divisions from
+    // the raw, inconsistently-formatted cpvCodes ("45", "45000000",
+    // "45000000-7") so new ingests are hasSome-filterable immediately —
+    // the ~1671 pre-existing rows are backfilled by a one-time migration
+    // (20260721150000_tender_cpv_divisions_backfill).
+    const cpvDivisions = deriveCpvDivisions(cpvCodes);
     const region = textValue(ocdsBuyer?.address?.region);
     const plz = textValue(ocdsBuyer?.address?.postalCode);
     // Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
@@ -69,6 +76,7 @@ export class TenderNormalizerService {
       title,
       buyerName,
       cpvCodes,
+      cpvDivisions,
       region,
       plz,
       bundesland,
@@ -221,6 +229,18 @@ function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] {
   return Array.from(new Set(codes));
 }
 
+/**
+ * Distinct CPV division codes (leading 2 digits) derived from the raw
+ * cpvCodes array via cpv-catalog.ts's divisionOf() — handles mixed formats
+ * ("45", "45000000", "45000000-7") uniformly (Pitfall 2). Never throws on
+ * an empty/malformed code; divisionOf('') === '' is filtered out below so
+ * cpvDivisions never contains an empty-string entry.
+ */
+function deriveCpvDivisions(cpvCodes: string[]): string[] {
+  const divisions = cpvCodes.map((code) => divisionOf(code)).filter(Boolean);
+  return Array.from(new Set(divisions));
+}
+
 function parseOcdsDate(value: unknown): Date | null {
   if (typeof value !== 'string' || !value) return null;
   const parsed = new Date(value);

apps/api/src/tenders/tender.types.ts

@@ -55,6 +55,15 @@ export interface NormalizedTenderFields {
   title: string;
   buyerName: string | null;
   cpvCodes: string[];
+  /**
+   * Distinct CPV division codes (leading 2 digits) derived from cpvCodes
+   * via cpv-catalog.ts's normalizeCpv/divisionOf (Phase 11, FILTER-03,
+   * Pitfall 2). Precomputed at normalization time so the query layer can
+   * use a typesafe, GIN-indexable `hasSome` match instead of raw-SQL
+   * per-element prefix matching against the inconsistently-formatted
+   * cpvCodes array.
+   */
+  cpvDivisions: string[];
   /** NUTS region code (e.g. "DEA41") from the buyer address. */
   region: string | null;
   plz: string | null;