feat(11-03): cpvDivisions column — normalizer + backfill migration
Adds Tender.cpvDivisions String[] (@@index Gin) derived at ingestion time via cpv-catalog.ts's divisionOf() — replaces exact-match cpvCodes comparison with a typesafe, GIN-indexable hasSome target (FILTER-03, Pitfall 2). Backfill migration 20260721150000_tender_cpv_divisions_backfill applied locally: 1612/1671 rows populated across all observed divisions (741 rows carry division '45' — Bauarbeiten); idempotent (second run: UPDATE 0, ADD COLUMN IF NOT EXISTS / CREATE INDEX IF NOT EXISTS both skip cleanly). Applied via docker exec psql + `prisma migrate resolve --applied` + `prisma generate` against the local dev DB only — no Docker deploy on the test server. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
-- FILTER-03 (Pitfall 2): Tender.cpvCodes is stored in inconsistent formats
|
||||
-- in the live DB (verified sample: "45", "45000000", "45000000-7" all
|
||||
-- referring to the same CPV division "45" = Bauarbeiten; 638 distinct raw
|
||||
-- values). An exact-equality `cpvCodes has '45000000'` filter would miss
|
||||
-- rows stored in a different format. This migration:
|
||||
-- 1. Adds the `cpvDivisions` String[] column (typesafe, GIN-indexable
|
||||
-- hasSome-match target — replaces raw-SQL per-element LIKE matching,
|
||||
-- RESEARCH Pattern 3 Option B / Don't Hand-Roll).
|
||||
-- 2. Adds the matching GIN index (matches schema.prisma
|
||||
-- @@index([cpvDivisions], type: Gin)).
|
||||
-- 3. Backfills the ~1671 pre-existing rows by deriving the distinct
|
||||
-- 2-digit CPV division from every element of cpvCodes, mirroring
|
||||
-- cpv/cpv-catalog.ts's normalizeCpv()+divisionOf() logic exactly
|
||||
-- (strip non-digit characters, take the leading 2 digits).
|
||||
--
|
||||
-- Idempotent: the backfill UPDATE only touches rows whose cpvDivisions is
|
||||
-- still the column default ('{}') — safe to run more than once, and never
|
||||
-- overwrites a value already set by the (now-fixed) normalizer for newer
|
||||
-- ingests. No user input is interpolated (T-11-06/07) — pure derived-field
|
||||
-- UPDATE over stored data.
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "Tender" ADD COLUMN IF NOT EXISTS "cpvDivisions" TEXT[] NOT NULL DEFAULT '{}';
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX IF NOT EXISTS "Tender_cpvDivisions_idx" ON "Tender" USING GIN ("cpvDivisions");
|
||||
|
||||
-- Backfill: distinct leading-2-digit division per row, derived from every
|
||||
-- element of cpvCodes (implicit LATERAL unnest — a set-returning function
|
||||
-- in FROM may refer to a preceding FROM item without the LATERAL keyword).
|
||||
UPDATE "Tender" t
|
||||
SET "cpvDivisions" = sub.divisions
|
||||
FROM (
|
||||
SELECT "Tender".id AS id,
|
||||
array_agg(DISTINCT left(regexp_replace(code, '\D', '', 'g'), 2)) AS divisions
|
||||
FROM "Tender", unnest("Tender"."cpvCodes") AS code
|
||||
WHERE length(regexp_replace(code, '\D', '', 'g')) > 0
|
||||
GROUP BY "Tender".id
|
||||
) AS sub
|
||||
WHERE t.id = sub.id
|
||||
AND (t."cpvDivisions" IS NULL OR t."cpvDivisions" = '{}');
|
||||
@@ -269,6 +269,7 @@ model Tender {
|
||||
title String
|
||||
buyerName String?
|
||||
cpvCodes String[] @default([])
|
||||
cpvDivisions String[] @default([]) // FILTER-03: normalized 2-digit CPV divisions (hasSome-filterable, Pitfall 2)
|
||||
region String?
|
||||
plz String?
|
||||
bundesland String?
|
||||
@@ -287,6 +288,7 @@ model Tender {
|
||||
@@index([deadlineAt])
|
||||
@@index([publishedAt])
|
||||
@@index([bundesland]) // FILTER-02: post-backfill Bundesland-Filter-Performance
|
||||
@@index([cpvDivisions], type: Gin) // FILTER-03: post-backfill CPV-Divisions-Filter-Performance (hasSome)
|
||||
// Deliberately NO tenant column and NO tenant index — this is global data (D-03)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user