import { Injectable } from '@nestjs/common'; import { createHash } from 'crypto'; import { divisionOf } from './cpv/cpv-catalog'; import { bundeslandFromRegion } from './geo/nuts-bundesland'; import type { NormalizedTenderFields, RawTenderRecord } from './tender.types'; /** * TenderNormalizerService — maps a RawTenderRecord into the unified Tender * field shape (SCHEMA-01), computing the dedupKey and contentHash * (SCHEMA-02 change-detection hook). * * Pure, no I/O (matches the DkvParserService transform-service shape: one * public entry point + private helpers). * * Per-sourceType dispatch (gap closure, 13-VERIFICATION.md): the DÖE source * pairs eForms-DE XML + OCDS JSON (see normalizeDoe field-authority notes * below); the NetServer/cosinex-DTVP scraper adapters instead carry a flat * `ocdsPayload` bag (see normalizeBag) with no eForms/OCDS structure at all. * Both paths converge on the shared `assemble()` tail so status/dedupKey/ * contentHash/publishedAt are computed identically for every source. * * Field authority for the DÖE path (RESEARCH.md Pattern 3 — reverses * ARCHITECTURE.md's blanket "prefer OCDS" guidance based on a live * cross-check): * - eForms-DE XML is PRIMARY for deadlineAt / estimatedValue / procedureType * (OCDS conversion measurably drops these for some notices). * - OCDS is PRIMARY for ocid / buyerName / title / cpvCodes / region / plz. * - deadlineAt / estimatedValue are nullable by mandate (RESEARCH Pattern 4) * — null is the common case for Unterschwelle notices, not an edge case. */ /** * The 10 source-specific fields each per-sourceType extraction path must * produce; everything else (sourcePortal, sourceNoticeId, ocid, dedupKey, * status, sourceUrl, contentHash, publishedAt, fingerprint) is assembled * identically for all sources by `assemble()`. */ type CoreTenderFields = Omit< NormalizedTenderFields, | 'sourcePortal' | 'sourceNoticeId' | 'ocid' | 'dedupKey' | 'status' | 'sourceUrl' | 'contentHash' | 'publishedAt' | 'fingerprint' >; @Injectable() export class TenderNormalizerService { normalize(raw: RawTenderRecord): NormalizedTenderFields { switch (raw.sourceType) { case 'ai-netserver': case 'cosinex-dtvp': case 'rss': case 'email-alert': return this.normalizeBag(raw); case 'doe-opendata': default: // default MUST stay on the DÖE path (never throw) so future // additive SourceType union members don't break existing callers. return this.normalizeDoe(raw); } } /** DÖE path: paired eForms-DE XML + OCDS JSON. Unchanged behavior (regression guard). */ private normalizeDoe(raw: RawTenderRecord): NormalizedTenderFields { const eformsRoot = getEformsRoot(raw.eformsPayload); const ocdsTender = getOcdsTender(raw.ocdsPayload); const ocdsBuyer = getOcdsBuyer(raw.ocdsPayload); const title = textValue(ocdsTender?.title) ?? extractEformsTitle(eformsRoot) ?? 'Unbenannte Ausschreibung'; const buyerName = textValue(ocdsBuyer?.name); const cpvCodes = extractCpvCodes(ocdsTender); // Phase 11 (FILTER-03, Pitfall 2): derive distinct CPV divisions from // the raw, inconsistently-formatted cpvCodes ("45", "45000000", // "45000000-7") so new ingests are hasSome-filterable immediately — // the ~1671 pre-existing rows are backfilled by a one-time migration // (20260721150000_tender_cpv_divisions_backfill). const cpvDivisions = deriveCpvDivisions(cpvCodes); const region = textValue(ocdsBuyer?.address?.region); const plz = textValue(ocdsBuyer?.address?.postalCode); // Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's // NUTS-1 prefix so new ingests are filterable immediately — the // ~1671 pre-existing rows are backfilled by a one-time migration // (20260721140000_tender_bundesland_backfill). const bundesland = bundeslandFromRegion(region); const deadlineAt = extractEformsDeadline(eformsRoot) ?? parseOcdsDate(ocdsTender?.tenderPeriod?.endDate); const estimatedValue = extractEformsEstimatedValue(eformsRoot) ?? parseOcdsAmount(ocdsTender?.value); const procedureType = extractEformsProcedureType(eformsRoot) ?? textValue(ocdsTender?.procurementMethodDetails); const core: CoreTenderFields = { title, buyerName, cpvCodes, cpvDivisions, region, plz, bundesland, deadlineAt, estimatedValue, procedureType, }; return this.assemble(raw, core); } /** * Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters * (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04; and since * Phase 14 Plan 03, the EmailAlertAdapter, INGEST-05) carry a flat * `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in * `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect. * `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has * no target field for it. RSS/email-alert records always have * `buyerName`/`procedureType`/`deadlineAt` null (baseline title/link/guid * mapping only, D-04/D-05 thin-fields deferral). */ private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields { const bag = getBagPayload(raw.ocdsPayload); const title = textValue(bag.title) || 'Unbenannte Ausschreibung'; const buyerName = textValue(bag.buyerName); const procedureType = textValue(bag.procedureType); const deadlineAt = parseOcdsDate(bag.deadlineAt); const core: CoreTenderFields = { title, buyerName, cpvCodes: [], cpvDivisions: [], region: null, plz: null, bundesland: null, deadlineAt, estimatedValue: null, procedureType, }; return this.assemble(raw, core); } /** * Shared tail: computes the source-invariant fields (status, dedupKey, * contentHash, publishedAt) plus the pass-through identity fields * (sourcePortal, sourceNoticeId, ocid, sourceUrl) identically regardless * of which extraction path produced `core`. */ private assemble( raw: RawTenderRecord, core: CoreTenderFields, ): NormalizedTenderFields { const status = 'active'; const dedupKey = raw.ocid ? raw.ocid : `${raw.sourcePortal}:${raw.sourceNoticeId}`; const contentHash = computeContentHash({ title: core.title, deadlineAt: core.deadlineAt, estimatedValue: core.estimatedValue, status, }); return { sourcePortal: raw.sourcePortal, sourceNoticeId: raw.sourceNoticeId, ocid: raw.ocid ?? null, dedupKey, ...core, status, sourceUrl: raw.sourceUrl ?? null, contentHash, publishedAt: raw.publishedAt ?? new Date(), // D-13: passed through unchanged — undefined for every source except // EmailAlertAdapter, which sets it to the configuring tenant's id. ownerTenantId: raw.ownerTenantId, }; } } // ─── Private: pure field-extraction helpers (module-level) ──────────────── interface OcdsAddress { region?: unknown; postalCode?: unknown; } interface OcdsParty { name?: unknown; address?: OcdsAddress; } interface OcdsClassification { scheme?: unknown; id?: unknown; } interface OcdsItem { classification?: OcdsClassification; } interface OcdsTender { title?: unknown; tenderPeriod?: { endDate?: unknown }; value?: { amount?: unknown }; procurementMethodDetails?: unknown; items?: OcdsItem[]; } function getOcdsTender(ocdsPayload: unknown): OcdsTender | undefined { if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined; return (ocdsPayload as { tender?: OcdsTender }).tender; } function getOcdsBuyer(ocdsPayload: unknown): OcdsParty | undefined { if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined; return (ocdsPayload as { buyer?: OcdsParty }).buyer; } /** Extract the sole root element's value from a fast-xml-parser document, robust across notice subtypes (ContractNotice, PriorInformationNotice, ...). */ function getEformsRoot(eformsPayload: unknown): Record { if (!eformsPayload || typeof eformsPayload !== 'object') return {}; const keys = Object.keys(eformsPayload as Record); const rootKey = keys[0]; if (!rootKey) return {}; const root = (eformsPayload as Record)[rootKey]; return root && typeof root === 'object' ? (root as Record) : {}; } /** * The flat scraper-adapter bag shape (NetServer/cosinex-DTVP): read * defensively — the field is `unknown` at the type level and adapters are * out of this plan's scope to change. */ interface OcdsBag { title?: unknown; buyerName?: unknown; procedureType?: unknown; legalFramework?: unknown; deadlineAt?: unknown; } function getBagPayload(ocdsPayload: unknown): OcdsBag { if (!ocdsPayload || typeof ocdsPayload !== 'object') return {}; return ocdsPayload as OcdsBag; } /** fast-xml-parser represents a tag with both attributes and text content as `{ '@_attr': ..., '#text': value }`; a plain tag is just the raw string/number. */ function textValue(node: unknown): string | null { if (node === null || node === undefined) return null; if (typeof node === 'string') return node || null; if (typeof node === 'number') return String(node); if (typeof node === 'object' && '#text' in (node as Record)) { const t = (node as Record)['#text']; if (t === null || t === undefined) return null; return String(t); } return null; } function firstLot( root: Record, ): Record | null { const lot = root.ProcurementProjectLot; if (!lot) return null; if (Array.isArray(lot)) { return (lot[0] as Record) ?? null; } return lot as Record; } function extractEformsTitle(root: Record): string | null { const project = root.ProcurementProject as | Record | undefined; return textValue(project?.Name); } /** * Primary structured deadline source (RESEARCH.md Pattern 3): per-lot * `TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate` (+ optional * `EndTime`), combined into a full ISO timestamp when both are present. */ function extractEformsDeadline(root: Record): Date | null { const lot = firstLot(root); const process = lot?.TenderingProcess as Record | undefined; const period = process?.TenderSubmissionDeadlinePeriod as | Record | undefined; const endDateRaw = textValue(period?.EndDate); if (!endDateRaw) return null; const endTimeRaw = textValue(period?.EndTime); const datePart = endDateRaw.replace(/[+-]\d{2}:\d{2}$/, ''); const combined = endTimeRaw ? `${datePart}T${endTimeRaw}` : datePart; const parsed = new Date(combined); return Number.isNaN(parsed.getTime()) ? null : parsed; } /** Primary structured value source (RESEARCH.md Pattern 3): root `ProcurementProject/RequestedTenderTotal/EstimatedOverallContractAmount`. */ function extractEformsEstimatedValue( root: Record, ): number | null { const project = root.ProcurementProject as | Record | undefined; const total = project?.RequestedTenderTotal as | Record | undefined; const amountRaw = textValue(total?.EstimatedOverallContractAmount); if (!amountRaw) return null; const parsed = Number.parseFloat(amountRaw); return Number.isNaN(parsed) ? null : parsed; } /** Primary structured procedure-type source: root `TenderingProcess/ProcedureCode`. */ function extractEformsProcedureType( root: Record, ): string | null { const process = root.TenderingProcess as Record | undefined; return textValue(process?.ProcedureCode); } function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] { const items = Array.isArray(ocdsTender?.items) ? ocdsTender.items : []; const codes = items .map((item) => item?.classification?.id) .filter((id): id is string => typeof id === 'string'); return Array.from(new Set(codes)); } /** * Distinct CPV division codes (leading 2 digits) derived from the raw * cpvCodes array via cpv-catalog.ts's divisionOf() — handles mixed formats * ("45", "45000000", "45000000-7") uniformly (Pitfall 2). Never throws on * an empty/malformed code; divisionOf('') === '' is filtered out below so * cpvDivisions never contains an empty-string entry. */ function deriveCpvDivisions(cpvCodes: string[]): string[] { const divisions = cpvCodes.map((code) => divisionOf(code)).filter(Boolean); return Array.from(new Set(divisions)); } function parseOcdsDate(value: unknown): Date | null { if (typeof value !== 'string' || !value) return null; const parsed = new Date(value); return Number.isNaN(parsed.getTime()) ? null : parsed; } function parseOcdsAmount(value: { amount?: unknown } | undefined): number | null { const amount = value?.amount; return typeof amount === 'number' ? amount : null; } function computeContentHash(fields: { title: string; deadlineAt: Date | null; estimatedValue: number | null; status: string; }): string { const raw = [ fields.title, fields.deadlineAt ? fields.deadlineAt.toISOString() : '', fields.estimatedValue !== null ? String(fields.estimatedValue) : '', fields.status, ].join('|'); return createHash('sha256').update(raw).digest('hex'); }