From 82c9a483058335131bc105cc810a1e53737310b2 Mon Sep 17 00:00:00 2001 From: Schalli Date: Thu, 23 Jul 2026 10:18:16 +0200 Subject: [PATCH] feat(quick-260723-e7i): per-sourceType dispatch in TenderNormalizerService - Extract shared assemble() tail (status/dedupKey/contentHash/publishedAt) so it is computed identically across all sources, not duplicated - Move existing DOE eForms/OCDS extraction into normalizeDoe() (byte-identical behavior, regression guard) - Add normalizeBag() for the flat ocdsPayload bag shared by the NetServer and cosinex/DTVP scraper adapters ({title, buyerName, procedureType, legalFramework, deadlineAt}); legalFramework deliberately not mapped - normalize() dispatches on raw.sourceType, defaulting to the DOE path so the additive SourceType union never throws Co-Authored-By: Claude Opus 4.8 --- .../src/tenders/tender-normalizer.service.ts | 150 +++++++++++++++--- 1 file changed, 129 insertions(+), 21 deletions(-) diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts index 99d66cd..55777f7 100644 --- a/apps/api/src/tenders/tender-normalizer.service.ts +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -5,24 +5,66 @@ import { bundeslandFromRegion } from './geo/nuts-bundesland'; import type { NormalizedTenderFields, RawTenderRecord } from './tender.types'; /** - * TenderNormalizerService — maps a RawTenderRecord (paired eForms-DE XML + - * OCDS JSON) into the unified Tender field shape (SCHEMA-01), computing the - * dedupKey and contentHash (SCHEMA-02 change-detection hook). + * TenderNormalizerService — maps a RawTenderRecord into the unified Tender + * field shape (SCHEMA-01), computing the dedupKey and contentHash + * (SCHEMA-02 change-detection hook). * * Pure, no I/O (matches the DkvParserService transform-service shape: one * public entry point + private helpers). * - * Field authority (RESEARCH.md Pattern 3 — reverses ARCHITECTURE.md's - * blanket "prefer OCDS" guidance based on a live cross-check): + * Per-sourceType dispatch (gap closure, 13-VERIFICATION.md): the DÖE source + * pairs eForms-DE XML + OCDS JSON (see normalizeDoe field-authority notes + * below); the NetServer/cosinex-DTVP scraper adapters instead carry a flat + * `ocdsPayload` bag (see normalizeBag) with no eForms/OCDS structure at all. + * Both paths converge on the shared `assemble()` tail so status/dedupKey/ + * contentHash/publishedAt are computed identically for every source. + * + * Field authority for the DÖE path (RESEARCH.md Pattern 3 — reverses + * ARCHITECTURE.md's blanket "prefer OCDS" guidance based on a live + * cross-check): * - eForms-DE XML is PRIMARY for deadlineAt / estimatedValue / procedureType * (OCDS conversion measurably drops these for some notices). * - OCDS is PRIMARY for ocid / buyerName / title / cpvCodes / region / plz. * - deadlineAt / estimatedValue are nullable by mandate (RESEARCH Pattern 4) * — null is the common case for Unterschwelle notices, not an edge case. */ + +/** + * The 10 source-specific fields each per-sourceType extraction path must + * produce; everything else (sourcePortal, sourceNoticeId, ocid, dedupKey, + * status, sourceUrl, contentHash, publishedAt, fingerprint) is assembled + * identically for all sources by `assemble()`. + */ +type CoreTenderFields = Omit< + NormalizedTenderFields, + | 'sourcePortal' + | 'sourceNoticeId' + | 'ocid' + | 'dedupKey' + | 'status' + | 'sourceUrl' + | 'contentHash' + | 'publishedAt' + | 'fingerprint' +>; + @Injectable() export class TenderNormalizerService { normalize(raw: RawTenderRecord): NormalizedTenderFields { + switch (raw.sourceType) { + case 'ai-netserver': + case 'cosinex-dtvp': + return this.normalizeBag(raw); + case 'doe-opendata': + default: + // default MUST stay on the DÖE path (never throw) so future + // additive SourceType union members don't break existing callers. + return this.normalizeDoe(raw); + } + } + + /** DÖE path: paired eForms-DE XML + OCDS JSON. Unchanged behavior (regression guard). */ + private normalizeDoe(raw: RawTenderRecord): NormalizedTenderFields { const eformsRoot = getEformsRoot(raw.eformsPayload); const ocdsTender = getOcdsTender(raw.ocdsPayload); const ocdsBuyer = getOcdsBuyer(raw.ocdsPayload); @@ -57,22 +99,7 @@ export class TenderNormalizerService { extractEformsProcedureType(eformsRoot) ?? textValue(ocdsTender?.procurementMethodDetails); - const status = 'active'; - const dedupKey = raw.ocid - ? raw.ocid - : `${raw.sourcePortal}:${raw.sourceNoticeId}`; - const contentHash = computeContentHash({ - title, - deadlineAt, - estimatedValue, - status, - }); - - return { - sourcePortal: raw.sourcePortal, - sourceNoticeId: raw.sourceNoticeId, - ocid: raw.ocid ?? null, - dedupKey, + const core: CoreTenderFields = { title, buyerName, cpvCodes, @@ -83,6 +110,69 @@ export class TenderNormalizerService { deadlineAt, estimatedValue, procedureType, + }; + + return this.assemble(raw, core); + } + + /** + * Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters + * carry a flat `{title, buyerName, procedureType, legalFramework, + * deadlineAt}` bag in `raw.ocdsPayload` — there is no eForms/OCDS + * structure to inspect. `legalFramework` is deliberately NOT mapped: + * NormalizedTenderFields has no target field for it. + */ + private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields { + const bag = getBagPayload(raw.ocdsPayload); + + const title = textValue(bag.title) || 'Unbenannte Ausschreibung'; + const buyerName = textValue(bag.buyerName); + const procedureType = textValue(bag.procedureType); + const deadlineAt = parseOcdsDate(bag.deadlineAt); + + const core: CoreTenderFields = { + title, + buyerName, + cpvCodes: [], + cpvDivisions: [], + region: null, + plz: null, + bundesland: null, + deadlineAt, + estimatedValue: null, + procedureType, + }; + + return this.assemble(raw, core); + } + + /** + * Shared tail: computes the source-invariant fields (status, dedupKey, + * contentHash, publishedAt) plus the pass-through identity fields + * (sourcePortal, sourceNoticeId, ocid, sourceUrl) identically regardless + * of which extraction path produced `core`. + */ + private assemble( + raw: RawTenderRecord, + core: CoreTenderFields, + ): NormalizedTenderFields { + const status = 'active'; + const dedupKey = raw.ocid + ? raw.ocid + : `${raw.sourcePortal}:${raw.sourceNoticeId}`; + const contentHash = computeContentHash({ + title: core.title, + deadlineAt: core.deadlineAt, + estimatedValue: core.estimatedValue, + status, + }); + + return { + sourcePortal: raw.sourcePortal, + sourceNoticeId: raw.sourceNoticeId, + ocid: raw.ocid ?? null, + dedupKey, + ...core, status, sourceUrl: raw.sourceUrl ?? null, contentHash, @@ -142,6 +232,24 @@ function getEformsRoot(eformsPayload: unknown): Record { : {}; } +/** + * The flat scraper-adapter bag shape (NetServer/cosinex-DTVP): read + * defensively — the field is `unknown` at the type level and adapters are + * out of this plan's scope to change. + */ +interface OcdsBag { + title?: unknown; + buyerName?: unknown; + procedureType?: unknown; + legalFramework?: unknown; + deadlineAt?: unknown; +} + +function getBagPayload(ocdsPayload: unknown): OcdsBag { + if (!ocdsPayload || typeof ocdsPayload !== 'object') return {}; + return ocdsPayload as OcdsBag; +} + /** fast-xml-parser represents a tag with both attributes and text content as `{ '@_attr': ..., '#text': value }`; a plain tag is just the raw string/number. */ function textValue(node: unknown): string | null { if (node === null || node === undefined) return null;