From 31607df48c74f0575b4b77b4c20a82e6391546b8 Mon Sep 17 00:00:00 2001 From: Schalli Date: Tue, 21 Jul 2026 10:58:33 +0200 Subject: [PATCH] =?UTF-8?q?feat(10-03):=20implement=20TenderNormalizerServ?= =?UTF-8?q?ice=20=E2=80=94=20fields,=20dedupKey,=20hash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - normalize(raw): eForms-DE XML primary for deadlineAt/estimatedValue/ procedureType (RESEARCH Pattern 3); OCDS primary for ocid/buyerName/ title/cpvCodes/region/plz - deadlineAt/estimatedValue nullable by mandate (RESEARCH Pattern 4) — missing data normalizes to null, never thrown or zero - dedupKey priority: ocid -> sourcePortal:sourceNoticeId fallback - contentHash = sha256(title+deadlineAt+estimatedValue+status), stable across repeat calls, changes when the deadline changes (SCHEMA-02 hook) - Register TenderNormalizerService in TendersModule.providers - All tender-normalizer.service.spec.ts tests green (6/6); full API suite green (59/59); tsc --noEmit clean --- .../src/tenders/tender-normalizer.service.ts | 246 ++++++++++++++++++ apps/api/src/tenders/tenders.module.ts | 3 +- 2 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/tenders/tender-normalizer.service.ts diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts new file mode 100644 index 0000000..51c1cfe --- /dev/null +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -0,0 +1,246 @@ +import { Injectable } from '@nestjs/common'; +import { createHash } from 'crypto'; +import type { NormalizedTenderFields, RawTenderRecord } from './tender.types'; + +/** + * TenderNormalizerService — maps a RawTenderRecord (paired eForms-DE XML + + * OCDS JSON) into the unified Tender field shape (SCHEMA-01), computing the + * dedupKey and contentHash (SCHEMA-02 change-detection hook). + * + * Pure, no I/O (matches the DkvParserService transform-service shape: one + * public entry point + private helpers). + * + * Field authority (RESEARCH.md Pattern 3 — reverses ARCHITECTURE.md's + * blanket "prefer OCDS" guidance based on a live cross-check): + * - eForms-DE XML is PRIMARY for deadlineAt / estimatedValue / procedureType + * (OCDS conversion measurably drops these for some notices). + * - OCDS is PRIMARY for ocid / buyerName / title / cpvCodes / region / plz. + * - deadlineAt / estimatedValue are nullable by mandate (RESEARCH Pattern 4) + * — null is the common case for Unterschwelle notices, not an edge case. + */ +@Injectable() +export class TenderNormalizerService { + normalize(raw: RawTenderRecord): NormalizedTenderFields { + const eformsRoot = getEformsRoot(raw.eformsPayload); + const ocdsTender = getOcdsTender(raw.ocdsPayload); + const ocdsBuyer = getOcdsBuyer(raw.ocdsPayload); + + const title = + textValue(ocdsTender?.title) ?? + extractEformsTitle(eformsRoot) ?? + 'Unbenannte Ausschreibung'; + const buyerName = textValue(ocdsBuyer?.name); + const cpvCodes = extractCpvCodes(ocdsTender); + const region = textValue(ocdsBuyer?.address?.region); + const plz = textValue(ocdsBuyer?.address?.postalCode); + // NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not + // required for this phase's ingestion core. + const bundesland: string | null = null; + + const deadlineAt = + extractEformsDeadline(eformsRoot) ?? + parseOcdsDate(ocdsTender?.tenderPeriod?.endDate); + const estimatedValue = + extractEformsEstimatedValue(eformsRoot) ?? + parseOcdsAmount(ocdsTender?.value); + const procedureType = + extractEformsProcedureType(eformsRoot) ?? + textValue(ocdsTender?.procurementMethodDetails); + + const status = 'active'; + const dedupKey = raw.ocid + ? raw.ocid + : `${raw.sourcePortal}:${raw.sourceNoticeId}`; + const contentHash = computeContentHash({ + title, + deadlineAt, + estimatedValue, + status, + }); + + return { + sourcePortal: raw.sourcePortal, + sourceNoticeId: raw.sourceNoticeId, + ocid: raw.ocid ?? null, + dedupKey, + title, + buyerName, + cpvCodes, + region, + plz, + bundesland, + deadlineAt, + estimatedValue, + procedureType, + status, + sourceUrl: raw.sourceUrl ?? null, + contentHash, + publishedAt: raw.publishedAt ?? new Date(), + }; + } +} + +// ─── Private: pure field-extraction helpers (module-level) ──────────────── + +interface OcdsAddress { + region?: unknown; + postalCode?: unknown; +} + +interface OcdsParty { + name?: unknown; + address?: OcdsAddress; +} + +interface OcdsClassification { + scheme?: unknown; + id?: unknown; +} + +interface OcdsItem { + classification?: OcdsClassification; +} + +interface OcdsTender { + title?: unknown; + tenderPeriod?: { endDate?: unknown }; + value?: { amount?: unknown }; + procurementMethodDetails?: unknown; + items?: OcdsItem[]; +} + +function getOcdsTender(ocdsPayload: unknown): OcdsTender | undefined { + if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined; + return (ocdsPayload as { tender?: OcdsTender }).tender; +} + +function getOcdsBuyer(ocdsPayload: unknown): OcdsParty | undefined { + if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined; + return (ocdsPayload as { buyer?: OcdsParty }).buyer; +} + +/** Extract the sole root element's value from a fast-xml-parser document, robust across notice subtypes (ContractNotice, PriorInformationNotice, ...). */ +function getEformsRoot(eformsPayload: unknown): Record { + if (!eformsPayload || typeof eformsPayload !== 'object') return {}; + const keys = Object.keys(eformsPayload as Record); + const rootKey = keys[0]; + if (!rootKey) return {}; + const root = (eformsPayload as Record)[rootKey]; + return root && typeof root === 'object' + ? (root as Record) + : {}; +} + +/** fast-xml-parser represents a tag with both attributes and text content as `{ '@_attr': ..., '#text': value }`; a plain tag is just the raw string/number. */ +function textValue(node: unknown): string | null { + if (node === null || node === undefined) return null; + if (typeof node === 'string') return node || null; + if (typeof node === 'number') return String(node); + if (typeof node === 'object' && '#text' in (node as Record)) { + const t = (node as Record)['#text']; + if (t === null || t === undefined) return null; + return String(t); + } + return null; +} + +function firstLot( + root: Record, +): Record | null { + const lot = root.ProcurementProjectLot; + if (!lot) return null; + if (Array.isArray(lot)) { + return (lot[0] as Record) ?? null; + } + return lot as Record; +} + +function extractEformsTitle(root: Record): string | null { + const project = root.ProcurementProject as + | Record + | undefined; + return textValue(project?.Name); +} + +/** + * Primary structured deadline source (RESEARCH.md Pattern 3): per-lot + * `TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate` (+ optional + * `EndTime`), combined into a full ISO timestamp when both are present. + */ +function extractEformsDeadline(root: Record): Date | null { + const lot = firstLot(root); + const process = lot?.TenderingProcess as Record | undefined; + const period = process?.TenderSubmissionDeadlinePeriod as + | Record + | undefined; + + const endDateRaw = textValue(period?.EndDate); + if (!endDateRaw) return null; + + const endTimeRaw = textValue(period?.EndTime); + const datePart = endDateRaw.replace(/[+-]\d{2}:\d{2}$/, ''); + const combined = endTimeRaw ? `${datePart}T${endTimeRaw}` : datePart; + + const parsed = new Date(combined); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +/** Primary structured value source (RESEARCH.md Pattern 3): root `ProcurementProject/RequestedTenderTotal/EstimatedOverallContractAmount`. */ +function extractEformsEstimatedValue( + root: Record, +): number | null { + const project = root.ProcurementProject as + | Record + | undefined; + const total = project?.RequestedTenderTotal as + | Record + | undefined; + const amountRaw = textValue(total?.EstimatedOverallContractAmount); + if (!amountRaw) return null; + + const parsed = Number.parseFloat(amountRaw); + return Number.isNaN(parsed) ? null : parsed; +} + +/** Primary structured procedure-type source: root `TenderingProcess/ProcedureCode`. */ +function extractEformsProcedureType( + root: Record, +): string | null { + const process = root.TenderingProcess as Record | undefined; + return textValue(process?.ProcedureCode); +} + +function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] { + const items = Array.isArray(ocdsTender?.items) ? ocdsTender.items : []; + const codes = items + .map((item) => item?.classification?.id) + .filter((id): id is string => typeof id === 'string'); + return Array.from(new Set(codes)); +} + +function parseOcdsDate(value: unknown): Date | null { + if (typeof value !== 'string' || !value) return null; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function parseOcdsAmount(value: { amount?: unknown } | undefined): number | null { + const amount = value?.amount; + return typeof amount === 'number' ? amount : null; +} + +function computeContentHash(fields: { + title: string; + deadlineAt: Date | null; + estimatedValue: number | null; + status: string; +}): string { + const raw = [ + fields.title, + fields.deadlineAt ? fields.deadlineAt.toISOString() : '', + fields.estimatedValue !== null ? String(fields.estimatedValue) : '', + fields.status, + ].join('|'); + + return createHash('sha256').update(raw).digest('hex'); +} diff --git a/apps/api/src/tenders/tenders.module.ts b/apps/api/src/tenders/tenders.module.ts index 53a115a..bff0ead 100644 --- a/apps/api/src/tenders/tenders.module.ts +++ b/apps/api/src/tenders/tenders.module.ts @@ -4,6 +4,7 @@ import { ModuleRegistryService } from '../module-registry/module-registry.servic import { PrismaService } from '../prisma/prisma.service'; import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter'; import { seedTendersModule } from './tenders.seed'; +import { TenderNormalizerService } from './tender-normalizer.service'; /** * NestJS module for the Ausschreibungs-Radar feature. @@ -22,7 +23,7 @@ import { seedTendersModule } from './tenders.seed'; @Module({ imports: [ModuleRegistryModule], controllers: [], - providers: [DoeOpenDataAdapter], + providers: [DoeOpenDataAdapter, TenderNormalizerService], }) export class TendersModule implements OnModuleInit { private readonly logger = new Logger(TendersModule.name);