feat(10-03): implement TenderNormalizerService — fields, dedupKey, hash

- normalize(raw): eForms-DE XML primary for deadlineAt/estimatedValue/
  procedureType (RESEARCH Pattern 3); OCDS primary for ocid/buyerName/
  title/cpvCodes/region/plz
- deadlineAt/estimatedValue nullable by mandate (RESEARCH Pattern 4) —
  missing data normalizes to null, never thrown or zero
- dedupKey priority: ocid -> sourcePortal:sourceNoticeId fallback
- contentHash = sha256(title+deadlineAt+estimatedValue+status), stable
  across repeat calls, changes when the deadline changes (SCHEMA-02 hook)
- Register TenderNormalizerService in TendersModule.providers
- All tender-normalizer.service.spec.ts tests green (6/6); full API
  suite green (59/59); tsc --noEmit clean
This commit is contained in:
2026-07-21 10:58:33 +02:00
parent 3764feb50b
commit 31607df48c
2 changed files with 248 additions and 1 deletions
@@ -0,0 +1,246 @@
import { Injectable } from '@nestjs/common';
import { createHash } from 'crypto';
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
/**
* TenderNormalizerService — maps a RawTenderRecord (paired eForms-DE XML +
* OCDS JSON) into the unified Tender field shape (SCHEMA-01), computing the
* dedupKey and contentHash (SCHEMA-02 change-detection hook).
*
* Pure, no I/O (matches the DkvParserService transform-service shape: one
* public entry point + private helpers).
*
* Field authority (RESEARCH.md Pattern 3 — reverses ARCHITECTURE.md's
* blanket "prefer OCDS" guidance based on a live cross-check):
* - eForms-DE XML is PRIMARY for deadlineAt / estimatedValue / procedureType
* (OCDS conversion measurably drops these for some notices).
* - OCDS is PRIMARY for ocid / buyerName / title / cpvCodes / region / plz.
* - deadlineAt / estimatedValue are nullable by mandate (RESEARCH Pattern 4)
* — null is the common case for Unterschwelle notices, not an edge case.
*/
@Injectable()
export class TenderNormalizerService {
normalize(raw: RawTenderRecord): NormalizedTenderFields {
const eformsRoot = getEformsRoot(raw.eformsPayload);
const ocdsTender = getOcdsTender(raw.ocdsPayload);
const ocdsBuyer = getOcdsBuyer(raw.ocdsPayload);
const title =
textValue(ocdsTender?.title) ??
extractEformsTitle(eformsRoot) ??
'Unbenannte Ausschreibung';
const buyerName = textValue(ocdsBuyer?.name);
const cpvCodes = extractCpvCodes(ocdsTender);
const region = textValue(ocdsBuyer?.address?.region);
const plz = textValue(ocdsBuyer?.address?.postalCode);
// NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not
// required for this phase's ingestion core.
const bundesland: string | null = null;
const deadlineAt =
extractEformsDeadline(eformsRoot) ??
parseOcdsDate(ocdsTender?.tenderPeriod?.endDate);
const estimatedValue =
extractEformsEstimatedValue(eformsRoot) ??
parseOcdsAmount(ocdsTender?.value);
const procedureType =
extractEformsProcedureType(eformsRoot) ??
textValue(ocdsTender?.procurementMethodDetails);
const status = 'active';
const dedupKey = raw.ocid
? raw.ocid
: `${raw.sourcePortal}:${raw.sourceNoticeId}`;
const contentHash = computeContentHash({
title,
deadlineAt,
estimatedValue,
status,
});
return {
sourcePortal: raw.sourcePortal,
sourceNoticeId: raw.sourceNoticeId,
ocid: raw.ocid ?? null,
dedupKey,
title,
buyerName,
cpvCodes,
region,
plz,
bundesland,
deadlineAt,
estimatedValue,
procedureType,
status,
sourceUrl: raw.sourceUrl ?? null,
contentHash,
publishedAt: raw.publishedAt ?? new Date(),
};
}
}
// ─── Private: pure field-extraction helpers (module-level) ────────────────
interface OcdsAddress {
region?: unknown;
postalCode?: unknown;
}
interface OcdsParty {
name?: unknown;
address?: OcdsAddress;
}
interface OcdsClassification {
scheme?: unknown;
id?: unknown;
}
interface OcdsItem {
classification?: OcdsClassification;
}
interface OcdsTender {
title?: unknown;
tenderPeriod?: { endDate?: unknown };
value?: { amount?: unknown };
procurementMethodDetails?: unknown;
items?: OcdsItem[];
}
function getOcdsTender(ocdsPayload: unknown): OcdsTender | undefined {
if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined;
return (ocdsPayload as { tender?: OcdsTender }).tender;
}
function getOcdsBuyer(ocdsPayload: unknown): OcdsParty | undefined {
if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined;
return (ocdsPayload as { buyer?: OcdsParty }).buyer;
}
/** Extract the sole root element's value from a fast-xml-parser document, robust across notice subtypes (ContractNotice, PriorInformationNotice, ...). */
function getEformsRoot(eformsPayload: unknown): Record<string, unknown> {
if (!eformsPayload || typeof eformsPayload !== 'object') return {};
const keys = Object.keys(eformsPayload as Record<string, unknown>);
const rootKey = keys[0];
if (!rootKey) return {};
const root = (eformsPayload as Record<string, unknown>)[rootKey];
return root && typeof root === 'object'
? (root as Record<string, unknown>)
: {};
}
/** fast-xml-parser represents a tag with both attributes and text content as `{ '@_attr': ..., '#text': value }`; a plain tag is just the raw string/number. */
function textValue(node: unknown): string | null {
if (node === null || node === undefined) return null;
if (typeof node === 'string') return node || null;
if (typeof node === 'number') return String(node);
if (typeof node === 'object' && '#text' in (node as Record<string, unknown>)) {
const t = (node as Record<string, unknown>)['#text'];
if (t === null || t === undefined) return null;
return String(t);
}
return null;
}
function firstLot(
root: Record<string, unknown>,
): Record<string, unknown> | null {
const lot = root.ProcurementProjectLot;
if (!lot) return null;
if (Array.isArray(lot)) {
return (lot[0] as Record<string, unknown>) ?? null;
}
return lot as Record<string, unknown>;
}
function extractEformsTitle(root: Record<string, unknown>): string | null {
const project = root.ProcurementProject as
| Record<string, unknown>
| undefined;
return textValue(project?.Name);
}
/**
* Primary structured deadline source (RESEARCH.md Pattern 3): per-lot
* `TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate` (+ optional
* `EndTime`), combined into a full ISO timestamp when both are present.
*/
function extractEformsDeadline(root: Record<string, unknown>): Date | null {
const lot = firstLot(root);
const process = lot?.TenderingProcess as Record<string, unknown> | undefined;
const period = process?.TenderSubmissionDeadlinePeriod as
| Record<string, unknown>
| undefined;
const endDateRaw = textValue(period?.EndDate);
if (!endDateRaw) return null;
const endTimeRaw = textValue(period?.EndTime);
const datePart = endDateRaw.replace(/[+-]\d{2}:\d{2}$/, '');
const combined = endTimeRaw ? `${datePart}T${endTimeRaw}` : datePart;
const parsed = new Date(combined);
return Number.isNaN(parsed.getTime()) ? null : parsed;
}
/** Primary structured value source (RESEARCH.md Pattern 3): root `ProcurementProject/RequestedTenderTotal/EstimatedOverallContractAmount`. */
function extractEformsEstimatedValue(
root: Record<string, unknown>,
): number | null {
const project = root.ProcurementProject as
| Record<string, unknown>
| undefined;
const total = project?.RequestedTenderTotal as
| Record<string, unknown>
| undefined;
const amountRaw = textValue(total?.EstimatedOverallContractAmount);
if (!amountRaw) return null;
const parsed = Number.parseFloat(amountRaw);
return Number.isNaN(parsed) ? null : parsed;
}
/** Primary structured procedure-type source: root `TenderingProcess/ProcedureCode`. */
function extractEformsProcedureType(
root: Record<string, unknown>,
): string | null {
const process = root.TenderingProcess as Record<string, unknown> | undefined;
return textValue(process?.ProcedureCode);
}
function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] {
const items = Array.isArray(ocdsTender?.items) ? ocdsTender.items : [];
const codes = items
.map((item) => item?.classification?.id)
.filter((id): id is string => typeof id === 'string');
return Array.from(new Set(codes));
}
function parseOcdsDate(value: unknown): Date | null {
if (typeof value !== 'string' || !value) return null;
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? null : parsed;
}
function parseOcdsAmount(value: { amount?: unknown } | undefined): number | null {
const amount = value?.amount;
return typeof amount === 'number' ? amount : null;
}
function computeContentHash(fields: {
title: string;
deadlineAt: Date | null;
estimatedValue: number | null;
status: string;
}): string {
const raw = [
fields.title,
fields.deadlineAt ? fields.deadlineAt.toISOString() : '',
fields.estimatedValue !== null ? String(fields.estimatedValue) : '',
fields.status,
].join('|');
return createHash('sha256').update(raw).digest('hex');
}
+2 -1
View File
@@ -4,6 +4,7 @@ import { ModuleRegistryService } from '../module-registry/module-registry.servic
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter'; import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter';
import { seedTendersModule } from './tenders.seed'; import { seedTendersModule } from './tenders.seed';
import { TenderNormalizerService } from './tender-normalizer.service';
/** /**
* NestJS module for the Ausschreibungs-Radar feature. * NestJS module for the Ausschreibungs-Radar feature.
@@ -22,7 +23,7 @@ import { seedTendersModule } from './tenders.seed';
@Module({ @Module({
imports: [ModuleRegistryModule], imports: [ModuleRegistryModule],
controllers: [], controllers: [],
providers: [DoeOpenDataAdapter], providers: [DoeOpenDataAdapter, TenderNormalizerService],
}) })
export class TendersModule implements OnModuleInit { export class TendersModule implements OnModuleInit {
private readonly logger = new Logger(TendersModule.name); private readonly logger = new Logger(TendersModule.name);