feat(10-03): implement TenderNormalizerService — fields, dedupKey, hash
- normalize(raw): eForms-DE XML primary for deadlineAt/estimatedValue/ procedureType (RESEARCH Pattern 3); OCDS primary for ocid/buyerName/ title/cpvCodes/region/plz - deadlineAt/estimatedValue nullable by mandate (RESEARCH Pattern 4) — missing data normalizes to null, never thrown or zero - dedupKey priority: ocid -> sourcePortal:sourceNoticeId fallback - contentHash = sha256(title+deadlineAt+estimatedValue+status), stable across repeat calls, changes when the deadline changes (SCHEMA-02 hook) - Register TenderNormalizerService in TendersModule.providers - All tender-normalizer.service.spec.ts tests green (6/6); full API suite green (59/59); tsc --noEmit clean
This commit is contained in:
@@ -0,0 +1,246 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TenderNormalizerService — maps a RawTenderRecord (paired eForms-DE XML +
|
||||||
|
* OCDS JSON) into the unified Tender field shape (SCHEMA-01), computing the
|
||||||
|
* dedupKey and contentHash (SCHEMA-02 change-detection hook).
|
||||||
|
*
|
||||||
|
* Pure, no I/O (matches the DkvParserService transform-service shape: one
|
||||||
|
* public entry point + private helpers).
|
||||||
|
*
|
||||||
|
* Field authority (RESEARCH.md Pattern 3 — reverses ARCHITECTURE.md's
|
||||||
|
* blanket "prefer OCDS" guidance based on a live cross-check):
|
||||||
|
* - eForms-DE XML is PRIMARY for deadlineAt / estimatedValue / procedureType
|
||||||
|
* (OCDS conversion measurably drops these for some notices).
|
||||||
|
* - OCDS is PRIMARY for ocid / buyerName / title / cpvCodes / region / plz.
|
||||||
|
* - deadlineAt / estimatedValue are nullable by mandate (RESEARCH Pattern 4)
|
||||||
|
* — null is the common case for Unterschwelle notices, not an edge case.
|
||||||
|
*/
|
||||||
|
@Injectable()
|
||||||
|
export class TenderNormalizerService {
|
||||||
|
normalize(raw: RawTenderRecord): NormalizedTenderFields {
|
||||||
|
const eformsRoot = getEformsRoot(raw.eformsPayload);
|
||||||
|
const ocdsTender = getOcdsTender(raw.ocdsPayload);
|
||||||
|
const ocdsBuyer = getOcdsBuyer(raw.ocdsPayload);
|
||||||
|
|
||||||
|
const title =
|
||||||
|
textValue(ocdsTender?.title) ??
|
||||||
|
extractEformsTitle(eformsRoot) ??
|
||||||
|
'Unbenannte Ausschreibung';
|
||||||
|
const buyerName = textValue(ocdsBuyer?.name);
|
||||||
|
const cpvCodes = extractCpvCodes(ocdsTender);
|
||||||
|
const region = textValue(ocdsBuyer?.address?.region);
|
||||||
|
const plz = textValue(ocdsBuyer?.address?.postalCode);
|
||||||
|
// NUTS->Bundesland name mapping deferred to Phase 11 (filter UI) — not
|
||||||
|
// required for this phase's ingestion core.
|
||||||
|
const bundesland: string | null = null;
|
||||||
|
|
||||||
|
const deadlineAt =
|
||||||
|
extractEformsDeadline(eformsRoot) ??
|
||||||
|
parseOcdsDate(ocdsTender?.tenderPeriod?.endDate);
|
||||||
|
const estimatedValue =
|
||||||
|
extractEformsEstimatedValue(eformsRoot) ??
|
||||||
|
parseOcdsAmount(ocdsTender?.value);
|
||||||
|
const procedureType =
|
||||||
|
extractEformsProcedureType(eformsRoot) ??
|
||||||
|
textValue(ocdsTender?.procurementMethodDetails);
|
||||||
|
|
||||||
|
const status = 'active';
|
||||||
|
const dedupKey = raw.ocid
|
||||||
|
? raw.ocid
|
||||||
|
: `${raw.sourcePortal}:${raw.sourceNoticeId}`;
|
||||||
|
const contentHash = computeContentHash({
|
||||||
|
title,
|
||||||
|
deadlineAt,
|
||||||
|
estimatedValue,
|
||||||
|
status,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
sourcePortal: raw.sourcePortal,
|
||||||
|
sourceNoticeId: raw.sourceNoticeId,
|
||||||
|
ocid: raw.ocid ?? null,
|
||||||
|
dedupKey,
|
||||||
|
title,
|
||||||
|
buyerName,
|
||||||
|
cpvCodes,
|
||||||
|
region,
|
||||||
|
plz,
|
||||||
|
bundesland,
|
||||||
|
deadlineAt,
|
||||||
|
estimatedValue,
|
||||||
|
procedureType,
|
||||||
|
status,
|
||||||
|
sourceUrl: raw.sourceUrl ?? null,
|
||||||
|
contentHash,
|
||||||
|
publishedAt: raw.publishedAt ?? new Date(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Private: pure field-extraction helpers (module-level) ────────────────
|
||||||
|
|
||||||
|
interface OcdsAddress {
|
||||||
|
region?: unknown;
|
||||||
|
postalCode?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OcdsParty {
|
||||||
|
name?: unknown;
|
||||||
|
address?: OcdsAddress;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OcdsClassification {
|
||||||
|
scheme?: unknown;
|
||||||
|
id?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OcdsItem {
|
||||||
|
classification?: OcdsClassification;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OcdsTender {
|
||||||
|
title?: unknown;
|
||||||
|
tenderPeriod?: { endDate?: unknown };
|
||||||
|
value?: { amount?: unknown };
|
||||||
|
procurementMethodDetails?: unknown;
|
||||||
|
items?: OcdsItem[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOcdsTender(ocdsPayload: unknown): OcdsTender | undefined {
|
||||||
|
if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined;
|
||||||
|
return (ocdsPayload as { tender?: OcdsTender }).tender;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOcdsBuyer(ocdsPayload: unknown): OcdsParty | undefined {
|
||||||
|
if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined;
|
||||||
|
return (ocdsPayload as { buyer?: OcdsParty }).buyer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Extract the sole root element's value from a fast-xml-parser document, robust across notice subtypes (ContractNotice, PriorInformationNotice, ...). */
|
||||||
|
function getEformsRoot(eformsPayload: unknown): Record<string, unknown> {
|
||||||
|
if (!eformsPayload || typeof eformsPayload !== 'object') return {};
|
||||||
|
const keys = Object.keys(eformsPayload as Record<string, unknown>);
|
||||||
|
const rootKey = keys[0];
|
||||||
|
if (!rootKey) return {};
|
||||||
|
const root = (eformsPayload as Record<string, unknown>)[rootKey];
|
||||||
|
return root && typeof root === 'object'
|
||||||
|
? (root as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** fast-xml-parser represents a tag with both attributes and text content as `{ '@_attr': ..., '#text': value }`; a plain tag is just the raw string/number. */
|
||||||
|
function textValue(node: unknown): string | null {
|
||||||
|
if (node === null || node === undefined) return null;
|
||||||
|
if (typeof node === 'string') return node || null;
|
||||||
|
if (typeof node === 'number') return String(node);
|
||||||
|
if (typeof node === 'object' && '#text' in (node as Record<string, unknown>)) {
|
||||||
|
const t = (node as Record<string, unknown>)['#text'];
|
||||||
|
if (t === null || t === undefined) return null;
|
||||||
|
return String(t);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstLot(
|
||||||
|
root: Record<string, unknown>,
|
||||||
|
): Record<string, unknown> | null {
|
||||||
|
const lot = root.ProcurementProjectLot;
|
||||||
|
if (!lot) return null;
|
||||||
|
if (Array.isArray(lot)) {
|
||||||
|
return (lot[0] as Record<string, unknown>) ?? null;
|
||||||
|
}
|
||||||
|
return lot as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractEformsTitle(root: Record<string, unknown>): string | null {
|
||||||
|
const project = root.ProcurementProject as
|
||||||
|
| Record<string, unknown>
|
||||||
|
| undefined;
|
||||||
|
return textValue(project?.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Primary structured deadline source (RESEARCH.md Pattern 3): per-lot
|
||||||
|
* `TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate` (+ optional
|
||||||
|
* `EndTime`), combined into a full ISO timestamp when both are present.
|
||||||
|
*/
|
||||||
|
function extractEformsDeadline(root: Record<string, unknown>): Date | null {
|
||||||
|
const lot = firstLot(root);
|
||||||
|
const process = lot?.TenderingProcess as Record<string, unknown> | undefined;
|
||||||
|
const period = process?.TenderSubmissionDeadlinePeriod as
|
||||||
|
| Record<string, unknown>
|
||||||
|
| undefined;
|
||||||
|
|
||||||
|
const endDateRaw = textValue(period?.EndDate);
|
||||||
|
if (!endDateRaw) return null;
|
||||||
|
|
||||||
|
const endTimeRaw = textValue(period?.EndTime);
|
||||||
|
const datePart = endDateRaw.replace(/[+-]\d{2}:\d{2}$/, '');
|
||||||
|
const combined = endTimeRaw ? `${datePart}T${endTimeRaw}` : datePart;
|
||||||
|
|
||||||
|
const parsed = new Date(combined);
|
||||||
|
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Primary structured value source (RESEARCH.md Pattern 3): root `ProcurementProject/RequestedTenderTotal/EstimatedOverallContractAmount`. */
|
||||||
|
function extractEformsEstimatedValue(
|
||||||
|
root: Record<string, unknown>,
|
||||||
|
): number | null {
|
||||||
|
const project = root.ProcurementProject as
|
||||||
|
| Record<string, unknown>
|
||||||
|
| undefined;
|
||||||
|
const total = project?.RequestedTenderTotal as
|
||||||
|
| Record<string, unknown>
|
||||||
|
| undefined;
|
||||||
|
const amountRaw = textValue(total?.EstimatedOverallContractAmount);
|
||||||
|
if (!amountRaw) return null;
|
||||||
|
|
||||||
|
const parsed = Number.parseFloat(amountRaw);
|
||||||
|
return Number.isNaN(parsed) ? null : parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Primary structured procedure-type source: root `TenderingProcess/ProcedureCode`. */
|
||||||
|
function extractEformsProcedureType(
|
||||||
|
root: Record<string, unknown>,
|
||||||
|
): string | null {
|
||||||
|
const process = root.TenderingProcess as Record<string, unknown> | undefined;
|
||||||
|
return textValue(process?.ProcedureCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] {
|
||||||
|
const items = Array.isArray(ocdsTender?.items) ? ocdsTender.items : [];
|
||||||
|
const codes = items
|
||||||
|
.map((item) => item?.classification?.id)
|
||||||
|
.filter((id): id is string => typeof id === 'string');
|
||||||
|
return Array.from(new Set(codes));
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseOcdsDate(value: unknown): Date | null {
|
||||||
|
if (typeof value !== 'string' || !value) return null;
|
||||||
|
const parsed = new Date(value);
|
||||||
|
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseOcdsAmount(value: { amount?: unknown } | undefined): number | null {
|
||||||
|
const amount = value?.amount;
|
||||||
|
return typeof amount === 'number' ? amount : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeContentHash(fields: {
|
||||||
|
title: string;
|
||||||
|
deadlineAt: Date | null;
|
||||||
|
estimatedValue: number | null;
|
||||||
|
status: string;
|
||||||
|
}): string {
|
||||||
|
const raw = [
|
||||||
|
fields.title,
|
||||||
|
fields.deadlineAt ? fields.deadlineAt.toISOString() : '',
|
||||||
|
fields.estimatedValue !== null ? String(fields.estimatedValue) : '',
|
||||||
|
fields.status,
|
||||||
|
].join('|');
|
||||||
|
|
||||||
|
return createHash('sha256').update(raw).digest('hex');
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import { ModuleRegistryService } from '../module-registry/module-registry.servic
|
|||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter';
|
import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter';
|
||||||
import { seedTendersModule } from './tenders.seed';
|
import { seedTendersModule } from './tenders.seed';
|
||||||
|
import { TenderNormalizerService } from './tender-normalizer.service';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* NestJS module for the Ausschreibungs-Radar feature.
|
* NestJS module for the Ausschreibungs-Radar feature.
|
||||||
@@ -22,7 +23,7 @@ import { seedTendersModule } from './tenders.seed';
|
|||||||
@Module({
|
@Module({
|
||||||
imports: [ModuleRegistryModule],
|
imports: [ModuleRegistryModule],
|
||||||
controllers: [],
|
controllers: [],
|
||||||
providers: [DoeOpenDataAdapter],
|
providers: [DoeOpenDataAdapter, TenderNormalizerService],
|
||||||
})
|
})
|
||||||
export class TendersModule implements OnModuleInit {
|
export class TendersModule implements OnModuleInit {
|
||||||
private readonly logger = new Logger(TendersModule.name);
|
private readonly logger = new Logger(TendersModule.name);
|
||||||
|
|||||||
Reference in New Issue
Block a user