tessera-ctl/apps/api/src/tenders/tender-normalizer.service.ts

import { Injectable } from '@nestjs/common';
import { createHash } from 'crypto';
import { divisionOf } from './cpv/cpv-catalog';
import { bundeslandFromRegion } from './geo/nuts-bundesland';
import type { NormalizedTenderFields, RawTenderRecord } from './tender.types';

/**
 * TenderNormalizerService — maps a RawTenderRecord into the unified Tender
 * field shape (SCHEMA-01), computing the dedupKey and contentHash
 * (SCHEMA-02 change-detection hook).
 *
 * Pure, no I/O (matches the DkvParserService transform-service shape: one
 * public entry point + private helpers).
 *
 * Per-sourceType dispatch (gap closure, 13-VERIFICATION.md): the DÖE source
 * pairs eForms-DE XML + OCDS JSON (see normalizeDoe field-authority notes
 * below); the NetServer/cosinex-DTVP scraper adapters instead carry a flat
 * `ocdsPayload` bag (see normalizeBag) with no eForms/OCDS structure at all.
 * Both paths converge on the shared `assemble()` tail so status/dedupKey/
 * contentHash/publishedAt are computed identically for every source.
 *
 * Field authority for the DÖE path (RESEARCH.md Pattern 3 — reverses
 * ARCHITECTURE.md's blanket "prefer OCDS" guidance based on a live
 * cross-check):
 * - eForms-DE XML is PRIMARY for deadlineAt / estimatedValue / procedureType
 *   (OCDS conversion measurably drops these for some notices).
 * - OCDS is PRIMARY for ocid / buyerName / title / cpvCodes / region / plz.
 * - deadlineAt / estimatedValue are nullable by mandate (RESEARCH Pattern 4)
 *   — null is the common case for Unterschwelle notices, not an edge case.
 */

/**
 * The 10 source-specific fields each per-sourceType extraction path must
 * produce; everything else (sourcePortal, sourceNoticeId, ocid, dedupKey,
 * status, sourceUrl, contentHash, publishedAt, fingerprint) is assembled
 * identically for all sources by `assemble()`.
 */
type CoreTenderFields = Omit<
  NormalizedTenderFields,
  | 'sourcePortal'
  | 'sourceNoticeId'
  | 'ocid'
  | 'dedupKey'
  | 'status'
  | 'sourceUrl'
  | 'contentHash'
  | 'publishedAt'
  | 'fingerprint'
>;

@Injectable()
export class TenderNormalizerService {
  normalize(raw: RawTenderRecord): NormalizedTenderFields {
    switch (raw.sourceType) {
      case 'ai-netserver':
      case 'cosinex-dtvp':
      case 'rss':
      case 'email-alert':
        return this.normalizeBag(raw);
      case 'doe-opendata':
      default:
        // default MUST stay on the DÖE path (never throw) so future
        // additive SourceType union members don't break existing callers.
        return this.normalizeDoe(raw);
    }
  }

  /** DÖE path: paired eForms-DE XML + OCDS JSON. Unchanged behavior (regression guard). */
  private normalizeDoe(raw: RawTenderRecord): NormalizedTenderFields {
    const eformsRoot = getEformsRoot(raw.eformsPayload);
    const ocdsTender = getOcdsTender(raw.ocdsPayload);
    const ocdsBuyer = getOcdsBuyer(raw.ocdsPayload);

    const title =
      textValue(ocdsTender?.title) ??
      extractEformsTitle(eformsRoot) ??
      'Unbenannte Ausschreibung';
    const buyerName = textValue(ocdsBuyer?.name);
    const cpvCodes = extractCpvCodes(ocdsTender);
    // Phase 11 (FILTER-03, Pitfall 2): derive distinct CPV divisions from
    // the raw, inconsistently-formatted cpvCodes ("45", "45000000",
    // "45000000-7") so new ingests are hasSome-filterable immediately —
    // the ~1671 pre-existing rows are backfilled by a one-time migration
    // (20260721150000_tender_cpv_divisions_backfill).
    const cpvDivisions = deriveCpvDivisions(cpvCodes);
    const region = textValue(ocdsBuyer?.address?.region);
    const plz = textValue(ocdsBuyer?.address?.postalCode);
    // Phase 11 (FILTER-02, Pitfall 1): derive bundesland from region's
    // NUTS-1 prefix so new ingests are filterable immediately — the
    // ~1671 pre-existing rows are backfilled by a one-time migration
    // (20260721140000_tender_bundesland_backfill).
    const bundesland = bundeslandFromRegion(region);

    const deadlineAt =
      extractEformsDeadline(eformsRoot) ??
      parseOcdsDate(ocdsTender?.tenderPeriod?.endDate);
    const estimatedValue =
      extractEformsEstimatedValue(eformsRoot) ??
      parseOcdsAmount(ocdsTender?.value);
    const procedureType =
      extractEformsProcedureType(eformsRoot) ??
      textValue(ocdsTender?.procurementMethodDetails);

    const core: CoreTenderFields = {
      title,
      buyerName,
      cpvCodes,
      cpvDivisions,
      region,
      plz,
      bundesland,
      deadlineAt,
      estimatedValue,
      procedureType,
    };

    return this.assemble(raw, core);
  }

  /**
   * Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters
   * (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04; and since
   * Phase 14 Plan 03, the EmailAlertAdapter, INGEST-05) carry a flat
   * `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in
   * `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect.
   * `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has
   * no target field for it. RSS/email-alert records always have
   * `buyerName`/`procedureType`/`deadlineAt` null (baseline title/link/guid
   * mapping only, D-04/D-05 thin-fields deferral).
   */
  private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields {
    const bag = getBagPayload(raw.ocdsPayload);

    const title = textValue(bag.title) || 'Unbenannte Ausschreibung';
    const buyerName = textValue(bag.buyerName);
    const procedureType = textValue(bag.procedureType);
    const deadlineAt = parseOcdsDate(bag.deadlineAt);

    const core: CoreTenderFields = {
      title,
      buyerName,
      cpvCodes: [],
      cpvDivisions: [],
      region: null,
      plz: null,
      bundesland: null,
      deadlineAt,
      estimatedValue: null,
      procedureType,
    };

    return this.assemble(raw, core);
  }

  /**
   * Shared tail: computes the source-invariant fields (status, dedupKey,
   * contentHash, publishedAt) plus the pass-through identity fields
   * (sourcePortal, sourceNoticeId, ocid, sourceUrl) identically regardless
   * of which extraction path produced `core`.
   */
  private assemble(
    raw: RawTenderRecord,
    core: CoreTenderFields,
  ): NormalizedTenderFields {
    const status = 'active';
    const dedupKey = raw.ocid
      ? raw.ocid
      : `${raw.sourcePortal}:${raw.sourceNoticeId}`;
    const contentHash = computeContentHash({
      title: core.title,
      deadlineAt: core.deadlineAt,
      estimatedValue: core.estimatedValue,
      status,
    });

    return {
      sourcePortal: raw.sourcePortal,
      sourceNoticeId: raw.sourceNoticeId,
      ocid: raw.ocid ?? null,
      dedupKey,
      ...core,
      status,
      sourceUrl: raw.sourceUrl ?? null,
      contentHash,
      publishedAt: raw.publishedAt ?? new Date(),
      // D-13: passed through unchanged — undefined for every source except
      // EmailAlertAdapter, which sets it to the configuring tenant's id.
      ownerTenantId: raw.ownerTenantId,
    };
  }
}

// ─── Private: pure field-extraction helpers (module-level) ────────────────

interface OcdsAddress {
  region?: unknown;
  postalCode?: unknown;
}

interface OcdsParty {
  name?: unknown;
  address?: OcdsAddress;
}

interface OcdsClassification {
  scheme?: unknown;
  id?: unknown;
}

interface OcdsItem {
  classification?: OcdsClassification;
}

interface OcdsTender {
  title?: unknown;
  tenderPeriod?: { endDate?: unknown };
  value?: { amount?: unknown };
  procurementMethodDetails?: unknown;
  items?: OcdsItem[];
}

function getOcdsTender(ocdsPayload: unknown): OcdsTender | undefined {
  if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined;
  return (ocdsPayload as { tender?: OcdsTender }).tender;
}

function getOcdsBuyer(ocdsPayload: unknown): OcdsParty | undefined {
  if (!ocdsPayload || typeof ocdsPayload !== 'object') return undefined;
  return (ocdsPayload as { buyer?: OcdsParty }).buyer;
}

/** Extract the sole root element's value from a fast-xml-parser document, robust across notice subtypes (ContractNotice, PriorInformationNotice, ...). */
function getEformsRoot(eformsPayload: unknown): Record<string, unknown> {
  if (!eformsPayload || typeof eformsPayload !== 'object') return {};
  const keys = Object.keys(eformsPayload as Record<string, unknown>);
  const rootKey = keys[0];
  if (!rootKey) return {};
  const root = (eformsPayload as Record<string, unknown>)[rootKey];
  return root && typeof root === 'object'
    ? (root as Record<string, unknown>)
    : {};
}

/**
 * The flat scraper-adapter bag shape (NetServer/cosinex-DTVP): read
 * defensively — the field is `unknown` at the type level and adapters are
 * out of this plan's scope to change.
 */
interface OcdsBag {
  title?: unknown;
  buyerName?: unknown;
  procedureType?: unknown;
  legalFramework?: unknown;
  deadlineAt?: unknown;
}

function getBagPayload(ocdsPayload: unknown): OcdsBag {
  if (!ocdsPayload || typeof ocdsPayload !== 'object') return {};
  return ocdsPayload as OcdsBag;
}

/** fast-xml-parser represents a tag with both attributes and text content as `{ '@_attr': ..., '#text': value }`; a plain tag is just the raw string/number. */
function textValue(node: unknown): string | null {
  if (node === null || node === undefined) return null;
  if (typeof node === 'string') return node || null;
  if (typeof node === 'number') return String(node);
  if (typeof node === 'object' && '#text' in (node as Record<string, unknown>)) {
    const t = (node as Record<string, unknown>)['#text'];
    if (t === null || t === undefined) return null;
    return String(t);
  }
  return null;
}

function firstLot(
  root: Record<string, unknown>,
): Record<string, unknown> | null {
  const lot = root.ProcurementProjectLot;
  if (!lot) return null;
  if (Array.isArray(lot)) {
    return (lot[0] as Record<string, unknown>) ?? null;
  }
  return lot as Record<string, unknown>;
}

function extractEformsTitle(root: Record<string, unknown>): string | null {
  const project = root.ProcurementProject as
    | Record<string, unknown>
    | undefined;
  return textValue(project?.Name);
}

/**
 * Primary structured deadline source (RESEARCH.md Pattern 3): per-lot
 * `TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate` (+ optional
 * `EndTime`), combined into a full ISO timestamp when both are present.
 */
function extractEformsDeadline(root: Record<string, unknown>): Date | null {
  const lot = firstLot(root);
  const process = lot?.TenderingProcess as Record<string, unknown> | undefined;
  const period = process?.TenderSubmissionDeadlinePeriod as
    | Record<string, unknown>
    | undefined;

  const endDateRaw = textValue(period?.EndDate);
  if (!endDateRaw) return null;

  const endTimeRaw = textValue(period?.EndTime);
  const datePart = endDateRaw.replace(/[+-]\d{2}:\d{2}$/, '');
  const combined = endTimeRaw ? `${datePart}T${endTimeRaw}` : datePart;

  const parsed = new Date(combined);
  return Number.isNaN(parsed.getTime()) ? null : parsed;
}

/** Primary structured value source (RESEARCH.md Pattern 3): root `ProcurementProject/RequestedTenderTotal/EstimatedOverallContractAmount`. */
function extractEformsEstimatedValue(
  root: Record<string, unknown>,
): number | null {
  const project = root.ProcurementProject as
    | Record<string, unknown>
    | undefined;
  const total = project?.RequestedTenderTotal as
    | Record<string, unknown>
    | undefined;
  const amountRaw = textValue(total?.EstimatedOverallContractAmount);
  if (!amountRaw) return null;

  const parsed = Number.parseFloat(amountRaw);
  return Number.isNaN(parsed) ? null : parsed;
}

/** Primary structured procedure-type source: root `TenderingProcess/ProcedureCode`. */
function extractEformsProcedureType(
  root: Record<string, unknown>,
): string | null {
  const process = root.TenderingProcess as Record<string, unknown> | undefined;
  return textValue(process?.ProcedureCode);
}

function extractCpvCodes(ocdsTender: OcdsTender | undefined): string[] {
  const items = Array.isArray(ocdsTender?.items) ? ocdsTender.items : [];
  const codes = items
    .map((item) => item?.classification?.id)
    .filter((id): id is string => typeof id === 'string');
  return Array.from(new Set(codes));
}

/**
 * Distinct CPV division codes (leading 2 digits) derived from the raw
 * cpvCodes array via cpv-catalog.ts's divisionOf() — handles mixed formats
 * ("45", "45000000", "45000000-7") uniformly (Pitfall 2). Never throws on
 * an empty/malformed code; divisionOf('') === '' is filtered out below so
 * cpvDivisions never contains an empty-string entry.
 */
function deriveCpvDivisions(cpvCodes: string[]): string[] {
  const divisions = cpvCodes.map((code) => divisionOf(code)).filter(Boolean);
  return Array.from(new Set(divisions));
}

function parseOcdsDate(value: unknown): Date | null {
  if (typeof value !== 'string' || !value) return null;
  const parsed = new Date(value);
  return Number.isNaN(parsed.getTime()) ? null : parsed;
}

function parseOcdsAmount(value: { amount?: unknown } | undefined): number | null {
  const amount = value?.amount;
  return typeof amount === 'number' ? amount : null;
}

function computeContentHash(fields: {
  title: string;
  deadlineAt: Date | null;
  estimatedValue: number | null;
  status: string;
}): string {
  const raw = [
    fields.title,
    fields.deadlineAt ? fields.deadlineAt.toISOString() : '',
    fields.estimatedValue !== null ? String(fields.estimatedValue) : '',
    fields.status,
  ].join('|');

  return createHash('sha256').update(raw).digest('hex');
}