tessera-ctl/apps/api/src/tenders/tender-normalizer.service.spec.ts

import AdmZip from 'adm-zip';
import { XMLParser } from 'fast-xml-parser';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import type { RawTenderRecord, SourceType } from './tender.types';
import { TenderNormalizerService } from './tender-normalizer.service';

/**
 * Loads a real notice pair (eForms-DE XML + OCDS JSON) directly from the
 * committed fixture ZIPs and assembles it into a RawTenderRecord — the same
 * real-fixture precedent as the DKV PDF parser, kept independent of
 * DoeOpenDataAdapter so this unit test exercises TenderNormalizerService in
 * isolation as a pure mapping function.
 */
const FIXTURES_DIR = join(__dirname, '__fixtures__');
const xmlParser = new XMLParser({
  removeNSPrefix: true,
  ignoreAttributes: false,
  attributeNamePrefix: '@_',
});

function loadRawNoticePair(basename: string): RawTenderRecord {
  const eformsZip = new AdmZip(
    readFileSync(join(FIXTURES_DIR, 'doe-eforms-sample.zip')),
  );
  const ocdsZip = new AdmZip(
    readFileSync(join(FIXTURES_DIR, 'doe-ocds-sample.zip')),
  );

  const xmlEntry = eformsZip.getEntry(`${basename}.xml`);
  const jsonEntry = ocdsZip.getEntry(`${basename}.json`);
  if (!xmlEntry || !jsonEntry) {
    throw new Error(`Fixture entry missing for ${basename}`);
  }

  const eformsPayload = xmlParser.parse(xmlEntry.getData().toString('utf8'));
  const ocdsDocument = JSON.parse(jsonEntry.getData().toString('utf8'));
  const release = ocdsDocument.releases[0];

  return {
    sourceType: 'doe-opendata',
    sourcePortal: 'doe-opendata',
    sourceNoticeId: release.id,
    ocid: release.ocid,
    sourceUrl: ocdsDocument.uri,
    fetchedAt: new Date(),
    publishedAt: ocdsDocument.publishedDate
      ? new Date(ocdsDocument.publishedDate)
      : null,
    eformsPayload,
    ocdsPayload: release,
  };
}

// Real notice: eForms XML has a structured
// ProcurementProjectLot/TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate
// (2026-08-18), while the paired OCDS release has no `tender.tenderPeriod` at
// all (RESEARCH.md Pattern 3 live cross-check).
const NOTICE_WITH_EFORMS_ONLY_DEADLINE =
  '019f64ec-cd64-4c09-8fd7-4fd12d8eea64-01';

// Real notice: neither the eForms XML nor the OCDS release carries a
// deadline or estimated value (RESEARCH.md Pattern 4 — the common case).
const NOTICE_WITH_NO_DEADLINE_OR_VALUE =
  '001c68dd-258a-4bf7-a264-478604ad9c3e-01';

/**
 * Builds a RawTenderRecord for the flat ocdsPayload-bag scraper adapters
 * (NetServer/cosinex-DTVP) — no fixture ZIPs exist for these sources, so
 * the record is constructed inline (gap closure, 13-VERIFICATION.md).
 */
function bagRecord(
  sourceType: SourceType,
  bag: {
    title?: string | null;
    buyerName?: string | null;
    procedureType?: string | null;
    legalFramework?: string | null;
    deadlineAt?: string | null;
  },
): RawTenderRecord {
  const sourcePortal =
    sourceType === 'cosinex-dtvp'
      ? 'cosinex-dtvp'
      : sourceType === 'rss'
        ? 'service-bund'
        : sourceType === 'email-alert'
          ? 'email-alert'
          : 'tender24';
  return {
    sourceType,
    sourcePortal,
    sourceNoticeId: 'bag-notice-1',
    ocid: undefined,
    sourceUrl: `https://example.invalid/${sourcePortal}`,
    fetchedAt: new Date(),
    publishedAt: null,
    eformsPayload: null,
    ocdsPayload: bag,
  };
}

describe('TenderNormalizerService', () => {
  const service = new TenderNormalizerService();

  it('recovers deadlineAt from eForms-DE XML when the OCDS release has no tenderPeriod (SCHEMA-01 key assertion)', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);

    const normalized = service.normalize(raw);

    expect(normalized.deadlineAt).not.toBeNull();
    expect(normalized.deadlineAt?.getUTCFullYear()).toBe(2026);
    expect(normalized.deadlineAt?.getUTCMonth()).toBe(7); // August (0-indexed)
    expect(normalized.deadlineAt?.getUTCDate()).toBe(18);
  });

  it('normalizes missing deadline/value to null, not thrown or zero', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);

    const normalized = service.normalize(raw);

    expect(normalized.deadlineAt).toBeNull();
    expect(normalized.estimatedValue).toBeNull();
    expect(normalized.title).toBeTruthy();
  });

  it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);

    const normalized = service.normalize(raw);

    if (normalized.region) {
      expect(normalized.bundesland).not.toBeNull();
    } else {
      expect(normalized.bundesland).toBeNull();
    }
  });

  it('bundesland is null when region is null/absent — never crashes', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
    const rawWithoutRegion: RawTenderRecord = {
      ...raw,
      ocdsPayload: {
        ...(raw.ocdsPayload as Record<string, unknown>),
        buyer: {
          ...(raw.ocdsPayload as { buyer?: Record<string, unknown> })?.buyer,
          address: { postalCode: '12345' },
        },
      },
    };

    const normalized = service.normalize(rawWithoutRegion);

    expect(normalized.region).toBeNull();
    expect(normalized.bundesland).toBeNull();
  });

  it('derives distinct cpvDivisions (leading 2 digits) from cpvCodes (FILTER-03, Pitfall 2)', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);

    const normalized = service.normalize(raw);

    expect(Array.isArray(normalized.cpvDivisions)).toBe(true);
    for (const division of normalized.cpvDivisions) {
      expect(division).toMatch(/^\d{2}$/);
    }
    // Every cpvCodes entry must normalize into one of the derived divisions.
    for (const code of normalized.cpvCodes) {
      const digits = code.replace(/\D/g, '');
      if (digits.length >= 2) {
        expect(normalized.cpvDivisions).toContain(digits.slice(0, 2));
      }
    }
  });

  it('cpvDivisions is an empty array when cpvCodes is empty — never crashes', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
    const rawWithoutCpv: RawTenderRecord = {
      ...raw,
      ocdsPayload: {
        ...(raw.ocdsPayload as Record<string, unknown>),
        tender: {
          ...(raw.ocdsPayload as { tender?: Record<string, unknown> })?.tender,
          items: [],
        },
      },
    };

    const normalized = service.normalize(rawWithoutCpv);

    expect(normalized.cpvDivisions).toEqual([]);
  });

  it('dedupKey = ocid when present', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);

    const normalized = service.normalize(raw);

    expect(raw.ocid).toBeTruthy();
    expect(normalized.dedupKey).toBe(raw.ocid);
  });

  it('dedupKey falls back to sourcePortal:sourceNoticeId when ocid is absent', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
    const rawWithoutOcid: RawTenderRecord = { ...raw, ocid: undefined };

    const normalized = service.normalize(rawWithoutOcid);

    expect(normalized.dedupKey).toBe(
      `${raw.sourcePortal}:${raw.sourceNoticeId}`,
    );
  });

  it('contentHash is a stable sha256 over title+deadline+value+status', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);

    const first = service.normalize(raw);
    const second = service.normalize(raw);

    expect(first.contentHash).toBe(second.contentHash);
    expect(first.contentHash).toMatch(/^[a-f0-9]{64}$/);
  });

  it('contentHash changes when the deadline changes', () => {
    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
    const original = service.normalize(raw);

    // Mutate the eForms deadline to simulate a Fristverlängerung (SCHEMA-02).
    const mutatedEformsPayload = JSON.parse(JSON.stringify(raw.eformsPayload));
    const root =
      mutatedEformsPayload[Object.keys(mutatedEformsPayload)[0]];
    const lots = Array.isArray(root.ProcurementProjectLot)
      ? root.ProcurementProjectLot
      : [root.ProcurementProjectLot];
    for (const lot of lots) {
      lot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate =
        '2026-12-31+01:00';
    }
    const mutatedRaw: RawTenderRecord = {
      ...raw,
      eformsPayload: mutatedEformsPayload,
    };

    const mutated = service.normalize(mutatedRaw);

    expect(mutated.contentHash).not.toBe(original.contentHash);
  });

  describe('generic ocdsPayload bag (ai-netserver / cosinex-dtvp gap closure)', () => {
    const fullBag = {
      title: 'Bauleistung X',
      buyerName: 'Stadt Y',
      procedureType: 'Offenes Verfahren',
      legalFramework: 'VOB/A',
      deadlineAt: '2026-09-01T10:00:00.000Z',
    };

    it('ai-netserver: maps the flat bag into NormalizedTenderFields', () => {
      const raw = bagRecord('ai-netserver', fullBag);

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe('Bauleistung X');
      expect(normalized.buyerName).toBe('Stadt Y');
      expect(normalized.procedureType).toBe('Offenes Verfahren');
      expect(normalized.deadlineAt).toBeInstanceOf(Date);
      expect(normalized.deadlineAt?.toISOString()).toBe(
        '2026-09-01T10:00:00.000Z',
      );
      expect(normalized.cpvCodes).toEqual([]);
      expect(normalized.cpvDivisions).toEqual([]);
      expect(normalized.region).toBeNull();
      expect(normalized.plz).toBeNull();
      expect(normalized.bundesland).toBeNull();
      expect(normalized.estimatedValue).toBeNull();
      expect(normalized.dedupKey).toBe(`${raw.sourcePortal}:${raw.sourceNoticeId}`);
      expect(normalized.status).toBe('active');
    });

    it('cosinex-dtvp: maps an identical bag shape identically', () => {
      const raw = bagRecord('cosinex-dtvp', fullBag);

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe('Bauleistung X');
      expect(normalized.buyerName).toBe('Stadt Y');
      expect(normalized.procedureType).toBe('Offenes Verfahren');
      expect(normalized.deadlineAt?.toISOString()).toBe(
        '2026-09-01T10:00:00.000Z',
      );
      expect(normalized.cpvCodes).toEqual([]);
      expect(normalized.cpvDivisions).toEqual([]);
      expect(normalized.region).toBeNull();
      expect(normalized.plz).toBeNull();
      expect(normalized.bundesland).toBeNull();
      expect(normalized.estimatedValue).toBeNull();
      expect(normalized.dedupKey).toBe(`${raw.sourcePortal}:${raw.sourceNoticeId}`);
    });

    it('null/empty bag fields: buyerName/procedureType/deadlineAt stay null, title falls back', () => {
      const raw = bagRecord('ai-netserver', {
        title: '',
        buyerName: null,
        procedureType: null,
        legalFramework: null,
        deadlineAt: null,
      });

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe('Unbenannte Ausschreibung');
      expect(normalized.buyerName).toBeNull();
      expect(normalized.procedureType).toBeNull();
      expect(normalized.deadlineAt).toBeNull();
    });

    it('contentHash for a bag source is a stable sha256 (proves shared assemble() reuse)', () => {
      const raw = bagRecord('ai-netserver', fullBag);

      const normalized = service.normalize(raw);

      expect(normalized.contentHash).toMatch(/^[a-f0-9]{64}$/);
    });
  });

  describe("'rss' sourceType (Phase 14, Plan 02, INGEST-04)", () => {
    it('maps an rss bag through normalizeBag(): title passes through, cpvDivisions stays empty', () => {
      const raw = bagRecord('rss', {
        title: 'Übermittlung von Bohrungsdaten',
        buyerName: null,
        procedureType: null,
        legalFramework: null,
        deadlineAt: null,
      });

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe('Übermittlung von Bohrungsdaten');
      expect(normalized.cpvDivisions).toEqual([]);
      expect(normalized.cpvCodes).toEqual([]);
      expect(normalized.region).toBeNull();
      expect(normalized.bundesland).toBeNull();
      expect(normalized.buyerName).toBeNull();
      expect(normalized.procedureType).toBeNull();
      expect(normalized.deadlineAt).toBeNull();
      expect(normalized.dedupKey).toBe(
        `${raw.sourcePortal}:${raw.sourceNoticeId}`,
      );
    });

    it('rss title falls back to "Unbenannte Ausschreibung" when empty', () => {
      const raw = bagRecord('rss', { title: '' });

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe('Unbenannte Ausschreibung');
    });
  });

  describe("'email-alert' sourceType (Phase 14, Plan 03, INGEST-05)", () => {
    it('maps an email-alert bag through normalizeBag(): title passes through, cpvDivisions stays empty', () => {
      const raw = bagRecord('email-alert', {
        title: 'Vergabebekanntmachung: Straßenbauarbeiten Musterstadt',
        buyerName: null,
        procedureType: null,
        legalFramework: null,
        deadlineAt: null,
      });

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe(
        'Vergabebekanntmachung: Straßenbauarbeiten Musterstadt',
      );
      expect(normalized.cpvDivisions).toEqual([]);
      expect(normalized.cpvCodes).toEqual([]);
      expect(normalized.region).toBeNull();
      expect(normalized.bundesland).toBeNull();
      expect(normalized.buyerName).toBeNull();
      expect(normalized.procedureType).toBeNull();
      expect(normalized.deadlineAt).toBeNull();
      expect(normalized.dedupKey).toBe(
        `${raw.sourcePortal}:${raw.sourceNoticeId}`,
      );
    });

    it('email-alert title falls back to "Unbenannte Ausschreibung" when empty', () => {
      const raw = bagRecord('email-alert', { title: '' });

      const normalized = service.normalize(raw);

      expect(normalized.title).toBe('Unbenannte Ausschreibung');
    });
  });
});