import AdmZip from 'adm-zip'; import { XMLParser } from 'fast-xml-parser'; import { readFileSync } from 'fs'; import { join } from 'path'; import { describe, expect, it } from 'vitest'; import type { RawTenderRecord, SourceType } from './tender.types'; import { TenderNormalizerService } from './tender-normalizer.service'; /** * Loads a real notice pair (eForms-DE XML + OCDS JSON) directly from the * committed fixture ZIPs and assembles it into a RawTenderRecord — the same * real-fixture precedent as the DKV PDF parser, kept independent of * DoeOpenDataAdapter so this unit test exercises TenderNormalizerService in * isolation as a pure mapping function. */ const FIXTURES_DIR = join(__dirname, '__fixtures__'); const xmlParser = new XMLParser({ removeNSPrefix: true, ignoreAttributes: false, attributeNamePrefix: '@_', }); function loadRawNoticePair(basename: string): RawTenderRecord { const eformsZip = new AdmZip( readFileSync(join(FIXTURES_DIR, 'doe-eforms-sample.zip')), ); const ocdsZip = new AdmZip( readFileSync(join(FIXTURES_DIR, 'doe-ocds-sample.zip')), ); const xmlEntry = eformsZip.getEntry(`${basename}.xml`); const jsonEntry = ocdsZip.getEntry(`${basename}.json`); if (!xmlEntry || !jsonEntry) { throw new Error(`Fixture entry missing for ${basename}`); } const eformsPayload = xmlParser.parse(xmlEntry.getData().toString('utf8')); const ocdsDocument = JSON.parse(jsonEntry.getData().toString('utf8')); const release = ocdsDocument.releases[0]; return { sourceType: 'doe-opendata', sourcePortal: 'doe-opendata', sourceNoticeId: release.id, ocid: release.ocid, sourceUrl: ocdsDocument.uri, fetchedAt: new Date(), publishedAt: ocdsDocument.publishedDate ? new Date(ocdsDocument.publishedDate) : null, eformsPayload, ocdsPayload: release, }; } // Real notice: eForms XML has a structured // ProcurementProjectLot/TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate // (2026-08-18), while the paired OCDS release has no `tender.tenderPeriod` at // all (RESEARCH.md Pattern 3 live cross-check). const NOTICE_WITH_EFORMS_ONLY_DEADLINE = '019f64ec-cd64-4c09-8fd7-4fd12d8eea64-01'; // Real notice: neither the eForms XML nor the OCDS release carries a // deadline or estimated value (RESEARCH.md Pattern 4 — the common case). const NOTICE_WITH_NO_DEADLINE_OR_VALUE = '001c68dd-258a-4bf7-a264-478604ad9c3e-01'; /** * Builds a RawTenderRecord for the flat ocdsPayload-bag scraper adapters * (NetServer/cosinex-DTVP) — no fixture ZIPs exist for these sources, so * the record is constructed inline (gap closure, 13-VERIFICATION.md). */ function bagRecord( sourceType: SourceType, bag: { title?: string | null; buyerName?: string | null; procedureType?: string | null; legalFramework?: string | null; deadlineAt?: string | null; }, ): RawTenderRecord { const sourcePortal = sourceType === 'cosinex-dtvp' ? 'cosinex-dtvp' : 'tender24'; return { sourceType, sourcePortal, sourceNoticeId: 'bag-notice-1', ocid: undefined, sourceUrl: `https://example.invalid/${sourcePortal}`, fetchedAt: new Date(), publishedAt: null, eformsPayload: null, ocdsPayload: bag, }; } describe('TenderNormalizerService', () => { const service = new TenderNormalizerService(); it('recovers deadlineAt from eForms-DE XML when the OCDS release has no tenderPeriod (SCHEMA-01 key assertion)', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const normalized = service.normalize(raw); expect(normalized.deadlineAt).not.toBeNull(); expect(normalized.deadlineAt?.getUTCFullYear()).toBe(2026); expect(normalized.deadlineAt?.getUTCMonth()).toBe(7); // August (0-indexed) expect(normalized.deadlineAt?.getUTCDate()).toBe(18); }); it('normalizes missing deadline/value to null, not thrown or zero', () => { const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); const normalized = service.normalize(raw); expect(normalized.deadlineAt).toBeNull(); expect(normalized.estimatedValue).toBeNull(); expect(normalized.title).toBeTruthy(); }); it('derives bundesland from region NUTS-1 prefix (FILTER-02, Pitfall 1)', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const normalized = service.normalize(raw); if (normalized.region) { expect(normalized.bundesland).not.toBeNull(); } else { expect(normalized.bundesland).toBeNull(); } }); it('bundesland is null when region is null/absent — never crashes', () => { const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); const rawWithoutRegion: RawTenderRecord = { ...raw, ocdsPayload: { ...(raw.ocdsPayload as Record), buyer: { ...(raw.ocdsPayload as { buyer?: Record })?.buyer, address: { postalCode: '12345' }, }, }, }; const normalized = service.normalize(rawWithoutRegion); expect(normalized.region).toBeNull(); expect(normalized.bundesland).toBeNull(); }); it('derives distinct cpvDivisions (leading 2 digits) from cpvCodes (FILTER-03, Pitfall 2)', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const normalized = service.normalize(raw); expect(Array.isArray(normalized.cpvDivisions)).toBe(true); for (const division of normalized.cpvDivisions) { expect(division).toMatch(/^\d{2}$/); } // Every cpvCodes entry must normalize into one of the derived divisions. for (const code of normalized.cpvCodes) { const digits = code.replace(/\D/g, ''); if (digits.length >= 2) { expect(normalized.cpvDivisions).toContain(digits.slice(0, 2)); } } }); it('cpvDivisions is an empty array when cpvCodes is empty — never crashes', () => { const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); const rawWithoutCpv: RawTenderRecord = { ...raw, ocdsPayload: { ...(raw.ocdsPayload as Record), tender: { ...(raw.ocdsPayload as { tender?: Record })?.tender, items: [], }, }, }; const normalized = service.normalize(rawWithoutCpv); expect(normalized.cpvDivisions).toEqual([]); }); it('dedupKey = ocid when present', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const normalized = service.normalize(raw); expect(raw.ocid).toBeTruthy(); expect(normalized.dedupKey).toBe(raw.ocid); }); it('dedupKey falls back to sourcePortal:sourceNoticeId when ocid is absent', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const rawWithoutOcid: RawTenderRecord = { ...raw, ocid: undefined }; const normalized = service.normalize(rawWithoutOcid); expect(normalized.dedupKey).toBe( `${raw.sourcePortal}:${raw.sourceNoticeId}`, ); }); it('contentHash is a stable sha256 over title+deadline+value+status', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const first = service.normalize(raw); const second = service.normalize(raw); expect(first.contentHash).toBe(second.contentHash); expect(first.contentHash).toMatch(/^[a-f0-9]{64}$/); }); it('contentHash changes when the deadline changes', () => { const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); const original = service.normalize(raw); // Mutate the eForms deadline to simulate a Fristverlängerung (SCHEMA-02). const mutatedEformsPayload = JSON.parse(JSON.stringify(raw.eformsPayload)); const root = mutatedEformsPayload[Object.keys(mutatedEformsPayload)[0]]; const lots = Array.isArray(root.ProcurementProjectLot) ? root.ProcurementProjectLot : [root.ProcurementProjectLot]; for (const lot of lots) { lot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate = '2026-12-31+01:00'; } const mutatedRaw: RawTenderRecord = { ...raw, eformsPayload: mutatedEformsPayload, }; const mutated = service.normalize(mutatedRaw); expect(mutated.contentHash).not.toBe(original.contentHash); }); describe('generic ocdsPayload bag (ai-netserver / cosinex-dtvp gap closure)', () => { const fullBag = { title: 'Bauleistung X', buyerName: 'Stadt Y', procedureType: 'Offenes Verfahren', legalFramework: 'VOB/A', deadlineAt: '2026-09-01T10:00:00.000Z', }; it('ai-netserver: maps the flat bag into NormalizedTenderFields', () => { const raw = bagRecord('ai-netserver', fullBag); const normalized = service.normalize(raw); expect(normalized.title).toBe('Bauleistung X'); expect(normalized.buyerName).toBe('Stadt Y'); expect(normalized.procedureType).toBe('Offenes Verfahren'); expect(normalized.deadlineAt).toBeInstanceOf(Date); expect(normalized.deadlineAt?.toISOString()).toBe( '2026-09-01T10:00:00.000Z', ); expect(normalized.cpvCodes).toEqual([]); expect(normalized.cpvDivisions).toEqual([]); expect(normalized.region).toBeNull(); expect(normalized.plz).toBeNull(); expect(normalized.bundesland).toBeNull(); expect(normalized.estimatedValue).toBeNull(); expect(normalized.dedupKey).toBe(`${raw.sourcePortal}:${raw.sourceNoticeId}`); expect(normalized.status).toBe('active'); }); it('cosinex-dtvp: maps an identical bag shape identically', () => { const raw = bagRecord('cosinex-dtvp', fullBag); const normalized = service.normalize(raw); expect(normalized.title).toBe('Bauleistung X'); expect(normalized.buyerName).toBe('Stadt Y'); expect(normalized.procedureType).toBe('Offenes Verfahren'); expect(normalized.deadlineAt?.toISOString()).toBe( '2026-09-01T10:00:00.000Z', ); expect(normalized.cpvCodes).toEqual([]); expect(normalized.cpvDivisions).toEqual([]); expect(normalized.region).toBeNull(); expect(normalized.plz).toBeNull(); expect(normalized.bundesland).toBeNull(); expect(normalized.estimatedValue).toBeNull(); expect(normalized.dedupKey).toBe(`${raw.sourcePortal}:${raw.sourceNoticeId}`); }); it('null/empty bag fields: buyerName/procedureType/deadlineAt stay null, title falls back', () => { const raw = bagRecord('ai-netserver', { title: '', buyerName: null, procedureType: null, legalFramework: null, deadlineAt: null, }); const normalized = service.normalize(raw); expect(normalized.title).toBe('Unbenannte Ausschreibung'); expect(normalized.buyerName).toBeNull(); expect(normalized.procedureType).toBeNull(); expect(normalized.deadlineAt).toBeNull(); }); it('contentHash for a bag source is a stable sha256 (proves shared assemble() reuse)', () => { const raw = bagRecord('ai-netserver', fullBag); const normalized = service.normalize(raw); expect(normalized.contentHash).toMatch(/^[a-f0-9]{64}$/); }); }); });