diff --git a/apps/api/src/tenders/__fixtures__/doe-eforms-sample.zip b/apps/api/src/tenders/__fixtures__/doe-eforms-sample.zip new file mode 100644 index 0000000..7ff5f9e Binary files /dev/null and b/apps/api/src/tenders/__fixtures__/doe-eforms-sample.zip differ diff --git a/apps/api/src/tenders/__fixtures__/doe-ocds-sample.zip b/apps/api/src/tenders/__fixtures__/doe-ocds-sample.zip new file mode 100644 index 0000000..a1e3b9d Binary files /dev/null and b/apps/api/src/tenders/__fixtures__/doe-ocds-sample.zip differ diff --git a/apps/api/src/tenders/adapters/doe-opendata.adapter.spec.ts b/apps/api/src/tenders/adapters/doe-opendata.adapter.spec.ts new file mode 100644 index 0000000..e008e08 --- /dev/null +++ b/apps/api/src/tenders/adapters/doe-opendata.adapter.spec.ts @@ -0,0 +1,139 @@ +import AdmZip from 'adm-zip'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { DoeOpenDataAdapter } from './doe-opendata.adapter'; + +/** + * Real, trimmed DÖE day-export fixtures (8 notices, captured live from + * oeffentlichevergabe.de/api/notice-exports for pubDay=2026-07-19, trimmed + * to a representative sample spanning D-02's tag classes): + * - 4x tag=["tender"] -> must survive the D-02 filter + * - 2x tag=["award"] -> must be excluded + * - 1x tag=["planning"] -> must be excluded + * - 1x no tag, populated awards -> must be excluded (Pitfall C) + */ +const FIXTURES_DIR = join(__dirname, '..', '__fixtures__'); +const EFORMS_FIXTURE = join(FIXTURES_DIR, 'doe-eforms-sample.zip'); +const OCDS_FIXTURE = join(FIXTURES_DIR, 'doe-ocds-sample.zip'); +const EXPECTED_TENDER_TAGGED_COUNT = 4; +const TEST_PUBDAY = '2026-07-19'; + +function stubFetchWithFixtures(): void { + const eformsBuffer = readFileSync(EFORMS_FIXTURE); + const ocdsBuffer = readFileSync(OCDS_FIXTURE); + + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + const isEforms = url.includes('format=eforms.zip'); + const buffer = isEforms ? eformsBuffer : ocdsBuffer; + return { + ok: true, + status: 200, + arrayBuffer: async () => + buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength, + ), + } as Response; + }), + ); +} + +function stub400Fetch(): void { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return { ok: false, status: 400 } as Response; + }), + ); +} + +describe('DoeOpenDataAdapter', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('has sourceType doe-opendata', () => { + const adapter = new DoeOpenDataAdapter(); + expect(adapter.sourceType).toBe('doe-opendata'); + }); + + it('parses the fixture ZIPs into RawTenderRecord[], D-02 filtered to only tag=["tender"] notices', async () => { + stubFetchWithFixtures(); + const adapter = new DoeOpenDataAdapter(); + + const records = await adapter.fetchTenders(TEST_PUBDAY); + + // (a) parsed records exist, (b) D-02 filter keeps only tag=["tender"], + // exact count assertion against the real fixture's known composition. + expect(records).toHaveLength(EXPECTED_TENDER_TAGGED_COUNT); + for (const record of records) { + expect(record.sourceType).toBe('doe-opendata'); + expect(record.ocdsPayload).toBeTruthy(); + expect(record.eformsPayload).toBeTruthy(); + } + }); + + it('excludes award/planning/untagged-with-awards notices from the returned records', async () => { + stubFetchWithFixtures(); + const adapter = new DoeOpenDataAdapter(); + + const records = await adapter.fetchTenders(TEST_PUBDAY); + const noticeIds = records.map((r) => r.sourceNoticeId); + + // Known award-tagged notice ids from the fixture — must NOT appear. + expect(noticeIds).not.toContain('006803d3-5b37-4362-bea9-124a44de67cb'); + expect(noticeIds).not.toContain('00a66578-d011-4a35-9628-eadd057228a0'); + // Known planning-tagged notice id — must NOT appear. + expect(noticeIds).not.toContain('0891b60a-846a-47c0-8d90-11bbde87d560'); + // Known untagged-with-populated-awards notice id — must NOT appear (Pitfall C). + expect(noticeIds).not.toContain('01726f63-0dbc-4456-b355-67082823199f'); + }); + + it('returns [] without throwing when the DÖE endpoint responds 400 (today/future pubDay, expected no-op)', async () => { + stub400Fetch(); + const adapter = new DoeOpenDataAdapter(); + + const records = await adapter.fetchTenders('2026-07-21'); + + expect(records).toEqual([]); + }); + + it('rejects an archive whose declared uncompressed size exceeds the decompression-bomb ceiling, before extracting entries (T-10-07)', async () => { + // Highly compressible synthetic archive: real (not corrupted) zip whose + // entries declare well over the ~50MB ceiling in their uncompressed + // size header, while the on-disk/compressed archive itself stays tiny. + const bomb = new AdmZip(); + bomb.addFile('bomb.xml', Buffer.alloc(60 * 1024 * 1024, 0)); + const bombBuffer = bomb.toBuffer(); + + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return { + ok: true, + status: 200, + arrayBuffer: async () => + bombBuffer.buffer.slice( + bombBuffer.byteOffset, + bombBuffer.byteOffset + bombBuffer.byteLength, + ), + } as Response; + }), + ); + + const adapter = new DoeOpenDataAdapter(); + + await expect(adapter.fetchTenders(TEST_PUBDAY)).rejects.toThrow(); + }); + + it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => { + const source = readFileSync( + join(__dirname, 'doe-opendata.adapter.ts'), + 'utf8', + ); + expect(source).not.toMatch(/from ['"]axios['"]/); + }); +}); diff --git a/apps/api/src/tenders/adapters/tender-source-adapter.interface.ts b/apps/api/src/tenders/adapters/tender-source-adapter.interface.ts new file mode 100644 index 0000000..3065114 --- /dev/null +++ b/apps/api/src/tenders/adapters/tender-source-adapter.interface.ts @@ -0,0 +1,29 @@ +import type { RawTenderRecord, SourceType } from '../tender.types'; + +/** + * Contract implemented by every tender source adapter (DÖE OpenData this + * phase; AI-NetServer/cosinex/RSS/email-alert in later phases). + * + * `fetchTenders` takes a day-cursor string (`YYYY-MM-DD`), NOT a `since: Date` + * timestamp — the DÖE OpenData API is a daily batch-export API with no + * incremental/since parameter (RESEARCH.md Pattern 1 corrects the + * `ARCHITECTURE.md` sketch's `since?: Date` signature). Day-cursor is kept + * as the shared interface shape so future adapters that ARE incremental + * (e.g. RSS) can still satisfy it by treating the day as a coarse filter, + * rather than forking the interface per source. + */ +export interface TenderSourceAdapter { + readonly sourceType: SourceType; + + /** + * Fetch and lightly parse all open-tender ("tender"-tagged", D-02) + * notices for the given calendar day (Europe/Berlin), already filtered — + * implementations must NOT return award/planning/untagged-with-awards + * notices. + * + * @param dayCursor - `YYYY-MM-DD`, must be strictly in the past (today/ + * future is rejected upstream by the DÖE API with HTTP 400). Implementations + * treat that 400 as an expected no-op and resolve to `[]`, never throw. + */ + fetchTenders(dayCursor: string): Promise; +} diff --git a/apps/api/src/tenders/tender-normalizer.service.spec.ts b/apps/api/src/tenders/tender-normalizer.service.spec.ts new file mode 100644 index 0000000..a26ed7f --- /dev/null +++ b/apps/api/src/tenders/tender-normalizer.service.spec.ts @@ -0,0 +1,146 @@ +import AdmZip from 'adm-zip'; +import { XMLParser } from 'fast-xml-parser'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { describe, expect, it } from 'vitest'; +import type { RawTenderRecord } from './tender.types'; +import { TenderNormalizerService } from './tender-normalizer.service'; + +/** + * Loads a real notice pair (eForms-DE XML + OCDS JSON) directly from the + * committed fixture ZIPs and assembles it into a RawTenderRecord — the same + * real-fixture precedent as the DKV PDF parser, kept independent of + * DoeOpenDataAdapter so this unit test exercises TenderNormalizerService in + * isolation as a pure mapping function. + */ +const FIXTURES_DIR = join(__dirname, '__fixtures__'); +const xmlParser = new XMLParser({ + removeNSPrefix: true, + ignoreAttributes: false, + attributeNamePrefix: '@_', +}); + +function loadRawNoticePair(basename: string): RawTenderRecord { + const eformsZip = new AdmZip( + readFileSync(join(FIXTURES_DIR, 'doe-eforms-sample.zip')), + ); + const ocdsZip = new AdmZip( + readFileSync(join(FIXTURES_DIR, 'doe-ocds-sample.zip')), + ); + + const xmlEntry = eformsZip.getEntry(`${basename}.xml`); + const jsonEntry = ocdsZip.getEntry(`${basename}.json`); + if (!xmlEntry || !jsonEntry) { + throw new Error(`Fixture entry missing for ${basename}`); + } + + const eformsPayload = xmlParser.parse(xmlEntry.getData().toString('utf8')); + const ocdsDocument = JSON.parse(jsonEntry.getData().toString('utf8')); + const release = ocdsDocument.releases[0]; + + return { + sourceType: 'doe-opendata', + sourcePortal: 'doe-opendata', + sourceNoticeId: release.id, + ocid: release.ocid, + sourceUrl: ocdsDocument.uri, + fetchedAt: new Date(), + publishedAt: ocdsDocument.publishedDate + ? new Date(ocdsDocument.publishedDate) + : null, + eformsPayload, + ocdsPayload: release, + }; +} + +// Real notice: eForms XML has a structured +// ProcurementProjectLot/TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate +// (2026-08-18), while the paired OCDS release has no `tender.tenderPeriod` at +// all (RESEARCH.md Pattern 3 live cross-check). +const NOTICE_WITH_EFORMS_ONLY_DEADLINE = + '019f64ec-cd64-4c09-8fd7-4fd12d8eea64-01'; + +// Real notice: neither the eForms XML nor the OCDS release carries a +// deadline or estimated value (RESEARCH.md Pattern 4 — the common case). +const NOTICE_WITH_NO_DEADLINE_OR_VALUE = + '001c68dd-258a-4bf7-a264-478604ad9c3e-01'; + +describe('TenderNormalizerService', () => { + const service = new TenderNormalizerService(); + + it('recovers deadlineAt from eForms-DE XML when the OCDS release has no tenderPeriod (SCHEMA-01 key assertion)', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + + const normalized = service.normalize(raw); + + expect(normalized.deadlineAt).not.toBeNull(); + expect(normalized.deadlineAt?.getUTCFullYear()).toBe(2026); + expect(normalized.deadlineAt?.getUTCMonth()).toBe(7); // August (0-indexed) + expect(normalized.deadlineAt?.getUTCDate()).toBe(18); + }); + + it('normalizes missing deadline/value to null, not thrown or zero', () => { + const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE); + + const normalized = service.normalize(raw); + + expect(normalized.deadlineAt).toBeNull(); + expect(normalized.estimatedValue).toBeNull(); + expect(normalized.title).toBeTruthy(); + }); + + it('dedupKey = ocid when present', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + + const normalized = service.normalize(raw); + + expect(raw.ocid).toBeTruthy(); + expect(normalized.dedupKey).toBe(raw.ocid); + }); + + it('dedupKey falls back to sourcePortal:sourceNoticeId when ocid is absent', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + const rawWithoutOcid: RawTenderRecord = { ...raw, ocid: undefined }; + + const normalized = service.normalize(rawWithoutOcid); + + expect(normalized.dedupKey).toBe( + `${raw.sourcePortal}:${raw.sourceNoticeId}`, + ); + }); + + it('contentHash is a stable sha256 over title+deadline+value+status', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + + const first = service.normalize(raw); + const second = service.normalize(raw); + + expect(first.contentHash).toBe(second.contentHash); + expect(first.contentHash).toMatch(/^[a-f0-9]{64}$/); + }); + + it('contentHash changes when the deadline changes', () => { + const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE); + const original = service.normalize(raw); + + // Mutate the eForms deadline to simulate a Fristverlängerung (SCHEMA-02). + const mutatedEformsPayload = JSON.parse(JSON.stringify(raw.eformsPayload)); + const root = + mutatedEformsPayload[Object.keys(mutatedEformsPayload)[0]]; + const lots = Array.isArray(root.ProcurementProjectLot) + ? root.ProcurementProjectLot + : [root.ProcurementProjectLot]; + for (const lot of lots) { + lot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate = + '2026-12-31+01:00'; + } + const mutatedRaw: RawTenderRecord = { + ...raw, + eformsPayload: mutatedEformsPayload, + }; + + const mutated = service.normalize(mutatedRaw); + + expect(mutated.contentHash).not.toBe(original.contentHash); + }); +}); diff --git a/apps/api/src/tenders/tender.types.ts b/apps/api/src/tenders/tender.types.ts new file mode 100644 index 0000000..70e8b6a --- /dev/null +++ b/apps/api/src/tenders/tender.types.ts @@ -0,0 +1,74 @@ +/** + * Shared types for the Ausschreibungs-Radar (tender ingestion) module. + * + * These interfaces are the single source of truth for the data shapes + * exchanged between source adapters, the normalizer, and (in Plan 04) the + * ingestion/upsert orchestrator. + */ + +/** + * Identifies which upstream source a raw record originated from. + * Only 'doe-opendata' exists in this phase (Phase 10); scraping/RSS/email + * sources are added in later phases (13/14) without changing this contract. + */ +export type SourceType = 'doe-opendata'; + +/** + * A single notice as fetched and lightly parsed from a source, before + * normalization. For the DÖE source this pairs the eForms-DE XML (primary + * structured-field source for deadline/value/procedureType, per RESEARCH + * Pattern 3) with the corresponding OCDS JSON release (primary source for + * `ocid`/buyer/party names). + */ +export interface RawTenderRecord { + sourceType: SourceType; + /** e.g. 'doe-opendata' */ + sourcePortal: string; + /** Stable per-notice id (OCDS release.id — no version suffix), used for the dedupKey fallback tier. */ + sourceNoticeId: string; + /** OCDS Open Contracting ID — present for DÖE, the primary dedup key when available. */ + ocid?: string; + /** Canonical URL to view/re-fetch this notice, when the source exposes one. */ + sourceUrl?: string; + /** When this record was fetched by the adapter (wall-clock, not notice publish date). */ + fetchedAt: Date; + /** The notice's own published date, when the source exposes one; null if absent. */ + publishedAt: Date | null; + /** Parsed eForms-DE XML document (fast-xml-parser output) for this notice, or null if unavailable/unpaired. */ + eformsPayload: unknown; + /** Parsed OCDS release object (the single `releases[0]` entry) for this notice. */ + ocdsPayload: unknown; +} + +/** + * The unified, normalized field shape a `TenderSourceAdapter`'s raw records + * are mapped into by `TenderNormalizerService`. This is the Tender column + * subset (SCHEMA-01) plus the computed dedup/change-detection keys. + */ +export interface NormalizedTenderFields { + sourcePortal: string; + sourceNoticeId: string; + /** OCDS Open Contracting ID, when present; null otherwise. */ + ocid: string | null; + /** Upsert target: ocid, else `${sourcePortal}:${sourceNoticeId}` (RESEARCH Pattern 2 priority order). */ + dedupKey: string; + title: string; + buyerName: string | null; + cpvCodes: string[]; + /** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */ + region: string | null; + plz: string | null; + /** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */ + bundesland: string | null; + /** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */ + deadlineAt: Date | null; + /** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */ + estimatedValue: number | null; + procedureType: string | null; + /** Initial value is always 'active' at normalization time (SCHEMA-01). */ + status: string; + sourceUrl: string | null; + /** sha256(title + deadlineAt + estimatedValue + status) — SCHEMA-02 change-detection hook. */ + contentHash: string; + publishedAt: Date; +}