test(10-03): add failing specs + real DÖE fixtures for adapter/normalizer

- Capture real, trimmed DÖE day-export fixtures (pubDay=2026-07-19, 8 notices spanning tender/award/planning/untagged-with-awards tag classes) under tenders/__fixtures__/, live-downloaded from oeffentlichevergabe.de - Define RawTenderRecord/NormalizedTenderFields/SourceType (tender.types.ts) and TenderSourceAdapter (day-cursor fetchTenders signature per RESEARCH Pattern 1) - RED: doe-opendata.adapter.spec.ts asserts D-02 exact-count filtering, HTTP-400 no-op, and a zip-bomb ceiling guard (T-10-07) - RED: tender-normalizer.service.spec.ts asserts eForms-primary deadline recovery where OCDS is null (RESEARCH Pattern 3), nullable deadline/value, dedupKey priority (ocid -> sourcePortal:sourceNoticeId), and a stable sha256 contentHash
2026-07-21 10:56:13 +02:00
parent f80d491adc
commit f88e2a8141
6 changed files with 388 additions and 0 deletions
@@ -0,0 +1,139 @@
+import AdmZip from 'adm-zip';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { DoeOpenDataAdapter } from './doe-opendata.adapter';
+
+/**
+ * Real, trimmed DÖE day-export fixtures (8 notices, captured live from
+ * oeffentlichevergabe.de/api/notice-exports for pubDay=2026-07-19, trimmed
+ * to a representative sample spanning D-02's tag classes):
+ *   - 4x tag=["tender"]           -> must survive the D-02 filter
+ *   - 2x tag=["award"]            -> must be excluded
+ *   - 1x tag=["planning"]         -> must be excluded
+ *   - 1x no tag, populated awards -> must be excluded (Pitfall C)
+ */
+const FIXTURES_DIR = join(__dirname, '..', '__fixtures__');
+const EFORMS_FIXTURE = join(FIXTURES_DIR, 'doe-eforms-sample.zip');
+const OCDS_FIXTURE = join(FIXTURES_DIR, 'doe-ocds-sample.zip');
+const EXPECTED_TENDER_TAGGED_COUNT = 4;
+const TEST_PUBDAY = '2026-07-19';
+
+function stubFetchWithFixtures(): void {
+  const eformsBuffer = readFileSync(EFORMS_FIXTURE);
+  const ocdsBuffer = readFileSync(OCDS_FIXTURE);
+
+  vi.stubGlobal(
+    'fetch',
+    vi.fn(async (url: string) => {
+      const isEforms = url.includes('format=eforms.zip');
+      const buffer = isEforms ? eformsBuffer : ocdsBuffer;
+      return {
+        ok: true,
+        status: 200,
+        arrayBuffer: async () =>
+          buffer.buffer.slice(
+            buffer.byteOffset,
+            buffer.byteOffset + buffer.byteLength,
+          ),
+      } as Response;
+    }),
+  );
+}
+
+function stub400Fetch(): void {
+  vi.stubGlobal(
+    'fetch',
+    vi.fn(async () => {
+      return { ok: false, status: 400 } as Response;
+    }),
+  );
+}
+
+describe('DoeOpenDataAdapter', () => {
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it('has sourceType doe-opendata', () => {
+    const adapter = new DoeOpenDataAdapter();
+    expect(adapter.sourceType).toBe('doe-opendata');
+  });
+
+  it('parses the fixture ZIPs into RawTenderRecord[], D-02 filtered to only tag=["tender"] notices', async () => {
+    stubFetchWithFixtures();
+    const adapter = new DoeOpenDataAdapter();
+
+    const records = await adapter.fetchTenders(TEST_PUBDAY);
+
+    // (a) parsed records exist, (b) D-02 filter keeps only tag=["tender"],
+    // exact count assertion against the real fixture's known composition.
+    expect(records).toHaveLength(EXPECTED_TENDER_TAGGED_COUNT);
+    for (const record of records) {
+      expect(record.sourceType).toBe('doe-opendata');
+      expect(record.ocdsPayload).toBeTruthy();
+      expect(record.eformsPayload).toBeTruthy();
+    }
+  });
+
+  it('excludes award/planning/untagged-with-awards notices from the returned records', async () => {
+    stubFetchWithFixtures();
+    const adapter = new DoeOpenDataAdapter();
+
+    const records = await adapter.fetchTenders(TEST_PUBDAY);
+    const noticeIds = records.map((r) => r.sourceNoticeId);
+
+    // Known award-tagged notice ids from the fixture — must NOT appear.
+    expect(noticeIds).not.toContain('006803d3-5b37-4362-bea9-124a44de67cb');
+    expect(noticeIds).not.toContain('00a66578-d011-4a35-9628-eadd057228a0');
+    // Known planning-tagged notice id — must NOT appear.
+    expect(noticeIds).not.toContain('0891b60a-846a-47c0-8d90-11bbde87d560');
+    // Known untagged-with-populated-awards notice id — must NOT appear (Pitfall C).
+    expect(noticeIds).not.toContain('01726f63-0dbc-4456-b355-67082823199f');
+  });
+
+  it('returns [] without throwing when the DÖE endpoint responds 400 (today/future pubDay, expected no-op)', async () => {
+    stub400Fetch();
+    const adapter = new DoeOpenDataAdapter();
+
+    const records = await adapter.fetchTenders('2026-07-21');
+
+    expect(records).toEqual([]);
+  });
+
+  it('rejects an archive whose declared uncompressed size exceeds the decompression-bomb ceiling, before extracting entries (T-10-07)', async () => {
+    // Highly compressible synthetic archive: real (not corrupted) zip whose
+    // entries declare well over the ~50MB ceiling in their uncompressed
+    // size header, while the on-disk/compressed archive itself stays tiny.
+    const bomb = new AdmZip();
+    bomb.addFile('bomb.xml', Buffer.alloc(60 * 1024 * 1024, 0));
+    const bombBuffer = bomb.toBuffer();
+
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => {
+        return {
+          ok: true,
+          status: 200,
+          arrayBuffer: async () =>
+            bombBuffer.buffer.slice(
+              bombBuffer.byteOffset,
+              bombBuffer.byteOffset + bombBuffer.byteLength,
+            ),
+        } as Response;
+      }),
+    );
+
+    const adapter = new DoeOpenDataAdapter();
+
+    await expect(adapter.fetchTenders(TEST_PUBDAY)).rejects.toThrow();
+  });
+
+  it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => {
+    const source = readFileSync(
+      join(__dirname, 'doe-opendata.adapter.ts'),
+      'utf8',
+    );
+    expect(source).not.toMatch(/from ['"]axios['"]/);
+  });
+});
@@ -0,0 +1,29 @@
+import type { RawTenderRecord, SourceType } from '../tender.types';
+
+/**
+ * Contract implemented by every tender source adapter (DÖE OpenData this
+ * phase; AI-NetServer/cosinex/RSS/email-alert in later phases).
+ *
+ * `fetchTenders` takes a day-cursor string (`YYYY-MM-DD`), NOT a `since: Date`
+ * timestamp — the DÖE OpenData API is a daily batch-export API with no
+ * incremental/since parameter (RESEARCH.md Pattern 1 corrects the
+ * `ARCHITECTURE.md` sketch's `since?: Date` signature). Day-cursor is kept
+ * as the shared interface shape so future adapters that ARE incremental
+ * (e.g. RSS) can still satisfy it by treating the day as a coarse filter,
+ * rather than forking the interface per source.
+ */
+export interface TenderSourceAdapter {
+  readonly sourceType: SourceType;
+
+  /**
+   * Fetch and lightly parse all open-tender ("tender"-tagged", D-02)
+   * notices for the given calendar day (Europe/Berlin), already filtered —
+   * implementations must NOT return award/planning/untagged-with-awards
+   * notices.
+   *
+   * @param dayCursor - `YYYY-MM-DD`, must be strictly in the past (today/
+   * future is rejected upstream by the DÖE API with HTTP 400). Implementations
+   * treat that 400 as an expected no-op and resolve to `[]`, never throw.
+   */
+  fetchTenders(dayCursor: string): Promise<RawTenderRecord[]>;
+}
@@ -0,0 +1,146 @@
+import AdmZip from 'adm-zip';
+import { XMLParser } from 'fast-xml-parser';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { describe, expect, it } from 'vitest';
+import type { RawTenderRecord } from './tender.types';
+import { TenderNormalizerService } from './tender-normalizer.service';
+
+/**
+ * Loads a real notice pair (eForms-DE XML + OCDS JSON) directly from the
+ * committed fixture ZIPs and assembles it into a RawTenderRecord — the same
+ * real-fixture precedent as the DKV PDF parser, kept independent of
+ * DoeOpenDataAdapter so this unit test exercises TenderNormalizerService in
+ * isolation as a pure mapping function.
+ */
+const FIXTURES_DIR = join(__dirname, '__fixtures__');
+const xmlParser = new XMLParser({
+  removeNSPrefix: true,
+  ignoreAttributes: false,
+  attributeNamePrefix: '@_',
+});
+
+function loadRawNoticePair(basename: string): RawTenderRecord {
+  const eformsZip = new AdmZip(
+    readFileSync(join(FIXTURES_DIR, 'doe-eforms-sample.zip')),
+  );
+  const ocdsZip = new AdmZip(
+    readFileSync(join(FIXTURES_DIR, 'doe-ocds-sample.zip')),
+  );
+
+  const xmlEntry = eformsZip.getEntry(`${basename}.xml`);
+  const jsonEntry = ocdsZip.getEntry(`${basename}.json`);
+  if (!xmlEntry || !jsonEntry) {
+    throw new Error(`Fixture entry missing for ${basename}`);
+  }
+
+  const eformsPayload = xmlParser.parse(xmlEntry.getData().toString('utf8'));
+  const ocdsDocument = JSON.parse(jsonEntry.getData().toString('utf8'));
+  const release = ocdsDocument.releases[0];
+
+  return {
+    sourceType: 'doe-opendata',
+    sourcePortal: 'doe-opendata',
+    sourceNoticeId: release.id,
+    ocid: release.ocid,
+    sourceUrl: ocdsDocument.uri,
+    fetchedAt: new Date(),
+    publishedAt: ocdsDocument.publishedDate
+      ? new Date(ocdsDocument.publishedDate)
+      : null,
+    eformsPayload,
+    ocdsPayload: release,
+  };
+}
+
+// Real notice: eForms XML has a structured
+// ProcurementProjectLot/TenderingProcess/TenderSubmissionDeadlinePeriod/EndDate
+// (2026-08-18), while the paired OCDS release has no `tender.tenderPeriod` at
+// all (RESEARCH.md Pattern 3 live cross-check).
+const NOTICE_WITH_EFORMS_ONLY_DEADLINE =
+  '019f64ec-cd64-4c09-8fd7-4fd12d8eea64-01';
+
+// Real notice: neither the eForms XML nor the OCDS release carries a
+// deadline or estimated value (RESEARCH.md Pattern 4 — the common case).
+const NOTICE_WITH_NO_DEADLINE_OR_VALUE =
+  '001c68dd-258a-4bf7-a264-478604ad9c3e-01';
+
+describe('TenderNormalizerService', () => {
+  const service = new TenderNormalizerService();
+
+  it('recovers deadlineAt from eForms-DE XML when the OCDS release has no tenderPeriod (SCHEMA-01 key assertion)', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
+
+    const normalized = service.normalize(raw);
+
+    expect(normalized.deadlineAt).not.toBeNull();
+    expect(normalized.deadlineAt?.getUTCFullYear()).toBe(2026);
+    expect(normalized.deadlineAt?.getUTCMonth()).toBe(7); // August (0-indexed)
+    expect(normalized.deadlineAt?.getUTCDate()).toBe(18);
+  });
+
+  it('normalizes missing deadline/value to null, not thrown or zero', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_NO_DEADLINE_OR_VALUE);
+
+    const normalized = service.normalize(raw);
+
+    expect(normalized.deadlineAt).toBeNull();
+    expect(normalized.estimatedValue).toBeNull();
+    expect(normalized.title).toBeTruthy();
+  });
+
+  it('dedupKey = ocid when present', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
+
+    const normalized = service.normalize(raw);
+
+    expect(raw.ocid).toBeTruthy();
+    expect(normalized.dedupKey).toBe(raw.ocid);
+  });
+
+  it('dedupKey falls back to sourcePortal:sourceNoticeId when ocid is absent', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
+    const rawWithoutOcid: RawTenderRecord = { ...raw, ocid: undefined };
+
+    const normalized = service.normalize(rawWithoutOcid);
+
+    expect(normalized.dedupKey).toBe(
+      `${raw.sourcePortal}:${raw.sourceNoticeId}`,
+    );
+  });
+
+  it('contentHash is a stable sha256 over title+deadline+value+status', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
+
+    const first = service.normalize(raw);
+    const second = service.normalize(raw);
+
+    expect(first.contentHash).toBe(second.contentHash);
+    expect(first.contentHash).toMatch(/^[a-f0-9]{64}$/);
+  });
+
+  it('contentHash changes when the deadline changes', () => {
+    const raw = loadRawNoticePair(NOTICE_WITH_EFORMS_ONLY_DEADLINE);
+    const original = service.normalize(raw);
+
+    // Mutate the eForms deadline to simulate a Fristverlängerung (SCHEMA-02).
+    const mutatedEformsPayload = JSON.parse(JSON.stringify(raw.eformsPayload));
+    const root =
+      mutatedEformsPayload[Object.keys(mutatedEformsPayload)[0]];
+    const lots = Array.isArray(root.ProcurementProjectLot)
+      ? root.ProcurementProjectLot
+      : [root.ProcurementProjectLot];
+    for (const lot of lots) {
+      lot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate =
+        '2026-12-31+01:00';
+    }
+    const mutatedRaw: RawTenderRecord = {
+      ...raw,
+      eformsPayload: mutatedEformsPayload,
+    };
+
+    const mutated = service.normalize(mutatedRaw);
+
+    expect(mutated.contentHash).not.toBe(original.contentHash);
+  });
+});
@@ -0,0 +1,74 @@
+/**
+ * Shared types for the Ausschreibungs-Radar (tender ingestion) module.
+ *
+ * These interfaces are the single source of truth for the data shapes
+ * exchanged between source adapters, the normalizer, and (in Plan 04) the
+ * ingestion/upsert orchestrator.
+ */
+
+/**
+ * Identifies which upstream source a raw record originated from.
+ * Only 'doe-opendata' exists in this phase (Phase 10); scraping/RSS/email
+ * sources are added in later phases (13/14) without changing this contract.
+ */
+export type SourceType = 'doe-opendata';
+
+/**
+ * A single notice as fetched and lightly parsed from a source, before
+ * normalization. For the DÖE source this pairs the eForms-DE XML (primary
+ * structured-field source for deadline/value/procedureType, per RESEARCH
+ * Pattern 3) with the corresponding OCDS JSON release (primary source for
+ * `ocid`/buyer/party names).
+ */
+export interface RawTenderRecord {
+  sourceType: SourceType;
+  /** e.g. 'doe-opendata' */
+  sourcePortal: string;
+  /** Stable per-notice id (OCDS release.id — no version suffix), used for the dedupKey fallback tier. */
+  sourceNoticeId: string;
+  /** OCDS Open Contracting ID — present for DÖE, the primary dedup key when available. */
+  ocid?: string;
+  /** Canonical URL to view/re-fetch this notice, when the source exposes one. */
+  sourceUrl?: string;
+  /** When this record was fetched by the adapter (wall-clock, not notice publish date). */
+  fetchedAt: Date;
+  /** The notice's own published date, when the source exposes one; null if absent. */
+  publishedAt: Date | null;
+  /** Parsed eForms-DE XML document (fast-xml-parser output) for this notice, or null if unavailable/unpaired. */
+  eformsPayload: unknown;
+  /** Parsed OCDS release object (the single `releases[0]` entry) for this notice. */
+  ocdsPayload: unknown;
+}
+
+/**
+ * The unified, normalized field shape a `TenderSourceAdapter`'s raw records
+ * are mapped into by `TenderNormalizerService`. This is the Tender column
+ * subset (SCHEMA-01) plus the computed dedup/change-detection keys.
+ */
+export interface NormalizedTenderFields {
+  sourcePortal: string;
+  sourceNoticeId: string;
+  /** OCDS Open Contracting ID, when present; null otherwise. */
+  ocid: string | null;
+  /** Upsert target: ocid, else `${sourcePortal}:${sourceNoticeId}` (RESEARCH Pattern 2 priority order). */
+  dedupKey: string;
+  title: string;
+  buyerName: string | null;
+  cpvCodes: string[];
+  /** NUTS region code (e.g. "DEA41") from the buyer address — not yet mapped to a human Bundesland name. */
+  region: string | null;
+  plz: string | null;
+  /** Deferred: NUTS→Bundesland name mapping is a Phase 11 filter-UI concern, not this phase's scope. */
+  bundesland: string | null;
+  /** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */
+  deadlineAt: Date | null;
+  /** Frequently null (RESEARCH Pattern 4) — nullable is a correctness requirement, not optional hardening. */
+  estimatedValue: number | null;
+  procedureType: string | null;
+  /** Initial value is always 'active' at normalization time (SCHEMA-01). */
+  status: string;
+  sourceUrl: string | null;
+  /** sha256(title + deadlineAt + estimatedValue + status) — SCHEMA-02 change-detection hook. */
+  contentHash: string;
+  publishedAt: Date;
+}