test(10-03): add failing specs + real DÖE fixtures for adapter/normalizer
- Capture real, trimmed DÖE day-export fixtures (pubDay=2026-07-19, 8 notices spanning tender/award/planning/untagged-with-awards tag classes) under tenders/__fixtures__/, live-downloaded from oeffentlichevergabe.de - Define RawTenderRecord/NormalizedTenderFields/SourceType (tender.types.ts) and TenderSourceAdapter (day-cursor fetchTenders signature per RESEARCH Pattern 1) - RED: doe-opendata.adapter.spec.ts asserts D-02 exact-count filtering, HTTP-400 no-op, and a zip-bomb ceiling guard (T-10-07) - RED: tender-normalizer.service.spec.ts asserts eForms-primary deadline recovery where OCDS is null (RESEARCH Pattern 3), nullable deadline/value, dedupKey priority (ocid -> sourcePortal:sourceNoticeId), and a stable sha256 contentHash
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
import AdmZip from 'adm-zip';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { DoeOpenDataAdapter } from './doe-opendata.adapter';
|
||||
|
||||
/**
|
||||
* Real, trimmed DÖE day-export fixtures (8 notices, captured live from
|
||||
* oeffentlichevergabe.de/api/notice-exports for pubDay=2026-07-19, trimmed
|
||||
* to a representative sample spanning D-02's tag classes):
|
||||
* - 4x tag=["tender"] -> must survive the D-02 filter
|
||||
* - 2x tag=["award"] -> must be excluded
|
||||
* - 1x tag=["planning"] -> must be excluded
|
||||
* - 1x no tag, populated awards -> must be excluded (Pitfall C)
|
||||
*/
|
||||
const FIXTURES_DIR = join(__dirname, '..', '__fixtures__');
|
||||
const EFORMS_FIXTURE = join(FIXTURES_DIR, 'doe-eforms-sample.zip');
|
||||
const OCDS_FIXTURE = join(FIXTURES_DIR, 'doe-ocds-sample.zip');
|
||||
const EXPECTED_TENDER_TAGGED_COUNT = 4;
|
||||
const TEST_PUBDAY = '2026-07-19';
|
||||
|
||||
function stubFetchWithFixtures(): void {
|
||||
const eformsBuffer = readFileSync(EFORMS_FIXTURE);
|
||||
const ocdsBuffer = readFileSync(OCDS_FIXTURE);
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async (url: string) => {
|
||||
const isEforms = url.includes('format=eforms.zip');
|
||||
const buffer = isEforms ? eformsBuffer : ocdsBuffer;
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () =>
|
||||
buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
),
|
||||
} as Response;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function stub400Fetch(): void {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async () => {
|
||||
return { ok: false, status: 400 } as Response;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe('DoeOpenDataAdapter', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('has sourceType doe-opendata', () => {
|
||||
const adapter = new DoeOpenDataAdapter();
|
||||
expect(adapter.sourceType).toBe('doe-opendata');
|
||||
});
|
||||
|
||||
it('parses the fixture ZIPs into RawTenderRecord[], D-02 filtered to only tag=["tender"] notices', async () => {
|
||||
stubFetchWithFixtures();
|
||||
const adapter = new DoeOpenDataAdapter();
|
||||
|
||||
const records = await adapter.fetchTenders(TEST_PUBDAY);
|
||||
|
||||
// (a) parsed records exist, (b) D-02 filter keeps only tag=["tender"],
|
||||
// exact count assertion against the real fixture's known composition.
|
||||
expect(records).toHaveLength(EXPECTED_TENDER_TAGGED_COUNT);
|
||||
for (const record of records) {
|
||||
expect(record.sourceType).toBe('doe-opendata');
|
||||
expect(record.ocdsPayload).toBeTruthy();
|
||||
expect(record.eformsPayload).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
it('excludes award/planning/untagged-with-awards notices from the returned records', async () => {
|
||||
stubFetchWithFixtures();
|
||||
const adapter = new DoeOpenDataAdapter();
|
||||
|
||||
const records = await adapter.fetchTenders(TEST_PUBDAY);
|
||||
const noticeIds = records.map((r) => r.sourceNoticeId);
|
||||
|
||||
// Known award-tagged notice ids from the fixture — must NOT appear.
|
||||
expect(noticeIds).not.toContain('006803d3-5b37-4362-bea9-124a44de67cb');
|
||||
expect(noticeIds).not.toContain('00a66578-d011-4a35-9628-eadd057228a0');
|
||||
// Known planning-tagged notice id — must NOT appear.
|
||||
expect(noticeIds).not.toContain('0891b60a-846a-47c0-8d90-11bbde87d560');
|
||||
// Known untagged-with-populated-awards notice id — must NOT appear (Pitfall C).
|
||||
expect(noticeIds).not.toContain('01726f63-0dbc-4456-b355-67082823199f');
|
||||
});
|
||||
|
||||
it('returns [] without throwing when the DÖE endpoint responds 400 (today/future pubDay, expected no-op)', async () => {
|
||||
stub400Fetch();
|
||||
const adapter = new DoeOpenDataAdapter();
|
||||
|
||||
const records = await adapter.fetchTenders('2026-07-21');
|
||||
|
||||
expect(records).toEqual([]);
|
||||
});
|
||||
|
||||
it('rejects an archive whose declared uncompressed size exceeds the decompression-bomb ceiling, before extracting entries (T-10-07)', async () => {
|
||||
// Highly compressible synthetic archive: real (not corrupted) zip whose
|
||||
// entries declare well over the ~50MB ceiling in their uncompressed
|
||||
// size header, while the on-disk/compressed archive itself stays tiny.
|
||||
const bomb = new AdmZip();
|
||||
bomb.addFile('bomb.xml', Buffer.alloc(60 * 1024 * 1024, 0));
|
||||
const bombBuffer = bomb.toBuffer();
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async () => {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () =>
|
||||
bombBuffer.buffer.slice(
|
||||
bombBuffer.byteOffset,
|
||||
bombBuffer.byteOffset + bombBuffer.byteLength,
|
||||
),
|
||||
} as Response;
|
||||
}),
|
||||
);
|
||||
|
||||
const adapter = new DoeOpenDataAdapter();
|
||||
|
||||
await expect(adapter.fetchTenders(TEST_PUBDAY)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => {
|
||||
const source = readFileSync(
|
||||
join(__dirname, 'doe-opendata.adapter.ts'),
|
||||
'utf8',
|
||||
);
|
||||
expect(source).not.toMatch(/from ['"]axios['"]/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { RawTenderRecord, SourceType } from '../tender.types';
|
||||
|
||||
/**
|
||||
* Contract implemented by every tender source adapter (DÖE OpenData this
|
||||
* phase; AI-NetServer/cosinex/RSS/email-alert in later phases).
|
||||
*
|
||||
* `fetchTenders` takes a day-cursor string (`YYYY-MM-DD`), NOT a `since: Date`
|
||||
* timestamp — the DÖE OpenData API is a daily batch-export API with no
|
||||
* incremental/since parameter (RESEARCH.md Pattern 1 corrects the
|
||||
* `ARCHITECTURE.md` sketch's `since?: Date` signature). Day-cursor is kept
|
||||
* as the shared interface shape so future adapters that ARE incremental
|
||||
* (e.g. RSS) can still satisfy it by treating the day as a coarse filter,
|
||||
* rather than forking the interface per source.
|
||||
*/
|
||||
export interface TenderSourceAdapter {
|
||||
readonly sourceType: SourceType;
|
||||
|
||||
/**
|
||||
* Fetch and lightly parse all open-tender ("tender"-tagged", D-02)
|
||||
* notices for the given calendar day (Europe/Berlin), already filtered —
|
||||
* implementations must NOT return award/planning/untagged-with-awards
|
||||
* notices.
|
||||
*
|
||||
* @param dayCursor - `YYYY-MM-DD`, must be strictly in the past (today/
|
||||
* future is rejected upstream by the DÖE API with HTTP 400). Implementations
|
||||
* treat that 400 as an expected no-op and resolve to `[]`, never throw.
|
||||
*/
|
||||
fetchTenders(dayCursor: string): Promise<RawTenderRecord[]>;
|
||||
}
|
||||
Reference in New Issue
Block a user