feat(13-05): add cosinex/DTVP source adapter with fixture tests
Separate HTML adapter for the cosinex Vergabemarktplatz (DTVP) satellite
(sourceType='cosinex-dtvp'), distinct from the NetServer adapter since
cosinex markup differs structurally. Live inspection (2026-07-23) found
the "Aktuelle Bekanntmachungen" results table is fully server-rendered
(not JS-dependent as D-01 anticipated), so selectors are fully populated
rather than falling back to a needs-JS stub — parses publish date,
deadline (or "nv"), title, legal framework/procedure type, buyer name,
and a real per-notice deep link (pid) into RawTenderRecord[].
Rule 1 fix: cosinex serves charset=ISO-8859-1 with raw Latin-1 bytes for
umlauts (not HTML entities); Response.text() always UTF-8-decodes per
the Fetch spec, so the adapter reads arrayBuffer() and decodes explicitly
via TextDecoder('iso-8859-1') to avoid mojibake.
16 spec tests pass against a live-captured fixture (20 rows, transcoded
to UTF-8 on disk): full-fixture parse, deadline/publish date parsing,
nested-<abbr> procedure-type extraction, umlaut decoding, empty/broken
HTML and missing-pid row fallback, fetch-throw/non-2xx fallback, no-axios
and no-input-interpolated-URL guards.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,271 @@
|
|||||||
|
import { readFileSync } from 'fs';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { CosinexAdapter } from './cosinex.adapter';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Real, live-captured cosinex/DTVP "Aktuelle Bekanntmachungen" results
|
||||||
|
* table (20 rows, from a live GET against
|
||||||
|
* `https://www.dtvp.de/Satellite/company/welcome.do`, 2026-07-23), stored
|
||||||
|
* as plain UTF-8 on disk — the live response itself is
|
||||||
|
* `charset=ISO-8859-1` (raw Latin-1 bytes, not HTML entities), which is
|
||||||
|
* exercised separately below via the encoding-conversion tests, so the
|
||||||
|
* on-disk fixture is transcoded to UTF-8 for a simple `readFileSync(...,
|
||||||
|
* 'utf8')` (matching netserver-search.html's convention).
|
||||||
|
*/
|
||||||
|
const FIXTURE_PATH = join(
|
||||||
|
__dirname,
|
||||||
|
'..',
|
||||||
|
'__fixtures__',
|
||||||
|
'cosinex-search.html',
|
||||||
|
);
|
||||||
|
const FIXTURE_HTML = readFileSync(FIXTURE_PATH, 'utf8');
|
||||||
|
const FIXTURE_ROW_COUNT = 20;
|
||||||
|
|
||||||
|
/** Encodes `html` as Latin-1 bytes and stubs `fetch` to resolve with them via `arrayBuffer()` — mirrors the adapter's real decode path (`TextDecoder('iso-8859-1')`). */
|
||||||
|
function stubFetchWithHtml(html: string): void {
|
||||||
|
const bytes = Buffer.from(html, 'latin1');
|
||||||
|
const arrayBuffer = bytes.buffer.slice(
|
||||||
|
bytes.byteOffset,
|
||||||
|
bytes.byteOffset + bytes.byteLength,
|
||||||
|
);
|
||||||
|
vi.stubGlobal(
|
||||||
|
'fetch',
|
||||||
|
vi.fn(async () => {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
arrayBuffer: async () => arrayBuffer,
|
||||||
|
} as unknown as Response;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stubFetchThrowing(): void {
|
||||||
|
vi.stubGlobal(
|
||||||
|
'fetch',
|
||||||
|
vi.fn(async () => {
|
||||||
|
throw new Error('network unreachable');
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('CosinexAdapter', () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('declares sourceType cosinex-dtvp and the single cosinex-dtvp portal', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
expect(adapter.sourceType).toBe('cosinex-dtvp');
|
||||||
|
expect(adapter.portals).toEqual(['cosinex-dtvp']);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('parseSearchResults (pure, fixture-driven)', () => {
|
||||||
|
it('parses all rows of the fixture into RawTenderRecord[], sourceType/sourcePortal set per row', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const fetchedAt = new Date();
|
||||||
|
|
||||||
|
const records = adapter.parseSearchResults(FIXTURE_HTML, fetchedAt);
|
||||||
|
|
||||||
|
expect(records.length).toBeGreaterThanOrEqual(1);
|
||||||
|
expect(records).toHaveLength(FIXTURE_ROW_COUNT);
|
||||||
|
for (const record of records) {
|
||||||
|
expect(record.sourceType).toBe('cosinex-dtvp');
|
||||||
|
expect(record.sourcePortal).toBe('cosinex-dtvp');
|
||||||
|
expect(record.sourceUrl).toMatch(
|
||||||
|
/^https:\/\/www\.dtvp\.de\/Satellite\/public\/company\/projectForwarding\.do\?pid=\d+$/,
|
||||||
|
);
|
||||||
|
expect(record.fetchedAt).toBe(fetchedAt);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the row detail-link pid as sourceNoticeId, unique per row', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const records = adapter.parseSearchResults(FIXTURE_HTML, new Date());
|
||||||
|
|
||||||
|
const ids = records.map((r) => r.sourceNoticeId);
|
||||||
|
expect(ids).toContain('335929');
|
||||||
|
expect(ids).toContain('335920');
|
||||||
|
expect(new Set(ids).size).toBe(ids.length);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses a populated publishedAt and an "nv" (nicht vorhanden) deadline to null', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const records = adapter.parseSearchResults(FIXTURE_HTML, new Date());
|
||||||
|
|
||||||
|
const row = records.find((r) => r.sourceNoticeId === '335929');
|
||||||
|
expect(row).toBeDefined();
|
||||||
|
expect(row!.publishedAt).toEqual(new Date('2026-07-23T09:07:00.000Z'));
|
||||||
|
expect(row!.sourceUrl).toBe(
|
||||||
|
'https://www.dtvp.de/Satellite/public/company/projectForwarding.do?pid=335929',
|
||||||
|
);
|
||||||
|
|
||||||
|
const payload = row!.ocdsPayload as {
|
||||||
|
title: string;
|
||||||
|
buyerName: string | null;
|
||||||
|
legalFramework: string | null;
|
||||||
|
procedureType: string | null;
|
||||||
|
deadlineAt: string | null;
|
||||||
|
};
|
||||||
|
expect(payload.deadlineAt).toBeNull();
|
||||||
|
expect(payload.buyerName).toBe('Stadt Wolfsburg');
|
||||||
|
expect(payload.legalFramework).toBe('UVgO');
|
||||||
|
expect(payload.procedureType).toBe('Vergebener Auftrag');
|
||||||
|
expect(payload.title).toMatch(/^26-0311 Beschaffung Ultraschallsysteme/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses a populated deadline row to a non-null ISO deadlineAt', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const records = adapter.parseSearchResults(FIXTURE_HTML, new Date());
|
||||||
|
|
||||||
|
const row = records.find((r) => r.sourceNoticeId === '335920');
|
||||||
|
expect(row).toBeDefined();
|
||||||
|
const payload = row!.ocdsPayload as { deadlineAt: string | null };
|
||||||
|
expect(payload.deadlineAt).toBe('2026-08-12T13:30:00.000Z');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extracts a nested-<abbr> procedure-type abbreviation (e.g. "TNW") via text, not HTML', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const records = adapter.parseSearchResults(FIXTURE_HTML, new Date());
|
||||||
|
|
||||||
|
const row = records.find((r) => r.sourceNoticeId === '335752');
|
||||||
|
expect(row).toBeDefined();
|
||||||
|
const payload = row!.ocdsPayload as {
|
||||||
|
legalFramework: string | null;
|
||||||
|
procedureType: string | null;
|
||||||
|
};
|
||||||
|
expect(payload.legalFramework).toBe('VgV');
|
||||||
|
expect(payload.procedureType).toBe('TNW');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('decodes a real umlaut buyer name correctly (Münster AöR)', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const records = adapter.parseSearchResults(FIXTURE_HTML, new Date());
|
||||||
|
|
||||||
|
const row = records.find((r) => r.sourceNoticeId === '328078');
|
||||||
|
expect(row).toBeDefined();
|
||||||
|
const payload = row!.ocdsPayload as { buyerName: string | null };
|
||||||
|
expect(payload.buyerName).toBe('Studierendenwerk Münster AöR');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns [] for empty HTML instead of throwing', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
expect(adapter.parseSearchResults('', new Date())).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns [] for HTML with no matching result table instead of throwing (e.g. future JS-rendered redesign)', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const broken = '<html><body><p>Keine Treffer</p></body></html>';
|
||||||
|
expect(adapter.parseSearchResults(broken, new Date())).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips a row without a pid-bearing detail link without throwing or affecting other rows', () => {
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
const html = `
|
||||||
|
<table class="csx-new-table">
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td class="text-center"><abbr title="22.07.2026 um 10:00 Uhr">22.07.2026</abbr></td>
|
||||||
|
<td class="text-center"><abbr title="nicht vorhanden">nv</abbr></td>
|
||||||
|
<td class="word-break">Ohne PID</td>
|
||||||
|
<td>VOB/A<br />Ausschreibung</td>
|
||||||
|
<td>Test-Vergabestelle</td>
|
||||||
|
<td class="text-center"></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="text-center"><abbr title="22.07.2026 um 10:00 Uhr">22.07.2026</abbr></td>
|
||||||
|
<td class="text-center"><abbr title="nicht vorhanden">nv</abbr></td>
|
||||||
|
<td class="word-break">Mit PID</td>
|
||||||
|
<td>VOB/A<br />Ausschreibung</td>
|
||||||
|
<td>Test-Vergabestelle 2</td>
|
||||||
|
<td class="text-center"><a href="/Satellite/public/company/projectForwarding.do?pid=999">Projektraum</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
`;
|
||||||
|
const records = adapter.parseSearchResults(html, new Date());
|
||||||
|
expect(records).toHaveLength(1);
|
||||||
|
expect(records[0]?.sourceNoticeId).toBe('999');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('fetchTenders (single-portal fetch, ISO-8859-1 decoding, fetch mocked)', () => {
|
||||||
|
it('fetches and parses the listing, decoding the real fixture bytes correctly', async () => {
|
||||||
|
stubFetchWithHtml(FIXTURE_HTML);
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
|
||||||
|
const records = await adapter.fetchTenders('2026-07-23');
|
||||||
|
|
||||||
|
expect(records).toHaveLength(FIXTURE_ROW_COUNT);
|
||||||
|
expect(records.every((r) => r.sourcePortal === 'cosinex-dtvp')).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
const umlautRow = records.find((r) => r.sourceNoticeId === '328078');
|
||||||
|
const payload = umlautRow!.ocdsPayload as { buyerName: string | null };
|
||||||
|
expect(payload.buyerName).toBe('Studierendenwerk Münster AöR');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('decodes raw Latin-1 umlaut bytes correctly (Rule 1 fix: arrayBuffer + TextDecoder(iso-8859-1), NOT res.text())', async () => {
|
||||||
|
// Raw Latin-1 byte 0xFC for 'ü' (as cosinex actually serves it, NOT
|
||||||
|
// an ü HTML entity) — Response.text() would UTF-8-decode this
|
||||||
|
// incorrectly per the WHATWG Fetch spec.
|
||||||
|
const html = `
|
||||||
|
<table class="csx-new-table">
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td class="text-center"><abbr title="22.07.2026 um 10:00 Uhr">22.07.2026</abbr></td>
|
||||||
|
<td class="text-center"><abbr title="nicht vorhanden">nv</abbr></td>
|
||||||
|
<td class="word-break">Müller-Test</td>
|
||||||
|
<td>VOB/A<br />Ausschreibung</td>
|
||||||
|
<td>Landkreis München</td>
|
||||||
|
<td class="text-center"><a href="/Satellite/public/company/projectForwarding.do?pid=42">Projektraum</a></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
`;
|
||||||
|
stubFetchWithHtml(html);
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
|
||||||
|
const records = await adapter.fetchTenders('2026-07-23');
|
||||||
|
|
||||||
|
expect(records).toHaveLength(1);
|
||||||
|
const payload = records[0]!.ocdsPayload as { buyerName: string | null };
|
||||||
|
expect(payload.buyerName).toBe('Landkreis München');
|
||||||
|
expect(records[0]!.ocdsPayload).toMatchObject({ title: 'Müller-Test' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns [] when fetch throws, without propagating (D-01 total-failure fallback)', async () => {
|
||||||
|
stubFetchThrowing();
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
|
||||||
|
const records = await adapter.fetchTenders('2026-07-23');
|
||||||
|
|
||||||
|
expect(records).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns [] without throwing when the listing responds non-2xx', async () => {
|
||||||
|
vi.stubGlobal(
|
||||||
|
'fetch',
|
||||||
|
vi.fn(async () => {
|
||||||
|
return { ok: false, status: 503 } as Response;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const adapter = new CosinexAdapter();
|
||||||
|
|
||||||
|
const records = await adapter.fetchTenders('2026-07-23');
|
||||||
|
|
||||||
|
expect(records).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => {
|
||||||
|
const source = readFileSync(join(__dirname, 'cosinex.adapter.ts'), 'utf8');
|
||||||
|
expect(source).not.toMatch(/from ['"]axios['"]/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('never interpolates the base URL from a variable/parameter (SSRF guard, T-13-05-01)', () => {
|
||||||
|
const source = readFileSync(join(__dirname, 'cosinex.adapter.ts'), 'utf8');
|
||||||
|
expect(source).toMatch(/COSINEX_BASE_URL = 'https:\/\/www\.dtvp\.de'/);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import * as cheerio from 'cheerio';
|
||||||
|
import type { RawTenderRecord, SourceType } from '../tender.types';
|
||||||
|
import type { TenderSourceAdapter } from './tender-source-adapter.interface';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CosinexAdapter — SEPARATE HTML adapter (INGEST-03) for the cosinex
|
||||||
|
* "Deutsches Vergabeportal" (DTVP) satellite. Deliberately NOT a
|
||||||
|
* NetServerAdapter reuse: cosinex-HTML is structurally different from
|
||||||
|
* AI-AG's NetServer markup (own table class, own column layout, own
|
||||||
|
* per-notice deep-link scheme) — see 13-RESEARCH.md/13-05-PLAN.md.
|
||||||
|
*
|
||||||
|
* Live verified (2026-07-23): `https://www.dtvp.de/Satellite/company/welcome.do`
|
||||||
|
* -> HTTP 200, ~40 KB, publicly reachable without auth. Open Question 1
|
||||||
|
* (13-RESEARCH.md) resolved by live inspection: the "Aktuelle
|
||||||
|
* Bekanntmachungen" results table is FULLY server-rendered (a plain
|
||||||
|
* `<table class="csx-new-table">` with real `<tr>` rows and a real
|
||||||
|
* `pid=`-bearing deep link per row) — NOT JS-dependent. This is a better
|
||||||
|
* outcome than the plan's best-effort D-01 fallback: no "needs-JS,
|
||||||
|
* deferred" stub was needed; selectors below are fully populated.
|
||||||
|
*
|
||||||
|
* Security notes (threat model T-13-05-01/02/03):
|
||||||
|
* - Base URL is a hardcoded constant (COSINEX_BASE_URL), NEVER interpolated
|
||||||
|
* from user/admin input (SSRF guard, T-10-06 pattern).
|
||||||
|
* - Native fetch + AbortController 15s timeout — no axios (project-wide
|
||||||
|
* convention, DoeOpenDataAdapter/NetServerAdapter idiom).
|
||||||
|
* - Per-row try/catch (Pitfall 2, D-01): a single malformed row is skipped
|
||||||
|
* (logger.warn), never aborts the whole parse. A totally unparsable page
|
||||||
|
* (cheerio.load throws) or a fetch failure resolves to `[]` instead of
|
||||||
|
* throwing — pollDueSources' catch-per-source (13-03) is a second,
|
||||||
|
* coarser safety net, but the adapter itself must never take down the
|
||||||
|
* rest of a fan-out tick.
|
||||||
|
* - Extracted title/buyerName/type fields are plain text (cheerio
|
||||||
|
* `.text()`, never `.html()`) — no raw HTML is ever carried into
|
||||||
|
* RawTenderRecord/Tender, preventing stored-XSS via portal markup (V5).
|
||||||
|
*/
|
||||||
|
const COSINEX_FETCH_TIMEOUT_MS = 15_000;
|
||||||
|
const COSINEX_BASE_URL = 'https://www.dtvp.de';
|
||||||
|
const LISTING_PATH = '/Satellite/company/welcome.do';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class CosinexAdapter implements TenderSourceAdapter {
|
||||||
|
readonly sourceType: SourceType = 'cosinex-dtvp';
|
||||||
|
readonly portals = ['cosinex-dtvp'] as const;
|
||||||
|
|
||||||
|
private readonly logger = new Logger(CosinexAdapter.name);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* dayCursor is accepted for interface conformance but NOT used as a
|
||||||
|
* query filter — like NetServerAdapter, the cosinex "Aktuelle
|
||||||
|
* Bekanntmachungen" listing has no documented incremental/date-range
|
||||||
|
* parameter; it is a "most recent first" paginated table (page 1 of
|
||||||
|
* ~331 live, 2026-07-23). First-page-only fetch is accepted as
|
||||||
|
* sufficient for the MVP proof (13-RESEARCH.md Open Question 1);
|
||||||
|
* pagination is out of scope for this plan.
|
||||||
|
*/
|
||||||
|
async fetchTenders(_dayCursor: string): Promise<RawTenderRecord[]> {
|
||||||
|
const fetchedAt = new Date();
|
||||||
|
|
||||||
|
let html: string;
|
||||||
|
try {
|
||||||
|
html = await this.fetchListingHtml();
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`cosinex/DTVP listing fetch failed, skipping this tick: ${(error as Error).message}`,
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.parseSearchResults(html, fetchedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* cosinex/DTVP serves `Content-Type: text/html;charset=ISO-8859-1` (live
|
||||||
|
* verified, 2026-07-23 response header) with RAW Latin-1 bytes for
|
||||||
|
* umlauts (not `ä`-style HTML entities, unlike NetServer/13-04).
|
||||||
|
* Per the WHATWG Fetch spec, `Response.text()` unconditionally UTF-8
|
||||||
|
* decodes the body regardless of the declared charset — using it here
|
||||||
|
* would silently mojibake every umlaut buyer/title. Rule 1 fix: read as
|
||||||
|
* `arrayBuffer()` and decode explicitly with `TextDecoder('iso-8859-1')`.
|
||||||
|
*/
|
||||||
|
private async fetchListingHtml(): Promise<string> {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeout = setTimeout(
|
||||||
|
() => controller.abort(),
|
||||||
|
COSINEX_FETCH_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${COSINEX_BASE_URL}${LISTING_PATH}`, {
|
||||||
|
signal: controller.signal,
|
||||||
|
redirect: 'follow',
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`cosinex/DTVP fetch failed: HTTP ${res.status}`);
|
||||||
|
}
|
||||||
|
const buffer = await res.arrayBuffer();
|
||||||
|
return new TextDecoder('iso-8859-1').decode(buffer);
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses the "Aktuelle Bekanntmachungen" `<table class="csx-new-table">`
|
||||||
|
* into RawTenderRecord[]. Tolerant by construction (Pitfall 2, D-01): a
|
||||||
|
* single row that throws during extraction is skipped (logger.warn)
|
||||||
|
* rather than aborting the whole parse; if the page itself is
|
||||||
|
* fundamentally unparsable (cheerio.load throws), or the table/rows are
|
||||||
|
* absent (e.g. a future JS-rendered redesign), this returns `[]`.
|
||||||
|
*/
|
||||||
|
parseSearchResults(html: string, fetchedAt: Date): RawTenderRecord[] {
|
||||||
|
const records: RawTenderRecord[] = [];
|
||||||
|
|
||||||
|
let $: cheerio.CheerioAPI;
|
||||||
|
try {
|
||||||
|
$ = cheerio.load(html);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`cosinex/DTVP HTML totally unparsable, returning [] (D-01): ${(error as Error).message}`,
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
$('table.csx-new-table tbody tr').each((_, el) => {
|
||||||
|
try {
|
||||||
|
const row = $(el);
|
||||||
|
const cells = row.find('td');
|
||||||
|
if (cells.length < 6) return; // e.g. empty-state/colspan row, not a result row
|
||||||
|
|
||||||
|
const actionCell = $(cells.get(5));
|
||||||
|
const detailHref = actionCell.find('a[href*="pid="]').attr('href');
|
||||||
|
const pidMatch = detailHref?.match(/[?&]pid=(\d+)/);
|
||||||
|
if (!pidMatch) return; // no stable id -> unusable row, skip (Pitfall 2)
|
||||||
|
const pid = pidMatch[1];
|
||||||
|
|
||||||
|
const publishedAt = parseCosinexDateTime(
|
||||||
|
$(cells.get(0)).find('abbr').attr('title') ?? '',
|
||||||
|
);
|
||||||
|
const deadlineAt = parseCosinexDateTime(
|
||||||
|
$(cells.get(1)).find('abbr').attr('title') ?? '',
|
||||||
|
);
|
||||||
|
const title =
|
||||||
|
$(cells.get(2)).text().trim() || 'Unbenannte Ausschreibung';
|
||||||
|
const [legalFramework = null, procedureType = null] = splitByBr(
|
||||||
|
$,
|
||||||
|
$(cells.get(3)),
|
||||||
|
);
|
||||||
|
const buyerName = $(cells.get(4)).text().trim() || null;
|
||||||
|
|
||||||
|
records.push({
|
||||||
|
sourceType: this.sourceType,
|
||||||
|
sourcePortal: 'cosinex-dtvp',
|
||||||
|
sourceNoticeId: pid,
|
||||||
|
sourceUrl: `${COSINEX_BASE_URL}${detailHref}`,
|
||||||
|
fetchedAt,
|
||||||
|
publishedAt,
|
||||||
|
eformsPayload: null,
|
||||||
|
// cosinex/DTVP has no eForms/OCDS structure; the extracted table
|
||||||
|
// fields are carried through this generic bag — same convention
|
||||||
|
// as NetServerAdapter (13-04) — so a future normalizer extension
|
||||||
|
// (out of scope for this plan) can map them without re-parsing
|
||||||
|
// HTML.
|
||||||
|
ocdsPayload: {
|
||||||
|
title,
|
||||||
|
buyerName,
|
||||||
|
procedureType,
|
||||||
|
legalFramework,
|
||||||
|
deadlineAt: deadlineAt ? deadlineAt.toISOString() : null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
`Skipping unparsable cosinex/DTVP row: ${(error as Error).message}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return records;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses cosinex's `dd.MM.yyyy um HH:mm Uhr` `<abbr title="...">` datetime
|
||||||
|
* (used for both the "Veröffentlicht" and "Angebots-/Teilnahmefrist"
|
||||||
|
* columns). Returns null for absent/malformed input — the deadline column
|
||||||
|
* frequently holds an `<abbr title="...">nv</abbr>` ("nicht vorhanden")
|
||||||
|
* reason string instead of a date, which never matches this pattern and
|
||||||
|
* therefore correctly falls through to null (matches the project-wide
|
||||||
|
* "deadline is often null" convention, RESEARCH.md Pattern 4).
|
||||||
|
*/
|
||||||
|
function parseCosinexDateTime(raw: string): Date | null {
|
||||||
|
const match = raw.match(
|
||||||
|
/^(\d{2})\.(\d{2})\.(\d{4})\s+um\s+(\d{2}):(\d{2})\s+Uhr$/,
|
||||||
|
);
|
||||||
|
if (!match) return null;
|
||||||
|
const [, dd, mm, yyyy, hh, min] = match;
|
||||||
|
const date = new Date(
|
||||||
|
Date.UTC(Number(yyyy), Number(mm) - 1, Number(dd), Number(hh), Number(min)),
|
||||||
|
);
|
||||||
|
return Number.isNaN(date.getTime()) ? null : date;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Splits a table cell's contents on `<br>` boundaries into trimmed text
|
||||||
|
* parts (e.g. cosinex's "Typ" column: `UVgO<br />Vergebener Auftrag` ->
|
||||||
|
* `['UVgO', 'Vergebener Auftrag']`). Nested tags (e.g. an `<abbr
|
||||||
|
* title="Teilnahmewettbewerb">TNW</abbr>` for procedure-type
|
||||||
|
* abbreviations) are text-extracted via cheerio `.text()`, never HTML
|
||||||
|
* (V5) — walks child nodes directly rather than `cell.html().split()` to
|
||||||
|
* avoid re-parsing entity-encoded fragments out of context.
|
||||||
|
*/
|
||||||
|
function splitByBr(
|
||||||
|
$: cheerio.CheerioAPI,
|
||||||
|
cell: ReturnType<cheerio.CheerioAPI>,
|
||||||
|
): string[] {
|
||||||
|
const parts: string[] = [];
|
||||||
|
let current = '';
|
||||||
|
|
||||||
|
cell.contents().each((_, node) => {
|
||||||
|
if (node.type === 'text') {
|
||||||
|
current += (node as unknown as { data: string }).data ?? '';
|
||||||
|
} else if (node.type === 'tag' && (node as unknown as { name: string }).name === 'br') {
|
||||||
|
parts.push(current.trim());
|
||||||
|
current = '';
|
||||||
|
} else {
|
||||||
|
current += $(node).text();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
parts.push(current.trim());
|
||||||
|
|
||||||
|
return parts.filter(Boolean);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user