feat(14-02): add RssAdapter.parseFeed + 'rss' normalizer dispatch
Fixture-first RSS parsing (INGEST-04): parses live-captured
service.bund.de (pubDate present, numeric-HTML-entity titles) and
subreport-elvis (pubDate absent, CDATA titles) feed shapes into
RawTenderRecord[] via fast-xml-parser, mirroring the DoeOpenDataAdapter
config. SourceType extended with 'rss'; normalize() dispatches 'rss'
through the existing normalizeBag() path unchanged (D-04/D-05).
Rule 1 fix: fast-xml-parser only decodes the 5 predefined XML entities,
not numeric character references — added an explicit decode step so
service.bund.de titles ("Übermittlung...") render correctly
instead of leaking raw entity syntax.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>service.bund.de - öffentliche Ausschreibungen</title>
|
||||
<link>http://www.service.bund.de/ausschreibungen</link>
|
||||
<description>RSS-Feed mit aktuellen Ausschreibungen der Vergabestellen der Bundes-, Landes- und Kommunalverwaltung.</description>
|
||||
<language>de-de</language>
|
||||
<pubDate>Thu, 23 Jul 2026 13:06:44 +0200</pubDate>
|
||||
<lastBuildDate>Thu, 23 Jul 2026 13:06:44 +0200</lastBuildDate>
|
||||
<managingEditor>redaktion-support@bva.bund.de</managingEditor>
|
||||
<webMaster>redaktion-support@bva.bund.de</webMaster>
|
||||
<generator>Government Site Builder (GSB7.0)</generator>
|
||||
<image>
|
||||
<url>http://www.service.bund.de/cae/servlet/contentblob/546/normal/66/logo_a.jpg</url>
|
||||
<title>service.bund.de - öffentliche Ausschreibungen</title>
|
||||
<link>http://www.service.bund.de/ausschreibungen</link>
|
||||
</image>
|
||||
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
|
||||
<ttl>60</ttl>
|
||||
<item>
|
||||
<title>Übermittlung von Bohrungsdaten in die Bohrpunktkarte Deutschland 4.0 Neuaufstellung des Datenformats</title>
|
||||
<link>https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Bundesanstalt-fuer-Geowissenschaften-und-Rohstoffe/2026/07/6585375.html#track=feed-callforbids</link>
|
||||
<description>
|
||||
<![CDATA[
|
||||
Erfüllungsort: <strong> Hannover</strong>
|
||||
<br />Vergabestelle: <strong>Bundesanstalt für Geowissenschaften und Rohstoffe</strong><br />
|
||||
<br />Angebotsfrist: <br />Veröffentlichungsende: <strong>23.10.2026 11:15</strong> <br />
|
||||
<br />
|
||||
]]>
|
||||
</description>
|
||||
<pubDate>Thu, 23 Jul 2026 11:15:00 +0200</pubDate>
|
||||
<guid>https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Bundesanstalt-fuer-Geowissenschaften-und-Rohstoffe/2026/07/6585375.html</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Vhv 120_26 UFZ - Bench-scale Anaerobic Bioreactor System</title>
|
||||
<link>https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Helmholtz-Zentrum-fuer-Umweltforschung-GmbH/2026/07/6585334.html#track=feed-callforbids</link>
|
||||
<description>
|
||||
<![CDATA[
|
||||
Erfüllungsort: <strong>04318 Leipzig</strong>
|
||||
<br />Vergabestelle: <strong>Helmholtz-Zentrum für Umweltforschung GmbH - UFZ</strong><br />
|
||||
<br />Angebotsfrist: <strong>07.08.2026 10:00</strong> <br />Veröffentlichungsende: <strong>07.08.2026 10:00</strong> <br />
|
||||
<br />
|
||||
]]>
|
||||
</description>
|
||||
<pubDate>Thu, 23 Jul 2026 11:00:00 +0200</pubDate>
|
||||
<guid>https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Helmholtz-Zentrum-fuer-Umweltforschung-GmbH/2026/07/6585334.html</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Verkehrsleistungen Linienbündel 5 "Birgland" (VGN-Linien 475 – 479)</title>
|
||||
<link>https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Zweckverband-Nahverkehr-Amberg-Sulzbach/2026/07/6585166.html#track=feed-callforbids</link>
|
||||
<description>
|
||||
<![CDATA[
|
||||
Erfüllungsort: <strong>92224 Amberg</strong>
|
||||
<br />Vergabestelle: <strong>Zweckverband Nahverkehr Amberg-Sulzbach (ZNAS)</strong><br />
|
||||
<br />Angebotsfrist: <strong>31.08.2026 11:00</strong> <br />Veröffentlichungsende: <strong>31.08.2026 11:00</strong> <br />
|
||||
<br />
|
||||
]]>
|
||||
</description>
|
||||
<pubDate>Thu, 23 Jul 2026 09:00:00 +0200</pubDate>
|
||||
<guid>https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Zweckverband-Nahverkehr-Amberg-Sulzbach/2026/07/6585166.html</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Plattformlifte</title>
|
||||
<link>https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196439.html#track=feed-callforbids</link>
|
||||
<description>
|
||||
<![CDATA[
|
||||
Erfüllungsort: <strong>86152 Augsburg</strong>
|
||||
<br />Vergabestelle: <strong>Stadt Augsburg</strong><br />
|
||||
<br />Angebotsfrist: <strong>27.08.2026 11:30</strong> <br />Veröffentlichungsende: <strong>27.08.2026 23:59</strong> <br />
|
||||
<br />
|
||||
]]>
|
||||
</description>
|
||||
<pubDate>Thu, 23 Jul 2026 07:28:16 +0200</pubDate>
|
||||
<guid>https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196439.html</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Hubbuehnen 1</title>
|
||||
<link>https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196438.html#track=feed-callforbids</link>
|
||||
<description>
|
||||
<![CDATA[
|
||||
Erfüllungsort: <strong>86152 Augsburg</strong>
|
||||
<br />Vergabestelle: <strong>Stadt Augsburg</strong><br />
|
||||
<br />Angebotsfrist: <strong>27.08.2026 11:00</strong> <br />Veröffentlichungsende: <strong>27.08.2026 23:59</strong> <br />
|
||||
<br />
|
||||
]]>
|
||||
</description>
|
||||
<pubDate>Thu, 23 Jul 2026 07:13:09 +0200</pubDate>
|
||||
<guid>https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196438.html</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>B 278 Flutgrabenbrücke bei Borsch, ASB-Nr. 5225 805; Ersatzneubau</title>
|
||||
<link>https://www.service.bund.de/IMPORTE/Ausschreibungen/eVergabe/877436.html#track=feed-callforbids</link>
|
||||
<description>
|
||||
<![CDATA[
|
||||
Erfüllungsort: <strong> Freistaat Thüringen, Landkreis Wartburgkreis, B 278 Flutgrabenbrücke bei Borsch ASB-Nr. 5225 805</strong>
|
||||
<br />Vergabestelle: <strong>Thüringer Landesamt für Bau und Verkehr, Straßenbauverwaltung und Zentrale Dienste</strong><br />
|
||||
<br />Angebotsfrist: <strong>19.08.2026 10:00</strong> <br />Veröffentlichungsende: <strong>19.08.2026 10:00</strong> <br />
|
||||
<br />
|
||||
]]>
|
||||
</description>
|
||||
<pubDate>Thu, 23 Jul 2026 05:00:00 +0200</pubDate>
|
||||
<guid>https://www.service.bund.de/IMPORTE/Ausschreibungen/eVergabe/877436.html</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
@@ -0,0 +1,45 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<rss version='2.0'>
|
||||
<channel>
|
||||
<title><![CDATA[subreport ELViS]]></title>
|
||||
<link>https://www.subreport-elvis.de/</link>
|
||||
<description><![CDATA[Aktuelle Ausschreibungen von Stadt Neuss, 41460 Neuss]]></description>
|
||||
<language>de-de</language>
|
||||
<copyright>subreport ELViS</copyright>
|
||||
<item>
|
||||
<title><![CDATA[E73433797:Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln-66-26-48 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln]]></title>
|
||||
<description><![CDATA[<table><tr><th style='text-align:left;'>Auftraggeber</th><td>Stadt Neuss, 41460 Neuss</td></tr><tr><th style='text-align:left;'>Vergabenummer</th><td>66-26-48</td></tr><tr><th style='text-align:left;'>Ausschreibungsart</th><td>Öffentliche Ausschreibung nach UVgO</td></tr><tr><th style='text-align:left;'>ELViS-ID</th><td>E73433797</td></tr><tr><th style='text-align:left;'>Ausschreibung</th><td>Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln</td></tr><tr><th style='text-align:left;'>Vergabeunterlagen</th><td>66-26-48 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln</td></tr><tr><th style='text-align:left;'>Gültig bis</th><td>27.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Angebotsabgabefrist</th><td>27.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Öffnungstermin</th><td>27.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>subreport</th><td><a href='https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E73433797,558464' target='_new'>zur Vorschau in die Vergabeunterlagen</a></td></tr></table>]]></description>
|
||||
<link>https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E73433797</link>
|
||||
<author><![CDATA[Stadt Neuss, vergabe@stadt.neuss.de]]></author>
|
||||
<guid isPermaLink="false">E73433797-558464</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title><![CDATA[E81865426:Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln-66-26-47 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln]]></title>
|
||||
<description><![CDATA[<table><tr><th style='text-align:left;'>Auftraggeber</th><td>Stadt Neuss, 41460 Neuss</td></tr><tr><th style='text-align:left;'>Vergabenummer</th><td>66-26-47</td></tr><tr><th style='text-align:left;'>Ausschreibungsart</th><td>Öffentliche Ausschreibung nach UVgO</td></tr><tr><th style='text-align:left;'>ELViS-ID</th><td>E81865426</td></tr><tr><th style='text-align:left;'>Ausschreibung</th><td>Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln</td></tr><tr><th style='text-align:left;'>Vergabeunterlagen</th><td>66-26-47 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln</td></tr><tr><th style='text-align:left;'>Gültig bis</th><td>26.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Angebotsabgabefrist</th><td>26.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Öffnungstermin</th><td>26.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>subreport</th><td><a href='https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E81865426,558461' target='_new'>zur Vorschau in die Vergabeunterlagen</a></td></tr></table>]]></description>
|
||||
<link>https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E81865426</link>
|
||||
<author><![CDATA[Stadt Neuss, vergabe@stadt.neuss.de]]></author>
|
||||
<guid isPermaLink="false">E81865426-558461</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title><![CDATA[E95748761:Lieferung eines Gerätewagens Logistik 2 (GW-L2)-37-26-44 Lieferung eines Gerätewagens Logistik 2 (GW-L2)]]></title>
|
||||
<description><![CDATA[<table><tr><th style='text-align:left;'>Auftraggeber</th><td>Stadt Neuss, 41460 Neuss</td></tr><tr><th style='text-align:left;'>Vergabenummer</th><td>37-26-44</td></tr><tr><th style='text-align:left;'>Ausschreibungsart</th><td>Offenes Verfahren nach VGV</td></tr><tr><th style='text-align:left;'>ELViS-ID</th><td>E95748761</td></tr><tr><th style='text-align:left;'>Ausschreibung</th><td>Lieferung eines Gerätewagens Logistik 2 (GW-L2)</td></tr><tr><th style='text-align:left;'>Vergabeunterlagen</th><td>37-26-44 Lieferung eines Gerätewagens Logistik 2 (GW-L2)</td></tr><tr><th style='text-align:left;'>Gültig bis</th><td>24.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Angebotsabgabefrist</th><td>24.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Öffnungstermin</th><td>24.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>subreport</th><td><a href='https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E95748761,557626' target='_new'>zur Vorschau in die Vergabeunterlagen</a></td></tr></table>]]></description>
|
||||
<link>https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E95748761</link>
|
||||
<author><![CDATA[Stadt Neuss, vergabe@stadt.neuss.de]]></author>
|
||||
<guid isPermaLink="false">E95748761-557626</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title><![CDATA[E96237741:Lieferung von Lichtmasten, Jahresbedarf 2026/2027-66-26-41 Lieferung von Lichtmasten, Jahresbedarf 2026/2027]]></title>
|
||||
<description><![CDATA[<table><tr><th style='text-align:left;'>Auftraggeber</th><td>Stadt Neuss, 41460 Neuss</td></tr><tr><th style='text-align:left;'>Vergabenummer</th><td>66-26-41</td></tr><tr><th style='text-align:left;'>Ausschreibungsart</th><td>Öffentliche Ausschreibung nach UVgO</td></tr><tr><th style='text-align:left;'>ELViS-ID</th><td>E96237741</td></tr><tr><th style='text-align:left;'>Ausschreibung</th><td>Lieferung von Lichtmasten, Jahresbedarf 2026/2027</td></tr><tr><th style='text-align:left;'>Vergabeunterlagen</th><td>66-26-41 Lieferung von Lichtmasten, Jahresbedarf 2026/2027</td></tr><tr><th style='text-align:left;'>Gültig bis</th><td>27.07.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Angebotsabgabefrist</th><td>27.07.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Öffnungstermin</th><td>27.07.2026 10:00:00</td></tr><tr><th style='text-align:left;'>subreport</th><td><a href='https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E96237741,556451' target='_new'>zur Vorschau in die Vergabeunterlagen</a></td></tr></table>]]></description>
|
||||
<link>https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E96237741</link>
|
||||
<author><![CDATA[Stadt Neuss, vergabe@stadt.neuss.de]]></author>
|
||||
<guid isPermaLink="false">E96237741-556451</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title><![CDATA[E87453261:Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt Neuss-53-26-40 Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt Neuss]]></title>
|
||||
<description><![CDATA[<table><tr><th style='text-align:left;'>Auftraggeber</th><td>Stadt Neuss, 41460 Neuss</td></tr><tr><th style='text-align:left;'>Vergabenummer</th><td>53-26-40</td></tr><tr><th style='text-align:left;'>Ausschreibungsart</th><td>Offenes Verfahren nach VGV</td></tr><tr><th style='text-align:left;'>ELViS-ID</th><td>E87453261</td></tr><tr><th style='text-align:left;'>Ausschreibung</th><td>Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt Neuss</td></tr><tr><th style='text-align:left;'>Vergabeunterlagen</th><td>53-26-40 Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt Neuss</td></tr><tr><th style='text-align:left;'>Gültig bis</th><td>13.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Angebotsabgabefrist</th><td>13.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>Öffnungstermin</th><td>13.08.2026 10:00:00</td></tr><tr><th style='text-align:left;'>subreport</th><td><a href='https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E87453261,555142' target='_new'>zur Vorschau in die Vergabeunterlagen</a></td></tr></table>]]></description>
|
||||
<link>https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E87453261</link>
|
||||
<author><![CDATA[Stadt Neuss, vergabe@stadt.neuss.de]]></author>
|
||||
<guid isPermaLink="false">E87453261-555142</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
@@ -0,0 +1,189 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { RssAdapter } from './rss.adapter';
|
||||
|
||||
/**
|
||||
* Real, live-captured RSS fixtures (both fetched directly against the real
|
||||
* endpoints, 2026-07-23) — mirrors cosinex.adapter.spec.ts's fixture/mock
|
||||
* spec style. `service-bund-feed.xml` proves the pubDate-present,
|
||||
* numeric-HTML-entity-title shape; `subreport-elvis-feed.xml` proves the
|
||||
* pubDate-absent, CDATA-title shape (14-RESEARCH.md live captures).
|
||||
*/
|
||||
const FIXTURES_DIR = join(__dirname, '..', '__fixtures__');
|
||||
const SERVICE_BUND_FIXTURE = readFileSync(
|
||||
join(FIXTURES_DIR, 'service-bund-feed.xml'),
|
||||
'utf8',
|
||||
);
|
||||
const SUBREPORT_ELVIS_FIXTURE = readFileSync(
|
||||
join(FIXTURES_DIR, 'subreport-elvis-feed.xml'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
describe('RssAdapter', () => {
|
||||
it('declares sourceType rss and the symbolic rss portal', () => {
|
||||
const adapter = new RssAdapter();
|
||||
expect(adapter.sourceType).toBe('rss');
|
||||
expect(adapter.portals).toEqual(['rss']);
|
||||
});
|
||||
|
||||
describe('parseFeed (pure, fixture-driven)', () => {
|
||||
it('parses service.bund.de items: sourceType/sourcePortal set, pubDate present, numeric HTML entities decoded in title', () => {
|
||||
const adapter = new RssAdapter();
|
||||
|
||||
const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund');
|
||||
|
||||
expect(records.length).toBeGreaterThanOrEqual(1);
|
||||
for (const record of records) {
|
||||
expect(record.sourceType).toBe('rss');
|
||||
expect(record.sourcePortal).toBe('service-bund');
|
||||
expect(record.sourceUrl).toMatch(/^https:\/\/www\.service\.bund\.de\//);
|
||||
expect(record.eformsPayload).toBeNull();
|
||||
}
|
||||
|
||||
// First fixture item's title is numeric-HTML-entity-encoded
|
||||
// (`Übermittlung...`), NOT CDATA — must decode to a real "Ü".
|
||||
const first = records[0];
|
||||
expect(first).toBeDefined();
|
||||
const payload = first!.ocdsPayload as { title: string };
|
||||
expect(payload.title).toMatch(/^Übermittlung von Bohrungsdaten/);
|
||||
expect(payload.title).not.toContain('&#');
|
||||
|
||||
// Every service.bund.de fixture item carries a <pubDate>.
|
||||
expect(records.every((r) => r.publishedAt instanceof Date)).toBe(true);
|
||||
});
|
||||
|
||||
it('uses the item guid as sourceNoticeId for service.bund.de (plain-text guid, no isPermaLink attribute)', () => {
|
||||
const adapter = new RssAdapter();
|
||||
const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund');
|
||||
|
||||
const ids = records.map((r) => r.sourceNoticeId);
|
||||
expect(ids.every((id) => id.startsWith('https://'))).toBe(true);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
});
|
||||
|
||||
it('parses subreport-elvis items: publishedAt null (no item pubDate), title from CDATA, guid isPermaLink=false extracted as plain text', () => {
|
||||
const adapter = new RssAdapter();
|
||||
|
||||
const records = adapter.parseFeed(
|
||||
SUBREPORT_ELVIS_FIXTURE,
|
||||
'subreport-neuss',
|
||||
);
|
||||
|
||||
expect(records.length).toBeGreaterThanOrEqual(1);
|
||||
for (const record of records) {
|
||||
expect(record.sourceType).toBe('rss');
|
||||
expect(record.sourcePortal).toBe('subreport-neuss');
|
||||
expect(record.publishedAt).toBeNull();
|
||||
expect(record.sourceUrl).toMatch(
|
||||
/^https:\/\/www\.subreport-elvis\.de\//,
|
||||
);
|
||||
}
|
||||
|
||||
const first = records[0];
|
||||
expect(first).toBeDefined();
|
||||
// guid is `<guid isPermaLink="false">E73433797-558464</guid>` —
|
||||
// fast-xml-parser represents this as { '#text': ..., '@_isPermaLink': ... },
|
||||
// extractTagText must pull out the plain '#text' value.
|
||||
expect(first!.sourceNoticeId).toBe('E73433797-558464');
|
||||
|
||||
const payload = first!.ocdsPayload as { title: string };
|
||||
expect(payload.title).toMatch(/^E73433797:/);
|
||||
});
|
||||
|
||||
it('never carries description HTML into the record (V5 — no stored-XSS surface)', () => {
|
||||
const adapter = new RssAdapter();
|
||||
const records = adapter.parseFeed(
|
||||
SUBREPORT_ELVIS_FIXTURE,
|
||||
'subreport-neuss',
|
||||
);
|
||||
|
||||
for (const record of records) {
|
||||
const serialized = JSON.stringify(record.ocdsPayload);
|
||||
expect(serialized).not.toContain('<table>');
|
||||
expect(serialized).not.toContain('<a href');
|
||||
}
|
||||
});
|
||||
|
||||
it('bare-minimum bag: buyerName/procedureType/deadlineAt always null (baseline mapping only, D-04/D-05)', () => {
|
||||
const adapter = new RssAdapter();
|
||||
const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund');
|
||||
|
||||
for (const record of records) {
|
||||
const payload = record.ocdsPayload as {
|
||||
buyerName: string | null;
|
||||
procedureType: string | null;
|
||||
legalFramework: string | null;
|
||||
deadlineAt: string | null;
|
||||
};
|
||||
expect(payload.buyerName).toBeNull();
|
||||
expect(payload.procedureType).toBeNull();
|
||||
expect(payload.legalFramework).toBeNull();
|
||||
expect(payload.deadlineAt).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it('returns [] for empty XML instead of throwing', () => {
|
||||
const adapter = new RssAdapter();
|
||||
expect(adapter.parseFeed('', 'empty-feed')).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] for malformed XML instead of throwing', () => {
|
||||
const adapter = new RssAdapter();
|
||||
expect(
|
||||
adapter.parseFeed('<rss><channel><item><title>', 'broken-feed'),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] for well-formed XML with no <item> at all', () => {
|
||||
const adapter = new RssAdapter();
|
||||
const xml =
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>Empty</title></channel></rss>';
|
||||
expect(adapter.parseFeed(xml, 'no-items-feed')).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips a single item missing <link> without aborting the rest', () => {
|
||||
const adapter = new RssAdapter();
|
||||
const xml = `<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<item><title>Ohne Link</title><guid>no-link-guid</guid></item>
|
||||
<item><title>Mit Link</title><link>https://example.invalid/a</link><guid>with-link-guid</guid></item>
|
||||
</channel></rss>`;
|
||||
|
||||
const records = adapter.parseFeed(xml, 'mixed-feed');
|
||||
|
||||
expect(records).toHaveLength(1);
|
||||
expect(records[0]?.sourceNoticeId).toBe('with-link-guid');
|
||||
});
|
||||
|
||||
it('falls back to sha256(link) for sourceNoticeId when guid is absent', () => {
|
||||
const adapter = new RssAdapter();
|
||||
const xml = `<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<item><title>No Guid</title><link>https://example.invalid/no-guid</link></item>
|
||||
</channel></rss>`;
|
||||
|
||||
const records = adapter.parseFeed(xml, 'no-guid-feed');
|
||||
|
||||
expect(records).toHaveLength(1);
|
||||
expect(records[0]?.sourceNoticeId).toMatch(/^[a-f0-9]{40}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalize() dispatch', () => {
|
||||
it("'rss' sourceType routes through TenderNormalizerService.normalizeBag() (proven end-to-end via tender-normalizer.service.spec.ts)", () => {
|
||||
// See tender-normalizer.service.spec.ts's "generic ocdsPayload bag"
|
||||
// describe block for the actual normalize() assertions — this spec
|
||||
// only proves RssAdapter's OWN output shape (parseFeed), not the
|
||||
// normalizer dispatch itself (kept in the normalizer's own spec file
|
||||
// to avoid duplicating TenderNormalizerService test infrastructure).
|
||||
const adapter = new RssAdapter();
|
||||
expect(adapter.sourceType).toBe('rss');
|
||||
});
|
||||
});
|
||||
|
||||
it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => {
|
||||
const source = readFileSync(join(__dirname, 'rss.adapter.ts'), 'utf8');
|
||||
expect(source).not.toMatch(/from ['"]axios['"]/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { createHash } from 'crypto';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import type { RawTenderRecord, SourceType } from '../tender.types';
|
||||
import type { TenderSourceAdapter } from './tender-source-adapter.interface';
|
||||
|
||||
/**
|
||||
* RssAdapter — INGEST-04. Parses admin-managed RSS feed URLs into
|
||||
* RawTenderRecord[] via the same `fast-xml-parser` configuration
|
||||
* doe-opendata.adapter.ts already uses for eForms-DE XML
|
||||
* (`removeNSPrefix`/`ignoreAttributes`/`attributeNamePrefix` —
|
||||
* 14-RESEARCH.md "RSS parsing").
|
||||
*
|
||||
* `portals: ['rss']` is a SYMBOLIC placeholder, not a real hostname list
|
||||
* (14-RESEARCH.md Pitfall 3): the actual feed hostnames are admin-supplied
|
||||
* RUNTIME data (`TenderRssFeedSource` rows, wired in Plan 14-02 Task 2),
|
||||
* added long after `SourceRegistry.register()`'s DI-boot-time denylist
|
||||
* check runs. The `DENYLISTED_PORTALS` gate therefore provides ZERO
|
||||
* protection for RSS feed URLs — a SEPARATE, save-time hostname/SSRF guard
|
||||
* is enforced in `tender-rss-feed.service.ts` (D-14, Plan 14-02 Task 2, see
|
||||
* threat T-14-02-01). Do not remove that guard under the assumption this
|
||||
* adapter's `portals` array already covers it.
|
||||
*
|
||||
* Live-verified shapes (14-RESEARCH.md, captured 2026-07-23):
|
||||
* - service.bund.de: `<item><pubDate>` present; `<title>` uses raw numeric
|
||||
* HTML character references (e.g. `Übermittlung...`), NOT CDATA —
|
||||
* decoded explicitly below (Rule 1: fast-xml-parser only decodes the 5
|
||||
* predefined XML entities by default, not numeric character refs).
|
||||
* - subreport-elvis: `<item><pubDate>` is ABSENT entirely (`publishedAt`
|
||||
* is always null for this source); `<title>`/`<description>` are
|
||||
* CDATA-wrapped (auto-merged to plain strings by fast-xml-parser, no
|
||||
* decoding needed).
|
||||
*
|
||||
* Baseline field mapping only (title/link/guid) — `buyerName`/
|
||||
* `procedureType`/`deadlineAt` stay null. The optional service.bund.de
|
||||
* `<description>` label-extraction enrichment (Erfüllungsort/
|
||||
* Vergabestelle/Angebotsfrist) is explicitly out of scope for this task
|
||||
* (RESEARCH.md Pattern 3 "optional enhancement"); `<description>` is never
|
||||
* read by this adapter, so no raw HTML fragment is ever carried into a
|
||||
* RawTenderRecord (V5 — no stored-XSS surface, same text-only discipline
|
||||
* as NetServerAdapter/CosinexAdapter).
|
||||
*/
|
||||
@Injectable()
|
||||
export class RssAdapter implements TenderSourceAdapter {
|
||||
readonly sourceType: SourceType = 'rss';
|
||||
/** Symbolic placeholder — see class docstring re: Pitfall 3. */
|
||||
readonly portals = ['rss'] as const;
|
||||
|
||||
private readonly logger = new Logger(RssAdapter.name);
|
||||
private readonly xmlParser = new XMLParser({
|
||||
removeNSPrefix: true,
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
});
|
||||
|
||||
/**
|
||||
* Placeholder — wired to the real `TenderRssFeedSource` internal fan-out
|
||||
* (native fetch + AbortController per admin-added feed URL) in Plan
|
||||
* 14-02 Task 2. Kept here only so the class satisfies
|
||||
* `TenderSourceAdapter` for this task's fixture-driven `parseFeed` proof.
|
||||
*/
|
||||
async fetchTenders(_dayCursor: string): Promise<RawTenderRecord[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure XML -> RawTenderRecord[] mapping. Never throws (Pitfall 2
|
||||
* discipline, mirrors NetServer/cosinex): malformed/empty XML, or a feed
|
||||
* with no `<channel>`/`<item>` at all, resolves to `[]`. A single item
|
||||
* missing `<link>` is skipped (logger.warn) without aborting the rest.
|
||||
*/
|
||||
parseFeed(xml: string, feedLabel: string): RawTenderRecord[] {
|
||||
const fetchedAt = new Date();
|
||||
let items: unknown[];
|
||||
|
||||
try {
|
||||
const parsed = this.xmlParser.parse(xml) as {
|
||||
rss?: { channel?: { item?: unknown | unknown[] } };
|
||||
};
|
||||
const channelItem = parsed.rss?.channel?.item;
|
||||
items = Array.isArray(channelItem)
|
||||
? channelItem
|
||||
: channelItem
|
||||
? [channelItem]
|
||||
: [];
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`RSS feed '${feedLabel}' totally unparsable, returning [] (mirrors NetServer/cosinex Pitfall 2): ${(error as Error).message}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
const records: RawTenderRecord[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
try {
|
||||
const node = item as Record<string, unknown>;
|
||||
const link = extractTagText(node.link);
|
||||
if (!link) {
|
||||
this.logger.warn(
|
||||
`Skipping RSS item without <link> (feed '${feedLabel}')`,
|
||||
);
|
||||
continue; // no stable URL -> unusable item, skip (Pitfall 2 discipline)
|
||||
}
|
||||
|
||||
const rawTitle = extractTagText(node.title);
|
||||
const title = rawTitle
|
||||
? decodeNumericEntities(rawTitle).trim()
|
||||
: 'Unbenannte Ausschreibung';
|
||||
|
||||
const guidText = extractTagText(node.guid);
|
||||
const sourceNoticeId =
|
||||
guidText ||
|
||||
createHash('sha256').update(link).digest('hex').slice(0, 40);
|
||||
|
||||
const pubDateRaw = extractTagText(node.pubDate);
|
||||
const publishedAt = parseRssDate(pubDateRaw);
|
||||
|
||||
records.push({
|
||||
sourceType: this.sourceType,
|
||||
sourcePortal: feedLabel,
|
||||
sourceNoticeId,
|
||||
sourceUrl: link,
|
||||
fetchedAt,
|
||||
publishedAt,
|
||||
eformsPayload: null,
|
||||
// RSS has no eForms/OCDS structure; the extracted baseline fields
|
||||
// are carried through this generic bag — same convention as
|
||||
// NetServerAdapter/CosinexAdapter (13-04/13-05) — so
|
||||
// TenderNormalizerService.normalizeBag() maps them without any
|
||||
// RSS-specific normalizer code (D-04/D-05).
|
||||
ocdsPayload: {
|
||||
title,
|
||||
buyerName: null,
|
||||
procedureType: null,
|
||||
legalFramework: null,
|
||||
deadlineAt: null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Skipping unparsable RSS item (feed '${feedLabel}'): ${(error as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* fast-xml-parser represents a plain tag as a raw string/number, and a tag
|
||||
* with attributes (e.g. subreport-elvis's `<guid isPermaLink="false">`) as
|
||||
* `{ '@_attr': ..., '#text': value }` — mirrors
|
||||
* tender-normalizer.service.ts's `textValue()` helper (kept local/
|
||||
* duplicated rather than imported: this is adapter-layer XML-shape
|
||||
* handling, not normalizer-layer field mapping).
|
||||
*/
|
||||
function extractTagText(node: unknown): string | null {
|
||||
if (node === null || node === undefined) return null;
|
||||
if (typeof node === 'string') return node || null;
|
||||
if (typeof node === 'number') return String(node);
|
||||
if (
|
||||
typeof node === 'object' &&
|
||||
'#text' in (node as Record<string, unknown>)
|
||||
) {
|
||||
const t = (node as Record<string, unknown>)['#text'];
|
||||
if (t === null || t === undefined) return null;
|
||||
return String(t);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rule 1 fix: fast-xml-parser only decodes the 5 predefined XML entities
|
||||
* (`&` `<` `>` `"` `'`) — numeric character references
|
||||
* (`Ü`, `ß`) pass through UNDECODED (confirmed live against
|
||||
* fast-xml-parser v5.10.1, see class docstring). service.bund.de titles use
|
||||
* numeric refs directly (not CDATA-wrapped, unlike subreport-elvis) —
|
||||
* without this decode step, titles would literally show "Übermittlung
|
||||
* ..." to admins/users instead of "Übermittlung...".
|
||||
*/
|
||||
function decodeNumericEntities(text: string): string {
|
||||
return text
|
||||
.replace(/&#(\d+);/g, (_match, dec: string) =>
|
||||
String.fromCodePoint(Number(dec)),
|
||||
)
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_match, hex: string) =>
|
||||
String.fromCodePoint(Number.parseInt(hex, 16)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* RSS `pubDate` is RFC-822 (`Thu, 23 Jul 2026 11:15:00 +0200`), parseable
|
||||
* directly by `Date`. Absent/malformed -> null (subreport-elvis has no
|
||||
* per-item `pubDate` at all — confirmed live, see class docstring).
|
||||
*/
|
||||
function parseRssDate(raw: string | null): Date | null {
|
||||
if (!raw) return null;
|
||||
const parsed = new Date(raw);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
@@ -81,7 +81,11 @@ function bagRecord(
|
||||
},
|
||||
): RawTenderRecord {
|
||||
const sourcePortal =
|
||||
sourceType === 'cosinex-dtvp' ? 'cosinex-dtvp' : 'tender24';
|
||||
sourceType === 'cosinex-dtvp'
|
||||
? 'cosinex-dtvp'
|
||||
: sourceType === 'rss'
|
||||
? 'service-bund'
|
||||
: 'tender24';
|
||||
return {
|
||||
sourceType,
|
||||
sourcePortal,
|
||||
@@ -317,4 +321,38 @@ describe('TenderNormalizerService', () => {
|
||||
expect(normalized.contentHash).toMatch(/^[a-f0-9]{64}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("'rss' sourceType (Phase 14, Plan 02, INGEST-04)", () => {
|
||||
it('maps an rss bag through normalizeBag(): title passes through, cpvDivisions stays empty', () => {
|
||||
const raw = bagRecord('rss', {
|
||||
title: 'Übermittlung von Bohrungsdaten',
|
||||
buyerName: null,
|
||||
procedureType: null,
|
||||
legalFramework: null,
|
||||
deadlineAt: null,
|
||||
});
|
||||
|
||||
const normalized = service.normalize(raw);
|
||||
|
||||
expect(normalized.title).toBe('Übermittlung von Bohrungsdaten');
|
||||
expect(normalized.cpvDivisions).toEqual([]);
|
||||
expect(normalized.cpvCodes).toEqual([]);
|
||||
expect(normalized.region).toBeNull();
|
||||
expect(normalized.bundesland).toBeNull();
|
||||
expect(normalized.buyerName).toBeNull();
|
||||
expect(normalized.procedureType).toBeNull();
|
||||
expect(normalized.deadlineAt).toBeNull();
|
||||
expect(normalized.dedupKey).toBe(
|
||||
`${raw.sourcePortal}:${raw.sourceNoticeId}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('rss title falls back to "Unbenannte Ausschreibung" when empty', () => {
|
||||
const raw = bagRecord('rss', { title: '' });
|
||||
|
||||
const normalized = service.normalize(raw);
|
||||
|
||||
expect(normalized.title).toBe('Unbenannte Ausschreibung');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -54,6 +54,7 @@ export class TenderNormalizerService {
|
||||
switch (raw.sourceType) {
|
||||
case 'ai-netserver':
|
||||
case 'cosinex-dtvp':
|
||||
case 'rss':
|
||||
return this.normalizeBag(raw);
|
||||
case 'doe-opendata':
|
||||
default:
|
||||
@@ -117,10 +118,13 @@ export class TenderNormalizerService {
|
||||
|
||||
/**
|
||||
* Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters
|
||||
* carry a flat `{title, buyerName, procedureType, legalFramework,
|
||||
* deadlineAt}` bag in `raw.ocdsPayload` — there is no eForms/OCDS
|
||||
* structure to inspect. `legalFramework` is deliberately NOT mapped:
|
||||
* NormalizedTenderFields has no target field for it.
|
||||
* (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04) carry a flat
|
||||
* `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in
|
||||
* `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect.
|
||||
* `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has
|
||||
* no target field for it. RSS records always have `buyerName`/
|
||||
* `procedureType`/`deadlineAt` null (baseline title/link/guid mapping
|
||||
* only, D-04/D-05 thin-fields deferral).
|
||||
*/
|
||||
private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields {
|
||||
const bag = getBagPayload(raw.ocdsPayload);
|
||||
|
||||
@@ -13,8 +13,16 @@
|
||||
* (Plan 13-05) adapters can be added without further breaking changes.
|
||||
* This union widening is the only breaking type change in Plan 13-01,
|
||||
* and it's contained locally (no runtime behavior change here).
|
||||
*
|
||||
* Phase 14, Plan 02 (INGEST-04) adds 'rss': admin-managed RSS feed URLs
|
||||
* (TenderRssFeedSource, D-14), routed through the same generic-bag
|
||||
* normalizer path as 'ai-netserver'/'cosinex-dtvp' (D-04/D-05).
|
||||
*/
|
||||
export type SourceType = 'doe-opendata' | 'ai-netserver' | 'cosinex-dtvp';
|
||||
export type SourceType =
|
||||
| 'doe-opendata'
|
||||
| 'ai-netserver'
|
||||
| 'cosinex-dtvp'
|
||||
| 'rss';
|
||||
|
||||
/**
|
||||
* A single notice as fetched and lightly parsed from a source, before
|
||||
|
||||
Reference in New Issue
Block a user