From 3a96cbbbe6e24fe2d4f6d01c7d8499e9d1a66bf4 Mon Sep 17 00:00:00 2001 From: Schalli Date: Thu, 23 Jul 2026 13:17:02 +0200 Subject: [PATCH] feat(14-02): add RssAdapter.parseFeed + 'rss' normalizer dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixture-first RSS parsing (INGEST-04): parses live-captured service.bund.de (pubDate present, numeric-HTML-entity titles) and subreport-elvis (pubDate absent, CDATA titles) feed shapes into RawTenderRecord[] via fast-xml-parser, mirroring the DoeOpenDataAdapter config. SourceType extended with 'rss'; normalize() dispatches 'rss' through the existing normalizeBag() path unchanged (D-04/D-05). Rule 1 fix: fast-xml-parser only decodes the 5 predefined XML entities, not numeric character references — added an explicit decode step so service.bund.de titles ("Übermittlung...") render correctly instead of leaking raw entity syntax. Co-Authored-By: Claude Opus 4.8 --- .../__fixtures__/service-bund-feed.xml | 105 +++++++++ .../__fixtures__/subreport-elvis-feed.xml | 45 ++++ .../src/tenders/adapters/rss.adapter.spec.ts | 189 ++++++++++++++++ apps/api/src/tenders/adapters/rss.adapter.ts | 202 ++++++++++++++++++ .../tenders/tender-normalizer.service.spec.ts | 40 +++- .../src/tenders/tender-normalizer.service.ts | 12 +- apps/api/src/tenders/tender.types.ts | 10 +- 7 files changed, 597 insertions(+), 6 deletions(-) create mode 100644 apps/api/src/tenders/__fixtures__/service-bund-feed.xml create mode 100644 apps/api/src/tenders/__fixtures__/subreport-elvis-feed.xml create mode 100644 apps/api/src/tenders/adapters/rss.adapter.spec.ts create mode 100644 apps/api/src/tenders/adapters/rss.adapter.ts diff --git a/apps/api/src/tenders/__fixtures__/service-bund-feed.xml b/apps/api/src/tenders/__fixtures__/service-bund-feed.xml new file mode 100644 index 0000000..30eae94 --- /dev/null +++ b/apps/api/src/tenders/__fixtures__/service-bund-feed.xml @@ -0,0 +1,105 @@ + + + + service.bund.de - öffentliche Ausschreibungen + http://www.service.bund.de/ausschreibungen + RSS-Feed mit aktuellen Ausschreibungen der Vergabestellen der Bundes-, Landes- und Kommunalverwaltung. + de-de + Thu, 23 Jul 2026 13:06:44 +0200 + Thu, 23 Jul 2026 13:06:44 +0200 + redaktion-support@bva.bund.de + redaktion-support@bva.bund.de + Government Site Builder (GSB7.0) + + http://www.service.bund.de/cae/servlet/contentblob/546/normal/66/logo_a.jpg + service.bund.de - öffentliche Ausschreibungen + http://www.service.bund.de/ausschreibungen + + http://blogs.law.harvard.edu/tech/rss + 60 + +Übermittlung von Bohrungsdaten in die Bohrpunktkarte Deutschland 4.0 Neuaufstellung des Datenformats +https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Bundesanstalt-fuer-Geowissenschaften-und-Rohstoffe/2026/07/6585375.html#track=feed-callforbids + + Hannover +
Vergabestelle: Bundesanstalt für Geowissenschaften und Rohstoffe
+
Angebotsfrist:
Veröffentlichungsende: 23.10.2026 11:15
+
+]]> + +Thu, 23 Jul 2026 11:15:00 +0200 +https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Bundesanstalt-fuer-Geowissenschaften-und-Rohstoffe/2026/07/6585375.html + + +Vhv 120_26 UFZ - Bench-scale Anaerobic Bioreactor System +https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Helmholtz-Zentrum-fuer-Umweltforschung-GmbH/2026/07/6585334.html#track=feed-callforbids + +04318 Leipzig +
Vergabestelle: Helmholtz-Zentrum für Umweltforschung GmbH - UFZ
+
Angebotsfrist: 07.08.2026 10:00
Veröffentlichungsende: 07.08.2026 10:00
+
+]]> + +Thu, 23 Jul 2026 11:00:00 +0200 +https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Helmholtz-Zentrum-fuer-Umweltforschung-GmbH/2026/07/6585334.html + + +Verkehrsleistungen Linienbündel 5 "Birgland" (VGN-Linien 475 – 479) +https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Zweckverband-Nahverkehr-Amberg-Sulzbach/2026/07/6585166.html#track=feed-callforbids + +92224 Amberg +
Vergabestelle: Zweckverband Nahverkehr Amberg-Sulzbach (ZNAS)
+
Angebotsfrist: 31.08.2026 11:00
Veröffentlichungsende: 31.08.2026 11:00
+
+]]> + +Thu, 23 Jul 2026 09:00:00 +0200 +https://www.service.bund.de/IMPORTE/Ausschreibungen/editor/Zweckverband-Nahverkehr-Amberg-Sulzbach/2026/07/6585166.html + + +Plattformlifte +https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196439.html#track=feed-callforbids + +86152 Augsburg +
Vergabestelle: Stadt Augsburg
+
Angebotsfrist: 27.08.2026 11:30
Veröffentlichungsende: 27.08.2026 23:59
+
+]]> + +Thu, 23 Jul 2026 07:28:16 +0200 +https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196439.html + + +Hubbuehnen 1 +https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196438.html#track=feed-callforbids + +86152 Augsburg +
Vergabestelle: Stadt Augsburg
+
Angebotsfrist: 27.08.2026 11:00
Veröffentlichungsende: 27.08.2026 23:59
+
+]]> + +Thu, 23 Jul 2026 07:13:09 +0200 +https://www.service.bund.de/IMPORTE/Ausschreibungen/asp/2026/07/196438.html + + +B 278 Flutgrabenbrücke bei Borsch, ASB-Nr. 5225 805; Ersatzneubau +https://www.service.bund.de/IMPORTE/Ausschreibungen/eVergabe/877436.html#track=feed-callforbids + + Freistaat Thüringen, Landkreis Wartburgkreis, B 278 Flutgrabenbrücke bei Borsch ASB-Nr. 5225 805 +
Vergabestelle: Thüringer Landesamt für Bau und Verkehr, Straßenbauverwaltung und Zentrale Dienste
+
Angebotsfrist: 19.08.2026 10:00
Veröffentlichungsende: 19.08.2026 10:00
+
+]]> + +Thu, 23 Jul 2026 05:00:00 +0200 +https://www.service.bund.de/IMPORTE/Ausschreibungen/eVergabe/877436.html + + + diff --git a/apps/api/src/tenders/__fixtures__/subreport-elvis-feed.xml b/apps/api/src/tenders/__fixtures__/subreport-elvis-feed.xml new file mode 100644 index 0000000..989ea6f --- /dev/null +++ b/apps/api/src/tenders/__fixtures__/subreport-elvis-feed.xml @@ -0,0 +1,45 @@ + + + +<![CDATA[subreport ELViS]]> +https://www.subreport-elvis.de/ + +de-de +subreport ELViS + +<![CDATA[E73433797:Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln-66-26-48 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln]]> +AuftraggeberStadt Neuss, 41460 NeussVergabenummer66-26-48AusschreibungsartÖffentliche Ausschreibung nach UVgOELViS-IDE73433797AusschreibungObjektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei MinkelnVergabeunterlagen66-26-48 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei MinkelnGültig bis27.08.2026 10:00:00Angebotsabgabefrist27.08.2026 10:00:00Öffnungstermin27.08.2026 10:00:00subreportzur Vorschau in die Vergabeunterlagen]]> +https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E73433797 + +E73433797-558464 + + +<![CDATA[E81865426:Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln-66-26-47 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei Minkeln]]> +AuftraggeberStadt Neuss, 41460 NeussVergabenummer66-26-47AusschreibungsartÖffentliche Ausschreibung nach UVgOELViS-IDE81865426AusschreibungObjektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei MinkelnVergabeunterlagen66-26-47 Objektbezogene Schadensanalysen für Brückenbauwerke im alten Bahndamm bei MinkelnGültig bis26.08.2026 10:00:00Angebotsabgabefrist26.08.2026 10:00:00Öffnungstermin26.08.2026 10:00:00subreportzur Vorschau in die Vergabeunterlagen]]> +https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E81865426 + +E81865426-558461 + + +<![CDATA[E95748761:Lieferung eines Gerätewagens Logistik 2 (GW-L2)-37-26-44 Lieferung eines Gerätewagens Logistik 2 (GW-L2)]]> +AuftraggeberStadt Neuss, 41460 NeussVergabenummer37-26-44AusschreibungsartOffenes Verfahren nach VGVELViS-IDE95748761AusschreibungLieferung eines Gerätewagens Logistik 2 (GW-L2)Vergabeunterlagen37-26-44 Lieferung eines Gerätewagens Logistik 2 (GW-L2)Gültig bis24.08.2026 10:00:00Angebotsabgabefrist24.08.2026 10:00:00Öffnungstermin24.08.2026 10:00:00subreportzur Vorschau in die Vergabeunterlagen]]> +https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E95748761 + +E95748761-557626 + + +<![CDATA[E96237741:Lieferung von Lichtmasten, Jahresbedarf 2026/2027-66-26-41 Lieferung von Lichtmasten, Jahresbedarf 2026/2027]]> +AuftraggeberStadt Neuss, 41460 NeussVergabenummer66-26-41AusschreibungsartÖffentliche Ausschreibung nach UVgOELViS-IDE96237741AusschreibungLieferung von Lichtmasten, Jahresbedarf 2026/2027Vergabeunterlagen66-26-41 Lieferung von Lichtmasten, Jahresbedarf 2026/2027Gültig bis27.07.2026 10:00:00Angebotsabgabefrist27.07.2026 10:00:00Öffnungstermin27.07.2026 10:00:00subreportzur Vorschau in die Vergabeunterlagen]]> +https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E96237741 + +E96237741-556451 + + +<![CDATA[E87453261:Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt Neuss-53-26-40 Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt Neuss]]> +AuftraggeberStadt Neuss, 41460 NeussVergabenummer53-26-40AusschreibungsartOffenes Verfahren nach VGVELViS-IDE87453261AusschreibungSicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt NeussVergabeunterlagen53-26-40 Sicherheitsdienstleistungen und Brandschutzwachdienst in städtischen Unterbringungseinrichtungen der Stadt NeussGültig bis13.08.2026 10:00:00Angebotsabgabefrist13.08.2026 10:00:00Öffnungstermin13.08.2026 10:00:00subreportzur Vorschau in die Vergabeunterlagen]]> +https://www.subreport-elvis.de/browseVerdingungsunterlagen.html#ELVISID:E87453261 + +E87453261-555142 + + + diff --git a/apps/api/src/tenders/adapters/rss.adapter.spec.ts b/apps/api/src/tenders/adapters/rss.adapter.spec.ts new file mode 100644 index 0000000..132a978 --- /dev/null +++ b/apps/api/src/tenders/adapters/rss.adapter.spec.ts @@ -0,0 +1,189 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { describe, expect, it } from 'vitest'; +import { RssAdapter } from './rss.adapter'; + +/** + * Real, live-captured RSS fixtures (both fetched directly against the real + * endpoints, 2026-07-23) — mirrors cosinex.adapter.spec.ts's fixture/mock + * spec style. `service-bund-feed.xml` proves the pubDate-present, + * numeric-HTML-entity-title shape; `subreport-elvis-feed.xml` proves the + * pubDate-absent, CDATA-title shape (14-RESEARCH.md live captures). + */ +const FIXTURES_DIR = join(__dirname, '..', '__fixtures__'); +const SERVICE_BUND_FIXTURE = readFileSync( + join(FIXTURES_DIR, 'service-bund-feed.xml'), + 'utf8', +); +const SUBREPORT_ELVIS_FIXTURE = readFileSync( + join(FIXTURES_DIR, 'subreport-elvis-feed.xml'), + 'utf8', +); + +describe('RssAdapter', () => { + it('declares sourceType rss and the symbolic rss portal', () => { + const adapter = new RssAdapter(); + expect(adapter.sourceType).toBe('rss'); + expect(adapter.portals).toEqual(['rss']); + }); + + describe('parseFeed (pure, fixture-driven)', () => { + it('parses service.bund.de items: sourceType/sourcePortal set, pubDate present, numeric HTML entities decoded in title', () => { + const adapter = new RssAdapter(); + + const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund'); + + expect(records.length).toBeGreaterThanOrEqual(1); + for (const record of records) { + expect(record.sourceType).toBe('rss'); + expect(record.sourcePortal).toBe('service-bund'); + expect(record.sourceUrl).toMatch(/^https:\/\/www\.service\.bund\.de\//); + expect(record.eformsPayload).toBeNull(); + } + + // First fixture item's title is numeric-HTML-entity-encoded + // (`Übermittlung...`), NOT CDATA — must decode to a real "Ü". + const first = records[0]; + expect(first).toBeDefined(); + const payload = first!.ocdsPayload as { title: string }; + expect(payload.title).toMatch(/^Übermittlung von Bohrungsdaten/); + expect(payload.title).not.toContain('&#'); + + // Every service.bund.de fixture item carries a . + expect(records.every((r) => r.publishedAt instanceof Date)).toBe(true); + }); + + it('uses the item guid as sourceNoticeId for service.bund.de (plain-text guid, no isPermaLink attribute)', () => { + const adapter = new RssAdapter(); + const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund'); + + const ids = records.map((r) => r.sourceNoticeId); + expect(ids.every((id) => id.startsWith('https://'))).toBe(true); + expect(new Set(ids).size).toBe(ids.length); + }); + + it('parses subreport-elvis items: publishedAt null (no item pubDate), title from CDATA, guid isPermaLink=false extracted as plain text', () => { + const adapter = new RssAdapter(); + + const records = adapter.parseFeed( + SUBREPORT_ELVIS_FIXTURE, + 'subreport-neuss', + ); + + expect(records.length).toBeGreaterThanOrEqual(1); + for (const record of records) { + expect(record.sourceType).toBe('rss'); + expect(record.sourcePortal).toBe('subreport-neuss'); + expect(record.publishedAt).toBeNull(); + expect(record.sourceUrl).toMatch( + /^https:\/\/www\.subreport-elvis\.de\//, + ); + } + + const first = records[0]; + expect(first).toBeDefined(); + // guid is `E73433797-558464` — + // fast-xml-parser represents this as { '#text': ..., '@_isPermaLink': ... }, + // extractTagText must pull out the plain '#text' value. + expect(first!.sourceNoticeId).toBe('E73433797-558464'); + + const payload = first!.ocdsPayload as { title: string }; + expect(payload.title).toMatch(/^E73433797:/); + }); + + it('never carries description HTML into the record (V5 — no stored-XSS surface)', () => { + const adapter = new RssAdapter(); + const records = adapter.parseFeed( + SUBREPORT_ELVIS_FIXTURE, + 'subreport-neuss', + ); + + for (const record of records) { + const serialized = JSON.stringify(record.ocdsPayload); + expect(serialized).not.toContain(''); + expect(serialized).not.toContain(' { + const adapter = new RssAdapter(); + const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund'); + + for (const record of records) { + const payload = record.ocdsPayload as { + buyerName: string | null; + procedureType: string | null; + legalFramework: string | null; + deadlineAt: string | null; + }; + expect(payload.buyerName).toBeNull(); + expect(payload.procedureType).toBeNull(); + expect(payload.legalFramework).toBeNull(); + expect(payload.deadlineAt).toBeNull(); + } + }); + + it('returns [] for empty XML instead of throwing', () => { + const adapter = new RssAdapter(); + expect(adapter.parseFeed('', 'empty-feed')).toEqual([]); + }); + + it('returns [] for malformed XML instead of throwing', () => { + const adapter = new RssAdapter(); + expect( + adapter.parseFeed('', 'broken-feed'), + ).toEqual([]); + }); + + it('returns [] for well-formed XML with no <item> at all', () => { + const adapter = new RssAdapter(); + const xml = + '<?xml version="1.0"?><rss version="2.0"><channel><title>Empty'; + expect(adapter.parseFeed(xml, 'no-items-feed')).toEqual([]); + }); + + it('skips a single item missing without aborting the rest', () => { + const adapter = new RssAdapter(); + const xml = ` + + Ohne Linkno-link-guid + Mit Linkhttps://example.invalid/awith-link-guid + `; + + const records = adapter.parseFeed(xml, 'mixed-feed'); + + expect(records).toHaveLength(1); + expect(records[0]?.sourceNoticeId).toBe('with-link-guid'); + }); + + it('falls back to sha256(link) for sourceNoticeId when guid is absent', () => { + const adapter = new RssAdapter(); + const xml = ` + + No Guidhttps://example.invalid/no-guid + `; + + const records = adapter.parseFeed(xml, 'no-guid-feed'); + + expect(records).toHaveLength(1); + expect(records[0]?.sourceNoticeId).toMatch(/^[a-f0-9]{40}$/); + }); + }); + + describe('normalize() dispatch', () => { + it("'rss' sourceType routes through TenderNormalizerService.normalizeBag() (proven end-to-end via tender-normalizer.service.spec.ts)", () => { + // See tender-normalizer.service.spec.ts's "generic ocdsPayload bag" + // describe block for the actual normalize() assertions — this spec + // only proves RssAdapter's OWN output shape (parseFeed), not the + // normalizer dispatch itself (kept in the normalizer's own spec file + // to avoid duplicating TenderNormalizerService test infrastructure). + const adapter = new RssAdapter(); + expect(adapter.sourceType).toBe('rss'); + }); + }); + + it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => { + const source = readFileSync(join(__dirname, 'rss.adapter.ts'), 'utf8'); + expect(source).not.toMatch(/from ['"]axios['"]/); + }); +}); diff --git a/apps/api/src/tenders/adapters/rss.adapter.ts b/apps/api/src/tenders/adapters/rss.adapter.ts new file mode 100644 index 0000000..84afb07 --- /dev/null +++ b/apps/api/src/tenders/adapters/rss.adapter.ts @@ -0,0 +1,202 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { createHash } from 'crypto'; +import { XMLParser } from 'fast-xml-parser'; +import type { RawTenderRecord, SourceType } from '../tender.types'; +import type { TenderSourceAdapter } from './tender-source-adapter.interface'; + +/** + * RssAdapter — INGEST-04. Parses admin-managed RSS feed URLs into + * RawTenderRecord[] via the same `fast-xml-parser` configuration + * doe-opendata.adapter.ts already uses for eForms-DE XML + * (`removeNSPrefix`/`ignoreAttributes`/`attributeNamePrefix` — + * 14-RESEARCH.md "RSS parsing"). + * + * `portals: ['rss']` is a SYMBOLIC placeholder, not a real hostname list + * (14-RESEARCH.md Pitfall 3): the actual feed hostnames are admin-supplied + * RUNTIME data (`TenderRssFeedSource` rows, wired in Plan 14-02 Task 2), + * added long after `SourceRegistry.register()`'s DI-boot-time denylist + * check runs. The `DENYLISTED_PORTALS` gate therefore provides ZERO + * protection for RSS feed URLs — a SEPARATE, save-time hostname/SSRF guard + * is enforced in `tender-rss-feed.service.ts` (D-14, Plan 14-02 Task 2, see + * threat T-14-02-01). Do not remove that guard under the assumption this + * adapter's `portals` array already covers it. + * + * Live-verified shapes (14-RESEARCH.md, captured 2026-07-23): + * - service.bund.de: `` present; `` uses raw numeric + * HTML character references (e.g. `Übermittlung...`), NOT CDATA — + * decoded explicitly below (Rule 1: fast-xml-parser only decodes the 5 + * predefined XML entities by default, not numeric character refs). + * - subreport-elvis: `<item><pubDate>` is ABSENT entirely (`publishedAt` + * is always null for this source); `<title>`/`<description>` are + * CDATA-wrapped (auto-merged to plain strings by fast-xml-parser, no + * decoding needed). + * + * Baseline field mapping only (title/link/guid) — `buyerName`/ + * `procedureType`/`deadlineAt` stay null. The optional service.bund.de + * `<description>` label-extraction enrichment (Erfüllungsort/ + * Vergabestelle/Angebotsfrist) is explicitly out of scope for this task + * (RESEARCH.md Pattern 3 "optional enhancement"); `<description>` is never + * read by this adapter, so no raw HTML fragment is ever carried into a + * RawTenderRecord (V5 — no stored-XSS surface, same text-only discipline + * as NetServerAdapter/CosinexAdapter). + */ +@Injectable() +export class RssAdapter implements TenderSourceAdapter { + readonly sourceType: SourceType = 'rss'; + /** Symbolic placeholder — see class docstring re: Pitfall 3. */ + readonly portals = ['rss'] as const; + + private readonly logger = new Logger(RssAdapter.name); + private readonly xmlParser = new XMLParser({ + removeNSPrefix: true, + ignoreAttributes: false, + attributeNamePrefix: '@_', + }); + + /** + * Placeholder — wired to the real `TenderRssFeedSource` internal fan-out + * (native fetch + AbortController per admin-added feed URL) in Plan + * 14-02 Task 2. Kept here only so the class satisfies + * `TenderSourceAdapter` for this task's fixture-driven `parseFeed` proof. + */ + async fetchTenders(_dayCursor: string): Promise<RawTenderRecord[]> { + return []; + } + + /** + * Pure XML -> RawTenderRecord[] mapping. Never throws (Pitfall 2 + * discipline, mirrors NetServer/cosinex): malformed/empty XML, or a feed + * with no `<channel>`/`<item>` at all, resolves to `[]`. A single item + * missing `<link>` is skipped (logger.warn) without aborting the rest. + */ + parseFeed(xml: string, feedLabel: string): RawTenderRecord[] { + const fetchedAt = new Date(); + let items: unknown[]; + + try { + const parsed = this.xmlParser.parse(xml) as { + rss?: { channel?: { item?: unknown | unknown[] } }; + }; + const channelItem = parsed.rss?.channel?.item; + items = Array.isArray(channelItem) + ? channelItem + : channelItem + ? [channelItem] + : []; + } catch (error) { + this.logger.warn( + `RSS feed '${feedLabel}' totally unparsable, returning [] (mirrors NetServer/cosinex Pitfall 2): ${(error as Error).message}`, + ); + return []; + } + + const records: RawTenderRecord[] = []; + + for (const item of items) { + try { + const node = item as Record<string, unknown>; + const link = extractTagText(node.link); + if (!link) { + this.logger.warn( + `Skipping RSS item without <link> (feed '${feedLabel}')`, + ); + continue; // no stable URL -> unusable item, skip (Pitfall 2 discipline) + } + + const rawTitle = extractTagText(node.title); + const title = rawTitle + ? decodeNumericEntities(rawTitle).trim() + : 'Unbenannte Ausschreibung'; + + const guidText = extractTagText(node.guid); + const sourceNoticeId = + guidText || + createHash('sha256').update(link).digest('hex').slice(0, 40); + + const pubDateRaw = extractTagText(node.pubDate); + const publishedAt = parseRssDate(pubDateRaw); + + records.push({ + sourceType: this.sourceType, + sourcePortal: feedLabel, + sourceNoticeId, + sourceUrl: link, + fetchedAt, + publishedAt, + eformsPayload: null, + // RSS has no eForms/OCDS structure; the extracted baseline fields + // are carried through this generic bag — same convention as + // NetServerAdapter/CosinexAdapter (13-04/13-05) — so + // TenderNormalizerService.normalizeBag() maps them without any + // RSS-specific normalizer code (D-04/D-05). + ocdsPayload: { + title, + buyerName: null, + procedureType: null, + legalFramework: null, + deadlineAt: null, + }, + }); + } catch (error) { + this.logger.warn( + `Skipping unparsable RSS item (feed '${feedLabel}'): ${(error as Error).message}`, + ); + } + } + + return records; + } +} + +/** + * fast-xml-parser represents a plain tag as a raw string/number, and a tag + * with attributes (e.g. subreport-elvis's `<guid isPermaLink="false">`) as + * `{ '@_attr': ..., '#text': value }` — mirrors + * tender-normalizer.service.ts's `textValue()` helper (kept local/ + * duplicated rather than imported: this is adapter-layer XML-shape + * handling, not normalizer-layer field mapping). + */ +function extractTagText(node: unknown): string | null { + if (node === null || node === undefined) return null; + if (typeof node === 'string') return node || null; + if (typeof node === 'number') return String(node); + if ( + typeof node === 'object' && + '#text' in (node as Record<string, unknown>) + ) { + const t = (node as Record<string, unknown>)['#text']; + if (t === null || t === undefined) return null; + return String(t); + } + return null; +} + +/** + * Rule 1 fix: fast-xml-parser only decodes the 5 predefined XML entities + * (`&` `<` `>` `"` `'`) — numeric character references + * (`Ü`, `ß`) pass through UNDECODED (confirmed live against + * fast-xml-parser v5.10.1, see class docstring). service.bund.de titles use + * numeric refs directly (not CDATA-wrapped, unlike subreport-elvis) — + * without this decode step, titles would literally show "Übermittlung + * ..." to admins/users instead of "Übermittlung...". + */ +function decodeNumericEntities(text: string): string { + return text + .replace(/&#(\d+);/g, (_match, dec: string) => + String.fromCodePoint(Number(dec)), + ) + .replace(/&#x([0-9a-fA-F]+);/g, (_match, hex: string) => + String.fromCodePoint(Number.parseInt(hex, 16)), + ); +} + +/** + * RSS `pubDate` is RFC-822 (`Thu, 23 Jul 2026 11:15:00 +0200`), parseable + * directly by `Date`. Absent/malformed -> null (subreport-elvis has no + * per-item `pubDate` at all — confirmed live, see class docstring). + */ +function parseRssDate(raw: string | null): Date | null { + if (!raw) return null; + const parsed = new Date(raw); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} diff --git a/apps/api/src/tenders/tender-normalizer.service.spec.ts b/apps/api/src/tenders/tender-normalizer.service.spec.ts index 0f26017..d65d0b7 100644 --- a/apps/api/src/tenders/tender-normalizer.service.spec.ts +++ b/apps/api/src/tenders/tender-normalizer.service.spec.ts @@ -81,7 +81,11 @@ function bagRecord( }, ): RawTenderRecord { const sourcePortal = - sourceType === 'cosinex-dtvp' ? 'cosinex-dtvp' : 'tender24'; + sourceType === 'cosinex-dtvp' + ? 'cosinex-dtvp' + : sourceType === 'rss' + ? 'service-bund' + : 'tender24'; return { sourceType, sourcePortal, @@ -317,4 +321,38 @@ describe('TenderNormalizerService', () => { expect(normalized.contentHash).toMatch(/^[a-f0-9]{64}$/); }); }); + + describe("'rss' sourceType (Phase 14, Plan 02, INGEST-04)", () => { + it('maps an rss bag through normalizeBag(): title passes through, cpvDivisions stays empty', () => { + const raw = bagRecord('rss', { + title: 'Übermittlung von Bohrungsdaten', + buyerName: null, + procedureType: null, + legalFramework: null, + deadlineAt: null, + }); + + const normalized = service.normalize(raw); + + expect(normalized.title).toBe('Übermittlung von Bohrungsdaten'); + expect(normalized.cpvDivisions).toEqual([]); + expect(normalized.cpvCodes).toEqual([]); + expect(normalized.region).toBeNull(); + expect(normalized.bundesland).toBeNull(); + expect(normalized.buyerName).toBeNull(); + expect(normalized.procedureType).toBeNull(); + expect(normalized.deadlineAt).toBeNull(); + expect(normalized.dedupKey).toBe( + `${raw.sourcePortal}:${raw.sourceNoticeId}`, + ); + }); + + it('rss title falls back to "Unbenannte Ausschreibung" when empty', () => { + const raw = bagRecord('rss', { title: '' }); + + const normalized = service.normalize(raw); + + expect(normalized.title).toBe('Unbenannte Ausschreibung'); + }); + }); }); diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts index 55777f7..4b4c110 100644 --- a/apps/api/src/tenders/tender-normalizer.service.ts +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -54,6 +54,7 @@ export class TenderNormalizerService { switch (raw.sourceType) { case 'ai-netserver': case 'cosinex-dtvp': + case 'rss': return this.normalizeBag(raw); case 'doe-opendata': default: @@ -117,10 +118,13 @@ export class TenderNormalizerService { /** * Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters - * carry a flat `{title, buyerName, procedureType, legalFramework, - * deadlineAt}` bag in `raw.ocdsPayload` — there is no eForms/OCDS - * structure to inspect. `legalFramework` is deliberately NOT mapped: - * NormalizedTenderFields has no target field for it. + * (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04) carry a flat + * `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in + * `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect. + * `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has + * no target field for it. RSS records always have `buyerName`/ + * `procedureType`/`deadlineAt` null (baseline title/link/guid mapping + * only, D-04/D-05 thin-fields deferral). */ private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields { const bag = getBagPayload(raw.ocdsPayload); diff --git a/apps/api/src/tenders/tender.types.ts b/apps/api/src/tenders/tender.types.ts index c11f8aa..6da2b34 100644 --- a/apps/api/src/tenders/tender.types.ts +++ b/apps/api/src/tenders/tender.types.ts @@ -13,8 +13,16 @@ * (Plan 13-05) adapters can be added without further breaking changes. * This union widening is the only breaking type change in Plan 13-01, * and it's contained locally (no runtime behavior change here). + * + * Phase 14, Plan 02 (INGEST-04) adds 'rss': admin-managed RSS feed URLs + * (TenderRssFeedSource, D-14), routed through the same generic-bag + * normalizer path as 'ai-netserver'/'cosinex-dtvp' (D-04/D-05). */ -export type SourceType = 'doe-opendata' | 'ai-netserver' | 'cosinex-dtvp'; +export type SourceType = + | 'doe-opendata' + | 'ai-netserver' + | 'cosinex-dtvp' + | 'rss'; /** * A single notice as fetched and lightly parsed from a source, before