From e812738c3a68768903d351248d8f70e9099004ef Mon Sep 17 00:00:00 2001 From: Schalli Date: Thu, 23 Jul 2026 13:24:36 +0200 Subject: [PATCH] feat(14-02): add TenderRssFeedSource CRUD, tick poll gate, and wire RssAdapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Global admin-managed RSS feed list (TenderRssFeedSource, D-08/D-14) with a save-time hostname/SSRF guard (TenderRssFeedSourceService) — RSS feed URLs are runtime admin input, so the code-level SourceRegistry denylist gate does not cover them; a separate check rejects DENYLISTED_PORTALS hostnames, non-http(s) schemes, and private/loopback hosts. Adds TenderSourcePollConfig.pollGranularity ('day' | 'tick', D-15): pollDueSources() branches per source — 'day' sources keep the existing lastIngestedDay gate byte-unchanged, 'tick' sources (rss) fetch on every active scheduler tick regardless of lastIngestedDay, since the day-cursor gate was built for a genuine daily batch-export API and would otherwise silently cap RSS to one fetch per calendar day. Wires RssAdapter.fetchTenders() to fan out over active feed rows (native fetch + AbortController 15s + response-size ceiling, catch-per-feed), registers it in tenders.module.ts, and seeds the 'rss' poll config active with pollGranularity='tick' plus a default-active service.bund.de feed row (subreport-elvis has no single canonical URL — zero rows seeded, admin adds relevant municipality feeds). Migration applied locally per project convention (host -> container IP). Co-Authored-By: Claude Opus 4.8 --- .../migration.sql | 20 ++ apps/api/prisma/schema.prisma | 25 ++ .../src/tenders/adapters/rss.adapter.spec.ts | 152 ++++++++++-- apps/api/src/tenders/adapters/rss.adapter.ts | 89 ++++++- .../src/tenders/dto/tender-rss-feed.dto.ts | 41 ++++ .../tenders/tender-ingestion.service.spec.ts | 53 +++++ .../src/tenders/tender-ingestion.service.ts | 35 +++ .../tenders/tender-rss-feed.service.spec.ts | 217 ++++++++++++++++++ .../src/tenders/tender-rss-feed.service.ts | 127 ++++++++++ apps/api/src/tenders/tenders.module.ts | 71 +++++- 10 files changed, 809 insertions(+), 21 deletions(-) create mode 100644 apps/api/prisma/migrations/20260723111946_add_rss_feed_source_and_poll_granularity/migration.sql create mode 100644 apps/api/src/tenders/dto/tender-rss-feed.dto.ts create mode 100644 apps/api/src/tenders/tender-rss-feed.service.spec.ts create mode 100644 apps/api/src/tenders/tender-rss-feed.service.ts diff --git a/apps/api/prisma/migrations/20260723111946_add_rss_feed_source_and_poll_granularity/migration.sql b/apps/api/prisma/migrations/20260723111946_add_rss_feed_source_and_poll_granularity/migration.sql new file mode 100644 index 0000000..150e46b --- /dev/null +++ b/apps/api/prisma/migrations/20260723111946_add_rss_feed_source_and_poll_granularity/migration.sql @@ -0,0 +1,20 @@ +-- AlterTable +ALTER TABLE "CalendarSource" ADD COLUMN "domain" TEXT; + +-- AlterTable +ALTER TABLE "TenderSourcePollConfig" ADD COLUMN "pollGranularity" TEXT NOT NULL DEFAULT 'day'; + +-- CreateTable +CREATE TABLE "TenderRssFeedSource" ( + "id" TEXT NOT NULL, + "url" TEXT NOT NULL, + "label" TEXT NOT NULL, + "isActive" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "TenderRssFeedSource_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "TenderRssFeedSource_url_key" ON "TenderRssFeedSource"("url"); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 54ef5f7..347b973 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -419,6 +419,31 @@ model TenderSourcePollConfig { pollIntervalMin Int @default(60) // D-04 default hourly isActive Boolean @default(false) lastIngestedDay DateTime? // day-cursor, NOT a timestamp (RESEARCH Pattern 1) + // Phase 14, Plan 02 (D-15/Pitfall 1): 'day' | 'tick'. 'day' sources + // (doe-opendata/ai-netserver/cosinex-dtvp) keep the lastIngestedDay-gated + // once-per-calendar-day fetch, unchanged. 'tick' sources (rss) are + // fetched on every active scheduler tick, ignoring lastIngestedDay + // entirely — the day-cursor gate was built for a genuine daily + // batch-export API and would otherwise silently cap RSS to one fetch + // per day regardless of pollIntervalMin. + pollGranularity String @default("day") createdAt DateTime @default(now()) updatedAt DateTime @updatedAt } + +// Phase 14, Plan 02 (INGEST-04, D-14) — admin-managed GLOBAL RSS feed list. +// Deliberately NO tenantId (mirrors TenderSourcePollConfig's global/ +// RLS-exempt stance, D-08: RSS feeds are public and identical for every +// tenant). Feed URLs are RUNTIME admin input — unlike the hardcoded +// NETSERVER_PORTALS/COSINEX_BASE_URL constants, the code-level +// SourceRegistry denylist gate does NOT cover this data (RESEARCH.md +// Pitfall 3); TenderRssFeedSourceService enforces a SEPARATE save-time +// hostname/SSRF guard (T-14-02-01) on create/update. +model TenderRssFeedSource { + id String @id @default(uuid()) + url String @unique + label String + isActive Boolean @default(true) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} diff --git a/apps/api/src/tenders/adapters/rss.adapter.spec.ts b/apps/api/src/tenders/adapters/rss.adapter.spec.ts index 132a978..e34ea90 100644 --- a/apps/api/src/tenders/adapters/rss.adapter.spec.ts +++ b/apps/api/src/tenders/adapters/rss.adapter.spec.ts @@ -1,6 +1,6 @@ import { readFileSync } from 'fs'; import { join } from 'path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { RssAdapter } from './rss.adapter'; /** @@ -20,16 +20,46 @@ const SUBREPORT_ELVIS_FIXTURE = readFileSync( 'utf8', ); +/** Minimal prisma-shaped fake — only `tenderRssFeedSource.findMany` is used by RssAdapter. */ +function makeFakePrisma(feeds: any[] = []) { + return { + tenderRssFeedSource: { + findMany: vi.fn(async () => feeds), + }, + } as any; +} + +/** Stubs global `fetch` to resolve with `xml` for every call (fan-out tests). */ +function stubFetchResolvingXml(xmlByUrl: Record): void { + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + const xml = xmlByUrl[url]; + if (xml === undefined) { + return { ok: false, status: 404 } as Response; + } + const bytes = Buffer.from(xml, 'utf8'); + return { + ok: true, + status: 200, + headers: new Headers({ 'content-length': String(bytes.byteLength) }), + arrayBuffer: async () => + bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), + } as unknown as Response; + }), + ); +} + describe('RssAdapter', () => { it('declares sourceType rss and the symbolic rss portal', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); expect(adapter.sourceType).toBe('rss'); expect(adapter.portals).toEqual(['rss']); }); describe('parseFeed (pure, fixture-driven)', () => { it('parses service.bund.de items: sourceType/sourcePortal set, pubDate present, numeric HTML entities decoded in title', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund'); @@ -54,7 +84,7 @@ describe('RssAdapter', () => { }); it('uses the item guid as sourceNoticeId for service.bund.de (plain-text guid, no isPermaLink attribute)', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund'); const ids = records.map((r) => r.sourceNoticeId); @@ -63,7 +93,7 @@ describe('RssAdapter', () => { }); it('parses subreport-elvis items: publishedAt null (no item pubDate), title from CDATA, guid isPermaLink=false extracted as plain text', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const records = adapter.parseFeed( SUBREPORT_ELVIS_FIXTURE, @@ -92,7 +122,7 @@ describe('RssAdapter', () => { }); it('never carries description HTML into the record (V5 — no stored-XSS surface)', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const records = adapter.parseFeed( SUBREPORT_ELVIS_FIXTURE, 'subreport-neuss', @@ -106,7 +136,7 @@ describe('RssAdapter', () => { }); it('bare-minimum bag: buyerName/procedureType/deadlineAt always null (baseline mapping only, D-04/D-05)', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const records = adapter.parseFeed(SERVICE_BUND_FIXTURE, 'service-bund'); for (const record of records) { @@ -124,26 +154,26 @@ describe('RssAdapter', () => { }); it('returns [] for empty XML instead of throwing', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); expect(adapter.parseFeed('', 'empty-feed')).toEqual([]); }); it('returns [] for malformed XML instead of throwing', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); expect( adapter.parseFeed('', 'broken-feed'), ).toEqual([]); }); it('returns [] for well-formed XML with no <item> at all', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const xml = '<?xml version="1.0"?><rss version="2.0"><channel><title>Empty'; expect(adapter.parseFeed(xml, 'no-items-feed')).toEqual([]); }); it('skips a single item missing without aborting the rest', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const xml = ` Ohne Linkno-link-guid @@ -157,7 +187,7 @@ describe('RssAdapter', () => { }); it('falls back to sha256(link) for sourceNoticeId when guid is absent', () => { - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); const xml = ` No Guidhttps://example.invalid/no-guid @@ -177,11 +207,107 @@ describe('RssAdapter', () => { // only proves RssAdapter's OWN output shape (parseFeed), not the // normalizer dispatch itself (kept in the normalizer's own spec file // to avoid duplicating TenderNormalizerService test infrastructure). - const adapter = new RssAdapter(); + const adapter = new RssAdapter(makeFakePrisma()); expect(adapter.sourceType).toBe('rss'); }); }); + describe('fetchTenders (internal fan-out over active TenderRssFeedSource rows, Plan 14-02 Task 2)', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('fetches and parses every active feed, fanning out over findMany({isActive:true})', async () => { + const feeds = [ + { id: 'f1', url: 'https://service-bund.invalid/rss.xml', label: 'service-bund', isActive: true }, + { id: 'f2', url: 'https://subreport.invalid/rss.xml', label: 'subreport-neuss', isActive: true }, + ]; + const prisma = makeFakePrisma(feeds); + stubFetchResolvingXml({ + 'https://service-bund.invalid/rss.xml': SERVICE_BUND_FIXTURE, + 'https://subreport.invalid/rss.xml': SUBREPORT_ELVIS_FIXTURE, + }); + const adapter = new RssAdapter(prisma); + + const records = await adapter.fetchTenders('2026-07-23'); + + expect(prisma.tenderRssFeedSource.findMany).toHaveBeenCalledWith({ + where: { isActive: true }, + }); + const portals = new Set(records.map((r) => r.sourcePortal)); + expect(portals).toEqual(new Set(['service-bund', 'subreport-neuss'])); + expect(records.length).toBeGreaterThan(1); + }); + + it('catch-per-feed: a feed that fails to fetch is skipped, the other feed still resolves (D-01)', async () => { + const feeds = [ + { id: 'f1', url: 'https://broken.invalid/rss.xml', label: 'broken', isActive: true }, + { id: 'f2', url: 'https://ok.invalid/rss.xml', label: 'ok-feed', isActive: true }, + ]; + const prisma = makeFakePrisma(feeds); + stubFetchResolvingXml({ + 'https://ok.invalid/rss.xml': SERVICE_BUND_FIXTURE, + // 'broken.invalid' deliberately absent -> stub resolves 404 -> throws + }); + const adapter = new RssAdapter(prisma); + + const records = await adapter.fetchTenders('2026-07-23'); + + expect(records.every((r) => r.sourcePortal === 'ok-feed')).toBe(true); + expect(records.length).toBeGreaterThanOrEqual(1); + }); + + it('returns [] when there are no active feeds', async () => { + const prisma = makeFakePrisma([]); + const adapter = new RssAdapter(prisma); + + const records = await adapter.fetchTenders('2026-07-23'); + + expect(records).toEqual([]); + }); + + it('returns [] without throwing when a feed fetch throws (network error)', async () => { + const feeds = [ + { id: 'f1', url: 'https://unreachable.invalid/rss.xml', label: 'unreachable', isActive: true }, + ]; + const prisma = makeFakePrisma(feeds); + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('network unreachable'); + }), + ); + const adapter = new RssAdapter(prisma); + + const records = await adapter.fetchTenders('2026-07-23'); + + expect(records).toEqual([]); + }); + + it('rejects a feed response exceeding the size ceiling (DoS mitigation, T-14-02-02)', async () => { + const feeds = [ + { id: 'f1', url: 'https://huge.invalid/rss.xml', label: 'huge', isActive: true }, + ]; + const prisma = makeFakePrisma(feeds); + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + return { + ok: true, + status: 200, + headers: new Headers({ 'content-length': String(11 * 1024 * 1024) }), + arrayBuffer: async () => new ArrayBuffer(0), + } as unknown as Response; + }), + ); + const adapter = new RssAdapter(prisma); + + const records = await adapter.fetchTenders('2026-07-23'); + + expect(records).toEqual([]); // oversized feed skipped, catch-per-feed (D-01) + }); + }); + it('never imports or uses axios (native fetch is the sole HTTP client convention)', () => { const source = readFileSync(join(__dirname, 'rss.adapter.ts'), 'utf8'); expect(source).not.toMatch(/from ['"]axios['"]/); diff --git a/apps/api/src/tenders/adapters/rss.adapter.ts b/apps/api/src/tenders/adapters/rss.adapter.ts index 84afb07..267b0ef 100644 --- a/apps/api/src/tenders/adapters/rss.adapter.ts +++ b/apps/api/src/tenders/adapters/rss.adapter.ts @@ -1,6 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { createHash } from 'crypto'; import { XMLParser } from 'fast-xml-parser'; +import { PrismaService } from '../../prisma/prisma.service'; import type { RawTenderRecord, SourceType } from '../tender.types'; import type { TenderSourceAdapter } from './tender-source-adapter.interface'; @@ -39,7 +40,21 @@ import type { TenderSourceAdapter } from './tender-source-adapter.interface'; * read by this adapter, so no raw HTML fragment is ever carried into a * RawTenderRecord (V5 — no stored-XSS surface, same text-only discipline * as NetServerAdapter/CosinexAdapter). + * + * `fetchTenders()` (Plan 14-02 Task 2) is an internal-fan-out adapter + * (14-RESEARCH.md Pattern 1, same shape as `NetServerAdapter`'s + * multi-portal loop): reads every `isActive` `TenderRssFeedSource` row and + * fetches+parses each feed independently, catch-per-feed (D-01 discipline) + * — one broken/unreachable feed never blocks the others in the same tick. + * `_dayCursor` is accepted for interface conformance but NOT used as a + * filter — RSS has no day-batch concept; it is polled every scheduler + * tick via `pollGranularity: 'tick'` (D-15, wired in + * `tender-ingestion.service.ts`), not gated by the day-cursor at all. */ +const RSS_FETCH_TIMEOUT_MS = 15_000; +/** RSS feeds are normally small; an admin-supplied URL is less trusted than a hardcoded portal (RESEARCH.md Security Domain, DoS mitigation). */ +const RSS_RESPONSE_SIZE_CEILING_BYTES = 10 * 1024 * 1024; + @Injectable() export class RssAdapter implements TenderSourceAdapter { readonly sourceType: SourceType = 'rss'; @@ -53,14 +68,78 @@ export class RssAdapter implements TenderSourceAdapter { attributeNamePrefix: '@_', }); + constructor(private readonly prisma: PrismaService) {} + /** - * Placeholder — wired to the real `TenderRssFeedSource` internal fan-out - * (native fetch + AbortController per admin-added feed URL) in Plan - * 14-02 Task 2. Kept here only so the class satisfies - * `TenderSourceAdapter` for this task's fixture-driven `parseFeed` proof. + * Internal fan-out over every active admin-managed feed (D-14/D-08, + * global — no tenantId). Catch-per-feed (D-01): a single feed that fails + * to fetch/parse is skipped (logger.warn), never aborting the rest. */ async fetchTenders(_dayCursor: string): Promise { - return []; + const feeds = await this.prisma.tenderRssFeedSource.findMany({ + where: { isActive: true }, + }); + + const records: RawTenderRecord[] = []; + + for (const feed of feeds) { + try { + const xml = await this.fetchFeedXml(feed.url); + records.push(...this.parseFeed(xml, feed.label)); + } catch (error) { + this.logger.warn( + `RSS feed '${feed.label}' (${feed.url}) fetch failed, skipping this feed for this tick: ${(error as Error).message}`, + ); + } + } + + return records; + } + + /** + * Native fetch + AbortController 15s timeout (project-wide convention, + * DoeOpenDataAdapter/NetServerAdapter/CosinexAdapter idiom) plus a + * response-size ceiling (T-14-02-02, DoS mitigation) — checked both via + * the `Content-Length` header (fast-path, may be absent/wrong) and the + * actually-received byte count (authoritative). + */ + private async fetchFeedXml(url: string): Promise { + const controller = new AbortController(); + const timeout = setTimeout( + () => controller.abort(), + RSS_FETCH_TIMEOUT_MS, + ); + + try { + const res = await fetch(url, { + signal: controller.signal, + redirect: 'follow', + }); + if (!res.ok) { + throw new Error(`RSS feed fetch failed: HTTP ${res.status}`); + } + + const contentLength = res.headers.get('content-length'); + if ( + contentLength && + Number(contentLength) > RSS_RESPONSE_SIZE_CEILING_BYTES + ) { + throw new Error( + `RSS feed exceeds size ceiling (Content-Length: ${contentLength} bytes)`, + ); + } + + const buffer = await res.arrayBuffer(); + if (buffer.byteLength > RSS_RESPONSE_SIZE_CEILING_BYTES) { + throw new Error( + `RSS feed exceeds size ceiling (${buffer.byteLength} bytes received)`, + ); + } + + return new TextDecoder('utf-8').decode(buffer); + } finally { + clearTimeout(timeout); + } } /** diff --git a/apps/api/src/tenders/dto/tender-rss-feed.dto.ts b/apps/api/src/tenders/dto/tender-rss-feed.dto.ts new file mode 100644 index 0000000..ec38df9 --- /dev/null +++ b/apps/api/src/tenders/dto/tender-rss-feed.dto.ts @@ -0,0 +1,41 @@ +import { + IsBoolean, + IsOptional, + IsString, + IsUrl, + MaxLength, + MinLength, +} from 'class-validator'; + +/** + * DTO for admin-managed RSS feed sources (`TenderRssFeedSource`, D-14/D-08). + * + * `@IsUrl` here is only a COARSE well-formedness check (http/https, + * protocol required). The SUBSTANTIVE SSRF/denylist guard — rejecting + * DENYLISTED_PORTALS hostnames and private/loopback hosts — is enforced in + * `TenderRssFeedSourceService.assertUrlAllowed()`, NOT here (RESEARCH.md + * Pitfall 3: an admin-supplied RSS feed URL is runtime data, added long + * after `SourceRegistry.register()`'s DI-boot-time denylist check runs — + * this DTO alone provides zero protection against a feed URL pointing at + * vergabe24/aumass or an internal host). `require_tld: false` deliberately + * lets IP-literal URLs pass THIS validation layer so the service-layer + * check can reject them with a clear, domain-specific SSRF error message + * instead of a generic "invalid URL" one. + */ +export class TenderRssFeedDto { + @IsUrl({ + protocols: ['http', 'https'], + require_protocol: true, + require_tld: false, + }) + url!: string; + + @IsString() + @MinLength(1) + @MaxLength(200) + label!: string; + + @IsOptional() + @IsBoolean() + isActive?: boolean; +} diff --git a/apps/api/src/tenders/tender-ingestion.service.spec.ts b/apps/api/src/tenders/tender-ingestion.service.spec.ts index 868b268..edd8461 100644 --- a/apps/api/src/tenders/tender-ingestion.service.spec.ts +++ b/apps/api/src/tenders/tender-ingestion.service.spec.ts @@ -299,6 +299,59 @@ describe('TenderIngestionService.pollDueSources — catch-per-source error isola }); }); +describe("TenderIngestionService.pollDueSources — pollGranularity 'tick' gate (D-15, Phase 14 Plan 02)", () => { + it("fetches a pollGranularity='tick' source on every tick, while a 'day' source with today's next-day gated-out lastIngestedDay is skipped in the SAME tick", async () => { + const prisma = makeFakePrisma({ + 'doe-opendata': { lastIngestedDay: dayDate(-1) }, // -> next day is today, gated out ('day' path unchanged) + rss: { pollGranularity: 'tick', lastIngestedDay: null }, // 'tick' — no day-cursor gate at all + }); + const doeAdapter = { fetchTenders: vi.fn() }; + const rssAdapter = { + fetchTenders: vi.fn().mockResolvedValue([ + { ...BASE_RECORD, sourcePortal: 'rss', sourceNoticeId: 'rss-1', ocid: null, dedupKey: 'rss:rss-1' }, + ]), + }; + const registry = makeFakeRegistry({ 'doe-opendata': doeAdapter, rss: rssAdapter }); + const normalizer = { normalize: vi.fn((raw: any) => raw) }; + const service = makeService(prisma, registry, normalizer); + + await service.pollDueSources(); + + expect(doeAdapter.fetchTenders).not.toHaveBeenCalled(); // 'day' gate unchanged + expect(rssAdapter.fetchTenders).toHaveBeenCalledTimes(1); // 'tick' — fetched regardless + expect(prisma.__store.tenders.size).toBe(1); + }); + + it("does NOT advance or read lastIngestedDay for a 'tick' source — it is fetched again next tick even with lastIngestedDay already set to today", async () => { + const prisma = makeFakePrisma({ + rss: { pollGranularity: 'tick', lastIngestedDay: dayDate(0) }, // today — would gate a 'day' source out entirely + }); + const rssAdapter = { fetchTenders: vi.fn().mockResolvedValue([]) }; + const registry = makeFakeRegistry({ rss: rssAdapter }); + const normalizer = { normalize: vi.fn() }; + const service = makeService(prisma, registry, normalizer); + + await service.pollDueSources(); + await service.pollDueSources(); + + expect(rssAdapter.fetchTenders).toHaveBeenCalledTimes(2); + // lastIngestedDay is never touched by the 'tick' path. + expect(prisma.__store.configs.get('rss').lastIngestedDay).toEqual(dayDate(0)); + }); + + it("passes berlinToday (not a day-cursor loop) as the single argument to a 'tick' adapter's fetchTenders", async () => { + const prisma = makeFakePrisma({ rss: { pollGranularity: 'tick' } }); + const rssAdapter = { fetchTenders: vi.fn().mockResolvedValue([]) }; + const registry = makeFakeRegistry({ rss: rssAdapter }); + const normalizer = { normalize: vi.fn() }; + const service = makeService(prisma, registry, normalizer); + + await service.pollDueSources(); + + expect(rssAdapter.fetchTenders).toHaveBeenCalledWith(berlinTodayStr()); + }); +}); + describe('TenderIngestionService.pollDueSources — dedupActive gate (D-05)', () => { it('passes dedupActive=false to resolve() when exactly one config is active', async () => { const prisma = makeFakePrisma({ 'doe-opendata': { lastIngestedDay: dayDate(-2) } }); diff --git a/apps/api/src/tenders/tender-ingestion.service.ts b/apps/api/src/tenders/tender-ingestion.service.ts index aa3ce53..aaed1da 100644 --- a/apps/api/src/tenders/tender-ingestion.service.ts +++ b/apps/api/src/tenders/tender-ingestion.service.ts @@ -70,6 +70,15 @@ function addOneDay(day: string): string { * structurally prevents a backfill flood. Rows that merely changed * (SCHEMA-02 contentHash update) or were merged as an additional source * (`created: false`) are NOT included. + * + * Phase 14, Plan 02 (D-15/RESEARCH.md Pitfall 1): each config now branches + * on `pollGranularity` ('day' | 'tick'). 'day' sources (doe-opendata/ + * ai-netserver/cosinex-dtvp) keep the exact pre-existing nextDayToFetch + * gate — zero regression. 'tick' sources (rss) skip the day-cursor gate + * entirely and fetch on every active tick, since the day-cursor mechanism + * was built for a genuine daily batch-export API and would otherwise + * silently cap a finer-grained source to one fetch per calendar day + * regardless of its configured `pollIntervalMin`. */ @Injectable() export class TenderIngestionService { @@ -123,6 +132,32 @@ export class TenderIngestionService { continue; } + if (config.pollGranularity === 'tick') { + // D-15/Pitfall 1: 'tick' sources (rss) are NOT gated on + // lastIngestedDay at all — the day-cursor gate was built for a + // genuine daily batch-export API (DÖE) and would otherwise + // silently cap these sources to one fetch per calendar day + // regardless of pollIntervalMin. Fetched unconditionally on + // every tick this config is active; lastIngestedDay is never + // read or advanced for 'tick' sources. + const rawRecords = await adapter.fetchTenders(berlinToday); + const normalized = rawRecords.map((r) => + this.normalizer.normalize(r), + ); + + for (const tender of normalized) { + const { tenderId, created } = await this.dedup.resolve(tender, { + dedupActive, + }); + if (created) newTenderIds.push(tenderId); + } + + anyDayFetched = true; // also triggers pruneExpiredTenders below, same as 'day' sources + continue; + } + + // 'day' path — UNCHANGED from before this plan (zero regression + // for doe-opendata/ai-netserver/cosinex-dtvp, D-15). let cursorDay = nextDayToFetch(config.lastIngestedDay); if (!cursorDay) { this.logger.debug( diff --git a/apps/api/src/tenders/tender-rss-feed.service.spec.ts b/apps/api/src/tenders/tender-rss-feed.service.spec.ts new file mode 100644 index 0000000..7a715a8 --- /dev/null +++ b/apps/api/src/tenders/tender-rss-feed.service.spec.ts @@ -0,0 +1,217 @@ +import { BadRequestException } from '@nestjs/common'; +import { describe, expect, it } from 'vitest'; +import { TenderRssFeedSourceService } from './tender-rss-feed.service'; + +/** + * TenderRssFeedSourceService.spec — proves the save-time hostname/SSRF + * guard (T-14-02-01, D-14/RESEARCH.md Pitfall 3): a runtime-admin-supplied + * RSS feed URL is NOT covered by the code-level SourceRegistry denylist + * gate (that only checks an adapter's statically-declared `portals` array + * at DI-boot time) — this service is the separate, independent + * enforcement point. + * + * Uses the same hand-rolled prisma-shaped fake convention as + * tender-notification-pref.service.spec.ts / tender-saved-search.service.spec.ts + * (in-memory Map, no live DB connection). + */ +function makeFakePrisma() { + const rows = new Map(); + let seq = 0; + + return { + tenderRssFeedSource: { + findMany: async ({ orderBy }: any) => { + const all = [...rows.values()]; + if (orderBy?.createdAt === 'asc') { + all.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()); + } + return all; + }, + create: async ({ data }: any) => { + seq += 1; + const row = { + id: `feed-${seq}`, + createdAt: new Date(Date.now() + seq), + updatedAt: new Date(Date.now() + seq), + ...data, + }; + rows.set(row.id, row); + return row; + }, + delete: async ({ where }: any) => { + const existing = rows.get(where.id); + rows.delete(where.id); + return existing; + }, + }, + __rows: rows, + }; +} + +describe('TenderRssFeedSourceService', () => { + describe('create() — save-time hostname/SSRF guard (T-14-02-01)', () => { + it('rejects a vergabe24.de feed URL (denylisted portal, D-14)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ + url: 'https://www.vergabe24.de/rss.xml', + label: 'vergabe24', + }), + ).rejects.toThrow(BadRequestException); + expect(prisma.__rows.size).toBe(0); + }); + + it('rejects an aumass.de feed URL (denylisted portal, D-14)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'https://aumass.de/feed', label: 'aumass' }), + ).rejects.toThrow(BadRequestException); + expect(prisma.__rows.size).toBe(0); + }); + + it('rejects a subdomain of a denylisted host (substring match on hostname)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ + url: 'https://feeds.vergabe24.de/rss.xml', + label: 'vergabe24-sub', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('accepts a valid service.bund.de feed URL', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + const created = await service.create({ + url: 'https://www.service.bund.de/Content/Globals/Functions/RSSFeed/RSSGenerator_Ausschreibungen.xml', + label: 'service-bund', + }); + + expect(created.url).toBe( + 'https://www.service.bund.de/Content/Globals/Functions/RSSFeed/RSSGenerator_Ausschreibungen.xml', + ); + expect(created.isActive).toBe(true); // default when omitted + expect(prisma.__rows.size).toBe(1); + }); + + it('rejects a non-http(s) scheme (e.g. file://)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'file:///etc/passwd', label: 'local-file' }), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects a malformed URL', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'not-a-url', label: 'broken' }), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects a loopback host (127.0.0.1, SSRF)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ + url: 'http://127.0.0.1:8080/internal-feed.xml', + label: 'internal', + }), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects "localhost" (SSRF)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'http://localhost:3000/feed', label: 'x' }), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects a private 10.x.x.x host (SSRF)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'http://10.0.0.5/feed', label: 'x' }), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects a private 192.168.x.x host (SSRF)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'http://192.168.1.1/feed', label: 'x' }), + ).rejects.toThrow(BadRequestException); + }); + + it('rejects an IPv6 loopback host ([::1], SSRF)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await expect( + service.create({ url: 'http://[::1]/feed', label: 'x' }), + ).rejects.toThrow(BadRequestException); + }); + + it('creates isActive=false when explicitly supplied', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + const created = await service.create({ + url: 'https://example-tenders.invalid/rss.xml', + label: 'inactive-feed', + isActive: false, + }); + + expect(created.isActive).toBe(false); + }); + }); + + describe('list()/remove()', () => { + it('list() returns created feeds ordered by createdAt asc', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + await service.create({ + url: 'https://a.example-tenders.invalid/rss.xml', + label: 'a', + }); + await service.create({ + url: 'https://b.example-tenders.invalid/rss.xml', + label: 'b', + }); + + const list = await service.list(); + expect(list.map((f: any) => f.label)).toEqual(['a', 'b']); + }); + + it('remove() deletes the feed by id', async () => { + const prisma = makeFakePrisma(); + const service = new TenderRssFeedSourceService(prisma as any); + + const created = await service.create({ + url: 'https://c.example-tenders.invalid/rss.xml', + label: 'c', + }); + + const result = await service.remove(created.id); + + expect(result).toEqual({ success: true }); + expect(prisma.__rows.size).toBe(0); + }); + }); +}); diff --git a/apps/api/src/tenders/tender-rss-feed.service.ts b/apps/api/src/tenders/tender-rss-feed.service.ts new file mode 100644 index 0000000..0f97055 --- /dev/null +++ b/apps/api/src/tenders/tender-rss-feed.service.ts @@ -0,0 +1,127 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import type { TenderRssFeedDto } from './dto/tender-rss-feed.dto'; +import { DENYLISTED_PORTALS } from './source-registry'; + +/** + * TenderRssFeedSourceService — admin CRUD for the GLOBAL RSS feed list + * (`TenderRssFeedSource`, D-14/D-08). No `forTenant()`/RLS — this is + * platform-wide config, mirroring `TenderSourcePollConfig`'s stance. + * + * Save-time hostname/SSRF guard (T-14-02-01, RESEARCH.md Pitfall 3): RSS + * feed URLs are RUNTIME admin input, added long after + * `SourceRegistry.register()`'s DI-boot-time `DENYLISTED_PORTALS` check + * runs — that gate provides ZERO protection here. This service is the + * SEPARATE, independent enforcement point: reject non-http(s) schemes, + * reject any hostname containing a `DENYLISTED_PORTALS` entry (same + * constant as the code-level gate — single source of truth, D-14), and + * reject private/loopback hosts (SSRF guard, analog to T-10-06). Runs on + * BOTH create and update paths. + */ +@Injectable() +export class TenderRssFeedSourceService { + constructor(private readonly prisma: PrismaService) {} + + async list() { + return this.prisma.tenderRssFeedSource.findMany({ + orderBy: { createdAt: 'asc' }, + }); + } + + async create(dto: TenderRssFeedDto) { + this.assertUrlAllowed(dto.url); + + return this.prisma.tenderRssFeedSource.create({ + data: { + url: dto.url, + label: dto.label, + isActive: dto.isActive ?? true, + }, + }); + } + + async remove(id: string) { + await this.prisma.tenderRssFeedSource.delete({ where: { id } }); + return { success: true }; + } + + /** + * Rejects (BadRequestException, HTTP 400): + * - non-http(s) schemes (e.g. `file://`, `ftp://`, `javascript:`) + * - hostnames containing a DENYLISTED_PORTALS entry (vergabe24/aumass) + * - private/loopback/link-local IP-literal hosts (SSRF) + * + * Deliberately string/IP-literal-based, not DNS-resolution-based — same + * scope as the existing SSRF-guard pattern documented for + * NETSERVER_PORTALS/COSINEX_BASE_URL (T-10-06); resolving a hostname to + * check its IP at save-time would itself be an SSRF-adjacent action and + * is out of scope for this task. + */ + private assertUrlAllowed(rawUrl: string): void { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new BadRequestException('Ungültige URL.'); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new BadRequestException( + 'Nur http(s)-URLs sind für RSS-Feeds erlaubt.', + ); + } + + const hostname = parsed.hostname.toLowerCase(); + + if ( + (DENYLISTED_PORTALS as readonly string[]).some((portal) => + hostname.includes(portal), + ) + ) { + throw new BadRequestException( + `Der Host '${hostname}' ist AGB-seitig für automatisierten Zugriff gesperrt (Denylist) und darf nicht als RSS-Feed hinterlegt werden.`, + ); + } + + if (isPrivateOrLoopbackHost(hostname)) { + throw new BadRequestException( + `Der Host '${hostname}' ist ein privater/loopback-Host und darf nicht als RSS-Feed hinterlegt werden (SSRF-Schutz).`, + ); + } + } +} + +/** + * Best-effort, literal-only private/loopback/link-local host check (SSRF + * guard, T-14-02-01) — matches on the hostname string as supplied, no DNS + * resolution (see assertUrlAllowed docstring). + */ +function isPrivateOrLoopbackHost(rawHostname: string): boolean { + // WHATWG URL keeps IPv6 literals bracketed (e.g. `new URL('http://[::1]/').hostname === '[::1]'`) — strip for the checks below. + const hostname = rawHostname.replace(/^\[|\]$/g, ''); + + if (hostname === 'localhost' || hostname.endsWith('.localhost')) { + return true; + } + if (hostname === '::1' || hostname === '0.0.0.0') { + return true; + } + + const ipv4Match = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); + if (ipv4Match) { + const [a, b] = ipv4Match.slice(1, 3).map(Number); + if (a === 127) return true; // loopback (127.0.0.0/8) + if (a === 10) return true; // private (10.0.0.0/8) + if (a === 172 && b >= 16 && b <= 31) return true; // private (172.16.0.0/12) + if (a === 192 && b === 168) return true; // private (192.168.0.0/16) + if (a === 169 && b === 254) return true; // link-local (169.254.0.0/16) + if (a === 0) return true; // "this network" + return false; + } + + // Bare IPv6 literal ranges (unique-local fc00::/7, link-local fe80::/10). + if (hostname.startsWith('fc') || hostname.startsWith('fd')) return true; + if (hostname.startsWith('fe80:')) return true; + + return false; +} diff --git a/apps/api/src/tenders/tenders.module.ts b/apps/api/src/tenders/tenders.module.ts index 7acd35c..141784f 100644 --- a/apps/api/src/tenders/tenders.module.ts +++ b/apps/api/src/tenders/tenders.module.ts @@ -6,6 +6,7 @@ import { SettingsModule } from '../settings/settings.module'; import { CosinexAdapter } from './adapters/cosinex.adapter'; import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter'; import { NetServerAdapter } from './adapters/netserver.adapter'; +import { RssAdapter } from './adapters/rss.adapter'; import { SourceRegistry } from './source-registry'; import { seedTendersModule } from './tenders.seed'; import { TenderDedupService } from './tender-dedup.service'; @@ -15,6 +16,7 @@ import { TenderMailService } from './tender-mail.service'; import { TenderMatchingService } from './tender-matching.service'; import { TenderNormalizerService } from './tender-normalizer.service'; import { TenderNotificationPrefService } from './tender-notification-pref.service'; +import { TenderRssFeedSourceService } from './tender-rss-feed.service'; import { TenderSavedSearchService } from './tender-saved-search.service'; import { TenderSchedulerService } from './tender-scheduler.service'; import { TenderTriageService } from './tender-triage.service'; @@ -93,6 +95,15 @@ import { TendersController } from './tenders.controller'; * final Wave-4 additive `registry.register(...)` call. Its * `TenderSourcePollConfig` row is likewise seeded with `isActive: false` * (D-02: activation is a later admin/seed decision, Phase 14 UI). + * + * Phase 14, Plan 02 (INGEST-04): adds `RssAdapter` (registered alongside + * the existing adapters — `rss` is not denylisted) and + * `TenderRssFeedSourceService` (admin CRUD for the global feed list, + * D-14). Unlike ai-netserver/cosinex-dtvp, the `rss` `TenderSourcePollConfig` + * seed is `isActive: true` with `pollGranularity: 'tick'` (D-15) — + * service.bund.de is seeded as a default-active `TenderRssFeedSource` row + * (RESEARCH.md Open Question 3), so RSS ingestion is live out of the box, + * not "framework ready, activation deferred" like the Phase 13 sources. */ @Module({ imports: [ModuleRegistryModule, SettingsModule], @@ -101,6 +112,7 @@ import { TendersController } from './tenders.controller'; DoeOpenDataAdapter, NetServerAdapter, CosinexAdapter, + RssAdapter, SourceRegistry, TenderNormalizerService, TenderDedupService, @@ -112,6 +124,7 @@ import { TendersController } from './tenders.controller'; TenderMailService, TenderDigestScheduler, TenderNotificationPrefService, + TenderRssFeedSourceService, ], }) export class TendersModule implements OnModuleInit { @@ -124,18 +137,24 @@ export class TendersModule implements OnModuleInit { private readonly doeAdapter: DoeOpenDataAdapter, private readonly netServerAdapter: NetServerAdapter, private readonly cosinexAdapter: CosinexAdapter, + private readonly rssAdapter: RssAdapter, ) {} async onModuleInit(): Promise { try { // D-06/INGEST-07: registration itself is the denylist-gate enforcement // point — SourceRegistry.register() throws for any adapter serving a - // denylisted portal. DoeOpenDataAdapter, NetServerAdapter, and - // CosinexAdapter are all legitimate (none of tender24/lhs-vpbw/ - // vergabe.landbw/cosinex-dtvp are on the denylist). + // denylisted portal. DoeOpenDataAdapter, NetServerAdapter, + // CosinexAdapter, and RssAdapter are all legitimate (none of + // tender24/lhs-vpbw/vergabe.landbw/cosinex-dtvp/rss are on the + // denylist). Note: RssAdapter's `portals: ['rss']` is a SYMBOLIC + // placeholder — the actual admin-supplied feed hostnames are NOT + // covered by this gate at all (RESEARCH.md Pitfall 3); that runtime + // check lives in TenderRssFeedSourceService instead (D-14). this.sourceRegistry.register(this.doeAdapter); this.sourceRegistry.register(this.netServerAdapter); this.sourceRegistry.register(this.cosinexAdapter); + this.sourceRegistry.register(this.rssAdapter); this.logger.log( `Registered source adapters: ${this.sourceRegistry .activeAdapters() @@ -211,5 +230,51 @@ export class TendersModule implements OnModuleInit { } catch (error) { this.logger.error('Failed to seed cosinex-dtvp poll config', error); } + + try { + // Phase 14, Plan 02 (INGEST-04, D-15): seed the rss poll config with + // pollGranularity: 'tick' (fetched every active scheduler tick, NOT + // gated by lastIngestedDay — see tender-ingestion.service.ts) and + // isActive: true — unlike ai-netserver/cosinex-dtvp, RSS is live by + // default (RESEARCH.md Open Question 3: service.bund.de is a safe, + // genuinely national default feed, seeded below). + await this.prisma.tenderSourcePollConfig.upsert({ + where: { sourceType: 'rss' }, + update: {}, + create: { + sourceType: 'rss', + pollIntervalMin: 60, + isActive: true, + pollGranularity: 'tick', + }, + }); + this.logger.log("rss poll config seeded (active, pollGranularity='tick')"); + } catch (error) { + this.logger.error('Failed to seed rss poll config', error); + } + + try { + // Phase 14, Plan 02 (D-14, RESEARCH.md Open Question 3): seed a + // single default-active service.bund.de feed row — the one genuinely + // national/global RSS source. subreport-elvis has no single + // canonical URL (per-municipality instances, RESEARCH.md Pitfall 2) + // — deliberately ZERO subreport-elvis rows seeded; admins add the + // municipality feeds relevant to them via the RSS-Feeds admin UI + // (Plan 14-02 Task 3). + await this.prisma.tenderRssFeedSource.upsert({ + where: { + url: 'https://www.service.bund.de/Content/Globals/Functions/RSSFeed/RSSGenerator_Ausschreibungen.xml', + }, + update: {}, + create: { + url: 'https://www.service.bund.de/Content/Globals/Functions/RSSFeed/RSSGenerator_Ausschreibungen.xml', + label: 'service-bund', + isActive: true, + }, + }); + this.logger.log('service.bund.de default RSS feed seeded (active)'); + } catch (error) { + this.logger.error('Failed to seed service.bund.de RSS feed', error); + } } }