feat(14-02): add TenderRssFeedSource CRUD, tick poll gate, and wire RssAdapter
Global admin-managed RSS feed list (TenderRssFeedSource, D-08/D-14) with
a save-time hostname/SSRF guard (TenderRssFeedSourceService) — RSS feed
URLs are runtime admin input, so the code-level SourceRegistry denylist
gate does not cover them; a separate check rejects DENYLISTED_PORTALS
hostnames, non-http(s) schemes, and private/loopback hosts.
Adds TenderSourcePollConfig.pollGranularity ('day' | 'tick', D-15):
pollDueSources() branches per source — 'day' sources keep the existing
lastIngestedDay gate byte-unchanged, 'tick' sources (rss) fetch on every
active scheduler tick regardless of lastIngestedDay, since the day-cursor
gate was built for a genuine daily batch-export API and would otherwise
silently cap RSS to one fetch per calendar day.
Wires RssAdapter.fetchTenders() to fan out over active feed rows (native
fetch + AbortController 15s + response-size ceiling, catch-per-feed),
registers it in tenders.module.ts, and seeds the 'rss' poll config
active with pollGranularity='tick' plus a default-active service.bund.de
feed row (subreport-elvis has no single canonical URL — zero rows seeded,
admin adds relevant municipality feeds).
Migration applied locally per project convention (host -> container IP).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { createHash } from 'crypto';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { PrismaService } from '../../prisma/prisma.service';
|
||||
import type { RawTenderRecord, SourceType } from '../tender.types';
|
||||
import type { TenderSourceAdapter } from './tender-source-adapter.interface';
|
||||
|
||||
@@ -39,7 +40,21 @@ import type { TenderSourceAdapter } from './tender-source-adapter.interface';
|
||||
* read by this adapter, so no raw HTML fragment is ever carried into a
|
||||
* RawTenderRecord (V5 — no stored-XSS surface, same text-only discipline
|
||||
* as NetServerAdapter/CosinexAdapter).
|
||||
*
|
||||
* `fetchTenders()` (Plan 14-02 Task 2) is an internal-fan-out adapter
|
||||
* (14-RESEARCH.md Pattern 1, same shape as `NetServerAdapter`'s
|
||||
* multi-portal loop): reads every `isActive` `TenderRssFeedSource` row and
|
||||
* fetches+parses each feed independently, catch-per-feed (D-01 discipline)
|
||||
* — one broken/unreachable feed never blocks the others in the same tick.
|
||||
* `_dayCursor` is accepted for interface conformance but NOT used as a
|
||||
* filter — RSS has no day-batch concept; it is polled every scheduler
|
||||
* tick via `pollGranularity: 'tick'` (D-15, wired in
|
||||
* `tender-ingestion.service.ts`), not gated by the day-cursor at all.
|
||||
*/
|
||||
const RSS_FETCH_TIMEOUT_MS = 15_000;
|
||||
/** RSS feeds are normally small; an admin-supplied URL is less trusted than a hardcoded portal (RESEARCH.md Security Domain, DoS mitigation). */
|
||||
const RSS_RESPONSE_SIZE_CEILING_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
@Injectable()
|
||||
export class RssAdapter implements TenderSourceAdapter {
|
||||
readonly sourceType: SourceType = 'rss';
|
||||
@@ -53,14 +68,78 @@ export class RssAdapter implements TenderSourceAdapter {
|
||||
attributeNamePrefix: '@_',
|
||||
});
|
||||
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
/**
|
||||
* Placeholder — wired to the real `TenderRssFeedSource` internal fan-out
|
||||
* (native fetch + AbortController per admin-added feed URL) in Plan
|
||||
* 14-02 Task 2. Kept here only so the class satisfies
|
||||
* `TenderSourceAdapter` for this task's fixture-driven `parseFeed` proof.
|
||||
* Internal fan-out over every active admin-managed feed (D-14/D-08,
|
||||
* global — no tenantId). Catch-per-feed (D-01): a single feed that fails
|
||||
* to fetch/parse is skipped (logger.warn), never aborting the rest.
|
||||
*/
|
||||
async fetchTenders(_dayCursor: string): Promise<RawTenderRecord[]> {
|
||||
return [];
|
||||
const feeds = await this.prisma.tenderRssFeedSource.findMany({
|
||||
where: { isActive: true },
|
||||
});
|
||||
|
||||
const records: RawTenderRecord[] = [];
|
||||
|
||||
for (const feed of feeds) {
|
||||
try {
|
||||
const xml = await this.fetchFeedXml(feed.url);
|
||||
records.push(...this.parseFeed(xml, feed.label));
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`RSS feed '${feed.label}' (${feed.url}) fetch failed, skipping this feed for this tick: ${(error as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Native fetch + AbortController 15s timeout (project-wide convention,
|
||||
* DoeOpenDataAdapter/NetServerAdapter/CosinexAdapter idiom) plus a
|
||||
* response-size ceiling (T-14-02-02, DoS mitigation) — checked both via
|
||||
* the `Content-Length` header (fast-path, may be absent/wrong) and the
|
||||
* actually-received byte count (authoritative).
|
||||
*/
|
||||
private async fetchFeedXml(url: string): Promise<string> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(
|
||||
() => controller.abort(),
|
||||
RSS_FETCH_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
redirect: 'follow',
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`RSS feed fetch failed: HTTP ${res.status}`);
|
||||
}
|
||||
|
||||
const contentLength = res.headers.get('content-length');
|
||||
if (
|
||||
contentLength &&
|
||||
Number(contentLength) > RSS_RESPONSE_SIZE_CEILING_BYTES
|
||||
) {
|
||||
throw new Error(
|
||||
`RSS feed exceeds size ceiling (Content-Length: ${contentLength} bytes)`,
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = await res.arrayBuffer();
|
||||
if (buffer.byteLength > RSS_RESPONSE_SIZE_CEILING_BYTES) {
|
||||
throw new Error(
|
||||
`RSS feed exceeds size ceiling (${buffer.byteLength} bytes received)`,
|
||||
);
|
||||
}
|
||||
|
||||
return new TextDecoder('utf-8').decode(buffer);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user