feat(14-02): add TenderRssFeedSource CRUD, tick poll gate, and wire RssAdapter
Global admin-managed RSS feed list (TenderRssFeedSource, D-08/D-14) with
a save-time hostname/SSRF guard (TenderRssFeedSourceService) — RSS feed
URLs are runtime admin input, so the code-level SourceRegistry denylist
gate does not cover them; a separate check rejects DENYLISTED_PORTALS
hostnames, non-http(s) schemes, and private/loopback hosts.
Adds TenderSourcePollConfig.pollGranularity ('day' | 'tick', D-15):
pollDueSources() branches per source — 'day' sources keep the existing
lastIngestedDay gate byte-unchanged, 'tick' sources (rss) fetch on every
active scheduler tick regardless of lastIngestedDay, since the day-cursor
gate was built for a genuine daily batch-export API and would otherwise
silently cap RSS to one fetch per calendar day.
Wires RssAdapter.fetchTenders() to fan out over active feed rows (native
fetch + AbortController 15s + response-size ceiling, catch-per-feed),
registers it in tenders.module.ts, and seeds the 'rss' poll config
active with pollGranularity='tick' plus a default-active service.bund.de
feed row (subreport-elvis has no single canonical URL — zero rows seeded,
admin adds relevant municipality feeds).
Migration applied locally per project convention (host -> container IP).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import { SettingsModule } from '../settings/settings.module';
|
||||
import { CosinexAdapter } from './adapters/cosinex.adapter';
|
||||
import { DoeOpenDataAdapter } from './adapters/doe-opendata.adapter';
|
||||
import { NetServerAdapter } from './adapters/netserver.adapter';
|
||||
import { RssAdapter } from './adapters/rss.adapter';
|
||||
import { SourceRegistry } from './source-registry';
|
||||
import { seedTendersModule } from './tenders.seed';
|
||||
import { TenderDedupService } from './tender-dedup.service';
|
||||
@@ -15,6 +16,7 @@ import { TenderMailService } from './tender-mail.service';
|
||||
import { TenderMatchingService } from './tender-matching.service';
|
||||
import { TenderNormalizerService } from './tender-normalizer.service';
|
||||
import { TenderNotificationPrefService } from './tender-notification-pref.service';
|
||||
import { TenderRssFeedSourceService } from './tender-rss-feed.service';
|
||||
import { TenderSavedSearchService } from './tender-saved-search.service';
|
||||
import { TenderSchedulerService } from './tender-scheduler.service';
|
||||
import { TenderTriageService } from './tender-triage.service';
|
||||
@@ -93,6 +95,15 @@ import { TendersController } from './tenders.controller';
|
||||
* final Wave-4 additive `registry.register(...)` call. Its
|
||||
* `TenderSourcePollConfig` row is likewise seeded with `isActive: false`
|
||||
* (D-02: activation is a later admin/seed decision, Phase 14 UI).
|
||||
*
|
||||
* Phase 14, Plan 02 (INGEST-04): adds `RssAdapter` (registered alongside
|
||||
* the existing adapters — `rss` is not denylisted) and
|
||||
* `TenderRssFeedSourceService` (admin CRUD for the global feed list,
|
||||
* D-14). Unlike ai-netserver/cosinex-dtvp, the `rss` `TenderSourcePollConfig`
|
||||
* seed is `isActive: true` with `pollGranularity: 'tick'` (D-15) —
|
||||
* service.bund.de is seeded as a default-active `TenderRssFeedSource` row
|
||||
* (RESEARCH.md Open Question 3), so RSS ingestion is live out of the box,
|
||||
* not "framework ready, activation deferred" like the Phase 13 sources.
|
||||
*/
|
||||
@Module({
|
||||
imports: [ModuleRegistryModule, SettingsModule],
|
||||
@@ -101,6 +112,7 @@ import { TendersController } from './tenders.controller';
|
||||
DoeOpenDataAdapter,
|
||||
NetServerAdapter,
|
||||
CosinexAdapter,
|
||||
RssAdapter,
|
||||
SourceRegistry,
|
||||
TenderNormalizerService,
|
||||
TenderDedupService,
|
||||
@@ -112,6 +124,7 @@ import { TendersController } from './tenders.controller';
|
||||
TenderMailService,
|
||||
TenderDigestScheduler,
|
||||
TenderNotificationPrefService,
|
||||
TenderRssFeedSourceService,
|
||||
],
|
||||
})
|
||||
export class TendersModule implements OnModuleInit {
|
||||
@@ -124,18 +137,24 @@ export class TendersModule implements OnModuleInit {
|
||||
private readonly doeAdapter: DoeOpenDataAdapter,
|
||||
private readonly netServerAdapter: NetServerAdapter,
|
||||
private readonly cosinexAdapter: CosinexAdapter,
|
||||
private readonly rssAdapter: RssAdapter,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
try {
|
||||
// D-06/INGEST-07: registration itself is the denylist-gate enforcement
|
||||
// point — SourceRegistry.register() throws for any adapter serving a
|
||||
// denylisted portal. DoeOpenDataAdapter, NetServerAdapter, and
|
||||
// CosinexAdapter are all legitimate (none of tender24/lhs-vpbw/
|
||||
// vergabe.landbw/cosinex-dtvp are on the denylist).
|
||||
// denylisted portal. DoeOpenDataAdapter, NetServerAdapter,
|
||||
// CosinexAdapter, and RssAdapter are all legitimate (none of
|
||||
// tender24/lhs-vpbw/vergabe.landbw/cosinex-dtvp/rss are on the
|
||||
// denylist). Note: RssAdapter's `portals: ['rss']` is a SYMBOLIC
|
||||
// placeholder — the actual admin-supplied feed hostnames are NOT
|
||||
// covered by this gate at all (RESEARCH.md Pitfall 3); that runtime
|
||||
// check lives in TenderRssFeedSourceService instead (D-14).
|
||||
this.sourceRegistry.register(this.doeAdapter);
|
||||
this.sourceRegistry.register(this.netServerAdapter);
|
||||
this.sourceRegistry.register(this.cosinexAdapter);
|
||||
this.sourceRegistry.register(this.rssAdapter);
|
||||
this.logger.log(
|
||||
`Registered source adapters: ${this.sourceRegistry
|
||||
.activeAdapters()
|
||||
@@ -211,5 +230,51 @@ export class TendersModule implements OnModuleInit {
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to seed cosinex-dtvp poll config', error);
|
||||
}
|
||||
|
||||
try {
|
||||
// Phase 14, Plan 02 (INGEST-04, D-15): seed the rss poll config with
|
||||
// pollGranularity: 'tick' (fetched every active scheduler tick, NOT
|
||||
// gated by lastIngestedDay — see tender-ingestion.service.ts) and
|
||||
// isActive: true — unlike ai-netserver/cosinex-dtvp, RSS is live by
|
||||
// default (RESEARCH.md Open Question 3: service.bund.de is a safe,
|
||||
// genuinely national default feed, seeded below).
|
||||
await this.prisma.tenderSourcePollConfig.upsert({
|
||||
where: { sourceType: 'rss' },
|
||||
update: {},
|
||||
create: {
|
||||
sourceType: 'rss',
|
||||
pollIntervalMin: 60,
|
||||
isActive: true,
|
||||
pollGranularity: 'tick',
|
||||
},
|
||||
});
|
||||
this.logger.log("rss poll config seeded (active, pollGranularity='tick')");
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to seed rss poll config', error);
|
||||
}
|
||||
|
||||
try {
|
||||
// Phase 14, Plan 02 (D-14, RESEARCH.md Open Question 3): seed a
|
||||
// single default-active service.bund.de feed row — the one genuinely
|
||||
// national/global RSS source. subreport-elvis has no single
|
||||
// canonical URL (per-municipality instances, RESEARCH.md Pitfall 2)
|
||||
// — deliberately ZERO subreport-elvis rows seeded; admins add the
|
||||
// municipality feeds relevant to them via the RSS-Feeds admin UI
|
||||
// (Plan 14-02 Task 3).
|
||||
await this.prisma.tenderRssFeedSource.upsert({
|
||||
where: {
|
||||
url: 'https://www.service.bund.de/Content/Globals/Functions/RSSFeed/RSSGenerator_Ausschreibungen.xml',
|
||||
},
|
||||
update: {},
|
||||
create: {
|
||||
url: 'https://www.service.bund.de/Content/Globals/Functions/RSSFeed/RSSGenerator_Ausschreibungen.xml',
|
||||
label: 'service-bund',
|
||||
isActive: true,
|
||||
},
|
||||
});
|
||||
this.logger.log('service.bund.de default RSS feed seeded (active)');
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to seed service.bund.de RSS feed', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user