diff --git a/apps/api/src/tenders/adapters/email-alert.adapter.ts b/apps/api/src/tenders/adapters/email-alert.adapter.ts new file mode 100644 index 0000000..2f6b7d3 --- /dev/null +++ b/apps/api/src/tenders/adapters/email-alert.adapter.ts @@ -0,0 +1,105 @@ +import { Injectable, Logger } from '@nestjs/common'; +import * as cheerio from 'cheerio'; +import { createHash } from 'crypto'; +import type { RawTenderRecord, SourceType } from '../tender.types'; +import type { TenderSourceAdapter } from './tender-source-adapter.interface'; + +/** + * EmailAlertAdapter — INGEST-05. Extracts candidate tender detail links and + * a display title from a tenant's portal-alert mailbox, generically (D-04): + * no per-portal parser, since no concrete alert-sending portals are known + * ahead of time (14-CONTEXT.md). Feld-Armut (no CPV/buyer/deadline) is a + * deliberate, documented consequence (D-05) — the same deferral already + * accepted for the NetServer/cosinex/RSS scraper adapters. + * + * Task 1 (this file, generic extraction + normalizer dispatch) ships the + * three pure helpers below plus a contract-complete but not-yet-wired + * `fetchTenders()`. Task 2 replaces the constructor/fetchTenders body with + * the real per-tenant fan-out over `TenderEmailConfig` rows, consuming + * `InboxProvider.fetchMessages` from the shared `inbox/` module (Plan 14-01). + * + * Security (T-14-03-03, stored-XSS guard): only plain-text `title` and raw + * `href` URL strings are ever extracted from an alert email body — cheerio + * `.attr('href')`/plaintext regex only, NEVER `.html()` — so no raw email + * markup ever crosses into a RawTenderRecord/Tender row. + */ + +/** + * Footer/unsubscribe-noise link fragments to strip from extracted candidate + * links (D-04) — matches RESEARCH.md's generic link-extraction example. + */ +const FOOTER_NOISE = ['unsubscribe', 'abmelden', 'einstellungen', 'preferences']; + +/** Safety valve against link-spam producing excess records per email. */ +const MAX_LINKS_PER_EMAIL = 5; + +/** + * Extracts deduped, absolute http(s) candidate detail links from an alert + * email body (D-04 — generic evaluation, no per-portal/sender-domain + * parser). HTML body is parsed with cheerio (`a[href]`, footer-noise + * filtered, capped at MAX_LINKS_PER_EMAIL); when no HTML body is available + * (`bodyHtml === null`), falls back to a plaintext URL regex over `bodyText`. + */ +export function extractCandidateLinks( + bodyHtml: string | null, + bodyText: string, +): string[] { + let links: string[]; + if (bodyHtml) { + const $ = cheerio.load(bodyHtml); + links = $('a[href]') + .map((_, el) => $(el).attr('href')) + .get() + .filter( + (href): href is string => + typeof href === 'string' && /^https?:\/\//i.test(href), + ); + } else { + links = Array.from(bodyText.matchAll(/https?:\/\/[^\s"'<>]+/g)).map( + (m) => m[0], + ); + } + + return Array.from(new Set(links)) + .filter((url) => !FOOTER_NOISE.some((noise) => url.toLowerCase().includes(noise))) + .slice(0, MAX_LINKS_PER_EMAIL); +} + +/** + * Derives a display title for an email-sourced tender: the trimmed subject, + * else the first non-empty body line, else the same fallback + * normalizeBag() already uses for other thin-field sources. + */ +export function titleFromEmail(subject: string, bodyText: string): string { + if (subject.trim()) return subject.trim(); + const firstLine = bodyText + .split('\n') + .map((l) => l.trim()) + .find(Boolean); + return firstLine || 'Unbenannte Ausschreibung'; +} + +/** + * Stable per-link id: sha256(link).slice(0,40) — used as sourceNoticeId + * since alert-email links carry no portal-native notice id. + */ +export function sourceNoticeIdFor(link: string): string { + return createHash('sha256').update(link).digest('hex').slice(0, 40); +} + +@Injectable() +export class EmailAlertAdapter implements TenderSourceAdapter { + readonly sourceType: SourceType = 'email-alert'; + readonly portals = ['email-alert'] as const; + + protected readonly logger = new Logger(EmailAlertAdapter.name); + + /** + * Task 1 placeholder — keeps the class contract-complete (TenderSourceAdapter) + * for this task's generic-extraction-only scope. Task 2 replaces this with + * the real per-tenant fan-out over TenderEmailConfig + InboxProvider. + */ + async fetchTenders(_dayCursor: string): Promise { + return []; + } +} diff --git a/apps/api/src/tenders/tender-normalizer.service.spec.ts b/apps/api/src/tenders/tender-normalizer.service.spec.ts index d65d0b7..a4b3afa 100644 --- a/apps/api/src/tenders/tender-normalizer.service.spec.ts +++ b/apps/api/src/tenders/tender-normalizer.service.spec.ts @@ -85,7 +85,9 @@ function bagRecord( ? 'cosinex-dtvp' : sourceType === 'rss' ? 'service-bund' - : 'tender24'; + : sourceType === 'email-alert' + ? 'email-alert' + : 'tender24'; return { sourceType, sourcePortal, @@ -355,4 +357,40 @@ describe('TenderNormalizerService', () => { expect(normalized.title).toBe('Unbenannte Ausschreibung'); }); }); + + describe("'email-alert' sourceType (Phase 14, Plan 03, INGEST-05)", () => { + it('maps an email-alert bag through normalizeBag(): title passes through, cpvDivisions stays empty', () => { + const raw = bagRecord('email-alert', { + title: 'Vergabebekanntmachung: Straßenbauarbeiten Musterstadt', + buyerName: null, + procedureType: null, + legalFramework: null, + deadlineAt: null, + }); + + const normalized = service.normalize(raw); + + expect(normalized.title).toBe( + 'Vergabebekanntmachung: Straßenbauarbeiten Musterstadt', + ); + expect(normalized.cpvDivisions).toEqual([]); + expect(normalized.cpvCodes).toEqual([]); + expect(normalized.region).toBeNull(); + expect(normalized.bundesland).toBeNull(); + expect(normalized.buyerName).toBeNull(); + expect(normalized.procedureType).toBeNull(); + expect(normalized.deadlineAt).toBeNull(); + expect(normalized.dedupKey).toBe( + `${raw.sourcePortal}:${raw.sourceNoticeId}`, + ); + }); + + it('email-alert title falls back to "Unbenannte Ausschreibung" when empty', () => { + const raw = bagRecord('email-alert', { title: '' }); + + const normalized = service.normalize(raw); + + expect(normalized.title).toBe('Unbenannte Ausschreibung'); + }); + }); }); diff --git a/apps/api/src/tenders/tender-normalizer.service.ts b/apps/api/src/tenders/tender-normalizer.service.ts index 4b4c110..d59a99c 100644 --- a/apps/api/src/tenders/tender-normalizer.service.ts +++ b/apps/api/src/tenders/tender-normalizer.service.ts @@ -55,6 +55,7 @@ export class TenderNormalizerService { case 'ai-netserver': case 'cosinex-dtvp': case 'rss': + case 'email-alert': return this.normalizeBag(raw); case 'doe-opendata': default: @@ -118,13 +119,14 @@ export class TenderNormalizerService { /** * Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters - * (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04) carry a flat + * (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04; and since + * Phase 14 Plan 03, the EmailAlertAdapter, INGEST-05) carry a flat * `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in * `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect. * `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has - * no target field for it. RSS records always have `buyerName`/ - * `procedureType`/`deadlineAt` null (baseline title/link/guid mapping - * only, D-04/D-05 thin-fields deferral). + * no target field for it. RSS/email-alert records always have + * `buyerName`/`procedureType`/`deadlineAt` null (baseline title/link/guid + * mapping only, D-04/D-05 thin-fields deferral). */ private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields { const bag = getBagPayload(raw.ocdsPayload); diff --git a/apps/api/src/tenders/tender.types.ts b/apps/api/src/tenders/tender.types.ts index 6da2b34..fa3e608 100644 --- a/apps/api/src/tenders/tender.types.ts +++ b/apps/api/src/tenders/tender.types.ts @@ -17,12 +17,19 @@ * Phase 14, Plan 02 (INGEST-04) adds 'rss': admin-managed RSS feed URLs * (TenderRssFeedSource, D-14), routed through the same generic-bag * normalizer path as 'ai-netserver'/'cosinex-dtvp' (D-04/D-05). + * + * Phase 14, Plan 03 (INGEST-05) adds 'email-alert': per-tenant portal-alert + * mailbox ingestion (TenderEmailConfig, D-06/D-07), also routed through the + * generic-bag normalizer path (D-04/D-05). Unlike every other SourceType, + * email-alert records carry a per-record `ownerTenantId` (D-13) so they are + * visible ONLY to the configuring tenant — see NormalizedTenderFields below. */ export type SourceType = | 'doe-opendata' | 'ai-netserver' | 'cosinex-dtvp' - | 'rss'; + | 'rss' + | 'email-alert'; /** * A single notice as fetched and lightly parsed from a source, before