feat(14-03): add EmailAlertAdapter generic extraction + 'email-alert' normalizer dispatch
GREEN phase (TDD) for Task 1: extractCandidateLinks (cheerio a[href] + footer-noise filter + MAX_LINKS_PER_EMAIL cap, plaintext regex fallback), titleFromEmail (subject -> first body line -> fallback), and sourceNoticeIdFor (sha256 link hash) implement D-04's generic, no-portal- specific-parser evaluation of alert emails. SourceType gains 'email-alert'; TenderNormalizerService routes it through the existing normalizeBag() path (same as ai-netserver/cosinex-dtvp/rss). EmailAlertAdapter.fetchTenders() is a Task-1 placeholder — Task 2 wires the real per-tenant fan-out. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,105 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import * as cheerio from 'cheerio';
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
import type { RawTenderRecord, SourceType } from '../tender.types';
|
||||||
|
import type { TenderSourceAdapter } from './tender-source-adapter.interface';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* EmailAlertAdapter — INGEST-05. Extracts candidate tender detail links and
|
||||||
|
* a display title from a tenant's portal-alert mailbox, generically (D-04):
|
||||||
|
* no per-portal parser, since no concrete alert-sending portals are known
|
||||||
|
* ahead of time (14-CONTEXT.md). Feld-Armut (no CPV/buyer/deadline) is a
|
||||||
|
* deliberate, documented consequence (D-05) — the same deferral already
|
||||||
|
* accepted for the NetServer/cosinex/RSS scraper adapters.
|
||||||
|
*
|
||||||
|
* Task 1 (this file, generic extraction + normalizer dispatch) ships the
|
||||||
|
* three pure helpers below plus a contract-complete but not-yet-wired
|
||||||
|
* `fetchTenders()`. Task 2 replaces the constructor/fetchTenders body with
|
||||||
|
* the real per-tenant fan-out over `TenderEmailConfig` rows, consuming
|
||||||
|
* `InboxProvider.fetchMessages` from the shared `inbox/` module (Plan 14-01).
|
||||||
|
*
|
||||||
|
* Security (T-14-03-03, stored-XSS guard): only plain-text `title` and raw
|
||||||
|
* `href` URL strings are ever extracted from an alert email body — cheerio
|
||||||
|
* `.attr('href')`/plaintext regex only, NEVER `.html()` — so no raw email
|
||||||
|
* markup ever crosses into a RawTenderRecord/Tender row.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Footer/unsubscribe-noise link fragments to strip from extracted candidate
|
||||||
|
* links (D-04) — matches RESEARCH.md's generic link-extraction example.
|
||||||
|
*/
|
||||||
|
const FOOTER_NOISE = ['unsubscribe', 'abmelden', 'einstellungen', 'preferences'];
|
||||||
|
|
||||||
|
/** Safety valve against link-spam producing excess records per email. */
|
||||||
|
const MAX_LINKS_PER_EMAIL = 5;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracts deduped, absolute http(s) candidate detail links from an alert
|
||||||
|
* email body (D-04 — generic evaluation, no per-portal/sender-domain
|
||||||
|
* parser). HTML body is parsed with cheerio (`a[href]`, footer-noise
|
||||||
|
* filtered, capped at MAX_LINKS_PER_EMAIL); when no HTML body is available
|
||||||
|
* (`bodyHtml === null`), falls back to a plaintext URL regex over `bodyText`.
|
||||||
|
*/
|
||||||
|
export function extractCandidateLinks(
|
||||||
|
bodyHtml: string | null,
|
||||||
|
bodyText: string,
|
||||||
|
): string[] {
|
||||||
|
let links: string[];
|
||||||
|
if (bodyHtml) {
|
||||||
|
const $ = cheerio.load(bodyHtml);
|
||||||
|
links = $('a[href]')
|
||||||
|
.map((_, el) => $(el).attr('href'))
|
||||||
|
.get()
|
||||||
|
.filter(
|
||||||
|
(href): href is string =>
|
||||||
|
typeof href === 'string' && /^https?:\/\//i.test(href),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
links = Array.from(bodyText.matchAll(/https?:\/\/[^\s"'<>]+/g)).map(
|
||||||
|
(m) => m[0],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(new Set(links))
|
||||||
|
.filter((url) => !FOOTER_NOISE.some((noise) => url.toLowerCase().includes(noise)))
|
||||||
|
.slice(0, MAX_LINKS_PER_EMAIL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Derives a display title for an email-sourced tender: the trimmed subject,
|
||||||
|
* else the first non-empty body line, else the same fallback
|
||||||
|
* normalizeBag() already uses for other thin-field sources.
|
||||||
|
*/
|
||||||
|
export function titleFromEmail(subject: string, bodyText: string): string {
|
||||||
|
if (subject.trim()) return subject.trim();
|
||||||
|
const firstLine = bodyText
|
||||||
|
.split('\n')
|
||||||
|
.map((l) => l.trim())
|
||||||
|
.find(Boolean);
|
||||||
|
return firstLine || 'Unbenannte Ausschreibung';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stable per-link id: sha256(link).slice(0,40) — used as sourceNoticeId
|
||||||
|
* since alert-email links carry no portal-native notice id.
|
||||||
|
*/
|
||||||
|
export function sourceNoticeIdFor(link: string): string {
|
||||||
|
return createHash('sha256').update(link).digest('hex').slice(0, 40);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class EmailAlertAdapter implements TenderSourceAdapter {
|
||||||
|
readonly sourceType: SourceType = 'email-alert';
|
||||||
|
readonly portals = ['email-alert'] as const;
|
||||||
|
|
||||||
|
protected readonly logger = new Logger(EmailAlertAdapter.name);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Task 1 placeholder — keeps the class contract-complete (TenderSourceAdapter)
|
||||||
|
* for this task's generic-extraction-only scope. Task 2 replaces this with
|
||||||
|
* the real per-tenant fan-out over TenderEmailConfig + InboxProvider.
|
||||||
|
*/
|
||||||
|
async fetchTenders(_dayCursor: string): Promise<RawTenderRecord[]> {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -85,7 +85,9 @@ function bagRecord(
|
|||||||
? 'cosinex-dtvp'
|
? 'cosinex-dtvp'
|
||||||
: sourceType === 'rss'
|
: sourceType === 'rss'
|
||||||
? 'service-bund'
|
? 'service-bund'
|
||||||
: 'tender24';
|
: sourceType === 'email-alert'
|
||||||
|
? 'email-alert'
|
||||||
|
: 'tender24';
|
||||||
return {
|
return {
|
||||||
sourceType,
|
sourceType,
|
||||||
sourcePortal,
|
sourcePortal,
|
||||||
@@ -355,4 +357,40 @@ describe('TenderNormalizerService', () => {
|
|||||||
expect(normalized.title).toBe('Unbenannte Ausschreibung');
|
expect(normalized.title).toBe('Unbenannte Ausschreibung');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("'email-alert' sourceType (Phase 14, Plan 03, INGEST-05)", () => {
|
||||||
|
it('maps an email-alert bag through normalizeBag(): title passes through, cpvDivisions stays empty', () => {
|
||||||
|
const raw = bagRecord('email-alert', {
|
||||||
|
title: 'Vergabebekanntmachung: Straßenbauarbeiten Musterstadt',
|
||||||
|
buyerName: null,
|
||||||
|
procedureType: null,
|
||||||
|
legalFramework: null,
|
||||||
|
deadlineAt: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const normalized = service.normalize(raw);
|
||||||
|
|
||||||
|
expect(normalized.title).toBe(
|
||||||
|
'Vergabebekanntmachung: Straßenbauarbeiten Musterstadt',
|
||||||
|
);
|
||||||
|
expect(normalized.cpvDivisions).toEqual([]);
|
||||||
|
expect(normalized.cpvCodes).toEqual([]);
|
||||||
|
expect(normalized.region).toBeNull();
|
||||||
|
expect(normalized.bundesland).toBeNull();
|
||||||
|
expect(normalized.buyerName).toBeNull();
|
||||||
|
expect(normalized.procedureType).toBeNull();
|
||||||
|
expect(normalized.deadlineAt).toBeNull();
|
||||||
|
expect(normalized.dedupKey).toBe(
|
||||||
|
`${raw.sourcePortal}:${raw.sourceNoticeId}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('email-alert title falls back to "Unbenannte Ausschreibung" when empty', () => {
|
||||||
|
const raw = bagRecord('email-alert', { title: '' });
|
||||||
|
|
||||||
|
const normalized = service.normalize(raw);
|
||||||
|
|
||||||
|
expect(normalized.title).toBe('Unbenannte Ausschreibung');
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -55,6 +55,7 @@ export class TenderNormalizerService {
|
|||||||
case 'ai-netserver':
|
case 'ai-netserver':
|
||||||
case 'cosinex-dtvp':
|
case 'cosinex-dtvp':
|
||||||
case 'rss':
|
case 'rss':
|
||||||
|
case 'email-alert':
|
||||||
return this.normalizeBag(raw);
|
return this.normalizeBag(raw);
|
||||||
case 'doe-opendata':
|
case 'doe-opendata':
|
||||||
default:
|
default:
|
||||||
@@ -118,13 +119,14 @@ export class TenderNormalizerService {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters
|
* Generic-bag path (gap closure): NetServer/cosinex-DTVP scraper adapters
|
||||||
* (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04) carry a flat
|
* (and, since Phase 14 Plan 02, the RssAdapter, INGEST-04; and since
|
||||||
|
* Phase 14 Plan 03, the EmailAlertAdapter, INGEST-05) carry a flat
|
||||||
* `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in
|
* `{title, buyerName, procedureType, legalFramework, deadlineAt}` bag in
|
||||||
* `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect.
|
* `raw.ocdsPayload` — there is no eForms/OCDS structure to inspect.
|
||||||
* `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has
|
* `legalFramework` is deliberately NOT mapped: NormalizedTenderFields has
|
||||||
* no target field for it. RSS records always have `buyerName`/
|
* no target field for it. RSS/email-alert records always have
|
||||||
* `procedureType`/`deadlineAt` null (baseline title/link/guid mapping
|
* `buyerName`/`procedureType`/`deadlineAt` null (baseline title/link/guid
|
||||||
* only, D-04/D-05 thin-fields deferral).
|
* mapping only, D-04/D-05 thin-fields deferral).
|
||||||
*/
|
*/
|
||||||
private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields {
|
private normalizeBag(raw: RawTenderRecord): NormalizedTenderFields {
|
||||||
const bag = getBagPayload(raw.ocdsPayload);
|
const bag = getBagPayload(raw.ocdsPayload);
|
||||||
|
|||||||
@@ -17,12 +17,19 @@
|
|||||||
* Phase 14, Plan 02 (INGEST-04) adds 'rss': admin-managed RSS feed URLs
|
* Phase 14, Plan 02 (INGEST-04) adds 'rss': admin-managed RSS feed URLs
|
||||||
* (TenderRssFeedSource, D-14), routed through the same generic-bag
|
* (TenderRssFeedSource, D-14), routed through the same generic-bag
|
||||||
* normalizer path as 'ai-netserver'/'cosinex-dtvp' (D-04/D-05).
|
* normalizer path as 'ai-netserver'/'cosinex-dtvp' (D-04/D-05).
|
||||||
|
*
|
||||||
|
* Phase 14, Plan 03 (INGEST-05) adds 'email-alert': per-tenant portal-alert
|
||||||
|
* mailbox ingestion (TenderEmailConfig, D-06/D-07), also routed through the
|
||||||
|
* generic-bag normalizer path (D-04/D-05). Unlike every other SourceType,
|
||||||
|
* email-alert records carry a per-record `ownerTenantId` (D-13) so they are
|
||||||
|
* visible ONLY to the configuring tenant — see NormalizedTenderFields below.
|
||||||
*/
|
*/
|
||||||
export type SourceType =
|
export type SourceType =
|
||||||
| 'doe-opendata'
|
| 'doe-opendata'
|
||||||
| 'ai-netserver'
|
| 'ai-netserver'
|
||||||
| 'cosinex-dtvp'
|
| 'cosinex-dtvp'
|
||||||
| 'rss';
|
| 'rss'
|
||||||
|
| 'email-alert';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A single notice as fetched and lightly parsed from a source, before
|
* A single notice as fetched and lightly parsed from a source, before
|
||||||
|
|||||||
Reference in New Issue
Block a user