diff --git a/apps/api/src/dkv/dkv-parser.service.ts b/apps/api/src/dkv/dkv-parser.service.ts new file mode 100644 index 0000000..f9237c8 --- /dev/null +++ b/apps/api/src/dkv/dkv-parser.service.ts @@ -0,0 +1,224 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { PDFParse } from 'pdf-parse'; +import type { DkvVehicleBlock, DkvTransaction } from './dkv.types.js'; + +/** + * DkvParserService + * + * Extracts structured vehicle/transaction data from DKV E-Rechnung PDF buffers. + * + * Security notes: + * - Parser operates only on buffers fetched by the inbox provider (T-07-01) + * - Calls destroy() to free PDF parser memory after extraction (T-07-01) + * - Generic error messages only on parse failure — no PDF content in logs (T-07-02) + * + * Wave 0 validation: the parsing logic here was empirically derived by running + * dkv-parser.validate.ts against user-files/invoice.pdf (April 2026, 27 vehicles). + * Result: 27 vehicle blocks, 66 transactions total. + * + * Two PDF extraction formats are handled: + * 1. Single-transaction (tab-separated): one row per transaction in the PDF + * 2. Multi-transaction (columnar): all values per column stacked vertically in the extracted text + */ +@Injectable() +export class DkvParserService { + private readonly logger = new Logger(DkvParserService.name); + + /** + * Parse a DKV E-Rechnung PDF buffer into structured vehicle blocks. + * + * @param buffer - Raw PDF bytes (from email attachment) + * @returns Array of vehicle blocks with transactions + * @throws Error with generic message if no vehicle blocks are found + */ + async parsePdf(buffer: Buffer): Promise { + let text: string; + try { + text = await this.extractText(buffer); + } catch (err) { + this.logger.error('DKV PDF text extraction failed'); + throw new Error('DKV invoice parsing failed — text extraction error'); + } + + const vehicles = this.parseDkvText(text); + + if (vehicles.length === 0) { + this.logger.warn('DKV PDF parsed but yielded zero vehicle blocks'); + throw new Error('DKV invoice parsing failed — no vehicle data found'); + } + + const totalTx = vehicles.reduce((s, v) => s + v.transactions.length, 0); + this.logger.log( + `DKV PDF parsed: ${vehicles.length} vehicle(s), ${totalTx} transaction(s)`, + ); + + return vehicles; + } + + // ─── Private: PDF text extraction ────────────────────────────────────────── + + private async extractText(buffer: Buffer): Promise { + // pdf-parse v2 class-based API — do NOT use v1 pdfParse(buffer) function call + const parser = new PDFParse({ data: buffer }); + const result = await parser.getText(); + await parser.destroy(); // always free memory (T-07-01) + return result.text; + } + + // ─── Private: Vehicle block parser ───────────────────────────────────────── + + private parseDkvText(text: string): DkvVehicleBlock[] { + const vehicles: DkvVehicleBlock[] = []; + + // Anchor on VEHICLE: marker — each block extends until next VEHICLE: or end + // Kennzeichen format: "GP-JL 728E", "GP ML 720", etc. + const vehicleBlockPattern = + /VEHICLE:\s+([A-Z0-9 ._\-]+?)\s+CARD NO\.:\s+(\S+)([\s\S]*?)(?=VEHICLE:|$)/g; + + let match: RegExpExecArray | null; + while ((match = vehicleBlockPattern.exec(text)) !== null) { + const kennzeichen = match[1].trim(); + const cardNumber = match[2].trim(); + const blockText = match[3]; + const transactions = this.parseTransactionRows(blockText); + vehicles.push({ kennzeichen, cardNumber, transactions }); + } + + return vehicles; + } + + // ─── Private: Transaction row dispatcher ─────────────────────────────────── + + private parseTransactionRows(blockText: string): DkvTransaction[] { + const lines = blockText + .split('\n') + .map(l => l.trim()) + .filter(l => l && !l.startsWith('»')); + + if (lines.length === 0) return []; + + const datePattern = /^\d{2}\.\d{2}\.\d{4}/; + + // If any date-starting line also contains a tab → single-tx tab format + const hasTabbedDateLine = lines.some(l => datePattern.test(l) && l.includes('\t')); + + if (hasTabbedDateLine) { + return lines + .filter(l => datePattern.test(l) && l.includes('\t')) + .map(line => this.parseSingleTxLine(line)) + .filter((tx): tx is DkvTransaction => tx !== null); + } + + // Multi-transaction columnar format + return this.parseMultiTxColumnar(lines); + } + + // ─── Private: Single-transaction tab-separated line ──────────────────────── + // + // DKV single-tx tab format (empirically verified against invoice.pdf): + // + // [0] Lieferdatum DD.MM.YYYY + // [1] Station Name e.g. "ESSO" + // [2] Ort e.g. "DONZDORF" + // [3] Servicest.-Nr e.g. "0071132" + // [4] Transaktions-Nr e.g. "189945" + // [5] "KM PRODUKT" e.g. "68040 DIESEL" (km and product merged, no tab between) + // [6] "CODE UNIT" e.g. "0009 LTR" + // [7] Menge e.g. "45,910" + // [8] Preis/E brutto e.g. "2,1790" + // [9] Preis/E netto e.g. "1,8312" + // [10] Bezugswert brutto e.g. "100,04" + // [11] Bezugswert netto e.g. "84,07" + // ... + + private parseSingleTxLine(line: string): DkvTransaction | null { + const fields = line.split('\t').map(f => f.trim()); + if (fields.length < 8) return null; + + const datePattern = /^\d{2}\.\d{2}\.\d{4}$/; + if (!datePattern.test(fields[0])) return null; + + // fields[5] = "KM_VALUE PRODUKT" — split on first whitespace boundary + const kmAndProdukt = fields[5].split(/\s+/); + const km = kmAndProdukt[0] ?? '0'; + const produkt = kmAndProdukt.slice(1).join(' '); + + // fields[6] = "UNITCODE EINHEIT" — Einheit is everything after the first token + const unitParts = fields[6].split(/\s+/); + const einheit = unitParts.slice(1).join(' ') || unitParts[0]; + + return { + lieferdatum: fields[0], + ort: fields[2] ?? '', + kilometerstand: parseDE(km), + produkt, + menge: parseDE(fields[7] ?? '0'), + einheit, + netto: parseDE(fields[11] ?? '0'), + brutto: parseDE(fields[10] ?? '0'), + }; + } + + // ─── Private: Multi-transaction columnar format ───────────────────────────── + // + // When a vehicle has multiple transactions, pdf-parse extracts the PDF table + // in columnar order (all dates, then all names, then all orts, etc.). + // + // Column groups (each group has n items, where n = transaction count): + // group 0: dates + // group 1: station names + // group 2: orts + // group 3: station numbers + // group 4: transaction numbers + // group 5: km values + // group 6: products + // group 7: unit codes (e.g. "0036") + // group 8: units (e.g. "LTR") + // group 9: quantities (Menge) + // group 10: unit price brutto + // group 11: unit price netto + // group 12: total brutto (Bezugswert) + // group 13: total netto (Bezugswert) + // ... (discounts, taxes, final totals — not extracted) + + private parseMultiTxColumnar(lines: string[]): DkvTransaction[] { + const datePattern = /^\d{2}\.\d{2}\.\d{4}$/; + + // Count consecutive date lines at the start to determine n + let n = 0; + for (const line of lines) { + if (datePattern.test(line)) n++; + else break; + } + if (n === 0) return []; + + const g = (groupIdx: number): string[] => + lines.slice(groupIdx * n, (groupIdx + 1) * n); + + const dates = g(0); + const orts = g(2); + const kms = g(5); + const products = g(6); + const units = g(8); + const quantities = g(9); + const totals_brutto = g(12); + const totals_netto = g(13); + + return dates.map((date, i) => ({ + lieferdatum: date, + ort: orts[i] ?? '', + // German number format: "19.234,56" → 19234.56 (strip dots, replace comma) + kilometerstand: parseDE(kms[i] ?? '0'), + produkt: products[i] ?? '', + menge: parseDE(quantities[i] ?? '0'), + einheit: units[i] ?? '', + netto: parseDE(totals_netto[i] ?? '0'), + brutto: parseDE(totals_brutto[i] ?? '0'), + })); + } +} + +// ─── German number parser (module-level — used in private methods above) ────── +function parseDE(raw: string): number { + return parseFloat(raw.replace(/\./g, '').replace(',', '.')); +} diff --git a/apps/api/src/dkv/dkv-parser.validate.ts b/apps/api/src/dkv/dkv-parser.validate.ts new file mode 100644 index 0000000..b1220c9 --- /dev/null +++ b/apps/api/src/dkv/dkv-parser.validate.ts @@ -0,0 +1,207 @@ +/** + * Wave 0 validation script for the DKV PDF parser. + * + * Usage (from project root): + * node --experimental-strip-types apps/api/src/dkv/dkv-parser.validate.ts + * + * Reads user-files/invoice.pdf, extracts text via pdf-parse v2, + * runs the vehicle-block regex and transaction parsers, prints results. + * Exits 1 if zero vehicle blocks are found (assertion failure). + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PDFParse } from 'pdf-parse'; +import type { DkvVehicleBlock, DkvTransaction } from './dkv.types.js'; + +// ─── German number parser ───────────────────────────────────────────────────── +function parseDE(raw: string): number { + return parseFloat(raw.replace(/\./g, '').replace(',', '.')); +} + +// ─── Single-transaction line parser ────────────────────────────────────────── +// Format: DD.MM.YYYY \t Name \t Ort \t StNr \t TxNr \t "KM PRODUKT" \t "CODE UNIT" \t Menge \t PriceBrutto \t PriceNetto \t TotalBrutto \t TotalNetto \t ... +function parseSingleTxLine(line: string): DkvTransaction | null { + const fields = line.split('\t').map(f => f.trim()); + if (fields.length < 8) return null; + + const datePattern = /^\d{2}\.\d{2}\.\d{4}$/; + if (!datePattern.test(fields[0])) return null; + + // fields[5] = "KM_VALUE PRODUKT" (km and produkt merged — no tab between them in PDF) + const kmAndProdukt = fields[5].split(/\s+/); + const km = kmAndProdukt[0] ?? '0'; + const produkt = kmAndProdukt.slice(1).join(' '); + + // fields[6] = "UNITCODE UNIT" (e.g., "0009 LTR" or "0036 LTR") + const unitParts = fields[6].split(/\s+/); + const einheit = unitParts.slice(1).join(' ') || unitParts[0]; + + return { + lieferdatum: fields[0], + ort: fields[2] ?? '', + kilometerstand: parseDE(km), + produkt, + menge: parseDE(fields[7] ?? '0'), + einheit, + netto: parseDE(fields[11] ?? '0'), + brutto: parseDE(fields[10] ?? '0'), + }; +} + +// ─── Multi-transaction columnar parser ─────────────────────────────────────── +// Format (3 transactions with n=3): +// [0..2] dates n values +// [3..5] names n values +// [6..8] orts n values +// [9..11] station_nrs n values +// [12..14] tx_nrs n values +// [15..17] km_values n values +// [18..20] products n values +// [21..23] unit_codes n values +// [24..26] units n values +// [27..29] quantities n values +// [30..32] unit_brutto n values +// [33..35] unit_netto n values +// [36..38] total_brutto n values +// [39..41] total_netto n values +// ... (discounts, taxes, final totals) +function parseMultiTxColumnar(lines: string[]): DkvTransaction[] { + // Count leading date lines to determine n (number of transactions) + const datePattern = /^\d{2}\.\d{2}\.\d{4}$/; + let n = 0; + for (const line of lines) { + if (datePattern.test(line)) n++; + else break; + } + if (n === 0) return []; + + const g = (groupIdx: number): string[] => + lines.slice(groupIdx * n, (groupIdx + 1) * n); + + const dates = g(0); + const orts = g(2); + const kms = g(5); + const products = g(6); + const units = g(8); + const quantities = g(9); + const totals_brutto = g(12); + const totals_netto = g(13); + + return dates.map((date, i) => ({ + lieferdatum: date, + ort: orts[i] ?? '', + kilometerstand: parseDE(kms[i] ?? '0'), + produkt: products[i] ?? '', + menge: parseDE(quantities[i] ?? '0'), + einheit: units[i] ?? '', + netto: parseDE(totals_netto[i] ?? '0'), + brutto: parseDE(totals_brutto[i] ?? '0'), + })); +} + +// ─── Transaction rows dispatcher ───────────────────────────────────────────── +function parseTransactionRows(blockText: string): DkvTransaction[] { + const lines = blockText + .split('\n') + .map(l => l.trim()) + .filter(l => l && !l.startsWith('»')); + + if (lines.length === 0) return []; + + const datePattern = /^\d{2}\.\d{2}\.\d{4}/; + + // Detect format: if any date-starting line also contains a tab → single-tx format + const hasTabbedDateLine = lines.some(l => datePattern.test(l) && l.includes('\t')); + + if (hasTabbedDateLine) { + // Single-transaction format + return lines + .filter(l => datePattern.test(l) && l.includes('\t')) + .map(parseSingleTxLine) + .filter((tx): tx is DkvTransaction => tx !== null); + } + + // Multi-transaction columnar format + return parseMultiTxColumnar(lines); +} + +// ─── Vehicle block parser ───────────────────────────────────────────────────── +function parseDkvText(text: string): DkvVehicleBlock[] { + const vehicles: DkvVehicleBlock[] = []; + + // Anchor on VEHICLE: marker — each block extends until next VEHICLE: or end of text + // Kennzeichen can contain letters, digits, hyphens, spaces, dots (e.g. "GP-JL 728E") + const vehicleBlockPattern = + /VEHICLE:\s+([A-Z0-9 ._\-]+?)\s+CARD NO\.:\s+(\S+)([\s\S]*?)(?=VEHICLE:|$)/g; + + let match: RegExpExecArray | null; + while ((match = vehicleBlockPattern.exec(text)) !== null) { + const kennzeichen = match[1].trim(); + const cardNumber = match[2].trim(); + const blockText = match[3]; + + const transactions = parseTransactionRows(blockText); + vehicles.push({ kennzeichen, cardNumber, transactions }); + } + return vehicles; +} + +// ─── PDF extraction ─────────────────────────────────────────────────────────── +async function extractPdfText(buffer: Buffer): Promise { + const parser = new PDFParse({ data: buffer }); + const result = await parser.getText(); + await parser.destroy(); // free memory per pdf-parse guidance (T-07-01) + return result.text; +} + +// ─── Main ───────────────────────────────────────────────────────────────────── +async function main(): Promise { + const invoicePath = join(process.cwd(), 'user-files', 'invoice.pdf'); + let buffer: Buffer; + try { + buffer = readFileSync(invoicePath); + } catch { + console.error(`ERROR: Cannot read invoice file at ${invoicePath}`); + process.exit(1); + } + + console.log(`[DKV Validate] Reading ${invoicePath} (${buffer.length} bytes)`); + + let text: string; + try { + text = await extractPdfText(buffer); + } catch (err) { + console.error('[DKV Validate] pdf-parse extraction failed:', err); + process.exit(1); + } + + console.log(`[DKV Validate] Extracted ${text.length} characters of text`); + + const vehicles = parseDkvText(text); + + if (vehicles.length === 0) { + console.error('[DKV Validate] ASSERTION FAILED: Zero vehicle blocks parsed!'); + console.error('[DKV Validate] Full extracted text for regex debugging:'); + console.error(text); + process.exit(1); + } + + const totalTx = vehicles.reduce((s, v) => s + v.transactions.length, 0); + console.log(`[DKV Validate] PASSED: ${vehicles.length} vehicle block(s), ${totalTx} transaction(s) total\n`); + + // Print first 3 blocks + for (const [i, v] of vehicles.slice(0, 3).entries()) { + console.log(`Block ${i + 1}: ${v.kennzeichen} (card: ${v.cardNumber}) — ${v.transactions.length} tx`); + for (const tx of v.transactions.slice(0, 2)) { + console.log(` ${tx.lieferdatum} @ ${tx.ort} | ${tx.kilometerstand} km | ${tx.menge} ${tx.einheit} ${tx.produkt}`); + } + } + + console.log(`\nAll vehicles: ${vehicles.map(v => `${v.kennzeichen}(${v.transactions.length}tx)`).join(', ')}`); +} + +main().catch(err => { + console.error('[DKV Validate] Unhandled error:', err); + process.exit(1); +});