feat(07-01): DKV PDF parser validated against real invoice + injectable service
- dkv-parser.validate.ts: empirical validation script against user-files/invoice.pdf
- 27 vehicle blocks, 66 transactions extracted (matches expected count)
- Handles two PDF extraction formats: single-tx (tab-separated) + multi-tx (columnar)
- German number parsing: replace(/\./g,'').replace(',','.') applied to km and menge
- Exits 1 with full raw text dump if zero vehicle blocks parsed (assertion guard)
- dkv-parser.service.ts: @Injectable() NestJS service wrapping validated logic
- parsePdf(buffer: Buffer): Promise<DkvVehicleBlock[]>
- Uses pdf-parse v2 class API: new PDFParse({data:buffer}) — NOT v1 pdfParse()
- Calls destroy() after extraction (T-07-01 memory safety)
- Generic error messages only on parse failure (T-07-02 info disclosure)
Regex adjustment vs Research Pattern 4: space-based regex replaced with
tab-split (single-tx) + columnar transpose (multi-tx) after empirical analysis
of actual invoice.pdf text extraction output.
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { PDFParse } from 'pdf-parse';
|
||||
import type { DkvVehicleBlock, DkvTransaction } from './dkv.types.js';
|
||||
|
||||
/**
|
||||
* DkvParserService
|
||||
*
|
||||
* Extracts structured vehicle/transaction data from DKV E-Rechnung PDF buffers.
|
||||
*
|
||||
* Security notes:
|
||||
* - Parser operates only on buffers fetched by the inbox provider (T-07-01)
|
||||
* - Calls destroy() to free PDF parser memory after extraction (T-07-01)
|
||||
* - Generic error messages only on parse failure — no PDF content in logs (T-07-02)
|
||||
*
|
||||
* Wave 0 validation: the parsing logic here was empirically derived by running
|
||||
* dkv-parser.validate.ts against user-files/invoice.pdf (April 2026, 27 vehicles).
|
||||
* Result: 27 vehicle blocks, 66 transactions total.
|
||||
*
|
||||
* Two PDF extraction formats are handled:
|
||||
* 1. Single-transaction (tab-separated): one row per transaction in the PDF
|
||||
* 2. Multi-transaction (columnar): all values per column stacked vertically in the extracted text
|
||||
*/
|
||||
@Injectable()
|
||||
export class DkvParserService {
|
||||
private readonly logger = new Logger(DkvParserService.name);
|
||||
|
||||
/**
|
||||
* Parse a DKV E-Rechnung PDF buffer into structured vehicle blocks.
|
||||
*
|
||||
* @param buffer - Raw PDF bytes (from email attachment)
|
||||
* @returns Array of vehicle blocks with transactions
|
||||
* @throws Error with generic message if no vehicle blocks are found
|
||||
*/
|
||||
async parsePdf(buffer: Buffer): Promise<DkvVehicleBlock[]> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await this.extractText(buffer);
|
||||
} catch (err) {
|
||||
this.logger.error('DKV PDF text extraction failed');
|
||||
throw new Error('DKV invoice parsing failed — text extraction error');
|
||||
}
|
||||
|
||||
const vehicles = this.parseDkvText(text);
|
||||
|
||||
if (vehicles.length === 0) {
|
||||
this.logger.warn('DKV PDF parsed but yielded zero vehicle blocks');
|
||||
throw new Error('DKV invoice parsing failed — no vehicle data found');
|
||||
}
|
||||
|
||||
const totalTx = vehicles.reduce((s, v) => s + v.transactions.length, 0);
|
||||
this.logger.log(
|
||||
`DKV PDF parsed: ${vehicles.length} vehicle(s), ${totalTx} transaction(s)`,
|
||||
);
|
||||
|
||||
return vehicles;
|
||||
}
|
||||
|
||||
// ─── Private: PDF text extraction ──────────────────────────────────────────
|
||||
|
||||
private async extractText(buffer: Buffer): Promise<string> {
|
||||
// pdf-parse v2 class-based API — do NOT use v1 pdfParse(buffer) function call
|
||||
const parser = new PDFParse({ data: buffer });
|
||||
const result = await parser.getText();
|
||||
await parser.destroy(); // always free memory (T-07-01)
|
||||
return result.text;
|
||||
}
|
||||
|
||||
// ─── Private: Vehicle block parser ─────────────────────────────────────────
|
||||
|
||||
private parseDkvText(text: string): DkvVehicleBlock[] {
|
||||
const vehicles: DkvVehicleBlock[] = [];
|
||||
|
||||
// Anchor on VEHICLE: marker — each block extends until next VEHICLE: or end
|
||||
// Kennzeichen format: "GP-JL 728E", "GP ML 720", etc.
|
||||
const vehicleBlockPattern =
|
||||
/VEHICLE:\s+([A-Z0-9 ._\-]+?)\s+CARD NO\.:\s+(\S+)([\s\S]*?)(?=VEHICLE:|$)/g;
|
||||
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = vehicleBlockPattern.exec(text)) !== null) {
|
||||
const kennzeichen = match[1].trim();
|
||||
const cardNumber = match[2].trim();
|
||||
const blockText = match[3];
|
||||
const transactions = this.parseTransactionRows(blockText);
|
||||
vehicles.push({ kennzeichen, cardNumber, transactions });
|
||||
}
|
||||
|
||||
return vehicles;
|
||||
}
|
||||
|
||||
// ─── Private: Transaction row dispatcher ───────────────────────────────────
|
||||
|
||||
private parseTransactionRows(blockText: string): DkvTransaction[] {
|
||||
const lines = blockText
|
||||
.split('\n')
|
||||
.map(l => l.trim())
|
||||
.filter(l => l && !l.startsWith('»'));
|
||||
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
const datePattern = /^\d{2}\.\d{2}\.\d{4}/;
|
||||
|
||||
// If any date-starting line also contains a tab → single-tx tab format
|
||||
const hasTabbedDateLine = lines.some(l => datePattern.test(l) && l.includes('\t'));
|
||||
|
||||
if (hasTabbedDateLine) {
|
||||
return lines
|
||||
.filter(l => datePattern.test(l) && l.includes('\t'))
|
||||
.map(line => this.parseSingleTxLine(line))
|
||||
.filter((tx): tx is DkvTransaction => tx !== null);
|
||||
}
|
||||
|
||||
// Multi-transaction columnar format
|
||||
return this.parseMultiTxColumnar(lines);
|
||||
}
|
||||
|
||||
// ─── Private: Single-transaction tab-separated line ────────────────────────
|
||||
//
|
||||
// DKV single-tx tab format (empirically verified against invoice.pdf):
|
||||
//
|
||||
// [0] Lieferdatum DD.MM.YYYY
|
||||
// [1] Station Name e.g. "ESSO"
|
||||
// [2] Ort e.g. "DONZDORF"
|
||||
// [3] Servicest.-Nr e.g. "0071132"
|
||||
// [4] Transaktions-Nr e.g. "189945"
|
||||
// [5] "KM PRODUKT" e.g. "68040 DIESEL" (km and product merged, no tab between)
|
||||
// [6] "CODE UNIT" e.g. "0009 LTR"
|
||||
// [7] Menge e.g. "45,910"
|
||||
// [8] Preis/E brutto e.g. "2,1790"
|
||||
// [9] Preis/E netto e.g. "1,8312"
|
||||
// [10] Bezugswert brutto e.g. "100,04"
|
||||
// [11] Bezugswert netto e.g. "84,07"
|
||||
// ...
|
||||
|
||||
private parseSingleTxLine(line: string): DkvTransaction | null {
|
||||
const fields = line.split('\t').map(f => f.trim());
|
||||
if (fields.length < 8) return null;
|
||||
|
||||
const datePattern = /^\d{2}\.\d{2}\.\d{4}$/;
|
||||
if (!datePattern.test(fields[0])) return null;
|
||||
|
||||
// fields[5] = "KM_VALUE PRODUKT" — split on first whitespace boundary
|
||||
const kmAndProdukt = fields[5].split(/\s+/);
|
||||
const km = kmAndProdukt[0] ?? '0';
|
||||
const produkt = kmAndProdukt.slice(1).join(' ');
|
||||
|
||||
// fields[6] = "UNITCODE EINHEIT" — Einheit is everything after the first token
|
||||
const unitParts = fields[6].split(/\s+/);
|
||||
const einheit = unitParts.slice(1).join(' ') || unitParts[0];
|
||||
|
||||
return {
|
||||
lieferdatum: fields[0],
|
||||
ort: fields[2] ?? '',
|
||||
kilometerstand: parseDE(km),
|
||||
produkt,
|
||||
menge: parseDE(fields[7] ?? '0'),
|
||||
einheit,
|
||||
netto: parseDE(fields[11] ?? '0'),
|
||||
brutto: parseDE(fields[10] ?? '0'),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Private: Multi-transaction columnar format ─────────────────────────────
|
||||
//
|
||||
// When a vehicle has multiple transactions, pdf-parse extracts the PDF table
|
||||
// in columnar order (all dates, then all names, then all orts, etc.).
|
||||
//
|
||||
// Column groups (each group has n items, where n = transaction count):
|
||||
// group 0: dates
|
||||
// group 1: station names
|
||||
// group 2: orts
|
||||
// group 3: station numbers
|
||||
// group 4: transaction numbers
|
||||
// group 5: km values
|
||||
// group 6: products
|
||||
// group 7: unit codes (e.g. "0036")
|
||||
// group 8: units (e.g. "LTR")
|
||||
// group 9: quantities (Menge)
|
||||
// group 10: unit price brutto
|
||||
// group 11: unit price netto
|
||||
// group 12: total brutto (Bezugswert)
|
||||
// group 13: total netto (Bezugswert)
|
||||
// ... (discounts, taxes, final totals — not extracted)
|
||||
|
||||
private parseMultiTxColumnar(lines: string[]): DkvTransaction[] {
|
||||
const datePattern = /^\d{2}\.\d{2}\.\d{4}$/;
|
||||
|
||||
// Count consecutive date lines at the start to determine n
|
||||
let n = 0;
|
||||
for (const line of lines) {
|
||||
if (datePattern.test(line)) n++;
|
||||
else break;
|
||||
}
|
||||
if (n === 0) return [];
|
||||
|
||||
const g = (groupIdx: number): string[] =>
|
||||
lines.slice(groupIdx * n, (groupIdx + 1) * n);
|
||||
|
||||
const dates = g(0);
|
||||
const orts = g(2);
|
||||
const kms = g(5);
|
||||
const products = g(6);
|
||||
const units = g(8);
|
||||
const quantities = g(9);
|
||||
const totals_brutto = g(12);
|
||||
const totals_netto = g(13);
|
||||
|
||||
return dates.map((date, i) => ({
|
||||
lieferdatum: date,
|
||||
ort: orts[i] ?? '',
|
||||
// German number format: "19.234,56" → 19234.56 (strip dots, replace comma)
|
||||
kilometerstand: parseDE(kms[i] ?? '0'),
|
||||
produkt: products[i] ?? '',
|
||||
menge: parseDE(quantities[i] ?? '0'),
|
||||
einheit: units[i] ?? '',
|
||||
netto: parseDE(totals_netto[i] ?? '0'),
|
||||
brutto: parseDE(totals_brutto[i] ?? '0'),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── German number parser (module-level — used in private methods above) ──────
|
||||
function parseDE(raw: string): number {
|
||||
return parseFloat(raw.replace(/\./g, '').replace(',', '.'));
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Wave 0 validation script for the DKV PDF parser.
|
||||
*
|
||||
* Usage (from project root):
|
||||
* node --experimental-strip-types apps/api/src/dkv/dkv-parser.validate.ts
|
||||
*
|
||||
* Reads user-files/invoice.pdf, extracts text via pdf-parse v2,
|
||||
* runs the vehicle-block regex and transaction parsers, prints results.
|
||||
* Exits 1 if zero vehicle blocks are found (assertion failure).
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { PDFParse } from 'pdf-parse';
|
||||
import type { DkvVehicleBlock, DkvTransaction } from './dkv.types.js';
|
||||
|
||||
// ─── German number parser ─────────────────────────────────────────────────────
|
||||
function parseDE(raw: string): number {
|
||||
return parseFloat(raw.replace(/\./g, '').replace(',', '.'));
|
||||
}
|
||||
|
||||
// ─── Single-transaction line parser ──────────────────────────────────────────
|
||||
// Format: DD.MM.YYYY \t Name \t Ort \t StNr \t TxNr \t "KM PRODUKT" \t "CODE UNIT" \t Menge \t PriceBrutto \t PriceNetto \t TotalBrutto \t TotalNetto \t ...
|
||||
function parseSingleTxLine(line: string): DkvTransaction | null {
|
||||
const fields = line.split('\t').map(f => f.trim());
|
||||
if (fields.length < 8) return null;
|
||||
|
||||
const datePattern = /^\d{2}\.\d{2}\.\d{4}$/;
|
||||
if (!datePattern.test(fields[0])) return null;
|
||||
|
||||
// fields[5] = "KM_VALUE PRODUKT" (km and produkt merged — no tab between them in PDF)
|
||||
const kmAndProdukt = fields[5].split(/\s+/);
|
||||
const km = kmAndProdukt[0] ?? '0';
|
||||
const produkt = kmAndProdukt.slice(1).join(' ');
|
||||
|
||||
// fields[6] = "UNITCODE UNIT" (e.g., "0009 LTR" or "0036 LTR")
|
||||
const unitParts = fields[6].split(/\s+/);
|
||||
const einheit = unitParts.slice(1).join(' ') || unitParts[0];
|
||||
|
||||
return {
|
||||
lieferdatum: fields[0],
|
||||
ort: fields[2] ?? '',
|
||||
kilometerstand: parseDE(km),
|
||||
produkt,
|
||||
menge: parseDE(fields[7] ?? '0'),
|
||||
einheit,
|
||||
netto: parseDE(fields[11] ?? '0'),
|
||||
brutto: parseDE(fields[10] ?? '0'),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Multi-transaction columnar parser ───────────────────────────────────────
|
||||
// Format (3 transactions with n=3):
|
||||
// [0..2] dates n values
|
||||
// [3..5] names n values
|
||||
// [6..8] orts n values
|
||||
// [9..11] station_nrs n values
|
||||
// [12..14] tx_nrs n values
|
||||
// [15..17] km_values n values
|
||||
// [18..20] products n values
|
||||
// [21..23] unit_codes n values
|
||||
// [24..26] units n values
|
||||
// [27..29] quantities n values
|
||||
// [30..32] unit_brutto n values
|
||||
// [33..35] unit_netto n values
|
||||
// [36..38] total_brutto n values
|
||||
// [39..41] total_netto n values
|
||||
// ... (discounts, taxes, final totals)
|
||||
function parseMultiTxColumnar(lines: string[]): DkvTransaction[] {
|
||||
// Count leading date lines to determine n (number of transactions)
|
||||
const datePattern = /^\d{2}\.\d{2}\.\d{4}$/;
|
||||
let n = 0;
|
||||
for (const line of lines) {
|
||||
if (datePattern.test(line)) n++;
|
||||
else break;
|
||||
}
|
||||
if (n === 0) return [];
|
||||
|
||||
const g = (groupIdx: number): string[] =>
|
||||
lines.slice(groupIdx * n, (groupIdx + 1) * n);
|
||||
|
||||
const dates = g(0);
|
||||
const orts = g(2);
|
||||
const kms = g(5);
|
||||
const products = g(6);
|
||||
const units = g(8);
|
||||
const quantities = g(9);
|
||||
const totals_brutto = g(12);
|
||||
const totals_netto = g(13);
|
||||
|
||||
return dates.map((date, i) => ({
|
||||
lieferdatum: date,
|
||||
ort: orts[i] ?? '',
|
||||
kilometerstand: parseDE(kms[i] ?? '0'),
|
||||
produkt: products[i] ?? '',
|
||||
menge: parseDE(quantities[i] ?? '0'),
|
||||
einheit: units[i] ?? '',
|
||||
netto: parseDE(totals_netto[i] ?? '0'),
|
||||
brutto: parseDE(totals_brutto[i] ?? '0'),
|
||||
}));
|
||||
}
|
||||
|
||||
// ─── Transaction rows dispatcher ─────────────────────────────────────────────
|
||||
function parseTransactionRows(blockText: string): DkvTransaction[] {
|
||||
const lines = blockText
|
||||
.split('\n')
|
||||
.map(l => l.trim())
|
||||
.filter(l => l && !l.startsWith('»'));
|
||||
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
const datePattern = /^\d{2}\.\d{2}\.\d{4}/;
|
||||
|
||||
// Detect format: if any date-starting line also contains a tab → single-tx format
|
||||
const hasTabbedDateLine = lines.some(l => datePattern.test(l) && l.includes('\t'));
|
||||
|
||||
if (hasTabbedDateLine) {
|
||||
// Single-transaction format
|
||||
return lines
|
||||
.filter(l => datePattern.test(l) && l.includes('\t'))
|
||||
.map(parseSingleTxLine)
|
||||
.filter((tx): tx is DkvTransaction => tx !== null);
|
||||
}
|
||||
|
||||
// Multi-transaction columnar format
|
||||
return parseMultiTxColumnar(lines);
|
||||
}
|
||||
|
||||
// ─── Vehicle block parser ─────────────────────────────────────────────────────
|
||||
function parseDkvText(text: string): DkvVehicleBlock[] {
|
||||
const vehicles: DkvVehicleBlock[] = [];
|
||||
|
||||
// Anchor on VEHICLE: marker — each block extends until next VEHICLE: or end of text
|
||||
// Kennzeichen can contain letters, digits, hyphens, spaces, dots (e.g. "GP-JL 728E")
|
||||
const vehicleBlockPattern =
|
||||
/VEHICLE:\s+([A-Z0-9 ._\-]+?)\s+CARD NO\.:\s+(\S+)([\s\S]*?)(?=VEHICLE:|$)/g;
|
||||
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = vehicleBlockPattern.exec(text)) !== null) {
|
||||
const kennzeichen = match[1].trim();
|
||||
const cardNumber = match[2].trim();
|
||||
const blockText = match[3];
|
||||
|
||||
const transactions = parseTransactionRows(blockText);
|
||||
vehicles.push({ kennzeichen, cardNumber, transactions });
|
||||
}
|
||||
return vehicles;
|
||||
}
|
||||
|
||||
// ─── PDF extraction ───────────────────────────────────────────────────────────
|
||||
async function extractPdfText(buffer: Buffer): Promise<string> {
|
||||
const parser = new PDFParse({ data: buffer });
|
||||
const result = await parser.getText();
|
||||
await parser.destroy(); // free memory per pdf-parse guidance (T-07-01)
|
||||
return result.text;
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
async function main(): Promise<void> {
|
||||
const invoicePath = join(process.cwd(), 'user-files', 'invoice.pdf');
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = readFileSync(invoicePath);
|
||||
} catch {
|
||||
console.error(`ERROR: Cannot read invoice file at ${invoicePath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`[DKV Validate] Reading ${invoicePath} (${buffer.length} bytes)`);
|
||||
|
||||
let text: string;
|
||||
try {
|
||||
text = await extractPdfText(buffer);
|
||||
} catch (err) {
|
||||
console.error('[DKV Validate] pdf-parse extraction failed:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`[DKV Validate] Extracted ${text.length} characters of text`);
|
||||
|
||||
const vehicles = parseDkvText(text);
|
||||
|
||||
if (vehicles.length === 0) {
|
||||
console.error('[DKV Validate] ASSERTION FAILED: Zero vehicle blocks parsed!');
|
||||
console.error('[DKV Validate] Full extracted text for regex debugging:');
|
||||
console.error(text);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const totalTx = vehicles.reduce((s, v) => s + v.transactions.length, 0);
|
||||
console.log(`[DKV Validate] PASSED: ${vehicles.length} vehicle block(s), ${totalTx} transaction(s) total\n`);
|
||||
|
||||
// Print first 3 blocks
|
||||
for (const [i, v] of vehicles.slice(0, 3).entries()) {
|
||||
console.log(`Block ${i + 1}: ${v.kennzeichen} (card: ${v.cardNumber}) — ${v.transactions.length} tx`);
|
||||
for (const tx of v.transactions.slice(0, 2)) {
|
||||
console.log(` ${tx.lieferdatum} @ ${tx.ort} | ${tx.kilometerstand} km | ${tx.menge} ${tx.einheit} ${tx.produkt}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nAll vehicles: ${vehicles.map(v => `${v.kennzeichen}(${v.transactions.length}tx)`).join(', ')}`);
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[DKV Validate] Unhandled error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user