From 9e5445e6b7c0e1fdf48ab9e437730d34fc8325be Mon Sep 17 00:00:00 2001 From: Schalli Date: Mon, 7 Sep 2026 10:38:37 +0200 Subject: [PATCH] fix(260907-efh): Wert-Veto in Stufe 3 und Fingerabdruck bei Update mitschreiben MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zwei Schaeden aus derselben Wurzel behoben (WINDOWS.md #11, beim Messen gefunden): der Aktualisierungszweig aendert Titel/Vergabestelle/Frist einer bestehenden Zeile, schrieb den Fingerabdruck dabei aber nie mit — der gespeicherte Wert driftete vom Inhalt der Zeile weg und Stufe 3 fand die Zeile nie wieder (an der Live-DB an drei Gruppen gemessen). - fingerprint wird einmal in resolve() berechnet und sowohl im Anlegezweig als auch im Aktualisierungszweig geschrieben - Neue Stufe-3-Pruefung: valueBucketsContradict() als Veto auf einen Fingerabdruck-Treffer, exakter Groessenordnungsvergleich, keine Aehnlichkeitssuche - toNumberOrNull() haendelt Prisma Decimal und den einfachen Zahlenwert der Test-Fakes gleichermassen - Vier neue Tests fuer die Wert-Faelle, bestehender Update-Test um die Fingerabdruck-Erwartung erweitert, alle vorherigen Tests bleiben gruen Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K5jtbGzC5Sf9npJ3JCjKhq --- .../src/tenders/tender-dedup.service.spec.ts | 97 +++++++++++++++++++ apps/api/src/tenders/tender-dedup.service.ts | 53 ++++++++-- 2 files changed, 143 insertions(+), 7 deletions(-) diff --git a/apps/api/src/tenders/tender-dedup.service.spec.ts b/apps/api/src/tenders/tender-dedup.service.spec.ts index f0ad314..75e686c 100644 --- a/apps/api/src/tenders/tender-dedup.service.spec.ts +++ b/apps/api/src/tenders/tender-dedup.service.spec.ts @@ -269,6 +269,12 @@ describe('TenderDedupService.resolve — SCHEMA-02 change-detection preservation expect(row.title).toBe('Sanierung Stadtbrücke (Frist verlängert)'); expect(row.deadlineAt).toEqual(new Date('2026-09-01T10:00:00Z')); expect(row.contentHash).toBe('hash-2'); + // WINDOWS.md #11 fix: this branch changes title/buyerName/deadlineAt — + // exactly the fields the fingerprint is built from — so the written + // data must carry the recomputed fingerprint, matching the updated + // content, not the stale one from creation. + expect(row.fingerprint).toBe(tenderFingerprint(rePoll)); + expect(row.fingerprint).not.toBe(tenderFingerprint(BASE)); }); it('OCID match with an UNCHANGED contentHash does NOT call tender.update', async () => { @@ -285,6 +291,97 @@ describe('TenderDedupService.resolve — SCHEMA-02 change-detection preservation }); }); +describe('TenderDedupService.resolve — Stufe-3-Wertveto (WINDOWS.md #11)', () => { + // All four cases reach tier 3 via a matching fingerprint (same + // buyerName/title/deadlineAt as BASE) with neither an ocid nor a + // source:noticeId overlap — the existing fingerprint test above builds + // it the same way. + const candidateSeed: NormalizedTenderFields = { + ...BASE, + ocid: 'ocds-veto-candidate', + sourceNoticeId: 'notice-veto-candidate', + }; + const incomingSeed: NormalizedTenderFields = { + ...BASE, + ocid: null, + dedupKey: 'ai-netserver:ns-veto', + sourcePortal: 'ai-netserver', + sourceNoticeId: 'ns-veto', + }; + + it('candidate has no value, incoming does: no veto, merges (created=false)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderDedupService(prisma as any); + + const first = await service.resolve( + { ...candidateSeed, estimatedValue: null }, + { dedupActive: true }, + ); + const second = await service.resolve( + { ...incomingSeed, estimatedValue: 54471 }, + { dedupActive: true }, + ); + + expect(second.created).toBe(false); + expect(second.tenderId).toBe(first.tenderId); + expect(prisma.__store.tenders.size).toBe(1); + }); + + it('incoming has no value, candidate does: no veto, merges (created=false)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderDedupService(prisma as any); + + const first = await service.resolve( + { ...candidateSeed, estimatedValue: 54471 }, + { dedupActive: true }, + ); + const second = await service.resolve( + { ...incomingSeed, estimatedValue: null }, + { dedupActive: true }, + ); + + expect(second.created).toBe(false); + expect(second.tenderId).toBe(first.tenderId); + expect(prisma.__store.tenders.size).toBe(1); + }); + + it('both sides carry a value of the same order of magnitude: no veto, merges (created=false)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderDedupService(prisma as any); + + const first = await service.resolve( + { ...candidateSeed, estimatedValue: 12000 }, + { dedupActive: true }, + ); + const second = await service.resolve( + { ...incomingSeed, estimatedValue: 15000 }, + { dedupActive: true }, + ); + + expect(second.created).toBe(false); + expect(second.tenderId).toBe(first.tenderId); + expect(prisma.__store.tenders.size).toBe(1); + }); + + it('both sides carry a value of a different order of magnitude: veto fires, two Tender rows (created=true)', async () => { + const prisma = makeFakePrisma(); + const service = new TenderDedupService(prisma as any); + + const first = await service.resolve( + { ...candidateSeed, estimatedValue: 54471 }, + { dedupActive: true }, + ); + const second = await service.resolve( + { ...incomingSeed, estimatedValue: 387291 }, + { dedupActive: true }, + ); + + expect(second.created).toBe(true); + expect(second.tenderId).not.toBe(first.tenderId); + expect(prisma.__store.tenders.size).toBe(2); + }); +}); + describe('TenderDedupService.resolve — D-13 ownerTenantId (Phase 14, Plan 03, INGEST-05)', () => { it('CREATE writes ownerTenantId from the normalized record (email-alert, private)', async () => { const prisma = makeFakePrisma(); diff --git a/apps/api/src/tenders/tender-dedup.service.ts b/apps/api/src/tenders/tender-dedup.service.ts index 47e6547..c89805f 100644 --- a/apps/api/src/tenders/tender-dedup.service.ts +++ b/apps/api/src/tenders/tender-dedup.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; -import { tenderFingerprint } from './tender-fingerprint'; +import { tenderFingerprint, valueBucketsContradict } from './tender-fingerprint'; import type { NormalizedTenderFields } from './tender.types'; export interface DedupResolveOptions { @@ -24,6 +24,15 @@ interface TenderMatchRow { contentHash: string; } +/** + * The value comes as a Prisma Decimal in production and as a plain number + * from the fake Prisma in tests. Decimal implements valueOf(), so Number() + * handles both transparently. null/undefined both collapse to null. + */ +function toNumberOrNull(v: unknown): number | null { + return v === null || v === undefined ? null : Number(v); +} + /** * TenderDedupService — SCHEMA-03 three-tier cross-source dedup resolver * (D-04): OCID -> source:noticeId -> fingerprint (13-RESEARCH Pattern 5). @@ -57,7 +66,11 @@ export class TenderDedupService { n: NormalizedTenderFields, opts: DedupResolveOptions, ): Promise { - const match = await this.findMatch(n, opts); + // Computed once, used both by the fingerprint tier below and (on a + // create or an update) as the value written to Tender.fingerprint — + // one value, not two independent computations that could drift. + const fingerprint = n.fingerprint ?? tenderFingerprint(n); + const match = await this.findMatch(n, opts, fingerprint); if (match) { await this.prisma.tenderSource.upsert({ @@ -79,7 +92,13 @@ export class TenderDedupService { }, }); - // SCHEMA-02 change-detection preservation — see class doc. + // SCHEMA-02 change-detection preservation — see class doc. This + // branch refreshes exactly the fields the v2 fingerprint is built + // from (buyerName, title, deadlineAt), so it must also refresh + // fingerprint itself — otherwise the stored value drifts from the + // row's actual content and Stufe 3 can never find this row again by + // its new content. Measured at the live DB: three groups of genuine + // duplicates stayed apart for exactly this reason before this fix. if (n.contentHash !== match.contentHash) { await this.prisma.tender.update({ where: { id: match.id }, @@ -96,6 +115,7 @@ export class TenderDedupService { procedureType: n.procedureType, sourceUrl: n.sourceUrl, contentHash: n.contentHash, + fingerprint, }, }); } @@ -103,7 +123,6 @@ export class TenderDedupService { return { tenderId: match.id, created: false }; } - const fingerprint = n.fingerprint ?? tenderFingerprint(n); const created = await this.prisma.tender.create({ data: { sourcePortal: n.sourcePortal, @@ -150,6 +169,7 @@ export class TenderDedupService { private async findMatch( n: NormalizedTenderFields, opts: DedupResolveOptions, + fingerprint: string, ): Promise { // Tier 1: OCID (D-04 primary key, present for DÖE — 0% NULL live). if (n.ocid) { @@ -183,12 +203,31 @@ export class TenderDedupService { // Tier 3: fingerprint — ONLY when dedupActive (D-05). With a single // active source this branch is structurally never reached. if (opts.dedupActive) { - const fingerprint = n.fingerprint ?? tenderFingerprint(n); const byFingerprint = await this.prisma.tender.findFirst({ where: { fingerprint }, - select: { id: true, contentHash: true }, + select: { id: true, contentHash: true, estimatedValue: true }, }); - if (byFingerprint) return byFingerprint; + if (byFingerprint) { + // Value veto (WINDOWS.md #11 fix): an exact order-of-magnitude + // contradiction between the incoming record and the candidate + // overrules an otherwise-matching fingerprint. This is an exact + // comparison of two magnitude buckets, not a similarity search — + // see valueBucketsContradict's own doc comment. It only speaks + // when BOTH sides carry a value, and it exists to rescue the case + // where two different lots of the same construction project share + // a title/buyer/deadline: two lots of the SAME order of magnitude + // are still merged incorrectly — measured and deliberately + // accepted, not an oversight (see tender-fingerprint.ts). + if ( + valueBucketsContradict( + toNumberOrNull(n.estimatedValue), + toNumberOrNull(byFingerprint.estimatedValue), + ) + ) { + return null; + } + return { id: byFingerprint.id, contentHash: byFingerprint.contentHash }; + } } return null;