fix(260907-efh): Wert-Veto in Stufe 3 und Fingerabdruck bei Update mitschreiben
Zwei Schaeden aus derselben Wurzel behoben (WINDOWS.md #11, beim Messen gefunden): der Aktualisierungszweig aendert Titel/Vergabestelle/Frist einer bestehenden Zeile, schrieb den Fingerabdruck dabei aber nie mit — der gespeicherte Wert driftete vom Inhalt der Zeile weg und Stufe 3 fand die Zeile nie wieder (an der Live-DB an drei Gruppen gemessen). - fingerprint wird einmal in resolve() berechnet und sowohl im Anlegezweig als auch im Aktualisierungszweig geschrieben - Neue Stufe-3-Pruefung: valueBucketsContradict() als Veto auf einen Fingerabdruck-Treffer, exakter Groessenordnungsvergleich, keine Aehnlichkeitssuche - toNumberOrNull() haendelt Prisma Decimal und den einfachen Zahlenwert der Test-Fakes gleichermassen - Vier neue Tests fuer die Wert-Faelle, bestehender Update-Test um die Fingerabdruck-Erwartung erweitert, alle vorherigen Tests bleiben gruen Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K5jtbGzC5Sf9npJ3JCjKhq
This commit is contained in:
@@ -269,6 +269,12 @@ describe('TenderDedupService.resolve — SCHEMA-02 change-detection preservation
|
|||||||
expect(row.title).toBe('Sanierung Stadtbrücke (Frist verlängert)');
|
expect(row.title).toBe('Sanierung Stadtbrücke (Frist verlängert)');
|
||||||
expect(row.deadlineAt).toEqual(new Date('2026-09-01T10:00:00Z'));
|
expect(row.deadlineAt).toEqual(new Date('2026-09-01T10:00:00Z'));
|
||||||
expect(row.contentHash).toBe('hash-2');
|
expect(row.contentHash).toBe('hash-2');
|
||||||
|
// WINDOWS.md #11 fix: this branch changes title/buyerName/deadlineAt —
|
||||||
|
// exactly the fields the fingerprint is built from — so the written
|
||||||
|
// data must carry the recomputed fingerprint, matching the updated
|
||||||
|
// content, not the stale one from creation.
|
||||||
|
expect(row.fingerprint).toBe(tenderFingerprint(rePoll));
|
||||||
|
expect(row.fingerprint).not.toBe(tenderFingerprint(BASE));
|
||||||
});
|
});
|
||||||
|
|
||||||
it('OCID match with an UNCHANGED contentHash does NOT call tender.update', async () => {
|
it('OCID match with an UNCHANGED contentHash does NOT call tender.update', async () => {
|
||||||
@@ -285,6 +291,97 @@ describe('TenderDedupService.resolve — SCHEMA-02 change-detection preservation
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('TenderDedupService.resolve — Stufe-3-Wertveto (WINDOWS.md #11)', () => {
|
||||||
|
// All four cases reach tier 3 via a matching fingerprint (same
|
||||||
|
// buyerName/title/deadlineAt as BASE) with neither an ocid nor a
|
||||||
|
// source:noticeId overlap — the existing fingerprint test above builds
|
||||||
|
// it the same way.
|
||||||
|
const candidateSeed: NormalizedTenderFields = {
|
||||||
|
...BASE,
|
||||||
|
ocid: 'ocds-veto-candidate',
|
||||||
|
sourceNoticeId: 'notice-veto-candidate',
|
||||||
|
};
|
||||||
|
const incomingSeed: NormalizedTenderFields = {
|
||||||
|
...BASE,
|
||||||
|
ocid: null,
|
||||||
|
dedupKey: 'ai-netserver:ns-veto',
|
||||||
|
sourcePortal: 'ai-netserver',
|
||||||
|
sourceNoticeId: 'ns-veto',
|
||||||
|
};
|
||||||
|
|
||||||
|
it('candidate has no value, incoming does: no veto, merges (created=false)', async () => {
|
||||||
|
const prisma = makeFakePrisma();
|
||||||
|
const service = new TenderDedupService(prisma as any);
|
||||||
|
|
||||||
|
const first = await service.resolve(
|
||||||
|
{ ...candidateSeed, estimatedValue: null },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
const second = await service.resolve(
|
||||||
|
{ ...incomingSeed, estimatedValue: 54471 },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(second.created).toBe(false);
|
||||||
|
expect(second.tenderId).toBe(first.tenderId);
|
||||||
|
expect(prisma.__store.tenders.size).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('incoming has no value, candidate does: no veto, merges (created=false)', async () => {
|
||||||
|
const prisma = makeFakePrisma();
|
||||||
|
const service = new TenderDedupService(prisma as any);
|
||||||
|
|
||||||
|
const first = await service.resolve(
|
||||||
|
{ ...candidateSeed, estimatedValue: 54471 },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
const second = await service.resolve(
|
||||||
|
{ ...incomingSeed, estimatedValue: null },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(second.created).toBe(false);
|
||||||
|
expect(second.tenderId).toBe(first.tenderId);
|
||||||
|
expect(prisma.__store.tenders.size).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('both sides carry a value of the same order of magnitude: no veto, merges (created=false)', async () => {
|
||||||
|
const prisma = makeFakePrisma();
|
||||||
|
const service = new TenderDedupService(prisma as any);
|
||||||
|
|
||||||
|
const first = await service.resolve(
|
||||||
|
{ ...candidateSeed, estimatedValue: 12000 },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
const second = await service.resolve(
|
||||||
|
{ ...incomingSeed, estimatedValue: 15000 },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(second.created).toBe(false);
|
||||||
|
expect(second.tenderId).toBe(first.tenderId);
|
||||||
|
expect(prisma.__store.tenders.size).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('both sides carry a value of a different order of magnitude: veto fires, two Tender rows (created=true)', async () => {
|
||||||
|
const prisma = makeFakePrisma();
|
||||||
|
const service = new TenderDedupService(prisma as any);
|
||||||
|
|
||||||
|
const first = await service.resolve(
|
||||||
|
{ ...candidateSeed, estimatedValue: 54471 },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
const second = await service.resolve(
|
||||||
|
{ ...incomingSeed, estimatedValue: 387291 },
|
||||||
|
{ dedupActive: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(second.created).toBe(true);
|
||||||
|
expect(second.tenderId).not.toBe(first.tenderId);
|
||||||
|
expect(prisma.__store.tenders.size).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('TenderDedupService.resolve — D-13 ownerTenantId (Phase 14, Plan 03, INGEST-05)', () => {
|
describe('TenderDedupService.resolve — D-13 ownerTenantId (Phase 14, Plan 03, INGEST-05)', () => {
|
||||||
it('CREATE writes ownerTenantId from the normalized record (email-alert, private)', async () => {
|
it('CREATE writes ownerTenantId from the normalized record (email-alert, private)', async () => {
|
||||||
const prisma = makeFakePrisma();
|
const prisma = makeFakePrisma();
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
import { tenderFingerprint } from './tender-fingerprint';
|
import { tenderFingerprint, valueBucketsContradict } from './tender-fingerprint';
|
||||||
import type { NormalizedTenderFields } from './tender.types';
|
import type { NormalizedTenderFields } from './tender.types';
|
||||||
|
|
||||||
export interface DedupResolveOptions {
|
export interface DedupResolveOptions {
|
||||||
@@ -24,6 +24,15 @@ interface TenderMatchRow {
|
|||||||
contentHash: string;
|
contentHash: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The value comes as a Prisma Decimal in production and as a plain number
|
||||||
|
* from the fake Prisma in tests. Decimal implements valueOf(), so Number()
|
||||||
|
* handles both transparently. null/undefined both collapse to null.
|
||||||
|
*/
|
||||||
|
function toNumberOrNull(v: unknown): number | null {
|
||||||
|
return v === null || v === undefined ? null : Number(v);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* TenderDedupService — SCHEMA-03 three-tier cross-source dedup resolver
|
* TenderDedupService — SCHEMA-03 three-tier cross-source dedup resolver
|
||||||
* (D-04): OCID -> source:noticeId -> fingerprint (13-RESEARCH Pattern 5).
|
* (D-04): OCID -> source:noticeId -> fingerprint (13-RESEARCH Pattern 5).
|
||||||
@@ -57,7 +66,11 @@ export class TenderDedupService {
|
|||||||
n: NormalizedTenderFields,
|
n: NormalizedTenderFields,
|
||||||
opts: DedupResolveOptions,
|
opts: DedupResolveOptions,
|
||||||
): Promise<DedupResolveResult> {
|
): Promise<DedupResolveResult> {
|
||||||
const match = await this.findMatch(n, opts);
|
// Computed once, used both by the fingerprint tier below and (on a
|
||||||
|
// create or an update) as the value written to Tender.fingerprint —
|
||||||
|
// one value, not two independent computations that could drift.
|
||||||
|
const fingerprint = n.fingerprint ?? tenderFingerprint(n);
|
||||||
|
const match = await this.findMatch(n, opts, fingerprint);
|
||||||
|
|
||||||
if (match) {
|
if (match) {
|
||||||
await this.prisma.tenderSource.upsert({
|
await this.prisma.tenderSource.upsert({
|
||||||
@@ -79,7 +92,13 @@ export class TenderDedupService {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// SCHEMA-02 change-detection preservation — see class doc.
|
// SCHEMA-02 change-detection preservation — see class doc. This
|
||||||
|
// branch refreshes exactly the fields the v2 fingerprint is built
|
||||||
|
// from (buyerName, title, deadlineAt), so it must also refresh
|
||||||
|
// fingerprint itself — otherwise the stored value drifts from the
|
||||||
|
// row's actual content and Stufe 3 can never find this row again by
|
||||||
|
// its new content. Measured at the live DB: three groups of genuine
|
||||||
|
// duplicates stayed apart for exactly this reason before this fix.
|
||||||
if (n.contentHash !== match.contentHash) {
|
if (n.contentHash !== match.contentHash) {
|
||||||
await this.prisma.tender.update({
|
await this.prisma.tender.update({
|
||||||
where: { id: match.id },
|
where: { id: match.id },
|
||||||
@@ -96,6 +115,7 @@ export class TenderDedupService {
|
|||||||
procedureType: n.procedureType,
|
procedureType: n.procedureType,
|
||||||
sourceUrl: n.sourceUrl,
|
sourceUrl: n.sourceUrl,
|
||||||
contentHash: n.contentHash,
|
contentHash: n.contentHash,
|
||||||
|
fingerprint,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -103,7 +123,6 @@ export class TenderDedupService {
|
|||||||
return { tenderId: match.id, created: false };
|
return { tenderId: match.id, created: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
const fingerprint = n.fingerprint ?? tenderFingerprint(n);
|
|
||||||
const created = await this.prisma.tender.create({
|
const created = await this.prisma.tender.create({
|
||||||
data: {
|
data: {
|
||||||
sourcePortal: n.sourcePortal,
|
sourcePortal: n.sourcePortal,
|
||||||
@@ -150,6 +169,7 @@ export class TenderDedupService {
|
|||||||
private async findMatch(
|
private async findMatch(
|
||||||
n: NormalizedTenderFields,
|
n: NormalizedTenderFields,
|
||||||
opts: DedupResolveOptions,
|
opts: DedupResolveOptions,
|
||||||
|
fingerprint: string,
|
||||||
): Promise<TenderMatchRow | null> {
|
): Promise<TenderMatchRow | null> {
|
||||||
// Tier 1: OCID (D-04 primary key, present for DÖE — 0% NULL live).
|
// Tier 1: OCID (D-04 primary key, present for DÖE — 0% NULL live).
|
||||||
if (n.ocid) {
|
if (n.ocid) {
|
||||||
@@ -183,12 +203,31 @@ export class TenderDedupService {
|
|||||||
// Tier 3: fingerprint — ONLY when dedupActive (D-05). With a single
|
// Tier 3: fingerprint — ONLY when dedupActive (D-05). With a single
|
||||||
// active source this branch is structurally never reached.
|
// active source this branch is structurally never reached.
|
||||||
if (opts.dedupActive) {
|
if (opts.dedupActive) {
|
||||||
const fingerprint = n.fingerprint ?? tenderFingerprint(n);
|
|
||||||
const byFingerprint = await this.prisma.tender.findFirst({
|
const byFingerprint = await this.prisma.tender.findFirst({
|
||||||
where: { fingerprint },
|
where: { fingerprint },
|
||||||
select: { id: true, contentHash: true },
|
select: { id: true, contentHash: true, estimatedValue: true },
|
||||||
});
|
});
|
||||||
if (byFingerprint) return byFingerprint;
|
if (byFingerprint) {
|
||||||
|
// Value veto (WINDOWS.md #11 fix): an exact order-of-magnitude
|
||||||
|
// contradiction between the incoming record and the candidate
|
||||||
|
// overrules an otherwise-matching fingerprint. This is an exact
|
||||||
|
// comparison of two magnitude buckets, not a similarity search —
|
||||||
|
// see valueBucketsContradict's own doc comment. It only speaks
|
||||||
|
// when BOTH sides carry a value, and it exists to rescue the case
|
||||||
|
// where two different lots of the same construction project share
|
||||||
|
// a title/buyer/deadline: two lots of the SAME order of magnitude
|
||||||
|
// are still merged incorrectly — measured and deliberately
|
||||||
|
// accepted, not an oversight (see tender-fingerprint.ts).
|
||||||
|
if (
|
||||||
|
valueBucketsContradict(
|
||||||
|
toNumberOrNull(n.estimatedValue),
|
||||||
|
toNumberOrNull(byFingerprint.estimatedValue),
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { id: byFingerprint.id, contentHash: byFingerprint.contentHash };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
Reference in New Issue
Block a user