From 713b1eb748734049003d577178e0d4d155e651e9 Mon Sep 17 00:00:00 2001 From: Schalli Date: Tue, 21 Jul 2026 10:33:24 +0200 Subject: [PATCH] feat(10-01): add Tender/TenderSourcePollConfig models + ingestion packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Install fast-xml-parser, adm-zip, csv-parse in @tessera/api - Add global Tender model (no tenantId — D-03 platform-global data) - Add singleton TenderSourcePollConfig (sourceType @unique) - Handwritten additive migration for both tables - Regenerate Prisma client --- apps/api/package.json | 3 + .../migration.sql | 54 +++++++++++++ apps/api/prisma/schema.prisma | 43 +++++++++++ pnpm-lock.yaml | 75 +++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 apps/api/prisma/migrations/20260721120000_add_tender_radar/migration.sql diff --git a/apps/api/package.json b/apps/api/package.json index 9ffe6b6..b8cf9c3 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -24,12 +24,15 @@ "@nestjs/schedule": "^6.1.3", "@prisma/client": "^6.0.0", "@tessera/shared": "workspace:*", + "adm-zip": "^0.6.0", "argon2": "^0.44.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", "cookie-parser": "^1.4.7", "cron": "4.4.0", + "csv-parse": "^7.0.1", "ews-javascript-api": "0.15.3", + "fast-xml-parser": "^5.10.1", "httpntlm": "^1.8.13", "imapflow": "^1.4.3", "ldapts": "^8.1.8", diff --git a/apps/api/prisma/migrations/20260721120000_add_tender_radar/migration.sql b/apps/api/prisma/migrations/20260721120000_add_tender_radar/migration.sql new file mode 100644 index 0000000..269fd97 --- /dev/null +++ b/apps/api/prisma/migrations/20260721120000_add_tender_radar/migration.sql @@ -0,0 +1,54 @@ +-- CreateTable +CREATE TABLE IF NOT EXISTS "Tender" ( + "id" TEXT NOT NULL, + "sourcePortal" TEXT NOT NULL, + "sourceNoticeId" TEXT NOT NULL, + "ocid" TEXT, + "dedupKey" TEXT NOT NULL, + "title" TEXT NOT NULL, + "buyerName" TEXT, + "cpvCodes" TEXT[] DEFAULT ARRAY[]::TEXT[], + "region" TEXT, + "plz" TEXT, + "bundesland" TEXT, + "deadlineAt" TIMESTAMP(3), + "estimatedValue" DECIMAL(14,2), + "procedureType" TEXT, + "status" TEXT NOT NULL DEFAULT 'active', + "sourceUrl" TEXT, + "contentHash" TEXT NOT NULL, + "rawPayload" JSONB, + "publishedAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "Tender_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE IF NOT EXISTS "TenderSourcePollConfig" ( + "id" TEXT NOT NULL, + "sourceType" TEXT NOT NULL, + "pollIntervalMin" INTEGER NOT NULL DEFAULT 60, + "isActive" BOOLEAN NOT NULL DEFAULT false, + "lastIngestedDay" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "TenderSourcePollConfig_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "Tender_dedupKey_key" ON "Tender"("dedupKey"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "Tender_status_idx" ON "Tender"("status"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "Tender_deadlineAt_idx" ON "Tender"("deadlineAt"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "Tender_publishedAt_idx" ON "Tender"("publishedAt"); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "TenderSourcePollConfig_sourceType_key" ON "TenderSourcePollConfig"("sourceType"); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 19034f5..48c9a53 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -256,3 +256,46 @@ model FavoriteLink { @@index([tenantId]) @@index([widgetId]) } + +// Ausschreibungs-Radar (Phase 10) — platform-global tender reference data. +// D-03: Tender carries NO tenantId and is NOT wrapped by forTenant()/RLS — +// per-tenant scoping (saved searches) lives one layer up in Phase 11. +model Tender { + id String @id @default(uuid()) + sourcePortal String // e.g. 'doe-opendata' + sourceNoticeId String + ocid String? // OCDS contracting id, when present + dedupKey String @unique // ocid, or fallback sourcePortal:sourceNoticeId — SCHEMA-02 upsert target + title String + buyerName String? + cpvCodes String[] @default([]) + region String? + plz String? + bundesland String? + deadlineAt DateTime? // frequently null (RESEARCH Pattern 4) — nullable is mandatory + estimatedValue Decimal? @db.Decimal(14, 2) + procedureType String? + status String @default("active") // 'active' | 'expired' (D-05 retention marking) + sourceUrl String? + contentHash String // SCHEMA-02 change detection + rawPayload Json? // debugging / re-normalization + publishedAt DateTime + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([status]) + @@index([deadlineAt]) + @@index([publishedAt]) + // Deliberately NO tenant column and NO tenant index — this is global data (D-03) +} + +// Singleton-per-source admin poll config (INGEST-06 foundation). +model TenderSourcePollConfig { + id String @id @default(uuid()) + sourceType String @unique // fixed slug 'doe-opendata' — @unique makes singleton intent explicit + pollIntervalMin Int @default(60) // D-04 default hourly + isActive Boolean @default(false) + lastIngestedDay DateTime? // day-cursor, NOT a timestamp (RESEARCH Pattern 1) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cdab648..fe04811 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: '@tessera/shared': specifier: workspace:* version: link:../../packages/shared + adm-zip: + specifier: ^0.6.0 + version: 0.6.0 argon2: specifier: ^0.44.0 version: 0.44.0 @@ -71,9 +74,15 @@ importers: cron: specifier: 4.4.0 version: 4.4.0 + csv-parse: + specifier: ^7.0.1 + version: 7.0.1 ews-javascript-api: specifier: 0.15.3 version: 0.15.3 + fast-xml-parser: + specifier: ^5.10.1 + version: 5.10.1 httpntlm: specifier: ^1.8.13 version: 1.8.13 @@ -1284,6 +1293,9 @@ packages: cpu: [x64] os: [win32] + '@nodable/entities@3.0.0': + resolution: {integrity: sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==} + '@one-ini/wasm@0.1.1': resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==} @@ -2280,6 +2292,10 @@ packages: resolution: {integrity: sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==} engines: {node: '>=0.8'} + adm-zip@0.6.0: + resolution: {integrity: sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==} + engines: {node: '>=14.0'} + agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -2355,6 +2371,9 @@ packages: resolution: {integrity: sha512-HqZ5rWlFjGiV0tDm3UxxgNRqsOTniqoKZu0pIAfh7TZQMGuZK+hH0drySty0si0QXj1ieop4+SkSfPZBPPkHig==} engines: {node: '>=14'} + anynum@1.0.1: + resolution: {integrity: sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==} + append-field@1.0.0: resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} @@ -2807,6 +2826,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + csv-parse@7.0.1: + resolution: {integrity: sha512-+2z7Ar0APQ7Uu6fX4cn+pitRmxjZ1WPBcGmZFKmA74FCyi7Et/XZx8cjNQ5CjbZ4HCOxXCOpRBYvYH08Qa003A==} + data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -3157,6 +3179,13 @@ packages: fast-uri@3.1.2: resolution: {integrity: sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==} + fast-xml-builder@1.3.0: + resolution: {integrity: sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==} + + fast-xml-parser@5.10.1: + resolution: {integrity: sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==} + hasBin: true + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -3556,6 +3585,9 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + is-unsafe@2.0.0: + resolution: {integrity: sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==} + is-wsl@2.2.0: resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} engines: {node: '>=8'} @@ -4396,6 +4428,10 @@ packages: resolution: {integrity: sha512-cPLl+qZpSc+ireUvt+IzqbED1cHHkDoVYMo30jbJIdOOjQ1MQYZBPiNvmi8UM6lJuOpTPXJGZQk0DtC4y61MYQ==} engines: {node: '>= 0.4.0'} + path-expression-matcher@1.6.2: + resolution: {integrity: sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==} + engines: {node: '>=14.0.0'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -5138,6 +5174,9 @@ packages: strip-literal@3.1.0: resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + strnum@2.4.1: + resolution: {integrity: sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==} + strtok3@10.3.5: resolution: {integrity: sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==} engines: {node: '>=18'} @@ -5753,6 +5792,10 @@ packages: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} + xml-naming@0.3.0: + resolution: {integrity: sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==} + engines: {node: '>=16.0.0'} + xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} @@ -6672,6 +6715,8 @@ snapshots: '@next/swc-win32-x64-msvc@15.5.19': optional: true + '@nodable/entities@3.0.0': {} + '@one-ini/wasm@0.1.1': optional: true @@ -7571,6 +7616,8 @@ snapshots: adler-32@1.3.1: {} + adm-zip@0.6.0: {} + agent-base@6.0.2: dependencies: debug: 4.4.3 @@ -7641,6 +7688,8 @@ snapshots: ansis@4.2.0: {} + anynum@1.0.1: {} + append-field@1.0.0: {} argon2@0.44.0: @@ -8145,6 +8194,8 @@ snapshots: csstype@3.2.3: {} + csv-parse@7.0.1: {} + data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -8527,6 +8578,20 @@ snapshots: fast-uri@3.1.2: {} + fast-xml-builder@1.3.0: + dependencies: + path-expression-matcher: 1.6.2 + xml-naming: 0.3.0 + + fast-xml-parser@5.10.1: + dependencies: + '@nodable/entities': 3.0.0 + fast-xml-builder: 1.3.0 + is-unsafe: 2.0.0 + path-expression-matcher: 1.6.2 + strnum: 2.4.1 + xml-naming: 0.3.0 + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -9048,6 +9113,8 @@ snapshots: is-unicode-supported@0.1.0: {} + is-unsafe@2.0.0: {} + is-wsl@2.2.0: dependencies: is-docker: 2.2.1 @@ -10504,6 +10571,8 @@ snapshots: pause: 0.0.1 utils-merge: 1.0.1 + path-expression-matcher@1.6.2: {} + path-key@3.1.1: {} path-parse@1.0.7: @@ -11494,6 +11563,10 @@ snapshots: dependencies: js-tokens: 9.0.1 + strnum@2.4.1: + dependencies: + anynum: 1.0.1 + strtok3@10.3.5: dependencies: '@tokenizer/token': 0.3.0 @@ -12081,6 +12154,8 @@ snapshots: xml-name-validator@5.0.0: {} + xml-naming@0.3.0: {} + xmlchars@2.2.0: {} y18n@5.0.8: