From f06a2ff39766566326b5b9957e4586d3443590e0 Mon Sep 17 00:00:00 2001 From: Schalli Date: Tue, 7 Jul 2026 15:41:32 +0200 Subject: [PATCH] fix(favorites): use realistic browser User-Agent for icon byte-fetch Reproducibly confirmed (3/3 vs 3/3 direct comparison inside the API container) that chatgpt.com's Cloudflare WAF returns 403 for the "tessera/1.0" User-Agent regardless of Accept header, and 200 for a real Chrome UA string. Parameterized fetchWithRedirectGuard's User-Agent (defaulting to the existing "tessera/1.0") and override it only for fetchIconBytes -- the HTML-discovery path (discoverFavoriteIconUrl) keeps its original User-Agent unchanged, per the no-regression constraint on that flow. Co-Authored-By: Claude Sonnet 5 --- apps/api/src/favorites/icon-discovery.service.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/api/src/favorites/icon-discovery.service.ts b/apps/api/src/favorites/icon-discovery.service.ts index 83ecff3..b73f6e8 100644 --- a/apps/api/src/favorites/icon-discovery.service.ts +++ b/apps/api/src/favorites/icon-discovery.service.ts @@ -227,7 +227,7 @@ function extractIconFromHtml(html: string, baseUrl: string): string | null { */ async function fetchWithRedirectGuard( pageUrl: URL, - options: { accept: string; timeoutMs: number }, + options: { accept: string; timeoutMs: number; userAgent?: string }, ): Promise<{ response: Response; finalUrl: URL } | null> { let currentUrl = pageUrl; @@ -245,7 +245,7 @@ async function fetchWithRedirectGuard( signal: controller.signal, headers: { Accept: options.accept, - 'User-Agent': 'tessera/1.0', + 'User-Agent': options.userAgent ?? 'tessera/1.0', }, }); @@ -330,11 +330,15 @@ export class IconDiscoveryService { const url = new URL(iconUrl); const result = await fetchWithRedirectGuard(url, { - // A bare "image/*" Accept header (paired with our non-browser - // User-Agent) trips bot-mitigation WAFs on some sites (observed: - // chatgpt.com/favicon.ico returns 403 with this combo) — a realistic - // browser-style image Accept list avoids that false positive. + // A browser-realistic Accept header and User-Agent avoid tripping + // bot-mitigation WAFs that block non-browser clients (observed + // reproducibly: chatgpt.com/favicon.ico returns 403 for our default + // "tessera/1.0" UA and 200 for a real Chrome UA string, confirmed by + // repeated direct comparison). The HTML-discovery path is unaffected — + // this User-Agent override only applies to this byte-fetch call. accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', + userAgent: + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', timeoutMs: ICON_FETCH_TIMEOUT_MS, });