nekomimi.pet / wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. wisp.place
fork atom

Remove on-demand recovery retry

nekomimi.pet 7f3576a9 6a9aa607

+4 -127
+2 -29
apps/hosting-service/src/lib/file-serving.ts
··· 409 409 filePath: string, 410 410 settings: WispSettings | null = null, 411 411 requestHeaders?: Record<string, string>, 412 - trace?: RequestTrace | null, 413 - allowExpectedMissRecovery = true, 412 + trace?: RequestTrace | null 414 413 ): Promise<Response> { 415 414 let expectedFileCids: Record<string, string> | null | undefined; 416 415 let expectedMissPath: string | null = null; ··· 436 435 } 437 436 }; 438 437 439 - const RECOVERED_RETRY = 'RECOVERED_RETRY' as const; 440 - const maybeReturnStorageMiss = async (): Promise<Response | typeof RECOVERED_RETRY | null> => { 438 + const maybeReturnStorageMiss = async (): Promise<Response | null> => { 441 439 if (!expectedMissPath) return null; 442 - 443 - if (allowExpectedMissRecovery) { 444 - logger.warn('Expected storage miss, attempting on-demand recovery before returning 503', { 445 - did, 446 - rkey, 447 - path: expectedMissPath, 448 - }); 449 - 450 - const recovered = await fetchAndCacheSite(did, rkey); 451 - if (recovered) { 452 - // Clear any per-site negative cache entries so retry can discover restored files. 453 - cache.deletePrefix('siteFiles', `${did}:${rkey}:`); 454 - return RECOVERED_RETRY; 455 - } 456 - } 457 - 458 440 recordStorageMiss(expectedMissPath); 459 441 await enqueueRevalidate(did, rkey, `storage-miss:${expectedMissPath}`); 460 442 return buildStorageMissResponse(requestHeaders); ··· 495 477 const directoryEntries = await listDirectoryEntries(did, rkey, requestPath, fileCids ? Object.keys(fileCids) : null); 496 478 if (directoryEntries.length > 0) { 497 479 const missResponse = await maybeReturnStorageMiss(); 498 - if (missResponse === RECOVERED_RETRY) { 499 - return serveFileInternal(did, rkey, filePath, settings, requestHeaders, trace, false); 500 - } 501 480 if (missResponse) return missResponse; 502 481 const html = generateDirectoryListing(requestPath, directoryEntries); 503 482 return new Response(html, { ··· 591 570 const rootEntries = await listDirectoryEntries(did, rkey, '', fileCids ? Object.keys(fileCids) : null); 592 571 if (rootEntries.length > 0) { 593 572 const missResponse = await maybeReturnStorageMiss(); 594 - if (missResponse === RECOVERED_RETRY) { 595 - return serveFileInternal(did, rkey, filePath, settings, requestHeaders, trace, false); 596 - } 597 573 if (missResponse) return missResponse; 598 574 const html = generateDirectoryListing('', rootEntries); 599 575 return new Response(html, { ··· 607 583 } 608 584 609 585 const missResponse = await maybeReturnStorageMiss(); 610 - if (missResponse === RECOVERED_RETRY) { 611 - return serveFileInternal(did, rkey, filePath, settings, requestHeaders, trace, false); 612 - } 613 586 if (missResponse) return missResponse; 614 587 615 588 // Last resort: if site not in DB at all, try on-demand fetch
+2 -98
apps/hosting-service/src/lib/on-demand-cache.ts
··· 26 26 27 27 // Track in-flight fetches to avoid duplicate work 28 28 const inFlightFetches = new Map<string, Promise<boolean>>(); 29 - const BLOB_500_BACKOFF_MS = 10 * 60 * 1000; 30 - const blob500BackoffUntil = new Map<string, number>(); 31 29 32 30 interface FileInfo { 33 31 path: string; ··· 36 34 encoding?: 'gzip'; 37 35 mimeType?: string; 38 36 base64?: boolean; 39 - } 40 - 41 - class Blob500BackoffError extends Error { 42 - constructor( 43 - public readonly blobKey: string, 44 - public readonly until: number, 45 - public readonly originalError?: unknown 46 - ) { 47 - super(`Blob fetch backoff active until ${new Date(until).toISOString()}`); 48 - this.name = 'Blob500BackoffError'; 49 - } 50 - } 51 - 52 - function isHttp500Error(err: unknown): boolean { 53 - if (typeof err === 'object' && err !== null) { 54 - const value = err as Record<string, unknown>; 55 - const status = value.status ?? value.statusCode; 56 - if (typeof status === 'number' && status === 500) return true; 57 - if (typeof status === 'string' && status === '500') return true; 58 - } 59 - 60 - const msg = err instanceof Error ? err.message : String(err); 61 - return /\bHTTP\s*500\b/i.test(msg); 62 - } 63 - 64 - function getBackoffUntil(blobKey: string): number | null { 65 - const until = blob500BackoffUntil.get(blobKey); 66 - if (!until) return null; 67 - if (Date.now() >= until) { 68 - blob500BackoffUntil.delete(blobKey); 69 - return null; 70 - } 71 - return until; 72 - } 73 - 74 - function set500Backoff(blobKey: string): number { 75 - const until = Date.now() + BLOB_500_BACKOFF_MS; 76 - blob500BackoffUntil.set(blobKey, until); 77 - return until; 78 37 } 79 38 80 39 function formatUnknownError(err: unknown): Record<string, unknown> { ··· 203 162 const CONCURRENCY = 10; 204 163 let downloaded = 0; 205 164 let failed = 0; 206 - const downloadedPaths = new Set<string>(); 207 165 208 166 for (let i = 0; i < files.length; i += CONCURRENCY) { 209 167 const batch = files.slice(i, i + CONCURRENCY); ··· 215 173 const file = batch[idx]; 216 174 if (result.status === 'fulfilled') { 217 175 downloaded++; 218 - if (file) downloadedPaths.add(file.path); 219 176 } else { 220 - if (result.reason instanceof Blob500BackoffError) { 221 - logger.warn('Skipping blob download due cached HTTP 500 backoff', { 222 - did, 223 - rkey, 224 - filePath: file?.path, 225 - cid: file?.cid, 226 - backoffUntil: new Date(result.reason.until).toISOString(), 227 - }); 228 - failed++; 229 - return; 230 - } 231 177 failed++; 232 178 logger.error('Failed to download blob', undefined, { 233 179 did, ··· 241 187 } 242 188 243 189 logger.info('Downloaded files', { did, rkey, downloaded, failed }); 244 - if (failed > 0) { 245 - logger.warn('Partial on-demand cache: some blobs could not be downloaded', { 246 - did, 247 - rkey, 248 - downloaded, 249 - failed, 250 - }); 251 - } 252 - 253 - // Keep site_cache aligned with what we actually fetched to avoid 254 - // expected-miss loops ("site is updating" forever) on permanently failing blobs. 255 - const successfullyCachedFileCids: Record<string, string> = {}; 256 - for (const path of downloadedPaths) { 257 - const cid = fileCids[path]; 258 - if (cid) { 259 - successfullyCachedFileCids[path] = cid; 260 - } 261 - } 262 - 263 - if (downloaded === 0) { 264 - logger.warn('On-demand cache could not fetch any blobs', { did, rkey, totalFiles: files.length }); 265 - return false; 266 - } 267 - 268 - // Update DB with file CIDs that are actually present in storage. 269 - await upsertSiteCache(did, rkey, recordCid, successfullyCachedFileCids); 190 + await upsertSiteCache(did, rkey, recordCid, fileCids); 270 191 271 192 // Enqueue revalidate so firehose-service backfills S3 (cold tier) 272 193 await enqueueRevalidate(did, rkey, `storage-miss:on-demand`); ··· 335 256 pdsEndpoint: string 336 257 ): Promise<void> { 337 258 const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(file.cid)}`; 338 - const blobKey = `${did}:${file.cid}`; 339 259 340 - const backoffUntil = getBackoffUntil(blobKey); 341 - if (backoffUntil) { 342 - throw new Blob500BackoffError(blobKey, backoffUntil); 343 - } 344 - 345 - let content: Uint8Array; 346 - try { 347 - content = await safeFetchBlob(blobUrl, { maxSize: MAX_BLOB_SIZE, timeout: 300000 }); 348 - } catch (err) { 349 - if (isHttp500Error(err)) { 350 - const until = set500Backoff(blobKey); 351 - throw new Blob500BackoffError(blobKey, until, err); 352 - } 353 - throw err; 354 - } 355 - // Successful fetch clears any stale backoff for this blob. 356 - blob500BackoffUntil.delete(blobKey); 260 + let content = await safeFetchBlob(blobUrl, { maxSize: MAX_BLOB_SIZE, timeout: 300000 }); 357 261 let encoding = file.encoding; 358 262 359 263 // Decode base64 if flagged