+220 -268

apps/main-app/src/routes/wisp.ts

reviewed

··· 19 19 compressFile, 20 20 computeCID, 21 21 extractBlobMap, 22 22 - extractSubfsUris 22 22 + extractSubfsUris, 23 23 + isTextMimeType 23 24 } from '@wispplace/atproto-utils' 24 25 import { createManifest } from '@wispplace/fs-utils' 25 25 - import { upsertSite, getDomainByDid, updateWispDomainSite, getWispDomainSite, upsertSiteCache } from '../lib/db' 26 26 + import { upsertSite, upsertSiteCache } from '../lib/db' 26 27 import { createLogger } from '@wispplace/observability' 27 28 // import { validateRecord, type Directory } from '@wispplace/lexicons/types/place/wisp/fs' 28 29 import { type Directory } from '@wispplace/lexicons/types/place/wisp/fs' ··· 222 223 // Determine if file should be compressed (pass filename to exclude _redirects) 223 224 const shouldCompress = shouldCompressFile(originalMimeType, normalizedPath); 224 225 225 225 - // Text files (HTML/CSS/JS) need base64 encoding to prevent PDS content sniffing 226 226 - // Audio files just need compression without base64 227 227 - const needsBase64 = 228 228 - originalMimeType.startsWith('text/') || 229 229 - originalMimeType.startsWith('application/json') || 230 230 - originalMimeType.startsWith('application/xml') || 231 231 - originalMimeType === 'image/svg+xml'; 232 232 - 233 226 let finalContent: Buffer; 234 227 let compressed = false; 235 235 - let base64Encoded = false; 236 228 237 229 if (shouldCompress) { 238 238 - const compressedContent = compressFile(originalContent); 230 230 + finalContent = compressFile(originalContent); 239 231 compressed = true; 240 240 - 241 241 - if (needsBase64) { 242 242 - // Text files: compress AND base64 encode 243 243 - finalContent = Buffer.from(compressedContent.toString('base64'), 'binary'); 244 244 - base64Encoded = true; 245 245 - } else { 246 246 - // Audio files: just compress, no base64 247 247 - finalContent = compressedContent; 248 248 - } 249 232 } else { 250 250 - // Binary files: upload directly 251 233 finalContent = originalContent; 252 234 } 253 235 ··· 257 239 mimeType: originalMimeType, 258 240 size: finalContent.length, 259 241 compressed, 260 260 - base64Encoded, 261 242 originalMimeType 262 243 }); 263 244 } ··· 320 301 logger.warn('Failed to cache site', err as any); 321 302 } 322 303 323 323 - // Auto-map claimed domain to this site if user has a claimed domain with no rkey 324 324 - try { 325 325 - const existingDomain = await getDomainByDid(did); 326 326 - if (existingDomain) { 327 327 - const currentSiteMapping = await getWispDomainSite(did); 328 328 - if (!currentSiteMapping) { 329 329 - // Domain is claimed but not mapped to any site, map it to this new site 330 330 - await updateWispDomainSite(existingDomain, rkey); 331 331 - logger.info(`Auto-mapped domain ${existingDomain} to new site ${siteName}`); 332 332 - } 333 333 - } 334 334 - } catch (err) { 335 335 - // Don't fail the upload if domain mapping fails 336 336 - logger.warn('Failed to auto-map domain to new site', err as any); 337 337 - } 338 338 - 339 304 completeUploadJob(jobId, { 340 305 success: true, 341 306 uri: record.data.uri, ··· 487 452 ...(file.compressed && { 488 453 encoding: 'gzip' as const, 489 454 mimeType: file.originalMimeType || file.mimeType, 490 490 - base64: file.base64Encoded || false 491 491 - }) 455 455 + }), 456 456 + base64: !!file.base64Encoded 492 457 }, 493 458 filePath: file.name, 494 459 sentMimeType: file.mimeType, ··· 528 493 ...(file.compressed && { 529 494 encoding: 'gzip' as const, 530 495 mimeType: file.originalMimeType || file.mimeType, 531 531 - base64: file.base64Encoded || false 532 532 - }) 496 496 + }), 497 497 + base64: !!file.base64Encoded 533 498 }, 534 499 filePath: file.name, 535 500 sentMimeType: file.mimeType, ··· 621 586 const uploadedCount = uploadedBlobs.filter(b => !b.reused).length; 622 587 logger.info(`[File Upload] 🎉 Upload phase complete! Total: ${successfulCount} files (${uploadedCount} uploaded, ${reusedCount} reused)`); 623 588 624 624 - const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result); 625 625 - const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath); 589 589 + // Build directory tree from blob results, split into subfs if needed, then put the manifest record. 590 590 + // Extracted so it can be retried with different blob sets (e.g. base64-encoded fallback). 591 591 + const buildManifestAndPut = async (blobs: typeof uploadedBlobs) => { 592 592 + const blobUploadResults: FileUploadResult[] = blobs.map(b => b.result); 593 593 + const blobFilePaths: string[] = blobs.map(b => b.filePath); 626 594 627 627 - // Update directory with file blobs (only for successfully uploaded files) 628 628 - console.log('Updating directory with blob references...'); 629 629 - updateJobProgress(jobId, { phase: 'creating_manifest' }); 595 595 + console.log('Updating directory with blob references...'); 596 596 + updateJobProgress(jobId, { phase: 'creating_manifest' }); 630 597 631 631 - // Create a set of successfully uploaded paths for quick lookup 632 632 - const successfulPaths = new Set(filePaths.map(path => path.replace(/^[^\/]*\//, ''))); 598 598 + const successfulPaths = new Set(blobFilePaths.map(path => path.replace(/^[^\/]*\//, ''))); 599 599 + const updatedDirectory = updateFileBlobs(directory, blobUploadResults, blobFilePaths, '', successfulPaths); 600 600 + const actualFileCount = blobs.length; 633 601 634 634 - const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths, '', successfulPaths); 602 602 + const MAX_MANIFEST_SIZE = 140 * 1024; 603 603 + const FILE_COUNT_THRESHOLD = 250; 604 604 + const TARGET_FILE_COUNT = 200; 605 605 + const MAX_SUBFS_SIZE = 75 * 1024; 606 606 + const subfsRecords: Array<{ uri: string; path: string }> = []; 607 607 + let workingDirectory = updatedDirectory; 608 608 + let currentFileCount = actualFileCount; 635 609 636 636 - // Calculate actual file count (only successfully uploaded files) 637 637 - const actualFileCount = uploadedBlobs.length; 610 610 + let manifest = createManifest(siteName, workingDirectory, actualFileCount); 611 611 + let manifestSize = JSON.stringify(manifest).length; 638 612 639 639 - // Check if we need to split into subfs records 640 640 - // Split proactively if we have lots of files to avoid hitting manifest size limits 641 641 - const MAX_MANIFEST_SIZE = 140 * 1024; // 140KB to be safe (PDS limit is 150KB) 642 642 - const FILE_COUNT_THRESHOLD = 250; // Start splitting at this many files 643 643 - const TARGET_FILE_COUNT = 200; // Try to keep main manifest under this many files 644 644 - const MAX_SUBFS_SIZE = 75 * 1024; // 75KB per subfs record 645 645 - const subfsRecords: Array<{ uri: string; path: string }> = []; 646 646 - let workingDirectory = updatedDirectory; 647 647 - let currentFileCount = actualFileCount; 613 613 + if (actualFileCount >= FILE_COUNT_THRESHOLD || manifestSize > MAX_MANIFEST_SIZE) { 614 614 + console.log(`⚠️ Large site detected (${actualFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB), splitting into subfs records...`); 615 615 + logger.info(`Large site with ${actualFileCount} files, splitting into subfs records`); 648 616 649 649 - // Create initial manifest to check size 650 650 - let manifest = createManifest(siteName, workingDirectory, actualFileCount); 651 651 - let manifestSize = JSON.stringify(manifest).length; 617 617 + let attempts = 0; 618 618 + const MAX_ATTEMPTS = 100; 652 619 653 653 - // Split if we have lots of files OR if manifest is already too large 654 654 - if (actualFileCount >= FILE_COUNT_THRESHOLD || manifestSize > MAX_MANIFEST_SIZE) { 655 655 - console.log(`⚠️ Large site detected (${actualFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB), splitting into subfs records...`); 656 656 - logger.info(`Large site with ${actualFileCount} files, splitting into subfs records`); 620 620 + while ((manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) && attempts < MAX_ATTEMPTS) { 621 621 + attempts++; 657 622 658 658 - // Keep splitting until manifest fits under limits (both size and file count) 659 659 - let attempts = 0; 660 660 - const MAX_ATTEMPTS = 100; // Allow many splits for very large sites 623 623 + const directories = findLargeDirectories(workingDirectory); 624 624 + directories.sort((a, b) => b.size - a.size); 661 625 662 662 - while ((manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) && attempts < MAX_ATTEMPTS) { 663 663 - attempts++; 626 626 + if (directories.length > 0) { 627 627 + const largestDir = directories[0]; 628 628 + console.log(` Split #${attempts}: ${largestDir.path} (${largestDir.fileCount} files, ${(largestDir.size / 1024).toFixed(1)}KB)`); 664 629 665 665 - // Find all directories sorted by size (largest first) 666 666 - const directories = findLargeDirectories(workingDirectory); 667 667 - directories.sort((a, b) => b.size - a.size); 630 630 + let subfsUri: string; 668 631 669 669 - // Check if we can split subdirectories or need to split flat files 670 670 - if (directories.length > 0) { 671 671 - // Split the largest subdirectory 672 672 - const largestDir = directories[0]; 673 673 - console.log(` Split #${attempts}: ${largestDir.path} (${largestDir.fileCount} files, ${(largestDir.size / 1024).toFixed(1)}KB)`); 632 632 + if (largestDir.size > MAX_SUBFS_SIZE) { 633 633 + console.log(` → Directory too large (${(largestDir.size / 1024).toFixed(1)}KB), splitting into chunks...`); 634 634 + const chunks = splitDirectoryIntoChunks(largestDir.directory, MAX_SUBFS_SIZE); 635 635 + console.log(` → Created ${chunks.length} chunks`); 674 636 675 675 - let subfsUri: string; 637 637 + const chunkUris: string[] = []; 638 638 + for (let i = 0; i < chunks.length; i++) { 639 639 + const chunk = chunks[i]!; 640 640 + const chunkRkey = TID.nextStr(); 641 641 + const chunkFileCount = countFilesInDirectory(chunk); 642 642 + console.log(` → Uploading chunk ${i + 1}/${chunks.length} (${chunkFileCount} files)...`); 643 643 + 644 644 + const chunkRecord = await agent.com.atproto.repo.putRecord({ 645 645 + repo: did, 646 646 + collection: 'place.wisp.subfs', 647 647 + rkey: chunkRkey, 648 648 + record: { 649 649 + $type: 'place.wisp.subfs' as const, 650 650 + root: chunk, 651 651 + fileCount: chunkFileCount, 652 652 + createdAt: new Date().toISOString() 653 653 + } 654 654 + }); 676 655 677 677 - if (largestDir.size > MAX_SUBFS_SIZE) { 678 678 - // Directory too large for a single subfs — split into chunks 679 679 - console.log(` → Directory too large (${(largestDir.size / 1024).toFixed(1)}KB), splitting into chunks...`); 680 680 - const chunks = splitDirectoryIntoChunks(largestDir.directory, MAX_SUBFS_SIZE); 681 681 - console.log(` → Created ${chunks.length} chunks`); 656 656 + chunkUris.push(chunkRecord.data.uri); 657 657 + } 682 658 683 683 - const chunkUris: string[] = []; 684 684 - for (let i = 0; i < chunks.length; i++) { 685 685 - const chunk = chunks[i]!; 686 686 - const chunkRkey = TID.nextStr(); 687 687 - const chunkFileCount = countFilesInDirectory(chunk); 688 688 - console.log(` → Uploading chunk ${i + 1}/${chunks.length} (${chunkFileCount} files)...`); 659 659 + console.log(` → Creating parent subfs with ${chunkUris.length} chunk references...`); 660 660 + const parentDirectory: Directory = { 661 661 + $type: 'place.wisp.fs#directory' as const, 662 662 + type: 'directory' as const, 663 663 + entries: chunkUris.map((uri, i) => ({ 664 664 + name: `chunk${i}`, 665 665 + node: { 666 666 + $type: 'place.wisp.fs#subfs' as const, 667 667 + type: 'subfs' as const, 668 668 + subject: uri, 669 669 + flat: true 670 670 + } 671 671 + })) 672 672 + }; 689 673 690 690 - const chunkRecord = await agent.com.atproto.repo.putRecord({ 674 674 + const parentRkey = TID.nextStr(); 675 675 + const parentRecord = await agent.com.atproto.repo.putRecord({ 691 676 repo: did, 692 677 collection: 'place.wisp.subfs', 693 693 - rkey: chunkRkey, 678 678 + rkey: parentRkey, 694 679 record: { 695 680 $type: 'place.wisp.subfs' as const, 696 696 - root: chunk, 697 697 - fileCount: chunkFileCount, 681 681 + root: parentDirectory, 682 682 + fileCount: largestDir.fileCount, 698 683 createdAt: new Date().toISOString() 699 684 } 700 685 }); 701 686 702 702 - chunkUris.push(chunkRecord.data.uri); 687 687 + subfsUri = parentRecord.data.uri; 688 688 + console.log(` ✓ Created parent subfs with ${chunks.length} chunks`); 689 689 + logger.info(`Created chunked subfs for ${largestDir.path}: ${subfsUri} (${chunks.length} chunks)`); 690 690 + } else { 691 691 + const subfsRkey = TID.nextStr(); 692 692 + const subfsRecord = await agent.com.atproto.repo.putRecord({ 693 693 + repo: did, 694 694 + collection: 'place.wisp.subfs', 695 695 + rkey: subfsRkey, 696 696 + record: { 697 697 + $type: 'place.wisp.subfs' as const, 698 698 + root: largestDir.directory, 699 699 + fileCount: largestDir.fileCount, 700 700 + createdAt: new Date().toISOString() 701 701 + } 702 702 + }); 703 703 + 704 704 + subfsUri = subfsRecord.data.uri; 705 705 + console.log(` ✅ Created subfs: ${subfsUri}`); 706 706 + logger.info(`Created subfs record for ${largestDir.path}: ${subfsUri}`); 703 707 } 704 708 705 705 - // Create parent subfs referencing all chunks with flat: true 706 706 - console.log(` → Creating parent subfs with ${chunkUris.length} chunk references...`); 707 707 - const parentDirectory: Directory = { 709 709 + subfsRecords.push({ uri: subfsUri, path: largestDir.path }); 710 710 + workingDirectory = replaceDirectoryWithSubfs(workingDirectory, largestDir.path, subfsUri); 711 711 + currentFileCount -= largestDir.fileCount; 712 712 + } else { 713 713 + const rootFiles = workingDirectory.entries.filter(e => 'type' in e.node && e.node.type === 'file'); 714 714 + 715 715 + if (rootFiles.length === 0) { 716 716 + throw new Error( 717 717 + `Cannot split manifest further - no files or directories available. ` + 718 718 + `Current: ${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB.` 719 719 + ); 720 720 + } 721 721 + 722 722 + const CHUNK_SIZE = 100; 723 723 + const chunkFiles = rootFiles.slice(0, Math.min(CHUNK_SIZE, rootFiles.length)); 724 724 + console.log(` Split #${attempts}: flat root (${chunkFiles.length} files)`); 725 725 + 726 726 + const chunkDirectory: Directory = { 708 727 $type: 'place.wisp.fs#directory' as const, 709 728 type: 'directory' as const, 710 710 - entries: chunkUris.map((uri, i) => ({ 711 711 - name: `chunk${i}`, 712 712 - node: { 713 713 - $type: 'place.wisp.fs#subfs' as const, 714 714 - type: 'subfs' as const, 715 715 - subject: uri, 716 716 - flat: true 717 717 - } 718 718 - })) 729 729 + entries: chunkFiles 719 730 }; 720 731 721 721 - const parentRkey = TID.nextStr(); 722 722 - const parentRecord = await agent.com.atproto.repo.putRecord({ 723 723 - repo: did, 724 724 - collection: 'place.wisp.subfs', 725 725 - rkey: parentRkey, 726 726 - record: { 727 727 - $type: 'place.wisp.subfs' as const, 728 728 - root: parentDirectory, 729 729 - fileCount: largestDir.fileCount, 730 730 - createdAt: new Date().toISOString() 731 731 - } 732 732 - }); 733 733 - 734 734 - subfsUri = parentRecord.data.uri; 735 735 - console.log(` ✓ Created parent subfs with ${chunks.length} chunks`); 736 736 - logger.info(`Created chunked subfs for ${largestDir.path}: ${subfsUri} (${chunks.length} chunks)`); 737 737 - } else { 738 738 - // Directory fits in a single subfs record 739 732 const subfsRkey = TID.nextStr(); 740 733 const subfsRecord = await agent.com.atproto.repo.putRecord({ 741 734 repo: did, ··· 743 736 rkey: subfsRkey, 744 737 record: { 745 738 $type: 'place.wisp.subfs' as const, 746 746 - root: largestDir.directory, 747 747 - fileCount: largestDir.fileCount, 739 739 + root: chunkDirectory, 740 740 + fileCount: chunkFiles.length, 748 741 createdAt: new Date().toISOString() 749 742 } 750 743 }); 751 744 752 752 - subfsUri = subfsRecord.data.uri; 753 753 - console.log(` ✅ Created subfs: ${subfsUri}`); 754 754 - logger.info(`Created subfs record for ${largestDir.path}: ${subfsUri}`); 755 755 - } 745 745 + const subfsUri = subfsRecord.data.uri; 746 746 + console.log(` ✅ Created flat subfs: ${subfsUri}`); 747 747 + logger.info(`Created flat subfs record with ${chunkFiles.length} files: ${subfsUri}`); 748 748 + 749 749 + const remainingEntries = workingDirectory.entries.filter( 750 750 + e => !chunkFiles.some(cf => cf.name === e.name) 751 751 + ); 756 752 757 757 - subfsRecords.push({ uri: subfsUri, path: largestDir.path }); 753 753 + remainingEntries.push({ 754 754 + name: `__subfs_${attempts}`, 755 755 + node: { 756 756 + $type: 'place.wisp.fs#subfs' as const, 757 757 + type: 'subfs' as const, 758 758 + subject: subfsUri, 759 759 + flat: true 760 760 + } 761 761 + }); 758 762 759 759 - // Replace directory with subfs node in the main tree 760 760 - workingDirectory = replaceDirectoryWithSubfs(workingDirectory, largestDir.path, subfsUri); 761 761 - currentFileCount -= largestDir.fileCount; 762 762 - } else { 763 763 - // No subdirectories - split flat files at root level 764 764 - const rootFiles = workingDirectory.entries.filter(e => 'type' in e.node && e.node.type === 'file'); 763 763 + workingDirectory = { 764 764 + $type: 'place.wisp.fs#directory' as const, 765 765 + type: 'directory' as const, 766 766 + entries: remainingEntries 767 767 + }; 765 768 766 766 - if (rootFiles.length === 0) { 767 767 - throw new Error( 768 768 - `Cannot split manifest further - no files or directories available. ` + 769 769 - `Current: ${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB.` 770 770 - ); 769 769 + subfsRecords.push({ uri: subfsUri, path: `__subfs_${attempts}` }); 770 770 + currentFileCount -= chunkFiles.length; 771 771 } 772 772 773 773 - // Take a chunk of files (aim for ~100 files per chunk) 774 774 - const CHUNK_SIZE = 100; 775 775 - const chunkFiles = rootFiles.slice(0, Math.min(CHUNK_SIZE, rootFiles.length)); 776 776 - console.log(` Split #${attempts}: flat root (${chunkFiles.length} files)`); 777 777 - 778 778 - // Create a directory with just these files 779 779 - const chunkDirectory: Directory = { 780 780 - $type: 'place.wisp.fs#directory' as const, 781 781 - type: 'directory' as const, 782 782 - entries: chunkFiles 783 783 - }; 773 773 + manifest = createManifest(siteName, workingDirectory, currentFileCount); 774 774 + manifestSize = JSON.stringify(manifest).length; 775 775 + const newSizeKB = (manifestSize / 1024).toFixed(1); 776 776 + console.log(` → Manifest now ${newSizeKB}KB with ${currentFileCount} files (${subfsRecords.length} subfs total)`); 784 777 785 785 - // Create subfs record for this chunk 786 786 - const subfsRkey = TID.nextStr(); 787 787 - const subfsManifest = { 788 788 - $type: 'place.wisp.subfs' as const, 789 789 - root: chunkDirectory, 790 790 - fileCount: chunkFiles.length, 791 791 - createdAt: new Date().toISOString() 792 792 - }; 778 778 + if (manifestSize <= MAX_MANIFEST_SIZE && currentFileCount <= TARGET_FILE_COUNT) { 779 779 + console.log(` ✅ Manifest fits! (${currentFileCount} files, ${newSizeKB}KB)`); 780 780 + break; 781 781 + } 782 782 + } 793 783 794 794 - // Validate subfs record 795 795 - // const subfsValidation = validateSubfsRecord(subfsManifest); 796 796 - // if (!subfsValidation.success) { 797 797 - // throw new Error(`Invalid subfs manifest: ${subfsValidation.error?.message || 'Validation failed'}`); 798 798 - // } 784 784 + if (manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) { 785 785 + throw new Error( 786 786 + `Failed to fit manifest after splitting ${attempts} directories. ` + 787 787 + `Current: ${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB. ` + 788 788 + `This should never happen - please report this issue.` 789 789 + ); 790 790 + } 799 791 800 800 - // Upload subfs record to PDS 801 801 - const subfsRecord = await agent.com.atproto.repo.putRecord({ 802 802 - repo: did, 803 803 - collection: 'place.wisp.subfs', 804 804 - rkey: subfsRkey, 805 805 - record: subfsManifest 806 806 - }); 792 792 + console.log(`✅ Split complete: ${subfsRecords.length} subfs records, ${currentFileCount} files in main, ${(manifestSize / 1024).toFixed(1)}KB manifest`); 793 793 + logger.info(`Split into ${subfsRecords.length} subfs records, ${currentFileCount} files remaining in main tree`); 794 794 + } else { 795 795 + const manifestSizeKB = (manifestSize / 1024).toFixed(1); 796 796 + console.log(`Manifest created (${actualFileCount} files, ${manifestSizeKB}KB JSON) - no splitting needed`); 797 797 + } 807 798 808 808 - const subfsUri = subfsRecord.data.uri; 809 809 - console.log(` ✅ Created flat subfs: ${subfsUri}`); 810 810 - logger.info(`Created flat subfs record with ${chunkFiles.length} files: ${subfsUri}`); 799 799 + updateJobProgress(jobId, { phase: 'finalizing' }); 800 800 + console.log('Putting record to PDS with rkey:', siteName); 801 801 + const record = await agent.com.atproto.repo.putRecord({ 802 802 + repo: did, 803 803 + collection: 'place.wisp.fs', 804 804 + rkey: siteName, 805 805 + record: manifest 806 806 + }); 807 807 + console.log('Record successfully created on PDS:', record.data.uri); 808 808 + return record; 809 809 + }; 811 810 812 812 - // Remove these files from the working directory and add a subfs entry 813 813 - const remainingEntries = workingDirectory.entries.filter( 814 814 - e => !chunkFiles.some(cf => cf.name === e.name) 815 815 - ); 811 811 + // First attempt: no base64 encoding 812 812 + let record: Awaited<ReturnType<typeof agent.com.atproto.repo.putRecord>>; 813 813 + let finalBlobs = uploadedBlobs; 814 814 + try { 815 815 + record = await buildManifestAndPut(uploadedBlobs); 816 816 + } catch (err: any) { 817 817 + if (err?.status !== 500) throw err; 816 818 817 817 - // Add subfs entry (will be merged flat when expanded) 818 818 - remainingEntries.push({ 819 819 - name: `__subfs_${attempts}`, // Placeholder name, will be merged away 820 820 - node: { 821 821 - $type: 'place.wisp.fs#subfs' as const, 822 822 - type: 'subfs' as const, 823 823 - subject: subfsUri, 824 824 - flat: true // Merge entries directly into parent (default, but explicit for clarity) 825 825 - } 826 826 - }); 819 819 + // On 500, retry with base64 encoding for compressed text files. 820 820 + // Re-read from the original File objects to avoid holding duplicate buffers in memory. 821 821 + logger.warn('Manifest put returned 500 — retrying with base64-encoded text files', err); 822 822 + console.warn('[Upload] Manifest put failed with 500, retrying with base64 encoding for text files...'); 827 823 828 828 - workingDirectory = { 829 829 - $type: 'place.wisp.fs#directory' as const, 830 830 - type: 'directory' as const, 831 831 - entries: remainingEntries 832 832 - }; 824 824 + const base64Blobs = [...uploadedBlobs]; 825 825 + for (const uploadedFile of validUploadedFiles) { 826 826 + if (!uploadedFile.compressed || !isTextMimeType(uploadedFile.mimeType)) continue; 833 827 834 834 - subfsRecords.push({ uri: subfsUri, path: `__subfs_${attempts}` }); 835 835 - currentFileCount -= chunkFiles.length; 836 836 - } 828 828 + // Find the original File object to re-read content without holding extra buffers 829 829 + const originalFile = fileArray.find(f => { 830 830 + if (!f) return false; 831 831 + const wp = 'webkitRelativePath' in f ? String(f.webkitRelativePath) : ''; 832 832 + return (wp || f.name) === uploadedFile.name; 833 833 + }); 834 834 + if (!originalFile) continue; 837 835 838 838 - // Recreate manifest and check new size 839 839 - manifest = createManifest(siteName, workingDirectory, currentFileCount); 840 840 - manifestSize = JSON.stringify(manifest).length; 841 841 - const newSizeKB = (manifestSize / 1024).toFixed(1); 842 842 - console.log(` → Manifest now ${newSizeKB}KB with ${currentFileCount} files (${subfsRecords.length} subfs total)`); 836 836 + const originalContent = Buffer.from(await originalFile.arrayBuffer()); 837 837 + const base64Content = Buffer.from(compressFile(originalContent).toString('base64'), 'binary'); 838 838 + const uploadResult = await uploadBlobWithRetry(agent, base64Content, 'application/octet-stream', uploadedFile.name); 839 839 + const newBlobRef = uploadResult.data.blob; 843 840 844 844 - // Check if we're under both limits now 845 845 - if (manifestSize <= MAX_MANIFEST_SIZE && currentFileCount <= TARGET_FILE_COUNT) { 846 846 - console.log(` ✅ Manifest fits! (${currentFileCount} files, ${newSizeKB}KB)`); 847 847 - break; 841 841 + const blobIdx = base64Blobs.findIndex(b => b.filePath === uploadedFile.name); 842 842 + if (blobIdx !== -1) { 843 843 + base64Blobs[blobIdx] = { 844 844 + ...base64Blobs[blobIdx]!, 845 845 + result: { 846 846 + hash: newBlobRef.ref.toString(), 847 847 + blobRef: newBlobRef, 848 848 + encoding: 'gzip', 849 849 + mimeType: uploadedFile.mimeType, 850 850 + base64: true 851 851 + } 852 852 + }; 848 853 } 849 854 } 850 855 851 851 - if (manifestSize > MAX_MANIFEST_SIZE || currentFileCount > TARGET_FILE_COUNT) { 852 852 - throw new Error( 853 853 - `Failed to fit manifest after splitting ${attempts} directories. ` + 854 854 - `Current: ${currentFileCount} files, ${(manifestSize / 1024).toFixed(1)}KB. ` + 855 855 - `This should never happen - please report this issue.` 856 856 - ); 857 857 - } 858 858 - 859 859 - console.log(`✅ Split complete: ${subfsRecords.length} subfs records, ${currentFileCount} files in main, ${(manifestSize / 1024).toFixed(1)}KB manifest`); 860 860 - logger.info(`Split into ${subfsRecords.length} subfs records, ${currentFileCount} files remaining in main tree`); 861 861 - } else { 862 862 - const manifestSizeKB = (manifestSize / 1024).toFixed(1); 863 863 - console.log(`Manifest created (${fileCount} files, ${manifestSizeKB}KB JSON) - no splitting needed`); 856 856 + finalBlobs = base64Blobs; 857 857 + record = await buildManifestAndPut(base64Blobs); 864 858 } 865 859 866 860 const rkey = siteName; 867 867 - updateJobProgress(jobId, { phase: 'finalizing' }); 868 868 - 869 869 - console.log('Putting record to PDS with rkey:', rkey); 870 870 - const record = await agent.com.atproto.repo.putRecord({ 871 871 - repo: did, 872 872 - collection: 'place.wisp.fs', 873 873 - rkey: rkey, 874 874 - record: manifest 875 875 - }); 876 876 - console.log('Record successfully created on PDS:', record.data.uri); 877 861 878 862 // Store site in database cache 879 863 await upsertSite(did, rkey, siteName); ··· 881 865 // Cache the site files for the hosting service 882 866 try { 883 867 const fileCids: Record<string, string> = {}; 884 884 - for (const blob of uploadedBlobs) { 868 868 + for (const blob of finalBlobs) { 885 869 const normalizedPath = blob.filePath.replace(/^[^\/]*\//, ''); 886 870 fileCids[normalizedPath] = blob.result.hash; 887 871 } ··· 889 873 } catch (err) { 890 874 // Don't fail the upload if caching fails 891 875 logger.warn('Failed to cache site files', err as any); 892 892 - } 893 893 - 894 894 - // Auto-map claimed domain to this site if user has a claimed domain with no rkey 895 895 - try { 896 896 - const existingDomain = await getDomainByDid(did); 897 897 - if (existingDomain) { 898 898 - const currentSiteMapping = await getWispDomainSite(did); 899 899 - if (!currentSiteMapping) { 900 900 - // Domain is claimed but not mapped to any site, map it to this new site 901 901 - await updateWispDomainSite(existingDomain, rkey); 902 902 - logger.info(`Auto-mapped domain ${existingDomain} to new site ${siteName}`); 903 903 - } 904 904 - } 905 905 - } catch (err) { 906 906 - // Don't fail the upload if domain mapping fails 907 907 - logger.warn('Failed to auto-map domain to new site', err as any); 908 876 } 909 877 910 878 // Clean up old subfs records if we had any ··· 1136 1104 } catch (err) { 1137 1105 // Don't fail the upload if caching fails 1138 1106 logger.warn('Failed to cache site', err as any); 1139 1139 - } 1140 1140 - 1141 1141 - // Auto-map claimed domain to this site if user has a claimed domain with no rkey 1142 1142 - try { 1143 1143 - const existingDomain = await getDomainByDid(auth.did); 1144 1144 - if (existingDomain) { 1145 1145 - const currentSiteMapping = await getWispDomainSite(auth.did); 1146 1146 - if (!currentSiteMapping) { 1147 1147 - // Domain is claimed but not mapped to any site, map it to this new site 1148 1148 - await updateWispDomainSite(existingDomain, rkey); 1149 1149 - logger.info(`Auto-mapped domain ${existingDomain} to new site ${siteName}`); 1150 1150 - } 1151 1151 - } 1152 1152 - } catch (err) { 1153 1153 - // Don't fail the upload if domain mapping fails 1154 1154 - logger.warn('Failed to auto-map domain to new site', err as any); 1155 1107 } 1156 1108 1157 1109 return {

+2

bun.lock

reviewed

··· 181 181 "dependencies": { 182 182 "@atproto/api": "^0.18.17", 183 183 "@wispplace/lexicons": "workspace:*", 184 184 + "mime-types": "^3.0.2", 184 185 "multiformats": "^13.3.1", 185 186 }, 186 187 "devDependencies": { 187 188 "@atproto/lexicon": "^0.6.1", 189 189 + "@types/mime-types": "^3.0.1", 188 190 }, 189 191 }, 190 192 "packages/@wispplace/bun-firehose": {

+66 -31

cli/commands/deploy.ts

reviewed

··· 14 14 type UploadedFile, 15 15 type FileUploadResult 16 16 } from '@wispplace/fs-utils'; 17 17 - import { computeCID, extractBlobMap, shouldCompressFile, compressFile, extractSubfsUris } from '@wispplace/atproto-utils'; 17 17 + import { computeCID, extractBlobMap, shouldCompressFile, compressFile, extractSubfsUris, isTextMimeType } from '@wispplace/atproto-utils'; 18 18 import { MAX_SITE_SIZE, MAX_FILE_COUNT, MAX_FILE_SIZE, DEFAULT_IGNORE_PATTERNS } from '@wispplace/constants'; 19 19 import { readdirSync, statSync, readFileSync, existsSync } from 'fs'; 20 20 import { join, relative, basename } from 'path'; ··· 167 167 agent: Agent, 168 168 files: FileInfo[], 169 169 existingBlobMap: Map<string, { blobRef: BlobRef; cid: string }>, 170 170 - concurrency: number 170 170 + concurrency: number, 171 171 + useBase64 = false 171 172 ): Promise<{ uploadedFiles: UploadedFile[]; uploadResults: FileUploadResult[]; filePaths: string[] }> { 172 173 const spinner = createSpinner(`Processing ${files.length} files...`).start(); 173 174 ··· 201 202 let base64Encoded = false; 202 203 203 204 if (shouldCompress) { 204 204 - // Compress with gzip 205 205 const compressed = compressFile(content); 206 206 - // Base64 encode 207 207 - processedContent = Buffer.from(compressed.toString('base64')); 206 206 + if (useBase64 && isTextMimeType(mimeType)) { 207 207 + // Fallback: base64-encode compressed text files for PDSes that reject them otherwise 208 208 + processedContent = Buffer.from(compressed.toString('base64'), 'binary'); 209 209 + base64Encoded = true; 210 210 + } else { 211 211 + processedContent = compressed; 212 212 + } 208 213 encoding = 'gzip'; 209 209 - base64Encoded = true; 210 214 } else { 211 215 processedContent = content; 212 216 } ··· 234 238 mimeType, 235 239 size: processedContent.length, 236 240 compressed: shouldCompress, 237 237 - base64Encoded, 241 241 + ...(base64Encoded && { base64Encoded }), 238 242 originalMimeType: mimeType 239 243 }); 240 244 ··· 243 247 blobRef, 244 248 encoding, 245 249 mimeType, 246 246 - base64: base64Encoded || undefined 250 250 + base64: base64Encoded 247 251 }); 248 252 249 253 filePaths.push(file.relativePath); ··· 510 514 } 511 515 512 516 // 4. Process and upload files 513 513 - const { uploadedFiles, uploadResults, filePaths } = await processAndUploadFiles( 517 517 + const concurrency = options.concurrency ?? DEFAULT_CONCURRENT_UPLOADS; 518 518 + let { uploadedFiles, uploadResults, filePaths } = await processAndUploadFiles( 514 519 agent, 515 520 files, 516 521 existingBlobMap, 517 517 - options.concurrency ?? DEFAULT_CONCURRENT_UPLOADS 522 522 + concurrency 518 523 ); 519 524 520 525 // 5. Build directory structure ··· 523 528 const successfulPaths = new Set(filePaths); 524 529 const directory = updateFileBlobs(rawDirectory, uploadResults, filePaths, '', successfulPaths, { skipNormalization: true }); 525 530 526 526 - // 6. Split into subfs if needed 527 527 - let finalDirectory = directory; 528 528 - let subfsRkeys: string[] = []; 531 531 + // 6+7. Create manifest and put record, retrying with base64 on 500 532 532 + const manifestSpinner = createSpinner('Creating manifest...').start(); 533 533 + 534 534 + // Returns subfsRkeys created so old ones can be cleaned up. 535 535 + const attemptManifest = async (useBase64: boolean): Promise<string[]> => { 536 536 + let curDirectory = directory; 537 537 + 538 538 + if (useBase64) { 539 539 + // Re-upload only compressed text files with base64 encoding. 540 540 + // Re-read from disk so we don't hold duplicate buffers in memory. 541 541 + const textFiles = files.filter(f => { 542 542 + const mimeType = lookup(f.relativePath) || 'application/octet-stream'; 543 543 + return shouldCompressFile(mimeType, f.relativePath) && isTextMimeType(mimeType); 544 544 + }); 545 545 + const retryResult = await processAndUploadFiles(agent, textFiles, existingBlobMap, concurrency, true); 546 546 + 547 547 + // Merge updated results back: replace matching paths 548 548 + const retryMap = new Map(retryResult.filePaths.map((p, i) => [p, retryResult.uploadResults[i]!])); 549 549 + const mergedResults = uploadResults.map((r, i) => retryMap.get(filePaths[i]!) ?? r); 550 550 + 551 551 + // Rebuild directory with updated blob refs 552 552 + curDirectory = updateFileBlobs(rawDirectory, mergedResults, filePaths, '', new Set(filePaths), { skipNormalization: true }); 553 553 + } 554 554 + 555 555 + let finalDirectory = curDirectory; 556 556 + let subfsRkeys: string[] = []; 529 557 530 530 - const initialManifest = createManifest(siteName, directory, fileCount); 531 531 - if ( 532 532 - fileCount >= FILE_COUNT_THRESHOLD || 533 533 - JSON.stringify(initialManifest).length > MAX_MANIFEST_SIZE 534 534 - ) { 535 535 - const result = await splitIntoSubfs(agent, did, directory, siteName); 536 536 - finalDirectory = result.directory; 537 537 - subfsRkeys = result.subfsRkeys; 538 538 - } 558 558 + if ( 559 559 + fileCount >= FILE_COUNT_THRESHOLD || 560 560 + JSON.stringify(createManifest(siteName, curDirectory, fileCount)).length > MAX_MANIFEST_SIZE 561 561 + ) { 562 562 + const result = await splitIntoSubfs(agent, did, curDirectory, siteName); 563 563 + finalDirectory = result.directory; 564 564 + subfsRkeys = result.subfsRkeys; 565 565 + } 539 566 540 540 - // 7. Create manifest 541 541 - const manifestSpinner = createSpinner('Creating manifest...').start(); 542 542 - const manifest = createManifest(siteName, finalDirectory, countFilesInDirectory(finalDirectory)); 567 567 + const manifest = createManifest(siteName, finalDirectory, countFilesInDirectory(finalDirectory)); 568 568 + await agent.com.atproto.repo.putRecord({ 569 569 + repo: did, 570 570 + collection: 'place.wisp.fs', 571 571 + rkey: siteName, 572 572 + record: manifest 573 573 + }); 574 574 + return subfsRkeys; 575 575 + }; 543 576 544 544 - await agent.com.atproto.repo.putRecord({ 545 545 - repo: did, 546 546 - collection: 'place.wisp.fs', 547 547 - rkey: siteName, 548 548 - record: manifest 549 549 - }); 577 577 + let subfsRkeys: string[]; 578 578 + try { 579 579 + subfsRkeys = await attemptManifest(false); 580 580 + } catch (err: any) { 581 581 + if (err?.status !== 500) throw err; 582 582 + console.warn('\n[Deploy] Manifest put failed with 500, retrying with base64 encoding for text files...'); 583 583 + subfsRkeys = await attemptManifest(true); 584 584 + } 550 585 551 586 manifestSpinner.succeed('Created manifest record'); 552 587

+3 -1

packages/@wispplace/atproto-utils/package.json

reviewed

··· 26 26 "dependencies": { 27 27 "@atproto/api": "^0.18.17", 28 28 "@wispplace/lexicons": "workspace:*", 29 29 + "mime-types": "^3.0.2", 29 30 "multiformats": "^13.3.1" 30 31 }, 31 32 "devDependencies": { 32 32 - "@atproto/lexicon": "^0.6.1" 33 33 + "@atproto/lexicon": "^0.6.1", 34 34 + "@types/mime-types": "^3.0.1" 33 35 } 34 36 }

+9

packages/@wispplace/atproto-utils/src/compression.ts

reviewed

··· 1 1 import { gzipSync } from 'zlib'; 2 2 + import { charsets } from 'mime-types'; 2 3 3 4 /** 4 5 * Determine if a file should be gzip compressed based on its MIME type and filename ··· 83 84 84 85 // Default to not compressing for unknown types 85 86 return false; 87 87 + } 88 88 + 89 89 + /** 90 90 + * Returns true if the given MIME type is a text-based type (charset UTF-8). 91 91 + * Uses mime-types library instead of a hardcoded list. 92 92 + */ 93 93 + export function isTextMimeType(mimeType: string): boolean { 94 94 + return charsets.lookup(mimeType) === 'UTF-8'; 86 95 } 87 96 88 97 /**

+1 -1

packages/@wispplace/atproto-utils/src/index.ts

reviewed

··· 2 2 export { computeCID, extractBlobMap, extractBlobCid } from './blob'; 3 3 4 4 // Compression utilities 5 5 - export { shouldCompressFile, shouldCompressMimeType, compressFile } from './compression'; 5 5 + export { shouldCompressFile, shouldCompressMimeType, compressFile, isTextMimeType } from './compression'; 6 6 7 7 // Subfs utilities 8 8 export { extractSubfsUris } from './subfs';

+2 -2

packages/@wispplace/fs-utils/src/tree.ts

reviewed

··· 17 17 blobRef: BlobRef; 18 18 encoding?: 'gzip'; 19 19 mimeType?: string; 20 20 - base64?: boolean; 20 20 + base64: boolean; 21 21 } 22 22 23 23 export interface ProcessedDirectory { ··· 204 204 blob: blobRef, 205 205 ...(result.encoding && { encoding: result.encoding }), 206 206 ...(result.mimeType && { mimeType: result.mimeType }), 207 207 - ...(result.base64 && { base64: result.base64 }) 207 207 + base64: result.base64 ?? false 208 208 } 209 209 }; 210 210 } else {