clean up file parsing · byarielm.fyi/atlast@4e0efdf

+10 -11

src/hooks/useFileUpload.ts

··· 1 - import { fileParser, FileParseError } from '../lib/fileParser'; 2 - import type { TikTokUser, SearchResult } from '../types'; 1 + import { parseFile, PlatformParseError } from '../lib/platforms/parser'; 2 + import type { SocialUser } from '../lib/platforms/types'; 3 + import type { SearchResult } from '../types'; 3 4 4 5 export function useFileUpload( 5 6 onSearchStart: (results: SearchResult[], platform: string) => void, ··· 10 11 if (!file) return; 11 12 12 13 onStatusUpdate(`Processing ${file.name}...`); 13 - let users: TikTokUser[] = []; 14 + let users: SocialUser[] = []; 14 15 15 16 try { 16 - users = await fileParser.parseFile(file); 17 + // Use the new platform-based parser 18 + users = await parseFile(file, platform); 17 19 18 - const fileType = file.name.endsWith('.zip') ? 'ZIP' : 19 - file.name.endsWith('.json') ? 'JSON' : 20 - file.name.endsWith('.html') ? 'HTML' : 'TXT'; 21 - console.log(`Loaded ${users.length} users from ${fileType} file`); 22 - onStatusUpdate(`Loaded ${users.length} users from ${fileType} file`); 20 + console.log(`Loaded ${users.length} users from ${platform} data`); 21 + onStatusUpdate(`Loaded ${users.length} users from ${platform} data`); 23 22 } catch (error) { 24 23 console.error("Error processing file:", error); 25 24 26 - const errorMsg = error instanceof FileParseError 25 + const errorMsg = error instanceof PlatformParseError 27 26 ? error.message 28 27 : "There was a problem processing the file. Please check that it's a valid data export."; 29 28 ··· 41 40 42 41 // Initialize search results 43 42 const initialResults: SearchResult[] = users.map(user => ({ 44 - tiktokUser: user, 43 + tiktokUser: user, // TODO: Rename to sourceUser in types 45 44 atprotoMatches: [], 46 45 isSearching: false, 47 46 selectedMatches: new Set<string>(),

+149

src/lib/platforms/instagram.ts

··· 1 + // src/lib/platforms/instagram.ts 2 + 3 + import type { PlatformConfig, PlatformParser, FileBundle, SocialUser } from './types'; 4 + import { PlatformParseError } from './types'; 5 + 6 + // HTML Parser for Instagram following.html 7 + const htmlParser: PlatformParser = { 8 + name: 'Instagram HTML', 9 + canParse: (bundle: FileBundle) => { 10 + for (const [_, file] of bundle.files) { 11 + if (file.type === 'html' && file.name.toLowerCase().includes('following')) { 12 + return file.content.includes('_a6-g') || file.content.includes('uiBoxWhite'); 13 + } 14 + } 15 + return false; 16 + }, 17 + parse: async (bundle: FileBundle) => { 18 + const users: SocialUser[] = []; 19 + 20 + // Find HTML file 21 + let htmlContent = ''; 22 + for (const [_, file] of bundle.files) { 23 + if (file.type === 'html' && file.name.toLowerCase().includes('following')) { 24 + htmlContent = file.content; 25 + break; 26 + } 27 + } 28 + 29 + if (!htmlContent) { 30 + throw new PlatformParseError('No Instagram following.html file found', 'instagram'); 31 + } 32 + 33 + // Parse the HTML 34 + const parser = new DOMParser(); 35 + const doc = parser.parseFromString(htmlContent, 'text/html'); 36 + 37 + // Instagram following data is in specific divs 38 + const userDivs = doc.querySelectorAll('div.pam._3-95._2ph-._a6-g.uiBoxWhite.noborder'); 39 + 40 + userDivs.forEach((div) => { 41 + const h2 = div.querySelector('h2._3-95._2pim._a6-h._a6-i'); 42 + const dateDiv = div.querySelector('div._a6-p > div > div:nth-child(2)'); 43 + 44 + if (h2) { 45 + const username = h2.textContent?.trim(); 46 + const date = dateDiv?.textContent?.trim() || ''; 47 + 48 + if (username) { 49 + users.push({ 50 + username: username, 51 + date: date 52 + }); 53 + } 54 + } 55 + }); 56 + 57 + if (users.length === 0) { 58 + throw new PlatformParseError( 59 + 'No following data found in Instagram HTML file', 60 + 'instagram' 61 + ); 62 + } 63 + 64 + return users; 65 + } 66 + }; 67 + 68 + // JSON Parser for Instagram JSON exports 69 + const jsonParser: PlatformParser = { 70 + name: 'Instagram JSON', 71 + canParse: (bundle: FileBundle) => { 72 + for (const [_, file] of bundle.files) { 73 + if (file.type === 'json') { 74 + try { 75 + const data = JSON.parse(file.content); 76 + return !!(data?.relationships_following || data?.following); 77 + } catch { 78 + return false; 79 + } 80 + } 81 + } 82 + return false; 83 + }, 84 + parse: async (bundle: FileBundle) => { 85 + const users: SocialUser[] = []; 86 + 87 + // Find and parse JSON file 88 + for (const [_, file] of bundle.files) { 89 + if (file.type === 'json') { 90 + try { 91 + const jsonData = JSON.parse(file.content); 92 + 93 + // Instagram JSON exports can have different structures 94 + let followingArray = jsonData?.relationships_following; 95 + 96 + if (!followingArray && jsonData?.following) { 97 + followingArray = jsonData.following; 98 + } 99 + 100 + if (!Array.isArray(followingArray)) { 101 + continue; 102 + } 103 + 104 + for (const entry of followingArray) { 105 + const username = entry.string_list_data?.[0]?.value || entry.username || entry.handle; 106 + const timestamp = entry.string_list_data?.[0]?.timestamp || entry.timestamp; 107 + 108 + if (username) { 109 + users.push({ 110 + username: username, 111 + date: timestamp ? new Date(timestamp * 1000).toISOString() : '' 112 + }); 113 + } 114 + } 115 + 116 + if (users.length > 0) { 117 + return users; 118 + } 119 + } catch (e) { 120 + continue; 121 + } 122 + } 123 + } 124 + 125 + throw new PlatformParseError( 126 + 'No valid Instagram JSON data found. Expected relationships_following or following array', 127 + 'instagram' 128 + ); 129 + } 130 + }; 131 + 132 + // Instagram Platform Configuration 133 + export const instagramPlatform: PlatformConfig = { 134 + id: 'instagram', 135 + name: 'Instagram', 136 + parsers: [htmlParser, jsonParser], // Try HTML first (most common) 137 + expectedFiles: ['following.html', 'connections.json', 'followers_and_following.json'], 138 + validate: (bundle: FileBundle) => { 139 + // Check if bundle contains Instagram-like files 140 + for (const [path, file] of bundle.files) { 141 + if (path.toLowerCase().includes('instagram') || 142 + path.toLowerCase().includes('connections') || 143 + (file.name.toLowerCase().includes('following') && file.type === 'html')) { 144 + return true; 145 + } 146 + } 147 + return false; 148 + } 149 + };

+120

src/lib/platforms/parser.ts

··· 1 + import JSZip from "jszip"; 2 + import type { FileBundle, SocialUser } from './types'; 3 + import { PlatformParseError } from './types'; 4 + import { getPlatform } from './registry'; 5 + 6 + // Convert a file into a FileBundle (extract ZIP if needed) 7 + async function createBundle(file: File): Promise<FileBundle> { 8 + const bundle: FileBundle = { 9 + files: new Map(), 10 + originalFileName: file.name 11 + }; 12 + 13 + if (file.name.endsWith('.zip')) { 14 + // Extract ZIP contents 15 + const zip = await JSZip.loadAsync(file); 16 + 17 + for (const [path, zipEntry] of Object.entries(zip.files)) { 18 + if (zipEntry.dir) continue; // Skip directories 19 + 20 + const content = await zipEntry.async('string'); 21 + const fileName = path.split('/').pop() || path; 22 + 23 + // Determine file type 24 + let type: 'text' | 'html' | 'json' = 'text'; 25 + if (fileName.endsWith('.html')) type = 'html'; 26 + else if (fileName.endsWith('.json')) type = 'json'; 27 + else if (fileName.endsWith('.txt')) type = 'text'; 28 + 29 + bundle.files.set(path, { 30 + name: fileName, 31 + content, 32 + type 33 + }); 34 + } 35 + } else { 36 + // Single file 37 + const content = await file.text(); 38 + let type: 'text' | 'html' | 'json' = 'text'; 39 + 40 + if (file.name.endsWith('.html')) type = 'html'; 41 + else if (file.name.endsWith('.json')) type = 'json'; 42 + else if (file.name.endsWith('.txt')) type = 'text'; 43 + 44 + bundle.files.set(file.name, { 45 + name: file.name, 46 + content, 47 + type 48 + }); 49 + } 50 + 51 + return bundle; 52 + } 53 + 54 + /** 55 + * Parse a file for a specific platform 56 + */ 57 + export async function parseFile(file: File, platformId: string): Promise<SocialUser[]> { 58 + // Get platform config 59 + const platform = getPlatform(platformId); 60 + if (!platform) { 61 + throw new PlatformParseError( 62 + `Platform '${platformId}' is not supported`, 63 + platformId 64 + ); 65 + } 66 + 67 + // Create file bundle 68 + const bundle = await createBundle(file); 69 + 70 + if (bundle.files.size === 0) { 71 + throw new PlatformParseError( 72 + 'No files found in upload', 73 + platformId 74 + ); 75 + } 76 + 77 + // Validate bundle contains expected files (optional check) 78 + if (!platform.validate(bundle)) { 79 + const expectedFiles = platform.expectedFiles.join(', '); 80 + throw new PlatformParseError( 81 + `File doesn't appear to be ${platform.name} data. Expected files like: ${expectedFiles}`, 82 + platformId 83 + ); 84 + } 85 + 86 + // Try each parser in order 87 + const errors: string[] = []; 88 + 89 + for (const parser of platform.parsers) { 90 + if (!parser.canParse(bundle)) { 91 + continue; // Skip parsers that can't handle this bundle 92 + } 93 + 94 + try { 95 + const users = await parser.parse(bundle); 96 + 97 + if (users.length === 0) { 98 + errors.push(`${parser.name}: No users found`); 99 + continue; 100 + } 101 + 102 + console.log(`Successfully parsed ${users.length} users using ${parser.name}`); 103 + return users; 104 + } catch (error) { 105 + const errorMsg = error instanceof Error ? error.message : 'Unknown error'; 106 + errors.push(`${parser.name}: ${errorMsg}`); 107 + console.warn(`${parser.name} failed:`, errorMsg); 108 + } 109 + } 110 + 111 + // All parsers failed 112 + throw new PlatformParseError( 113 + `Could not parse ${platform.name} data. Tried: ${errors.join('; ')}`, 114 + platformId 115 + ); 116 + } 117 + 118 + // Export for backwards compatibility 119 + export { PlatformParseError } from './types'; 120 + export type { SocialUser } from './types';

+26

src/lib/platforms/registry.ts

··· 1 + import type { PlatformConfig } from './types'; 2 + import { tiktokPlatform } from './tiktok'; 3 + import { instagramPlatform } from './instagram'; 4 + 5 + // Registry of all supported platforms 6 + const platformRegistry = new Map<string, PlatformConfig>(); 7 + 8 + // Register platforms 9 + platformRegistry.set('tiktok', tiktokPlatform); 10 + platformRegistry.set('instagram', instagramPlatform); 11 + 12 + // Future platforms can be added here: 13 + // platformRegistry.set('twitter', twitterPlatform); 14 + // platformRegistry.set('youtube', youtubePlatform); 15 + 16 + export function getPlatform(platformId: string): PlatformConfig | undefined { 17 + return platformRegistry.get(platformId); 18 + } 19 + 20 + export function getAllPlatforms(): PlatformConfig[] { 21 + return Array.from(platformRegistry.values()); 22 + } 23 + 24 + export function isPlatformSupported(platformId: string): boolean { 25 + return platformRegistry.has(platformId); 26 + }

+114

src/lib/platforms/tiktok.ts

··· 1 + import type { PlatformConfig, PlatformParser, FileBundle, SocialUser } from './types'; 2 + import { PlatformParseError } from './types'; 3 + 4 + // TXT Parser for TikTok Following.txt format 5 + const txtParser: PlatformParser = { 6 + name: 'TikTok TXT', 7 + canParse: (bundle: FileBundle) => { 8 + // Look for .txt files that might be TikTok format 9 + for (const [_, file] of bundle.files) { 10 + if (file.name.toLowerCase().includes('following') && file.type === 'text') { 11 + return file.content.includes('Username:'); 12 + } 13 + } 14 + return false; 15 + }, 16 + parse: async (bundle: FileBundle) => { 17 + const users: SocialUser[] = []; 18 + 19 + // Find the TikTok following.txt file 20 + let content = ''; 21 + for (const [_, file] of bundle.files) { 22 + if (file.name.toLowerCase().includes('following') && file.type === 'text') { 23 + content = file.content; 24 + break; 25 + } 26 + } 27 + 28 + if (!content) { 29 + throw new PlatformParseError('No TikTok following.txt file found', 'tiktok'); 30 + } 31 + 32 + const entries = content.split("\n\n").map((b) => b.trim()).filter(Boolean); 33 + 34 + for (const entry of entries) { 35 + const userMatch = entry.match(/Username:\s*(.+)/); 36 + if (userMatch) { 37 + users.push({ username: userMatch[1].trim(), date: "" }); 38 + } 39 + } 40 + 41 + return users; 42 + } 43 + }; 44 + 45 + // JSON Parser for TikTok JSON exports 46 + const jsonParser: PlatformParser = { 47 + name: 'TikTok JSON', 48 + canParse: (bundle: FileBundle) => { 49 + for (const [_, file] of bundle.files) { 50 + if (file.type === 'json') { 51 + try { 52 + const data = JSON.parse(file.content); 53 + return !!data?.["Your Activity"]?.["Following"]?.["Following"]; 54 + } catch { 55 + return false; 56 + } 57 + } 58 + } 59 + return false; 60 + }, 61 + parse: async (bundle: FileBundle) => { 62 + const users: SocialUser[] = []; 63 + 64 + // Find and parse JSON file 65 + for (const [_, file] of bundle.files) { 66 + if (file.type === 'json') { 67 + try { 68 + const jsonData = JSON.parse(file.content); 69 + const followingArray = jsonData?.["Your Activity"]?.["Following"]?.["Following"]; 70 + 71 + if (!followingArray || !Array.isArray(followingArray)) { 72 + continue; 73 + } 74 + 75 + for (const entry of followingArray) { 76 + users.push({ 77 + username: entry.UserName, 78 + date: entry.Date || "", 79 + }); 80 + } 81 + 82 + if (users.length > 0) { 83 + return users; 84 + } 85 + } catch (e) { 86 + continue; 87 + } 88 + } 89 + } 90 + 91 + throw new PlatformParseError( 92 + 'No valid TikTok JSON data found. Expected path: Your Activity > Following > Following', 93 + 'tiktok' 94 + ); 95 + } 96 + }; 97 + 98 + // TikTok Platform Configuration 99 + export const tiktokPlatform: PlatformConfig = { 100 + id: 'tiktok', 101 + name: 'TikTok', 102 + parsers: [txtParser, jsonParser], // Try TXT first (most common) 103 + expectedFiles: ['Following.txt', 'user_data.json'], 104 + validate: (bundle: FileBundle) => { 105 + // Check if bundle contains TikTok-like files 106 + for (const [path, file] of bundle.files) { 107 + if (path.toLowerCase().includes('tiktok') || 108 + (file.name.toLowerCase().includes('following') && file.type === 'text')) { 109 + return true; 110 + } 111 + } 112 + return false; 113 + } 114 + };

+30

src/lib/platforms/types.ts

··· 1 + export interface SocialUser { 2 + username: string; 3 + date: string; 4 + } 5 + 6 + export interface FileBundle { 7 + files: Map<string, { name: string; content: string; type: 'text' | 'html' | 'json' }>; 8 + originalFileName: string; 9 + } 10 + 11 + export interface PlatformParser { 12 + name: string; 13 + parse: (bundle: FileBundle) => Promise<SocialUser[]>; 14 + canParse: (bundle: FileBundle) => boolean; 15 + } 16 + 17 + export interface PlatformConfig { 18 + id: string; 19 + name: string; 20 + parsers: PlatformParser[]; 21 + expectedFiles: string[]; // File patterns to look for 22 + validate: (bundle: FileBundle) => boolean; 23 + } 24 + 25 + export class PlatformParseError extends Error { 26 + constructor(message: string, public platform: string) { 27 + super(message); 28 + this.name = 'PlatformParseError'; 29 + } 30 + }