Add full-text search with SQLite FTS5 · aparker.io/papili.one@6a0a45c

+2 -1

package.json

··· 33 33 "db:migrate:local": "drizzle-kit migrate --config drizzle.local.config.ts", 34 34 "db:studio": "drizzle-kit studio", 35 35 "db:studio:content": "drizzle-kit studio --config drizzle.content.config.ts", 36 - "db:studio:local": "drizzle-kit studio --config drizzle.local.config.ts" 36 + "db:studio:local": "drizzle-kit studio --config drizzle.local.config.ts", 37 + "db:fts": "tsx scripts/setup-fts.ts" 37 38 }, 38 39 "devDependencies": { 39 40 "@atproto/lex": "^0.0.5",

+98

scripts/setup-fts.ts

··· 1 + /** 2 + * Set up FTS5 full-text search tables and triggers 3 + * Run with: pnpm db:fts 4 + */ 5 + 6 + import { createClient } from '@libsql/client'; 7 + 8 + const CONTENT_DB_PATH = process.env.CONTENT_DB_PATH || './data/content.db'; 9 + 10 + // FTS setup statements - each is a complete statement 11 + const statements = [ 12 + // Posts FTS table 13 + `CREATE VIRTUAL TABLE IF NOT EXISTS posts_fts USING fts5( 14 + uri UNINDEXED, 15 + title, 16 + text, 17 + url, 18 + content='posts', 19 + content_rowid='rowid' 20 + )`, 21 + 22 + // Posts triggers 23 + `CREATE TRIGGER IF NOT EXISTS posts_fts_insert AFTER INSERT ON posts BEGIN 24 + INSERT INTO posts_fts(rowid, uri, title, text, url) 25 + VALUES (new.rowid, new.uri, new.title, new.text, new.url); 26 + END`, 27 + 28 + `CREATE TRIGGER IF NOT EXISTS posts_fts_delete AFTER DELETE ON posts BEGIN 29 + INSERT INTO posts_fts(posts_fts, rowid, uri, title, text, url) 30 + VALUES ('delete', old.rowid, old.uri, old.title, old.text, old.url); 31 + END`, 32 + 33 + `CREATE TRIGGER IF NOT EXISTS posts_fts_update AFTER UPDATE ON posts BEGIN 34 + INSERT INTO posts_fts(posts_fts, rowid, uri, title, text, url) 35 + VALUES ('delete', old.rowid, old.uri, old.title, old.text, old.url); 36 + INSERT INTO posts_fts(rowid, uri, title, text, url) 37 + VALUES (new.rowid, new.uri, new.title, new.text, new.url); 38 + END`, 39 + 40 + // Comments FTS table 41 + `CREATE VIRTUAL TABLE IF NOT EXISTS comments_fts USING fts5( 42 + uri UNINDEXED, 43 + post_uri UNINDEXED, 44 + text, 45 + content='comments', 46 + content_rowid='rowid' 47 + )`, 48 + 49 + // Comments triggers 50 + `CREATE TRIGGER IF NOT EXISTS comments_fts_insert AFTER INSERT ON comments BEGIN 51 + INSERT INTO comments_fts(rowid, uri, post_uri, text) 52 + VALUES (new.rowid, new.uri, new.post_uri, new.text); 53 + END`, 54 + 55 + `CREATE TRIGGER IF NOT EXISTS comments_fts_delete AFTER DELETE ON comments BEGIN 56 + INSERT INTO comments_fts(comments_fts, rowid, uri, post_uri, text) 57 + VALUES ('delete', old.rowid, old.uri, old.post_uri, old.text); 58 + END`, 59 + 60 + `CREATE TRIGGER IF NOT EXISTS comments_fts_update AFTER UPDATE ON comments BEGIN 61 + INSERT INTO comments_fts(comments_fts, rowid, uri, post_uri, text) 62 + VALUES ('delete', old.rowid, old.uri, old.post_uri, old.text); 63 + INSERT INTO comments_fts(rowid, uri, post_uri, text) 64 + VALUES (new.rowid, new.uri, new.post_uri, new.text); 65 + END`, 66 + 67 + // Rebuild FTS indexes from existing data 68 + `INSERT INTO posts_fts(posts_fts) VALUES('rebuild')`, 69 + `INSERT INTO comments_fts(comments_fts) VALUES('rebuild')` 70 + ]; 71 + 72 + async function setupFTS() { 73 + console.log(`Setting up FTS5 for ${CONTENT_DB_PATH}...`); 74 + 75 + const client = createClient({ 76 + url: `file:${CONTENT_DB_PATH}` 77 + }); 78 + 79 + for (const sql of statements) { 80 + const preview = sql.slice(0, 60).replace(/\s+/g, ' ').trim(); 81 + try { 82 + await client.execute(sql); 83 + console.log('✓', preview + '...'); 84 + } catch (err) { 85 + if (err instanceof Error && err.message.includes('already exists')) { 86 + console.log('⊘', preview + '... (already exists)'); 87 + } else { 88 + console.error('✗', preview); 89 + console.error(' Error:', err instanceof Error ? err.message : err); 90 + } 91 + } 92 + } 93 + 94 + console.log('\nFTS5 setup complete!'); 95 + process.exit(0); 96 + } 97 + 98 + setupFTS();

+55

src/lib/components/SearchBox.svelte

···

+1 -1

src/lib/server/db/index.ts

··· 19 19 const LOCAL_DB_PATH = process.env.LOCAL_DB_PATH || './data/local.db'; 20 20 21 21 // Content DB - posts, comments, accounts (read-only in webapp, write in ingester) 22 - const contentClient = createClient({ 22 + export const contentClient = createClient({ 23 23 url: `file:${CONTENT_DB_PATH}` 24 24 }); 25 25 export const contentDb = drizzle(contentClient, { schema: contentSchema });

+12

src/routes/+layout.svelte

··· 2 2 import '../app.css'; 3 3 import favicon from '$lib/assets/favicon.svg'; 4 4 import Logo from '$lib/components/Logo.svelte'; 5 + import SearchBox from '$lib/components/SearchBox.svelte'; 5 6 6 7 let { children, data } = $props(); 7 8 let menuOpen = $state(false); ··· 28 29 <Logo /> 29 30 <a href="/new" class="text-violet-200 hover:text-white">new</a> 30 31 <a href="/comments" class="text-violet-200 hover:text-white">comments</a> 32 + 33 + <div class="hidden sm:block"> 34 + <SearchBox compact /> 35 + </div> 31 36 32 37 <div class="flex-1"></div> 33 38 ··· 74 79 {#if menuOpen} 75 80 <div class="absolute right-0 top-full mt-2 w-48 rounded-md bg-white dark:bg-gray-800 shadow-lg ring-1 ring-black/5 dark:ring-white/10 z-50"> 76 81 <div class="py-1"> 82 + <a 83 + href="/search" 84 + class="block px-4 py-2 text-sm text-gray-700 dark:text-gray-200 hover:bg-gray-100 dark:hover:bg-gray-700" 85 + onclick={closeMenu} 86 + > 87 + Search 88 + </a> 77 89 <a 78 90 href="/submit" 79 91 class="block px-4 py-2 text-sm text-gray-700 dark:text-gray-200 hover:bg-gray-100 dark:hover:bg-gray-700"

+122

src/routes/api/search/+server.ts

··· 1 + import { json } from '@sveltejs/kit'; 2 + import type { RequestHandler } from './$types'; 3 + import { contentClient } from '$lib/server/db'; 4 + import { fetchProfiles, getProfileOrFallback } from '$lib/server/profiles'; 5 + 6 + export const GET: RequestHandler = async ({ url }) => { 7 + const query = url.searchParams.get('q')?.trim(); 8 + const type = url.searchParams.get('type') || 'posts'; // 'posts' | 'comments' | 'all' 9 + const limit = Math.min(parseInt(url.searchParams.get('limit') || '20'), 100); 10 + 11 + if (!query || query.length < 2) { 12 + return json({ posts: [], comments: [], error: 'Query must be at least 2 characters' }); 13 + } 14 + 15 + // Escape special FTS5 characters and prepare query 16 + const ftsQuery = query 17 + .replace(/['"]/g, '') // Remove quotes 18 + .split(/\s+/) 19 + .filter((term) => term.length > 0) 20 + .map((term) => `"${term}"*`) // Prefix match each term 21 + .join(' '); 22 + 23 + const results: { posts: unknown[]; comments: unknown[] } = { posts: [], comments: [] }; 24 + 25 + try { 26 + // Search posts 27 + if (type === 'posts' || type === 'all') { 28 + const postsResult = await contentClient.execute({ 29 + sql: ` 30 + SELECT 31 + p.uri, 32 + p.cid, 33 + p.author_did as authorDid, 34 + p.rkey, 35 + p.url, 36 + p.title, 37 + p.text, 38 + p.created_at as createdAt, 39 + p.vote_count as voteCount, 40 + snippet(posts_fts, 1, '<mark>', '</mark>', '...', 32) as titleSnippet, 41 + snippet(posts_fts, 2, '<mark>', '</mark>', '...', 64) as textSnippet 42 + FROM posts_fts 43 + JOIN posts p ON posts_fts.uri = p.uri 44 + WHERE posts_fts MATCH ? 45 + ORDER BY rank 46 + LIMIT ? 47 + `, 48 + args: [ftsQuery, limit] 49 + }); 50 + 51 + // Get author profiles 52 + const authorDids = postsResult.rows.map((r) => r.authorDid as string); 53 + const profiles = await fetchProfiles(authorDids); 54 + 55 + results.posts = postsResult.rows.map((row) => ({ 56 + uri: row.uri, 57 + cid: row.cid, 58 + authorDid: row.authorDid, 59 + rkey: row.rkey, 60 + url: row.url, 61 + title: row.title, 62 + text: row.text, 63 + createdAt: row.createdAt, 64 + voteCount: row.voteCount, 65 + titleSnippet: row.titleSnippet, 66 + textSnippet: row.textSnippet, 67 + author: getProfileOrFallback(profiles, row.authorDid as string) 68 + })); 69 + } 70 + 71 + // Search comments 72 + if (type === 'comments' || type === 'all') { 73 + const commentsResult = await contentClient.execute({ 74 + sql: ` 75 + SELECT 76 + c.uri, 77 + c.cid, 78 + c.author_did as authorDid, 79 + c.rkey, 80 + c.post_uri as postUri, 81 + c.text, 82 + c.created_at as createdAt, 83 + c.vote_count as voteCount, 84 + p.rkey as postRkey, 85 + p.title as postTitle, 86 + snippet(comments_fts, 2, '<mark>', '</mark>', '...', 64) as textSnippet 87 + FROM comments_fts 88 + JOIN comments c ON comments_fts.uri = c.uri 89 + JOIN posts p ON c.post_uri = p.uri 90 + WHERE comments_fts MATCH ? 91 + ORDER BY rank 92 + LIMIT ? 93 + `, 94 + args: [ftsQuery, limit] 95 + }); 96 + 97 + // Get author profiles 98 + const authorDids = commentsResult.rows.map((r) => r.authorDid as string); 99 + const profiles = await fetchProfiles(authorDids); 100 + 101 + results.comments = commentsResult.rows.map((row) => ({ 102 + uri: row.uri, 103 + cid: row.cid, 104 + authorDid: row.authorDid, 105 + rkey: row.rkey, 106 + postUri: row.postUri, 107 + postRkey: row.postRkey, 108 + postTitle: row.postTitle, 109 + text: row.text, 110 + createdAt: row.createdAt, 111 + voteCount: row.voteCount, 112 + textSnippet: row.textSnippet, 113 + author: getProfileOrFallback(profiles, row.authorDid as string) 114 + })); 115 + } 116 + 117 + return json(results); 118 + } catch (err) { 119 + console.error('[search] FTS query error:', err); 120 + return json({ posts: [], comments: [], error: 'Search failed' }, { status: 500 }); 121 + } 122 + };

+155

src/routes/search/+page.server.ts

··· 1 + import type { PageServerLoad } from './$types'; 2 + import { contentClient } from '$lib/server/db'; 3 + import { fetchProfiles, getProfileOrFallback } from '$lib/server/profiles'; 4 + import type { AuthorProfile } from '$lib/types'; 5 + 6 + export interface SearchPost { 7 + uri: string; 8 + cid: string; 9 + authorDid: string; 10 + rkey: string; 11 + url: string | null; 12 + title: string; 13 + text: string | null; 14 + createdAt: string; 15 + voteCount: number; 16 + titleSnippet: string | null; 17 + textSnippet: string | null; 18 + author: AuthorProfile; 19 + } 20 + 21 + export interface SearchComment { 22 + uri: string; 23 + cid: string; 24 + authorDid: string; 25 + rkey: string; 26 + postUri: string; 27 + postRkey: string; 28 + postTitle: string; 29 + text: string; 30 + createdAt: string; 31 + voteCount: number; 32 + textSnippet: string | null; 33 + author: AuthorProfile; 34 + } 35 + 36 + export const load: PageServerLoad = async ({ url }) => { 37 + const query = url.searchParams.get('q')?.trim() || ''; 38 + const type = url.searchParams.get('type') || 'posts'; 39 + 40 + if (!query || query.length < 2) { 41 + return { query, type, posts: [] as SearchPost[], comments: [] as SearchComment[] }; 42 + } 43 + 44 + // Escape special FTS5 characters and prepare query 45 + const ftsQuery = query 46 + .replace(/['"]/g, '') 47 + .split(/\s+/) 48 + .filter((term) => term.length > 0) 49 + .map((term) => `"${term}"*`) 50 + .join(' '); 51 + 52 + const results: { posts: SearchPost[]; comments: SearchComment[] } = { posts: [], comments: [] }; 53 + 54 + try { 55 + // Search posts 56 + if (type === 'posts' || type === 'all') { 57 + const postsResult = await contentClient.execute({ 58 + sql: ` 59 + SELECT 60 + p.uri, 61 + p.cid, 62 + p.author_did as authorDid, 63 + p.rkey, 64 + p.url, 65 + p.title, 66 + p.text, 67 + p.created_at as createdAt, 68 + p.vote_count as voteCount, 69 + snippet(posts_fts, 1, '<mark>', '</mark>', '...', 32) as titleSnippet, 70 + snippet(posts_fts, 2, '<mark>', '</mark>', '...', 64) as textSnippet 71 + FROM posts_fts 72 + JOIN posts p ON posts_fts.uri = p.uri 73 + WHERE posts_fts MATCH ? 74 + ORDER BY rank 75 + LIMIT 50 76 + `, 77 + args: [ftsQuery] 78 + }); 79 + 80 + const authorDids = postsResult.rows.map((r) => r.authorDid as string); 81 + const profiles = await fetchProfiles(authorDids); 82 + 83 + results.posts = postsResult.rows.map((row) => ({ 84 + uri: row.uri as string, 85 + cid: row.cid as string, 86 + authorDid: row.authorDid as string, 87 + rkey: row.rkey as string, 88 + url: row.url as string | null, 89 + title: row.title as string, 90 + text: row.text as string | null, 91 + createdAt: row.createdAt as string, 92 + voteCount: row.voteCount as number, 93 + titleSnippet: row.titleSnippet as string | null, 94 + textSnippet: row.textSnippet as string | null, 95 + author: getProfileOrFallback(profiles, row.authorDid as string) 96 + })); 97 + } 98 + 99 + // Search comments 100 + if (type === 'comments' || type === 'all') { 101 + const commentsResult = await contentClient.execute({ 102 + sql: ` 103 + SELECT 104 + c.uri, 105 + c.cid, 106 + c.author_did as authorDid, 107 + c.rkey, 108 + c.post_uri as postUri, 109 + c.text, 110 + c.created_at as createdAt, 111 + c.vote_count as voteCount, 112 + p.rkey as postRkey, 113 + p.title as postTitle, 114 + snippet(comments_fts, 2, '<mark>', '</mark>', '...', 64) as textSnippet 115 + FROM comments_fts 116 + JOIN comments c ON comments_fts.uri = c.uri 117 + JOIN posts p ON c.post_uri = p.uri 118 + WHERE comments_fts MATCH ? 119 + ORDER BY rank 120 + LIMIT 50 121 + `, 122 + args: [ftsQuery] 123 + }); 124 + 125 + const authorDids = commentsResult.rows.map((r) => r.authorDid as string); 126 + const profiles = await fetchProfiles(authorDids); 127 + 128 + results.comments = commentsResult.rows.map((row) => ({ 129 + uri: row.uri as string, 130 + cid: row.cid as string, 131 + authorDid: row.authorDid as string, 132 + rkey: row.rkey as string, 133 + postUri: row.postUri as string, 134 + postRkey: row.postRkey as string, 135 + postTitle: row.postTitle as string, 136 + text: row.text as string, 137 + createdAt: row.createdAt as string, 138 + voteCount: row.voteCount as number, 139 + textSnippet: row.textSnippet as string | null, 140 + author: getProfileOrFallback(profiles, row.authorDid as string) 141 + })); 142 + } 143 + 144 + return { query, type, ...results }; 145 + } catch (err) { 146 + console.error('[search] FTS query error:', err); 147 + return { 148 + query, 149 + type, 150 + posts: [] as SearchPost[], 151 + comments: [] as SearchComment[], 152 + error: 'Search failed' 153 + }; 154 + } 155 + };

+178

src/routes/search/+page.svelte

··· 1 + <script lang="ts"> 2 + import { goto } from '$app/navigation'; 3 + import Avatar from '$lib/components/Avatar.svelte'; 4 + import { formatTimeAgo, getDomain } from '$lib/utils/formatting'; 5 + import type { SearchPost, SearchComment } from './+page.server'; 6 + 7 + let { data } = $props(); 8 + 9 + // Type the data 10 + const posts = $derived(data.posts as SearchPost[]); 11 + const comments = $derived(data.comments as SearchComment[]); 12 + 13 + let searchInput = $state(data.query); 14 + let selectedType = $state(data.type); 15 + 16 + function handleSearch(e: Event) { 17 + e.preventDefault(); 18 + if (searchInput.trim().length >= 2) { 19 + goto(`/search?q=${encodeURIComponent(searchInput.trim())}&type=${selectedType}`); 20 + } 21 + } 22 + 23 + function switchType(type: string) { 24 + selectedType = type; 25 + if (data.query) { 26 + goto(`/search?q=${encodeURIComponent(data.query)}&type=${type}`); 27 + } 28 + } 29 + </script> 30 + 31 + <svelte:head> 32 + <title>{data.query ? `Search: ${data.query}` : 'Search'} - papili</title> 33 + </svelte:head> 34 + 35 + <div class="space-y-4"> 36 +  37 + <form onsubmit={handleSearch} class="flex gap-2"> 38 + <input 39 + type="search" 40 + bind:value={searchInput} 41 + placeholder="Search posts and comments..." 42 + class="flex-1 px-3 py-2 rounded-md border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100 focus:outline-none focus:ring-2 focus:ring-violet-500" 43 + /> 44 + <button 45 + type="submit" 46 + class="px-4 py-2 bg-violet-600 text-white rounded-md hover:bg-violet-700 transition-colors" 47 + > 48 + Search 49 + </button> 50 + </form> 51 + 52 +  53 + {#if data.query} 54 + <div class="flex gap-4 border-b border-gray-200 dark:border-gray-700"> 55 + <button 56 + onclick={() => switchType('posts')} 57 + class="pb-2 text-sm font-medium transition-colors {selectedType === 'posts' 58 + ? 'text-violet-600 dark:text-violet-400 border-b-2 border-violet-600 dark:border-violet-400' 59 + : 'text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-300'}" 60 + > 61 + Posts ({posts.length}) 62 + </button> 63 + <button 64 + onclick={() => switchType('comments')} 65 + class="pb-2 text-sm font-medium transition-colors {selectedType === 'comments' 66 + ? 'text-violet-600 dark:text-violet-400 border-b-2 border-violet-600 dark:border-violet-400' 67 + : 'text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-300'}" 68 + > 69 + Comments ({comments.length}) 70 + </button> 71 + </div> 72 + {/if} 73 + 74 +  75 + {#if data.query && data.query.length >= 2} 76 + {#if selectedType === 'posts'} 77 + {#if posts.length === 0} 78 + <p class="text-gray-500 dark:text-gray-400 py-8 text-center"> 79 + No posts found for "{data.query}" 80 + </p> 81 + {:else} 82 + <ol class="space-y-3"> 83 + {#each posts as post (post.uri)} 84 + <li class="flex gap-2 text-sm"> 85 + <div class="flex-1 min-w-0"> 86 + <div> 87 + {#if post.url} 88 + <a 89 + href={post.url} 90 + target="_blank" 91 + rel="noopener noreferrer" 92 + class="text-gray-900 dark:text-gray-100 hover:underline" 93 + > 94 + {@html post.titleSnippet || post.title} 95 + </a> 96 + <a 97 + href="/from/{getDomain(post.url)}" 98 + class="text-xs text-gray-400 dark:text-gray-500 ml-1 hover:text-violet-600 dark:hover:text-violet-400" 99 + > 100 + ({getDomain(post.url)}) 101 + </a> 102 + {:else} 103 + <a 104 + href="/post/{post.rkey}" 105 + class="text-gray-900 dark:text-gray-100 hover:underline" 106 + > 107 + {@html post.titleSnippet || post.title} 108 + </a> 109 + {/if} 110 + </div> 111 + {#if post.textSnippet} 112 + <p class="text-xs text-gray-600 dark:text-gray-400 mt-0.5"> 113 + {@html post.textSnippet} 114 + </p> 115 + {/if} 116 + <div class="text-xs text-gray-500 dark:text-gray-400 flex items-center gap-1 mt-0.5"> 117 + <span>{post.voteCount} points</span> 118 + <span>by</span> 119 + <Avatar 120 + handle={post.author.handle} 121 + avatar={post.author.avatar} 122 + did={post.author.did} 123 + size="xs" 124 + showHandle 125 + link 126 + /> 127 + <span>{formatTimeAgo(post.createdAt)}</span> 128 + </div> 129 + </div> 130 + </li> 131 + {/each} 132 + </ol> 133 + {/if} 134 + {:else if selectedType === 'comments'} 135 + {#if comments.length === 0} 136 + <p class="text-gray-500 dark:text-gray-400 py-8 text-center"> 137 + No comments found for "{data.query}" 138 + </p> 139 + {:else} 140 + <ol class="space-y-3"> 141 + {#each comments as comment (comment.uri)} 142 + <li class="text-sm border-l-2 border-gray-200 dark:border-gray-700 pl-3"> 143 + <div class="text-xs text-gray-500 dark:text-gray-400 flex items-center gap-1 mb-1"> 144 + <Avatar 145 + handle={comment.author.handle} 146 + avatar={comment.author.avatar} 147 + did={comment.author.did} 148 + size="xs" 149 + showHandle 150 + link 151 + /> 152 + <span>{formatTimeAgo(comment.createdAt)}</span> 153 + <span>on</span> 154 + <a 155 + href="/post/{comment.postRkey}" 156 + class="text-violet-600 dark:text-violet-400 hover:underline truncate max-w-xs" 157 + > 158 + {comment.postTitle} 159 + </a> 160 + </div> 161 + <p class="text-gray-700 dark:text-gray-300"> 162 + {@html comment.textSnippet || comment.text} 163 + </p> 164 + </li> 165 + {/each} 166 + </ol> 167 + {/if} 168 + {/if} 169 + {:else if data.query} 170 + <p class="text-gray-500 dark:text-gray-400 py-8 text-center"> 171 + Search query must be at least 2 characters 172 + </p> 173 + {:else} 174 + <p class="text-gray-500 dark:text-gray-400 py-8 text-center"> 175 + Enter a search term to find posts and comments 176 + </p> 177 + {/if} 178 + </div>