···11+# Tiered Storage Configuration
22+# Copy this file to .env and configure for your environment
33+44+# ============================================================================
55+# S3 Configuration (Cold Tier - Required)
66+# ============================================================================
77+88+# AWS S3 bucket name (or S3-compatible bucket)
99+S3_BUCKET=tiered-storage-cache
1010+1111+# Optional: Separate bucket for metadata (RECOMMENDED for production!)
1212+# When set, metadata is stored as separate JSON objects instead of S3 object metadata.
1313+# This allows fast, cheap metadata updates without copying entire objects.
1414+# Leave blank to store metadata in S3 object metadata fields (slower, more expensive).
1515+S3_METADATA_BUCKET=tiered-storage-metadata
1616+1717+# AWS region
1818+S3_REGION=us-east-1
1919+2020+# S3 endpoint (optional - for S3-compatible services like R2, Minio)
2121+# Leave blank for AWS S3
2222+# For Cloudflare R2: https://YOUR-ACCOUNT-ID.r2.cloudflarestorage.com
2323+# For MinIO: http://localhost:9000
2424+# For other S3-compatible: https://s3.your-service.com
2525+# S3_ENDPOINT=
2626+2727+# Force path-style URLs (usually needed for S3-compatible services)
2828+# Default: true (recommended for most S3-compatible services)
2929+# Set to false only if your service requires virtual-host-style URLs
3030+# S3_FORCE_PATH_STYLE=true
3131+3232+# AWS credentials
3333+# If not provided, uses default AWS credential chain
3434+# (environment variables, ~/.aws/credentials, IAM roles, etc.)
3535+AWS_ACCESS_KEY_ID=your_access_key_id
3636+AWS_SECRET_ACCESS_KEY=your_secret_access_key
3737+3838+# ============================================================================
3939+# Cloudflare R2 Example Configuration
4040+# ============================================================================
4141+# Uncomment these to use Cloudflare R2 instead of AWS S3:
4242+#
4343+# S3_BUCKET=my-r2-bucket
4444+# S3_METADATA_BUCKET=my-r2-metadata-bucket
4545+# S3_REGION=auto
4646+# S3_ENDPOINT=https://YOUR-ACCOUNT-ID.r2.cloudflarestorage.com
4747+# AWS_ACCESS_KEY_ID=your_r2_access_key_id
4848+# AWS_SECRET_ACCESS_KEY=your_r2_secret_access_key
4949+5050+# ============================================================================
5151+# Memory Tier Configuration (Hot)
5252+# ============================================================================
5353+5454+# Maximum size in bytes for hot (memory) tier
5555+# Default: 100MB
5656+MEMORY_MAX_SIZE_BYTES=104857600
5757+5858+# Maximum number of items in hot tier
5959+# Optional - if not set, only size limit applies
6060+MEMORY_MAX_ITEMS=1000
6161+6262+# ============================================================================
6363+# Disk Tier Configuration (Warm)
6464+# ============================================================================
6565+6666+# Directory for warm tier cache
6767+# Default: ./cache/warm
6868+DISK_WARM_DIRECTORY=./cache/warm
6969+7070+# Maximum size in bytes for warm tier
7171+# Optional - if not set, no size limit
7272+DISK_WARM_MAX_SIZE_BYTES=10737418240
7373+7474+# Eviction policy when size limit reached
7575+# Options: lru, fifo, size
7676+# Default: lru
7777+DISK_WARM_EVICTION_POLICY=lru
7878+7979+# ============================================================================
8080+# Storage Options
8181+# ============================================================================
8282+8383+# Enable compression (gzip)
8484+# Default: false
8585+COMPRESSION_ENABLED=true
8686+8787+# Default TTL in milliseconds
8888+# Optional - if not set, data never expires
8989+# Example: 1209600000 = 14 days
9090+DEFAULT_TTL_MS=1209600000
9191+9292+# Promotion strategy: 'eager' or 'lazy'
9393+# eager: Automatically promote data to upper tiers on read
9494+# lazy: Only promote on explicit bootstrap or write
9595+# Default: lazy
9696+PROMOTION_STRATEGY=lazy
9797+9898+# ============================================================================
9999+# Bootstrap Configuration
100100+# ============================================================================
101101+102102+# Number of items to load into hot tier on bootstrap
103103+# Optional - if not set, loads all items
104104+BOOTSTRAP_HOT_LIMIT=1000
105105+106106+# Number of days to look back when bootstrapping warm tier
107107+# Example: 7 = only load items accessed in last 7 days
108108+BOOTSTRAP_WARM_DAYS=7
109109+110110+# Maximum items to load into warm tier on bootstrap
111111+# Optional - if not set, loads all matching items
112112+BOOTSTRAP_WARM_LIMIT=10000
113113+114114+# ============================================================================
115115+# Performance Tuning
116116+# ============================================================================
117117+118118+# Maximum concurrent operations for bootstrap
119119+# Default: 10
120120+BOOTSTRAP_CONCURRENCY=10
121121+122122+# Timeout for tier operations in milliseconds
123123+# Default: 30000 (30 seconds)
124124+TIER_OPERATION_TIMEOUT_MS=30000
125125+126126+# ============================================================================
127127+# Monitoring & Observability
128128+# ============================================================================
129129+130130+# Enable statistics tracking
131131+# Default: true
132132+STATS_ENABLED=true
133133+134134+# Log level: debug, info, warn, error
135135+# Default: info
136136+LOG_LEVEL=info
···11+# tiered-storage
22+33+Cascading cache that flows hot → warm → cold. Memory, disk, S3—or bring your own.
44+55+## Features
66+77+- **Cascading writes** - data flows down through all tiers
88+- **Bubbling reads** - check hot first, fall back to warm, then cold
99+- **Pluggable backends** - memory, disk, S3, or implement your own
1010+- **Selective placement** - skip tiers for big files that don't need memory caching
1111+- **Prefix invalidation** - `invalidate('user:')` nukes all user keys
1212+- **Optional compression** - transparent gzip
1313+1414+## Install
1515+1616+```bash
1717+npm install tiered-storage
1818+```
1919+2020+## Example
2121+2222+```typescript
2323+import { TieredStorage, MemoryStorageTier, DiskStorageTier, S3StorageTier } from 'tiered-storage'
2424+2525+const storage = new TieredStorage({
2626+ tiers: {
2727+ hot: new MemoryStorageTier({ maxSizeBytes: 100 * 1024 * 1024 }),
2828+ warm: new DiskStorageTier({ directory: './cache' }),
2929+ cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),
3030+ },
3131+ placementRules: [
3232+ { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
3333+ { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },
3434+ { pattern: '**', tiers: ['warm', 'cold'] },
3535+ ],
3636+})
3737+3838+// just set - rules decide where it goes
3939+await storage.set('site:abc/index.html', indexHtml) // → hot + warm + cold
4040+await storage.set('site:abc/hero.png', imageData) // → warm + cold
4141+await storage.set('site:abc/video.mp4', videoData) // → warm + cold
4242+4343+// reads bubble up from wherever it lives
4444+const page = await storage.getWithMetadata('site:abc/index.html')
4545+console.log(page.source) // 'hot'
4646+4747+const video = await storage.getWithMetadata('site:abc/video.mp4')
4848+console.log(video.source) // 'warm'
4949+5050+// nuke entire site
5151+await storage.invalidate('site:abc/')
5252+```
5353+5454+Hot tier stays small and fast. Warm tier has everything. Cold tier is the source of truth.
5555+5656+## How it works
5757+5858+```
5959+┌─────────────────────────────────────────────┐
6060+│ Cold (S3) - source of truth, all data │
6161+│ ↑ │
6262+│ Warm (disk) - everything hot has + more │
6363+│ ↑ │
6464+│ Hot (memory) - just the hottest stuff │
6565+└─────────────────────────────────────────────┘
6666+```
6767+6868+Writes cascade **down**. Reads bubble **up**.
6969+7070+## Eviction
7171+7272+Items leave upper tiers through eviction or TTL expiration:
7373+7474+```typescript
7575+const storage = new TieredStorage({
7676+ tiers: {
7777+ // hot: LRU eviction when size/count limits hit
7878+ hot: new MemoryStorageTier({
7979+ maxSizeBytes: 100 * 1024 * 1024,
8080+ maxItems: 500,
8181+ }),
8282+8383+ // warm: evicts when maxSizeBytes hit, policy controls which items go
8484+ warm: new DiskStorageTier({
8585+ directory: './cache',
8686+ maxSizeBytes: 10 * 1024 * 1024 * 1024,
8787+ evictionPolicy: 'lru', // 'lru' | 'fifo' | 'size'
8888+ }),
8989+9090+ // cold: never evicts, keeps everything
9191+ cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),
9292+ },
9393+ defaultTTL: 14 * 24 * 60 * 60 * 1000, // TTL checked on read
9494+})
9595+```
9696+9797+A file that hasn't been accessed eventually gets evicted from hot (LRU), then warm (size limit + policy). Next request fetches from cold and promotes it back up.
9898+9999+## Placement rules
100100+101101+Define once which keys go where, instead of passing `skipTiers` on every `set()`:
102102+103103+```typescript
104104+const storage = new TieredStorage({
105105+ tiers: {
106106+ hot: new MemoryStorageTier({ maxSizeBytes: 50 * 1024 * 1024 }),
107107+ warm: new DiskStorageTier({ directory: './cache' }),
108108+ cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),
109109+ },
110110+ placementRules: [
111111+ // index.html goes everywhere for instant serving
112112+ { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
113113+114114+ // images and video skip hot
115115+ { pattern: '**/*.{jpg,png,gif,webp,mp4}', tiers: ['warm', 'cold'] },
116116+117117+ // assets directory skips hot
118118+ { pattern: 'assets/**', tiers: ['warm', 'cold'] },
119119+120120+ // everything else: warm + cold only
121121+ { pattern: '**', tiers: ['warm', 'cold'] },
122122+ ],
123123+})
124124+125125+// just call set() - rules handle placement
126126+await storage.set('site:abc/index.html', html) // → hot + warm + cold
127127+await storage.set('site:abc/hero.png', image) // → warm + cold
128128+await storage.set('site:abc/assets/font.woff', font) // → warm + cold
129129+await storage.set('site:abc/about.html', html) // → warm + cold
130130+```
131131+132132+Rules are evaluated in order. First match wins. Cold is always included.
133133+134134+## API
135135+136136+### `storage.get(key)`
137137+138138+Get data. Returns `null` if missing or expired.
139139+140140+### `storage.getWithMetadata(key)`
141141+142142+Get data plus which tier served it.
143143+144144+### `storage.set(key, data, options?)`
145145+146146+Store data. Options:
147147+148148+```typescript
149149+{
150150+ ttl: 86400000, // custom TTL
151151+ skipTiers: ['hot'], // skip specific tiers
152152+ metadata: { ... }, // custom metadata
153153+}
154154+```
155155+156156+### `storage.delete(key)`
157157+158158+Delete from all tiers.
159159+160160+### `storage.invalidate(prefix)`
161161+162162+Delete all keys matching prefix. Returns count.
163163+164164+### `storage.touch(key, ttl?)`
165165+166166+Renew TTL.
167167+168168+### `storage.listKeys(prefix?)`
169169+170170+Async iterator over keys.
171171+172172+### `storage.getStats()`
173173+174174+Stats across all tiers.
175175+176176+### `storage.bootstrapHot(limit?)`
177177+178178+Warm up hot tier from warm tier. Run on startup.
179179+180180+### `storage.bootstrapWarm(options?)`
181181+182182+Warm up warm tier from cold tier.
183183+184184+## Built-in tiers
185185+186186+### MemoryStorageTier
187187+188188+```typescript
189189+new MemoryStorageTier({
190190+ maxSizeBytes: 100 * 1024 * 1024,
191191+ maxItems: 1000,
192192+})
193193+```
194194+195195+LRU eviction. Fast. Single process only.
196196+197197+### DiskStorageTier
198198+199199+```typescript
200200+new DiskStorageTier({
201201+ directory: './cache',
202202+ maxSizeBytes: 10 * 1024 * 1024 * 1024,
203203+ evictionPolicy: 'lru', // or 'fifo', 'size'
204204+})
205205+```
206206+207207+Files on disk with `.meta` sidecars.
208208+209209+### S3StorageTier
210210+211211+```typescript
212212+new S3StorageTier({
213213+ bucket: 'data',
214214+ metadataBucket: 'metadata', // recommended!
215215+ region: 'us-east-1',
216216+})
217217+```
218218+219219+Works with AWS S3, Cloudflare R2, MinIO. Use a separate metadata bucket—otherwise updating access counts requires copying entire objects.
220220+221221+## Custom tiers
222222+223223+Implement `StorageTier`:
224224+225225+```typescript
226226+interface StorageTier {
227227+ get(key: string): Promise<Uint8Array | null>
228228+ set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void>
229229+ delete(key: string): Promise<void>
230230+ exists(key: string): Promise<boolean>
231231+ listKeys(prefix?: string): AsyncIterableIterator<string>
232232+ deleteMany(keys: string[]): Promise<void>
233233+ getMetadata(key: string): Promise<StorageMetadata | null>
234234+ setMetadata(key: string, metadata: StorageMetadata): Promise<void>
235235+ getStats(): Promise<TierStats>
236236+ clear(): Promise<void>
237237+238238+ // Optional: combine get + getMetadata for better performance
239239+ getWithMetadata?(key: string): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null>
240240+}
241241+```
242242+243243+The optional `getWithMetadata` method returns both data and metadata in a single call. Implement it if your backend can fetch both efficiently (e.g., parallel I/O, single query). Falls back to separate `get()` + `getMetadata()` calls if not implemented.
244244+245245+## Running the demo
246246+247247+```bash
248248+cp .env.example .env # add S3 creds
249249+bun run serve
250250+```
251251+252252+Visit http://localhost:3000 to see it work. Check http://localhost:3000/admin/stats for live cache stats.
253253+254254+## License
255255+256256+MIT
+27
packages/@wispplace/tiered-storage/agents.md
···11+You are working on a project that stores cold objects in s3, warm objects on disk, and hot objects in memory. It serves APIs for library consumers to use.
22+33+## Package management
44+- Use bun for package management (bun install)
55+- Use npm run test to test
66+- Use npm run lint to lint
77+- Use npm run check to typecheck
88+99+Please test and typecheck always whenever you think you are done.
1010+1111+## Code style
1212+- Use tabs for indentation, spaces allowed for diagrams in comments
1313+- Use single quotes and add trailing commas
1414+- Use template literals for user-facing strings and error messages
1515+1616+## Commenting
1717+Add JSDoc comments to all new exported functions, methods, classes, fields, and enums
1818+JSDoc should include proper annotations:
1919+- use @param for parameters (no dashes after param names)
2020+- use @returns for return values
2121+- use @throws for exceptions when applicable
2222+- keep descriptions concise but informative
2323+2424+## Misc
2525+the .research/ directory serves as a workspace for temporary experiments, analysis, and planning materials. create it if necessary (it's gitignored). this directory may contain cloned repositories or other reference materials that can help inform implementation decisions
2626+2727+**don't make assumptions or speculate about code, plans, or requirements without exploring first; pause and ask for clarification when you're still unsure after looking into it**
···11+# Example Static Site
22+33+This is a demonstration static website used in the tiered-storage library examples.
44+55+## Files
66+77+- **index.html** (3.5 KB) - Homepage, stored in hot + warm + cold tiers
88+- **about.html** (4.2 KB) - About page, stored in warm + cold (skips hot)
99+- **docs.html** (3.8 KB) - Documentation, stored in warm + cold (skips hot)
1010+- **style.css** (7.1 KB) - Stylesheet, stored in warm + cold (skips hot)
1111+- **script.js** (1.9 KB) - JavaScript, stored in warm + cold (skips hot)
1212+1313+## Usage in Examples
1414+1515+The `example.ts` file demonstrates how this site would be stored using the tiered-storage library:
1616+1717+1. **index.html** is stored in all tiers (hot + warm + cold) because it's the entry point and needs instant serving
1818+2. Other HTML pages skip the hot tier to save memory
1919+3. CSS and JS files are stored in warm + cold
2020+4. If there were large media files, they would be stored in cold tier only
2121+2222+## Tier Strategy
2323+2424+```
2525+Hot Tier (Memory - 100MB):
2626+ └── index.html (3.5 KB)
2727+2828+Warm Tier (Disk - 10GB):
2929+ ├── index.html (3.5 KB)
3030+ ├── about.html (4.2 KB)
3131+ ├── docs.html (3.8 KB)
3232+ ├── style.css (7.1 KB)
3333+ └── script.js (1.9 KB)
3434+3535+Cold Tier (S3 - Unlimited):
3636+ ├── index.html (3.5 KB)
3737+ ├── about.html (4.2 KB)
3838+ ├── docs.html (3.8 KB)
3939+ ├── style.css (7.1 KB)
4040+ └── script.js (1.9 KB)
4141+```
4242+4343+This strategy ensures:
4444+- Lightning-fast serving of the homepage (from memory)
4545+- Efficient use of hot tier capacity
4646+- All files are cached on disk for fast local access
4747+- S3 acts as the source of truth for disaster recovery
···11+/**
22+ * Metadata associated with stored data in a tier.
33+ *
44+ * @remarks
55+ * This metadata is stored alongside the actual data and is used for:
66+ * - TTL management and expiration
77+ * - Access tracking for LRU/eviction policies
88+ * - Data integrity verification via checksum
99+ * - Content type information for HTTP serving
1010+ */
1111+export interface StorageMetadata {
1212+ /** Original key used to store the data (human-readable) */
1313+ key: string;
1414+1515+ /** Size of the data in bytes (uncompressed size) */
1616+ size: number;
1717+1818+ /** Timestamp when the data was first created */
1919+ createdAt: Date;
2020+2121+ /** Timestamp when the data was last accessed */
2222+ lastAccessed: Date;
2323+2424+ /** Number of times this data has been accessed */
2525+ accessCount: number;
2626+2727+ /** Optional expiration timestamp. Data expires when current time > ttl */
2828+ ttl?: Date;
2929+3030+ /** Whether the data is compressed (e.g., with gzip) */
3131+ compressed: boolean;
3232+3333+ /** SHA256 checksum of the data for integrity verification */
3434+ checksum: string;
3535+3636+ /** Optional MIME type (e.g., 'text/html', 'application/json') */
3737+ mimeType?: string;
3838+3939+ /** Optional encoding (e.g., 'gzip', 'base64') */
4040+ encoding?: string;
4141+4242+ /** User-defined metadata fields */
4343+ customMetadata?: Record<string, string>;
4444+}
4545+4646+/**
4747+ * Statistics for a single storage tier.
4848+ *
4949+ * @remarks
5050+ * Used for monitoring cache performance and capacity planning.
5151+ */
5252+export interface TierStats {
5353+ /** Total bytes stored in this tier */
5454+ bytes: number;
5555+5656+ /** Total number of items stored in this tier */
5757+ items: number;
5858+5959+ /** Number of cache hits (only tracked if tier implements hit tracking) */
6060+ hits?: number;
6161+6262+ /** Number of cache misses (only tracked if tier implements miss tracking) */
6363+ misses?: number;
6464+6565+ /** Number of evictions due to size/count limits (only tracked if tier implements eviction) */
6666+ evictions?: number;
6767+}
6868+6969+/**
7070+ * Aggregated statistics across all configured tiers.
7171+ *
7272+ * @remarks
7373+ * Provides a complete view of cache performance across the entire storage hierarchy.
7474+ */
7575+export interface AllTierStats {
7676+ /** Statistics for hot tier (if configured) */
7777+ hot?: TierStats;
7878+7979+ /** Statistics for warm tier (if configured) */
8080+ warm?: TierStats;
8181+8282+ /** Statistics for cold tier (always present) */
8383+ cold: TierStats;
8484+8585+ /** Total hits across all tiers */
8686+ totalHits: number;
8787+8888+ /** Total misses across all tiers */
8989+ totalMisses: number;
9090+9191+ /** Hit rate as a percentage (0-1) */
9292+ hitRate: number;
9393+}
9494+9595+/**
9696+ * Interface that all storage tier implementations must satisfy.
9797+ *
9898+ * @remarks
9999+ * This is the core abstraction that allows pluggable backends.
100100+ * Implementations can be memory-based (Map, Redis), disk-based (filesystem, SQLite),
101101+ * or cloud-based (S3, R2, etc.).
102102+ *
103103+ * @example
104104+ * ```typescript
105105+ * class RedisStorageTier implements StorageTier {
106106+ * constructor(private client: RedisClient) {}
107107+ *
108108+ * async get(key: string): Promise<Uint8Array | null> {
109109+ * const buffer = await this.client.getBuffer(key);
110110+ * return buffer ? new Uint8Array(buffer) : null;
111111+ * }
112112+ *
113113+ * // ... implement other methods
114114+ * }
115115+ * ```
116116+ */
117117+/**
118118+ * Result from a combined get+metadata operation on a tier.
119119+ */
120120+export interface TierGetResult {
121121+ /** The retrieved data */
122122+ data: Uint8Array;
123123+ /** Metadata associated with the data */
124124+ metadata: StorageMetadata;
125125+}
126126+127127+/**
128128+ * Result from a streaming get operation on a tier.
129129+ */
130130+export interface TierStreamResult {
131131+ /** Readable stream of the data */
132132+ stream: NodeJS.ReadableStream;
133133+ /** Metadata associated with the data */
134134+ metadata: StorageMetadata;
135135+}
136136+137137+/**
138138+ * Result from a streaming get operation on TieredStorage.
139139+ *
140140+ * @remarks
141141+ * Includes the source tier for observability.
142142+ */
143143+export interface StreamResult {
144144+ /** Readable stream of the data */
145145+ stream: NodeJS.ReadableStream;
146146+ /** Metadata associated with the data */
147147+ metadata: StorageMetadata;
148148+ /** Which tier the data was served from */
149149+ source: 'hot' | 'warm' | 'cold';
150150+}
151151+152152+/**
153153+ * Options for streaming set operations.
154154+ */
155155+export interface StreamSetOptions extends SetOptions {
156156+ /**
157157+ * Size of the data being streamed in bytes.
158158+ *
159159+ * @remarks
160160+ * Required for streaming writes because the size cannot be determined
161161+ * until the stream is fully consumed. This is used for:
162162+ * - Metadata creation before streaming starts
163163+ * - Capacity checks and eviction in tiers with size limits
164164+ */
165165+ size: number;
166166+167167+ /**
168168+ * Pre-computed checksum of the data.
169169+ *
170170+ * @remarks
171171+ * If not provided, checksum will be computed during streaming.
172172+ * Providing it upfront is useful when the checksum is already known
173173+ * (e.g., from a previous upload or external source).
174174+ */
175175+ checksum?: string;
176176+177177+ /**
178178+ * MIME type of the content.
179179+ */
180180+ mimeType?: string;
181181+}
182182+183183+export interface StorageTier {
184184+ /**
185185+ * Retrieve data for a key.
186186+ *
187187+ * @param key - The key to retrieve
188188+ * @returns The data as a Uint8Array, or null if not found
189189+ */
190190+ get(key: string): Promise<Uint8Array | null>;
191191+192192+ /**
193193+ * Retrieve data and metadata together in a single operation.
194194+ *
195195+ * @param key - The key to retrieve
196196+ * @returns The data and metadata, or null if not found
197197+ *
198198+ * @remarks
199199+ * This is more efficient than calling get() and getMetadata() separately,
200200+ * especially for disk and network-based tiers.
201201+ */
202202+ getWithMetadata?(key: string): Promise<TierGetResult | null>;
203203+204204+ /**
205205+ * Retrieve data as a readable stream with metadata.
206206+ *
207207+ * @param key - The key to retrieve
208208+ * @returns A readable stream and metadata, or null if not found
209209+ *
210210+ * @remarks
211211+ * Use this for large files to avoid loading entire content into memory.
212212+ * The stream must be consumed or destroyed by the caller.
213213+ */
214214+ getStream?(key: string): Promise<TierStreamResult | null>;
215215+216216+ /**
217217+ * Store data from a readable stream.
218218+ *
219219+ * @param key - The key to store under
220220+ * @param stream - Readable stream of data to store
221221+ * @param metadata - Metadata to store alongside the data
222222+ *
223223+ * @remarks
224224+ * Use this for large files to avoid loading entire content into memory.
225225+ * The stream will be fully consumed by this operation.
226226+ */
227227+ setStream?(
228228+ key: string,
229229+ stream: NodeJS.ReadableStream,
230230+ metadata: StorageMetadata,
231231+ ): Promise<void>;
232232+233233+ /**
234234+ * Store data with associated metadata.
235235+ *
236236+ * @param key - The key to store under
237237+ * @param data - The data to store (as Uint8Array)
238238+ * @param metadata - Metadata to store alongside the data
239239+ *
240240+ * @remarks
241241+ * If the key already exists, it should be overwritten.
242242+ */
243243+ set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void>;
244244+245245+ /**
246246+ * Delete data for a key.
247247+ *
248248+ * @param key - The key to delete
249249+ *
250250+ * @remarks
251251+ * Should not throw if the key doesn't exist.
252252+ */
253253+ delete(key: string): Promise<void>;
254254+255255+ /**
256256+ * Check if a key exists in this tier.
257257+ *
258258+ * @param key - The key to check
259259+ * @returns true if the key exists, false otherwise
260260+ */
261261+ exists(key: string): Promise<boolean>;
262262+263263+ /**
264264+ * List all keys in this tier, optionally filtered by prefix.
265265+ *
266266+ * @param prefix - Optional prefix to filter keys (e.g., 'user:' matches 'user:123', 'user:456')
267267+ * @returns An async iterator of keys
268268+ *
269269+ * @remarks
270270+ * This should be memory-efficient and stream keys rather than loading all into memory.
271271+ * Useful for prefix-based invalidation and cache warming.
272272+ *
273273+ * @example
274274+ * ```typescript
275275+ * for await (const key of tier.listKeys('site:')) {
276276+ * console.log(key); // 'site:abc', 'site:xyz', etc.
277277+ * }
278278+ * ```
279279+ */
280280+ listKeys(prefix?: string): AsyncIterableIterator<string>;
281281+282282+ /**
283283+ * Delete multiple keys in a single operation.
284284+ *
285285+ * @param keys - Array of keys to delete
286286+ *
287287+ * @remarks
288288+ * This is more efficient than calling delete() in a loop.
289289+ * Implementations should batch deletions where possible.
290290+ */
291291+ deleteMany(keys: string[]): Promise<void>;
292292+293293+ /**
294294+ * Retrieve metadata for a key without fetching the data.
295295+ *
296296+ * @param key - The key to get metadata for
297297+ * @returns The metadata, or null if not found
298298+ *
299299+ * @remarks
300300+ * This is useful for checking TTL, access counts, etc. without loading large data.
301301+ */
302302+ getMetadata(key: string): Promise<StorageMetadata | null>;
303303+304304+ /**
305305+ * Update metadata for a key without modifying the data.
306306+ *
307307+ * @param key - The key to update metadata for
308308+ * @param metadata - The new metadata
309309+ *
310310+ * @remarks
311311+ * Useful for updating TTL (via touch()) or access counts.
312312+ */
313313+ setMetadata(key: string, metadata: StorageMetadata): Promise<void>;
314314+315315+ /**
316316+ * Get statistics about this tier.
317317+ *
318318+ * @returns Statistics including size, item count, hits, misses, etc.
319319+ */
320320+ getStats(): Promise<TierStats>;
321321+322322+ /**
323323+ * Clear all data from this tier.
324324+ *
325325+ * @remarks
326326+ * Use with caution! This will delete all data in the tier.
327327+ */
328328+ clear(): Promise<void>;
329329+}
330330+331331+/**
332332+ * Rule for automatic tier placement based on key patterns.
333333+ *
334334+ * @remarks
335335+ * Rules are evaluated in order. First matching rule wins.
336336+ * Use this to define which keys go to which tiers without
337337+ * specifying skipTiers on every set() call.
338338+ *
339339+ * @example
340340+ * ```typescript
341341+ * placementRules: [
342342+ * { pattern: 'index.html', tiers: ['hot', 'warm', 'cold'] },
343343+ * { pattern: '*.html', tiers: ['warm', 'cold'] },
344344+ * { pattern: 'assets/**', tiers: ['warm', 'cold'] },
345345+ * { pattern: '**', tiers: ['warm', 'cold'] }, // default
346346+ * ]
347347+ * ```
348348+ */
349349+export interface PlacementRule {
350350+ /**
351351+ * Glob pattern to match against keys.
352352+ *
353353+ * @remarks
354354+ * Supports basic globs:
355355+ * - `*` matches any characters except `/`
356356+ * - `**` matches any characters including `/`
357357+ * - Exact matches work too: `index.html`
358358+ */
359359+ pattern: string;
360360+361361+ /**
362362+ * Which tiers to write to for matching keys.
363363+ *
364364+ * @remarks
365365+ * Cold is always included (source of truth).
366366+ * Use `['hot', 'warm', 'cold']` for critical files.
367367+ * Use `['warm', 'cold']` for large files.
368368+ * Use `['cold']` for archival only.
369369+ */
370370+ tiers: ('hot' | 'warm' | 'cold')[];
371371+}
372372+373373+/**
374374+ * Configuration for the TieredStorage system.
375375+ *
376376+ * @typeParam T - The type of data being stored (for serialization)
377377+ *
378378+ * @remarks
379379+ * The tiered storage system uses a cascading containment model:
380380+ * - Hot tier (optional): Fastest, smallest capacity (memory/Redis)
381381+ * - Warm tier (optional): Medium speed, medium capacity (disk/database)
382382+ * - Cold tier (required): Slowest, unlimited capacity (S3/object storage)
383383+ *
384384+ * Data flows down on writes (hot → warm → cold) and bubbles up on reads (cold → warm → hot).
385385+ */
386386+export interface TieredStorageConfig {
387387+ /** Storage tier configuration */
388388+ tiers: {
389389+ /** Optional hot tier - fastest, smallest capacity (e.g., in-memory, Redis) */
390390+ hot?: StorageTier;
391391+392392+ /** Optional warm tier - medium speed, medium capacity (e.g., disk, SQLite, Postgres) */
393393+ warm?: StorageTier;
394394+395395+ /** Required cold tier - slowest, largest capacity (e.g., S3, R2, object storage) */
396396+ cold: StorageTier;
397397+ };
398398+399399+ /** Rules for automatic tier placement based on key patterns. First match wins. */
400400+ placementRules?: PlacementRule[];
401401+402402+ /**
403403+ * Whether to automatically compress data before storing.
404404+ *
405405+ * @defaultValue false
406406+ *
407407+ * @remarks
408408+ * Uses gzip compression. Compression is transparent - data is automatically
409409+ * decompressed on retrieval. The `compressed` flag in metadata indicates compression state.
410410+ */
411411+ compression?: boolean;
412412+413413+ /**
414414+ * Default TTL (time-to-live) in milliseconds.
415415+ *
416416+ * @remarks
417417+ * Data will expire after this duration. Can be overridden per-key via SetOptions.
418418+ * If not set, data never expires.
419419+ */
420420+ defaultTTL?: number;
421421+422422+ /**
423423+ * Strategy for promoting data to upper tiers on cache miss.
424424+ *
425425+ * @defaultValue 'lazy'
426426+ *
427427+ * @remarks
428428+ * - 'eager': Immediately promote data to all upper tiers on read
429429+ * - 'lazy': Don't automatically promote; rely on explicit promotion or next write
430430+ *
431431+ * Eager promotion increases hot tier hit rate but adds write overhead.
432432+ * Lazy promotion reduces writes but may serve from lower tiers more often.
433433+ */
434434+ promotionStrategy?: 'eager' | 'lazy';
435435+436436+ /**
437437+ * Custom serialization/deserialization functions.
438438+ *
439439+ * @remarks
440440+ * By default, JSON serialization is used. Provide custom functions for:
441441+ * - Non-JSON types (e.g., Buffer, custom classes)
442442+ * - Performance optimization (e.g., msgpack, protobuf)
443443+ * - Encryption (serialize includes encryption, deserialize includes decryption)
444444+ */
445445+ serialization?: {
446446+ /** Convert data to Uint8Array for storage */
447447+ serialize: (data: unknown) => Promise<Uint8Array>;
448448+449449+ /** Convert Uint8Array back to original data */
450450+ deserialize: (data: Uint8Array) => Promise<unknown>;
451451+ };
452452+}
453453+454454+/**
455455+ * Options for setting data in the cache.
456456+ *
457457+ * @remarks
458458+ * These options allow fine-grained control over where and how data is stored.
459459+ */
460460+export interface SetOptions {
461461+ /**
462462+ * Custom TTL in milliseconds for this specific key.
463463+ *
464464+ * @remarks
465465+ * Overrides the default TTL from TieredStorageConfig.
466466+ * Data will expire after this duration from the current time.
467467+ */
468468+ ttl?: number;
469469+470470+ /**
471471+ * Custom metadata to attach to this key.
472472+ *
473473+ * @remarks
474474+ * Merged with system-generated metadata (size, checksum, timestamps).
475475+ * Useful for storing application-specific information like content-type, encoding, etc.
476476+ */
477477+ metadata?: Record<string, string>;
478478+479479+ /**
480480+ * Skip writing to specific tiers.
481481+ *
482482+ * @remarks
483483+ * Useful for controlling which tiers receive data. For example:
484484+ * - Large files: `skipTiers: ['hot']` to avoid filling memory
485485+ * - Small critical files: Write to hot only for fastest access
486486+ *
487487+ * Note: Cold tier can never be skipped (it's the source of truth).
488488+ * Mutually exclusive with `onlyTiers`.
489489+ *
490490+ * @example
491491+ * ```typescript
492492+ * // Store large file only in warm and cold (skip memory)
493493+ * await storage.set('large-video.mp4', videoData, { skipTiers: ['hot'] });
494494+ *
495495+ * // Store index.html in all tiers for fast access
496496+ * await storage.set('index.html', htmlData); // No skipping
497497+ * ```
498498+ */
499499+ skipTiers?: ('hot' | 'warm')[];
500500+501501+ /**
502502+ * Write only to specific tiers.
503503+ *
504504+ * @remarks
505505+ * Unlike `skipTiers`, this explicitly specifies which tiers to write to.
506506+ * Useful for write-only services that should only populate cold storage.
507507+ * Mutually exclusive with `skipTiers`.
508508+ *
509509+ * @example
510510+ * ```typescript
511511+ * // Write only to cold tier (S3) - useful for firehose/ingestion services
512512+ * await storage.set('site/index.html', htmlData, { onlyTiers: ['cold'] });
513513+ *
514514+ * // Write to warm and cold, skip hot
515515+ * await storage.set('large-file.mp4', videoData, { onlyTiers: ['warm', 'cold'] });
516516+ * ```
517517+ */
518518+ onlyTiers?: ('hot' | 'warm' | 'cold')[];
519519+}
520520+521521+/**
522522+ * Result from retrieving data with metadata.
523523+ *
524524+ * @typeParam T - The type of data being retrieved
525525+ *
526526+ * @remarks
527527+ * Includes both the data and information about where it was served from.
528528+ */
529529+export interface StorageResult<T> {
530530+ /** The retrieved data */
531531+ data: T;
532532+533533+ /** Metadata associated with the data */
534534+ metadata: StorageMetadata;
535535+536536+ /** Which tier the data was served from */
537537+ source: 'hot' | 'warm' | 'cold';
538538+}
539539+540540+/**
541541+ * Result from setting data in the cache.
542542+ *
543543+ * @remarks
544544+ * Indicates which tiers successfully received the data.
545545+ */
546546+export interface SetResult {
547547+ /** The key that was set */
548548+ key: string;
549549+550550+ /** Metadata that was stored with the data */
551551+ metadata: StorageMetadata;
552552+553553+ /** Which tiers received the data */
554554+ tiersWritten: ('hot' | 'warm' | 'cold')[];
555555+}
556556+557557+/**
558558+ * Snapshot of the entire storage state.
559559+ *
560560+ * @remarks
561561+ * Used for export/import, backup, and migration scenarios.
562562+ * The snapshot includes metadata but not the actual data (data remains in tiers).
563563+ */
564564+export interface StorageSnapshot {
565565+ /** Snapshot format version (for compatibility) */
566566+ version: number;
567567+568568+ /** When this snapshot was created */
569569+ exportedAt: Date;
570570+571571+ /** All keys present in cold tier (source of truth) */
572572+ keys: string[];
573573+574574+ /** Metadata for each key */
575575+ metadata: Record<string, StorageMetadata>;
576576+577577+ /** Statistics at time of export */
578578+ stats: AllTierStats;
579579+}