+3 -1

.gitignore

reviewed

··· 5 5 test_bundles 6 6 test_bundles* 7 7 .claude 8 8 - .DS_Store 8 8 + .DS_Store 9 9 + .trae 10 10 + .github

-61

.trae/documents/Restrict Public API to BundleManager.md

reviewed

··· 1 1 - ## Goal 2 2 - Expose only the `BundleManager` API (and the minimal inputs/outputs it requires) to library users, hiding all other modules and helpers. 3 3 - 4 4 - ## Strategy 5 5 - - Turn crate root into a narrow facade that re-exports `BundleManager` and its associated types. 6 6 - - Make all non-essential modules private (`mod` or `pub(crate)`) instead of `pub mod`. 7 7 - - Keep only the re-exports required by `BundleManager` method signatures. 8 8 - - Gate CLI/server/FFI pieces behind features so they are not part of the default public surface. 9 9 - 10 10 - ## Current Surface (reference) 11 11 - - `BundleManager` is defined at `src/manager.rs:83` and implemented starting `src/manager.rs:204`. 12 12 - - Crate root re-exports a large set starting at `src/lib.rs:73`. 13 13 - - Many modules are public at `src/lib.rs:33–55`. 14 14 - 15 15 - ## Changes in `src/lib.rs` 16 16 - - Module visibility: 17 17 - - Change all `pub mod ...` to `pub(crate) mod ...` (or `mod ...`) except `manager`. 18 18 - - Examples: `bundle_format`, `cache`, `constants`, `did_index`, `format`, `handle_resolver`, `index`, `iterators`, `mempool`, `operations`, `options`, `plc_client`, `processor`, `remote`, `resolver`, `runtime`, `sync`, `verification`, `server` become crate-private. 19 19 - - Re-export trimming: 20 20 - - Remove broad re-exports at `src/lib.rs:57–93`. 21 21 - - Keep only items needed by `BundleManager` signatures: 22 22 - - Constructors/options: `ManagerOptions`, `IntoManagerOptions`. 23 23 - - Query/export: `QuerySpec`, `QueryMode`, `ExportSpec`, `ExportIterator`, `QueryIterator`, `RangeIterator`. 24 24 - - Loading: `LoadOptions`, `LoadResult`. 25 25 - - Operations: `Operation`, `OperationRequest`, `OperationWithLocation`, `OperationResult`. 26 26 - - DID: `ResolveResult`, `DIDIndexStats`, `DIDLookupStats`, `DIDLookupTimings`, `DIDOperationsResult` (if present; otherwise keep the return struct used in `get_did_operations()` at `src/manager.rs:510–586`). 27 27 - - Verification: `VerifySpec`, `VerifyResult`, `ChainVerifySpec`, `ChainVerifyResult`. 28 28 - - Info/Stats: `InfoFlags`, `BundleInfo`, `ManagerStats`, `SizeInfo`. 29 29 - - Rollback: `RollbackSpec`, `RollbackPlan`, `RollbackResult`. 30 30 - - Warm-up: `WarmUpSpec`, `WarmUpStrategy`. 31 31 - - Re-export from `manager` when possible, otherwise re-export specific types from their modules without exposing the whole module. 32 32 - - Optional prelude: 33 33 - - Add `pub mod prelude` (facade-only) that re-exports the curated set for `use plcbundle::prelude::*;` consumers. 34 34 - 35 35 - ## Changes in internal modules 36 36 - - Where an internal type is only used by `BundleManager`, make it `pub(crate)` and reference via `crate::module::Type` from `manager`. 37 37 - - Avoid leaking helpers (functions and structs) by keeping them `pub(crate)` or private. 38 38 - - If a `BundleManager` method requires a type from another module, prefer re-exporting that type at the crate root rather than making the module public. 39 39 - 40 40 - ## Feature gating 41 41 - - Hide CLI/server by default: 42 42 - - Ensure `server` remains behind `#[cfg(feature = "server")]` and is not re-exported. 43 43 - - Keep `ffi` off by default; expose only via `feature = "ffi"` for `cdylib` consumers. 44 44 - - Confirm `[lib] crate-type = ["cdylib", "rlib"]` remains, but do not publicly expose `ffi` unless the feature is enabled. 45 45 - 46 46 - ## Docs and examples 47 47 - - Update crate-level docs to show only `use plcbundle::{BundleManager, ManagerOptions, QuerySpec, BundleRange, QueryMode};` (already present in `src/lib.rs:11–25`). 48 48 - - Ensure rustdoc for internal modules is hidden or marked as crate-private. 49 49 - - Optionally add `#![deny(missing_docs)]` and document the curated public types. 50 50 - 51 51 - ## Validation 52 52 - - Build and run rustdoc to verify only the curated items appear in the public API. 53 53 - - Compile with `--all-features` to ensure feature-gated code compiles. 54 54 - - Run existing tests; add a test that `use plcbundle::*;` only imports the intended items. 55 55 - 56 56 - ## Result 57 57 - Library users see a minimal surface: 58 58 - - `use plcbundle::{BundleManager, ManagerOptions, QuerySpec, BundleRange, QueryMode, ...}` 59 59 - - Everything else is crate-private, accessible only via `BundleManager` methods. 60 60 - 61 61 - Confirm, and I will implement these visibility and re-export adjustments in `src/lib.rs` and tighten module visibilities accordingly.

+19 -13

src/bundle_format.rs

reviewed

··· 310 310 }) 311 311 } 312 312 313 313 - 314 313 /// Serialize operations to JSONL (uncompressed) 315 314 /// 316 315 /// CRITICAL: This function implements the V1 specification requirement (docs/specification.md § 4.2) ··· 349 348 Ok(format!("{:x}", hasher.finalize())) 350 349 } 351 350 352 352 - 353 353 - 354 351 /// Create bundle metadata structure 355 352 #[allow(clippy::too_many_arguments)] 356 353 pub fn create_bundle_metadata( ··· 359 356 content_hash: &str, 360 357 parent_hash: Option<&str>, 361 358 uncompressed_size: Option<u64>, 362 362 - compressed_size: Option<u64>, 359 359 + compressed_size: Option<u64>, 363 360 operation_count: usize, 364 361 did_count: usize, 365 362 start_time: &str, ··· 375 372 content_hash: content_hash.to_string(), 376 373 parent_hash: parent_hash.map(|s| s.to_string()), 377 374 uncompressed_size, 378 378 - compressed_size, 375 375 + compressed_size, 379 376 operation_count, 380 380 - did_count, 377 377 + did_count, 381 378 start_time: start_time.to_string(), 382 379 end_time: end_time.to_string(), 383 380 created_at: chrono::Utc::now().to_rfc3339(), ··· 462 459 let mut operations = Vec::new(); 463 460 for i in 0..250 { 464 461 // Multiple frames (250 ops = 3 frames with FRAME_SIZE=100) 465 465 - let operation_json = format!( 466 466 - r#"{{"type":"create","data":"test data {}"}}"#, 467 467 - i 468 468 - ); 462 462 + let operation_json = format!(r#"{{"type":"create","data":"test data {}"}}"#, i); 469 463 let operation_value: Value = from_str(&operation_json).unwrap(); 470 464 let extra_value: Value = from_str("{}").unwrap_or_else(|_| Value::new()); 471 465 operations.push(Operation { ··· 599 593 600 594 // Write with invalid magic (not in skippable range) 601 595 buffer.write_all(&0x12345678u32.to_le_bytes()).unwrap(); 602 602 - buffer.write_all(&(data.len() as u32).to_le_bytes()).unwrap(); 596 596 + buffer 597 597 + .write_all(&(data.len() as u32).to_le_bytes()) 598 598 + .unwrap(); 603 599 buffer.write_all(data).unwrap(); 604 600 605 601 let mut cursor = std::io::Cursor::new(&buffer); 606 602 let result = read_skippable_frame(&mut cursor); 607 603 assert!(result.is_err()); 608 608 - assert!(result.unwrap_err().to_string().contains("Not a skippable frame")); 604 604 + assert!( 605 605 + result 606 606 + .unwrap_err() 607 607 + .to_string() 608 608 + .contains("Not a skippable frame") 609 609 + ); 609 610 } 610 611 611 612 #[test] ··· 682 683 683 684 let mut buffer = Vec::new(); 684 685 // Write with wrong magic 685 685 - write_skippable_frame(&mut buffer, 0x184D2A51, &sonic_rs::to_vec(&metadata).unwrap()).unwrap(); 686 686 + write_skippable_frame( 687 687 + &mut buffer, 688 688 + 0x184D2A51, 689 689 + &sonic_rs::to_vec(&metadata).unwrap(), 690 690 + ) 691 691 + .unwrap(); 686 692 687 693 let mut cursor = std::io::Cursor::new(&buffer); 688 694 let result = read_metadata_frame(&mut cursor);

+16 -11

src/cache.rs

reviewed

··· 23 23 24 24 pub fn insert(&self, bundle: u32, ops: Vec<Operation>) { 25 25 let mut cache = self.cache.write().unwrap(); 26 26 - if cache.len() >= self.capacity && let Some(&key) = cache.keys().next() { 26 26 + if cache.len() >= self.capacity 27 27 + && let Some(&key) = cache.keys().next() 28 28 + { 27 29 cache.remove(&key); 28 30 } 29 31 cache.insert(bundle, ops); ··· 76 78 77 79 cache.insert(1, ops.clone()); 78 80 assert!(cache.contains(1)); 79 79 - 81 81 + 80 82 let retrieved = cache.get(1); 81 83 assert!(retrieved.is_some()); 82 84 assert_eq!(retrieved.unwrap().len(), 2); ··· 86 88 fn test_cache_contains() { 87 89 let cache = BundleCache::new(10); 88 90 assert!(!cache.contains(1)); 89 89 - 91 91 + 90 92 cache.insert(1, vec![create_test_operation("did:plc:test1")]); 91 93 assert!(cache.contains(1)); 92 94 assert!(!cache.contains(2)); ··· 97 99 let cache = BundleCache::new(10); 98 100 cache.insert(1, vec![create_test_operation("did:plc:test1")]); 99 101 assert!(cache.contains(1)); 100 100 - 102 102 + 101 103 cache.remove(1); 102 104 assert!(!cache.contains(1)); 103 105 } ··· 109 111 cache.insert(2, vec![create_test_operation("did:plc:test2")]); 110 112 assert!(cache.contains(1)); 111 113 assert!(cache.contains(2)); 112 112 - 114 114 + 113 115 cache.clear(); 114 116 assert!(!cache.contains(1)); 115 117 assert!(!cache.contains(2)); ··· 118 120 #[test] 119 121 fn test_cache_capacity_eviction() { 120 122 let cache = BundleCache::new(2); 121 121 - 123 123 + 122 124 // Fill cache to capacity 123 125 cache.insert(1, vec![create_test_operation("did:plc:test1")]); 124 126 cache.insert(2, vec![create_test_operation("did:plc:test2")]); 125 127 assert!(cache.contains(1)); 126 128 assert!(cache.contains(2)); 127 127 - 129 129 + 128 130 // Adding third should evict one (HashMap iteration order is not guaranteed) 129 131 cache.insert(3, vec![create_test_operation("did:plc:test3")]); 130 132 // One of the first two should be evicted, and 3 should be present ··· 140 142 #[test] 141 143 fn test_cache_multiple_bundles() { 142 144 let cache = BundleCache::new(10); 143 143 - 145 145 + 144 146 for i in 1..=5 { 145 145 - cache.insert(i, vec![create_test_operation(&format!("did:plc:test{}", i))]); 147 147 + cache.insert( 148 148 + i, 149 149 + vec![create_test_operation(&format!("did:plc:test{}", i))], 150 150 + ); 146 151 } 147 147 - 152 152 + 148 153 for i in 1..=5 { 149 154 assert!(cache.contains(i)); 150 155 let ops = cache.get(i).unwrap(); ··· 157 162 fn test_cache_empty_operations() { 158 163 let cache = BundleCache::new(10); 159 164 cache.insert(1, vec![]); 160 160 - 165 165 + 161 166 let ops = cache.get(1); 162 167 assert!(ops.is_some()); 163 168 assert_eq!(ops.unwrap().len(), 0);

+14 -8

src/cli/cmd_bench.rs

reviewed

··· 403 403 let mut bundle_op_counts = Vec::with_capacity(bundles.len()); 404 404 for &bundle_num in &bundles { 405 405 if let Ok(bundle) = manager.load_bundle(bundle_num, LoadOptions::default()) 406 406 - && !bundle.operations.is_empty() { 407 407 - bundle_op_counts.push((bundle_num, bundle.operations.len())); 408 408 - } 406 406 + && !bundle.operations.is_empty() 407 407 + { 408 408 + bundle_op_counts.push((bundle_num, bundle.operations.len())); 409 409 + } 409 410 } 410 411 411 412 if bundle_op_counts.is_empty() { ··· 421 422 422 423 // Benchmark - random bundle and random position each iteration 423 424 let mut timings = Vec::with_capacity(iterations); 424 424 - 425 425 + 425 426 let pb = if interactive { 426 427 Some(ProgressBar::new(iterations)) 427 428 } else { ··· 481 482 // Ensure DID index is loaded (sample_random_dids already does this, but be explicit) 482 483 // The did_index will be loaded by sample_random_dids above 483 484 let did_index = manager.get_did_index(); 484 484 - 485 485 + 485 486 // Ensure it's actually loaded (in case sample_random_dids didn't load it) 486 487 { 487 488 let guard = did_index.read().unwrap(); ··· 502 503 503 504 // Benchmark - different DID each iteration 504 505 let mut timings = Vec::with_capacity(iterations); 505 505 - 506 506 + 506 507 let pb = if interactive { 507 508 Some(ProgressBar::new(iterations)) 508 509 } else { ··· 516 517 517 518 let did = &dids[i % dids.len()]; 518 519 let start = Instant::now(); 519 519 - let _ = did_index.read().unwrap().as_ref().unwrap().get_did_locations(did)?; 520 520 + let _ = did_index 521 521 + .read() 522 522 + .unwrap() 523 523 + .as_ref() 524 524 + .unwrap() 525 525 + .get_did_locations(did)?; 520 526 timings.push(start.elapsed().as_secs_f64() * 1000.0); 521 527 } 522 528 ··· 554 560 // Benchmark - different DID each iteration 555 561 manager.clear_caches(); 556 562 let mut timings = Vec::with_capacity(iterations); 557 557 - 563 563 + 558 564 let pb = if interactive { 559 565 Some(ProgressBar::new(iterations)) 560 566 } else {

+20 -10

src/cli/cmd_clean.rs

reviewed

··· 82 82 83 83 // Show errors if any 84 84 if let Some(errors) = &result.errors 85 85 - && !errors.is_empty() { 86 86 - eprintln!("\n⚠ Warning: Some errors occurred during cleanup:"); 87 87 - for error in errors { 88 88 - eprintln!(" - {}", error); 89 89 - } 85 85 + && !errors.is_empty() 86 86 + { 87 87 + eprintln!("\n⚠ Warning: Some errors occurred during cleanup:"); 88 88 + for error in errors { 89 89 + eprintln!(" - {}", error); 90 90 } 91 91 + } 91 92 92 93 Ok(()) 93 94 } ··· 120 121 if !root_files.is_empty() { 121 122 println!(" Repository root:"); 122 123 for file in &root_files { 123 123 - let rel_path = file.path.strip_prefix(dir) 124 124 + let rel_path = file 125 125 + .path 126 126 + .strip_prefix(dir) 124 127 .unwrap_or(&file.path) 125 128 .to_string_lossy(); 126 129 println!(" • {} ({})", rel_path, utils::format_bytes(file.size)); ··· 132 135 if !config_files.is_empty() { 133 136 println!(" DID index directory:"); 134 137 for file in &config_files { 135 135 - let rel_path = file.path.strip_prefix(dir) 138 138 + let rel_path = file 139 139 + .path 140 140 + .strip_prefix(dir) 136 141 .unwrap_or(&file.path) 137 142 .to_string_lossy(); 138 143 println!(" • {} ({})", rel_path, utils::format_bytes(file.size)); ··· 144 149 if !shard_files.is_empty() { 145 150 println!(" Shards directory:"); 146 151 for file in &shard_files { 147 147 - let rel_path = file.path.strip_prefix(dir) 152 152 + let rel_path = file 153 153 + .path 154 154 + .strip_prefix(dir) 148 155 .unwrap_or(&file.path) 149 156 .to_string_lossy(); 150 157 println!(" • {} ({})", rel_path, utils::format_bytes(file.size)); ··· 152 159 println!(); 153 160 } 154 161 155 155 - println!(" Total: {} file(s), {}", preview.files.len(), utils::format_bytes(preview.total_size)); 162 162 + println!( 163 163 + " Total: {} file(s), {}", 164 164 + preview.files.len(), 165 165 + utils::format_bytes(preview.total_size) 166 166 + ); 156 167 println!(); 157 168 158 169 Ok(()) ··· 168 179 let response = response.trim().to_lowercase(); 169 180 Ok(response == "y" || response == "yes") 170 181 } 171 171 -

+24 -15

src/cli/cmd_clone.rs

reviewed

··· 1 1 use anyhow::{Context, Result}; 2 2 use clap::{Args, ValueHint}; 3 3 - use plcbundle::{constants, remote::RemoteClient, BundleManager}; 3 3 + use plcbundle::{BundleManager, constants, remote::RemoteClient}; 4 4 use std::path::PathBuf; 5 5 use std::sync::Arc; 6 6 ··· 51 51 52 52 pub fn run(cmd: CloneCommand) -> Result<()> { 53 53 // Create tokio runtime for async operations 54 54 - tokio::runtime::Runtime::new()?.block_on(async { 55 55 - run_async(cmd).await 56 56 - }) 54 54 + tokio::runtime::Runtime::new()?.block_on(async { run_async(cmd).await }) 57 55 } 58 56 59 57 async fn run_async(cmd: CloneCommand) -> Result<()> { ··· 115 113 116 114 // Create target directory if it doesn't exist 117 115 if !target_dir.exists() { 118 118 - std::fs::create_dir_all(&target_dir) 119 119 - .context("Failed to create target directory")?; 116 116 + std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?; 120 117 } 121 118 122 119 // Check if target directory is empty or if resuming ··· 181 178 if let Some(free_space) = super::utils::get_free_disk_space(&target_dir) { 182 179 // Add 10% buffer for safety (filesystem overhead, temporary files, etc.) 183 180 let required_space = total_bytes + (total_bytes / 10); 184 184 - 181 181 + 185 182 if free_space < required_space { 186 183 let free_display = plcbundle::format::format_bytes(free_space); 187 184 let required_display = plcbundle::format::format_bytes(required_space); 188 185 let shortfall = required_space - free_space; 189 186 let shortfall_display = plcbundle::format::format_bytes(shortfall); 190 190 - 187 187 + 191 188 eprintln!("⚠️ Warning: Insufficient disk space"); 192 189 eprintln!(" Required: {}", required_display); 193 190 eprintln!(" Available: {}", free_display); 194 191 eprintln!(" Shortfall: {}", shortfall_display); 195 192 eprintln!(); 196 196 - 193 193 + 197 194 // Prompt user to continue 198 195 use dialoguer::Confirm; 199 196 let proceed = Confirm::new() ··· 201 198 .default(false) 202 199 .interact() 203 200 .context("Failed to read user input")?; 204 204 - 201 201 + 205 202 if !proceed { 206 203 anyhow::bail!("Clone cancelled by user"); 207 204 } 208 208 - 205 205 + 209 206 println!(); 210 207 } 211 208 } ··· 214 211 println!(); 215 212 216 213 // Create progress bar with byte tracking 217 217 - let progress = Arc::new(super::progress::ProgressBar::with_bytes(bundles_count, total_bytes)); 214 214 + let progress = Arc::new(super::progress::ProgressBar::with_bytes( 215 215 + bundles_count, 216 216 + total_bytes, 217 217 + )); 218 218 219 219 // Clone using BundleManager API with progress callback 220 220 let progress_clone = Arc::clone(&progress); ··· 233 233 println!(); 234 234 235 235 if failed_count > 0 { 236 236 - eprintln!("✗ Clone incomplete: {} succeeded, {} failed", downloaded_count, failed_count); 236 236 + eprintln!( 237 237 + "✗ Clone incomplete: {} succeeded, {} failed", 238 238 + downloaded_count, failed_count 239 239 + ); 237 240 eprintln!(" Use --resume to retry failed downloads"); 238 241 anyhow::bail!("Clone failed"); 239 242 } ··· 244 247 println!(); 245 248 println!("Next steps:"); 246 249 println!(" cd {}", display_path(&target_dir).display()); 247 247 - println!(" {} status # Check repository status", constants::BINARY_NAME); 248 248 - println!(" {} sync # Sync to latest", constants::BINARY_NAME); 250 250 + println!( 251 251 + " {} status # Check repository status", 252 252 + constants::BINARY_NAME 253 253 + ); 254 254 + println!( 255 255 + " {} sync # Sync to latest", 256 256 + constants::BINARY_NAME 257 257 + ); 249 258 println!(" {} server --sync # Run server", constants::BINARY_NAME); 250 259 251 260 Ok(())

+40 -32

src/cli/cmd_compare.rs

reviewed

··· 1 1 // Compare command - compare repositories 2 2 use anyhow::{Context, Result, bail}; 3 3 use clap::{Args, ValueHint}; 4 4 - use plcbundle::{BundleManager, constants, remote}; 5 4 use plcbundle::constants::bundle_position_to_global; 5 5 + use plcbundle::{BundleManager, constants, remote}; 6 6 use sonic_rs::JsonValueTrait; 7 7 use std::collections::HashMap; 8 8 use std::path::{Path, PathBuf}; ··· 73 73 let last_bundle = manager.get_last_bundle(); 74 74 let bundle_nums = super::utils::parse_bundle_spec(Some(bundles_str), last_bundle)?; 75 75 if bundle_nums.len() != 1 { 76 76 - anyhow::bail!("--bundles must specify a single bundle number for comparison (e.g., \"23\")"); 76 76 + anyhow::bail!( 77 77 + "--bundles must specify a single bundle number for comparison (e.g., \"23\")" 78 78 + ); 77 79 } 78 80 let bundle_num = bundle_nums[0]; 79 81 rt.block_on(diff_specific_bundle( ··· 126 128 eprintln!("📥 Loading target index..."); 127 129 let target_index = if target.starts_with("http://") || target.starts_with("https://") { 128 130 let client = remote::RemoteClient::new(target)?; 129 129 - client.fetch_index().await 131 131 + client 132 132 + .fetch_index() 133 133 + .await 130 134 .context("Failed to load target index")? 131 135 } else { 132 132 - remote::load_local_index(target) 133 133 - .context("Failed to load target index")? 136 136 + remote::load_local_index(target).context("Failed to load target index")? 134 137 }; 135 138 136 139 // Check origins - CRITICAL: must match for valid comparison ··· 204 207 eprintln!("📥 Loading remote index..."); 205 208 let remote_index = if target.starts_with("http://") || target.starts_with("https://") { 206 209 let client = remote::RemoteClient::new(target)?; 207 207 - client.fetch_index().await 210 210 + client 211 211 + .fetch_index() 212 212 + .await 208 213 .context("Failed to load remote index")? 209 214 } else { 210 210 - remote::load_local_index(target) 211 211 - .context("Failed to load remote index")? 215 215 + remote::load_local_index(target).context("Failed to load remote index")? 212 216 }; 213 217 let target_origin = &remote_index.origin; 214 218 ··· 256 260 eprintln!("📦 Loading remote bundle {}...\n", bundle_num); 257 261 let remote_ops = if target.starts_with("http://") || target.starts_with("https://") { 258 262 let client = remote::RemoteClient::new(target)?; 259 259 - client.fetch_bundle_operations(bundle_num) 263 263 + client 264 264 + .fetch_bundle_operations(bundle_num) 260 265 .await 261 266 .context("Failed to load remote bundle")? 262 267 } else { ··· 270 275 }; 271 276 272 277 // Try to load bundle from target directory 273 273 - let target_manager = super::utils::create_manager(target_dir.to_path_buf(), false, false, false)?; 278 278 + let target_manager = 279 279 + super::utils::create_manager(target_dir.to_path_buf(), false, false, false)?; 274 280 let target_result = target_manager 275 281 .load_bundle(bundle_num, plcbundle::LoadOptions::default()) 276 282 .context("Failed to load bundle from target directory")?; ··· 612 618 } else { 613 619 // Just missing/extra bundles - not critical 614 620 eprintln!("ℹ️ Indexes differ (missing or extra bundles, but hashes match)"); 615 615 - eprintln!( 616 616 - " This is normal when comparing repositories at different sync states." 617 617 - ); 621 621 + eprintln!(" This is normal when comparing repositories at different sync states."); 618 622 } 619 623 } 620 624 } ··· 1113 1117 1114 1118 // Add hints for exploring missing operations 1115 1119 if let Some((first_cid, first_pos)) = missing_in_local.first() 1116 1116 - && target.starts_with("http") { 1117 1117 - let base_url = if let Some(stripped) = target.strip_suffix('/') { 1118 1118 - stripped 1119 1119 - } else { 1120 1120 - target 1121 1121 - }; 1122 1122 - let global_pos = bundle_position_to_global(bundle_num, *first_pos); 1123 1123 - eprintln!(" 💡 To explore missing operations:"); 1124 1124 - eprintln!( 1125 1125 - " • Global position: {} (bundle {} position {})", 1126 1126 - global_pos, bundle_num, first_pos 1127 1127 - ); 1128 1128 - eprintln!( 1129 1129 - " • View in remote: curl '{}/op/{}' | grep '{}' | jq .", 1130 1130 - base_url, global_pos, first_cid 1131 1131 - ); 1120 1120 + && target.starts_with("http") 1121 1121 + { 1122 1122 + let base_url = if let Some(stripped) = target.strip_suffix('/') { 1123 1123 + stripped 1124 1124 + } else { 1125 1125 + target 1126 1126 + }; 1127 1127 + let global_pos = bundle_position_to_global(bundle_num, *first_pos); 1128 1128 + eprintln!(" 💡 To explore missing operations:"); 1129 1129 + eprintln!( 1130 1130 + " • Global position: {} (bundle {} position {})", 1131 1131 + global_pos, bundle_num, first_pos 1132 1132 + ); 1133 1133 + eprintln!( 1134 1134 + " • View in remote: curl '{}/op/{}' | grep '{}' | jq .", 1135 1135 + base_url, global_pos, first_cid 1136 1136 + ); 1132 1137 } 1133 1138 eprintln!(); 1134 1139 } ··· 1162 1167 1163 1168 // Add hints for exploring missing operations 1164 1169 if let Some((_first_cid, first_pos)) = missing_in_remote.first() { 1165 1165 - let global_pos = 1166 1166 - bundle_position_to_global(bundle_num, *first_pos); 1170 1170 + let global_pos = bundle_position_to_global(bundle_num, *first_pos); 1167 1171 eprintln!(" 💡 To explore missing operations:"); 1168 1172 eprintln!( 1169 1173 " • Global position: {} (bundle {} position {})", ··· 1211 1215 sample_size.min(local_ops.len()) 1212 1216 ); 1213 1217 eprintln!("────────────────────────────────"); 1214 1214 - for (i, op) in local_ops.iter().enumerate().take(sample_size.min(local_ops.len())) { 1218 1218 + for (i, op) in local_ops 1219 1219 + .iter() 1220 1220 + .enumerate() 1221 1221 + .take(sample_size.min(local_ops.len())) 1222 1222 + { 1215 1223 let remote_match = if let Some(ref cid) = op.cid { 1216 1224 if let Some(&remote_pos) = remote_cids.get(cid) { 1217 1225 if remote_pos == i {

+33 -15

src/cli/cmd_completions.rs

reviewed

··· 1 1 // Completions command - generate shell completion scripts 2 2 use anyhow::Result; 3 3 use clap::{Args, CommandFactory, ValueEnum}; 4 4 - use clap_complete::{generate, Shell}; 4 4 + use clap_complete::{Shell, generate}; 5 5 use plcbundle::constants; 6 6 use std::io; 7 7 ··· 84 84 let shell: Shell = shell_arg.into(); 85 85 let mut app = super::Cli::command(); 86 86 let bin_name = app.get_name().to_string(); 87 87 - 87 87 + 88 88 generate(shell, &mut app, bin_name, &mut io::stdout()); 89 89 } else { 90 90 // Show instructions 91 91 show_instructions(); 92 92 } 93 93 - 93 93 + 94 94 Ok(()) 95 95 } 96 96 97 97 fn show_instructions() { 98 98 let bin_name = constants::BINARY_NAME; 99 99 - 99 99 + 100 100 println!("Shell Completion Setup Instructions"); 101 101 println!("════════════════════════════════════\n"); 102 102 - println!("Generate completion scripts for your shell to enable tab completion for {}.\n", bin_name); 102 102 + println!( 103 103 + "Generate completion scripts for your shell to enable tab completion for {}.\n", 104 104 + bin_name 105 105 + ); 103 106 println!("Usage:"); 104 107 println!(" {} completions <SHELL>\n", bin_name); 105 108 println!("Supported shells:"); ··· 108 111 println!(" fish - Fish completion script"); 109 112 println!(" powershell - PowerShell completion script\n"); 110 113 println!("Examples:\n"); 111 111 - 114 114 + 112 115 // Bash 113 116 println!("Bash:"); 114 117 println!(" # Generate completion script"); 115 115 - println!(" {} completions bash > ~/.bash_completion.d/{}", bin_name, bin_name); 118 118 + println!( 119 119 + " {} completions bash > ~/.bash_completion.d/{}", 120 120 + bin_name, bin_name 121 121 + ); 116 122 println!(" # Add to ~/.bashrc:"); 117 117 - println!(" echo 'source ~/.bash_completion.d/{}' >> ~/.bashrc", bin_name); 123 123 + println!( 124 124 + " echo 'source ~/.bash_completion.d/{}' >> ~/.bashrc", 125 125 + bin_name 126 126 + ); 118 127 println!(" source ~/.bashrc\n"); 119 119 - 128 128 + 120 129 // Zsh 121 130 println!("Zsh:"); 122 131 println!(" # Generate completion script"); 123 132 println!(" mkdir -p ~/.zsh/completions"); 124 124 - println!(" {} completions zsh > ~/.zsh/completions/_{}", bin_name, bin_name); 133 133 + println!( 134 134 + " {} completions zsh > ~/.zsh/completions/_{}", 135 135 + bin_name, bin_name 136 136 + ); 125 137 println!(" # Add to ~/.zshrc:"); 126 138 println!(" echo 'fpath=(~/.zsh/completions $fpath)' >> ~/.zshrc"); 127 139 println!(" echo 'autoload -U compinit && compinit' >> ~/.zshrc"); 128 140 println!(" source ~/.zshrc\n"); 129 129 - 141 141 + 130 142 // Fish 131 143 println!("Fish:"); 132 144 println!(" # Generate completion script"); 133 145 println!(" mkdir -p ~/.config/fish/completions"); 134 134 - println!(" {} completions fish > ~/.config/fish/completions/{}.fish", bin_name, bin_name); 146 146 + println!( 147 147 + " {} completions fish > ~/.config/fish/completions/{}.fish", 148 148 + bin_name, bin_name 149 149 + ); 135 150 println!(" # Fish will automatically load completions from this directory\n"); 136 136 - 151 151 + 137 152 // PowerShell 138 153 println!("PowerShell:"); 139 154 println!(" # Generate completion script"); ··· 143 158 println!(" # Or add to your PowerShell profile:"); 144 159 println!(" . $PROFILE # if it exists, or create it"); 145 160 println!(" echo '. ./{}.ps1' >> $PROFILE\n", bin_name); 146 146 - 161 161 + 147 162 println!("After installation, restart your shell or source the configuration file."); 148 148 - println!("Then you can use tab completion for {} commands and arguments!", bin_name); 163 163 + println!( 164 164 + "Then you can use tab completion for {} commands and arguments!", 165 165 + bin_name 166 166 + ); 149 167 }

+184 -138

src/cli/cmd_did.rs

reviewed

··· 210 210 raw, 211 211 compare, 212 212 } => { 213 213 - cmd_did_resolve(dir, did, handle_resolver, global_verbose, query, raw, compare)?; 213 213 + cmd_did_resolve( 214 214 + dir, 215 215 + did, 216 216 + handle_resolver, 217 217 + global_verbose, 218 218 + query, 219 219 + raw, 220 220 + compare, 221 221 + )?; 214 222 } 215 215 - DIDCommands::Log { did, json, format, no_header, separator, reverse } => { 216 216 - cmd_did_lookup(dir, did, global_verbose, json, format, no_header, separator, reverse)?; 223 223 + DIDCommands::Log { 224 224 + did, 225 225 + json, 226 226 + format, 227 227 + no_header, 228 228 + separator, 229 229 + reverse, 230 230 + } => { 231 231 + cmd_did_lookup( 232 232 + dir, 233 233 + did, 234 234 + global_verbose, 235 235 + json, 236 236 + format, 237 237 + no_header, 238 238 + separator, 239 239 + reverse, 240 240 + )?; 217 241 } 218 242 DIDCommands::Batch { 219 243 action, ··· 285 309 286 310 // If compare is enabled, fetch remote document and compare 287 311 if let Some(maybe_url) = compare { 288 288 - use tokio::runtime::Runtime; 289 312 use crate::plc_client::PLCClient; 290 313 use std::time::Instant; 314 314 + use tokio::runtime::Runtime; 291 315 292 316 // Use provided URL, or use repository origin, or fall back to default 293 317 let plc_url = match maybe_url { ··· 296 320 log::info!("Using provided PLC directory URL: {}", url); 297 321 } 298 322 url 299 299 - }, 323 323 + } 300 324 _ => { 301 325 // Get origin from local repository index 302 326 let local_index = manager.get_index(); 303 327 let origin = &local_index.origin; 304 304 - 328 328 + 305 329 // If origin is "local" or empty, use default PLC directory 306 330 if origin == "local" || origin.is_empty() { 307 331 if verbose { 308 308 - log::info!("Origin is '{}', using default PLC directory: {}", origin, constants::DEFAULT_PLC_DIRECTORY_URL); 332 332 + log::info!( 333 333 + "Origin is '{}', using default PLC directory: {}", 334 334 + origin, 335 335 + constants::DEFAULT_PLC_DIRECTORY_URL 336 336 + ); 309 337 } 310 338 constants::DEFAULT_PLC_DIRECTORY_URL.to_string() 311 339 } else { ··· 318 346 }; 319 347 320 348 eprintln!("🔍 Comparing with remote PLC directory..."); 321 321 - 349 349 + 322 350 if verbose { 323 351 log::info!("Target PLC directory: {}", plc_url); 324 352 log::info!("DID to fetch: {}", did); 325 353 } 326 326 - 354 354 + 327 355 let fetch_start = Instant::now(); 328 356 let rt = Runtime::new().context("Failed to create tokio runtime")?; 329 357 use plcbundle::resolver::DIDDocument; 330 330 - let (remote_doc, remote_json_raw): (DIDDocument, String) = rt.block_on(async { 331 331 - let client = PLCClient::new(&plc_url) 332 332 - .context("Failed to create PLC client")?; 333 333 - if verbose { 334 334 - log::info!("Created PLC client, fetching DID document..."); 335 335 - } 336 336 - // Fetch both the parsed document and raw JSON for accurate comparison 337 337 - let raw_json = client.fetch_did_document_raw(&did).await?; 338 338 - let parsed_doc = client.fetch_did_document(&did).await?; 339 339 - Ok::<(DIDDocument, String), anyhow::Error>((parsed_doc, raw_json)) 340 340 - }) 341 341 - .context("Failed to fetch DID document from remote PLC directory")?; 358 358 + let (remote_doc, remote_json_raw): (DIDDocument, String) = rt 359 359 + .block_on(async { 360 360 + let client = PLCClient::new(&plc_url).context("Failed to create PLC client")?; 361 361 + if verbose { 362 362 + log::info!("Created PLC client, fetching DID document..."); 363 363 + } 364 364 + // Fetch both the parsed document and raw JSON for accurate comparison 365 365 + let raw_json = client.fetch_did_document_raw(&did).await?; 366 366 + let parsed_doc = client.fetch_did_document(&did).await?; 367 367 + Ok::<(DIDDocument, String), anyhow::Error>((parsed_doc, raw_json)) 368 368 + }) 369 369 + .context("Failed to fetch DID document from remote PLC directory")?; 342 370 let fetch_duration = fetch_start.elapsed(); 343 371 344 372 if verbose { ··· 349 377 eprintln!(); 350 378 351 379 // Compare documents and return early (don't show full document) 352 352 - compare_did_documents(&result.document, &remote_doc, &remote_json_raw, &did, &plc_url, fetch_duration)?; 380 380 + compare_did_documents( 381 381 + &result.document, 382 382 + &remote_doc, 383 383 + &remote_json_raw, 384 384 + &did, 385 385 + &plc_url, 386 386 + fetch_duration, 387 387 + )?; 353 388 return Ok(()); 354 389 } 355 390 ··· 479 514 if result.bundle_number == 0 { 480 515 // Operation from mempool 481 516 let index = manager.get_index(); 482 482 - let global_pos = plcbundle::constants::mempool_position_to_global(index.last_bundle, result.position); 517 517 + let global_pos = plcbundle::constants::mempool_position_to_global( 518 518 + index.last_bundle, 519 519 + result.position, 520 520 + ); 483 521 log::info!( 484 522 "Source: mempool position {} (global: {})\n", 485 523 result.position, ··· 487 525 ); 488 526 } else { 489 527 // Operation from bundle 490 490 - let global_pos = plcbundle::constants::bundle_position_to_global(result.bundle_number, result.position); 528 528 + let global_pos = plcbundle::constants::bundle_position_to_global( 529 529 + result.bundle_number, 530 530 + result.position, 531 531 + ); 491 532 log::info!( 492 533 "Source: bundle {}, position {} (global: {})\n", 493 534 result.bundle_number, ··· 503 544 // If query is provided, apply JMESPath query 504 545 let output_json = if let Some(query_expr) = query { 505 546 // Compile JMESPath expression 506 506 - let expr = jmespath::compile(&query_expr) 507 507 - .map_err(|e| anyhow::anyhow!("Failed to compile JMESPath query '{}': {}", query_expr, e))?; 547 547 + let expr = jmespath::compile(&query_expr).map_err(|e| { 548 548 + anyhow::anyhow!("Failed to compile JMESPath query '{}': {}", query_expr, e) 549 549 + })?; 508 550 509 551 // Execute query 510 552 let data = jmespath::Variable::from_json(&document_json) 511 553 .map_err(|e| anyhow::anyhow!("Failed to parse DID document JSON: {}", e))?; 512 554 513 513 - let result = expr.search(&data) 555 555 + let result = expr 556 556 + .search(&data) 514 557 .map_err(|e| anyhow::anyhow!("JMESPath query failed: {}", e))?; 515 558 516 559 if result.is_null() { ··· 578 621 579 622 eprintln!("📊 Document Comparison"); 580 623 eprintln!("═══════════════════════"); 581 581 - 624 624 + 582 625 // Construct the full URL that was fetched 583 626 let full_url = format!("{}/{}", remote_url.trim_end_matches('/'), _did); 584 627 eprintln!(" Remote URL: {}", full_url); ··· 655 698 656 699 eprintln!(); 657 700 use super::utils::colors; 658 658 - eprintln!("Legend: {}Local{} {}Remote{}", colors::RED, colors::RESET, colors::GREEN, colors::RESET); 701 701 + eprintln!( 702 702 + "Legend: {}Local{} {}Remote{}", 703 703 + colors::RED, 704 704 + colors::RESET, 705 705 + colors::GREEN, 706 706 + colors::RESET 707 707 + ); 659 708 eprintln!(); 660 709 } 661 710 ··· 674 723 // Parse JSON using sonic_rs (faster than serde_json) 675 724 let value: sonic_rs::Value = sonic_rs::from_str(json) 676 725 .map_err(|e| anyhow::anyhow!("Failed to parse JSON for normalization: {}", e))?; 677 677 - 726 726 + 678 727 // Re-serialize with consistent formatting (compact, no whitespace) 679 728 // This normalizes key ordering and whitespace differences 680 729 let normalized = sonic_rs::to_string(&value) 681 730 .map_err(|e| anyhow::anyhow!("Failed to serialize normalized JSON: {}", e))?; 682 682 - 731 731 + 683 732 Ok(normalized) 684 733 } 685 734 ··· 687 736 fn json_to_pretty(json: &str) -> Result<String> { 688 737 let value: sonic_rs::Value = sonic_rs::from_str(json) 689 738 .map_err(|e| anyhow::anyhow!("Failed to parse JSON for pretty printing: {}", e))?; 690 690 - 739 739 + 691 740 let pretty = sonic_rs::to_string_pretty(&value) 692 741 .map_err(|e| anyhow::anyhow!("Failed to serialize pretty JSON: {}", e))?; 693 693 - 742 742 + 694 743 Ok(pretty) 695 744 } 696 745 ··· 1036 1085 1037 1086 // Calculate column widths for alignment 1038 1087 let mut column_widths = vec![0; fields.len()]; 1039 1039 - 1088 1088 + 1040 1089 // Check header widths 1041 1090 for (i, field) in fields.iter().enumerate() { 1042 1091 let header = get_lookup_field_header(field); 1043 1092 column_widths[i] = column_widths[i].max(header.len()); 1044 1093 } 1045 1045 - 1094 1094 + 1046 1095 // Check data widths 1047 1096 for owl in bundled_ops.iter() { 1048 1097 for (i, field) in fields.iter().enumerate() { ··· 1050 1099 column_widths[i] = column_widths[i].max(value.len()); 1051 1100 } 1052 1101 } 1053 1053 - 1102 1102 + 1054 1103 for op in mempool_ops.iter() { 1055 1104 for (i, field) in fields.iter().enumerate() { 1056 1105 let value = get_lookup_field_value_mempool(op, field); ··· 1060 1109 1061 1110 // Print column header (unless disabled) 1062 1111 if !no_header { 1063 1063 - let headers: Vec<String> = fields.iter() 1112 1112 + let headers: Vec<String> = fields 1113 1113 + .iter() 1064 1114 .enumerate() 1065 1115 .map(|(i, f)| { 1066 1116 let header = get_lookup_field_header(f); ··· 1083 1133 }; 1084 1134 1085 1135 for owl in bundled_iter { 1086 1086 - let values: Vec<String> = fields.iter() 1136 1136 + let values: Vec<String> = fields 1137 1137 + .iter() 1087 1138 .enumerate() 1088 1139 .map(|(i, f)| { 1089 1140 let value = get_lookup_field_value(owl, None, f); ··· 1096 1147 }) 1097 1148 .collect(); 1098 1149 println!("{}", values.join(separator)); 1099 1099 - 1150 1150 + 1100 1151 if verbose && !owl.nullified { 1101 1152 let op_val = &owl.operation.operation; 1102 1153 if let Some(op_type) = op_val.get("type").and_then(|v| v.as_str()) { ··· 1105 1156 if let Some(handle) = op_val.get("handle").and_then(|v| v.as_str()) { 1106 1157 eprintln!(" handle: {}", handle); 1107 1158 } else if let Some(aka) = op_val.get("alsoKnownAs").and_then(|v| v.as_array()) 1108 1108 - && let Some(aka_str) = aka.first().and_then(|v| v.as_str()) { 1109 1109 - let handle = aka_str.strip_prefix("at://").unwrap_or(aka_str); 1110 1110 - eprintln!(" handle: {}", handle); 1159 1159 + && let Some(aka_str) = aka.first().and_then(|v| v.as_str()) 1160 1160 + { 1161 1161 + let handle = aka_str.strip_prefix("at://").unwrap_or(aka_str); 1162 1162 + eprintln!(" handle: {}", handle); 1111 1163 } 1112 1164 } 1113 1165 } ··· 1120 1172 }; 1121 1173 1122 1174 for op in mempool_iter { 1123 1123 - let values: Vec<String> = fields.iter() 1175 1175 + let values: Vec<String> = fields 1176 1176 + .iter() 1124 1177 .enumerate() 1125 1178 .map(|(i, f)| { 1126 1179 let value = get_lookup_field_value_mempool(op, f); ··· 1171 1224 "bundle" => format!("{}", owl.bundle), 1172 1225 "position" | "pos" => format!("{:04}", owl.position), 1173 1226 "global" | "global_pos" => { 1174 1174 - let global_pos = plcbundle::constants::bundle_position_to_global(owl.bundle, owl.position); 1227 1227 + let global_pos = 1228 1228 + plcbundle::constants::bundle_position_to_global(owl.bundle, owl.position); 1175 1229 format!("{}", global_pos) 1176 1176 - }, 1230 1230 + } 1177 1231 "status" => { 1178 1232 if owl.nullified { 1179 1233 "✗".to_string() ··· 1181 1235 "✓".to_string() 1182 1236 } 1183 1237 } 1184 1184 - "cid" => { 1185 1185 - owl.operation.cid.clone() 1186 1186 - .unwrap_or_default() 1187 1187 - } 1238 1238 + "cid" => owl.operation.cid.clone().unwrap_or_default(), 1188 1239 "created_at" | "created" | "date" | "time" => owl.operation.created_at.clone(), 1189 1240 "nullified" => { 1190 1241 if owl.nullified { ··· 1217 1268 "position" | "pos" => "".to_string(), 1218 1269 "global" | "global_pos" => "".to_string(), 1219 1270 "status" => "✓".to_string(), 1220 1220 - "cid" => { 1221 1221 - op.cid.clone() 1222 1222 - .unwrap_or_default() 1223 1223 - } 1271 1271 + "cid" => op.cid.clone().unwrap_or_default(), 1224 1272 "created_at" | "created" | "date" | "time" => op.created_at.clone(), 1225 1273 "nullified" => "false".to_string(), 1226 1274 "date_short" => { ··· 1240 1288 _ => String::new(), 1241 1289 } 1242 1290 } 1243 1243 - 1244 1291 1245 1292 // DID BATCH - Process multiple DIDs (TODO) 1246 1293 ··· 1394 1441 if verbose { 1395 1442 println!("📋 Operations:"); 1396 1443 for (i, entry) in audit_log.iter().enumerate() { 1397 1397 - let status = if entry.nullified { "❌ NULLIFIED" } else { "✅" }; 1398 1398 - println!( 1399 1399 - " [{}] {} {} - {}", 1400 1400 - i, status, entry.cid, entry.created_at 1401 1401 - ); 1444 1444 + let status = if entry.nullified { 1445 1445 + "❌ NULLIFIED" 1446 1446 + } else { 1447 1447 + "✅" 1448 1448 + }; 1449 1449 + println!(" [{}] {} {} - {}", i, status, entry.cid, entry.created_at); 1402 1450 if entry.operation.is_genesis() { 1403 1451 println!(" Type: Genesis (creates the DID)"); 1404 1452 } else { ··· 1576 1624 for (j, key) in rotation_keys.iter().enumerate() { 1577 1625 println!(" [{}] {}", j, key); 1578 1626 } 1579 1579 - println!(" ⚠️ Genesis signature cannot be verified (bootstrapping trust)"); 1627 1627 + println!( 1628 1628 + " ⚠️ Genesis signature cannot be verified (bootstrapping trust)" 1629 1629 + ); 1580 1630 } 1581 1631 } 1582 1632 continue; ··· 1591 1641 // Validate signature using current rotation keys 1592 1642 if !current_rotation_keys.is_empty() { 1593 1643 if verbose { 1594 1594 - println!(" Available rotation keys: {}", current_rotation_keys.len()); 1644 1644 + println!( 1645 1645 + " Available rotation keys: {}", 1646 1646 + current_rotation_keys.len() 1647 1647 + ); 1595 1648 for (j, key) in current_rotation_keys.iter().enumerate() { 1596 1649 println!(" [{}] {}", j, key); 1597 1650 } ··· 1626 1679 // Final attempt with all keys (for comprehensive error) 1627 1680 if let Err(e) = entry.operation.verify(&verifying_keys) { 1628 1681 eprintln!(); 1629 1629 - eprintln!( 1630 1630 - "❌ Validation failed: Invalid signature at operation {}", 1631 1631 - i 1632 1632 - ); 1682 1682 + eprintln!("❌ Validation failed: Invalid signature at operation {}", i); 1633 1683 eprintln!(" Error: {}", e); 1634 1684 eprintln!(" CID: {}", entry.cid); 1635 1685 eprintln!( ··· 1655 1705 1656 1706 // Update rotation keys if this operation changes them 1657 1707 if let Some(new_rotation_keys) = entry.operation.rotation_keys() 1658 1658 - && new_rotation_keys != current_rotation_keys { 1659 1659 - if verbose { 1660 1660 - println!(" 🔄 Rotation keys updated by this operation"); 1661 1661 - println!(" Old keys: {}", current_rotation_keys.len()); 1662 1662 - println!(" New keys: {}", new_rotation_keys.len()); 1663 1663 - for (j, key) in new_rotation_keys.iter().enumerate() { 1664 1664 - println!(" [{}] {}", j, key); 1665 1665 - } 1708 1708 + && new_rotation_keys != current_rotation_keys 1709 1709 + { 1710 1710 + if verbose { 1711 1711 + println!(" 🔄 Rotation keys updated by this operation"); 1712 1712 + println!(" Old keys: {}", current_rotation_keys.len()); 1713 1713 + println!(" New keys: {}", new_rotation_keys.len()); 1714 1714 + for (j, key) in new_rotation_keys.iter().enumerate() { 1715 1715 + println!(" [{}] {}", j, key); 1666 1716 } 1667 1667 - current_rotation_keys = new_rotation_keys.to_vec(); 1717 1717 + } 1718 1718 + current_rotation_keys = new_rotation_keys.to_vec(); 1668 1719 } 1669 1720 } 1670 1721 ··· 1720 1771 let mut prev_to_indices: HashMap<String, Vec<usize>> = HashMap::new(); 1721 1772 for (i, entry) in audit_log.iter().enumerate() { 1722 1773 if let Some(prev) = entry.operation.prev() { 1723 1723 - prev_to_indices 1724 1724 - .entry(prev.to_string()) 1725 1725 - .or_default() 1726 1726 - .push(i); 1774 1774 + prev_to_indices.entry(prev.to_string()).or_default().push(i); 1727 1775 } 1728 1776 } 1729 1777 ··· 1817 1865 .map_err(|e| anyhow::anyhow!("Failed to parse operation: {}", e))?; 1818 1866 1819 1867 // Get CID from bundle operation (should always be present) 1820 1820 - let cid = owl 1821 1821 - .operation 1822 1822 - .cid 1823 1823 - .clone() 1824 1824 - .unwrap_or_else(|| { 1825 1825 - // Fallback: this shouldn't happen in real data, but provide a placeholder 1826 1826 - format!("bundle_{}_pos_{}", owl.bundle, owl.position) 1827 1827 - }); 1868 1868 + let cid = owl.operation.cid.clone().unwrap_or_else(|| { 1869 1869 + // Fallback: this shouldn't happen in real data, but provide a placeholder 1870 1870 + format!("bundle_{}_pos_{}", owl.bundle, owl.position) 1871 1871 + }); 1828 1872 1829 1873 audit_log.push(AuditLogEntry { 1830 1874 did: owl.operation.did.clone(), ··· 1839 1883 } 1840 1884 1841 1885 /// Visualize forks in the audit log 1842 1842 - fn visualize_forks( 1843 1843 - audit_log: &[AuditLogEntry], 1844 1844 - did_str: &str, 1845 1845 - verbose: bool, 1846 1846 - ) -> Result<()> { 1886 1886 + fn visualize_forks(audit_log: &[AuditLogEntry], did_str: &str, verbose: bool) -> Result<()> { 1847 1887 println!("🔍 Analyzing forks in: {}", did_str); 1848 1888 println!(" Source: local bundles"); 1849 1889 println!(); ··· 2013 2053 } 2014 2054 2015 2055 /// Find which rotation key signed an operation 2016 2016 - fn find_signing_key(operation: &Operation, rotation_keys: &[String]) -> (Option<usize>, Option<String>) { 2056 2056 + fn find_signing_key( 2057 2057 + operation: &Operation, 2058 2058 + rotation_keys: &[String], 2059 2059 + ) -> (Option<usize>, Option<String>) { 2017 2060 for (index, key_did) in rotation_keys.iter().enumerate() { 2018 2061 if let Ok(verifying_key) = VerifyingKey::from_did_key(key_did) 2019 2019 - && operation.verify(&[verifying_key]).is_ok() { 2020 2020 - return (Some(index), Some(key_did.clone())); 2062 2062 + && operation.verify(&[verifying_key]).is_ok() 2063 2063 + { 2064 2064 + return (Some(index), Some(key_did.clone())); 2021 2065 } 2022 2066 } 2023 2067 (None, None) ··· 2072 2116 2073 2117 // Check if this operation is part of a fork 2074 2118 if let Some(_prev_cid) = prev 2075 2075 - && let Some(fork) = fork_map.get(&entry.cid) { 2076 2076 - // This is a fork point 2077 2077 - if !processed_forks.contains(&fork.prev_cid) { 2078 2078 - processed_forks.insert(fork.prev_cid.clone()); 2119 2119 + && let Some(fork) = fork_map.get(&entry.cid) 2120 2120 + { 2121 2121 + // This is a fork point 2122 2122 + if !processed_forks.contains(&fork.prev_cid) { 2123 2123 + processed_forks.insert(fork.prev_cid.clone()); 2079 2124 2080 2080 - println!("Fork at operation referencing {}", truncate_cid(&fork.prev_cid)); 2125 2125 + println!( 2126 2126 + "Fork at operation referencing {}", 2127 2127 + truncate_cid(&fork.prev_cid) 2128 2128 + ); 2081 2129 2082 2082 - for (j, fork_op) in fork.operations.iter().enumerate() { 2083 2083 - let symbol = if fork_op.is_winner { "✓" } else { "✗" }; 2084 2084 - let color = if fork_op.is_winner { "🟢" } else { "🔴" }; 2085 2085 - let prefix = if j == fork.operations.len() - 1 { 2086 2086 - "└─" 2087 2087 - } else { 2088 2088 - "├─" 2089 2089 - }; 2130 2130 + for (j, fork_op) in fork.operations.iter().enumerate() { 2131 2131 + let symbol = if fork_op.is_winner { "✓" } else { "✗" }; 2132 2132 + let color = if fork_op.is_winner { "🟢" } else { "🔴" }; 2133 2133 + let prefix = if j == fork.operations.len() - 1 { 2134 2134 + "└─" 2135 2135 + } else { 2136 2136 + "├─" 2137 2137 + }; 2090 2138 2091 2091 - println!( 2092 2092 - " {} {} {} CID: {}", 2093 2093 - prefix, 2094 2094 - color, 2095 2095 - symbol, 2096 2096 - truncate_cid(&fork_op.cid) 2097 2097 - ); 2139 2139 + println!( 2140 2140 + " {} {} {} CID: {}", 2141 2141 + prefix, 2142 2142 + color, 2143 2143 + symbol, 2144 2144 + truncate_cid(&fork_op.cid) 2145 2145 + ); 2098 2146 2099 2099 - if let Some(key_idx) = fork_op.signing_key_index { 2100 2100 - println!(" │ Signed by: rotation_key[{}]", key_idx); 2101 2101 - if verbose 2102 2102 - && let Some(key) = &fork_op.signing_key { 2103 2103 - println!(" │ Key: {}", truncate_cid(key)); 2104 2104 - } 2147 2147 + if let Some(key_idx) = fork_op.signing_key_index { 2148 2148 + println!(" │ Signed by: rotation_key[{}]", key_idx); 2149 2149 + if verbose && let Some(key) = &fork_op.signing_key { 2150 2150 + println!(" │ Key: {}", truncate_cid(key)); 2105 2151 } 2152 2152 + } 2106 2153 2107 2107 - println!( 2108 2108 - " │ Timestamp: {}", 2109 2109 - fork_op.timestamp.format("%Y-%m-%d %H:%M:%S UTC") 2110 2110 - ); 2154 2154 + println!( 2155 2155 + " │ Timestamp: {}", 2156 2156 + fork_op.timestamp.format("%Y-%m-%d %H:%M:%S UTC") 2157 2157 + ); 2111 2158 2112 2112 - if !fork_op.is_winner { 2113 2113 - if let Some(reason) = &fork_op.rejection_reason { 2114 2114 - println!(" │ Reason: {}", reason); 2115 2115 - } 2116 2116 - } else { 2117 2117 - println!(" │ Status: CANONICAL (winner)"); 2159 2159 + if !fork_op.is_winner { 2160 2160 + if let Some(reason) = &fork_op.rejection_reason { 2161 2161 + println!(" │ Reason: {}", reason); 2118 2162 } 2163 2163 + } else { 2164 2164 + println!(" │ Status: CANONICAL (winner)"); 2165 2165 + } 2119 2166 2120 2120 - if j < fork.operations.len() - 1 { 2121 2121 - println!(" │"); 2122 2122 - } 2167 2167 + if j < fork.operations.len() - 1 { 2168 2168 + println!(" │"); 2123 2169 } 2124 2124 - println!(); 2125 2170 } 2126 2126 - continue; 2171 2171 + println!(); 2172 2172 + } 2173 2173 + continue; 2127 2174 } 2128 2175 2129 2176 // Regular operation (not part of a fork) ··· 2131 2178 println!("🌱 Genesis"); 2132 2179 println!(" CID: {}", truncate_cid(&entry.cid)); 2133 2180 println!(" Timestamp: {}", entry.created_at); 2134 2134 - if verbose 2135 2135 - && let Operation::PlcOperation { rotation_keys, .. } = &entry.operation { 2136 2136 - println!(" Rotation keys: {}", rotation_keys.len()); 2181 2181 + if verbose && let Operation::PlcOperation { rotation_keys, .. } = &entry.operation { 2182 2182 + println!(" Rotation keys: {}", rotation_keys.len()); 2137 2183 } 2138 2184 println!(); 2139 2185 }

+70 -44

src/cli/cmd_export.rs

reviewed

··· 10 10 11 11 use super::progress::ProgressBar; 12 12 use super::utils; 13 13 - use plcbundle::constants::{global_to_bundle_position, total_operations_from_bundles, bundle_position_to_global}; 13 13 + use plcbundle::constants::{ 14 14 + bundle_position_to_global, global_to_bundle_position, total_operations_from_bundles, 15 15 + }; 14 16 use plcbundle::format::format_std_duration_auto; 15 17 16 18 #[derive(Args)] ··· 74 76 pub reverse: bool, 75 77 } 76 78 77 77 - 78 79 pub fn run(cmd: ExportCommand, dir: PathBuf, quiet: bool, verbose: bool) -> Result<()> { 79 80 let output = cmd.output; 80 81 let count = cmd.count; ··· 132 133 eprintln!("Exporting {} operations", utils::format_number(op_count)); 133 134 } else { 134 135 let bundle_range_str = format_bundle_range(&bundle_numbers); 135 135 - eprintln!("Exporting bundles: {} ({})", bundle_range_str, bundle_numbers.len()); 136 136 + eprintln!( 137 137 + "Exporting bundles: {} ({})", 138 138 + bundle_range_str, 139 139 + bundle_numbers.len() 140 140 + ); 136 141 } 137 142 } 138 143 ··· 156 161 let total_compressed_size: u64 = bundle_numbers 157 162 .iter() 158 163 .filter_map(|bundle_num| { 159 159 - index.get_bundle(*bundle_num).map(|meta| meta.compressed_size) 164 164 + index 165 165 + .get_bundle(*bundle_num) 166 166 + .map(|meta| meta.compressed_size) 160 167 }) 161 168 .sum(); 162 162 - 169 169 + 163 170 // Create progress bar tracking bundles processed 164 171 // Skip progress bar if quiet mode or if only one bundle needs to be loaded 165 172 let pb = if quiet || bundle_numbers.len() == 1 { 166 173 None 167 174 } else { 168 168 - Some(ProgressBar::with_bytes(bundle_numbers.len(), total_uncompressed_size)) 175 175 + Some(ProgressBar::with_bytes( 176 176 + bundle_numbers.len(), 177 177 + total_uncompressed_size, 178 178 + )) 169 179 }; 170 180 171 181 let start = Instant::now(); ··· 179 189 for bundle_num in bundle_numbers { 180 190 // Check count limit 181 191 if let Some(limit) = count 182 182 - && exported_count >= limit { 183 183 - break; 192 192 + && exported_count >= limit 193 193 + { 194 194 + break; 184 195 } 185 196 186 197 // Use BundleManager API to get decompressed stream ··· 226 237 227 238 // Check count limit 228 239 if let Some(limit) = count 229 229 - && exported_count >= limit { 230 230 - break; 240 240 + && exported_count >= limit 241 241 + { 242 242 + break; 231 243 } 232 244 233 245 output_buffer.push_str(&line); ··· 246 258 247 259 // Progress update (operations count in message, but bundles in progress bar) 248 260 if let Some(ref pb) = pb 249 249 - && (exported_count % BATCH_SIZE == 0 || exported_count == 1) { 250 250 - let bytes = bytes_written.lock().unwrap(); 251 251 - let total_bytes = *bytes + output_buffer.len() as u64; 252 252 - drop(bytes); 253 253 - pb.set_with_bytes(bundles_processed, total_bytes); 254 254 - pb.set_message(format!("{} ops", utils::format_number(exported_count as u64))); 261 261 + && (exported_count % BATCH_SIZE == 0 || exported_count == 1) 262 262 + { 263 263 + let bytes = bytes_written.lock().unwrap(); 264 264 + let total_bytes = *bytes + output_buffer.len() as u64; 265 265 + drop(bytes); 266 266 + pb.set_with_bytes(bundles_processed, total_bytes); 267 267 + pb.set_message(format!( 268 268 + "{} ops", 269 269 + utils::format_number(exported_count as u64) 270 270 + )); 255 271 } 256 272 } 257 257 - 273 273 + 258 274 // Update progress after processing each bundle 259 275 bundles_processed += 1; 260 276 if let Some(ref pb) = pb { ··· 262 278 let total_bytes = *bytes + output_buffer.len() as u64; 263 279 drop(bytes); 264 280 pb.set_with_bytes(bundles_processed, total_bytes); 265 265 - pb.set_message(format!("{} ops", utils::format_number(exported_count as u64))); 281 281 + pb.set_message(format!( 282 282 + "{} ops", 283 283 + utils::format_number(exported_count as u64) 284 284 + )); 266 285 } 267 286 } 268 287 ··· 287 306 if !quiet { 288 307 let elapsed = start.elapsed(); 289 308 let elapsed_secs = elapsed.as_secs_f64(); 290 290 - 309 309 + 291 310 // Format duration with auto-scaling using utility function 292 311 let duration_str = format_std_duration_auto(elapsed); 293 293 - 312 312 + 294 313 let mut parts = Vec::new(); 295 295 - parts.push(format!("Exported: {} ops", utils::format_number(exported_count as u64))); 314 314 + parts.push(format!( 315 315 + "Exported: {} ops", 316 316 + utils::format_number(exported_count as u64) 317 317 + )); 296 318 parts.push(format!("in {}", duration_str)); 297 297 - 319 319 + 298 320 if elapsed_secs > 0.0 { 299 321 // Calculate throughputs 300 300 - let uncompressed_throughput_mb = (total_uncompressed_size as f64 / elapsed_secs) / (1024.0 * 1024.0); 301 301 - let compressed_throughput_mb = (total_compressed_size as f64 / elapsed_secs) / (1024.0 * 1024.0); 302 302 - 303 303 - parts.push(format!("({:.1} MB/s uncompressed, {:.1} MB/s compressed)", 304 304 - uncompressed_throughput_mb, 305 305 - compressed_throughput_mb 322 322 + let uncompressed_throughput_mb = 323 323 + (total_uncompressed_size as f64 / elapsed_secs) / (1024.0 * 1024.0); 324 324 + let compressed_throughput_mb = 325 325 + (total_compressed_size as f64 / elapsed_secs) / (1024.0 * 1024.0); 326 326 + 327 327 + parts.push(format!( 328 328 + "({:.1} MB/s uncompressed, {:.1} MB/s compressed)", 329 329 + uncompressed_throughput_mb, compressed_throughput_mb 306 330 )); 307 331 } 308 308 - 332 332 + 309 333 eprintln!("{}", parts.join(" ")); 310 334 } 311 335 ··· 355 379 } 356 380 357 381 let result = ranges.join(", "); 358 358 - 382 382 + 359 383 // If the formatted string is too long, fall back to min-max 360 384 if result.len() > 200 { 361 385 let min = bundles.iter().min().copied().unwrap_or(0); ··· 380 404 /// This is much more efficient for large ranges like "0-10000000" 381 405 fn parse(spec: &str, max_operation: u64) -> Result<Self> { 382 406 use anyhow::Context; 383 383 - 407 407 + 384 408 if max_operation == 0 { 385 409 anyhow::bail!("No operations available"); 386 410 } 387 411 388 412 let mut ranges = Vec::new(); 389 413 let mut individual = HashSet::new(); 390 390 - 414 414 + 391 415 for part in spec.split(',') { 392 416 let part = part.trim(); 393 417 if part.is_empty() { ··· 435 459 individual.insert(num); 436 460 } 437 461 } 438 438 - 462 462 + 439 463 // Sort ranges for efficient lookup 440 464 ranges.sort_unstable(); 441 441 - 465 465 + 442 466 Ok(OperationFilter { ranges, individual }) 443 467 } 444 444 - 468 468 + 445 469 /// Check if a global operation position is included in the filter 446 470 fn contains(&self, global_pos: u64) -> bool { 447 471 // Check individual operations first (O(1)) 448 472 if self.individual.contains(&global_pos) { 449 473 return true; 450 474 } 451 451 - 475 475 + 452 476 // Check ranges (O(log n) with binary search, but we use linear for simplicity) 453 477 // For small number of ranges, linear is fine 454 478 for &(start, end) in &self.ranges { ··· 456 480 return true; 457 481 } 458 482 } 459 459 - 483 483 + 460 484 false 461 485 } 462 462 - 486 486 + 463 487 /// Get bundle numbers that contain operations in this filter 464 488 /// This calculates bundles from range bounds without materializing all operations 465 489 fn get_bundle_numbers(&self, max_bundle: u32) -> Vec<u32> { 466 490 let mut bundle_set = HashSet::new(); 467 467 - 491 491 + 468 492 // Process ranges 469 493 for &(start, end) in &self.ranges { 470 494 // Convert start and end to bundle numbers 471 495 let (start_bundle, _) = global_to_bundle_position(start); 472 496 let (end_bundle, _) = global_to_bundle_position(end); 473 473 - 497 497 + 474 498 // Add all bundles in the range 475 499 for bundle in start_bundle..=end_bundle.min(max_bundle) { 476 500 bundle_set.insert(bundle); 477 501 } 478 502 } 479 479 - 503 503 + 480 504 // Process individual operations 481 505 for &op in &self.individual { 482 506 let (bundle, _) = global_to_bundle_position(op); ··· 484 508 bundle_set.insert(bundle); 485 509 } 486 510 } 487 487 - 511 511 + 488 512 let mut bundles: Vec<u32> = bundle_set.into_iter().collect(); 489 513 bundles.sort_unstable(); 490 514 bundles 491 515 } 492 492 - 516 516 + 493 517 /// Estimate the number of operations (for display purposes) 494 518 fn estimated_count(&self) -> u64 { 495 495 - let range_count: u64 = self.ranges.iter() 519 519 + let range_count: u64 = self 520 520 + .ranges 521 521 + .iter() 496 522 .map(|&(start, end)| end.saturating_sub(start).saturating_add(1)) 497 523 .sum(); 498 524 let individual_count = self.individual.len() as u64;

+289 -148

src/cli/cmd_index.rs

reviewed

··· 150 150 151 151 pub fn run(cmd: IndexCommand, dir: PathBuf, global_verbose: bool) -> Result<()> { 152 152 match cmd.command { 153 153 - IndexCommands::Build { force, threads, flush_interval } => { 153 153 + IndexCommands::Build { 154 154 + force, 155 155 + threads, 156 156 + flush_interval, 157 157 + } => { 154 158 cmd_index_build(dir, force, threads, flush_interval)?; 155 159 } 156 156 - IndexCommands::Repair { threads, flush_interval } => { 160 160 + IndexCommands::Repair { 161 161 + threads, 162 162 + flush_interval, 163 163 + } => { 157 164 cmd_index_repair(dir, threads, flush_interval)?; 158 165 } 159 166 IndexCommands::Status { json } => { 160 167 cmd_index_stats(dir, json)?; 161 168 } 162 162 - IndexCommands::Verify { flush_interval, full } => { 169 169 + IndexCommands::Verify { 170 170 + flush_interval, 171 171 + full, 172 172 + } => { 163 173 cmd_index_verify(dir, global_verbose, flush_interval, full)?; 164 174 } 165 175 IndexCommands::Debug { shard, did, json } => { ··· 184 194 /// Parse shard number from string (supports hex 0xac or decimal) 185 195 fn parse_shard(s: &str) -> Result<u8> { 186 196 if s.starts_with("0x") || s.starts_with("0X") { 187 187 - u8::from_str_radix(&s[2..], 16) 188 188 - .map_err(|_| anyhow::anyhow!("Invalid shard number: {}", s)) 197 197 + u8::from_str_radix(&s[2..], 16).map_err(|_| anyhow::anyhow!("Invalid shard number: {}", s)) 189 198 } else { 190 199 s.parse::<u8>() 191 200 .map_err(|_| anyhow::anyhow!("Invalid shard number: {}", s)) 192 201 } 193 202 } 194 203 195 195 - pub fn cmd_index_build(dir: PathBuf, force: bool, threads: usize, flush_interval: u32) -> Result<()> { 204 204 + pub fn cmd_index_build( 205 205 + dir: PathBuf, 206 206 + force: bool, 207 207 + threads: usize, 208 208 + flush_interval: u32, 209 209 + ) -> Result<()> { 196 210 // Set thread pool size 197 211 let num_threads = super::utils::get_worker_threads(threads, 4); 198 212 rayon::ThreadPoolBuilder::new() ··· 212 226 .get("exists") 213 227 .and_then(|v| v.as_bool()) 214 228 .unwrap_or(false); 215 215 - 229 229 + 216 230 if index_exists && total_dids > 0 && !force { 217 231 let last_bundle = manager.get_last_bundle(); 218 232 let index_last_bundle = did_index ··· 223 237 .get("shards_with_data") 224 238 .and_then(|v| v.as_i64()) 225 239 .unwrap_or(0); 226 226 - 240 240 + 227 241 eprintln!("\n✅ DID Index Already Built"); 228 228 - eprintln!(" Total DIDs: {}", utils::format_number(total_dids as u64)); 242 242 + eprintln!( 243 243 + " Total DIDs: {}", 244 244 + utils::format_number(total_dids as u64) 245 245 + ); 229 246 eprintln!(" Last bundle: {} / {}", index_last_bundle, last_bundle); 230 247 eprintln!(" Shards: {} with data", shards_with_data); 231 231 - eprintln!(" Location: {}/{}/", utils::display_path(&dir).display(), constants::DID_INDEX_DIR); 232 232 - 248 248 + eprintln!( 249 249 + " Location: {}/{}/", 250 250 + utils::display_path(&dir).display(), 251 251 + constants::DID_INDEX_DIR 252 252 + ); 253 253 + 233 254 if index_last_bundle < last_bundle { 234 255 let missing = last_bundle - index_last_bundle; 235 256 eprintln!(); ··· 239 260 eprintln!(); 240 261 eprintln!(" ✓ Index is up-to-date"); 241 262 } 242 242 - 263 263 + 243 264 eprintln!(); 244 265 eprintln!(" 💡 Use --force to rebuild from scratch"); 245 266 return Ok(()); ··· 260 281 use std::sync::Arc; 261 282 let last_bundle = manager.get_last_bundle() as u32; 262 283 let progress = TwoStageProgress::new(last_bundle, total_bytes); 263 263 - 284 284 + 264 285 // Set up cleanup guard to ensure temp files are deleted on CTRL+C or panic 265 286 // Use Arc to share manager for cleanup 266 287 let manager_arc = Arc::new(manager); 267 288 let manager_for_cleanup = manager_arc.clone(); 268 268 - 289 289 + 269 290 struct IndexBuildCleanup { 270 291 manager: Arc<BundleManager>, 271 292 } 272 272 - 293 293 + 273 294 impl Drop for IndexBuildCleanup { 274 295 fn drop(&mut self) { 275 296 // Cleanup temp files on drop (CTRL+C, panic, or normal exit) 276 297 let did_index = self.manager.get_did_index(); 277 298 if let Some(idx) = did_index.read().unwrap().as_ref() 278 278 - && let Err(e) = idx.cleanup_temp_files() { 279 279 - log::warn!("[Index Build] Failed to cleanup temp files: {}", e); 299 299 + && let Err(e) = idx.cleanup_temp_files() 300 300 + { 301 301 + log::warn!("[Index Build] Failed to cleanup temp files: {}", e); 280 302 } 281 303 } 282 304 } 283 283 - 305 305 + 284 306 let _cleanup_guard = IndexBuildCleanup { 285 307 manager: manager_for_cleanup.clone(), 286 308 }; ··· 290 312 flush_interval, 291 313 Some(progress.callback_for_build_did_index()), 292 314 Some(num_threads), 293 293 - Some(progress.interrupted()) 315 315 + Some(progress.interrupted()), 294 316 ); 295 317 296 318 // Handle build result - ensure cleanup happens on error ··· 302 324 // Error occurred - explicitly cleanup temp files before returning 303 325 let did_index = manager_arc.get_did_index(); 304 326 if let Some(idx) = did_index.read().unwrap().as_ref() 305 305 - && let Err(cleanup_err) = idx.cleanup_temp_files() { 306 306 - log::warn!("[Index Build] Failed to cleanup temp files after error: {}", cleanup_err); 327 327 + && let Err(cleanup_err) = idx.cleanup_temp_files() 328 328 + { 329 329 + log::warn!( 330 330 + "[Index Build] Failed to cleanup temp files after error: {}", 331 331 + cleanup_err 332 332 + ); 307 333 } 308 334 return Err(e); 309 335 } ··· 361 387 // Check if repair is needed before setting up progress bar 362 388 let did_index = manager.get_did_index(); 363 389 let idx = did_index.read().unwrap(); 364 364 - let idx_ref = idx.as_ref().ok_or_else(|| anyhow::anyhow!("DID index not initialized"))?; 390 390 + let idx_ref = idx 391 391 + .as_ref() 392 392 + .ok_or_else(|| anyhow::anyhow!("DID index not initialized"))?; 365 393 let repair_info = idx_ref.get_repair_info(last_bundle)?; 366 394 let needs_work = repair_info.needs_rebuild || repair_info.needs_compact; 367 395 drop(idx); 368 368 - 396 396 + 369 397 // Use BundleManager API for repair 370 398 let start = Instant::now(); 371 371 - 399 399 + 372 400 // Set up progress callback only if work is needed 373 401 use super::progress::TwoStageProgress; 374 402 let progress = if needs_work { ··· 379 407 } else { 380 408 None 381 409 }; 382 382 - 410 410 + 383 411 let repair_result = manager.repair_did_index( 384 412 num_threads, 385 413 flush_interval, 386 414 progress.as_ref().map(|p| p.callback_for_build_did_index()), 387 415 )?; 388 388 - 416 416 + 389 417 if let Some(progress) = progress { 390 418 progress.finish(); 391 419 } ··· 409 437 use super::utils::colors; 410 438 eprintln!(); 411 439 if repair_result.repaired || repair_result.compacted { 412 412 - eprintln!("{}✓{} Index repair completed successfully in {:?}", colors::GREEN, colors::RESET, elapsed); 440 440 + eprintln!( 441 441 + "{}✓{} Index repair completed successfully in {:?}", 442 442 + colors::GREEN, 443 443 + colors::RESET, 444 444 + elapsed 445 445 + ); 413 446 eprintln!(); 414 447 eprintln!("📊 Index Statistics:"); 415 415 - eprintln!(" Total DIDs: {}", utils::format_number(total_dids as u64)); 448 448 + eprintln!( 449 449 + " Total DIDs: {}", 450 450 + utils::format_number(total_dids as u64) 451 451 + ); 416 452 eprintln!(" Last bundle: {}", last_bundle); 417 453 eprintln!(" Shards: {}", shard_count); 418 454 eprintln!(" Delta segments: {}", final_delta_segments); 419 419 - 455 455 + 420 456 // Show what was fixed 421 457 eprintln!(); 422 458 eprintln!("🔧 Repairs performed:"); 423 459 if repair_result.repaired { 424 460 eprintln!(" • Processed {} bundles", repair_result.bundles_processed); 425 461 if repair_result.segments_rebuilt > 0 { 426 426 - eprintln!(" • Rebuilt {} delta segment(s)", repair_result.segments_rebuilt); 462 462 + eprintln!( 463 463 + " • Rebuilt {} delta segment(s)", 464 464 + repair_result.segments_rebuilt 465 465 + ); 427 466 } 428 467 } 429 468 if repair_result.compacted { 430 469 eprintln!(" • Compacted delta segments"); 431 470 } 432 432 - 471 471 + 433 472 // Show compaction recommendation if needed 434 473 if (50..100).contains(&final_delta_segments) { 435 474 eprintln!(); 436 436 - eprintln!("💡 Tip: Consider running '{} index compact' to optimize performance", constants::BINARY_NAME); 475 475 + eprintln!( 476 476 + "💡 Tip: Consider running '{} index compact' to optimize performance", 477 477 + constants::BINARY_NAME 478 478 + ); 437 479 } 438 480 } else { 439 439 - eprintln!("{}✓{} Index is up-to-date and optimized", colors::GREEN, colors::RESET); 481 481 + eprintln!( 482 482 + "{}✓{} Index is up-to-date and optimized", 483 483 + colors::GREEN, 484 484 + colors::RESET 485 485 + ); 440 486 eprintln!(" Last bundle: {}", index_last_bundle); 441 487 eprintln!(" Delta segments: {}", delta_segments); 442 488 } ··· 449 495 450 496 // Get raw stats from did_index 451 497 let stats_map = manager.get_did_index_stats(); 452 452 - 498 498 + 453 499 // Check if index has been built 454 500 let is_built = stats_map 455 501 .get("exists") ··· 596 642 Ok(()) 597 643 } 598 644 599 599 - 600 600 - pub fn cmd_index_verify(dir: PathBuf, verbose: bool, flush_interval: u32, full: bool) -> Result<()> { 645 645 + pub fn cmd_index_verify( 646 646 + dir: PathBuf, 647 647 + verbose: bool, 648 648 + flush_interval: u32, 649 649 + full: bool, 650 650 + ) -> Result<()> { 601 651 let manager = super::utils::create_manager(dir.clone(), false, false, false)?; 602 652 603 653 let stats_map = manager.get_did_index_stats(); ··· 631 681 let bundle_numbers: Vec<u32> = (1..=last_bundle).collect(); 632 682 let total_bytes = index.total_uncompressed_size_for_bundles(&bundle_numbers); 633 683 let progress = TwoStageProgress::new(last_bundle, total_bytes); 634 634 - 684 684 + 635 685 // Print header info like build command 636 686 eprintln!("\n📦 Building Temporary DID Index (for verification)"); 637 687 eprintln!(" Strategy: Streaming (memory-efficient)"); ··· 639 689 if flush_interval > 0 { 640 690 if flush_interval == constants::DID_INDEX_FLUSH_INTERVAL { 641 691 // Default value - show with tuning hint 642 642 - eprintln!(" Flush: Every {} bundles (tune with --flush-interval)", flush_interval); 692 692 + eprintln!( 693 693 + " Flush: Every {} bundles (tune with --flush-interval)", 694 694 + flush_interval 695 695 + ); 643 696 } else { 644 697 // Non-default value - show with tuning hint 645 645 - eprintln!(" Flush: {} bundles (you can tune with --flush-interval)", flush_interval); 698 698 + eprintln!( 699 699 + " Flush: {} bundles (you can tune with --flush-interval)", 700 700 + flush_interval 701 701 + ); 646 702 } 647 703 } else { 648 704 eprintln!(" Flush: Only at end (maximum memory usage)"); 649 705 } 650 706 eprintln!(); 651 707 eprintln!("📊 Stage 1: Processing bundles..."); 652 652 - 708 708 + 653 709 let result = manager.verify_did_index( 654 710 verbose, 655 711 flush_interval, 656 712 full, 657 713 Some(progress.callback_for_build_did_index()), 658 714 )?; 659 659 - 715 715 + 660 716 progress.finish(); 661 717 eprintln!("\n"); 662 718 result 663 719 } else { 664 720 // Standard verification: no progress bar 665 665 - manager.verify_did_index::<fn(u32, u32, u64, u64)>( 666 666 - verbose, 667 667 - flush_interval, 668 668 - full, 669 669 - None, 670 670 - )? 721 721 + manager.verify_did_index::<fn(u32, u32, u64, u64)>(verbose, flush_interval, full, None)? 671 722 }; 672 723 673 724 let errors = verify_result.errors; ··· 781 832 // Summary 782 833 if errors > 0 { 783 834 use super::utils::colors; 784 784 - eprintln!("{}✗{} Index verification failed", colors::RED, colors::RESET); 835 835 + eprintln!( 836 836 + "{}✗{} Index verification failed", 837 837 + colors::RED, 838 838 + colors::RESET 839 839 + ); 785 840 eprintln!(" Errors: {}", errors); 786 841 eprintln!(" Warnings: {}", warnings); 787 787 - 842 842 + 788 843 // Print error breakdown 789 844 if missing_base_shards > 0 { 790 845 eprintln!(" • Missing base shards: {}", missing_base_shards); ··· 792 847 if missing_delta_segments > 0 { 793 848 eprintln!(" • Missing delta segments: {}", missing_delta_segments); 794 849 } 795 795 - 850 850 + 796 851 // Print other error categories 797 852 for (category, count) in &error_categories { 798 853 eprintln!(" • {}: {}", category, count); 799 854 } 800 800 - 855 855 + 801 856 eprintln!("\n Run: {} index repair", constants::BINARY_NAME); 802 857 std::process::exit(1); 803 858 } else if warnings > 0 { 804 859 use super::utils::colors; 805 860 use super::utils::format_number; 806 806 - eprintln!("\x1b[33m⚠️{} Index verification passed with warnings", colors::RESET); 861 861 + eprintln!( 862 862 + "\x1b[33m⚠️{} Index verification passed with warnings", 863 863 + colors::RESET 864 864 + ); 807 865 eprintln!(" Warnings: {}", warnings); 808 866 eprintln!(" Total DIDs: {}", format_number(total_dids as u64)); 809 867 eprintln!(" Last bundle: {}", format_number(last_bundle as u64)); ··· 830 888 831 889 /// Get raw shard data as JSON 832 890 fn get_raw_shard_data_json(dir: &Path, shard_num: u8) -> Result<serde_json::Value> { 833 833 - use std::fs; 834 891 use plcbundle::constants; 835 892 use plcbundle::did_index::OpLocation; 836 893 use serde_json::json; 894 894 + use std::fs; 837 895 838 896 const DID_IDENTIFIER_LEN: usize = 24; 839 897 ··· 859 917 let offset_table_start = 1056; 860 918 861 919 let shard_filename = format!("{:02x}.idx", shard_num); 862 862 - 920 920 + 863 921 // Parse entries 864 922 let max_entries_to_show = 10; 865 923 let entries_to_show = entry_count.min(max_entries_to_show); ··· 891 949 current_offset += DID_IDENTIFIER_LEN; 892 950 893 951 // Read location count 894 894 - let loc_count = u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 952 952 + let loc_count = 953 953 + u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 895 954 current_offset += 2; 896 955 897 956 // Read locations ··· 934 993 } 935 994 936 995 /// Get raw delta segment data as JSON 937 937 - fn get_raw_segment_data_json(dir: &Path, shard_num: u8, file_name: &str) -> Result<serde_json::Value> { 938 938 - use std::fs; 996 996 + fn get_raw_segment_data_json( 997 997 + dir: &Path, 998 998 + shard_num: u8, 999 999 + file_name: &str, 1000 1000 + ) -> Result<serde_json::Value> { 939 1001 use plcbundle::constants; 940 1002 use plcbundle::did_index::OpLocation; 941 1003 use serde_json::json; 1004 1004 + use std::fs; 942 1005 943 1006 const DID_IDENTIFIER_LEN: usize = 24; 944 1007 ··· 995 1058 current_offset += DID_IDENTIFIER_LEN; 996 1059 997 1060 // Read location count 998 998 - let loc_count = u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 1061 1061 + let loc_count = 1062 1062 + u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 999 1063 current_offset += 2; 1000 1064 1001 1065 // Read locations ··· 1039 1103 1040 1104 /// Display raw shard data in a readable format 1041 1105 fn display_raw_shard_data(dir: &Path, shard_num: u8) -> Result<()> { 1042 1042 - use std::fs; 1043 1106 use plcbundle::constants; 1044 1107 use plcbundle::did_index::OpLocation; 1108 1108 + use std::fs; 1045 1109 1046 1110 const DID_IDENTIFIER_LEN: usize = 24; 1047 1111 ··· 1104 1168 current_offset += DID_IDENTIFIER_LEN; 1105 1169 1106 1170 // Read location count 1107 1107 - let loc_count = u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 1171 1171 + let loc_count = 1172 1172 + u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 1108 1173 current_offset += 2; 1109 1174 1110 1175 // Read locations ··· 1124 1189 } 1125 1190 1126 1191 println!(" Entry {}:", i); 1127 1127 - 1192 1192 + 1128 1193 let identifier_clean = identifier.trim_end_matches('\0'); 1129 1194 let full_did = format!("did:plc:{}", identifier_clean); 1130 1130 - println!(" Identifier: {} [did={}]", identifier_clean, full_did); 1131 1131 - 1195 1195 + println!( 1196 1196 + " Identifier: {} [did={}]", 1197 1197 + identifier_clean, full_did 1198 1198 + ); 1199 1199 + 1132 1200 print!(" Locations ({}): [ ", loc_count); 1133 1201 for (idx, loc) in locations.iter().enumerate() { 1134 1202 if idx > 0 { ··· 1143 1211 } 1144 1212 1145 1213 if entry_count > max_entries_to_show { 1146 1146 - println!("\n ... ({} more entries)", entry_count - max_entries_to_show); 1214 1214 + println!( 1215 1215 + "\n ... ({} more entries)", 1216 1216 + entry_count - max_entries_to_show 1217 1217 + ); 1147 1218 } 1148 1219 } else { 1149 1220 println!(" No entries in shard"); ··· 1154 1225 1155 1226 /// Display raw delta segment data in a readable format 1156 1227 fn display_raw_segment_data(dir: &Path, shard_num: u8, file_name: &str) -> Result<()> { 1157 1157 - use std::fs; 1158 1158 - use plcbundle::constants; 1159 1228 use crate::did_index::OpLocation; 1229 1229 + use plcbundle::constants; 1230 1230 + use std::fs; 1160 1231 1161 1232 const DID_IDENTIFIER_LEN: usize = 24; 1162 1233 ··· 1218 1289 current_offset += DID_IDENTIFIER_LEN; 1219 1290 1220 1291 // Read location count 1221 1221 - let loc_count = u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 1292 1292 + let loc_count = 1293 1293 + u16::from_le_bytes([data[current_offset], data[current_offset + 1]]) as usize; 1222 1294 current_offset += 2; 1223 1295 1224 1296 // Read locations ··· 1238 1310 } 1239 1311 1240 1312 println!(" Entry {}:", i); 1241 1241 - 1313 1313 + 1242 1314 let identifier_clean = identifier.trim_end_matches('\0'); 1243 1315 let full_did = format!("did:plc:{}", identifier_clean); 1244 1244 - println!(" Identifier: {} [did={}]", identifier_clean, full_did); 1245 1245 - 1316 1316 + println!( 1317 1317 + " Identifier: {} [did={}]", 1318 1318 + identifier_clean, full_did 1319 1319 + ); 1320 1320 + 1246 1321 print!(" Locations ({}): [ ", loc_count); 1247 1322 for (idx, loc) in locations.iter().enumerate() { 1248 1323 if idx > 0 { ··· 1257 1332 } 1258 1333 1259 1334 if entry_count > max_entries_to_show { 1260 1260 - println!("\n ... ({} more entries)", entry_count - max_entries_to_show); 1335 1335 + println!( 1336 1336 + "\n ... ({} more entries)", 1337 1337 + entry_count - max_entries_to_show 1338 1338 + ); 1261 1339 } 1262 1340 } else { 1263 1341 println!(" No entries in segment"); ··· 1284 1362 1285 1363 // Resolve handle to DID if needed 1286 1364 let (did, handle_resolve_time) = manager.resolve_handle_or_did(&input)?; 1287 1287 - 1365 1365 + 1288 1366 let is_handle = did != input; 1289 1367 if is_handle { 1290 1368 log::info!("Resolved handle: {} → {}", input, did); ··· 1297 1375 .and_then(|v| v.as_bool()) 1298 1376 .unwrap_or(false) 1299 1377 { 1300 1300 - anyhow::bail!("DID index does not exist. Run: {} index build", constants::BINARY_NAME); 1378 1378 + anyhow::bail!( 1379 1379 + "DID index does not exist. Run: {} index build", 1380 1380 + constants::BINARY_NAME 1381 1381 + ); 1301 1382 } 1302 1383 1303 1384 // Ensure the index is loaded 1304 1385 manager.ensure_did_index()?; 1305 1305 - 1386 1386 + 1306 1387 // Get DID index and lookup DID with stats 1307 1388 let did_index = manager.get_did_index(); 1308 1389 let lookup_start = Instant::now(); ··· 1392 1473 1393 1474 println!(" Locations ({}):", locations.len()); 1394 1475 println!(" ───────────────────────────────────────"); 1395 1395 - 1476 1476 + 1396 1477 // Group locations for more compact display 1397 1478 let mut current_line = String::new(); 1398 1479 for (idx, loc) in locations.iter().enumerate() { ··· 1400 1481 println!(" {}", current_line); 1401 1482 current_line.clear(); 1402 1483 } 1403 1403 - 1484 1484 + 1404 1485 if !current_line.is_empty() { 1405 1486 current_line.push_str(", "); 1406 1487 } 1407 1407 - 1488 1488 + 1408 1489 let mut loc_str = format!("{}", loc.global_position()); 1409 1490 if loc.nullified() { 1410 1491 loc_str.push_str(" (nullified)"); 1411 1492 } 1412 1493 current_line.push_str(&loc_str); 1413 1494 } 1414 1414 - 1495 1495 + 1415 1496 if !current_line.is_empty() { 1416 1497 println!(" {}", current_line); 1417 1498 } ··· 1419 1500 1420 1501 println!(" Lookup Statistics:"); 1421 1502 println!(" ───────────────────────────────────────"); 1422 1422 - println!(" Shard size: {} bytes", utils::format_bytes(shard_stats.shard_size as u64)); 1423 1423 - println!(" Total entries: {}", utils::format_number(shard_stats.total_entries as u64)); 1424 1424 - println!(" Binary search attempts: {}", shard_stats.binary_search_attempts); 1425 1425 - println!(" Prefix narrowed to: {}", shard_stats.prefix_narrowed_to); 1426 1426 - println!(" Locations found: {}", shard_stats.locations_found); 1503 1503 + println!( 1504 1504 + " Shard size: {} bytes", 1505 1505 + utils::format_bytes(shard_stats.shard_size as u64) 1506 1506 + ); 1507 1507 + println!( 1508 1508 + " Total entries: {}", 1509 1509 + utils::format_number(shard_stats.total_entries as u64) 1510 1510 + ); 1511 1511 + println!( 1512 1512 + " Binary search attempts: {}", 1513 1513 + shard_stats.binary_search_attempts 1514 1514 + ); 1515 1515 + println!( 1516 1516 + " Prefix narrowed to: {}", 1517 1517 + shard_stats.prefix_narrowed_to 1518 1518 + ); 1519 1519 + println!( 1520 1520 + " Locations found: {}", 1521 1521 + shard_stats.locations_found 1522 1522 + ); 1427 1523 println!(); 1428 1524 1429 1525 println!(" Timing:"); 1430 1526 println!(" ───────────────────────────────────────"); 1431 1431 - println!(" Extract identifier: {:.3}ms", lookup_timings.extract_identifier.as_secs_f64() * 1000.0); 1432 1432 - println!(" Calculate shard: {:.3}ms", lookup_timings.calculate_shard.as_secs_f64() * 1000.0); 1433 1433 - println!(" Load shard: {:.3}ms ({})", 1527 1527 + println!( 1528 1528 + " Extract identifier: {:.3}ms", 1529 1529 + lookup_timings.extract_identifier.as_secs_f64() * 1000.0 1530 1530 + ); 1531 1531 + println!( 1532 1532 + " Calculate shard: {:.3}ms", 1533 1533 + lookup_timings.calculate_shard.as_secs_f64() * 1000.0 1534 1534 + ); 1535 1535 + println!( 1536 1536 + " Load shard: {:.3}ms ({})", 1434 1537 lookup_timings.load_shard.as_secs_f64() * 1000.0, 1435 1435 - if lookup_timings.cache_hit { "cache hit" } else { "cache miss" }); 1436 1436 - println!(" Search: {:.3}ms", lookup_timings.search.as_secs_f64() * 1000.0); 1538 1538 + if lookup_timings.cache_hit { 1539 1539 + "cache hit" 1540 1540 + } else { 1541 1541 + "cache miss" 1542 1542 + } 1543 1543 + ); 1544 1544 + println!( 1545 1545 + " Search: {:.3}ms", 1546 1546 + lookup_timings.search.as_secs_f64() * 1000.0 1547 1547 + ); 1437 1548 if let Some(base_time) = lookup_timings.base_search_time { 1438 1438 - println!(" Base search: {:.3}ms", base_time.as_secs_f64() * 1000.0); 1549 1549 + println!( 1550 1550 + " Base search: {:.3}ms", 1551 1551 + base_time.as_secs_f64() * 1000.0 1552 1552 + ); 1439 1553 } 1440 1554 if !lookup_timings.delta_segment_times.is_empty() { 1441 1441 - println!(" Delta segments: {} segments", lookup_timings.delta_segment_times.len()); 1555 1555 + println!( 1556 1556 + " Delta segments: {} segments", 1557 1557 + lookup_timings.delta_segment_times.len() 1558 1558 + ); 1442 1559 for (idx, (name, time)) in lookup_timings.delta_segment_times.iter().enumerate() { 1443 1443 - println!(" Segment {} ({:20}): {:.3}ms", idx + 1, name, time.as_secs_f64() * 1000.0); 1560 1560 + println!( 1561 1561 + " Segment {} ({:20}): {:.3}ms", 1562 1562 + idx + 1, 1563 1563 + name, 1564 1564 + time.as_secs_f64() * 1000.0 1565 1565 + ); 1444 1566 } 1445 1567 } 1446 1446 - println!(" Merge: {:.3}ms", lookup_timings.merge_time.as_secs_f64() * 1000.0); 1568 1568 + println!( 1569 1569 + " Merge: {:.3}ms", 1570 1570 + lookup_timings.merge_time.as_secs_f64() * 1000.0 1571 1571 + ); 1447 1572 if handle_resolve_time > 0 { 1448 1573 println!(" Handle resolve: {}ms", handle_resolve_time); 1449 1574 } 1450 1450 - println!(" Total: {:.3}ms", lookup_elapsed.as_secs_f64() * 1000.0); 1575 1575 + println!( 1576 1576 + " Total: {:.3}ms", 1577 1577 + lookup_elapsed.as_secs_f64() * 1000.0 1578 1578 + ); 1451 1579 println!(); 1452 1580 1453 1581 Ok(()) 1454 1582 } 1455 1583 1456 1456 - pub fn cmd_index_debug(dir: PathBuf, shard: Option<u8>, did: Option<String>, json: bool) -> Result<()> { 1584 1584 + pub fn cmd_index_debug( 1585 1585 + dir: PathBuf, 1586 1586 + shard: Option<u8>, 1587 1587 + did: Option<String>, 1588 1588 + json: bool, 1589 1589 + ) -> Result<()> { 1457 1590 let manager = super::utils::create_manager(dir.clone(), false, false, false)?; 1458 1591 1459 1592 let stats_map = manager.get_did_index_stats(); ··· 1474 1607 } 1475 1608 1476 1609 let did_index = manager.get_did_index(); 1477 1477 - let mut shard_details = did_index.read().unwrap().as_ref().unwrap().get_shard_details(shard)?; 1610 1610 + let mut shard_details = did_index 1611 1611 + .read() 1612 1612 + .unwrap() 1613 1613 + .as_ref() 1614 1614 + .unwrap() 1615 1615 + .get_shard_details(shard)?; 1478 1616 1479 1617 if json { 1480 1618 // Add raw data to JSON output if a specific shard is requested 1481 1619 if let Some(shard_num) = shard 1482 1482 - && let Some(detail) = shard_details.first_mut() { 1483 1483 - let base_exists = detail 1484 1484 - .get("base_exists") 1485 1485 - .and_then(|v| v.as_bool()) 1486 1486 - .unwrap_or(false); 1487 1487 - 1488 1488 - if base_exists 1489 1489 - && let Ok(raw_data) = get_raw_shard_data_json(&dir, shard_num) { 1490 1490 - detail.insert("raw_data".to_string(), raw_data); 1491 1491 - } 1492 1492 - 1493 1493 - // Add raw data for delta segments 1494 1494 - if let Some(segments) = detail.get_mut("segments").and_then(|v| v.as_array_mut()) { 1495 1495 - for seg in segments { 1496 1496 - let file_name = seg.get("file_name").and_then(|v| v.as_str()).unwrap_or(""); 1497 1497 - let exists = seg.get("exists").and_then(|v| v.as_bool()).unwrap_or(false); 1498 1498 - if exists && !file_name.is_empty() 1499 1499 - && let Ok(raw_data) = get_raw_segment_data_json(&dir, shard_num, file_name) { 1500 1500 - seg.as_object_mut().unwrap().insert("raw_data".to_string(), raw_data); 1501 1501 - } 1620 1620 + && let Some(detail) = shard_details.first_mut() 1621 1621 + { 1622 1622 + let base_exists = detail 1623 1623 + .get("base_exists") 1624 1624 + .and_then(|v| v.as_bool()) 1625 1625 + .unwrap_or(false); 1626 1626 + 1627 1627 + if base_exists && let Ok(raw_data) = get_raw_shard_data_json(&dir, shard_num) { 1628 1628 + detail.insert("raw_data".to_string(), raw_data); 1629 1629 + } 1630 1630 + 1631 1631 + // Add raw data for delta segments 1632 1632 + if let Some(segments) = detail.get_mut("segments").and_then(|v| v.as_array_mut()) { 1633 1633 + for seg in segments { 1634 1634 + let file_name = seg.get("file_name").and_then(|v| v.as_str()).unwrap_or(""); 1635 1635 + let exists = seg.get("exists").and_then(|v| v.as_bool()).unwrap_or(false); 1636 1636 + if exists 1637 1637 + && !file_name.is_empty() 1638 1638 + && let Ok(raw_data) = get_raw_segment_data_json(&dir, shard_num, file_name) 1639 1639 + { 1640 1640 + seg.as_object_mut() 1641 1641 + .unwrap() 1642 1642 + .insert("raw_data".to_string(), raw_data); 1502 1643 } 1503 1644 } 1645 1645 + } 1504 1646 } 1505 1505 - 1647 1647 + 1506 1648 let json_str = serde_json::to_string_pretty(&shard_details)?; 1507 1649 println!("{}", json_str); 1508 1650 return Ok(()); ··· 1558 1700 println!(" Next segment ID: {}", next_segment_id); 1559 1701 1560 1702 if let Some(segments) = detail.get("segments").and_then(|v| v.as_array()) 1561 1561 - && !segments.is_empty() { 1562 1562 - println!("\n Delta Segments:"); 1563 1563 - println!(" ───────────────────────────────────────"); 1564 1564 - for (idx, seg) in segments.iter().enumerate() { 1565 1565 - let file_name = 1566 1566 - seg.get("file_name").and_then(|v| v.as_str()).unwrap_or("?"); 1567 1567 - let exists = seg.get("exists").and_then(|v| v.as_bool()).unwrap_or(false); 1568 1568 - let size = seg.get("size_bytes").and_then(|v| v.as_u64()).unwrap_or(0); 1569 1569 - let did_count = seg.get("did_count").and_then(|v| v.as_u64()).unwrap_or(0); 1570 1570 - let bundle_start = seg 1571 1571 - .get("bundle_start") 1572 1572 - .and_then(|v| v.as_u64()) 1573 1573 - .unwrap_or(0); 1574 1574 - let bundle_end = 1575 1575 - seg.get("bundle_end").and_then(|v| v.as_u64()).unwrap_or(0); 1703 1703 + && !segments.is_empty() 1704 1704 + { 1705 1705 + println!("\n Delta Segments:"); 1706 1706 + println!(" ───────────────────────────────────────"); 1707 1707 + for (idx, seg) in segments.iter().enumerate() { 1708 1708 + let file_name = seg.get("file_name").and_then(|v| v.as_str()).unwrap_or("?"); 1709 1709 + let exists = seg.get("exists").and_then(|v| v.as_bool()).unwrap_or(false); 1710 1710 + let size = seg.get("size_bytes").and_then(|v| v.as_u64()).unwrap_or(0); 1711 1711 + let did_count = seg.get("did_count").and_then(|v| v.as_u64()).unwrap_or(0); 1712 1712 + let bundle_start = seg 1713 1713 + .get("bundle_start") 1714 1714 + .and_then(|v| v.as_u64()) 1715 1715 + .unwrap_or(0); 1716 1716 + let bundle_end = seg.get("bundle_end").and_then(|v| v.as_u64()).unwrap_or(0); 1576 1717 1577 1577 - println!( 1578 1578 - " [{:2}] {} {} ({})", 1579 1579 - idx + 1, 1580 1580 - if exists { "✓" } else { "✗" }, 1581 1581 - file_name, 1582 1582 - utils::format_bytes(size) 1583 1583 - ); 1584 1584 - println!( 1585 1585 - " Bundles: {}-{}, DIDs: {}, Locations: {}", 1586 1586 - bundle_start, 1587 1587 - bundle_end, 1588 1588 - utils::format_number(did_count), 1589 1589 - seg.get("location_count") 1590 1590 - .and_then(|v| v.as_u64()) 1591 1591 - .unwrap_or(0) 1592 1592 - ); 1593 1593 - } 1718 1718 + println!( 1719 1719 + " [{:2}] {} {} ({})", 1720 1720 + idx + 1, 1721 1721 + if exists { "✓" } else { "✗" }, 1722 1722 + file_name, 1723 1723 + utils::format_bytes(size) 1724 1724 + ); 1725 1725 + println!( 1726 1726 + " Bundles: {}-{}, DIDs: {}, Locations: {}", 1727 1727 + bundle_start, 1728 1728 + bundle_end, 1729 1729 + utils::format_number(did_count), 1730 1730 + seg.get("location_count") 1731 1731 + .and_then(|v| v.as_u64()) 1732 1732 + .unwrap_or(0) 1733 1733 + ); 1734 1734 + } 1594 1735 } 1595 1736 1596 1737 // Show raw shard data

+5 -2

src/cli/cmd_init.rs

reviewed

··· 1 1 use anyhow::Result; 2 2 use clap::{Args, ValueHint}; 3 3 - use plcbundle::{constants, BundleManager}; 3 3 + use plcbundle::{BundleManager, constants}; 4 4 use std::path::{Path, PathBuf}; 5 5 6 6 #[derive(Args)] ··· 218 218 origin: None, 219 219 force: false, 220 220 }; 221 221 - assert!(run(cmd).is_err(), "Should fail when trying to initialize already-initialized repository without --force"); 221 221 + assert!( 222 222 + run(cmd).is_err(), 223 223 + "Should fail when trying to initialize already-initialized repository without --force" 224 224 + ); 222 225 223 226 // Verify the origin is still "first" (not overwritten) 224 227 let index = Index::load(temp.path()).unwrap();

+111 -106

src/cli/cmd_inspect.rs

reviewed

··· 201 201 // For arbitrary file paths, we still need filesystem access - this should be refactored 202 202 // to use a manager method for loading from arbitrary paths in the future if supported. 203 203 // For now, it will return an error as per `resolve_bundle_target`. 204 204 - anyhow::bail!("Loading from arbitrary paths not yet implemented. Please specify a bundle number."); 204 204 + anyhow::bail!( 205 205 + "Loading from arbitrary paths not yet implemented. Please specify a bundle number." 206 206 + ); 205 207 }; 206 208 207 209 if !cmd.json { ··· 380 382 if let Some(aka) = op_val.get("alsoKnownAs").and_then(|v| v.as_array()) { 381 383 for item in aka.iter() { 382 384 if let Some(aka_str) = item.as_str() 383 383 - && aka_str.starts_with("at://") { 384 384 - total_handles += 1; 385 385 + && aka_str.starts_with("at://") 386 386 + { 387 387 + total_handles += 1; 385 388 386 386 - // Extract domain 387 387 - let handle = aka_str.strip_prefix("at://").unwrap_or(""); 388 388 - let handle = handle.split('/').next().unwrap_or(""); 389 389 + // Extract domain 390 390 + let handle = aka_str.strip_prefix("at://").unwrap_or(""); 391 391 + let handle = handle.split('/').next().unwrap_or(""); 389 392 390 390 - // Count domain (TLD) 391 391 - let parts: Vec<&str> = handle.split('.').collect(); 392 392 - if parts.len() >= 2 { 393 393 - let domain = format!( 394 394 - "{}.{}", 395 395 - parts[parts.len() - 2], 396 396 - parts[parts.len() - 1] 397 397 - ); 398 398 - *domain_counts.entry(domain).or_insert(0) += 1; 399 399 - } 393 393 + // Count domain (TLD) 394 394 + let parts: Vec<&str> = handle.split('.').collect(); 395 395 + if parts.len() >= 2 { 396 396 + let domain = 397 397 + format!("{}.{}", parts[parts.len() - 2], parts[parts.len() - 1]); 398 398 + *domain_counts.entry(domain).or_insert(0) += 1; 399 399 + } 400 400 401 401 - // Check for invalid patterns 402 402 - if handle.contains('_') { 403 403 - invalid_handles += 1; 404 404 - } 401 401 + // Check for invalid patterns 402 402 + if handle.contains('_') { 403 403 + invalid_handles += 1; 404 404 + } 405 405 } 406 406 } 407 407 } ··· 413 413 // Extract PDS endpoints 414 414 if let Some(pds_val) = op_val.get("services").and_then(|v| v.get("atproto_pds")) 415 415 && let Some(_pds) = pds_val.as_object() 416 416 - && let Some(endpoint) = pds_val.get("endpoint").and_then(|v| v.as_str()) { 417 417 - // Normalize endpoint 418 418 - let endpoint = endpoint 419 419 - .strip_prefix("https://") 420 420 - .or_else(|| endpoint.strip_prefix("http://")) 421 421 - .unwrap_or(endpoint); 422 422 - let endpoint = endpoint.split('/').next().unwrap_or(endpoint); 423 423 - *endpoint_counts.entry(endpoint.to_string()).or_insert(0) += 1; 416 416 + && let Some(endpoint) = pds_val.get("endpoint").and_then(|v| v.as_str()) 417 417 + { 418 418 + // Normalize endpoint 419 419 + let endpoint = endpoint 420 420 + .strip_prefix("https://") 421 421 + .or_else(|| endpoint.strip_prefix("http://")) 422 422 + .unwrap_or(endpoint); 423 423 + let endpoint = endpoint.split('/').next().unwrap_or(endpoint); 424 424 + *endpoint_counts.entry(endpoint.to_string()).or_insert(0) += 1; 424 425 } 425 426 } 426 427 } ··· 663 664 println!(); 664 665 665 666 // Embedded metadata (if available and not skipped) 666 666 - if !cmd.skip_metadata && result.has_metadata_frame 667 667 - && let Some(ref meta) = result.embedded_metadata { 668 668 - println!("📋 Embedded Metadata (Skippable Frame)"); 669 669 - println!("───────────────────────────────────────"); 670 670 - println!(" Format: {}", meta.format); 671 671 - println!(" Origin: {}", meta.origin); 672 672 - println!(" Bundle Number: {}", meta.bundle_number); 667 667 + if !cmd.skip_metadata 668 668 + && result.has_metadata_frame 669 669 + && let Some(ref meta) = result.embedded_metadata 670 670 + { 671 671 + println!("📋 Embedded Metadata (Skippable Frame)"); 672 672 + println!("───────────────────────────────────────"); 673 673 + println!(" Format: {}", meta.format); 674 674 + println!(" Origin: {}", meta.origin); 675 675 + println!(" Bundle Number: {}", meta.bundle_number); 673 676 674 674 - if !meta.created_by.is_empty() { 675 675 - println!(" Created by: {}", meta.created_by); 676 676 - } 677 677 - println!(" Created at: {}", meta.created_at); 677 677 + if !meta.created_by.is_empty() { 678 678 + println!(" Created by: {}", meta.created_by); 679 679 + } 680 680 + println!(" Created at: {}", meta.created_at); 678 681 679 679 - println!("\n Content:"); 680 680 - println!( 681 681 - " Operations: {}", 682 682 - format_number(meta.operation_count) 683 683 - ); 684 684 - println!(" Unique DIDs: {}", format_number(meta.did_count)); 685 685 - println!( 686 686 - " Frames: {} × {} ops", 687 687 - meta.frame_count, 688 688 - format_number(meta.frame_size) 689 689 - ); 690 690 - println!( 691 691 - " Timespan: {} → {}", 692 692 - meta.start_time, meta.end_time 693 693 - ); 682 682 + println!("\n Content:"); 683 683 + println!( 684 684 + " Operations: {}", 685 685 + format_number(meta.operation_count) 686 686 + ); 687 687 + println!(" Unique DIDs: {}", format_number(meta.did_count)); 688 688 + println!( 689 689 + " Frames: {} × {} ops", 690 690 + meta.frame_count, 691 691 + format_number(meta.frame_size) 692 692 + ); 693 693 + println!( 694 694 + " Timespan: {} → {}", 695 695 + meta.start_time, meta.end_time 696 696 + ); 697 697 + 698 698 + let duration = if let (Ok(start), Ok(end)) = ( 699 699 + DateTime::parse_from_rfc3339(&meta.start_time), 700 700 + DateTime::parse_from_rfc3339(&meta.end_time), 701 701 + ) { 702 702 + end.signed_duration_since(start) 703 703 + } else { 704 704 + chrono::Duration::seconds(0) 705 705 + }; 706 706 + println!( 707 707 + " Duration: {}", 708 708 + format_duration_verbose(duration) 709 709 + ); 694 710 695 695 - let duration = if let (Ok(start), Ok(end)) = ( 696 696 - DateTime::parse_from_rfc3339(&meta.start_time), 697 697 - DateTime::parse_from_rfc3339(&meta.end_time), 698 698 - ) { 699 699 - end.signed_duration_since(start) 700 700 - } else { 701 701 - chrono::Duration::seconds(0) 702 702 - }; 703 703 - println!( 704 704 - " Duration: {}", 705 705 - format_duration_verbose(duration) 706 706 - ); 711 711 + println!("\n Integrity:"); 712 712 + println!(" Content hash: {}", meta.content_hash); 713 713 + if let Some(ref parent) = meta.parent_hash 714 714 + && !parent.is_empty() 715 715 + { 716 716 + println!(" Parent hash: {}", parent); 717 717 + } 707 718 708 708 - println!("\n Integrity:"); 709 709 - println!(" Content hash: {}", meta.content_hash); 710 710 - if let Some(ref parent) = meta.parent_hash 711 711 - && !parent.is_empty() { 712 712 - println!(" Parent hash: {}", parent); 719 719 + // Index metadata for chain info 720 720 + if let Some(ref index_meta) = result.index_metadata { 721 721 + println!("\n Chain:"); 722 722 + println!(" Chain hash: {}", index_meta.hash); 723 723 + if !index_meta.parent.is_empty() { 724 724 + println!(" Parent: {}", index_meta.parent); 713 725 } 714 714 - 715 715 - // Index metadata for chain info 716 716 - if let Some(ref index_meta) = result.index_metadata { 717 717 - println!("\n Chain:"); 718 718 - println!(" Chain hash: {}", index_meta.hash); 719 719 - if !index_meta.parent.is_empty() { 720 720 - println!(" Parent: {}", index_meta.parent); 721 721 - } 722 722 - if !index_meta.cursor.is_empty() { 723 723 - println!(" Cursor: {}", index_meta.cursor); 724 724 - } 726 726 + if !index_meta.cursor.is_empty() { 727 727 + println!(" Cursor: {}", index_meta.cursor); 725 728 } 729 729 + } 726 730 727 727 - if !meta.frame_offsets.is_empty() { 728 728 - println!("\n Frame Index:"); 729 729 - println!(" {} frame offsets (embedded)", meta.frame_offsets.len()); 731 731 + if !meta.frame_offsets.is_empty() { 732 732 + println!("\n Frame Index:"); 733 733 + println!(" {} frame offsets (embedded)", meta.frame_offsets.len()); 730 734 731 731 - if let Some(metadata_size) = meta.metadata_frame_size { 732 732 - println!(" Metadata size: {}", format_bytes(metadata_size)); 733 733 - } 735 735 + if let Some(metadata_size) = meta.metadata_frame_size { 736 736 + println!(" Metadata size: {}", format_bytes(metadata_size)); 737 737 + } 734 738 735 735 - // Show compact list of first few offsets 736 736 - if meta.frame_offsets.len() <= 10 { 737 737 - println!(" Offsets: {:?}", meta.frame_offsets); 738 738 - } else { 739 739 - println!( 740 740 - " First offsets: {:?} ... ({} more)", 741 741 - &meta.frame_offsets[..5], 742 742 - meta.frame_offsets.len() - 5 743 743 - ); 744 744 - } 739 739 + // Show compact list of first few offsets 740 740 + if meta.frame_offsets.len() <= 10 { 741 741 + println!(" Offsets: {:?}", meta.frame_offsets); 742 742 + } else { 743 743 + println!( 744 744 + " First offsets: {:?} ... ({} more)", 745 745 + &meta.frame_offsets[..5], 746 746 + meta.frame_offsets.len() - 5 747 747 + ); 745 748 } 749 749 + } 746 750 747 747 - println!(); 748 748 - } 751 751 + println!(); 752 752 + } 749 753 750 754 // Operations breakdown 751 755 println!("📊 Operations Analysis"); ··· 812 816 println!("────────────────────"); 813 817 println!(" Total handles: {}", format_number(total_handles)); 814 818 if let Some(invalid) = result.invalid_handles 815 815 - && invalid > 0 { 816 816 - println!( 817 817 - " Invalid patterns: {} ({:.1}%)", 818 818 - format_number(invalid), 819 819 - (invalid as f64 / total_handles as f64 * 100.0) 820 820 - ); 819 819 + && invalid > 0 820 820 + { 821 821 + println!( 822 822 + " Invalid patterns: {} ({:.1}%)", 823 823 + format_number(invalid), 824 824 + (invalid as f64 / total_handles as f64 * 100.0) 825 825 + ); 821 826 } 822 827 823 828 if !result.top_domains.is_empty() {

+17 -5

src/cli/cmd_ls.rs

reviewed

··· 163 163 .to_string() 164 164 } 165 165 166 166 - fn print_bundle_fields(meta: &plcbundle::index::BundleMetadata, fields: &[String], sep: &str, human_readable: bool) { 167 167 - let values: Vec<String> = fields.iter().map(|f| get_field_value(meta, f, human_readable)).collect(); 166 166 + fn print_bundle_fields( 167 167 + meta: &plcbundle::index::BundleMetadata, 168 168 + fields: &[String], 169 169 + sep: &str, 170 170 + human_readable: bool, 171 171 + ) { 172 172 + let values: Vec<String> = fields 173 173 + .iter() 174 174 + .map(|f| get_field_value(meta, f, human_readable)) 175 175 + .collect(); 168 176 println!("{}", values.join(sep)); 169 177 } 170 178 171 171 - fn get_field_value(meta: &plcbundle::index::BundleMetadata, field: &str, human_readable: bool) -> String { 179 179 + fn get_field_value( 180 180 + meta: &plcbundle::index::BundleMetadata, 181 181 + field: &str, 182 182 + human_readable: bool, 183 183 + ) -> String { 172 184 match field { 173 185 "bundle" => format!("{}", meta.bundle_number), 174 186 ··· 250 262 } else { 251 263 format!("{}", meta.compressed_size) 252 264 } 253 253 - }, 265 265 + } 254 266 "size_mb" => format!("{:.2}", meta.compressed_size as f64 / (1024.0 * 1024.0)), 255 267 "size_h" | "size_human" => format_bytes_compact(meta.compressed_size), 256 268 ··· 260 272 } else { 261 273 format!("{}", meta.uncompressed_size) 262 274 } 263 263 - }, 275 275 + } 264 276 "uncompressed_mb" => format!("{:.2}", meta.uncompressed_size as f64 / (1024.0 * 1024.0)), 265 277 "uncompressed_h" | "uncompressed_human" => format_bytes_compact(meta.uncompressed_size), 266 278

+7 -3

src/cli/cmd_mempool.rs

reviewed

··· 4 4 use anyhow::Result; 5 5 use clap::{Args, Subcommand, ValueHint}; 6 6 use plcbundle::format::format_number; 7 7 - use std::path::Path; 8 7 use plcbundle::{BundleManager, constants}; 9 8 use std::io::{self, Write}; 9 9 + use std::path::Path; 10 10 use std::path::PathBuf; 11 11 12 12 #[derive(Args)] ··· 75 75 } 76 76 77 77 impl HasGlobalFlags for MempoolCommand { 78 78 - fn verbose(&self) -> bool { false } 79 79 - fn quiet(&self) -> bool { false } 78 78 + fn verbose(&self) -> bool { 79 79 + false 80 80 + } 81 81 + fn quiet(&self) -> bool { 82 82 + false 83 83 + } 80 84 } 81 85 82 86 pub fn run(cmd: MempoolCommand, dir: PathBuf, global_verbose: bool) -> Result<()> {

+26 -16

src/cli/cmd_migrate.rs

reviewed

··· 1 1 // Migrate command - convert bundles to multi-frame format 2 2 use super::progress::ProgressBar; 3 3 - use super::utils::{format_bytes, HasGlobalFlags}; 3 3 + use super::utils::{HasGlobalFlags, format_bytes}; 4 4 use anyhow::{Result, bail}; 5 5 use clap::Args; 6 6 use plcbundle::BundleManager; ··· 65 65 } 66 66 67 67 impl HasGlobalFlags for MigrateCommand { 68 68 - fn verbose(&self) -> bool { false } 69 69 - fn quiet(&self) -> bool { false } 68 68 + fn verbose(&self) -> bool { 69 69 + false 70 70 + } 71 71 + fn quiet(&self) -> bool { 72 72 + false 73 73 + } 70 74 } 71 75 72 76 pub fn run(cmd: MigrateCommand, dir: PathBuf, global_verbose: bool) -> Result<()> { ··· 75 79 // Auto-detect number of threads if 0 76 80 let workers = super::utils::get_worker_threads(cmd.threads, 4); 77 81 78 78 - eprintln!("Scanning for legacy bundles in: {}\n", super::utils::display_path(&dir).display()); 82 82 + eprintln!( 83 83 + "Scanning for legacy bundles in: {}\n", 84 84 + super::utils::display_path(&dir).display() 85 85 + ); 79 86 80 87 let index = manager.get_index(); 81 88 let bundles = &index.bundles; ··· 87 94 88 95 // Determine which bundles to consider for migration 89 96 let last_bundle = index.last_bundle; 90 90 - let target_bundles: Option<std::collections::HashSet<u32>> = if let Some(ref bundles_spec) = cmd.bundles { 97 97 + let target_bundles: Option<std::collections::HashSet<u32>> = if let Some(ref bundles_spec) = 98 98 + cmd.bundles 99 99 + { 91 100 let bundle_list = super::utils::parse_bundle_spec(Some(bundles_spec.clone()), last_bundle)?; 92 101 Some(bundle_list.into_iter().collect()) 93 102 } else { ··· 102 111 for meta in bundles { 103 112 // Filter by bundle range if specified 104 113 if let Some(ref target_set) = target_bundles 105 105 - && !target_set.contains(&meta.bundle_number) { 106 106 - continue; 114 114 + && !target_set.contains(&meta.bundle_number) 115 115 + { 116 116 + continue; 107 117 } 108 118 109 119 let embedded_meta = manager.get_embedded_metadata(meta.bundle_number)?; ··· 145 155 // Show migration plan 146 156 eprintln!("Migration Plan"); 147 157 eprintln!("══════════════\n"); 148 148 - 158 158 + 149 159 if let Some(ref bundles_spec) = cmd.bundles { 150 160 eprintln!(" Range: {}", bundles_spec); 151 161 } ··· 211 221 212 222 let progress_arc = Arc::new(Mutex::new(progress)); 213 223 // Use atomics for counters to reduce lock contention 214 214 - use std::sync::atomic::{AtomicUsize, AtomicU64, Ordering}; 224 224 + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; 215 225 let count_atomic = Arc::new(AtomicUsize::new(0)); 216 226 let bytes_atomic = Arc::new(AtomicU64::new(0)); 217 217 - 227 227 + 218 228 // Update progress bar less frequently to reduce contention 219 229 // Update every N bundles or every 100ms, whichever comes first 220 230 let update_interval = (workers.max(1) * 4).max(10); // At least every 10 bundles, or 4x workers 221 221 - 231 231 + 222 232 let results: Vec<_> = if workers > 1 { 223 233 // Parallel mode: process in chunks to maintain some ordering 224 234 // Increase chunk size to reduce contention ··· 235 245 236 246 // Update atomics (lock-free) 237 247 let current_count = count_atomic.fetch_add(1, Ordering::Relaxed) + 1; 238 238 - let total_bytes = bytes_atomic.fetch_add(info.old_size, Ordering::Relaxed) + info.old_size; 248 248 + let total_bytes = bytes_atomic.fetch_add(info.old_size, Ordering::Relaxed) 249 249 + + info.old_size; 239 250 240 251 // Only update progress bar periodically to reduce lock contention 241 252 if current_count.is_multiple_of(update_interval) || current_count == 1 { ··· 257 268 let result = manager.migrate_bundle(info.bundle_number); 258 269 259 270 let current_count = i + 1; 260 260 - let total_bytes = bytes_atomic.fetch_add(info.old_size, Ordering::Relaxed) + info.old_size; 271 271 + let total_bytes = 272 272 + bytes_atomic.fetch_add(info.old_size, Ordering::Relaxed) + info.old_size; 261 273 262 274 // Update every bundle in sequential mode (no contention) 263 275 let prog = progress_arc.lock().unwrap(); ··· 376 388 let ratio_diff_str = format!("{:+.3}x", ratio_diff); 377 389 eprintln!( 378 390 "Ratio {:<13} {:<13} {}", 379 379 - old_ratio_str, 380 380 - new_ratio_str, 381 381 - ratio_diff_str 391 391 + old_ratio_str, new_ratio_str, ratio_diff_str 382 392 ); 383 393 let avg_change = size_diff / success as i64; 384 394 let avg_change_str = if avg_change >= 0 {

+23 -10

src/cli/cmd_op.rs

reviewed

··· 138 138 139 139 pub fn run(cmd: OpCommand, dir: PathBuf, quiet: bool) -> Result<()> { 140 140 match cmd.command { 141 141 - OpCommands::Get { bundle, position, query, raw } => { 141 141 + OpCommands::Get { 142 142 + bundle, 143 143 + position, 144 144 + query, 145 145 + raw, 146 146 + } => { 142 147 cmd_op_get(dir, bundle, position, query, raw, quiet)?; 143 148 } 144 149 OpCommands::Show { bundle, position } => { ··· 170 175 // op get 10000 → bundle 2, position 0 171 176 // op get 88410345 → bundle 8842, position 345 172 177 let global_pos = bundle as u64; 173 173 - let (bundle_num, op_pos) = plcbundle::constants::global_to_bundle_position(global_pos); 178 178 + let (bundle_num, op_pos) = 179 179 + plcbundle::constants::global_to_bundle_position(global_pos); 174 180 (bundle_num, op_pos) 175 181 } 176 182 } 177 183 } 178 184 } 179 185 180 180 - pub fn cmd_op_get(dir: PathBuf, bundle: u32, position: Option<usize>, query: Option<String>, raw: bool, quiet: bool) -> Result<()> { 186 186 + pub fn cmd_op_get( 187 187 + dir: PathBuf, 188 188 + bundle: u32, 189 189 + position: Option<usize>, 190 190 + query: Option<String>, 191 191 + raw: bool, 192 192 + quiet: bool, 193 193 + ) -> Result<()> { 181 194 let (bundle_num, op_index) = parse_op_position(bundle, position); 182 195 183 196 let manager = super::utils::create_manager(dir, false, quiet, false)?; ··· 187 200 manager.get_operation_raw(bundle_num, op_index)? 188 201 } else { 189 202 let result = manager.get_operation_with_stats(bundle_num, op_index)?; 190 190 - let global_pos = 191 191 - plcbundle::constants::bundle_position_to_global(bundle_num, op_index); 203 203 + let global_pos = plcbundle::constants::bundle_position_to_global(bundle_num, op_index); 192 204 193 205 log::info!( 194 206 "[Load] Bundle {}:{:04} (pos={}) in {:?} | {} bytes", ··· 205 217 // If query is provided, apply JMESPath query 206 218 let output_json = if let Some(query_expr) = query { 207 219 // Compile JMESPath expression 208 208 - let expr = jmespath::compile(&query_expr) 209 209 - .map_err(|e| anyhow::anyhow!("Failed to compile JMESPath query '{}': {}", query_expr, e))?; 220 220 + let expr = jmespath::compile(&query_expr).map_err(|e| { 221 221 + anyhow::anyhow!("Failed to compile JMESPath query '{}': {}", query_expr, e) 222 222 + })?; 210 223 211 224 // Execute query 212 225 let data = jmespath::Variable::from_json(&json) 213 226 .map_err(|e| anyhow::anyhow!("Failed to parse operation JSON: {}", e))?; 214 227 215 215 - let result = expr.search(&data) 228 228 + let result = expr 229 229 + .search(&data) 216 230 .map_err(|e| anyhow::anyhow!("JMESPath query failed: {}", e))?; 217 231 218 232 if result.is_null() { ··· 409 423 let cid_matches = op.cid.as_ref() == Some(&cid); 410 424 411 425 if cid_matches { 412 412 - let global_pos = 413 413 - plcbundle::constants::bundle_position_to_global(bundle_num, i); 426 426 + let global_pos = plcbundle::constants::bundle_position_to_global(bundle_num, i); 414 427 415 428 println!("Found: bundle {}, position {}", bundle_num, i); 416 429 println!("Global position: {}\n", global_pos);

+5 -5

src/cli/cmd_query.rs

reviewed

··· 1 1 use anyhow::Result; 2 2 - use plcbundle::processor::{Stats, OutputHandler}; 3 2 use clap::{Args, ValueEnum, ValueHint}; 3 3 + use plcbundle::processor::{OutputHandler, Stats}; 4 4 use plcbundle::*; 5 5 use std::io::Write; 6 6 use std::path::PathBuf; ··· 175 175 176 176 let start = Instant::now(); 177 177 let output = Arc::new(StdoutHandler::new(stats_only)); 178 178 - 178 178 + 179 179 // Track bundle count separately since callback gives increment, not total 180 180 let bundle_count = Arc::new(Mutex::new(0usize)); 181 181 let pb_arc = pb.as_ref().map(|pb| Arc::new(Mutex::new(pb))); ··· 192 192 *count += 1; 193 193 let current_bundles = *count; 194 194 drop(count); 195 195 - 195 195 + 196 196 let pb = pb_mutex.lock().unwrap(); 197 197 - 197 197 + 198 198 // Update progress with bundles processed and bytes 199 199 pb.set_with_bytes(current_bundles, stats.total_bytes); 200 200 - 200 200 + 201 201 // Set message with matches 202 202 pb.set_message(format!( 203 203 "✓ {} matches",

+21 -7

src/cli/cmd_rebuild.rs

reviewed

··· 1 1 // Rebuild plc_bundles.json from existing bundle files 2 2 use super::progress::ProgressBar; 3 3 - use super::utils::{format_bytes, HasGlobalFlags}; 3 3 + use super::utils::{HasGlobalFlags, format_bytes}; 4 4 use anyhow::Result; 5 5 use clap::{Args, ValueHint}; 6 6 use plcbundle::BundleManager; ··· 47 47 } 48 48 49 49 impl HasGlobalFlags for RebuildCommand { 50 50 - fn verbose(&self) -> bool { false } 51 51 - fn quiet(&self) -> bool { false } 50 50 + fn verbose(&self) -> bool { 51 51 + false 52 52 + } 53 53 + fn quiet(&self) -> bool { 54 54 + false 55 55 + } 52 56 } 53 57 54 58 pub fn run(cmd: RebuildCommand, dir: PathBuf, _global_verbose: bool) -> Result<()> { 55 55 - eprintln!("Rebuilding bundle index from: {}\n", super::utils::display_path(&dir).display()); 59 59 + eprintln!( 60 60 + "Rebuilding bundle index from: {}\n", 61 61 + super::utils::display_path(&dir).display() 62 62 + ); 56 63 57 64 let start = Instant::now(); 58 65 ··· 90 97 eprintln!("Rebuild Summary"); 91 98 eprintln!("═══════════════"); 92 99 eprintln!(" Bundles: {}", index.bundles.len()); 93 93 - eprintln!(" Range: {} - {}", 100 100 + eprintln!( 101 101 + " Range: {} - {}", 94 102 index.bundles.first().map(|b| b.bundle_number).unwrap_or(0), 95 103 index.last_bundle 96 104 ); 97 105 eprintln!(" Origin: {}", index.origin); 98 98 - eprintln!(" Compressed size: {}", format_bytes(index.total_size_bytes)); 99 99 - eprintln!(" Uncompressed size: {}", format_bytes(index.total_uncompressed_size_bytes)); 106 106 + eprintln!( 107 107 + " Compressed size: {}", 108 108 + format_bytes(index.total_size_bytes) 109 109 + ); 110 110 + eprintln!( 111 111 + " Uncompressed size: {}", 112 112 + format_bytes(index.total_uncompressed_size_bytes) 113 113 + ); 100 114 eprintln!(" Scan time: {:?}", elapsed); 101 115 eprintln!(); 102 116

+6 -1

src/cli/cmd_rollback.rs

reviewed

··· 374 374 } 375 375 376 376 // Use default flush interval for rollback 377 377 - let _stats = manager.build_did_index(crate::constants::DID_INDEX_FLUSH_INTERVAL, None::<fn(u32, u32, u64, u64)>, None, None)?; 377 377 + let _stats = manager.build_did_index( 378 378 + crate::constants::DID_INDEX_FLUSH_INTERVAL, 379 379 + None::<fn(u32, u32, u64, u64)>, 380 380 + None, 381 381 + None, 382 382 + )?; 378 383 println!( 379 384 " ✓ DID index rebuilt ({} bundles)", 380 385 plan.bundles_to_keep

+26 -19

src/cli/cmd_server.rs

reviewed

··· 7 7 use tokio::time::Duration; 8 8 9 9 #[cfg(feature = "server")] 10 10 - use plcbundle::server::{start_server, StartupConfig, ProgressCallbackFactory}; 11 11 - #[cfg(feature = "server")] 12 10 use super::progress::ProgressBar; 11 11 + #[cfg(feature = "server")] 12 12 + use plcbundle::server::{ProgressCallbackFactory, StartupConfig, start_server}; 13 13 #[cfg(feature = "server")] 14 14 use std::sync::{Arc, Mutex}; 15 15 16 16 #[cfg(feature = "server")] 17 17 fn parse_duration_for_clap(s: &str) -> Result<Duration, String> { 18 18 - plcbundle::server::parse_duration(s) 19 19 - .map_err(|e| e.to_string()) 18 18 + plcbundle::server::parse_duration(s).map_err(|e| e.to_string()) 20 19 } 21 20 22 21 #[derive(Args)] ··· 76 75 #[cfg(feature = "server")] 77 76 #[arg(long, default_value = "60s", value_parser = parse_duration_for_clap, help_heading = "Sync Options")] 78 77 pub interval: Duration, 79 79 - 78 78 + 80 79 #[cfg(not(feature = "server"))] 81 80 #[arg(long, default_value = "60s", help_heading = "Sync Options")] 82 81 pub interval: String, ··· 96 95 /// Handle resolver URL (defaults to quickdid.smokesignal.tools if not provided) 97 96 #[arg(long, help_heading = "Feature Options", value_hint = ValueHint::Url)] 98 97 pub handle_resolver: Option<String>, 99 99 - 100 98 } 101 99 102 100 pub fn run(cmd: ServerCommand, dir: PathBuf, global_verbose: bool) -> Result<()> { ··· 115 113 #[cfg(feature = "server")] 116 114 fn run_server(cmd: ServerCommand, dir: PathBuf, global_verbose: bool) -> Result<()> { 117 115 use anyhow::Context; 118 118 - use tokio::runtime::Runtime; 119 116 use plcbundle::constants; 117 117 + use tokio::runtime::Runtime; 120 118 121 119 // Create tokio runtime for async operations 122 120 let rt = Runtime::new().context("Failed to create tokio runtime")?; 123 123 - 121 121 + 124 122 // Determine handle resolver URL 125 123 let handle_resolver_url = if cmd.handle_resolver.is_none() { 126 124 Some(constants::DEFAULT_HANDLE_RESOLVER_URL.to_string()) ··· 147 145 // This factory will be called with (last_bundle, total_bytes) when the index needs to be built 148 146 let progress_callback_factory: Option<ProgressCallbackFactory> = if cmd.resolver { 149 147 Some(Box::new(move |last_bundle: u32, total_bytes: u64| { 150 150 - let progress = Arc::new(Mutex::new(ProgressBar::with_bytes(last_bundle as usize, total_bytes))); 148 148 + let progress = Arc::new(Mutex::new(ProgressBar::with_bytes( 149 149 + last_bundle as usize, 150 150 + total_bytes, 151 151 + ))); 151 152 let progress_clone = progress.clone(); 152 153 let progress_finish = progress.clone(); 153 154 let verbose = global_verbose; 154 154 - 155 155 - let callback = Box::new(move |current: u32, _total: u32, bytes_processed: u64, _total_bytes: u64| { 156 156 - let pb = progress_clone.lock().unwrap(); 157 157 - pb.set_with_bytes(current as usize, bytes_processed); 158 158 - if verbose && current.is_multiple_of(100) { 159 159 - log::debug!("[DIDResolver] Index progress: {}/{} bundles", current, _total); 160 160 - } 161 161 - }) as Box<dyn Fn(u32, u32, u64, u64) + Send + Sync>; 162 162 - 155 155 + 156 156 + let callback = Box::new( 157 157 + move |current: u32, _total: u32, bytes_processed: u64, _total_bytes: u64| { 158 158 + let pb = progress_clone.lock().unwrap(); 159 159 + pb.set_with_bytes(current as usize, bytes_processed); 160 160 + if verbose && current.is_multiple_of(100) { 161 161 + log::debug!( 162 162 + "[DIDResolver] Index progress: {}/{} bundles", 163 163 + current, 164 164 + _total 165 165 + ); 166 166 + } 167 167 + }, 168 168 + ) as Box<dyn Fn(u32, u32, u64, u64) + Send + Sync>; 169 169 + 163 170 let finish = Box::new(move || { 164 171 let pb = progress_finish.lock().unwrap(); 165 172 pb.finish(); 166 173 }) as Box<dyn FnOnce() + Send + Sync>; 167 167 - 174 174 + 168 175 (callback, Some(finish)) 169 176 })) 170 177 } else {

+82 -40

src/cli/cmd_stats.rs

reviewed

··· 160 160 .collect(); 161 161 162 162 let bundle_count = bundle_metadatas.len(); 163 163 - let total_operations: u64 = bundle_metadatas.iter().map(|b| b.operation_count as u64).sum(); 164 164 - let total_dids: u64 = bundle_metadatas.iter().map(|b| b.did_count as u64).sum(); 165 165 - let total_compressed_size: u64 = bundle_metadatas.iter().map(|b| b.compressed_size).sum(); 166 166 - let total_uncompressed_size: u64 = bundle_metadatas 163 163 + let total_operations: u64 = bundle_metadatas 167 164 .iter() 168 168 - .map(|b| b.uncompressed_size) 165 165 + .map(|b| b.operation_count as u64) 169 166 .sum(); 167 167 + let total_dids: u64 = bundle_metadatas.iter().map(|b| b.did_count as u64).sum(); 168 168 + let total_compressed_size: u64 = bundle_metadatas.iter().map(|b| b.compressed_size).sum(); 169 169 + let total_uncompressed_size: u64 = bundle_metadatas.iter().map(|b| b.uncompressed_size).sum(); 170 170 171 171 let compression_ratio = if total_uncompressed_size > 0 { 172 172 (1.0 - total_compressed_size as f64 / total_uncompressed_size as f64) * 100.0 ··· 268 268 .filter(|b| bundle_nums.contains(&b.bundle_number)) 269 269 .collect(); 270 270 271 271 - let total_did_operations: u64 = bundle_metadatas.iter().map(|b| b.operation_count as u64).sum(); 271 271 + let total_did_operations: u64 = bundle_metadatas 272 272 + .iter() 273 273 + .map(|b| b.operation_count as u64) 274 274 + .sum(); 272 275 let total_unique_dids: usize = bundle_metadatas.iter().map(|b| b.did_count as usize).sum(); 273 276 274 277 // For more detailed stats, we'd need to iterate operations, but that's expensive ··· 322 325 .map(|b| &b.start_time) 323 326 .min() 324 327 .cloned(); 325 325 - let latest_time = bundle_metadatas 326 326 - .iter() 327 327 - .map(|b| &b.end_time) 328 328 - .max() 329 329 - .cloned(); 328 328 + let latest_time = bundle_metadatas.iter().map(|b| &b.end_time).max().cloned(); 330 329 331 331 - let time_span_days = if let (Some(earliest), Some(latest)) = (earliest_time.as_ref(), latest_time.as_ref()) { 332 332 - if let (Ok(e), Ok(l)) = ( 333 333 - chrono::DateTime::parse_from_rfc3339(earliest), 334 334 - chrono::DateTime::parse_from_rfc3339(latest), 335 335 - ) { 336 336 - let duration = l.signed_duration_since(e); 337 337 - Some(duration.num_seconds() as f64 / 86400.0) 330 330 + let time_span_days = 331 331 + if let (Some(earliest), Some(latest)) = (earliest_time.as_ref(), latest_time.as_ref()) { 332 332 + if let (Ok(e), Ok(l)) = ( 333 333 + chrono::DateTime::parse_from_rfc3339(earliest), 334 334 + chrono::DateTime::parse_from_rfc3339(latest), 335 335 + ) { 336 336 + let duration = l.signed_duration_since(e); 337 337 + Some(duration.num_seconds() as f64 / 86400.0) 338 338 + } else { 339 339 + None 340 340 + } 338 341 } else { 339 342 None 340 340 - } 341 341 - } else { 342 342 - None 343 343 - }; 343 343 + }; 344 344 345 345 - let total_operations: u64 = bundle_metadatas.iter().map(|b| b.operation_count as u64).sum(); 345 345 + let total_operations: u64 = bundle_metadatas 346 346 + .iter() 347 347 + .map(|b| b.operation_count as u64) 348 348 + .sum(); 346 349 let operations_per_day = time_span_days.and_then(|days| { 347 350 if days > 0.0 { 348 351 Some(total_operations as f64 / days) ··· 396 399 println!("═══════════════════════════════════════════════════════════════"); 397 400 println!(); 398 401 println!(" Bundle Range: {}", stats["bundle_range"]); 399 399 - println!(" Total Bundles: {}", utils::format_number(stats["bundle_count"].as_u64().unwrap_or(0))); 400 400 - println!(" Total Operations: {}", utils::format_number(stats["total_operations"].as_u64().unwrap_or(0))); 401 401 - println!(" Total DIDs: {}", utils::format_number(stats["total_dids"].as_u64().unwrap_or(0))); 402 402 + println!( 403 403 + " Total Bundles: {}", 404 404 + utils::format_number(stats["bundle_count"].as_u64().unwrap_or(0)) 405 405 + ); 406 406 + println!( 407 407 + " Total Operations: {}", 408 408 + utils::format_number(stats["total_operations"].as_u64().unwrap_or(0)) 409 409 + ); 410 410 + println!( 411 411 + " Total DIDs: {}", 412 412 + utils::format_number(stats["total_dids"].as_u64().unwrap_or(0)) 413 413 + ); 402 414 println!(); 403 415 println!(" Storage:"); 404 404 - println!(" Compressed: {}", utils::format_bytes(stats["total_compressed_size"].as_u64().unwrap_or(0))); 405 405 - println!(" Uncompressed: {}", utils::format_bytes(stats["total_uncompressed_size"].as_u64().unwrap_or(0))); 406 406 - println!(" Compression: {:.1}%", stats["compression_ratio"].as_f64().unwrap_or(0.0)); 416 416 + println!( 417 417 + " Compressed: {}", 418 418 + utils::format_bytes(stats["total_compressed_size"].as_u64().unwrap_or(0)) 419 419 + ); 420 420 + println!( 421 421 + " Uncompressed: {}", 422 422 + utils::format_bytes(stats["total_uncompressed_size"].as_u64().unwrap_or(0)) 423 423 + ); 424 424 + println!( 425 425 + " Compression: {:.1}%", 426 426 + stats["compression_ratio"].as_f64().unwrap_or(0.0) 427 427 + ); 407 428 println!(); 408 429 println!(" Averages:"); 409 409 - println!(" Ops per Bundle: {:.1}", stats["avg_operations_per_bundle"].as_f64().unwrap_or(0.0)); 410 410 - println!(" DIDs per Bundle: {:.1}", stats["avg_dids_per_bundle"].as_f64().unwrap_or(0.0)); 430 430 + println!( 431 431 + " Ops per Bundle: {:.1}", 432 432 + stats["avg_operations_per_bundle"].as_f64().unwrap_or(0.0) 433 433 + ); 434 434 + println!( 435 435 + " DIDs per Bundle: {:.1}", 436 436 + stats["avg_dids_per_bundle"].as_f64().unwrap_or(0.0) 437 437 + ); 411 438 } 412 439 StatType::Operations => { 413 440 println!("🔧 Operation Statistics"); 414 441 println!("═══════════════════════════════════════════════════════════════"); 415 442 println!(); 416 416 - println!(" Total Operations: {}", utils::format_number(stats["total_operations"].as_u64().unwrap_or(0))); 417 417 - println!(" Nullified: {}", utils::format_number(stats["nullified_operations"].as_u64().unwrap_or(0))); 443 443 + println!( 444 444 + " Total Operations: {}", 445 445 + utils::format_number(stats["total_operations"].as_u64().unwrap_or(0)) 446 446 + ); 447 447 + println!( 448 448 + " Nullified: {}", 449 449 + utils::format_number(stats["nullified_operations"].as_u64().unwrap_or(0)) 450 450 + ); 418 451 println!(); 419 419 - 452 452 + 420 453 if let Some(types) = stats.get("operation_types").and_then(|v| v.as_object()) { 421 454 println!(" Operation Types:"); 422 455 let mut type_vec: Vec<_> = types.iter().collect(); ··· 433 466 .and_then(|p| p.get(op_type)) 434 467 .and_then(|v| v.as_f64()) 435 468 .unwrap_or(0.0); 436 436 - println!(" {:<20} {:>10} ({:>5.1}%)", 437 437 - op_type, 469 469 + println!( 470 470 + " {:<20} {:>10} ({:>5.1}%)", 471 471 + op_type, 438 472 utils::format_number(count), 439 473 percentage 440 474 ); ··· 445 479 println!("🆔 DID Statistics"); 446 480 println!("═══════════════════════════════════════════════════════════════"); 447 481 println!(); 448 448 - println!(" Unique DIDs: {}", utils::format_number(stats["total_unique_dids"].as_u64().unwrap_or(0))); 449 449 - println!(" Total Operations: {}", utils::format_number(stats["total_did_operations"].as_u64().unwrap_or(0))); 450 450 - println!(" Avg Ops per DID: {:.2}", stats["avg_operations_per_did"].as_f64().unwrap_or(0.0)); 482 482 + println!( 483 483 + " Unique DIDs: {}", 484 484 + utils::format_number(stats["total_unique_dids"].as_u64().unwrap_or(0)) 485 485 + ); 486 486 + println!( 487 487 + " Total Operations: {}", 488 488 + utils::format_number(stats["total_did_operations"].as_u64().unwrap_or(0)) 489 489 + ); 490 490 + println!( 491 491 + " Avg Ops per DID: {:.2}", 492 492 + stats["avg_operations_per_did"].as_f64().unwrap_or(0.0) 493 493 + ); 451 494 } 452 495 StatType::Timeline => { 453 496 println!("⏰ Timeline Statistics"); ··· 473 516 println!(); 474 517 Ok(()) 475 518 } 476 476 -

+52 -22

src/cli/cmd_status.rs

reviewed

··· 1 1 use anyhow::Result; 2 2 use clap::Parser; 3 3 - use plcbundle::*; 4 3 use plcbundle::index::Index; 4 4 + use plcbundle::*; 5 5 use std::path::{Path, PathBuf}; 6 6 7 7 use super::utils; ··· 65 65 manager: &BundleManager, 66 66 index: &Index, 67 67 dir: &Path, 68 68 - detailed: bool 68 68 + detailed: bool, 69 69 ) -> Result<()> { 70 70 let dir_display = utils::display_path(dir); 71 71 ··· 96 96 let total_bundles = index.last_bundle; 97 97 let total_operations = plcbundle::constants::total_operations_from_bundles(total_bundles); 98 98 99 99 - println!(" Total Bundles: {}", utils::format_number(total_bundles as u64)); 100 100 - println!(" Operations: ~{}", utils::format_number(total_operations)); 101 101 - println!(" Unique DIDs: {}", utils::format_number(calculate_total_dids(index) as u64)); 99 99 + println!( 100 100 + " Total Bundles: {}", 101 101 + utils::format_number(total_bundles as u64) 102 102 + ); 103 103 + println!( 104 104 + " Operations: ~{}", 105 105 + utils::format_number(total_operations) 106 106 + ); 107 107 + println!( 108 108 + " Unique DIDs: {}", 109 109 + utils::format_number(calculate_total_dids(index) as u64) 110 110 + ); 102 111 println!(); 103 112 104 113 // Storage 105 105 - println!(" Compressed: {}", utils::format_bytes(index.total_size_bytes)); 106 106 - println!(" Uncompressed: {}", utils::format_bytes(index.total_uncompressed_size_bytes)); 114 114 + println!( 115 115 + " Compressed: {}", 116 116 + utils::format_bytes(index.total_size_bytes) 117 117 + ); 118 118 + println!( 119 119 + " Uncompressed: {}", 120 120 + utils::format_bytes(index.total_uncompressed_size_bytes) 121 121 + ); 107 122 108 123 let compression_ratio = if index.total_uncompressed_size_bytes > 0 { 109 109 - (1.0 - index.total_size_bytes as f64 / index.total_uncompressed_size_bytes as f64) * 100.0 124 124 + (1.0 - index.total_size_bytes as f64 / index.total_uncompressed_size_bytes as f64) 125 125 + * 100.0 110 126 } else { 111 127 0.0 112 128 }; ··· 121 137 122 138 println!(" Recent Bundles:"); 123 139 for meta in recent_bundles { 124 124 - println!(" #{:<6} {} → {} ({} ops, {} DIDs)", 140 140 + println!( 141 141 + " #{:<6} {} → {} ({} ops, {} DIDs)", 125 142 meta.bundle_number, 126 143 &meta.start_time[..19], // Just date and time without timezone 127 144 &meta.end_time[..19], ··· 144 161 145 162 let stats = manager.get_did_index_stats_struct(); 146 163 if stats.total_dids > 0 { 147 147 - println!(" Indexed DIDs: {}", utils::format_number(stats.total_dids as u64)); 164 164 + println!( 165 165 + " Indexed DIDs: {}", 166 166 + utils::format_number(stats.total_dids as u64) 167 167 + ); 148 168 } 149 169 } else { 150 170 println!(" Status: ✗ Not built"); ··· 158 178 159 179 if let Ok(mempool_stats) = manager.get_mempool_stats() { 160 180 if mempool_stats.count > 0 { 161 161 - println!(" Operations: {}", utils::format_number(mempool_stats.count as u64)); 181 181 + println!( 182 182 + " Operations: {}", 183 183 + utils::format_number(mempool_stats.count as u64) 184 184 + ); 162 185 println!(" Target Bundle: #{}", mempool_stats.target_bundle); 163 163 - println!(" Can Bundle: {}", if mempool_stats.can_create_bundle { "Yes" } else { "No" }); 186 186 + println!( 187 187 + " Can Bundle: {}", 188 188 + if mempool_stats.can_create_bundle { 189 189 + "Yes" 190 190 + } else { 191 191 + "No" 192 192 + } 193 193 + ); 164 194 165 195 if let Some(first) = mempool_stats.first_time { 166 196 println!(" First Op: {}", first); ··· 189 219 } 190 220 191 221 if let Ok(mempool_stats) = manager.get_mempool_stats() 192 192 - && mempool_stats.count >= plcbundle::constants::BUNDLE_SIZE { 193 193 - suggestions.push("Mempool ready to create new bundle: plcbundle sync"); 222 222 + && mempool_stats.count >= plcbundle::constants::BUNDLE_SIZE 223 223 + { 224 224 + suggestions.push("Mempool ready to create new bundle: plcbundle sync"); 194 225 } 195 226 196 227 // Add general hints ··· 228 259 Ok(()) 229 260 } 230 261 231 231 - fn print_json_status( 232 232 - manager: &BundleManager, 233 233 - index: &Index, 234 234 - dir: &Path, 235 235 - ) -> Result<()> { 262 262 + fn print_json_status(manager: &BundleManager, index: &Index, dir: &Path) -> Result<()> { 236 263 use serde_json::json; 237 264 238 265 let dir_display = utils::display_path(dir); ··· 291 318 health.push("did_index_not_built"); 292 319 } 293 320 if let Ok(mempool_stats) = manager.get_mempool_stats() 294 294 - && mempool_stats.count >= plcbundle::constants::BUNDLE_SIZE { 295 295 - health.push("mempool_ready_to_bundle"); 321 321 + && mempool_stats.count >= plcbundle::constants::BUNDLE_SIZE 322 322 + { 323 323 + health.push("mempool_ready_to_bundle"); 296 324 } 297 325 status["health"] = json!(health); 298 326 ··· 302 330 } 303 331 304 332 fn check_did_index_exists(dir: &Path) -> bool { 305 305 - let did_index_dir = dir.join(plcbundle::constants::DID_INDEX_DIR).join(plcbundle::constants::DID_INDEX_SHARDS); 333 333 + let did_index_dir = dir 334 334 + .join(plcbundle::constants::DID_INDEX_DIR) 335 335 + .join(plcbundle::constants::DID_INDEX_SHARDS); 306 336 did_index_dir.exists() && did_index_dir.is_dir() 307 337 } 308 338

+16 -6

src/cli/cmd_sync.rs

reviewed

··· 5 5 use plcbundle::{ 6 6 constants, 7 7 plc_client::PLCClient, 8 8 - sync::{SyncLoggerImpl, SyncConfig, SyncManager}, 9 8 runtime::BundleRuntime, 9 9 + sync::{SyncConfig, SyncLoggerImpl, SyncManager}, 10 10 }; 11 11 use std::path::PathBuf; 12 12 use std::sync::Arc; ··· 76 76 /// Maximum bundles to fetch (0 = all, only for one-time sync) 77 77 #[arg(long, default_value = "0", conflicts_with = "continuous")] 78 78 pub max_bundles: usize, 79 79 - 80 79 } 81 80 82 81 impl HasGlobalFlags for SyncCommand { 83 83 - fn verbose(&self) -> bool { false } 84 84 - fn quiet(&self) -> bool { false } 82 82 + fn verbose(&self) -> bool { 83 83 + false 84 84 + } 85 85 + fn quiet(&self) -> bool { 86 86 + false 87 87 + } 85 88 } 86 89 87 90 pub fn run(cmd: SyncCommand, dir: PathBuf, global_quiet: bool, global_verbose: bool) -> Result<()> { ··· 95 98 } 96 99 97 100 let client = PLCClient::new(&cmd.plc)?; 98 98 - let manager = Arc::new(super::utils::create_manager(dir.clone(), global_verbose, global_quiet, false)?); 101 101 + let manager = Arc::new(super::utils::create_manager( 102 102 + dir.clone(), 103 103 + global_verbose, 104 104 + global_quiet, 105 105 + false, 106 106 + )?); 99 107 100 108 let config = SyncConfig { 101 109 plc_url: cmd.plc.clone(), ··· 145 153 } 146 154 147 155 // Use common shutdown cleanup handler (no tasks to clean up for sync) 148 148 - runtime.wait_for_shutdown_cleanup::<()>("Sync", None, None).await; 156 156 + runtime 157 157 + .wait_for_shutdown_cleanup::<()>("Sync", None, None) 158 158 + .await; 149 159 150 160 Ok(()) 151 161 } else {

+25 -24

src/cli/cmd_verify.rs

reviewed

··· 1 1 use super::progress::ProgressBar; 2 2 - use super::utils::{format_bytes, format_bytes_per_sec, format_number, HasGlobalFlags}; 2 2 + use super::utils::{HasGlobalFlags, format_bytes, format_bytes_per_sec, format_number}; 3 3 use anyhow::{Result, bail}; 4 4 use clap::Args; 5 5 use plcbundle::{BundleManager, VerifyResult, VerifySpec}; ··· 69 69 } 70 70 71 71 impl HasGlobalFlags for VerifyCommand { 72 72 - fn verbose(&self) -> bool { false } 73 73 - fn quiet(&self) -> bool { false } 72 72 + fn verbose(&self) -> bool { 73 73 + false 74 74 + } 75 75 + fn quiet(&self) -> bool { 76 76 + false 77 77 + } 74 78 } 75 79 76 80 pub fn run(cmd: VerifyCommand, dir: PathBuf, global_verbose: bool) -> Result<()> { ··· 101 105 if let Some(bundles_str) = cmd.bundles { 102 106 let last_bundle = manager.get_last_bundle(); 103 107 let bundle_nums = super::utils::parse_bundle_spec(Some(bundles_str), last_bundle)?; 104 104 - 108 108 + 105 109 if bundle_nums.len() == 1 { 106 110 verify_single_bundle(&manager, bundle_nums[0], global_verbose, cmd.full, cmd.fast)?; 107 111 } else { ··· 388 392 } 389 393 390 394 // Provide helpful hint for common issues 391 391 - let has_hash_mismatch = errors.iter().any(|e| { 392 392 - e.contains("hash") && e.contains("mismatch") 393 393 - }); 395 395 + let has_hash_mismatch = errors 396 396 + .iter() 397 397 + .any(|e| e.contains("hash") && e.contains("mismatch")); 394 398 if has_hash_mismatch { 395 399 eprintln!( 396 400 " 💡 Hint: Bundle file may have been migrated but index wasn't updated." ··· 404 408 405 409 // Store first error for summary 406 410 if first_error.is_none() 407 407 - && let Some(first_err) = errors.first() { 408 408 - first_error = Some(anyhow::anyhow!("{}", first_err)); 411 411 + && let Some(first_err) = errors.first() 412 412 + { 413 413 + first_error = Some(anyhow::anyhow!("{}", first_err)); 409 414 } 410 415 } else { 411 416 verified_count += 1; ··· 531 536 } 532 537 533 538 if total_uncompressed_size > 0 { 534 534 - let bytes_per_sec_uncompressed = total_uncompressed_size as f64 / elapsed.as_secs_f64(); 539 539 + let bytes_per_sec_uncompressed = 540 540 + total_uncompressed_size as f64 / elapsed.as_secs_f64(); 535 541 eprintln!( 536 542 " Data rate: {} (uncompressed)", 537 543 format_bytes_per_sec(bytes_per_sec_uncompressed) ··· 543 549 eprintln!("\n❌ Chain verification failed"); 544 550 eprintln!(" Verified: {}/{} bundles", verified_count, bundles.len()); 545 551 eprintln!(" Errors: {}", error_count); 546 546 - 552 552 + 547 553 // Show failed bundles with their error messages 548 554 print_failed_bundles(&failed_bundles, 10); 549 549 - 555 555 + 550 556 eprintln!(" Time: {:?}", elapsed); 551 557 552 558 // Show helpful hint if hash mismatch detected ··· 582 588 num_threads: usize, 583 589 ) -> Result<()> { 584 590 let use_parallel = num_threads > 1; 585 585 - 591 591 + 586 592 eprintln!("\n🔬 Verifying bundles {} - {}", start, end); 587 593 if use_parallel { 588 594 eprintln!(" Using {} worker thread(s)", num_threads); ··· 593 599 let overall_start = Instant::now(); 594 600 595 601 let verify_err = if use_parallel { 596 596 - verify_range_parallel( 597 597 - manager, 598 598 - start, 599 599 - end, 600 600 - num_threads, 601 601 - verbose, 602 602 - full, 603 603 - fast, 604 604 - ) 602 602 + verify_range_parallel(manager, start, end, num_threads, verbose, full, fast) 605 603 } else { 606 604 verify_range_sequential(manager, start, end, total as usize, verbose, full, fast) 607 605 }; ··· 763 761 } 764 762 765 763 /// Print failed bundles with their error messages 766 766 - /// 764 764 + /// 767 765 /// # Arguments 768 766 /// * `failed_bundles` - Vector of (bundle_num, errors) tuples 769 767 /// * `threshold` - Maximum number of bundles to list in full before truncating (e.g., 10 or 20) ··· 804 802 } 805 803 } 806 804 } 807 807 - eprintln!(" ... and {} more failed bundles", failed_bundles.len() - 5); 805 805 + eprintln!( 806 806 + " ... and {} more failed bundles", 807 807 + failed_bundles.len() - 5 808 808 + ); 808 809 } 809 810 }

+7 -4

src/cli/mod.rs

reviewed

··· 7 7 mod cmd_bench; 8 8 mod cmd_clean; 9 9 mod cmd_clone; 10 10 - mod cmd_completions; 11 10 mod cmd_compare; 11 11 + mod cmd_completions; 12 12 mod cmd_did; 13 13 mod cmd_export; 14 14 mod cmd_index; ··· 195 195 "{all-args}\n\n", 196 196 "Examples:\n", 197 197 $examples 198 198 - ).replace("{bin}", BIN) 198 198 + ) 199 199 + .replace("{bin}", BIN) 199 200 }}; 200 201 201 202 (before: $before:literal, examples: $examples:literal) => {{ 202 203 const BIN: &str = env!("CARGO_PKG_NAME"); 203 204 concat!( 204 205 "{about-with-newline}\n", 205 205 - $before, "\n\n", 206 206 + $before, 207 207 + "\n\n", 206 208 "{usage-heading} {usage}\n\n", 207 209 "{all-args}\n\n", 208 210 "Examples:\n", 209 211 $examples 210 210 - ).replace("{bin}", BIN) 212 212 + ) 213 213 + .replace("{bin}", BIN) 211 214 }}; 212 215 }

+29 -23

src/cli/progress.rs

reviewed

··· 1 1 - use plcbundle::format::format_bytes_per_sec; 2 1 use indicatif::{ProgressBar as IndicatifProgressBar, ProgressStyle}; 2 2 + use plcbundle::format::format_bytes_per_sec; 3 3 use std::sync::Arc; 4 4 use std::sync::Mutex; 5 5 ··· 24 24 .unwrap() 25 25 .progress_chars("█▓▒░ "), 26 26 ); 27 27 - 27 27 + 28 28 Self { 29 29 pb, 30 30 show_bytes: false, ··· 35 35 /// Create a progress bar with byte tracking 36 36 pub fn with_bytes(total: usize, _total_bytes: u64) -> Self { 37 37 let pb = IndicatifProgressBar::new(total as u64); 38 38 - 38 38 + 39 39 // Custom template that shows data rate calculated from our tracked bytes 40 40 // We'll calculate bytes/sec manually and include it in the message 41 41 pb.set_style( ··· 44 44 .unwrap() 45 45 .progress_chars("█▓▒░ "), 46 46 ); 47 47 - 47 47 + 48 48 Self { 49 49 pb, 50 50 show_bytes: true, ··· 64 64 *bytes_guard = bytes; 65 65 let current_msg = self.pb.message().to_string(); 66 66 drop(bytes_guard); 67 67 - 67 67 + 68 68 // Update message to include data rate, preserving user message if present 69 69 let elapsed = self.pb.elapsed().as_secs_f64(); 70 70 let bytes_guard = self.current_bytes.lock().unwrap(); 71 71 let bytes = *bytes_guard; 72 72 drop(bytes_guard); 73 73 - 73 73 + 74 74 let bytes_per_sec = if elapsed > 0.0 { 75 75 bytes as f64 / elapsed 76 76 } else { 77 77 0.0 78 78 }; 79 79 - 79 79 + 80 80 // Extract user message (everything after " | " if present) 81 81 let user_msg = if let Some(pos) = current_msg.find(" | ") { 82 82 &current_msg[pos + 3..] 83 83 } else { 84 84 "" 85 85 }; 86 86 - 86 86 + 87 87 let new_msg = if user_msg.is_empty() { 88 88 format_bytes_per_sec(bytes_per_sec) 89 89 } else { ··· 100 100 let bytes_guard = self.current_bytes.lock().unwrap(); 101 101 let bytes = *bytes_guard; 102 102 drop(bytes_guard); 103 103 - 103 103 + 104 104 let bytes_per_sec = if elapsed > 0.0 { 105 105 bytes as f64 / elapsed 106 106 } else { 107 107 0.0 108 108 }; 109 109 - 109 109 + 110 110 let new_msg = if msg_str.is_empty() { 111 111 format_bytes_per_sec(bytes_per_sec) 112 112 } else { ··· 138 138 139 139 impl TwoStageProgress { 140 140 /// Create a new two-stage progress bar 141 141 - /// 141 141 + /// 142 142 /// # Arguments 143 143 /// * `last_bundle` - Total number of bundles to process in stage 1 144 144 /// * `total_bytes` - Total uncompressed bytes to process in stage 1 145 145 pub fn new(last_bundle: u32, total_bytes: u64) -> Self { 146 146 use std::sync::atomic::AtomicBool; 147 147 - 147 147 + 148 148 Self { 149 149 - stage1_progress: Arc::new(Mutex::new(Some(ProgressBar::with_bytes(last_bundle as usize, total_bytes)))), 149 149 + stage1_progress: Arc::new(Mutex::new(Some(ProgressBar::with_bytes( 150 150 + last_bundle as usize, 151 151 + total_bytes, 152 152 + )))), 150 153 stage2_progress: Arc::new(Mutex::new(None)), 151 154 stage2_started: Arc::new(Mutex::new(false)), 152 155 stage1_finished: Arc::new(Mutex::new(false)), ··· 161 164 } 162 165 163 166 /// Create a callback function for build_did_index (signature: Fn(u32, u32, u64, u64)) 164 164 - pub fn callback_for_build_did_index(&self) -> impl Fn(u32, u32, u64, u64) + Send + Sync + 'static { 167 167 + pub fn callback_for_build_did_index( 168 168 + &self, 169 169 + ) -> impl Fn(u32, u32, u64, u64) + Send + Sync + 'static { 165 170 let progress = self.clone_for_callback(); 166 171 move |current, total, bytes_processed, _total_bytes| { 167 172 progress.update_progress(current, total, bytes_processed); ··· 206 211 impl TwoStageProgressCallback { 207 212 fn update_progress(&self, current: u32, total: u32, bytes_processed: u64) { 208 213 use std::sync::atomic::Ordering; 209 209 - 214 214 + 210 215 // Stop updating progress bar if interrupted 211 216 if self.interrupted.load(Ordering::Relaxed) { 212 217 return; 213 218 } 214 214 - 219 219 + 215 220 // Detect stage change: if total changes from bundle count to shard count (256), we're in stage 2 216 221 let is_stage_2 = total == 256 && current <= 256; 217 217 - 222 222 + 218 223 if is_stage_2 { 219 224 // Check if this is the first time we're entering stage 2 220 225 let mut started = self.stage2_started.lock().unwrap(); 221 226 if !*started { 222 227 *started = true; 223 228 drop(started); 224 224 - 229 229 + 225 230 // Finish Stage 1 progress bar if not already finished 226 231 // (did_index.rs already printed empty line + Stage 2 header before this callback) 227 232 let mut finished = self.stage1_finished.lock().unwrap(); ··· 234 239 // Add extra newline after Stage 1 (did_index.rs already printed one before Stage 2 header) 235 240 eprintln!(); 236 241 } 237 237 - 242 242 + 238 243 // Create new simple progress bar for Stage 2 (256 shards, no byte tracking) 239 244 let mut stage2_pb = self.stage2_progress.lock().unwrap(); 240 245 *stage2_pb = Some(ProgressBar::new(256)); 241 246 } 242 242 - 247 247 + 243 248 // Update Stage 2 progress bar (pos/len already shows the count, no message needed) 244 249 let mut stage2_pb_guard = self.stage2_progress.lock().unwrap(); 245 250 if let Some(ref pb) = *stage2_pb_guard { 246 251 pb.set(current as usize); 247 252 // Don't set message - progress bar template will show pos/len without extra message 248 248 - 253 253 + 249 254 // Finish progress bar when stage 2 completes (256/256) 250 255 if current == 256 251 251 - && let Some(pb) = stage2_pb_guard.take() { 252 252 - pb.finish(); 256 256 + && let Some(pb) = stage2_pb_guard.take() 257 257 + { 258 258 + pb.finish(); 253 259 } 254 260 } 255 261 } else {

+29 -15

src/cli/utils.rs

reviewed

··· 9 9 pub mod colors { 10 10 /// Standard green color (used for success, matches, etc.) 11 11 pub const GREEN: &str = "\x1b[32m"; 12 12 - 12 12 + 13 13 /// Standard red color (used for errors, deletions, etc.) 14 14 pub const RED: &str = "\x1b[31m"; 15 15 - 15 15 + 16 16 /// Reset color code 17 17 pub const RESET: &str = "\x1b[0m"; 18 18 - 18 18 + 19 19 /// Dim/bright black color (used for context, unchanged lines, etc.) 20 20 pub const DIM: &str = "\x1b[2m"; 21 21 } ··· 76 76 } 77 77 } 78 78 79 79 - 80 79 /// Display path resolving "." to absolute path 81 80 /// Per RULES.md: NEVER display "." in user-facing output 82 81 pub fn display_path(path: &Path) -> PathBuf { ··· 110 109 /// 111 110 /// This is the standard way to create a BundleManager from CLI commands. 112 111 /// It respects the verbose and quiet flags for logging. 113 113 - /// 112 112 + /// 114 113 /// # Arguments 115 114 /// * `dir` - Directory path 116 115 /// * `verbose` - Enable verbose logging 117 116 /// * `_quiet` - Quiet mode (currently unused) 118 117 /// * `preload_mempool` - If true, preload mempool at initialization (for commands that need it) 119 119 - pub fn create_manager(dir: PathBuf, verbose: bool, _quiet: bool, preload_mempool: bool) -> Result<BundleManager> { 118 118 + pub fn create_manager( 119 119 + dir: PathBuf, 120 120 + verbose: bool, 121 121 + _quiet: bool, 122 122 + preload_mempool: bool, 123 123 + ) -> Result<BundleManager> { 120 124 use anyhow::Context; 121 121 - 125 125 + 122 126 // Check if directory exists 123 127 if !dir.exists() { 124 128 anyhow::bail!( ··· 128 132 plcbundle::constants::BINARY_NAME 129 133 ); 130 134 } 131 131 - 135 135 + 132 136 // Check if it's a bundle directory (has plc_bundles.json) 133 137 let index_path = dir.join("plc_bundles.json"); 134 138 if !index_path.exists() { ··· 139 143 plcbundle::constants::BINARY_NAME 140 144 ); 141 145 } 142 142 - 146 146 + 143 147 let display_dir = display_path(&dir); 144 148 let options = plcbundle::ManagerOptions { 145 149 handle_resolver_url: None, 146 150 preload_mempool, 147 151 verbose, 148 152 }; 149 149 - let manager = BundleManager::new(dir, options) 150 150 - .with_context(|| format!("Failed to load bundle repository from: {}", display_dir.display()))?; 153 153 + let manager = BundleManager::new(dir, options).with_context(|| { 154 154 + format!( 155 155 + "Failed to load bundle repository from: {}", 156 156 + display_dir.display() 157 157 + ) 158 158 + })?; 151 159 152 160 Ok(manager) 153 161 } ··· 156 164 /// 157 165 /// Convenience function for commands that implement `HasGlobalFlags`. 158 166 /// The global flags (verbose, quiet) are automatically extracted from the command. 159 159 - pub fn create_manager_from_cmd<C: HasGlobalFlags>(dir: PathBuf, cmd: &C, preload_mempool: bool) -> Result<BundleManager> { 167 167 + pub fn create_manager_from_cmd<C: HasGlobalFlags>( 168 168 + dir: PathBuf, 169 169 + cmd: &C, 170 170 + preload_mempool: bool, 171 171 + ) -> Result<BundleManager> { 160 172 create_manager(dir, cmd.verbose(), cmd.quiet(), preload_mempool) 161 173 } 162 174 ··· 178 190 } 179 191 } else { 180 192 // Otherwise treat as file path. For now, this is an error as direct file access is disallowed. 181 181 - anyhow::bail!("Loading from arbitrary paths not yet implemented. Please specify a bundle number."); 193 193 + anyhow::bail!( 194 194 + "Loading from arbitrary paths not yet implemented. Please specify a bundle number." 195 195 + ); 182 196 } 183 197 } 184 198 ··· 197 211 pub fn get_free_disk_space(path: &Path) -> Option<u64> { 198 212 use std::ffi::CString; 199 213 use std::os::unix::ffi::OsStrExt; 200 200 - 214 214 + 201 215 let c_path = match CString::new(path.as_os_str().as_bytes()) { 202 216 Ok(p) => p, 203 217 Err(_) => return None, 204 218 }; 205 205 - 219 219 + 206 220 unsafe { 207 221 let mut stat: libc::statvfs = std::mem::zeroed(); 208 222 if libc::statvfs(c_path.as_ptr(), &mut stat) == 0 {

+23 -16

src/constants.rs

reviewed

··· 128 128 129 129 /// Calculate global position from bundle number and position within bundle 130 130 /// Global position = ((bundle_number - 1) * BUNDLE_SIZE) + position 131 131 - /// 131 131 + /// 132 132 /// # Examples 133 133 /// - Bundle 1, position 0 → global 0 134 134 /// - Bundle 1, position 9999 → global 9999 ··· 152 152 153 153 /// Convert global position to bundle number and position 154 154 /// Returns (bundle_number, position) where bundle_number is 1-indexed and position is 0-indexed 155 155 - /// 155 155 + /// 156 156 /// # Examples 157 157 /// - Global 0 → bundle 1, position 0 158 158 /// - Global 9999 → bundle 1, position 9999 ··· 205 205 fn test_bundle_position_to_global() { 206 206 // Bundle 1, position 0 → global 0 207 207 assert_eq!(bundle_position_to_global(1, 0), 0); 208 208 - 208 208 + 209 209 // Bundle 1, position 9999 → global 9999 210 210 assert_eq!(bundle_position_to_global(1, 9999), 9999); 211 211 - 211 211 + 212 212 // Bundle 2, position 0 → global 10000 213 213 assert_eq!(bundle_position_to_global(2, 0), 10000); 214 214 - 214 214 + 215 215 // Bundle 2, position 500 → global 10500 216 216 assert_eq!(bundle_position_to_global(2, 500), 10500); 217 217 - 217 217 + 218 218 // Bundle 3, position 42 → global 20042 219 219 assert_eq!(bundle_position_to_global(3, 42), 20042); 220 220 } ··· 223 223 fn test_bundle_position_to_global_edge_cases() { 224 224 // Bundle 0 (edge case, should handle gracefully) 225 225 assert_eq!(bundle_position_to_global(0, 0), 0); 226 226 - 226 226 + 227 227 // Large bundle numbers 228 228 assert_eq!(bundle_position_to_global(100, 0), 990000); 229 229 assert_eq!(bundle_position_to_global(100, 5000), 995000); ··· 241 241 fn test_mempool_position_to_global() { 242 242 // With last_bundle = 0, mempool position 0 → global 0 243 243 assert_eq!(mempool_position_to_global(0, 0), 0); 244 244 - 244 244 + 245 245 // With last_bundle = 1, mempool position 0 → global 10000 246 246 assert_eq!(mempool_position_to_global(1, 0), 10000); 247 247 - 247 247 + 248 248 // With last_bundle = 1, mempool position 42 → global 10042 249 249 assert_eq!(mempool_position_to_global(1, 42), 10042); 250 250 - 250 250 + 251 251 // With last_bundle = 2, mempool position 100 → global 20100 252 252 assert_eq!(mempool_position_to_global(2, 100), 20100); 253 253 } ··· 258 258 let (bundle, pos) = global_to_bundle_position(0); 259 259 assert_eq!(bundle, 1); 260 260 assert_eq!(pos, 0); 261 261 - 261 261 + 262 262 // Global 9999 → bundle 1, position 9999 263 263 let (bundle, pos) = global_to_bundle_position(9999); 264 264 assert_eq!(bundle, 1); 265 265 assert_eq!(pos, 9999); 266 266 - 266 266 + 267 267 // Global 10000 → bundle 2, position 0 268 268 let (bundle, pos) = global_to_bundle_position(10000); 269 269 assert_eq!(bundle, 2); 270 270 assert_eq!(pos, 0); 271 271 - 271 271 + 272 272 // Global 10500 → bundle 2, position 500 273 273 let (bundle, pos) = global_to_bundle_position(10500); 274 274 assert_eq!(bundle, 2); 275 275 assert_eq!(pos, 500); 276 276 - 276 276 + 277 277 // Global 20042 → bundle 3, position 42 278 278 let (bundle, pos) = global_to_bundle_position(20042); 279 279 assert_eq!(bundle, 3); ··· 288 288 let global = bundle_position_to_global(bundle, position); 289 289 let (bundle_back, pos_back) = global_to_bundle_position(global); 290 290 assert_eq!(bundle, bundle_back, "Bundle mismatch for global {}", global); 291 291 - assert_eq!(position, pos_back, "Position mismatch for global {}", global); 291 291 + assert_eq!( 292 292 + position, pos_back, 293 293 + "Position mismatch for global {}", 294 294 + global 295 295 + ); 292 296 } 293 297 } 294 298 } ··· 311 315 assert_eq!(DID_INDEX_DELTAS, "deltas"); 312 316 assert_eq!(DID_INDEX_CONFIG, "config.json"); 313 317 assert_eq!(DEFAULT_PLC_DIRECTORY_URL, "https://plc.directory"); 314 314 - assert_eq!(DEFAULT_HANDLE_RESOLVER_URL, "https://quickdid.smokesignal.tools"); 318 318 + assert_eq!( 319 319 + DEFAULT_HANDLE_RESOLVER_URL, 320 320 + "https://quickdid.smokesignal.tools" 321 321 + ); 315 322 assert_eq!(DEFAULT_ORIGIN, "local"); 316 323 assert_eq!(ZSTD_COMPRESSION_LEVEL, 1); 317 324 assert_eq!(DID_INDEX_FLUSH_INTERVAL, 64);

+302 -173

src/did_index.rs

reviewed

··· 3 3 use crate::constants; 4 4 use anyhow::{Context, Result}; 5 5 use memmap2::{Mmap, MmapMut, MmapOptions}; 6 6 + use rayon::prelude::*; 6 7 use serde::{Deserialize, Serialize}; 7 8 use std::collections::HashMap; 8 9 use std::collections::hash_map::DefaultHasher; ··· 13 14 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; 14 15 use std::sync::{Arc, RwLock}; 15 16 use std::time::{SystemTime, UNIX_EPOCH}; 16 16 - use rayon::prelude::*; 17 17 const DID_SHARD_COUNT: usize = 256; 18 18 const DID_PREFIX: &str = "did:plc:"; 19 19 const DID_IDENTIFIER_LEN: usize = 24; ··· 27 27 28 28 // ============================================================================ 29 29 // OpLocation - Packed 32-bit global position with nullified flag 30 30 - // 30 30 + // 31 31 // Storage format: 32 32 // - Bits 31-1: Global position (0-indexed across all bundles) 33 33 // - Bit 0: Nullified flag (1 if nullified, 0 otherwise) 34 34 - // 34 34 + // 35 35 // Global position = ((bundle - 1) * BUNDLE_SIZE) + position 36 36 // ============================================================================ 37 37 ··· 256 256 } 257 257 258 258 fn add(&mut self, identifier: String, loc: OpLocation) { 259 259 - self.entries 260 260 - .entry(identifier) 261 261 - .or_default() 262 262 - .push(loc); 259 259 + self.entries.entry(identifier).or_default().push(loc); 263 260 } 264 261 265 262 fn merge(&mut self, other: HashMap<String, Vec<OpLocation>>) { ··· 704 701 total_flush_time.as_secs_f64(), 705 702 total_flush_time.as_secs_f64() / duration.as_secs_f64() * 100.0, 706 703 (duration - total_write_time - total_flush_time).as_secs_f64(), 707 707 - (duration - total_write_time - total_flush_time).as_secs_f64() / duration.as_secs_f64() * 100.0 704 704 + (duration - total_write_time - total_flush_time).as_secs_f64() 705 705 + / duration.as_secs_f64() 706 706 + * 100.0 708 707 ); 709 708 log::debug!( 710 709 "[DID Index] Throughput: {:.0} entries/sec, {:.0} syscalls/sec (est.)", ··· 718 717 719 718 /// Process a single bundle and return entries grouped by shard 720 719 /// This is the shared implementation used by both parallel and sequential processing 721 721 - fn process_bundle_for_index( 722 722 - bundle_dir: &PathBuf, 723 723 - bundle_num: u32, 724 724 - ) -> ShardProcessResult { 720 720 + fn process_bundle_for_index(bundle_dir: &PathBuf, bundle_num: u32) -> ShardProcessResult { 725 721 use std::fs::File; 726 722 use std::io::BufReader; 727 723 ··· 746 742 747 743 /// Core bundle processing logic - processes lines from a reader and groups by shard 748 744 /// This shared function eliminates duplication between parallel and sequential processing 749 749 - fn process_bundle_lines<R: std::io::BufRead>( 750 750 - bundle_num: u32, 751 751 - reader: R, 752 752 - ) -> ShardProcessResult { 745 745 + fn process_bundle_lines<R: std::io::BufRead>(bundle_num: u32, reader: R) -> ShardProcessResult { 753 746 use sonic_rs::JsonValueTrait; 754 747 755 748 let mut shard_entries: HashMap<u8, Vec<(String, OpLocation)>> = HashMap::new(); ··· 772 765 773 766 // Parse only the fields we need (did and nullified) 774 767 if let Ok(value) = sonic_rs::from_str::<sonic_rs::Value>(&line) 775 775 - && let Some(did) = value.get("did").and_then(|v| v.as_str()) { 776 776 - let nullified = value.get("nullified") 768 768 + && let Some(did) = value.get("did").and_then(|v| v.as_str()) 769 769 + { 770 770 + let nullified = value 771 771 + .get("nullified") 777 772 .and_then(|v| v.as_bool()) 778 773 .unwrap_or(false); 779 774 780 780 - // Calculate shard directly from DID bytes (no String allocation) 781 781 - if let Some(shard_num) = calculate_shard_from_did(did) { 782 782 - // Only allocate String when we actually need to store it 783 783 - let identifier = &did[DID_PREFIX.len()..DID_PREFIX.len() + DID_IDENTIFIER_LEN]; 784 784 - let identifier = identifier.to_string(); 785 785 - 786 786 - let global_pos = crate::constants::bundle_position_to_global(bundle_num, position as usize) as u32; 787 787 - let loc = OpLocation::new(global_pos, nullified); 775 775 + // Calculate shard directly from DID bytes (no String allocation) 776 776 + if let Some(shard_num) = calculate_shard_from_did(did) { 777 777 + // Only allocate String when we actually need to store it 778 778 + let identifier = &did[DID_PREFIX.len()..DID_PREFIX.len() + DID_IDENTIFIER_LEN]; 779 779 + let identifier = identifier.to_string(); 780 780 + 781 781 + let global_pos = 782 782 + crate::constants::bundle_position_to_global(bundle_num, position as usize) 783 783 + as u32; 784 784 + let loc = OpLocation::new(global_pos, nullified); 788 785 789 789 - shard_entries 790 790 - .entry(shard_num) 791 791 - .or_default() 792 792 - .push((identifier, loc)); 793 793 - } 786 786 + shard_entries 787 787 + .entry(shard_num) 788 788 + .or_default() 789 789 + .push((identifier, loc)); 794 790 } 791 791 + } 795 792 796 793 position += 1; 797 794 if position >= constants::BUNDLE_SIZE as u16 { ··· 821 818 progress_callback: Option<F>, 822 819 num_threads: usize, 823 820 interrupted: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>, 824 824 - ) -> Result<(u64, u64, std::time::Duration, std::time::Duration)> // Returns (total_operations, bundles_processed, stage1_duration, stage2_duration) 821 821 + ) -> Result<(u64, u64, std::time::Duration, std::time::Duration)> 822 822 + // Returns (total_operations, bundles_processed, stage1_duration, stage2_duration) 825 823 where 826 824 F: Fn(u32, u32, u64, Option<String>) + Send + Sync, // (current, total, bytes_processed, stage) 827 825 { 828 828 - use std::time::Instant; 829 826 use std::sync::atomic::AtomicU64; 827 827 + use std::time::Instant; 830 828 831 829 let build_start = Instant::now(); 832 830 833 833 - log::debug!("[DID Index] Starting streaming build for {} bundles", last_bundle); 831 831 + log::debug!( 832 832 + "[DID Index] Starting streaming build for {} bundles", 833 833 + last_bundle 834 834 + ); 834 835 835 836 // Clear existing index 836 837 fs::create_dir_all(&self.shard_dir)?; 837 838 fs::remove_dir_all(&self.delta_dir).ok(); 838 838 - 839 839 + 839 840 // Cleanup any leftover temp files from previous interrupted builds 840 841 self.cleanup_temp_files()?; 841 841 - 842 842 + 842 843 // Set up cleanup guard to remove temp files on panic or early exit 843 844 struct CleanupGuard { 844 845 shard_dir: PathBuf, 845 846 } 846 846 - 847 847 + 847 848 impl Drop for CleanupGuard { 848 849 fn drop(&mut self) { 849 850 // Clean up temp files on drop (panic or normal exit) ··· 852 853 } 853 854 } 854 855 } 855 855 - 856 856 + 856 857 impl CleanupGuard { 857 858 fn cleanup(shard_dir: &Path) -> Result<()> { 858 859 if !shard_dir.exists() { ··· 873 874 Ok(()) 874 875 } 875 876 } 876 876 - 877 877 + 877 878 let _cleanup_guard = CleanupGuard { 878 879 shard_dir: self.shard_dir.clone(), 879 880 }; ··· 904 905 Err(e) => { 905 906 // Handler might already be set (e.g., by CLI layer) 906 907 // This is okay, but log it for debugging 907 907 - log::debug!("[DID Index] CTRL+C handler registration: {} (may already be set)", e); 908 908 + log::debug!( 909 909 + "[DID Index] CTRL+C handler registration: {} (may already be set)", 910 910 + e 911 911 + ); 908 912 } 909 913 } 910 914 } ··· 920 924 }; 921 925 922 926 if use_parallel { 923 923 - log::debug!("[DID Index] Using {} threads for parallel bundle processing", actual_threads); 927 927 + log::debug!( 928 928 + "[DID Index] Using {} threads for parallel bundle processing", 929 929 + actual_threads 930 930 + ); 924 931 rayon::ThreadPoolBuilder::new() 925 932 .num_threads(actual_threads) 926 933 .build_global() ··· 933 940 let bytes_processed = AtomicU64::new(0); 934 941 let mut flush_count = 0usize; 935 942 let mut total_flush_time = std::time::Duration::ZERO; 936 936 - 943 943 + 937 944 // Metrics tracking (aggregated every N bundles) 938 945 let metrics_interval = 100u32; // Log metrics every 100 bundles 939 946 let mut metrics_start = Instant::now(); ··· 944 951 // Process bundles in batches (parallel or sequential) 945 952 let batch_size = if use_parallel { 100 } else { 1 }; // Process 100 bundles at a time in parallel 946 953 let bundle_numbers: Vec<u32> = (1..=last_bundle).collect(); 947 947 - 954 954 + 948 955 for batch_start in (0..bundle_numbers.len()).step_by(batch_size) { 949 956 let batch_end = (batch_start + batch_size).min(bundle_numbers.len()); 950 957 let batch: Vec<u32> = bundle_numbers[batch_start..batch_end].to_vec(); 951 951 - 958 958 + 952 959 let batch_results: Vec<ShardProcessResult> = if use_parallel { 953 960 // Process batch in parallel 954 954 - batch.par_iter() 961 961 + batch 962 962 + .par_iter() 955 963 .map(|&bundle_num| Self::process_bundle_for_index(bundle_dir, bundle_num)) 956 964 .collect() 957 965 } else { 958 966 // Process batch sequentially (for metrics tracking) 959 967 // Use the shared process_bundle_for_index function to avoid code duplication 960 960 - batch.iter() 968 968 + batch 969 969 + .iter() 961 970 .map(|&bundle_num| { 962 971 let bundle_processing_start = Instant::now(); 963 972 let result = Self::process_bundle_for_index(bundle_dir, bundle_num); 964 973 metrics_bundle_total += bundle_processing_start.elapsed(); 965 965 - 974 974 + 966 975 // Track aggregate metrics from result 967 976 if let Ok((_, bundle_bytes, bundle_operations)) = &result { 968 977 metrics_bytes += bundle_bytes; 969 978 metrics_ops += bundle_operations; 970 979 } 971 971 - 980 980 + 972 981 result 973 982 }) 974 983 .collect() 975 984 }; 976 976 - 985 985 + 977 986 // Merge results from batch into main shard_entries 978 987 // Zip batch with results to track bundle numbers (rayon preserves order) 979 988 for (bundle_num, result) in batch.iter().zip(batch_results.iter()) { 980 989 let (mut batch_entries, batch_bytes, batch_ops) = match result { 981 990 Ok(r) => r.clone(), 982 982 - Err(e) => return Err(anyhow::anyhow!("Error processing bundle {}: {}", bundle_num, e)), 991 991 + Err(e) => { 992 992 + return Err(anyhow::anyhow!( 993 993 + "Error processing bundle {}: {}", 994 994 + bundle_num, 995 995 + e 996 996 + )); 997 997 + } 983 998 }; 984 999 bytes_processed.fetch_add(batch_bytes, Ordering::Relaxed); 985 985 - 1000 1000 + 986 1001 // Track basic metrics from batch results (works in both parallel and sequential mode) 987 1002 metrics_ops += batch_ops; 988 1003 metrics_bytes += batch_bytes; ··· 994 1009 batch_bytes, 995 1010 total_operations 996 1011 ); 997 997 - 1012 1012 + 998 1013 // Merge batch entries into main shard_entries 999 1014 for (shard_num, mut entries) in batch_entries.drain() { 1000 1000 - shard_entries.entry(shard_num) 1015 1015 + shard_entries 1016 1016 + .entry(shard_num) 1001 1017 .or_default() 1002 1018 .append(&mut entries); 1003 1019 } 1004 1004 - 1020 1020 + 1005 1021 // Update progress after each bundle (more granular updates) 1006 1022 let current_bytes = bytes_processed.load(Ordering::Relaxed); 1007 1023 if let Some(ref cb) = progress_callback { 1008 1008 - cb(*bundle_num, last_bundle, current_bytes, Some("Stage 1/2: Processing bundles".to_string())); 1024 1024 + cb( 1025 1025 + *bundle_num, 1026 1026 + last_bundle, 1027 1027 + current_bytes, 1028 1028 + Some("Stage 1/2: Processing bundles".to_string()), 1029 1029 + ); 1009 1030 } 1010 1010 - 1031 1031 + 1011 1032 // Flush to disk periodically to avoid excessive memory (check after each bundle) 1012 1033 if flush_interval > 0 && *bundle_num % flush_interval == 0 { 1013 1034 let flush_start = Instant::now(); 1014 1035 let mem_before = shard_entries.values().map(|v| v.len()).sum::<usize>(); 1015 1036 let shards_used = shard_entries.len(); 1016 1016 - 1037 1037 + 1017 1038 self.flush_shard_entries(&mut shard_entries)?; 1018 1039 flush_count += 1; 1019 1019 - 1040 1040 + 1020 1041 let flush_duration = flush_start.elapsed(); 1021 1042 total_flush_time += flush_duration; 1022 1022 - 1043 1043 + 1023 1044 log::info!( 1024 1045 "[DID Index] Flush #{}: {} entries across {} shards (bundle {}/{}), took {:.3}s", 1025 1046 flush_count, ··· 1031 1052 ); 1032 1053 } 1033 1054 } 1034 1034 - 1055 1055 + 1035 1056 // Check for metrics logging (using last bundle in batch) 1036 1057 let last_bundle_in_batch = batch.last().copied().unwrap_or(0); 1037 1058 if last_bundle_in_batch > 0 { 1038 1038 - 1039 1059 // Log detailed metrics every N bundles 1040 1040 - if last_bundle_in_batch % metrics_interval == 0 || last_bundle_in_batch == last_bundle { 1060 1060 + if last_bundle_in_batch % metrics_interval == 0 1061 1061 + || last_bundle_in_batch == last_bundle 1062 1062 + { 1041 1063 let metrics_duration = metrics_start.elapsed(); 1042 1064 let ops_per_sec = if metrics_duration.as_secs_f64() > 0.0 { 1043 1065 metrics_ops as f64 / metrics_duration.as_secs_f64() ··· 1049 1071 } else { 1050 1072 0.0 1051 1073 }; 1052 1052 - 1074 1074 + 1053 1075 log::info!( 1054 1076 "[DID Index] Metrics (bundles {}..{}): {} ops, {:.1} MB | {:.1} ops/sec, {:.1} MB/sec", 1055 1055 - last_bundle_in_batch.saturating_sub(metrics_interval - 1).max(1), 1077 1077 + last_bundle_in_batch 1078 1078 + .saturating_sub(metrics_interval - 1) 1079 1079 + .max(1), 1056 1080 last_bundle_in_batch, 1057 1081 metrics_ops, 1058 1082 metrics_bytes as f64 / 1_000_000.0, 1059 1083 ops_per_sec, 1060 1084 mb_per_sec 1061 1085 ); 1062 1062 - 1086 1086 + 1063 1087 // Show bundle processing time (simplified metrics) 1064 1088 if metrics_bundle_total.as_secs_f64() > 0.0 { 1065 1089 log::info!( ··· 1067 1091 metrics_bundle_total.as_secs_f64() * 1000.0 1068 1092 ); 1069 1093 } 1070 1070 - 1094 1094 + 1071 1095 // Reset metrics for next interval 1072 1096 metrics_start = Instant::now(); 1073 1097 metrics_ops = 0; ··· 1085 1109 self.flush_shard_entries(&mut shard_entries)?; 1086 1110 let final_flush_duration = final_flush_start.elapsed(); 1087 1111 total_flush_time += final_flush_duration; 1088 1088 - log::info!("[DID Index] Final flush completed in {:.3}s", final_flush_duration.as_secs_f64()); 1112 1112 + log::info!( 1113 1113 + "[DID Index] Final flush completed in {:.3}s", 1114 1114 + final_flush_duration.as_secs_f64() 1115 1115 + ); 1089 1116 } 1090 1117 1091 1118 // CRITICAL: Ensure all file handles are closed and data is synced to disk ··· 1106 1133 let use_parallel_consolidation = use_parallel; 1107 1134 // CRITICAL: Use usize range then map to u8, because DID_SHARD_COUNT (256) as u8 wraps to 0 1108 1135 let shard_numbers: Vec<u8> = (0..DID_SHARD_COUNT).map(|i| i as u8).collect(); 1109 1109 - 1136 1136 + 1110 1137 // Use atomic counter for progress tracking in parallel mode 1111 1138 let progress_counter = std::sync::atomic::AtomicU32::new(0); 1112 1139 let progress_cb = progress_callback.as_ref(); 1113 1113 - 1140 1140 + 1114 1141 let consolidation_results: Vec<Result<(u8, i64)>> = if use_parallel_consolidation { 1115 1142 // Process shards in parallel 1116 1143 // IMPORTANT: Use try_reduce or collect with explicit error handling 1117 1144 // to ensure all shards are processed even if some fail 1118 1118 - shard_numbers.par_iter() 1145 1145 + shard_numbers 1146 1146 + .par_iter() 1119 1147 .map(|&shard_num| { 1120 1120 - log::debug!("[DID Index] Parallel consolidation starting for shard {:02x}", shard_num); 1148 1148 + log::debug!( 1149 1149 + "[DID Index] Parallel consolidation starting for shard {:02x}", 1150 1150 + shard_num 1151 1151 + ); 1121 1152 // Wrap in explicit error handling to ensure we process all shards 1122 1153 let result = match self.consolidate_shard(shard_num) { 1123 1154 Ok(shard_did_count) => { 1124 1155 // Update progress atomically 1125 1156 if let Some(ref cb) = progress_cb { 1126 1157 let count = progress_counter.fetch_add(1, Ordering::Relaxed) + 1; 1127 1127 - cb(count, DID_SHARD_COUNT as u32, 0, Some("Stage 2/2: Consolidating shards".to_string())); 1158 1158 + cb( 1159 1159 + count, 1160 1160 + DID_SHARD_COUNT as u32, 1161 1161 + 0, 1162 1162 + Some("Stage 2/2: Consolidating shards".to_string()), 1163 1163 + ); 1128 1164 } 1129 1129 - log::debug!("[DID Index] Shard {:02x} consolidation succeeded: {} DIDs", shard_num, shard_did_count); 1165 1165 + log::debug!( 1166 1166 + "[DID Index] Shard {:02x} consolidation succeeded: {} DIDs", 1167 1167 + shard_num, 1168 1168 + shard_did_count 1169 1169 + ); 1130 1170 Ok((shard_num, shard_did_count)) 1131 1171 } 1132 1172 Err(e) => { ··· 1138 1178 // Still update progress even on error 1139 1179 if let Some(ref cb) = progress_cb { 1140 1180 let count = progress_counter.fetch_add(1, Ordering::Relaxed) + 1; 1141 1141 - cb(count, DID_SHARD_COUNT as u32, 0, Some("Stage 2/2: Consolidating shards".to_string())); 1181 1181 + cb( 1182 1182 + count, 1183 1183 + DID_SHARD_COUNT as u32, 1184 1184 + 0, 1185 1185 + Some("Stage 2/2: Consolidating shards".to_string()), 1186 1186 + ); 1142 1187 } 1143 1188 // Return 0 DIDs for failed shard, but don't fail the entire operation 1144 1189 log::debug!( ··· 1148 1193 Ok((shard_num, 0)) 1149 1194 } 1150 1195 }; 1151 1151 - log::debug!("[DID Index] Parallel consolidation result for shard {:02x}: {:?}", shard_num, result); 1196 1196 + log::debug!( 1197 1197 + "[DID Index] Parallel consolidation result for shard {:02x}: {:?}", 1198 1198 + shard_num, 1199 1199 + result 1200 1200 + ); 1152 1201 result 1153 1202 }) 1154 1203 .collect() 1155 1204 } else { 1156 1205 // Process shards sequentially with progress updates 1157 1206 log::debug!("[DID Index] Using sequential consolidation"); 1158 1158 - shard_numbers.iter() 1207 1207 + shard_numbers 1208 1208 + .iter() 1159 1209 .map(|&shard_num| { 1160 1160 - log::debug!("[DID Index] Sequential consolidation starting for shard {:02x}", shard_num); 1210 1210 + log::debug!( 1211 1211 + "[DID Index] Sequential consolidation starting for shard {:02x}", 1212 1212 + shard_num 1213 1213 + ); 1161 1214 // Update progress callback for consolidation phase 1162 1215 if let Some(ref cb) = progress_callback { 1163 1216 // For consolidation, we use shard number as progress indicator 1164 1217 // Total is DID_SHARD_COUNT, current is shard_num + 1 1165 1165 - cb((shard_num + 1) as u32, DID_SHARD_COUNT as u32, 0, Some("Stage 2/2: Consolidating shards".to_string())); 1218 1218 + cb( 1219 1219 + (shard_num + 1) as u32, 1220 1220 + DID_SHARD_COUNT as u32, 1221 1221 + 0, 1222 1222 + Some("Stage 2/2: Consolidating shards".to_string()), 1223 1223 + ); 1166 1224 } 1167 1225 let result = self.consolidate_shard(shard_num); 1168 1168 - log::debug!("[DID Index] Sequential consolidation result for shard {:02x}: {:?}", shard_num, result); 1226 1226 + log::debug!( 1227 1227 + "[DID Index] Sequential consolidation result for shard {:02x}: {:?}", 1228 1228 + shard_num, 1229 1229 + result 1230 1230 + ); 1169 1231 match result { 1170 1232 Ok(shard_did_count) => Ok((shard_num, shard_did_count)), 1171 1233 Err(e) => { 1172 1172 - log::warn!("[DID Index] Sequential consolidation failed for shard {:02x}: {}", shard_num, e); 1234 1234 + log::warn!( 1235 1235 + "[DID Index] Sequential consolidation failed for shard {:02x}: {}", 1236 1236 + shard_num, 1237 1237 + e 1238 1238 + ); 1173 1239 Ok((shard_num, 0)) 1174 1240 } 1175 1241 } ··· 1192 1258 shard_did_count, 1193 1259 total_dids 1194 1260 ); 1195 1195 - 1261 1261 + 1196 1262 // Update shard metadata with did_count 1197 1263 self.modify_config(|config| { 1198 1264 if let Some(shard_meta) = config.shards.get_mut(shard_num as usize) { ··· 1200 1266 } 1201 1267 })?; 1202 1268 } 1203 1203 - 1269 1269 + 1204 1270 log::debug!( 1205 1271 "[DID Index] Final total_dids after collecting all shard results: {}", 1206 1272 total_dids 1207 1273 ); 1208 1208 - 1274 1274 + 1209 1275 // Explicitly cleanup temp files on successful completion 1210 1276 // (Drop guard will also handle this, but explicit cleanup ensures it happens) 1211 1277 self.cleanup_temp_files()?; ··· 1239 1305 total_duration.as_secs_f64() 1240 1306 ); 1241 1307 1242 1242 - Ok((total_operations, last_bundle as u64, stage1_duration, pass2_duration)) 1308 1308 + Ok(( 1309 1309 + total_operations, 1310 1310 + last_bundle as u64, 1311 1311 + stage1_duration, 1312 1312 + pass2_duration, 1313 1313 + )) 1243 1314 } 1244 1315 1245 1316 // Update index for new bundle (incremental) ··· 1273 1344 1274 1345 valid_dids += 1; 1275 1346 let shard_num = self.calculate_shard(&identifier); 1276 1276 - let global_pos = crate::constants::bundle_position_to_global(bundle_num, position) as u32; 1347 1347 + let global_pos = 1348 1348 + crate::constants::bundle_position_to_global(bundle_num, position) as u32; 1277 1349 let loc = OpLocation::new(global_pos, *nullified); 1278 1350 1279 1351 shard_ops ··· 1473 1545 /// Returns (intact_shards, corrupted_shards, missing_shards) 1474 1546 pub fn verify_base_shard_integrity(&self) -> Result<(Vec<u8>, Vec<u8>, Vec<u8>)> { 1475 1547 use sha2::{Digest, Sha256}; 1476 1476 - 1548 1548 + 1477 1549 let config = self.config.read().unwrap(); 1478 1550 let mut intact = Vec::new(); 1479 1551 let mut corrupted = Vec::new(); 1480 1552 let mut missing = Vec::new(); 1481 1481 - 1553 1553 + 1482 1554 for shard_num in 0..DID_SHARD_COUNT { 1483 1555 let shard_meta = config.shards.get(shard_num).unwrap(); 1484 1556 let shard_path = self.shard_path(shard_num as u8); 1485 1485 - 1557 1557 + 1486 1558 if !shard_path.exists() { 1487 1559 if shard_meta.did_count > 0 { 1488 1560 missing.push(shard_num as u8); 1489 1561 } 1490 1562 continue; 1491 1563 } 1492 1492 - 1564 1564 + 1493 1565 // If no hash stored, we can't verify (old index or empty shard) 1494 1566 if let Some(expected_hash) = &shard_meta.base_shard_hash { 1495 1567 let file_data = match fs::read(&shard_path) { ··· 1499 1571 continue; 1500 1572 } 1501 1573 }; 1502 1502 - 1574 1574 + 1503 1575 let mut hasher = Sha256::new(); 1504 1576 hasher.update(&file_data); 1505 1577 let actual_hash = format!("{:x}", hasher.finalize()); 1506 1506 - 1578 1578 + 1507 1579 if actual_hash == *expected_hash { 1508 1580 intact.push(shard_num as u8); 1509 1581 } else { ··· 1515 1587 intact.push(shard_num as u8); 1516 1588 } 1517 1589 } 1518 1518 - 1590 1590 + 1519 1591 Ok((intact, corrupted, missing)) 1520 1592 } 1521 1593 ··· 2048 2120 struct TempFileGuard { 2049 2121 path: PathBuf, 2050 2122 } 2051 2051 - 2123 2123 + 2052 2124 impl Drop for TempFileGuard { 2053 2125 fn drop(&mut self) { 2054 2054 - if self.path.exists() && let Err(e) = fs::remove_file(&self.path) { 2055 2055 - log::warn!("[DID Index] Failed to remove temp file {}: {}", self.path.display(), e); 2126 2126 + if self.path.exists() 2127 2127 + && let Err(e) = fs::remove_file(&self.path) 2128 2128 + { 2129 2129 + log::warn!( 2130 2130 + "[DID Index] Failed to remove temp file {}: {}", 2131 2131 + self.path.display(), 2132 2132 + e 2133 2133 + ); 2056 2134 } 2057 2135 } 2058 2136 } 2059 2059 - 2137 2137 + 2060 2138 let _temp_guard = TempFileGuard { 2061 2139 path: temp_path.clone(), 2062 2140 }; ··· 2175 2253 // Parse entries (28 bytes each) 2176 2254 let parse_start = Instant::now(); 2177 2255 let entry_count = data.len() / 28; 2178 2178 - 2256 2256 + 2179 2257 if entry_count == 0 { 2180 2258 log::warn!( 2181 2259 "[DID Index] Shard {:02x} consolidate: temp file {} has {} bytes but no complete entries (need at least 28 bytes per entry)", ··· 2187 2265 return Ok(0); 2188 2266 } 2189 2267 // Store (identifier_string, raw_bytes, location) so we can preserve exact bytes 2190 2190 - let mut entries: Vec<(String, [u8; DID_IDENTIFIER_LEN], OpLocation)> = Vec::with_capacity(entry_count); 2268 2268 + let mut entries: Vec<(String, [u8; DID_IDENTIFIER_LEN], OpLocation)> = 2269 2269 + Vec::with_capacity(entry_count); 2191 2270 2192 2271 for i in 0..entry_count { 2193 2272 let offset = i * 28; ··· 2202 2281 let identifier_bytes = &data[offset..offset + DID_IDENTIFIER_LEN]; 2203 2282 // Create a string from the bytes for HashMap key (lossy conversion handles invalid UTF-8) 2204 2283 let identifier = String::from_utf8_lossy(identifier_bytes).to_string(); 2205 2205 - 2284 2284 + 2206 2285 // Preserve the exact raw bytes for writing back 2207 2286 let mut raw_bytes = [0u8; DID_IDENTIFIER_LEN]; 2208 2287 raw_bytes.copy_from_slice(identifier_bytes); 2209 2209 - 2288 2288 + 2210 2289 let loc_bytes = [ 2211 2290 data[offset + 24], 2212 2291 data[offset + 25], ··· 2238 2317 // Write final shard, using raw bytes when available 2239 2318 let write_start = Instant::now(); 2240 2319 let target = self.shard_path(shard_num); 2241 2241 - self.write_shard_to_path_with_bytes(shard_num, &builder, &target, "base", Some(&identifier_to_bytes))?; 2320 2320 + self.write_shard_to_path_with_bytes( 2321 2321 + shard_num, 2322 2322 + &builder, 2323 2323 + &target, 2324 2324 + "base", 2325 2325 + Some(&identifier_to_bytes), 2326 2326 + )?; 2242 2327 let write_duration = write_start.elapsed(); 2243 2328 2244 2329 // Explicitly remove temp file immediately after successful write 2245 2330 // (Drop guard will also handle this, but explicit removal ensures immediate cleanup) 2246 2331 if let Err(e) = fs::remove_file(&temp_path) { 2247 2247 - log::warn!("[DID Index] Failed to remove temp file {}: {}", temp_path.display(), e); 2332 2332 + log::warn!( 2333 2333 + "[DID Index] Failed to remove temp file {}: {}", 2334 2334 + temp_path.display(), 2335 2335 + e 2336 2336 + ); 2248 2337 } 2249 2338 2250 2339 let total_duration = start.elapsed(); ··· 2456 2545 let start = Instant::now(); 2457 2546 2458 2547 // Ensure shard directory exists 2459 2459 - if let Some(parent) = target.parent() && !parent.exists() { 2548 2548 + if let Some(parent) = target.parent() 2549 2549 + && !parent.exists() 2550 2550 + { 2460 2551 fs::create_dir_all(parent)?; 2461 2552 } 2462 2553 ··· 2556 2647 for id in identifiers { 2557 2648 // Use raw bytes if available, otherwise normalize from string 2558 2649 let id_bytes = if let Some(bytes_map) = identifier_bytes { 2559 2559 - bytes_map.get(id).copied().unwrap_or_else(|| normalize_identifier_bytes(id)) 2650 2650 + bytes_map 2651 2651 + .get(id) 2652 2652 + .copied() 2653 2653 + .unwrap_or_else(|| normalize_identifier_bytes(id)) 2560 2654 } else { 2561 2655 normalize_identifier_bytes(id) 2562 2656 }; ··· 2601 2695 // Read from final file location to ensure we hash exactly what's on disk 2602 2696 let shard_hash = if label == "base" { 2603 2697 use sha2::{Digest, Sha256}; 2604 2604 - let file_data = fs::read(target) 2605 2605 - .with_context(|| format!("Failed to read shard file for hashing {}", target.display()))?; 2698 2698 + let file_data = fs::read(target).with_context(|| { 2699 2699 + format!("Failed to read shard file for hashing {}", target.display()) 2700 2700 + })?; 2606 2701 let mut hasher = Sha256::new(); 2607 2702 hasher.update(&file_data); 2608 2703 Some(format!("{:x}", hasher.finalize())) ··· 2728 2823 fn invalidate_shard_cache(&self, shard_num: u8) { 2729 2824 self.shard_cache.write().unwrap().remove(&shard_num); 2730 2825 } 2731 2731 - 2826 2826 + 2732 2827 /// Clean up temporary files from interrupted builds 2733 2828 pub fn cleanup_temp_files(&self) -> Result<()> { 2734 2829 if !self.shard_dir.exists() { ··· 2738 2833 for entry in fs::read_dir(&self.shard_dir)? { 2739 2834 let entry = entry?; 2740 2835 let path = entry.path(); 2741 2741 - if let Some(ext) = path.extension() && ext == "tmp" { 2836 2836 + if let Some(ext) = path.extension() 2837 2837 + && ext == "tmp" 2838 2838 + { 2742 2839 fs::remove_file(&path)?; 2743 2840 cleaned += 1; 2744 2841 } 2745 2842 } 2746 2843 if cleaned > 0 { 2747 2747 - log::info!("[DID Index] Cleaned up {} leftover temp files from previous build", cleaned); 2844 2844 + log::info!( 2845 2845 + "[DID Index] Cleaned up {} leftover temp files from previous build", 2846 2846 + cleaned 2847 2847 + ); 2748 2848 } 2749 2849 Ok(()) 2750 2850 } ··· 2757 2857 .get("last_bundle") 2758 2858 .and_then(|v| v.as_i64()) 2759 2859 .unwrap_or(0) as u32; 2760 2760 - 2860 2860 + 2761 2861 let mut errors = 0; 2762 2862 let mut warnings = 0; 2763 2863 let mut error_categories: Vec<(String, usize)> = Vec::new(); 2764 2764 - 2864 2864 + 2765 2865 // Check 1: Last bundle consistency 2766 2866 if index_last_bundle < last_bundle_in_repo { 2767 2867 warnings += 1; 2768 2868 } 2769 2769 - 2869 2869 + 2770 2870 // Check 2: Verify shard files exist 2771 2871 let shard_details = self.get_shard_details(None)?; 2772 2872 let mut missing_base_shards = 0; 2773 2873 let mut missing_delta_segments = 0; 2774 2874 let mut shards_checked = 0; 2775 2875 let mut segments_checked = 0; 2776 2776 - 2876 2876 + 2777 2877 for detail in &shard_details { 2778 2878 let base_exists = detail 2779 2879 .get("base_exists") ··· 2783 2883 .get("did_count") 2784 2884 .and_then(|v| v.as_u64()) 2785 2885 .unwrap_or(0); 2786 2786 - 2886 2886 + 2787 2887 if did_count > 0 { 2788 2888 shards_checked += 1; 2789 2889 if !base_exists { ··· 2791 2891 errors += 1; 2792 2892 } 2793 2893 } 2794 2794 - 2894 2894 + 2795 2895 if let Some(segments) = detail.get("segments").and_then(|v| v.as_array()) { 2796 2896 for seg in segments { 2797 2897 segments_checked += 1; ··· 2803 2903 } 2804 2904 } 2805 2905 } 2806 2806 - 2906 2906 + 2807 2907 if missing_base_shards > 0 { 2808 2908 error_categories.push(("Missing base shards".to_string(), missing_base_shards)); 2809 2909 } 2810 2910 if missing_delta_segments > 0 { 2811 2911 error_categories.push(("Missing delta segments".to_string(), missing_delta_segments)); 2812 2912 } 2813 2813 - 2913 2913 + 2814 2914 // Check 3: Verify index configuration 2815 2915 let shard_count = stats 2816 2916 .get("shard_count") 2817 2917 .and_then(|v| v.as_i64()) 2818 2918 .unwrap_or(0); 2819 2819 - 2919 2919 + 2820 2920 if shard_count != 256 { 2821 2921 warnings += 1; 2822 2922 } 2823 2823 - 2923 2923 + 2824 2924 // Check 4: Check delta segment accumulation 2825 2925 let delta_segments = stats 2826 2926 .get("delta_segments") 2827 2927 .and_then(|v| v.as_u64()) 2828 2928 .unwrap_or(0); 2829 2829 - 2929 2929 + 2830 2930 const SEGMENT_WARNING_THRESHOLD: u64 = 1280; 2831 2931 const SEGMENT_ERROR_THRESHOLD: u64 = 5120; 2832 2832 - 2932 2932 + 2833 2933 if delta_segments >= SEGMENT_ERROR_THRESHOLD { 2834 2934 errors += 1; 2835 2935 error_categories.push(("Too many delta segments".to_string(), 1)); 2836 2936 } else if delta_segments >= SEGMENT_WARNING_THRESHOLD { 2837 2937 warnings += 1; 2838 2938 } 2839 2839 - 2939 2939 + 2840 2940 Ok(VerifyResult { 2841 2941 errors, 2842 2942 warnings, ··· 2861 2961 .get("delta_segments") 2862 2962 .and_then(|v| v.as_u64()) 2863 2963 .unwrap_or(0); 2864 2864 - 2964 2964 + 2865 2965 let shard_details = self.get_shard_details(None)?; 2866 2966 let mut missing_base_shards = 0; 2867 2967 let mut missing_delta_segments = 0; 2868 2968 let mut missing_segment_bundles = std::collections::HashSet::new(); 2869 2869 - 2969 2969 + 2870 2970 for detail in &shard_details { 2871 2971 let base_exists = detail 2872 2972 .get("base_exists") ··· 2876 2976 .get("did_count") 2877 2977 .and_then(|v| v.as_u64()) 2878 2978 .unwrap_or(0); 2879 2879 - 2979 2979 + 2880 2980 if did_count > 0 && !base_exists { 2881 2981 missing_base_shards += 1; 2882 2982 } 2883 2883 - 2983 2983 + 2884 2984 if let Some(segments) = detail.get("segments").and_then(|v| v.as_array()) { 2885 2985 for seg in segments { 2886 2986 let exists = seg.get("exists").and_then(|v| v.as_bool()).unwrap_or(false); 2887 2987 if !exists { 2888 2988 missing_delta_segments += 1; 2889 2889 - if let Some(start) = seg.get("bundle_start").and_then(|v| v.as_u64()).map(|v| v as u32) 2890 2890 - && let Some(end) = seg.get("bundle_end").and_then(|v| v.as_u64()).map(|v| v as u32) { 2891 2891 - for bundle_num in start..=end { 2892 2892 - missing_segment_bundles.insert(bundle_num); 2893 2893 - } 2989 2989 + if let Some(start) = seg 2990 2990 + .get("bundle_start") 2991 2991 + .and_then(|v| v.as_u64()) 2992 2992 + .map(|v| v as u32) 2993 2993 + && let Some(end) = seg 2994 2994 + .get("bundle_end") 2995 2995 + .and_then(|v| v.as_u64()) 2996 2996 + .map(|v| v as u32) 2997 2997 + { 2998 2998 + for bundle_num in start..=end { 2999 2999 + missing_segment_bundles.insert(bundle_num); 2894 3000 } 3001 3001 + } 2895 3002 } 2896 3003 } 2897 3004 } 2898 3005 } 2899 2899 - 3006 3006 + 2900 3007 let missing_bundles = last_bundle_in_repo.saturating_sub(index_last_bundle); 2901 2901 - 2902 2902 - let needs_rebuild = index_last_bundle < last_bundle_in_repo 2903 2903 - || missing_base_shards > 0 3008 3008 + 3009 3009 + let needs_rebuild = index_last_bundle < last_bundle_in_repo 3010 3010 + || missing_base_shards > 0 2904 3011 || missing_delta_segments > 0; 2905 3012 let needs_compact = delta_segments > 50; 2906 2906 - 3013 3013 + 2907 3014 Ok(RepairInfo { 2908 3015 needs_rebuild, 2909 3016 needs_compact, ··· 2932 3039 use std::fs; 2933 3040 2934 3041 // Create temporary directory for rebuilt index 2935 2935 - let temp_dir = std::env::temp_dir().join(format!("plcbundle_verify_{}", std::process::id())); 3042 3042 + let temp_dir = 3043 3043 + std::env::temp_dir().join(format!("plcbundle_verify_{}", std::process::id())); 2936 3044 fs::create_dir_all(&temp_dir)?; 2937 3045 2938 3046 // Create temporary DID index ··· 2944 3052 last_bundle, 2945 3053 flush_interval, 2946 3054 progress_callback, 2947 2947 - 0, // num_threads: auto-detect 3055 3055 + 0, // num_threads: auto-detect 2948 3056 None, // interrupted flag 2949 3057 )?; 2950 3058 ··· 3066 3174 } 3067 3175 3068 3176 /// Repair index - intelligently rebuilds or updates as needed 3069 3069 - /// 3177 3177 + /// 3070 3178 /// `bundle_loader` is a callback that loads a bundle and returns operations as (did, nullified) pairs 3071 3179 /// Returns a RepairResult indicating what was done. If `repaired` is true but `bundles_processed` is 0, 3072 3180 /// it means a full rebuild is needed (caller should call build_from_scratch). 3073 3073 - pub fn repair<L>( 3074 3074 - &mut self, 3075 3075 - last_bundle: u32, 3076 3076 - bundle_loader: L, 3077 3077 - ) -> Result<RepairResult> 3181 3181 + pub fn repair<L>(&mut self, last_bundle: u32, bundle_loader: L) -> Result<RepairResult> 3078 3182 where 3079 3183 L: Fn(u32) -> Result<Vec<(String, bool)>>, // Load bundle and return operations 3080 3184 { 3081 3185 let repair_info = self.get_repair_info(last_bundle)?; 3082 3082 - 3186 3186 + 3083 3187 if !repair_info.needs_rebuild && !repair_info.needs_compact { 3084 3188 return Ok(RepairResult { 3085 3189 repaired: false, ··· 3088 3192 segments_rebuilt: 0, 3089 3193 }); 3090 3194 } 3091 3091 - 3195 3195 + 3092 3196 let mut segments_rebuilt = 0; 3093 3197 let mut bundles_processed = 0; 3094 3094 - 3198 3198 + 3095 3199 // Case 1: Rebuild if needed 3096 3200 if repair_info.needs_rebuild { 3097 3201 // Optimization: If only delta segments are missing, verify base shard integrity 3098 3202 // and rebuild only missing segments if base shards are intact 3099 3099 - let can_rebuild_segments_only = repair_info.missing_base_shards == 0 3100 3100 - && repair_info.missing_delta_segments > 0 3203 3203 + let can_rebuild_segments_only = repair_info.missing_base_shards == 0 3204 3204 + && repair_info.missing_delta_segments > 0 3101 3205 && repair_info.missing_bundles == 0; 3102 3102 - 3206 3206 + 3103 3207 if can_rebuild_segments_only { 3104 3208 let (_intact, corrupted, missing) = self.verify_base_shard_integrity()?; 3105 3105 - 3209 3209 + 3106 3210 if corrupted.is_empty() && missing.is_empty() { 3107 3211 // Rebuild only missing segments 3108 3212 let mut bundle_list = repair_info.missing_segment_bundles.clone(); 3109 3213 bundle_list.sort(); 3110 3110 - 3214 3214 + 3111 3215 if !bundle_list.is_empty() { 3112 3216 for bundle_num in &bundle_list { 3113 3217 let operations = bundle_loader(*bundle_num)?; 3114 3218 self.update_for_bundle(*bundle_num, operations)?; 3115 3219 } 3116 3116 - 3220 3220 + 3117 3221 bundles_processed = bundle_list.len() as u32; 3118 3222 segments_rebuilt = repair_info.missing_delta_segments; 3119 3223 } ··· 3129 3233 } 3130 3234 } else { 3131 3235 // Need full rebuild or incremental update 3132 3132 - let force_full_rebuild = repair_info.missing_base_shards > 0 || repair_info.missing_bundles > 1000; 3133 3133 - 3236 3236 + let force_full_rebuild = 3237 3237 + repair_info.missing_base_shards > 0 || repair_info.missing_bundles > 1000; 3238 3238 + 3134 3239 if force_full_rebuild || repair_info.missing_bundles > 1000 { 3135 3240 // Full rebuild - signal to caller 3136 3241 return Ok(RepairResult { ··· 3145 3250 let operations = bundle_loader(bundle_num)?; 3146 3251 self.update_for_bundle(bundle_num, operations)?; 3147 3252 } 3148 3148 - 3253 3253 + 3149 3254 bundles_processed = repair_info.missing_bundles; 3150 3255 } 3151 3256 } 3152 3257 } 3153 3153 - 3258 3258 + 3154 3259 // Case 2: Compact if needed 3155 3260 let compacted = if repair_info.needs_compact { 3156 3261 self.compact_pending_segments(None)?; ··· 3158 3263 } else { 3159 3264 false 3160 3265 }; 3161 3161 - 3266 3266 + 3162 3267 Ok(RepairResult { 3163 3268 repaired: repair_info.needs_rebuild, 3164 3269 compacted, ··· 3217 3322 3218 3323 let identifier = &did[DID_PREFIX.len()..]; 3219 3324 if identifier.len() != DID_IDENTIFIER_LEN { 3220 3220 - anyhow::bail!("Invalid DID identifier length: expected {} bytes, got {} bytes", DID_IDENTIFIER_LEN, identifier.len()); 3325 3325 + anyhow::bail!( 3326 3326 + "Invalid DID identifier length: expected {} bytes, got {} bytes", 3327 3327 + DID_IDENTIFIER_LEN, 3328 3328 + identifier.len() 3329 3329 + ); 3221 3330 } 3222 3331 3223 3332 Ok(identifier.to_string()) ··· 3229 3338 if !did.starts_with(DID_PREFIX) { 3230 3339 return None; 3231 3340 } 3232 3232 - 3341 3341 + 3233 3342 let identifier_start = DID_PREFIX.len(); 3234 3343 let identifier_end = identifier_start + DID_IDENTIFIER_LEN; 3235 3235 - 3344 3344 + 3236 3345 if did.len() < identifier_end { 3237 3346 return None; 3238 3347 } 3239 3239 - 3348 3348 + 3240 3349 let identifier_bytes = &did.as_bytes()[identifier_start..identifier_end]; 3241 3241 - 3350 3350 + 3242 3351 use fnv::FnvHasher; 3243 3352 use std::hash::Hasher; 3244 3244 - 3353 3353 + 3245 3354 let mut hasher = FnvHasher::default(); 3246 3355 hasher.write(identifier_bytes); 3247 3356 let hash = hasher.finish() as u32; ··· 3433 3542 let loc1 = OpLocation::new(global_pos, nullified); 3434 3543 let u32_val = loc1.as_u32(); 3435 3544 let loc2 = OpLocation::from_u32(u32_val); 3436 3436 - 3437 3437 - assert_eq!(loc1.global_position(), loc2.global_position(), 3438 3438 - "Global position mismatch for {} (nullified={})", global_pos, nullified); 3439 3439 - assert_eq!(loc1.nullified(), loc2.nullified(), 3440 3440 - "Nullified flag mismatch for {} (nullified={})", global_pos, nullified); 3441 3441 - assert_eq!(loc1.bundle(), loc2.bundle(), 3442 3442 - "Bundle mismatch for {} (nullified={})", global_pos, nullified); 3443 3443 - assert_eq!(loc1.position(), loc2.position(), 3444 3444 - "Position mismatch for {} (nullified={})", global_pos, nullified); 3545 3545 + 3546 3546 + assert_eq!( 3547 3547 + loc1.global_position(), 3548 3548 + loc2.global_position(), 3549 3549 + "Global position mismatch for {} (nullified={})", 3550 3550 + global_pos, 3551 3551 + nullified 3552 3552 + ); 3553 3553 + assert_eq!( 3554 3554 + loc1.nullified(), 3555 3555 + loc2.nullified(), 3556 3556 + "Nullified flag mismatch for {} (nullified={})", 3557 3557 + global_pos, 3558 3558 + nullified 3559 3559 + ); 3560 3560 + assert_eq!( 3561 3561 + loc1.bundle(), 3562 3562 + loc2.bundle(), 3563 3563 + "Bundle mismatch for {} (nullified={})", 3564 3564 + global_pos, 3565 3565 + nullified 3566 3566 + ); 3567 3567 + assert_eq!( 3568 3568 + loc1.position(), 3569 3569 + loc2.position(), 3570 3570 + "Position mismatch for {} (nullified={})", 3571 3571 + global_pos, 3572 3572 + nullified 3573 3573 + ); 3445 3574 } 3446 3575 } 3447 3576 }

+9 -4

src/ffi.rs

reviewed

··· 656 656 let manager = unsafe { &*manager }; 657 657 658 658 let callback = progress_callback.map(|cb| { 659 659 - Box::new(move |current: u32, total: u32, bytes_processed: u64, total_bytes: u64| { 660 660 - cb(current, total, bytes_processed, total_bytes); 661 661 - }) as Box<dyn Fn(u32, u32, u64, u64) + Send + Sync> 659 659 + Box::new( 660 660 + move |current: u32, total: u32, bytes_processed: u64, total_bytes: u64| { 661 661 + cb(current, total, bytes_processed, total_bytes); 662 662 + }, 663 663 + ) as Box<dyn Fn(u32, u32, u64, u64) + Send + Sync> 662 664 }); 663 665 664 666 // Use default flush interval for FFI 665 665 - match manager.manager.build_did_index(constants::DID_INDEX_FLUSH_INTERVAL, callback, None, None) { 667 667 + match manager 668 668 + .manager 669 669 + .build_did_index(constants::DID_INDEX_FLUSH_INTERVAL, callback, None, None) 670 670 + { 666 671 Ok(stats) => { 667 672 if !out_stats.is_null() { 668 673 unsafe {

+36 -9

src/format.rs

reviewed

··· 226 226 fn test_format_std_duration_verbose() { 227 227 assert_eq!(format_std_duration_verbose(StdDuration::from_secs(0)), "0s"); 228 228 assert_eq!(format_std_duration_verbose(StdDuration::from_secs(5)), "5s"); 229 229 - assert_eq!(format_std_duration_verbose(StdDuration::from_secs(65)), "1m 5s"); 230 230 - assert_eq!(format_std_duration_verbose(StdDuration::from_secs(3665)), "1h 1m 5s"); 231 231 - assert_eq!(format_std_duration_verbose(StdDuration::from_secs(90065)), "1d 1h 1m 5s"); 229 229 + assert_eq!( 230 230 + format_std_duration_verbose(StdDuration::from_secs(65)), 231 231 + "1m 5s" 232 232 + ); 233 233 + assert_eq!( 234 234 + format_std_duration_verbose(StdDuration::from_secs(3665)), 235 235 + "1h 1m 5s" 236 236 + ); 237 237 + assert_eq!( 238 238 + format_std_duration_verbose(StdDuration::from_secs(90065)), 239 239 + "1d 1h 1m 5s" 240 240 + ); 232 241 } 233 242 234 243 #[test] 235 244 fn test_format_std_duration_ms() { 236 236 - assert_eq!(format_std_duration_ms(StdDuration::from_millis(0)), "0.000ms"); 237 237 - assert_eq!(format_std_duration_ms(StdDuration::from_millis(50)), "50.000ms"); 238 238 - assert_eq!(format_std_duration_ms(StdDuration::from_millis(100)), "100ms"); 239 239 - assert_eq!(format_std_duration_ms(StdDuration::from_millis(1234)), "1234ms"); 245 245 + assert_eq!( 246 246 + format_std_duration_ms(StdDuration::from_millis(0)), 247 247 + "0.000ms" 248 248 + ); 249 249 + assert_eq!( 250 250 + format_std_duration_ms(StdDuration::from_millis(50)), 251 251 + "50.000ms" 252 252 + ); 253 253 + assert_eq!( 254 254 + format_std_duration_ms(StdDuration::from_millis(100)), 255 255 + "100ms" 256 256 + ); 257 257 + assert_eq!( 258 258 + format_std_duration_ms(StdDuration::from_millis(1234)), 259 259 + "1234ms" 260 260 + ); 240 261 } 241 262 242 263 #[test] 243 264 fn test_format_duration_verbose() { 244 265 assert_eq!(format_duration_verbose(ChronoDuration::seconds(0)), "0s"); 245 245 - assert_eq!(format_duration_verbose(ChronoDuration::seconds(65)), "1m 5s"); 246 246 - assert_eq!(format_duration_verbose(ChronoDuration::seconds(-65)), "-1m 5s"); 266 266 + assert_eq!( 267 267 + format_duration_verbose(ChronoDuration::seconds(65)), 268 268 + "1m 5s" 269 269 + ); 270 270 + assert_eq!( 271 271 + format_duration_verbose(ChronoDuration::seconds(-65)), 272 272 + "-1m 5s" 273 273 + ); 247 274 } 248 275 249 276 #[test]

+8 -2

src/handle_resolver.rs

reviewed

··· 228 228 #[test] 229 229 fn test_normalize_handle() { 230 230 assert_eq!(normalize_handle("user.bsky.social"), "user.bsky.social"); 231 231 - assert_eq!(normalize_handle("at://user.bsky.social"), "user.bsky.social"); 231 231 + assert_eq!( 232 232 + normalize_handle("at://user.bsky.social"), 233 233 + "user.bsky.social" 234 234 + ); 232 235 assert_eq!(normalize_handle("@user.bsky.social"), "user.bsky.social"); 233 236 // Note: trim_start_matches removes all matches, so "at://@user" becomes "user" (both prefixes removed) 234 234 - assert_eq!(normalize_handle("at://@user.bsky.social"), "user.bsky.social"); 237 237 + assert_eq!( 238 238 + normalize_handle("at://@user.bsky.social"), 239 239 + "user.bsky.social" 240 240 + ); 235 241 assert_eq!(normalize_handle("example.com"), "example.com"); 236 242 } 237 243

+66 -27

src/index.rs

reviewed

··· 39 39 impl Index { 40 40 pub fn load<P: AsRef<Path>>(directory: P) -> Result<Self> { 41 41 let index_path = directory.as_ref().join("plc_bundles.json"); 42 42 - let display_path = index_path.canonicalize().unwrap_or_else(|_| index_path.clone()); 43 43 - log::debug!("[BundleManager] Loading index from: {}", display_path.display()); 42 42 + let display_path = index_path 43 43 + .canonicalize() 44 44 + .unwrap_or_else(|_| index_path.clone()); 45 45 + log::debug!( 46 46 + "[BundleManager] Loading index from: {}", 47 47 + display_path.display() 48 48 + ); 44 49 let start = std::time::Instant::now(); 45 50 let file = File::open(&index_path)?; 46 51 let index: Index = sonic_rs::from_reader(file)?; ··· 114 119 let index_path = directory.as_ref().join("plc_bundles.json"); 115 120 let temp_path = index_path.with_extension("json.tmp"); 116 121 117 117 - let json = sonic_rs::to_string_pretty(self) 118 118 - .context("Failed to serialize index")?; 122 122 + let json = sonic_rs::to_string_pretty(self).context("Failed to serialize index")?; 119 123 120 124 std::fs::write(&temp_path, json) 121 125 .with_context(|| format!("Failed to write temp index: {}", temp_path.display()))?; ··· 159 163 // Create .plcbundle directory for DID index 160 164 let plcbundle_dir = dir.join(crate::constants::DID_INDEX_DIR); 161 165 if !plcbundle_dir.exists() { 162 162 - std::fs::create_dir_all(&plcbundle_dir) 163 163 - .with_context(|| format!("Failed to create DID index directory: {}", plcbundle_dir.display()))?; 166 166 + std::fs::create_dir_all(&plcbundle_dir).with_context(|| { 167 167 + format!( 168 168 + "Failed to create DID index directory: {}", 169 169 + plcbundle_dir.display() 170 170 + ) 171 171 + })?; 164 172 } 165 173 166 174 // Create and save empty index ··· 216 224 continue; 217 225 } 218 226 219 219 - let filename = path.file_name() 220 220 - .and_then(|n| n.to_str()) 221 221 - .unwrap_or(""); 227 227 + let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); 222 228 223 229 // Match pattern: NNNNNN.jsonl.zst (16 chars: 6 digits + 10 chars for .jsonl.zst) 224 224 - if filename.ends_with(".jsonl.zst") && filename.len() == 16 225 225 - && let Ok(bundle_num) = filename[0..6].parse::<u32>() { 230 230 + if filename.ends_with(".jsonl.zst") 231 231 + && filename.len() == 16 232 232 + && let Ok(bundle_num) = filename[0..6].parse::<u32>() 233 233 + { 226 234 bundle_files.push((bundle_num, path)); 227 235 } 228 236 } ··· 264 272 .sum(); 265 273 266 274 // Extract metadata from each bundle in parallel 275 275 + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; 267 276 use std::sync::{Arc, Mutex}; 268 268 - use std::sync::atomic::{AtomicUsize, AtomicU64, Ordering}; 269 269 - 277 277 + 270 278 let detected_origin: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(origin)); 271 279 let progress_cb_arc: Arc<Mutex<Option<F>>> = Arc::new(Mutex::new(progress_cb)); 272 280 let count_atomic = Arc::new(AtomicUsize::new(0)); 273 281 let bytes_atomic = Arc::new(AtomicU64::new(0)); 274 274 - 282 282 + 275 283 // Update progress bar less frequently to reduce contention 276 284 let update_interval = (rayon::current_num_threads().max(1) * 4).max(10); 277 285 ··· 283 291 // Get file size 284 292 let metadata = std::fs::metadata(bundle_path)?; 285 293 let compressed_size = metadata.len(); 286 286 - let bytes_processed = bytes_atomic.fetch_add(compressed_size, Ordering::Relaxed) + compressed_size; 294 294 + let bytes_processed = 295 295 + bytes_atomic.fetch_add(compressed_size, Ordering::Relaxed) + compressed_size; 287 296 let current_count = count_atomic.fetch_add(1, Ordering::Relaxed) + 1; 288 297 289 298 // Update progress periodically 290 290 - if (current_count.is_multiple_of(update_interval) || current_count == 1 || current_count == bundle_count) 299 299 + if (current_count.is_multiple_of(update_interval) 300 300 + || current_count == 1 301 301 + || current_count == bundle_count) 291 302 && let Ok(cb_guard) = progress_cb_arc.lock() 292 292 - && let Some(ref cb) = *cb_guard { 303 303 + && let Some(ref cb) = *cb_guard 304 304 + { 293 305 cb(current_count, bundle_count, bytes_processed, total_bytes); 294 306 } 295 307 296 308 // Extract embedded metadata from bundle file 297 309 let embedded = crate::bundle_format::extract_metadata_from_file(bundle_path) 298 298 - .with_context(|| format!("Failed to extract metadata from bundle {}", bundle_num))?; 310 310 + .with_context(|| { 311 311 + format!("Failed to extract metadata from bundle {}", bundle_num) 312 312 + })?; 299 313 300 314 // Auto-detect origin from first bundle if not provided 301 315 { ··· 309 323 { 310 324 let origin_guard = detected_origin.lock().unwrap(); 311 325 if let Some(ref expected_origin) = *origin_guard 312 312 - && embedded.origin != *expected_origin { 326 326 + && embedded.origin != *expected_origin 327 327 + { 313 328 anyhow::bail!( 314 329 "Bundle {:06}: origin mismatch (expected '{}', got '{}')", 315 330 bundle_num, ··· 359 374 360 375 // Calculate total uncompressed size 361 376 // Note: For legacy bundles without uncompressed_size in metadata, it will be 0 362 362 - let total_uncompressed_size: u64 = bundles_metadata.iter().map(|b| b.uncompressed_size).sum(); 377 377 + let total_uncompressed_size: u64 = 378 378 + bundles_metadata.iter().map(|b| b.uncompressed_size).sum(); 363 379 364 380 // Calculate chain hashes sequentially (depends on previous bundles) 365 381 for i in 0..bundles_metadata.len() { ··· 390 406 } 391 407 392 408 let last_bundle = bundles_metadata.last().unwrap().bundle_number; 393 393 - let origin_str = detected_origin.unwrap_or_else(|| crate::constants::DEFAULT_ORIGIN.to_string()); 409 409 + let origin_str = 410 410 + detected_origin.unwrap_or_else(|| crate::constants::DEFAULT_ORIGIN.to_string()); 394 411 395 412 Ok(Index { 396 413 version: "1.0".to_string(), ··· 432 449 bundle_numbers 433 450 .iter() 434 451 .filter_map(|bundle_num| { 435 435 - self.get_bundle(*bundle_num).map(|meta| meta.uncompressed_size) 452 452 + self.get_bundle(*bundle_num) 453 453 + .map(|meta| meta.uncompressed_size) 436 454 }) 437 455 .sum() 438 456 } ··· 515 533 516 534 let result = index.validate_bundle_sequence(); 517 535 assert!(result.is_err()); 518 518 - assert!(result.unwrap_err().to_string().contains("first bundle is 2 but must be 1")); 536 536 + assert!( 537 537 + result 538 538 + .unwrap_err() 539 539 + .to_string() 540 540 + .contains("first bundle is 2 but must be 1") 541 541 + ); 519 542 } 520 543 521 544 #[test] ··· 563 586 564 587 let result = index.validate_bundle_sequence(); 565 588 assert!(result.is_err()); 566 566 - assert!(result.unwrap_err().to_string().contains("expected bundle 2, found bundle 3")); 589 589 + assert!( 590 590 + result 591 591 + .unwrap_err() 592 592 + .to_string() 593 593 + .contains("expected bundle 2, found bundle 3") 594 594 + ); 567 595 } 568 596 569 597 #[test] ··· 823 851 824 852 // Test subset - should sum individual bundles 825 853 let subset = vec![1, 3]; 826 826 - assert_eq!(index.total_uncompressed_size_for_bundles(&subset), 200 + 400); 854 854 + assert_eq!( 855 855 + index.total_uncompressed_size_for_bundles(&subset), 856 856 + 200 + 400 857 857 + ); 827 858 828 859 // Test single bundle 829 860 let single = vec![2]; ··· 890 921 891 922 // Requesting non-existent bundle should be ignored 892 923 let with_missing = vec![1, 3, 2]; 893 893 - assert_eq!(index.total_uncompressed_size_for_bundles(&with_missing), 200 + 200); 924 924 + assert_eq!( 925 925 + index.total_uncompressed_size_for_bundles(&with_missing), 926 926 + 200 + 200 927 927 + ); 894 928 } 895 929 896 930 #[test] ··· 938 972 939 973 let result = index.validate_bundle_sequence(); 940 974 assert!(result.is_err()); 941 941 - assert!(result.unwrap_err().to_string().contains("last_bundle: index says 5, but last bundle in array is 2")); 975 975 + assert!( 976 976 + result 977 977 + .unwrap_err() 978 978 + .to_string() 979 979 + .contains("last_bundle: index says 5, but last bundle in array is 2") 980 980 + ); 942 981 } 943 982 }

+4 -4

src/lib.rs

reviewed

··· 57 57 pub use iterators::{ExportIterator, QueryIterator, RangeIterator}; 58 58 pub use manager::{ 59 59 BundleInfo, BundleManager, BundleRange, ChainVerifyResult, ChainVerifySpec, CleanPreview, 60 60 - CleanPreviewFile, CleanResult, DIDIndexStats, ExportFormat, ExportSpec, 61 61 - InfoFlags, IntoManagerOptions, LoadOptions, LoadResult, ManagerOptions, ManagerStats, 62 62 - OperationResult, QuerySpec, RebuildStats, ResolveResult, RollbackFileStats, RollbackPlan, 63 63 - RollbackResult, RollbackSpec, SyncResult, VerifyResult, VerifySpec, WarmUpSpec, WarmUpStrategy, 60 60 + CleanPreviewFile, CleanResult, DIDIndexStats, ExportFormat, ExportSpec, InfoFlags, 61 61 + IntoManagerOptions, LoadOptions, LoadResult, ManagerOptions, ManagerStats, OperationResult, 62 62 + QuerySpec, RebuildStats, ResolveResult, RollbackFileStats, RollbackPlan, RollbackResult, 63 63 + RollbackSpec, SyncResult, VerifyResult, VerifySpec, WarmUpSpec, WarmUpStrategy, 64 64 }; 65 65 pub use operations::{Operation, OperationFilter, OperationRequest, OperationWithLocation}; 66 66 pub use options::{Options, OptionsBuilder, QueryMode};

+317 -187

src/manager.rs

reviewed

··· 149 149 pub errors: Option<Vec<String>>, 150 150 } 151 151 152 152 - 153 152 #[derive(Debug, Clone)] 154 153 pub struct CleanPreview { 155 154 pub files: Vec<CleanPreviewFile>, ··· 225 224 pub fn new<O: IntoManagerOptions>(directory: PathBuf, options: O) -> Result<Self> { 226 225 let options = options.into_options(); 227 226 let init_start = std::time::Instant::now(); 228 228 - let display_dir = directory.canonicalize().unwrap_or_else(|_| directory.clone()); 229 229 - log::debug!("[BundleManager] Initializing BundleManager from: {}", display_dir.display()); 227 227 + let display_dir = directory 228 228 + .canonicalize() 229 229 + .unwrap_or_else(|_| directory.clone()); 230 230 + log::debug!( 231 231 + "[BundleManager] Initializing BundleManager from: {}", 232 232 + display_dir.display() 233 233 + ); 230 234 let index = Index::load(&directory)?; 231 235 232 236 let handle_resolver = options ··· 257 261 } else { 258 262 let mempool_preload_time = mempool_preload_start.elapsed(); 259 263 let mempool_preload_ms = mempool_preload_time.as_secs_f64() * 1000.0; 260 260 - if let Ok(stats) = manager.get_mempool_stats() && stats.count > 0 { 264 264 + if let Ok(stats) = manager.get_mempool_stats() 265 265 + && stats.count > 0 266 266 + { 261 267 log::debug!( 262 268 "[BundleManager] Pre-loaded mempool: {} operations for bundle {} ({:.3}ms)", 263 269 stats.count, ··· 270 276 271 277 let total_elapsed = init_start.elapsed(); 272 278 let total_elapsed_ms = total_elapsed.as_secs_f64() * 1000.0; 273 273 - log::debug!("[BundleManager] BundleManager initialized successfully ({:.3}ms total)", total_elapsed_ms); 279 279 + log::debug!( 280 280 + "[BundleManager] BundleManager initialized successfully ({:.3}ms total)", 281 281 + total_elapsed_ms 282 282 + ); 274 283 Ok(manager) 275 284 } 276 285 ··· 283 292 let did_index = did_index::Manager::new(self.directory.clone())?; 284 293 let did_index_elapsed = did_index_start.elapsed(); 285 294 let did_index_elapsed_ms = did_index_elapsed.as_secs_f64() * 1000.0; 286 286 - log::debug!("[BundleManager] DID index loaded ({:.3}ms)", did_index_elapsed_ms); 295 295 + log::debug!( 296 296 + "[BundleManager] DID index loaded ({:.3}ms)", 297 297 + did_index_elapsed_ms 298 298 + ); 287 299 *did_index_guard = Some(did_index); 288 300 } 289 301 Ok(()) ··· 430 442 /// Use `get_operation_raw()` if you only need the JSON. 431 443 pub fn get_operation(&self, bundle_num: u32, position: usize) -> Result<Operation> { 432 444 let json = self.get_operation_raw(bundle_num, position)?; 433 433 - let op = Operation::from_json(&json) 434 434 - .with_context(|| format!("Failed to parse operation JSON (bundle {}, position {})", bundle_num, position))?; 445 445 + let op = Operation::from_json(&json).with_context(|| { 446 446 + format!( 447 447 + "Failed to parse operation JSON (bundle {}, position {})", 448 448 + bundle_num, position 449 449 + ) 450 450 + })?; 435 451 Ok(op) 436 452 } 437 453 ··· 501 517 502 518 // === DID Operations === 503 519 /// Get all operations for a DID from both bundles and mempool 504 504 - /// 520 520 + /// 505 521 /// # Arguments 506 522 /// * `did` - The DID to look up 507 523 /// * `include_locations` - If true, also return operations with location information ··· 512 528 include_locations: bool, 513 529 include_stats: bool, 514 530 ) -> Result<DIDOperationsResult> { 515 515 - use std::time::Instant; 516 531 use chrono::DateTime; 532 532 + use std::time::Instant; 517 533 518 534 self.ensure_did_index()?; 519 535 520 536 let index_start = Instant::now(); 521 537 let (locations, shard_stats, shard_num, lookup_timings) = if include_stats { 522 538 let did_index = self.did_index.read().unwrap(); 523 523 - did_index.as_ref().unwrap().get_did_locations_with_stats(did)? 539 539 + did_index 540 540 + .as_ref() 541 541 + .unwrap() 542 542 + .get_did_locations_with_stats(did)? 524 543 } else { 525 544 let did_index = self.did_index.read().unwrap(); 526 545 let locs = did_index.as_ref().unwrap().get_did_locations(did)?; 527 527 - (locs, did_index::DIDLookupStats::default(), 0, did_index::DIDLookupTimings::default()) 546 546 + ( 547 547 + locs, 548 548 + did_index::DIDLookupStats::default(), 549 549 + 0, 550 550 + did_index::DIDLookupTimings::default(), 551 551 + ) 528 552 }; 529 553 let _index_time = index_start.elapsed(); 530 554 531 555 // Get operations from bundles 532 556 let (bundle_ops_with_loc, load_time) = self.collect_operations_for_locations(&locations)?; 533 533 - let mut bundle_operations: Vec<Operation> = bundle_ops_with_loc.iter().map(|owl| owl.operation.clone()).collect(); 557 557 + let mut bundle_operations: Vec<Operation> = bundle_ops_with_loc 558 558 + .iter() 559 559 + .map(|owl| owl.operation.clone()) 560 560 + .collect(); 534 561 535 562 // Get operations from mempool (only once) 536 563 let (mempool_ops, _mempool_load_time) = self.get_did_operations_from_mempool(did)?; ··· 563 590 564 591 // Sort all operations by timestamp 565 592 ops_with_loc.sort_by(|a, b| { 566 566 - let time_a = DateTime::parse_from_rfc3339(&a.operation.created_at) 567 567 - .unwrap_or_else(|_| DateTime::parse_from_rfc3339("1970-01-01T00:00:00Z").unwrap()); 568 568 - let time_b = DateTime::parse_from_rfc3339(&b.operation.created_at) 569 569 - .unwrap_or_else(|_| DateTime::parse_from_rfc3339("1970-01-01T00:00:00Z").unwrap()); 593 593 + let time_a = 594 594 + DateTime::parse_from_rfc3339(&a.operation.created_at).unwrap_or_else(|_| { 595 595 + DateTime::parse_from_rfc3339("1970-01-01T00:00:00Z").unwrap() 596 596 + }); 597 597 + let time_b = 598 598 + DateTime::parse_from_rfc3339(&b.operation.created_at).unwrap_or_else(|_| { 599 599 + DateTime::parse_from_rfc3339("1970-01-01T00:00:00Z").unwrap() 600 600 + }); 570 601 time_a.cmp(&time_b) 571 602 }); 572 603 ··· 578 609 Ok(DIDOperationsResult { 579 610 operations: bundle_operations, 580 611 operations_with_locations, 581 581 - stats: if include_stats { Some(shard_stats) } else { None }, 612 612 + stats: if include_stats { 613 613 + Some(shard_stats) 614 614 + } else { 615 615 + None 616 616 + }, 582 617 shard_num: if include_stats { Some(shard_num) } else { None }, 583 583 - lookup_timings: if include_stats { Some(lookup_timings) } else { None }, 618 618 + lookup_timings: if include_stats { 619 619 + Some(lookup_timings) 620 620 + } else { 621 621 + None 622 622 + }, 584 623 load_time: if include_stats { Some(load_time) } else { None }, 585 624 }) 586 625 } 587 587 - 588 626 589 627 /// Sample random DIDs directly from the DID index without reading bundles. 590 628 pub fn sample_random_dids(&self, count: usize, seed: Option<u64>) -> Result<Vec<String>> { ··· 596 634 /// Get DID operations from mempool (internal helper) 597 635 /// Mempool should be preloaded at initialization, so this is just a fast in-memory lookup 598 636 /// Returns (operations, load_time) where load_time is always ZERO (no lazy loading) 599 599 - fn get_did_operations_from_mempool(&self, did: &str) -> Result<(Vec<Operation>, std::time::Duration)> { 637 637 + fn get_did_operations_from_mempool( 638 638 + &self, 639 639 + did: &str, 640 640 + ) -> Result<(Vec<Operation>, std::time::Duration)> { 600 641 use std::time::Instant; 601 601 - 642 642 + 602 643 let mempool_start = Instant::now(); 603 603 - 644 644 + 604 645 // Mempool should be preloaded at initialization (no lazy loading) 605 646 let mempool_guard = self.mempool.read().unwrap(); 606 647 match mempool_guard.as_ref() { ··· 628 669 } 629 670 } 630 671 631 631 - fn get_latest_did_operation_from_mempool(&self, did: &str) -> Result<(Option<(Operation, usize)>, std::time::Duration)> { 672 672 + fn get_latest_did_operation_from_mempool( 673 673 + &self, 674 674 + did: &str, 675 675 + ) -> Result<(Option<(Operation, usize)>, std::time::Duration)> { 632 676 use std::time::Instant; 633 633 - 677 677 + 634 678 let mempool_start = Instant::now(); 635 635 - 679 679 + 636 680 // Mempool should be preloaded at initialization (no lazy loading) 637 681 let mempool_guard = self.mempool.read().unwrap(); 638 682 let result = match mempool_guard.as_ref() { ··· 645 689 None 646 690 } 647 691 }; 648 648 - 692 692 + 649 693 let mempool_elapsed = mempool_start.elapsed(); 650 694 log::debug!( 651 695 "[Mempool] Latest operation lookup for DID {} in {:?}", 652 696 did, 653 697 mempool_elapsed 654 698 ); 655 655 - 699 699 + 656 700 Ok((result, std::time::Duration::ZERO)) 657 701 } 658 702 ··· 660 704 /// Returns the latest non-nullified DID document. 661 705 /// If mempool has operations, uses the latest from mempool and skips bundle/index lookup. 662 706 pub fn resolve_did(&self, did: &str) -> Result<ResolveResult> { 663 663 - use std::time::Instant; 664 707 use chrono::DateTime; 708 708 + use std::time::Instant; 665 709 666 710 let total_start = Instant::now(); 667 711 ··· 671 715 // Check mempool first (most recent operations) 672 716 log::debug!("[Resolve] Checking mempool first for DID: {}", did); 673 717 let mempool_start = Instant::now(); 674 674 - let (latest_mempool_op, mempool_load_time) = self.get_latest_did_operation_from_mempool(did)?; 718 718 + let (latest_mempool_op, mempool_load_time) = 719 719 + self.get_latest_did_operation_from_mempool(did)?; 675 720 let mempool_time = mempool_start.elapsed(); 676 676 - log::debug!("[Resolve] Mempool check: found latest operation in {:?} (load: {:?})", mempool_time, mempool_load_time); 721 721 + log::debug!( 722 722 + "[Resolve] Mempool check: found latest operation in {:?} (load: {:?})", 723 723 + mempool_time, 724 724 + mempool_load_time 725 725 + ); 677 726 678 727 // If mempool has a non-nullified operation, use it and skip bundle lookup 679 728 if let Some((operation, position)) = latest_mempool_op { 680 729 let load_start = Instant::now(); 681 681 - log::debug!("[Resolve] Found latest non-nullified operation in mempool, skipping bundle lookup"); 730 730 + log::debug!( 731 731 + "[Resolve] Found latest non-nullified operation in mempool, skipping bundle lookup" 732 732 + ); 682 733 683 734 // Build document from latest mempool operation 684 684 - let document = crate::resolver::resolve_did_document(did, std::slice::from_ref(&operation))?; 735 735 + let document = 736 736 + crate::resolver::resolve_did_document(did, std::slice::from_ref(&operation))?; 685 737 let load_time = load_start.elapsed(); 686 738 687 739 return Ok(ResolveResult { ··· 695 747 load_time, 696 748 total_time: total_start.elapsed(), 697 749 locations_found: 1, // Found one operation in mempool 698 698 - shard_num: 0, // No shard for mempool 750 750 + shard_num: 0, // No shard for mempool 699 751 shard_stats: None, 700 752 lookup_timings: None, 701 753 }); 702 754 } 703 755 704 756 // Mempool is empty or all nullified - check bundles 705 705 - log::debug!("[Resolve] No non-nullified operations in mempool, checking bundles for DID: {}", did); 757 757 + log::debug!( 758 758 + "[Resolve] No non-nullified operations in mempool, checking bundles for DID: {}", 759 759 + did 760 760 + ); 706 761 self.ensure_did_index()?; 707 762 let index_start = Instant::now(); 708 763 let did_index = self.did_index.read().unwrap(); 709 709 - let (locations, shard_stats, shard_num, lookup_timings) = 710 710 - did_index.as_ref().unwrap().get_did_locations_with_stats(did)?; 764 764 + let (locations, shard_stats, shard_num, lookup_timings) = did_index 765 765 + .as_ref() 766 766 + .unwrap() 767 767 + .get_did_locations_with_stats(did)?; 711 768 let index_time = index_start.elapsed(); 712 712 - log::debug!("[Resolve] Bundle index lookup: {} locations found in {:?}", locations.len(), index_time); 769 769 + log::debug!( 770 770 + "[Resolve] Bundle index lookup: {} locations found in {:?}", 771 771 + locations.len(), 772 772 + index_time 773 773 + ); 713 774 714 775 // Find latest non-nullified operation from bundles 715 776 let load_start = Instant::now(); 716 777 let mut latest_operation: Option<(Operation, u32, usize)> = None; 717 778 let mut latest_time = DateTime::parse_from_rfc3339("1970-01-01T00:00:00Z").unwrap(); 718 718 - 779 779 + 719 780 for loc in &locations { 720 781 if !loc.nullified() 721 782 && let Ok(op) = self.get_operation(loc.bundle() as u32, loc.position() as usize) ··· 728 789 } 729 790 let load_time = load_start.elapsed(); 730 791 731 731 - let (operation, bundle_number, position) = latest_operation 732 732 - .ok_or_else(|| anyhow::anyhow!("DID not found: {} (checked bundles and mempool)", did))?; 792 792 + let (operation, bundle_number, position) = latest_operation.ok_or_else(|| { 793 793 + anyhow::anyhow!("DID not found: {} (checked bundles and mempool)", did) 794 794 + })?; 733 795 734 796 // Build document from latest bundle operation 735 735 - let document = crate::resolver::resolve_did_document(did, std::slice::from_ref(&operation))?; 797 797 + let document = 798 798 + crate::resolver::resolve_did_document(did, std::slice::from_ref(&operation))?; 736 799 737 800 Ok(ResolveResult { 738 801 document, ··· 783 846 } 784 847 } 785 848 786 786 - ops_with_loc.sort_by_key(|owl| { 787 787 - bundle_position_to_global(owl.bundle, owl.position) 788 788 - }); 849 849 + ops_with_loc.sort_by_key(|owl| bundle_position_to_global(owl.bundle, owl.position)); 789 850 790 851 Ok((ops_with_loc, load_start.elapsed())) 791 852 } ··· 942 1003 .retain(|b| b.bundle_number <= spec.target_bundle); 943 1004 944 1005 // Use default flush interval for rollback 945 945 - self.build_did_index(crate::constants::DID_INDEX_FLUSH_INTERVAL, None::<fn(u32, u32, u64, u64)>, None, None)?; 1006 1006 + self.build_did_index( 1007 1007 + crate::constants::DID_INDEX_FLUSH_INTERVAL, 1008 1008 + None::<fn(u32, u32, u64, u64)>, 1009 1009 + None, 1010 1010 + None, 1011 1011 + )?; 946 1012 947 1013 Ok(RollbackResult { 948 1014 success: true, ··· 981 1047 } 982 1048 983 1049 // === DID Index === 984 984 - pub fn build_did_index<F>(&self, flush_interval: u32, progress_cb: Option<F>, num_threads: Option<usize>, interrupted: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>) -> Result<RebuildStats> 1050 1050 + pub fn build_did_index<F>( 1051 1051 + &self, 1052 1052 + flush_interval: u32, 1053 1053 + progress_cb: Option<F>, 1054 1054 + num_threads: Option<usize>, 1055 1055 + interrupted: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>, 1056 1056 + ) -> Result<RebuildStats> 985 1057 where 986 1058 F: Fn(u32, u32, u64, u64) + Send + Sync, // (current, total, bytes_processed, total_bytes) 987 1059 { 988 1060 use std::time::Instant; 989 1061 990 1062 let actual_threads = num_threads.unwrap_or(0); // 0 = auto-detect 991 991 - 1063 1063 + 992 1064 let last_bundle = self.get_last_bundle(); 993 1065 let mut stats = RebuildStats::default(); 994 1066 ··· 1010 1082 if flush_interval > 0 { 1011 1083 if flush_interval == crate::constants::DID_INDEX_FLUSH_INTERVAL { 1012 1084 // Default value - show with tuning hint 1013 1013 - eprintln!(" Flush: Every {} bundles (tune with --flush-interval)", flush_interval); 1085 1085 + eprintln!( 1086 1086 + " Flush: Every {} bundles (tune with --flush-interval)", 1087 1087 + flush_interval 1088 1088 + ); 1014 1089 } else { 1015 1090 // Non-default value - show with tuning hint 1016 1016 - eprintln!(" Flush: {} bundles (you can tune with --flush-interval)", flush_interval); 1091 1091 + eprintln!( 1092 1092 + " Flush: {} bundles (you can tune with --flush-interval)", 1093 1093 + flush_interval 1094 1094 + ); 1017 1095 } 1018 1096 } else { 1019 1097 eprintln!(" Flush: Only at end (maximum memory usage)"); ··· 1051 1129 } 1052 1130 }), 1053 1131 actual_threads, 1054 1054 - interrupted 1132 1132 + interrupted, 1055 1133 )? 1056 1134 } else { 1057 1135 return Err(anyhow::anyhow!("DID index not initialized")); ··· 1065 1143 1066 1144 eprintln!("\n"); 1067 1145 eprintln!("✅ Index Build Complete"); 1068 1068 - eprintln!(" Time: {:.1}s (Stage 1: {:.1}s, Stage 2: {:.1}s)", 1146 1146 + eprintln!( 1147 1147 + " Time: {:.1}s (Stage 1: {:.1}s, Stage 2: {:.1}s)", 1069 1148 total_duration.as_secs_f64(), 1070 1149 stage1_duration.as_secs_f64(), 1071 1150 stage2_duration.as_secs_f64() 1072 1151 ); 1073 1073 - eprintln!(" Operations: {}", crate::format::format_number(total_operations)); 1152 1152 + eprintln!( 1153 1153 + " Operations: {}", 1154 1154 + crate::format::format_number(total_operations) 1155 1155 + ); 1074 1156 1075 1157 // Get final stats 1076 1158 let final_stats = self.get_did_index_stats(); ··· 1079 1161 .and_then(|v| v.as_i64()) 1080 1162 .unwrap_or(0); 1081 1163 1082 1082 - eprintln!(" Total DIDs: {}", crate::format::format_number(total_dids as u64)); 1164 1164 + eprintln!( 1165 1165 + " Total DIDs: {}", 1166 1166 + crate::format::format_number(total_dids as u64) 1167 1167 + ); 1083 1168 1084 1169 Ok(stats) 1085 1170 } ··· 1089 1174 /// Returns keys like `exists`, `total_dids`, `last_bundle`, `delta_segments`, `shard_count` when available. 1090 1175 pub fn get_did_index_stats(&self) -> HashMap<String, serde_json::Value> { 1091 1176 self.ensure_did_index().ok(); // Stats might be called even if index doesn't exist 1092 1092 - self.did_index.read().unwrap().as_ref().map(|idx| idx.get_stats()).unwrap_or_default() 1177 1177 + self.did_index 1178 1178 + .read() 1179 1179 + .unwrap() 1180 1180 + .as_ref() 1181 1181 + .map(|idx| idx.get_stats()) 1182 1182 + .unwrap_or_default() 1093 1183 } 1094 1184 1095 1185 /// Get DID index stats as struct (legacy format) ··· 1112 1202 } 1113 1203 1114 1204 /// Verify DID index and return detailed result 1115 1115 - /// 1205 1205 + /// 1116 1206 /// Performs standard integrity check by default. If `full` is true, also rebuilds 1117 1207 /// the index in a temporary directory and compares with the existing index. 1118 1118 - /// 1208 1208 + /// 1119 1209 /// For server startup checks, call with `full=false` and check `verify_result.missing_base_shards` 1120 1210 /// and `verify_result.missing_delta_segments` to determine if the index is corrupted. 1121 1211 pub fn verify_did_index<F>( ··· 1129 1219 F: Fn(u32, u32, u64, u64) + Send + Sync, // (current, total, bytes_processed, total_bytes) 1130 1220 { 1131 1221 self.ensure_did_index()?; 1132 1132 - 1222 1222 + 1133 1223 let did_index = self.did_index.read().unwrap(); 1134 1134 - let idx = did_index.as_ref().ok_or_else(|| anyhow::anyhow!("DID index not initialized"))?; 1135 1135 - 1224 1224 + let idx = did_index 1225 1225 + .as_ref() 1226 1226 + .ok_or_else(|| anyhow::anyhow!("DID index not initialized"))?; 1227 1227 + 1136 1228 let last_bundle = self.get_last_bundle(); 1137 1229 let mut verify_result = idx.verify_integrity(last_bundle)?; 1138 1138 - 1230 1230 + 1139 1231 // If full verification requested, rebuild and compare 1140 1232 if full { 1141 1233 // Adapt callback for build_from_scratch which expects Option<String> as 4th param ··· 1152 1244 build_callback, 1153 1245 )?; 1154 1246 verify_result.errors += rebuild_result.errors; 1155 1155 - verify_result.error_categories.extend(rebuild_result.error_categories); 1247 1247 + verify_result 1248 1248 + .error_categories 1249 1249 + .extend(rebuild_result.error_categories); 1156 1250 } 1157 1157 - 1251 1251 + 1158 1252 Ok(verify_result) 1159 1253 } 1160 1254 ··· 1169 1263 F: Fn(u32, u32, u64, u64) + Send + Sync, // (current, total, bytes_processed, total_bytes) 1170 1264 { 1171 1265 self.ensure_did_index()?; 1172 1172 - 1266 1266 + 1173 1267 let last_bundle = self.get_last_bundle(); 1174 1174 - 1268 1268 + 1175 1269 // Create bundle loader closure 1176 1270 let bundle_loader = |bundle_num: u32| -> Result<Vec<(String, bool)>> { 1177 1271 let result = self.load_bundle(bundle_num, LoadOptions::default())?; ··· 1181 1275 .map(|op| (op.did.clone(), op.nullified)) 1182 1276 .collect()) 1183 1277 }; 1184 1184 - 1278 1278 + 1185 1279 let mut did_index = self.did_index.write().unwrap(); 1186 1186 - let idx = did_index.as_mut().ok_or_else(|| anyhow::anyhow!("DID index not initialized"))?; 1187 1187 - 1280 1280 + let idx = did_index 1281 1281 + .as_mut() 1282 1282 + .ok_or_else(|| anyhow::anyhow!("DID index not initialized"))?; 1283 1283 + 1188 1284 let mut repair_result = idx.repair(last_bundle, bundle_loader)?; 1189 1189 - 1285 1285 + 1190 1286 // If repair indicates full rebuild is needed, do it 1191 1287 if repair_result.repaired && repair_result.bundles_processed == 0 { 1192 1288 drop(did_index); 1193 1193 - 1289 1289 + 1194 1290 // Adapt callback signature for build_did_index 1195 1291 let build_callback = progress_callback.map(|cb| { 1196 1292 move |current: u32, total: u32, bytes: u64, total_bytes: u64| { ··· 1198 1294 } 1199 1295 }); 1200 1296 self.build_did_index(flush_interval, build_callback, Some(num_threads), None)?; 1201 1201 - 1297 1297 + 1202 1298 repair_result.bundles_processed = last_bundle; 1203 1299 } 1204 1204 - 1300 1300 + 1205 1301 Ok(repair_result) 1206 1302 } 1207 1303 ··· 1334 1430 if let Ok(entries) = std::fs::read_dir(&self.directory) { 1335 1431 for entry in entries.flatten() { 1336 1432 if let Some(name) = entry.file_name().to_str() 1337 1337 - && name.starts_with(constants::MEMPOOL_FILE_PREFIX) && name.ends_with(".jsonl") 1433 1433 + && name.starts_with(constants::MEMPOOL_FILE_PREFIX) 1434 1434 + && name.ends_with(".jsonl") 1338 1435 { 1339 1436 let _ = std::fs::remove_file(entry.path()); 1340 1437 } ··· 1394 1491 if let Ok(entries) = std::fs::read_dir(&self.directory) { 1395 1492 for entry in entries.flatten() { 1396 1493 if let Some(name) = entry.file_name().to_str() 1397 1397 - && name.starts_with(constants::MEMPOOL_FILE_PREFIX) && name.ends_with(".jsonl") 1494 1494 + && name.starts_with(constants::MEMPOOL_FILE_PREFIX) 1495 1495 + && name.ends_with(".jsonl") 1398 1496 { 1399 1497 // Extract bundle number from filename: plc_mempool_NNNNNN.jsonl 1400 1498 if let Some(num_str) = name 1401 1499 .strip_prefix(constants::MEMPOOL_FILE_PREFIX) 1402 1500 .and_then(|s| s.strip_suffix(".jsonl")) 1403 1403 - && let Ok(bundle_num) = num_str.parse::<u32>() { 1501 1501 + && let Ok(bundle_num) = num_str.parse::<u32>() 1502 1502 + { 1404 1503 // Delete mempool files for completed bundles or way future bundles 1405 1504 if bundle_num <= last_bundle || bundle_num > next_bundle_num { 1406 1406 - log::warn!( 1407 1407 - "Removing stale mempool file for bundle {:06}", 1408 1408 - bundle_num 1409 1409 - ); 1505 1505 + log::warn!("Removing stale mempool file for bundle {:06}", bundle_num); 1410 1506 let _ = std::fs::remove_file(entry.path()); 1411 1507 found_stale_files = true; 1412 1508 } ··· 1432 1528 } 1433 1529 1434 1530 // Get the last operation from the previous bundle 1435 1435 - let last_bundle_time = if next_bundle_num > 1 1436 1436 - && let Ok(last_bundle_result) = self.load_bundle(next_bundle_num - 1, LoadOptions::default()) { 1437 1437 - last_bundle_result.operations.last().and_then(|last_op| { 1438 1438 - chrono::DateTime::parse_from_rfc3339(&last_op.created_at) 1439 1439 - .ok() 1440 1440 - .map(|dt| dt.with_timezone(&chrono::Utc)) 1441 1441 - }) 1442 1442 - } else { 1443 1443 - None 1444 1444 - }; 1531 1531 + let last_bundle_time = if next_bundle_num > 1 1532 1532 + && let Ok(last_bundle_result) = 1533 1533 + self.load_bundle(next_bundle_num - 1, LoadOptions::default()) 1534 1534 + { 1535 1535 + last_bundle_result.operations.last().and_then(|last_op| { 1536 1536 + chrono::DateTime::parse_from_rfc3339(&last_op.created_at) 1537 1537 + .ok() 1538 1538 + .map(|dt| dt.with_timezone(&chrono::Utc)) 1539 1539 + }) 1540 1540 + } else { 1541 1541 + None 1542 1542 + }; 1445 1543 1446 1544 // Special case: When creating the first bundle (next_bundle_num == 1, meaning 1447 1545 // last_bundle == 0, i.e., empty repository), any existing mempool is likely stale ··· 1464 1562 1465 1563 // Check if mempool operations are chronologically valid relative to last bundle 1466 1564 if let Some(last_time) = last_bundle_time 1467 1467 - && let Some(first_mempool_time) = mempool_stats.first_time { 1468 1468 - // Case 1: Mempool operations are BEFORE the last bundle (definitely stale) 1469 1469 - if first_mempool_time < last_time { 1470 1470 - log::warn!("Detected stale mempool data (operations before last bundle)"); 1471 1471 - log::warn!( 1472 1472 - "First mempool op: {}, Last bundle op: {}", 1473 1473 - first_mempool_time.format("%Y-%m-%d %H:%M:%S"), 1474 1474 - last_time.format("%Y-%m-%d %H:%M:%S") 1475 1475 - ); 1476 1476 - log::warn!("Clearing mempool to start fresh..."); 1477 1477 - self.clear_mempool()?; 1478 1478 - return Ok(()); 1479 1479 - } 1565 1565 + && let Some(first_mempool_time) = mempool_stats.first_time 1566 1566 + { 1567 1567 + // Case 1: Mempool operations are BEFORE the last bundle (definitely stale) 1568 1568 + if first_mempool_time < last_time { 1569 1569 + log::warn!("Detected stale mempool data (operations before last bundle)"); 1570 1570 + log::warn!( 1571 1571 + "First mempool op: {}, Last bundle op: {}", 1572 1572 + first_mempool_time.format("%Y-%m-%d %H:%M:%S"), 1573 1573 + last_time.format("%Y-%m-%d %H:%M:%S") 1574 1574 + ); 1575 1575 + log::warn!("Clearing mempool to start fresh..."); 1576 1576 + self.clear_mempool()?; 1577 1577 + return Ok(()); 1578 1578 + } 1480 1579 1481 1481 - // Case 2: Mempool operations are slightly after last bundle, but way too close 1482 1482 - // This indicates they're from a previous failed attempt at this bundle 1483 1483 - // BUT: Only clear if the mempool file is old (modified > 1 hour ago) 1484 1484 - // If it's recent, it might be a legitimate resume of a slow sync 1485 1485 - let time_diff = first_mempool_time.signed_duration_since(last_time); 1486 1486 - if time_diff 1487 1487 - < chrono::Duration::seconds(constants::MIN_BUNDLE_CREATION_INTERVAL_SECS) 1488 1488 - && mempool_stats.count < constants::BUNDLE_SIZE 1489 1489 - { 1490 1490 - // Check mempool file modification time 1491 1491 - let mempool_filename = format!( 1492 1492 - "{}{:06}.jsonl", 1493 1493 - constants::MEMPOOL_FILE_PREFIX, 1494 1494 - next_bundle_num 1495 1495 - ); 1496 1496 - let mempool_path = self.directory.join(mempool_filename); 1580 1580 + // Case 2: Mempool operations are slightly after last bundle, but way too close 1581 1581 + // This indicates they're from a previous failed attempt at this bundle 1582 1582 + // BUT: Only clear if the mempool file is old (modified > 1 hour ago) 1583 1583 + // If it's recent, it might be a legitimate resume of a slow sync 1584 1584 + let time_diff = first_mempool_time.signed_duration_since(last_time); 1585 1585 + if time_diff < chrono::Duration::seconds(constants::MIN_BUNDLE_CREATION_INTERVAL_SECS) 1586 1586 + && mempool_stats.count < constants::BUNDLE_SIZE 1587 1587 + { 1588 1588 + // Check mempool file modification time 1589 1589 + let mempool_filename = format!( 1590 1590 + "{}{:06}.jsonl", 1591 1591 + constants::MEMPOOL_FILE_PREFIX, 1592 1592 + next_bundle_num 1593 1593 + ); 1594 1594 + let mempool_path = self.directory.join(mempool_filename); 1497 1595 1498 1498 - let is_stale = if let Ok(metadata) = std::fs::metadata(&mempool_path) { 1499 1499 - if let Ok(modified) = metadata.modified() { 1500 1500 - let modified_time = std::time::SystemTime::now() 1501 1501 - .duration_since(modified) 1502 1502 - .unwrap_or(std::time::Duration::from_secs(0)); 1503 1503 - modified_time > std::time::Duration::from_secs(3600) // 1 hour 1504 1504 - } else { 1505 1505 - false // Can't get modified time, assume not stale 1506 1506 - } 1596 1596 + let is_stale = if let Ok(metadata) = std::fs::metadata(&mempool_path) { 1597 1597 + if let Ok(modified) = metadata.modified() { 1598 1598 + let modified_time = std::time::SystemTime::now() 1599 1599 + .duration_since(modified) 1600 1600 + .unwrap_or(std::time::Duration::from_secs(0)); 1601 1601 + modified_time > std::time::Duration::from_secs(3600) // 1 hour 1507 1602 } else { 1508 1508 - false // File doesn't exist, assume not stale 1509 1509 - }; 1603 1603 + false // Can't get modified time, assume not stale 1604 1604 + } 1605 1605 + } else { 1606 1606 + false // File doesn't exist, assume not stale 1607 1607 + }; 1510 1608 1511 1511 - if is_stale { 1512 1512 - log::warn!( 1513 1513 - "Detected potentially stale mempool data (too close to last bundle timestamp)" 1514 1514 - ); 1515 1515 - log::warn!( 1516 1516 - "Time difference: {}s, Operations: {}/{}", 1517 1517 - time_diff.num_seconds(), 1518 1518 - mempool_stats.count, 1519 1519 - constants::BUNDLE_SIZE 1520 1520 - ); 1521 1521 - log::warn!( 1522 1522 - "This likely indicates a previous failed sync attempt. Clearing mempool..." 1523 1523 - ); 1524 1524 - self.clear_mempool()?; 1525 1525 - } else if *self.verbose.lock().unwrap() { 1526 1526 - log::debug!( 1527 1527 - "Mempool appears recent, allowing resume despite close timestamp" 1528 1528 - ); 1529 1529 - } 1530 1530 - return Ok(()); 1609 1609 + if is_stale { 1610 1610 + log::warn!( 1611 1611 + "Detected potentially stale mempool data (too close to last bundle timestamp)" 1612 1612 + ); 1613 1613 + log::warn!( 1614 1614 + "Time difference: {}s, Operations: {}/{}", 1615 1615 + time_diff.num_seconds(), 1616 1616 + mempool_stats.count, 1617 1617 + constants::BUNDLE_SIZE 1618 1618 + ); 1619 1619 + log::warn!( 1620 1620 + "This likely indicates a previous failed sync attempt. Clearing mempool..." 1621 1621 + ); 1622 1622 + self.clear_mempool()?; 1623 1623 + } else if *self.verbose.lock().unwrap() { 1624 1624 + log::debug!("Mempool appears recent, allowing resume despite close timestamp"); 1531 1625 } 1626 1626 + return Ok(()); 1627 1627 + } 1532 1628 } 1533 1629 1534 1630 // Check if mempool has way too many operations (likely from failed previous attempt) ··· 1648 1744 1649 1745 /// Fetch and save next bundle from PLC directory 1650 1746 /// DID index is updated on every bundle (fast with delta segments) 1651 1651 - pub async fn sync_next_bundle(&self, client: &crate::plc_client::PLCClient) -> Result<SyncResult> { 1747 1747 + pub async fn sync_next_bundle( 1748 1748 + &self, 1749 1749 + client: &crate::plc_client::PLCClient, 1750 1750 + ) -> Result<SyncResult> { 1652 1751 use crate::sync::{get_boundary_cids, strip_boundary_duplicates}; 1653 1752 use std::time::Instant; 1654 1753 ··· 1684 1783 // If mempool has operations, update cursor AND boundaries from mempool 1685 1784 // (mempool operations already had boundary dedup applied when they were added) 1686 1785 let mempool_stats = self.get_mempool_stats()?; 1687 1687 - if mempool_stats.count > 0 && let Some(last_time) = mempool_stats.last_time { 1786 1786 + if mempool_stats.count > 0 1787 1787 + && let Some(last_time) = mempool_stats.last_time 1788 1788 + { 1688 1789 if *self.verbose.lock().unwrap() { 1689 1790 log::debug!( 1690 1791 "Mempool has {} ops, resuming from {}", ··· 2050 2151 synced += 1; 2051 2152 2052 2153 // Check if we've reached the limit 2053 2053 - if let Some(max) = max_bundles && synced >= max { 2154 2154 + if let Some(max) = max_bundles 2155 2155 + && synced >= max 2156 2156 + { 2054 2157 break; 2055 2158 } 2056 2159 } ··· 2102 2205 // Use multi-frame compression for better performance on large bundles 2103 2206 2104 2207 // Compress operations to frames using parallel compression 2105 2105 - let compress_result = crate::bundle_format::compress_operations_to_frames_parallel(&operations)?; 2208 2208 + let compress_result = 2209 2209 + crate::bundle_format::compress_operations_to_frames_parallel(&operations)?; 2106 2210 2107 2107 - let serialize_time = std::time::Duration::from_secs_f64(compress_result.serialize_time_ms / 1000.0); 2108 2108 - let compress_time = std::time::Duration::from_secs_f64(compress_result.compress_time_ms / 1000.0); 2211 2211 + let serialize_time = 2212 2212 + std::time::Duration::from_secs_f64(compress_result.serialize_time_ms / 1000.0); 2213 2213 + let compress_time = 2214 2214 + std::time::Duration::from_secs_f64(compress_result.compress_time_ms / 1000.0); 2109 2215 2110 2216 let uncompressed_size = compress_result.uncompressed_size; 2111 2217 let compressed_size = compress_result.compressed_size; ··· 2421 2527 // Serialize and write index in blocking task to avoid blocking async runtime 2422 2528 // Use Index::save() which does atomic write (temp file + rename) 2423 2529 let directory = self.directory.clone(); 2424 2424 - tokio::task::spawn_blocking(move || { 2425 2425 - index_clone.save(directory) 2426 2426 - }) 2427 2427 - .await 2428 2428 - .context("Index write task failed")??; 2530 2530 + tokio::task::spawn_blocking(move || index_clone.save(directory)) 2531 2531 + .await 2532 2532 + .context("Index write task failed")??; 2429 2533 let index_write_time = index_write_start.elapsed(); 2430 2534 2431 2535 Ok(( ··· 2915 3019 2916 3020 // Scan shards directory (.plcbundle/shards/) 2917 3021 let shards_dir = did_index_dir.join(constants::DID_INDEX_SHARDS); 2918 2918 - if shards_dir.exists() && let Ok(entries) = fs::read_dir(&shards_dir) { 3022 3022 + if shards_dir.exists() 3023 3023 + && let Ok(entries) = fs::read_dir(&shards_dir) 3024 3024 + { 2919 3025 for entry in entries { 2920 3026 let entry = match entry { 2921 3027 Ok(e) => e, ··· 2958 3064 use std::fs; 2959 3065 2960 3066 let verbose = *self.verbose.lock().unwrap(); 2961 2961 - 3067 3067 + 2962 3068 if verbose { 2963 3069 log::info!("Starting repository cleanup..."); 2964 3070 } ··· 2972 3078 if verbose { 2973 3079 log::info!("Scanning repository root directory: {}", root_dir.display()); 2974 3080 } 2975 2975 - 3081 3081 + 2976 3082 if let Ok(entries) = fs::read_dir(root_dir) { 2977 3083 for entry in entries { 2978 3084 let entry = match entry { ··· 2995 3101 bytes_freed += size; 2996 3102 size 2997 3103 } 2998 2998 - Err(_) => 0 3104 3104 + Err(_) => 0, 2999 3105 }; 3000 3106 3001 3107 match fs::remove_file(&path) { 3002 3108 Ok(_) => { 3003 3109 files_removed += 1; 3004 3110 if verbose { 3005 3005 - log::info!(" ✓ Removed: {} ({})", 3111 3111 + log::info!( 3112 3112 + " ✓ Removed: {} ({})", 3006 3113 path.file_name().and_then(|n| n.to_str()).unwrap_or("?"), 3007 3007 - crate::format::format_bytes(file_size)); 3114 3114 + crate::format::format_bytes(file_size) 3115 3115 + ); 3008 3116 } 3009 3117 } 3010 3118 Err(e) => { ··· 3025 3133 if verbose { 3026 3134 log::info!("Scanning DID index directory: {}", did_index_dir.display()); 3027 3135 } 3028 3028 - 3136 3136 + 3029 3137 // Clean config.json.tmp 3030 3138 let config_tmp = did_index_dir.join(format!("{}.tmp", constants::DID_INDEX_CONFIG)); 3031 3139 if config_tmp.exists() { ··· 3035 3143 bytes_freed += size; 3036 3144 size 3037 3145 } 3038 3038 - Err(_) => 0 3146 3146 + Err(_) => 0, 3039 3147 }; 3040 3148 3041 3149 match fs::remove_file(&config_tmp) { 3042 3150 Ok(_) => { 3043 3151 files_removed += 1; 3044 3152 if verbose { 3045 3045 - log::info!(" ✓ Removed: {} ({})", 3046 3046 - config_tmp.file_name().and_then(|n| n.to_str()).unwrap_or("?"), 3047 3047 - crate::format::format_bytes(file_size)); 3153 3153 + log::info!( 3154 3154 + " ✓ Removed: {} ({})", 3155 3155 + config_tmp 3156 3156 + .file_name() 3157 3157 + .and_then(|n| n.to_str()) 3158 3158 + .unwrap_or("?"), 3159 3159 + crate::format::format_bytes(file_size) 3160 3160 + ); 3048 3161 } 3049 3162 } 3050 3163 Err(e) => { ··· 3068 3181 let entry = match entry { 3069 3182 Ok(e) => e, 3070 3183 Err(e) => { 3071 3071 - errors.push(format!("Failed to read shards directory entry: {}", e)); 3184 3184 + errors 3185 3185 + .push(format!("Failed to read shards directory entry: {}", e)); 3072 3186 continue; 3073 3187 } 3074 3188 }; ··· 3085 3199 bytes_freed += size; 3086 3200 size 3087 3201 } 3088 3088 - Err(_) => 0 3202 3202 + Err(_) => 0, 3089 3203 }; 3090 3204 3091 3205 match fs::remove_file(&path) { 3092 3206 Ok(_) => { 3093 3207 files_removed += 1; 3094 3208 if verbose { 3095 3095 - log::info!(" ✓ Removed: {} ({})", 3096 3096 - path.file_name().and_then(|n| n.to_str()).unwrap_or("?"), 3097 3097 - crate::format::format_bytes(file_size)); 3209 3209 + log::info!( 3210 3210 + " ✓ Removed: {} ({})", 3211 3211 + path.file_name() 3212 3212 + .and_then(|n| n.to_str()) 3213 3213 + .unwrap_or("?"), 3214 3214 + crate::format::format_bytes(file_size) 3215 3215 + ); 3098 3216 } 3099 3217 } 3100 3218 Err(e) => { 3101 3101 - let error_msg = format!("Failed to remove {}: {}", path.display(), e); 3219 3219 + let error_msg = 3220 3220 + format!("Failed to remove {}: {}", path.display(), e); 3102 3221 errors.push(error_msg.clone()); 3103 3222 if verbose { 3104 3223 log::warn!(" ✗ {}", error_msg); ··· 3112 3231 log::debug!("Shards directory does not exist: {}", shards_dir.display()); 3113 3232 } 3114 3233 } else if verbose { 3115 3115 - log::debug!("DID index directory does not exist: {}", did_index_dir.display()); 3234 3234 + log::debug!( 3235 3235 + "DID index directory does not exist: {}", 3236 3236 + did_index_dir.display() 3237 3237 + ); 3116 3238 } 3117 3239 3118 3240 // Summary logging 3119 3241 if verbose { 3120 3242 if files_removed > 0 { 3121 3121 - log::info!("Cleanup complete: removed {} file(s), freed {}", 3243 3243 + log::info!( 3244 3244 + "Cleanup complete: removed {} file(s), freed {}", 3122 3245 files_removed, 3123 3123 - crate::format::format_bytes(bytes_freed)); 3246 3246 + crate::format::format_bytes(bytes_freed) 3247 3247 + ); 3124 3248 } else { 3125 3249 log::info!("Cleanup complete: no temporary files found"); 3126 3250 } 3127 3127 - 3251 3251 + 3128 3252 if !errors.is_empty() { 3129 3253 log::warn!("Encountered {} error(s) during cleanup", errors.len()); 3130 3254 } ··· 3133 3257 Ok(CleanResult { 3134 3258 files_removed, 3135 3259 bytes_freed, 3136 3136 - errors: if errors.is_empty() { None } else { Some(errors) }, 3260 3260 + errors: if errors.is_empty() { 3261 3261 + None 3262 3262 + } else { 3263 3263 + Some(errors) 3264 3264 + }, 3137 3265 }) 3138 3266 } 3139 3267 ··· 3374 3502 3375 3503 fn matches_filter(&self, op: &Operation, filter: &OperationFilter) -> bool { 3376 3504 if let Some(ref did) = filter.did 3377 3377 - && &op.did != did { 3505 3505 + && &op.did != did 3506 3506 + { 3378 3507 return false; 3379 3508 } 3380 3509 3381 3510 if let Some(ref op_type) = filter.operation_type 3382 3382 - && &op.operation != op_type { 3511 3511 + && &op.operation != op_type 3512 3512 + { 3383 3513 return false; 3384 3514 } 3385 3515 ··· 3471 3601 F: Fn(u32, usize, usize, u64) + Send + Sync + 'static, 3472 3602 { 3473 3603 use crate::remote::RemoteClient; 3474 3474 - use std::sync::atomic::{AtomicUsize, AtomicU64, Ordering}; 3604 3604 + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; 3475 3605 3476 3606 let target_dir = target_dir.as_ref(); 3477 3607 ··· 3522 3652 } 3523 3653 3524 3654 let count = downloaded.fetch_add(1, Ordering::SeqCst) + 1; 3525 3525 - let bytes = bytes_downloaded.fetch_add(data_len, Ordering::SeqCst) + data_len; 3655 3655 + let bytes = 3656 3656 + bytes_downloaded.fetch_add(data_len, Ordering::SeqCst) + data_len; 3526 3657 3527 3658 // Call progress callback 3528 3659 if let Some(ref cb) = progress_cb { ··· 3669 3800 pub enum ExportFormat { 3670 3801 JsonLines, 3671 3802 } 3672 3672 - 3673 3803 3674 3804 /// Statistics collected during export 3675 3805 #[derive(Debug, Default)]

+57 -43

src/mempool.rs

reviewed

··· 1 1 //! Persistent pre-bundle operation store with strict chronological validation, CID deduplication, incremental saving, and fast DID lookups 2 2 // src/mempool.rs 3 3 use crate::constants; 4 4 + use crate::format::format_std_duration_ms; 4 5 use crate::operations::Operation; 5 6 use anyhow::{Result, bail}; 6 7 use chrono::{DateTime, Utc}; 7 8 use log::{debug, info}; 8 8 - use crate::format::format_std_duration_ms; 9 9 use std::collections::{HashMap, HashSet}; 10 10 use std::fs::{self, File, OpenOptions}; 11 11 use std::io::{BufRead, BufReader, BufWriter, Write}; ··· 104 104 None => { 105 105 skipped_no_cid += 1; 106 106 continue; 107 107 - }, 107 107 + } 108 108 }; 109 109 110 110 // Skip duplicates ··· 149 149 // Add new operations and update DID index 150 150 let start_idx = self.operations.len(); 151 151 self.operations.extend(new_ops); 152 152 - 152 152 + 153 153 // Update DID index for new operations 154 154 for (offset, op) in self.operations[start_idx..].iter().enumerate() { 155 155 let idx = start_idx + offset; 156 156 self.did_index.entry(op.did.clone()).or_default().push(idx); 157 157 } 158 158 - 158 158 + 159 159 self.validated = true; 160 160 self.dirty = true; 161 161 ··· 449 449 let op = Operation::from_json(&line)?; 450 450 let idx = self.operations.len(); 451 451 self.operations.push(op); 452 452 - 452 452 + 453 453 // Update DID index 454 454 let did = self.operations[idx].did.clone(); 455 455 self.did_index.entry(did).or_default().push(idx); ··· 562 562 pub fn find_did_operations(&self, did: &str) -> Vec<Operation> { 563 563 if let Some(indices) = self.did_index.get(did) { 564 564 // Fast path: use index 565 565 - indices.iter().map(|&idx| self.operations[idx].clone()).collect() 565 565 + indices 566 566 + .iter() 567 567 + .map(|&idx| self.operations[idx].clone()) 568 568 + .collect() 566 569 } else { 567 570 // DID not in index (shouldn't happen if index is maintained correctly) 568 571 Vec::new() 569 572 } 570 573 } 571 571 - 574 574 + 572 575 /// Rebuild DID index from operations (used after take/clear) 573 576 fn rebuild_did_index(&mut self) { 574 577 self.did_index.clear(); ··· 584 587 if let Some(indices) = self.did_index.get(did) { 585 588 // Operations are in chronological order, so highest index = latest 586 589 // Find the highest index that's not nullified 587 587 - indices 588 588 - .iter() 589 589 - .rev() 590 590 - .find_map(|&idx| { 591 591 - let op = &self.operations[idx]; 592 592 - if !op.nullified { 593 593 - Some((op.clone(), idx)) 594 594 - } else { 595 595 - None 596 596 - } 597 597 - }) 590 590 + indices.iter().rev().find_map(|&idx| { 591 591 + let op = &self.operations[idx]; 592 592 + if !op.nullified { 593 593 + Some((op.clone(), idx)) 594 594 + } else { 595 595 + None 596 596 + } 597 597 + }) 598 598 } else { 599 599 None 600 600 } ··· 650 650 .unwrap() 651 651 .with_timezone(&Utc); 652 652 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 653 653 - 653 653 + 654 654 assert_eq!(mempool.count(), 0); 655 655 - 655 655 + 656 656 let ops = vec![ 657 657 create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z"), 658 658 create_test_operation("did:plc:test2", "cid2", "2024-01-01T00:00:02Z"), ··· 688 688 .unwrap() 689 689 .with_timezone(&Utc); 690 690 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 691 691 - 691 691 + 692 692 assert_eq!(mempool.get_first_time(), None); 693 693 - 694 694 - let ops = vec![create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z")]; 693 693 + 694 694 + let ops = vec![create_test_operation( 695 695 + "did:plc:test1", 696 696 + "cid1", 697 697 + "2024-01-01T00:00:01Z", 698 698 + )]; 695 699 mempool.add(ops).unwrap(); 696 696 - assert_eq!(mempool.get_first_time(), Some("2024-01-01T00:00:01Z".to_string())); 700 700 + assert_eq!( 701 701 + mempool.get_first_time(), 702 702 + Some("2024-01-01T00:00:01Z".to_string()) 703 703 + ); 697 704 } 698 705 699 706 #[test] ··· 703 710 .unwrap() 704 711 .with_timezone(&Utc); 705 712 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 706 706 - 713 713 + 707 714 assert_eq!(mempool.get_last_time(), None); 708 708 - 715 715 + 709 716 let ops = vec![ 710 717 create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z"), 711 718 create_test_operation("did:plc:test2", "cid2", "2024-01-01T00:00:02Z"), 712 719 ]; 713 720 mempool.add(ops).unwrap(); 714 714 - assert_eq!(mempool.get_last_time(), Some("2024-01-01T00:00:02Z".to_string())); 721 721 + assert_eq!( 722 722 + mempool.get_last_time(), 723 723 + Some("2024-01-01T00:00:02Z".to_string()) 724 724 + ); 715 725 } 716 726 717 727 #[test] ··· 721 731 .unwrap() 722 732 .with_timezone(&Utc); 723 733 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 724 724 - 734 734 + 725 735 let ops = vec![ 726 736 create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z"), 727 737 create_test_operation("did:plc:test2", "cid2", "2024-01-01T00:00:02Z"), 728 738 create_test_operation("did:plc:test3", "cid3", "2024-01-01T00:00:03Z"), 729 739 ]; 730 740 mempool.add(ops).unwrap(); 731 731 - 741 741 + 732 742 let peeked = mempool.peek(2); 733 743 assert_eq!(peeked.len(), 2); 734 744 assert_eq!(peeked[0].cid, Some("cid1".to_string())); 735 745 assert_eq!(peeked[1].cid, Some("cid2".to_string())); 736 736 - 746 746 + 737 747 // Peek should not remove operations 738 748 assert_eq!(mempool.count(), 3); 739 749 } ··· 745 755 .unwrap() 746 756 .with_timezone(&Utc); 747 757 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 748 748 - 749 749 - let ops = vec![create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z")]; 758 758 + 759 759 + let ops = vec![create_test_operation( 760 760 + "did:plc:test1", 761 761 + "cid1", 762 762 + "2024-01-01T00:00:01Z", 763 763 + )]; 750 764 mempool.add(ops).unwrap(); 751 751 - 765 765 + 752 766 let peeked = mempool.peek(10); 753 767 assert_eq!(peeked.len(), 1); 754 768 } ··· 760 774 .unwrap() 761 775 .with_timezone(&Utc); 762 776 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 763 763 - 777 777 + 764 778 let ops = vec![ 765 779 create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z"), 766 780 create_test_operation("did:plc:test2", "cid2", "2024-01-01T00:00:02Z"), 767 781 ]; 768 782 mempool.add(ops).unwrap(); 769 783 assert_eq!(mempool.count(), 2); 770 770 - 784 784 + 771 785 mempool.clear(); 772 786 assert_eq!(mempool.count(), 0); 773 787 } ··· 779 793 .unwrap() 780 794 .with_timezone(&Utc); 781 795 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 782 782 - 796 796 + 783 797 let ops = vec![ 784 798 create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z"), 785 799 create_test_operation("did:plc:test1", "cid2", "2024-01-01T00:00:02Z"), 786 800 create_test_operation("did:plc:test2", "cid3", "2024-01-01T00:00:03Z"), 787 801 ]; 788 802 mempool.add(ops).unwrap(); 789 789 - 803 803 + 790 804 let found = mempool.find_did_operations("did:plc:test1"); 791 805 assert_eq!(found.len(), 2); 792 806 assert_eq!(found[0].cid, Some("cid1".to_string())); 793 807 assert_eq!(found[1].cid, Some("cid2".to_string())); 794 794 - 808 808 + 795 809 let found = mempool.find_did_operations("did:plc:test2"); 796 810 assert_eq!(found.len(), 1); 797 811 assert_eq!(found[0].cid, Some("cid3".to_string())); 798 798 - 812 812 + 799 813 let found = mempool.find_did_operations("did:plc:nonexistent"); 800 814 assert_eq!(found.len(), 0); 801 815 } ··· 807 821 .unwrap() 808 822 .with_timezone(&Utc); 809 823 let mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 810 810 - 824 824 + 811 825 let stats = mempool.stats(); 812 826 assert_eq!(stats.count, 0); 813 827 assert!(!stats.can_create_bundle); ··· 821 835 .unwrap() 822 836 .with_timezone(&Utc); 823 837 let mut mempool = Mempool::new(tmp.path(), 1, min_time, false).unwrap(); 824 824 - 838 838 + 825 839 let ops = vec![ 826 840 create_test_operation("did:plc:test1", "cid1", "2024-01-01T00:00:01Z"), 827 841 create_test_operation("did:plc:test2", "cid2", "2024-01-01T00:00:02Z"), 828 842 ]; 829 843 mempool.add(ops).unwrap(); 830 830 - 844 844 + 831 845 let stats = mempool.stats(); 832 846 assert_eq!(stats.count, 2); 833 847 assert!(!stats.can_create_bundle); // Need BUNDLE_SIZE (10000) ops ··· 844 858 .unwrap() 845 859 .with_timezone(&Utc); 846 860 let mempool = Mempool::new(tmp.path(), 42, min_time, false).unwrap(); 847 847 - 861 861 + 848 862 let filename = mempool.get_filename(); 849 863 assert!(filename.contains("plc_mempool_")); 850 864 assert!(filename.contains("000042"));

+42 -25

src/operations.rs

reviewed

··· 6 6 /// PLC Operation 7 7 /// 8 8 /// Represents a single operation from the PLC directory. 9 9 - /// 9 9 + /// 10 10 /// **IMPORTANT**: This struct uses `sonic_rs` for JSON parsing (not serde). 11 11 /// Serialization still uses serde for compatibility with JMESPath queries. 12 12 #[derive(Debug, Clone, Serialize, Deserialize)] ··· 39 39 40 40 impl Operation { 41 41 /// Parse an Operation from JSON using sonic_rs (not serde) 42 42 - /// 42 42 + /// 43 43 /// This method manually extracts fields from the JSON to avoid issues with 44 44 /// serde attributes like `#[serde(flatten)]` that sonic_rs may not fully support. 45 45 pub fn from_json(json: &str) -> anyhow::Result<Self> { 46 46 use anyhow::Context; 47 47 use sonic_rs::JsonValueTrait; 48 48 - 49 49 - let value: Value = sonic_rs::from_str(json) 50 50 - .context("Failed to parse JSON")?; 51 51 - 48 48 + 49 49 + let value: Value = sonic_rs::from_str(json).context("Failed to parse JSON")?; 50 50 + 52 51 // Extract required fields 53 53 - let did = value.get("did") 52 52 + let did = value 53 53 + .get("did") 54 54 .and_then(|v| v.as_str()) 55 55 .ok_or_else(|| anyhow::anyhow!("missing 'did' field"))? 56 56 .to_string(); 57 57 - 58 58 - let operation = value.get("operation") 59 59 - .cloned() 60 60 - .unwrap_or_else(Value::new); 61 61 - 57 57 + 58 58 + let operation = value.get("operation").cloned().unwrap_or_else(Value::new); 59 59 + 62 60 // Extract optional fields with defaults 63 63 - let cid = value.get("cid") 61 61 + let cid = value 62 62 + .get("cid") 64 63 .and_then(|v| v.as_str()) 65 64 .map(|s| s.to_string()); 66 66 - 67 67 - let nullified = value.get("nullified") 65 65 + 66 66 + let nullified = value 67 67 + .get("nullified") 68 68 .and_then(|v| v.as_bool()) 69 69 .unwrap_or(false); 70 70 - 70 70 + 71 71 // Handle both "createdAt" and "created_at" field names 72 72 - let created_at = value.get("createdAt") 72 72 + let created_at = value 73 73 + .get("createdAt") 73 74 .or_else(|| value.get("created_at")) 74 75 .and_then(|v| v.as_str()) 75 76 .ok_or_else(|| anyhow::anyhow!("missing 'createdAt' or 'created_at' field"))? 76 77 .to_string(); 77 77 - 78 78 + 78 79 // Extract extra fields (everything except known fields) 79 80 // Since sonic_rs::Value doesn't provide easy iteration, we'll parse the JSON 80 81 // and manually extract extra fields by checking for unknown keys ··· 82 83 // The extra field was used with #[serde(flatten)] but we can reconstruct it if needed 83 84 // For performance, we'll just use an empty Value since most operations don't have extra fields 84 85 let extra = Value::new(); 85 85 - 86 86 + 86 87 Ok(Operation { 87 88 did, 88 89 operation, ··· 155 156 assert_eq!(op.did, "did:plc:abcdefghijklmnopqrstuvwx"); 156 157 assert!(op.nullified); 157 158 assert_eq!(op.created_at, "2024-01-01T12:34:56Z"); 158 158 - assert_eq!(op.cid, Some("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi".to_string())); 159 159 - 159 159 + assert_eq!( 160 160 + op.cid, 161 161 + Some("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi".to_string()) 162 162 + ); 163 163 + 160 164 // Check operation field 161 165 let op_type = op.operation.get("type").and_then(|v| v.as_str()).unwrap(); 162 166 assert_eq!(op_type, "create"); ··· 179 183 180 184 let op1 = Operation::from_json(json1).unwrap(); 181 185 let op2 = Operation::from_json(json2).unwrap(); 182 182 - 186 186 + 183 187 assert_eq!(op1.created_at, "2024-01-01T00:00:00Z"); 184 188 assert_eq!(op2.created_at, "2024-01-01T00:00:00Z"); 185 189 } ··· 193 197 194 198 let result = Operation::from_json(json); 195 199 assert!(result.is_err()); 196 196 - assert!(result.unwrap_err().to_string().contains("missing 'did' field")); 200 200 + assert!( 201 201 + result 202 202 + .unwrap_err() 203 203 + .to_string() 204 204 + .contains("missing 'did' field") 205 205 + ); 197 206 } 198 207 199 208 #[test] ··· 205 214 206 215 let result = Operation::from_json(json); 207 216 assert!(result.is_err()); 208 208 - assert!(result.unwrap_err().to_string().contains("missing 'createdAt' or 'created_at' field")); 217 217 + assert!( 218 218 + result 219 219 + .unwrap_err() 220 220 + .to_string() 221 221 + .contains("missing 'createdAt' or 'created_at' field") 222 222 + ); 209 223 } 210 224 211 225 #[test] ··· 275 289 assert_eq!(request.bundle, 1); 276 290 assert_eq!(request.index, Some(5)); 277 291 assert!(request.filter.is_some()); 278 278 - assert_eq!(request.filter.as_ref().unwrap().did, Some("did:plc:test".to_string())); 292 292 + assert_eq!( 293 293 + request.filter.as_ref().unwrap().did, 294 294 + Some("did:plc:test".to_string()) 295 295 + ); 279 296 } 280 297 281 298 #[test]

+5 -13

src/options.rs

reviewed

··· 120 120 121 121 #[test] 122 122 fn test_options_builder_directory() { 123 123 - let opts = OptionsBuilder::new() 124 124 - .directory("/tmp/test") 125 125 - .build(); 123 123 + let opts = OptionsBuilder::new().directory("/tmp/test").build(); 126 124 assert_eq!(opts.directory, PathBuf::from("/tmp/test")); 127 125 } 128 126 129 127 #[test] 130 128 fn test_options_builder_query() { 131 131 - let opts = OptionsBuilder::new() 132 132 - .query("operation.type") 133 133 - .build(); 129 129 + let opts = OptionsBuilder::new().query("operation.type").build(); 134 130 assert_eq!(opts.query, "operation.type"); 135 131 } 136 132 ··· 144 140 145 141 #[test] 146 142 fn test_options_builder_num_threads() { 147 147 - let opts = OptionsBuilder::new() 148 148 - .num_threads(4) 149 149 - .build(); 143 143 + let opts = OptionsBuilder::new().num_threads(4).build(); 150 144 assert_eq!(opts.num_threads, 4); 151 145 } 152 146 153 147 #[test] 154 148 fn test_options_builder_batch_size() { 155 155 - let opts = OptionsBuilder::new() 156 156 - .batch_size(5000) 157 157 - .build(); 149 149 + let opts = OptionsBuilder::new().batch_size(5000).build(); 158 150 assert_eq!(opts.batch_size, 5000); 159 151 } 160 152 ··· 167 159 .num_threads(8) 168 160 .batch_size(1000) 169 161 .build(); 170 170 - 162 162 + 171 163 assert_eq!(opts.directory, PathBuf::from("/tmp/test")); 172 164 assert_eq!(opts.query, "did"); 173 165 assert_eq!(opts.query_mode, QueryMode::JmesPath);

+13 -7

src/plc_client.rs

reviewed

··· 205 205 206 206 log::debug!("Fetching DID document from: {}", url); 207 207 let request_start = Instant::now(); 208 208 - 208 208 + 209 209 let response = self 210 210 .client 211 211 .get(&url) ··· 215 215 .context(format!("Failed to fetch DID document from {}", url))?; 216 216 217 217 let request_duration = request_start.elapsed(); 218 218 - log::debug!("HTTP request completed in {:?}, status: {}", request_duration, response.status()); 218 218 + log::debug!( 219 219 + "HTTP request completed in {:?}, status: {}", 220 220 + request_duration, 221 221 + response.status() 222 222 + ); 219 223 220 224 // Handle rate limiting (429) 221 225 if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS { ··· 226 230 } 227 231 228 232 if !response.status().is_success() { 229 229 - log::error!("Unexpected status code: {} for DID document at {}", response.status(), url); 233 233 + log::error!( 234 234 + "Unexpected status code: {} for DID document at {}", 235 235 + response.status(), 236 236 + url 237 237 + ); 230 238 anyhow::bail!( 231 239 "Unexpected status code: {} for DID document at {}", 232 240 response.status(), ··· 247 255 /// Uses the /did/{did} endpoint. 248 256 pub async fn fetch_did_document(&self, did: &str) -> Result<DIDDocument> { 249 257 let data = self.fetch_did_document_raw(did).await?; 250 250 - let document: DIDDocument = sonic_rs::from_str(&data) 251 251 - .context("Failed to parse DID document JSON")?; 258 258 + let document: DIDDocument = 259 259 + sonic_rs::from_str(&data).context("Failed to parse DID document JSON")?; 252 260 Ok(document) 253 261 } 254 262 } ··· 280 288 // Default to 60 seconds if no header or parsing fails 281 289 Duration::from_secs(MAX_RETRY_SECONDS) 282 290 } 283 283 - 284 291 285 292 /// Simple token bucket rate limiter 286 293 /// Prevents burst requests by starting with 0 permits and refilling at steady rate ··· 347 354 assert!(client.base_url.contains("plc.directory")); 348 355 } 349 356 } 350 350 -

+85 -19

src/processor.rs

reviewed

··· 273 273 274 274 // Handle "head~N" syntax 275 275 if let Some(rest) = keyword.strip_prefix("head~") { 276 276 - let offset: u32 = rest.parse() 276 276 + let offset: u32 = rest 277 277 + .parse() 277 278 .map_err(|_| anyhow::anyhow!("Invalid offset in 'head~{}': expected a number", rest))?; 278 279 if offset >= max_bundle { 279 279 - anyhow::bail!("Offset {} in 'head~{}' exceeds maximum bundle {}", offset, rest, max_bundle); 280 280 + anyhow::bail!( 281 281 + "Offset {} in 'head~{}' exceeds maximum bundle {}", 282 282 + offset, 283 283 + rest, 284 284 + max_bundle 285 285 + ); 280 286 } 281 287 return Ok(max_bundle - offset); 282 288 } 283 289 284 290 // Handle "~N" shorthand syntax 285 291 if let Some(rest) = keyword.strip_prefix('~') { 286 286 - let offset: u32 = rest.parse() 292 292 + let offset: u32 = rest 293 293 + .parse() 287 294 .map_err(|_| anyhow::anyhow!("Invalid offset in '~{}': expected a number", rest))?; 288 295 if offset >= max_bundle { 289 289 - anyhow::bail!("Offset {} in '~{}' exceeds maximum bundle {}", offset, rest, max_bundle); 296 296 + anyhow::bail!( 297 297 + "Offset {} in '~{}' exceeds maximum bundle {}", 298 298 + offset, 299 299 + rest, 300 300 + max_bundle 301 301 + ); 290 302 } 291 303 return Ok(max_bundle - offset); 292 304 } 293 305 294 306 // Not a keyword, try parsing as number 295 295 - let num: u32 = keyword.parse() 296 296 - .map_err(|_| anyhow::anyhow!("Invalid bundle specifier: '{}' (expected number, 'root', 'head', 'head~N', or '~N')", keyword))?; 307 307 + let num: u32 = keyword.parse().map_err(|_| { 308 308 + anyhow::anyhow!( 309 309 + "Invalid bundle specifier: '{}' (expected number, 'root', 'head', 'head~N', or '~N')", 310 310 + keyword 311 311 + ) 312 312 + })?; 297 313 Ok(num) 298 314 } 299 315 ··· 328 344 anyhow::bail!("Invalid range: {} > {} (start must be <= end)", start, end); 329 345 } 330 346 if start > max_bundle || end > max_bundle { 331 331 - anyhow::bail!("Invalid range: {}-{} (exceeds maximum bundle {})", start, end, max_bundle); 347 347 + anyhow::bail!( 348 348 + "Invalid range: {}-{} (exceeds maximum bundle {})", 349 349 + start, 350 350 + end, 351 351 + max_bundle 352 352 + ); 332 353 } 333 354 bundles.extend(start..=end); 334 355 } else { ··· 351 372 /// Operations are 0-indexed global positions (0 = first operation, bundle 1 position 0) 352 373 pub fn parse_operation_range(spec: &str, max_operation: u64) -> Result<Vec<u64>> { 353 374 use anyhow::Context; 354 354 - 375 375 + 355 376 if max_operation == 0 { 356 377 anyhow::bail!("No operations available"); 357 378 } ··· 484 505 let mut buffer = OutputBuffer::new(10); 485 506 buffer.push("line1"); 486 507 buffer.push("line2"); 487 487 - 508 508 + 488 509 let flushed = buffer.flush(); 489 510 assert_eq!(flushed, "line1\nline2\n"); 490 511 assert!(buffer.is_empty()); ··· 495 516 let mut buffer = OutputBuffer::new(10); 496 517 buffer.push("line1"); 497 518 buffer.push("line2"); 498 498 - 519 519 + 499 520 // Each line adds len + 1 (for newline) 500 521 assert_eq!(buffer.get_matched_bytes(), 5 + 1 + 5 + 1); 501 522 } ··· 534 555 fn test_parse_bundle_range_empty_max() { 535 556 let result = parse_bundle_range("1", 0); 536 557 assert!(result.is_err()); 537 537 - assert!(result.unwrap_err().to_string().contains("No bundles available")); 558 558 + assert!( 559 559 + result 560 560 + .unwrap_err() 561 561 + .to_string() 562 562 + .contains("No bundles available") 563 563 + ); 538 564 } 539 565 540 566 #[test] ··· 548 574 fn test_parse_bundle_range_invalid_range() { 549 575 let result = parse_bundle_range("5-3", 10); 550 576 assert!(result.is_err()); 551 551 - assert!(result.unwrap_err().to_string().contains("start must be <= end")); 577 577 + assert!( 578 578 + result 579 579 + .unwrap_err() 580 580 + .to_string() 581 581 + .contains("start must be <= end") 582 582 + ); 552 583 } 553 584 554 585 #[test] 555 586 fn test_parse_bundle_range_invalid_format() { 556 587 let result = parse_bundle_range("1-2-3", 10); 557 588 assert!(result.is_err()); 558 558 - assert!(result.unwrap_err().to_string().contains("Invalid range format")); 589 589 + assert!( 590 590 + result 591 591 + .unwrap_err() 592 592 + .to_string() 593 593 + .contains("Invalid range format") 594 594 + ); 559 595 } 560 596 561 597 #[test] ··· 592 628 fn test_parse_bundle_range_keyword_head_offset_invalid() { 593 629 let result = parse_bundle_range("head~10", 10); 594 630 assert!(result.is_err()); 595 595 - assert!(result.unwrap_err().to_string().contains("exceeds maximum bundle")); 631 631 + assert!( 632 632 + result 633 633 + .unwrap_err() 634 634 + .to_string() 635 635 + .contains("exceeds maximum bundle") 636 636 + ); 596 637 } 597 638 598 639 #[test] 599 640 fn test_parse_bundle_range_keyword_shorthand_offset_invalid() { 600 641 let result = parse_bundle_range("~10", 10); 601 642 assert!(result.is_err()); 602 602 - assert!(result.unwrap_err().to_string().contains("exceeds maximum bundle")); 643 643 + assert!( 644 644 + result 645 645 + .unwrap_err() 646 646 + .to_string() 647 647 + .contains("exceeds maximum bundle") 648 648 + ); 603 649 } 604 650 605 651 #[test] ··· 643 689 fn test_parse_operation_range_empty_max() { 644 690 let result = parse_operation_range("0", 0); 645 691 assert!(result.is_err()); 646 646 - assert!(result.unwrap_err().to_string().contains("No operations available")); 692 692 + assert!( 693 693 + result 694 694 + .unwrap_err() 695 695 + .to_string() 696 696 + .contains("No operations available") 697 697 + ); 647 698 } 648 699 649 700 #[test] ··· 657 708 fn test_parse_operation_range_invalid_range() { 658 709 let result = parse_operation_range("10-5", 1000); 659 710 assert!(result.is_err()); 660 660 - assert!(result.unwrap_err().to_string().contains("start must be <= end")); 711 711 + assert!( 712 712 + result 713 713 + .unwrap_err() 714 714 + .to_string() 715 715 + .contains("start must be <= end") 716 716 + ); 661 717 } 662 718 663 719 #[test] 664 720 fn test_parse_operation_range_invalid_format() { 665 721 let result = parse_operation_range("1-2-3", 1000); 666 722 assert!(result.is_err()); 667 667 - assert!(result.unwrap_err().to_string().contains("Invalid range format")); 723 723 + assert!( 724 724 + result 725 725 + .unwrap_err() 726 726 + .to_string() 727 727 + .contains("Invalid range format") 728 728 + ); 668 729 } 669 730 670 731 #[test] 671 732 fn test_parse_operation_range_invalid_number() { 672 733 let result = parse_operation_range("abc", 1000); 673 734 assert!(result.is_err()); 674 674 - assert!(result.unwrap_err().to_string().contains("Invalid operation number")); 735 735 + assert!( 736 736 + result 737 737 + .unwrap_err() 738 738 + .to_string() 739 739 + .contains("Invalid operation number") 740 740 + ); 675 741 } 676 742 677 743 #[test]

+35 -17

src/remote.rs

reviewed

··· 29 29 pub async fn fetch_index(&self) -> Result<Index> { 30 30 // Try both common index file names 31 31 let mut url = format!("{}/index.json", self.base_url); 32 32 - 32 32 + 33 33 let response = self 34 34 .client 35 35 .get(&url) ··· 48 48 .send() 49 49 .await 50 50 .context("Failed to fetch index")?; 51 51 - 51 51 + 52 52 if !response2.status().is_success() { 53 53 anyhow::bail!("Unexpected status code: {}", response2.status()); 54 54 } 55 55 - 55 55 + 56 56 let data = response2.text().await?; 57 57 let index: Index = sonic_rs::from_str(&data).context("Failed to parse index JSON")?; 58 58 return Ok(index); ··· 155 155 } 156 156 157 157 let data = response.text().await?; 158 158 - let document: DIDDocument = sonic_rs::from_str(&data) 159 159 - .context("Failed to parse DID document JSON")?; 158 158 + let document: DIDDocument = 159 159 + sonic_rs::from_str(&data).context("Failed to parse DID document JSON")?; 160 160 161 161 Ok(document) 162 162 } ··· 227 227 #[test] 228 228 fn test_normalize_base_url() { 229 229 // Test removing trailing slash 230 230 - assert_eq!(normalize_base_url("https://example.com/".to_string()), "https://example.com"); 231 231 - 230 230 + assert_eq!( 231 231 + normalize_base_url("https://example.com/".to_string()), 232 232 + "https://example.com" 233 233 + ); 234 234 + 232 235 // Test removing index.json 233 233 - assert_eq!(normalize_base_url("https://example.com/index.json".to_string()), "https://example.com"); 234 234 - 236 236 + assert_eq!( 237 237 + normalize_base_url("https://example.com/index.json".to_string()), 238 238 + "https://example.com" 239 239 + ); 240 240 + 235 241 // Test removing plc_bundles.json 236 236 - assert_eq!(normalize_base_url("https://example.com/plc_bundles.json".to_string()), "https://example.com"); 237 237 - 242 242 + assert_eq!( 243 243 + normalize_base_url("https://example.com/plc_bundles.json".to_string()), 244 244 + "https://example.com" 245 245 + ); 246 246 + 238 247 // Test removing both trailing slash and index file 239 239 - assert_eq!(normalize_base_url("https://example.com/index.json/".to_string()), "https://example.com"); 240 240 - 248 248 + assert_eq!( 249 249 + normalize_base_url("https://example.com/index.json/".to_string()), 250 250 + "https://example.com" 251 251 + ); 252 252 + 241 253 // Test already normalized URL 242 242 - assert_eq!(normalize_base_url("https://example.com".to_string()), "https://example.com"); 243 243 - 254 254 + assert_eq!( 255 255 + normalize_base_url("https://example.com".to_string()), 256 256 + "https://example.com" 257 257 + ); 258 258 + 244 259 // Test with path 245 245 - assert_eq!(normalize_base_url("https://example.com/api/".to_string()), "https://example.com/api"); 260 260 + assert_eq!( 261 261 + normalize_base_url("https://example.com/api/".to_string()), 262 262 + "https://example.com/api" 263 263 + ); 246 264 } 247 265 248 266 #[test] ··· 256 274 fn test_remote_client_new_normalizes_url() { 257 275 let client = RemoteClient::new("https://example.com/").unwrap(); 258 276 assert!(!client.base_url.ends_with('/')); 259 259 - 277 277 + 260 278 let client2 = RemoteClient::new("https://example.com/index.json").unwrap(); 261 279 assert!(!client2.base_url.ends_with("index.json")); 262 280 }

+12 -2

src/resolver.rs

reviewed

··· 377 377 fn test_validate_did_format_wrong_method() { 378 378 let result = validate_did_format("did:web:example.com"); 379 379 assert!(result.is_err()); 380 380 - assert!(result.unwrap_err().to_string().contains("invalid DID method")); 380 380 + assert!( 381 381 + result 382 382 + .unwrap_err() 383 383 + .to_string() 384 384 + .contains("invalid DID method") 385 385 + ); 381 386 } 382 387 383 388 #[test] ··· 385 390 // Too short 386 391 let result = validate_did_format("did:plc:short"); 387 392 assert!(result.is_err()); 388 388 - assert!(result.unwrap_err().to_string().contains("invalid DID length")); 393 393 + assert!( 394 394 + result 395 395 + .unwrap_err() 396 396 + .to_string() 397 397 + .contains("invalid DID length") 398 398 + ); 389 399 390 400 // Too long 391 401 let result = validate_did_format("did:plc:abcdefghijklmnopqrstuvwxyz");

+15 -10

src/runtime.rs

reviewed

··· 1 1 //! Graceful shutdown coordination for server and background tasks, with unified shutdown future and fatal-error handling 2 2 // Runtime module - shutdown coordination for server and background tasks 3 3 + use std::future::Future; 4 4 + use std::sync::Arc; 5 5 + use std::sync::atomic::{AtomicBool, Ordering}; 3 6 use tokio::signal; 4 7 use tokio::sync::watch; 5 8 use tokio::task::JoinSet; 6 6 - use std::future::Future; 7 7 - use std::sync::Arc; 8 8 - use std::sync::atomic::{AtomicBool, Ordering}; 9 9 10 10 /// Lightweight coordination for bundle operations shutdown and background tasks 11 11 /// Used by both server and sync continuous mode for graceful shutdown handling ··· 77 77 } 78 78 79 79 /// Common shutdown cleanup handler for server and sync commands 80 80 - /// 80 80 + /// 81 81 /// This method handles the common pattern of: 82 82 /// 1. Triggering shutdown (if not already triggered) 83 83 /// 2. Aborting resolver tasks immediately (if any) 84 84 /// 3. Handling background tasks based on shutdown type (fatal vs normal) 85 85 /// 4. Printing completion message 86 86 - /// 86 86 + /// 87 87 /// # Arguments 88 88 /// * `service_name` - Name of the service (e.g., "Server", "Sync") for messages 89 89 /// * `resolver_tasks` - Optional resolver tasks to abort immediately ··· 98 98 self.trigger_shutdown(); 99 99 100 100 // Always abort resolver tasks immediately - they're just keep-alive pings 101 101 - if let Some(resolver_tasks) = resolver_tasks && !resolver_tasks.is_empty() { 101 101 + if let Some(resolver_tasks) = resolver_tasks 102 102 + && !resolver_tasks.is_empty() 103 103 + { 102 104 resolver_tasks.abort_all(); 103 105 while let Some(result) = resolver_tasks.join_next().await { 104 104 - if let Err(e) = result && !e.is_cancelled() { 106 106 + if let Err(e) = result 107 107 + && !e.is_cancelled() 108 108 + { 105 109 eprintln!("Resolver task error: {}", e); 106 110 } 107 111 } ··· 115 119 background_tasks.abort_all(); 116 120 // Wait briefly for aborted tasks to finish 117 121 while let Some(result) = background_tasks.join_next().await { 118 118 - if let Err(e) = result && !e.is_cancelled() { 122 122 + if let Err(e) = result 123 123 + && !e.is_cancelled() 124 124 + { 119 125 eprintln!("Background task error: {}", e); 120 126 } 121 127 } ··· 145 151 #[cfg(test)] 146 152 mod tests { 147 153 use super::*; 148 148 - use tokio::time::{sleep, Duration}; 154 154 + use tokio::time::{Duration, sleep}; 149 155 150 156 #[tokio::test] 151 157 async fn test_programmatic_shutdown() { ··· 199 205 assert!(*rx.borrow()); 200 206 } 201 207 } 202 202 -

+1 -5

src/server/error.rs

reviewed

··· 1 1 // Error handling utilities and response helpers 2 2 3 3 - use axum::{ 4 4 - http::StatusCode, 5 5 - response::IntoResponse, 6 6 - }; 3 3 + use axum::{http::StatusCode, response::IntoResponse}; 7 4 use serde_json::json; 8 5 9 6 /// Helper to create a JSON error response ··· 45 42 let msg = e.to_string(); 46 43 msg.contains("not found") || msg.contains("not in index") 47 44 } 48 48 -

+5 -6

src/server/handle_bundle.rs

reviewed

··· 1 1 // Bundle-related handlers 2 2 3 3 use crate::constants; 4 4 - use crate::server::error::{bad_request, internal_error, is_not_found_error, not_found, task_join_error}; 5 4 use crate::server::ServerState; 5 5 + use crate::server::error::{ 6 6 + bad_request, internal_error, is_not_found_error, not_found, task_join_error, 7 7 + }; 6 8 use crate::server::utils::{bundle_download_headers, parse_operation_pointer}; 7 9 use axum::{ 8 10 body::Body, ··· 43 45 let stream = ReaderStream::new(file); 44 46 let body = Body::from_stream(stream); 45 47 46 46 - let headers = bundle_download_headers( 47 47 - "application/zstd", 48 48 - &constants::bundle_filename(number), 49 49 - ); 48 48 + let headers = 49 49 + bundle_download_headers("application/zstd", &constants::bundle_filename(number)); 50 50 51 51 (StatusCode::OK, headers, body).into_response() 52 52 } ··· 167 167 Err(e) => task_join_error(e).into_response(), 168 168 } 169 169 } 170 170 -

+3 -8

src/server/handle_debug.rs

reviewed

··· 1 1 // Debug handlers 2 2 3 3 - use crate::server::error::not_found; 4 3 use crate::server::ServerState; 4 4 + use crate::server::error::not_found; 5 5 use axum::{ 6 6 extract::State, 7 7 http::{HeaderMap, HeaderValue, StatusCode}, ··· 14 14 // Get DID index stats for memory info (avoid holding lock in async context) 15 15 let did_stats = tokio::task::spawn_blocking({ 16 16 let manager = Arc::clone(&state.manager); 17 17 - move || { 18 18 - manager.get_did_index_stats() 19 19 - } 17 17 + move || manager.get_did_index_stats() 20 18 }) 21 19 .await 22 20 .unwrap_or_default(); ··· 43 41 // Avoid holding lock in async context 44 42 let stats = tokio::task::spawn_blocking({ 45 43 let manager = Arc::clone(&state.manager); 46 46 - move || { 47 47 - manager.get_did_index_stats() 48 48 - } 44 44 + move || manager.get_did_index_stats() 49 45 }) 50 46 .await 51 47 .unwrap_or_default(); ··· 60 56 let resolver_stats = state.manager.get_resolver_stats(); 61 57 (StatusCode::OK, axum::Json(json!(resolver_stats))).into_response() 62 58 } 63 63 -

+46 -18

src/server/handle_did.rs

reviewed

··· 1 1 // DID resolution handlers 2 2 3 3 use crate::resolver::{build_did_state, format_audit_log}; 4 4 - use crate::server::error::{bad_request, gone, internal_error, is_not_found_error, not_found, task_join_error}; 5 4 use crate::server::ServerState; 5 5 + use crate::server::error::{ 6 6 + bad_request, gone, internal_error, is_not_found_error, not_found, task_join_error, 7 7 + }; 6 8 use crate::server::utils::{is_common_browser_file, is_valid_did_or_handle}; 7 9 use axum::{ 8 10 extract::State, ··· 12 14 use serde_json::json; 13 15 use std::sync::Arc; 14 16 15 15 - pub async fn handle_did_routing_guard(State(state): State<ServerState>, uri: Uri) -> impl IntoResponse { 17 17 + pub async fn handle_did_routing_guard( 18 18 + State(state): State<ServerState>, 19 19 + uri: Uri, 20 20 + ) -> impl IntoResponse { 16 21 // Only handle if resolver is enabled, otherwise return 404 17 22 if !state.config.enable_resolver { 18 23 return not_found("not found").into_response(); ··· 109 114 let resolution_source = if is_mempool { "mempool" } else { "bundle" }; 110 115 111 116 // Calculate operation age 112 112 - let operation_age_seconds = if let Ok(op_time) = chrono::DateTime::parse_from_rfc3339(&result.operation.created_at) { 113 113 - let now = chrono::Utc::now(); 114 114 - let op_utc = op_time.with_timezone(&chrono::Utc); 115 115 - (now - op_utc).num_seconds().max(0) as u64 116 116 - } else { 117 117 - 0 118 118 - }; 117 117 + let operation_age_seconds = 118 118 + if let Ok(op_time) = chrono::DateTime::parse_from_rfc3339(&result.operation.created_at) { 119 119 + let now = chrono::Utc::now(); 120 120 + let op_utc = op_time.with_timezone(&chrono::Utc); 121 121 + (now - op_utc).num_seconds().max(0) as u64 122 122 + } else { 123 123 + 0 124 124 + }; 119 125 120 126 // Get operation size 121 121 - let operation_size = result.operation.raw_json.as_ref().map(|s| s.len()).unwrap_or(0); 127 127 + let operation_size = result 128 128 + .operation 129 129 + .raw_json 130 130 + .as_ref() 131 131 + .map(|s| s.len()) 132 132 + .unwrap_or(0); 122 133 123 134 // Set headers 124 135 let mut headers = HeaderMap::new(); ··· 133 144 } else { 134 145 headers.insert("X-Request-Type", HeaderValue::from_static("did")); 135 146 } 136 136 - headers.insert("X-Resolution-Source", HeaderValue::from_str(resolution_source).unwrap()); 137 137 - headers.insert("X-Mempool", HeaderValue::from_str(if is_mempool { "true" } else { "false" }).unwrap()); 147 147 + headers.insert( 148 148 + "X-Resolution-Source", 149 149 + HeaderValue::from_str(resolution_source).unwrap(), 150 150 + ); 151 151 + headers.insert( 152 152 + "X-Mempool", 153 153 + HeaderValue::from_str(if is_mempool { "true" } else { "false" }).unwrap(), 154 154 + ); 138 155 headers.insert( 139 156 "X-Mempool-Time-Ms", 140 140 - HeaderValue::from_str(&format!("{:.3}", result.mempool_time.as_secs_f64() * 1000.0)).unwrap(), 157 157 + HeaderValue::from_str(&format!( 158 158 + "{:.3}", 159 159 + result.mempool_time.as_secs_f64() * 1000.0 160 160 + )) 161 161 + .unwrap(), 141 162 ); 142 163 headers.insert( 143 164 "X-Index-Time-Ms", ··· 147 168 "X-Load-Time-Ms", 148 169 HeaderValue::from_str(&format!("{:.3}", result.load_time.as_secs_f64() * 1000.0)).unwrap(), 149 170 ); 150 150 - 171 171 + 151 172 // Calculate global position 152 173 let global_position = if result.bundle_number == 0 { 153 174 // From mempool ··· 161 182 "X-Global-Position", 162 183 HeaderValue::from_str(&global_position.to_string()).unwrap(), 163 184 ); 164 164 - 185 185 + 165 186 headers.insert("X-Bundle-Number", HeaderValue::from(result.bundle_number)); 166 187 headers.insert("X-Bundle-Position", HeaderValue::from(result.position)); 167 188 headers.insert( ··· 171 192 if let Some(cid) = &result.operation.cid { 172 193 headers.insert("X-Operation-CID", HeaderValue::from_str(cid).unwrap()); 173 194 } 174 174 - headers.insert("X-Operation-Created", HeaderValue::from_str(&result.operation.created_at).unwrap()); 195 195 + headers.insert( 196 196 + "X-Operation-Created", 197 197 + HeaderValue::from_str(&result.operation.created_at).unwrap(), 198 198 + ); 175 199 headers.insert( 176 200 "X-Operation-Nullified", 177 177 - HeaderValue::from_str(if result.operation.nullified { "true" } else { "false" }).unwrap(), 201 201 + HeaderValue::from_str(if result.operation.nullified { 202 202 + "true" 203 203 + } else { 204 204 + "false" 205 205 + }) 206 206 + .unwrap(), 178 207 ); 179 208 headers.insert( 180 209 "X-Operation-Size", ··· 255 284 let audit_log = format_audit_log(&operations); 256 285 (StatusCode::OK, axum::Json(audit_log)).into_response() 257 286 } 258 258 -

+47 -40

src/server/handle_root.rs

reviewed

··· 82 82 } 83 83 } 84 84 85 85 - if state.config.sync_mode && let Ok(mempool_stats) = state.manager.get_mempool_stats() { 86 86 - response.push_str("\nMempool\n"); 87 87 - response.push_str("━━━━━━━\n"); 88 88 - response.push_str(&format!( 89 89 - " Target bundle: {}\n", 90 90 - mempool_stats.target_bundle 91 91 - )); 92 92 - response.push_str(&format!( 93 93 - " Operations: {} / {}\n", 94 94 - mempool_stats.count, 95 95 - constants::BUNDLE_SIZE 96 96 - )); 85 85 + if state.config.sync_mode 86 86 + && let Ok(mempool_stats) = state.manager.get_mempool_stats() 87 87 + { 88 88 + response.push_str("\nMempool\n"); 89 89 + response.push_str("━━━━━━━\n"); 90 90 + response.push_str(&format!( 91 91 + " Target bundle: {}\n", 92 92 + mempool_stats.target_bundle 93 93 + )); 94 94 + response.push_str(&format!( 95 95 + " Operations: {} / {}\n", 96 96 + mempool_stats.count, 97 97 + constants::BUNDLE_SIZE 98 98 + )); 97 99 98 98 - if mempool_stats.count > 0 { 99 99 - let progress = (mempool_stats.count as f64 / constants::BUNDLE_SIZE as f64) * 100.0; 100 100 - response.push_str(&format!(" Progress: {:.1}%\n", progress)); 100 100 + if mempool_stats.count > 0 { 101 101 + let progress = (mempool_stats.count as f64 / constants::BUNDLE_SIZE as f64) * 100.0; 102 102 + response.push_str(&format!(" Progress: {:.1}%\n", progress)); 101 103 102 102 - let bar_width = 50; 103 103 - let filled = ((bar_width as f64) 104 104 - * (mempool_stats.count as f64 / constants::BUNDLE_SIZE as f64)) 105 105 - as usize; 106 106 - let bar = "█".repeat(filled.min(bar_width)) 107 107 - + &"░".repeat(bar_width.saturating_sub(filled)); 108 108 - response.push_str(&format!(" [{}]\n", bar)); 104 104 + let bar_width = 50; 105 105 + let filled = ((bar_width as f64) 106 106 + * (mempool_stats.count as f64 / constants::BUNDLE_SIZE as f64)) 107 107 + as usize; 108 108 + let bar = 109 109 + "█".repeat(filled.min(bar_width)) + &"░".repeat(bar_width.saturating_sub(filled)); 110 110 + response.push_str(&format!(" [{}]\n", bar)); 109 111 110 110 - if let Some(first_time) = mempool_stats.first_time { 111 111 - response.push_str(&format!( 112 112 - " First op: {}\n", 113 113 - first_time.format("%Y-%m-%d %H:%M:%S") 114 114 - )); 115 115 - } 116 116 - if let Some(last_time) = mempool_stats.last_time { 117 117 - response.push_str(&format!( 118 118 - " Last op: {}\n", 119 119 - last_time.format("%Y-%m-%d %H:%M:%S") 120 120 - )); 121 121 - } 122 122 - } else { 123 123 - response.push_str(" (empty)\n"); 112 112 + if let Some(first_time) = mempool_stats.first_time { 113 113 + response.push_str(&format!( 114 114 + " First op: {}\n", 115 115 + first_time.format("%Y-%m-%d %H:%M:%S") 116 116 + )); 117 117 + } 118 118 + if let Some(last_time) = mempool_stats.last_time { 119 119 + response.push_str(&format!( 120 120 + " Last op: {}\n", 121 121 + last_time.format("%Y-%m-%d %H:%M:%S") 122 122 + )); 124 123 } 124 124 + } else { 125 125 + response.push_str(" (empty)\n"); 126 126 + } 125 127 } 126 128 127 129 if state.config.enable_resolver { ··· 162 164 )); 163 165 } 164 166 } 165 165 - response.push('\n'); 167 167 + response.push('\n'); 166 168 } 167 169 168 170 response.push_str("Server Stats\n"); 169 171 response.push_str("━━━━━━━━━━━━\n"); 170 170 - response.push_str(&format!(" Version: v{} (rust)\n", state.config.version)); 172 172 + response.push_str(&format!( 173 173 + " Version: v{} (rust)\n", 174 174 + state.config.version 175 175 + )); 171 176 response.push_str(&format!( 172 177 " Sync mode: {}\n", 173 178 state.config.sync_mode ··· 181 186 } else { 182 187 response.push_str(" Handle Resolver: (not configured)\n"); 183 188 } 184 184 - response.push_str(&format!(" Uptime: {}\n", format_std_duration_verbose(uptime))); 189 189 + response.push_str(&format!( 190 190 + " Uptime: {}\n", 191 191 + format_std_duration_verbose(uptime) 192 192 + )); 185 193 186 194 // Get base URL from request 187 195 let base_url = extract_base_url(&headers, &uri); ··· 249 257 ); 250 258 (StatusCode::OK, headers, response).into_response() 251 259 } 252 252 -

+13 -14

src/server/handle_status.rs

reviewed

··· 1 1 // Status, mempool, and index handlers 2 2 3 3 use crate::constants; 4 4 - use crate::server::error::{internal_error, not_found}; 5 4 use crate::server::ServerState; 5 5 + use crate::server::error::{internal_error, not_found}; 6 6 use axum::{ 7 7 extract::State, 8 8 http::{HeaderMap, HeaderValue, StatusCode}, ··· 66 66 response["bundles"]["total_operations"] = json!(total_ops); 67 67 } 68 68 69 69 - if state.config.sync_mode && let Ok(mempool_stats) = state.manager.get_mempool_stats() { 70 70 - response["mempool"] = json!({ 71 71 - "count": mempool_stats.count, 72 72 - "target_bundle": mempool_stats.target_bundle, 73 73 - "can_create_bundle": mempool_stats.count >= constants::BUNDLE_SIZE, 74 74 - "progress_percent": (mempool_stats.count as f64 / constants::BUNDLE_SIZE as f64) * 100.0, 75 75 - "bundle_size": constants::BUNDLE_SIZE, 76 76 - "operations_needed": constants::BUNDLE_SIZE - mempool_stats.count, 77 77 - }); 69 69 + if state.config.sync_mode 70 70 + && let Ok(mempool_stats) = state.manager.get_mempool_stats() 71 71 + { 72 72 + response["mempool"] = json!({ 73 73 + "count": mempool_stats.count, 74 74 + "target_bundle": mempool_stats.target_bundle, 75 75 + "can_create_bundle": mempool_stats.count >= constants::BUNDLE_SIZE, 76 76 + "progress_percent": (mempool_stats.count as f64 / constants::BUNDLE_SIZE as f64) * 100.0, 77 77 + "bundle_size": constants::BUNDLE_SIZE, 78 78 + "operations_needed": constants::BUNDLE_SIZE - mempool_stats.count, 79 79 + }); 78 80 } 79 81 80 82 // DID Index stats (get_stats is fast, but we should still avoid holding lock in async context) 81 83 let did_stats = tokio::task::spawn_blocking({ 82 84 let manager = Arc::clone(&state.manager); 83 83 - move || { 84 84 - manager.get_did_index_stats() 85 85 - } 85 85 + move || manager.get_did_index_stats() 86 86 }) 87 87 .await 88 88 .unwrap_or_default(); ··· 143 143 Err(e) => internal_error(&e.to_string()).into_response(), 144 144 } 145 145 } 146 146 -

+7 -4

src/server/mod.rs

reviewed

··· 18 18 #[cfg(feature = "server")] 19 19 mod routes; 20 20 #[cfg(feature = "server")] 21 21 - mod utils; 22 22 - #[cfg(feature = "server")] 23 21 mod startup; 22 22 + #[cfg(feature = "server")] 23 23 + mod utils; 24 24 #[cfg(feature = "server")] 25 25 mod websocket; 26 26 ··· 36 36 #[cfg(feature = "server")] 37 37 pub use config::ServerConfig; 38 38 #[cfg(feature = "server")] 39 39 - pub use startup::{StartupConfig, start_server, ProgressCallback, ProgressCallbackFactory, ProgressFinish}; 39 39 + pub use startup::{ 40 40 + ProgressCallback, ProgressCallbackFactory, ProgressFinish, StartupConfig, start_server, 41 41 + }; 40 42 #[cfg(feature = "server")] 41 43 pub use utils::parse_duration; 42 44 ··· 115 117 ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠉⠂⠂⠏⠿⢻⣥⡪⢽⣳⣳⣥⡶⣫⣍⢐⣥⣻⣾⡻⣅⢭⡴⢭⣿⠕⣧⡭⣞⣻⣣⣻⢿⠟⠛⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 116 118 ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠋⠫⠯⣍⢻⣿⣿⣷⣕⣵⣹⣽⣿⣷⣇⡏⣿⡿⣍⡝⠵⠯⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 117 119 ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠐⠠⠁⠋⢣⠓⡍⣫⠹⣿⣿⣷⡿⠯⠺⠁⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 118 118 - ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠋⢀⠋⢈⡿⠿⠁⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀"#.to_string() 120 120 + ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠋⢀⠋⢈⡿⠿⠁⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀"# 121 121 + .to_string() 119 122 }

+6 -4

src/server/routes.rs

reviewed

··· 1 1 // Route setup and configuration 2 2 3 3 + use crate::manager::BundleManager; 4 4 + use crate::server::ServerState; 3 5 use crate::server::config::ServerConfig; 4 4 - use crate::server::ServerState; 5 6 use crate::server::{ 6 7 handle_bundle, handle_bundle_data, handle_bundle_jsonl, handle_debug_didindex, 7 8 handle_debug_memory, handle_debug_resolver, handle_did_routing_guard, handle_index_json, 8 9 handle_mempool, handle_operation, handle_root, handle_status, 9 10 }; 10 10 - use crate::manager::BundleManager; 11 11 use axum::Router; 12 12 use std::sync::Arc; 13 13 use std::time::Instant; ··· 34 34 35 35 // WebSocket route (if enabled) 36 36 if config.enable_websocket { 37 37 - router = router.route("/ws", axum::routing::get(crate::server::websocket::handle_websocket)); 37 37 + router = router.route( 38 38 + "/ws", 39 39 + axum::routing::get(crate::server::websocket::handle_websocket), 40 40 + ); 38 41 } 39 42 40 43 // DID resolution routes (if enabled) - must be last to catch all other paths ··· 48 51 start_time, 49 52 }) 50 53 } 51 51 -

+70 -58

src/server/startup.rs

reviewed

··· 2 2 // This module handles all the complex setup logic for starting the server, 3 3 // including manager initialization, DID index building, handle resolver setup, etc. 4 4 5 5 + use crate::constants; 5 6 use crate::manager::BundleManager; 6 7 use crate::runtime::BundleRuntime; 7 8 use crate::server::{Server, ServerConfig}; 8 8 - use crate::constants; 9 9 use anyhow::{Context, Result}; 10 10 use std::path::PathBuf; 11 11 use std::sync::Arc; ··· 56 56 57 57 // Preload mempool for server use (faster responses) 58 58 let preload_mempool = true; 59 59 - 59 59 + 60 60 let options = crate::ManagerOptions { 61 61 handle_resolver_url: handle_resolver_url.clone(), 62 62 preload_mempool, 63 63 verbose: config.verbose, 64 64 }; 65 65 - 65 65 + 66 66 let manager = if config.sync { 67 67 // Sync mode can auto-init 68 68 match BundleManager::new(config.dir.clone(), options.clone()) { ··· 87 87 // Log handle resolver configuration 88 88 if config.verbose { 89 89 if let Some(url) = manager.get_handle_resolver_base_url() { 90 90 - log::debug!("[HandleResolver] External handle resolver configured: {}", url); 90 90 + log::debug!( 91 91 + "[HandleResolver] External handle resolver configured: {}", 92 92 + url 93 93 + ); 91 94 } else { 92 95 log::debug!("[HandleResolver] External handle resolver not configured"); 93 96 } ··· 96 99 Ok(manager) 97 100 } 98 101 99 99 - 100 102 /// Build or verify DID index if resolver is enabled 101 101 - /// 103 103 + /// 102 104 /// `progress_callback_factory` is an optional function that creates a progress callback 103 103 - /// and finish function given the total bundle count and total bytes. This allows the caller 105 105 + /// and finish function given the total bundle count and total bytes. This allows the caller 104 106 /// to create progress tracking (e.g., a progress bar) after knowing the bundle count. 105 107 pub fn setup_did_index<F>( 106 108 manager: &mut BundleManager, ··· 131 133 .get("total_entries") 132 134 .and_then(|v| v.as_i64()) 133 135 .unwrap_or(0); 134 134 - 136 136 + 135 137 if verbose { 136 138 log::debug!( 137 139 "[DIDResolver] DID index stats: total_dids={}, total_entries={}", ··· 147 149 148 150 let last_bundle = manager.get_last_bundle(); 149 151 if verbose { 150 150 - log::debug!( 151 151 - "[DIDResolver] Last bundle number: {}", 152 152 - last_bundle 153 153 - ); 152 152 + log::debug!("[DIDResolver] Last bundle number: {}", last_bundle); 154 153 } 155 154 156 155 // Check if repository is empty ··· 179 178 } 180 179 181 180 let start_time = std::time::Instant::now(); 182 182 - 181 181 + 183 182 // Calculate total uncompressed size for progress tracking 184 183 let bundle_numbers: Vec<u32> = (1..=last_bundle).collect(); 185 184 let total_uncompressed_size = index.total_uncompressed_size_for_bundles(&bundle_numbers); 186 186 - 185 185 + 187 186 // Build index with progress callback (create it now that we know the totals) 188 187 let (callback, finish_fn) = if let Some(factory) = progress_callback_factory { 189 188 let (cb, finish) = factory(last_bundle, total_uncompressed_size); 190 190 - let callback = move |current: u32, total: u32, bytes_processed: u64, total_bytes: u64| { 191 191 - cb(current, total, bytes_processed, total_bytes); 192 192 - }; 189 189 + let callback = 190 190 + move |current: u32, total: u32, bytes_processed: u64, total_bytes: u64| { 191 191 + cb(current, total, bytes_processed, total_bytes); 192 192 + }; 193 193 (Some(callback), finish) 194 194 } else { 195 195 (None, None) 196 196 }; 197 197 - 198 198 - manager.build_did_index( 199 199 - constants::DID_INDEX_FLUSH_INTERVAL, 200 200 - callback, 201 201 - None, 202 202 - None, 203 203 - )?; 204 204 - 197 197 + 198 198 + manager.build_did_index(constants::DID_INDEX_FLUSH_INTERVAL, callback, None, None)?; 199 199 + 205 200 // Finish progress tracking if provided 206 201 if let Some(finish) = finish_fn { 207 202 finish(); ··· 218 213 .get("total_entries") 219 214 .and_then(|v| v.as_i64()) 220 215 .unwrap_or(0); 221 221 - 216 216 + 222 217 eprintln!("✓ DID index built"); 223 218 eprintln!(" Total DIDs: {}\n", final_total_dids); 224 219 ··· 230 225 final_total_entries 231 226 ); 232 227 } 233 233 - 228 228 + 234 229 // Verify index integrity after building 235 230 if verbose { 236 231 log::debug!("[DIDResolver] Verifying DID index integrity..."); 237 232 } 238 238 - let verify_result = manager.verify_did_index::<fn(u32, u32, u64, u64)>(false, 64, false, None)?; 233 233 + let verify_result = 234 234 + manager.verify_did_index::<fn(u32, u32, u64, u64)>(false, 64, false, None)?; 239 235 if verify_result.missing_base_shards > 0 { 240 236 anyhow::bail!( 241 237 "DID index is corrupted: {} missing base shard(s). Run '{} index repair' to fix.", ··· 260 256 total_dids 261 257 ); 262 258 } 263 263 - 259 259 + 264 260 // Verify index integrity before starting server 265 261 if verbose { 266 262 log::debug!("[DIDResolver] Verifying DID index integrity..."); 267 263 } 268 268 - let verify_result = manager.verify_did_index::<fn(u32, u32, u64, u64)>(false, 64, false, None)?; 264 264 + let verify_result = 265 265 + manager.verify_did_index::<fn(u32, u32, u64, u64)>(false, 64, false, None)?; 269 266 if verify_result.missing_base_shards > 0 { 270 267 anyhow::bail!( 271 268 "DID index is corrupted: {} missing base shard(s). Run '{} index repair' to fix.", ··· 289 286 } 290 287 291 288 /// Test handle resolver on startup 292 292 - pub async fn test_handle_resolver( 293 293 - manager: &BundleManager, 294 294 - verbose: bool, 295 295 - ) -> Result<()> { 289 289 + pub async fn test_handle_resolver(manager: &BundleManager, verbose: bool) -> Result<()> { 296 290 if let Some(handle_resolver) = manager.get_handle_resolver() { 297 291 if verbose { 298 298 - log::debug!("[HandleResolver] Testing external handle resolver with handle resolution..."); 292 292 + log::debug!( 293 293 + "[HandleResolver] Testing external handle resolver with handle resolution..." 294 294 + ); 299 295 } 300 296 eprintln!("Testing handle resolver..."); 301 301 - 297 297 + 302 298 // Use a well-known handle that should always resolve 303 299 let test_handle = "bsky.app"; 304 300 let start = std::time::Instant::now(); 305 301 match handle_resolver.resolve_handle(test_handle).await { 306 302 Ok(did) => { 307 303 let duration = start.elapsed(); 308 308 - eprintln!("✓ Handle resolver test successful ({:.3}s)", duration.as_secs_f64()); 304 304 + eprintln!( 305 305 + "✓ Handle resolver test successful ({:.3}s)", 306 306 + duration.as_secs_f64() 307 307 + ); 309 308 if verbose { 310 309 log::debug!( 311 310 "[HandleResolver] Test resolution of '{}' -> '{}' successful in {:.3}s", ··· 317 316 } 318 317 Err(e) => { 319 318 let duration = start.elapsed(); 320 320 - eprintln!("⚠️ Handle resolver test failed ({:.3}s): {}", duration.as_secs_f64(), e); 319 319 + eprintln!( 320 320 + "⚠️ Handle resolver test failed ({:.3}s): {}", 321 321 + duration.as_secs_f64(), 322 322 + e 323 323 + ); 321 324 if verbose { 322 325 log::warn!( 323 326 "[HandleResolver] Test resolution of '{}' failed after {:.3}s: {}", ··· 350 353 "[HandleResolver] Starting keep-alive ping loop (interval: {:?})", 351 354 ping_interval 352 355 ); 353 353 - log::debug!("[HandleResolver] Handle resolver URL: {}", resolver.get_base_url()); 356 356 + log::debug!( 357 357 + "[HandleResolver] Handle resolver URL: {}", 358 358 + resolver.get_base_url() 359 359 + ); 354 360 } 355 361 356 362 // Initial delay before first ping ··· 507 513 508 514 /// Progress callback factory type 509 515 /// Takes (last_bundle, total_bytes) and returns (progress_callback, finish_function) 510 510 - pub type ProgressCallbackFactory = Box<dyn FnOnce(u32, u64) -> (ProgressCallback, Option<ProgressFinish>) + Send + Sync>; 516 516 + pub type ProgressCallbackFactory = 517 517 + Box<dyn FnOnce(u32, u64) -> (ProgressCallback, Option<ProgressFinish>) + Send + Sync>; 511 518 512 519 /// Main server startup function that orchestrates all initialization 513 520 pub async fn start_server( ··· 520 527 let mut manager = initialize_manager(&config)?; 521 528 522 529 // Setup DID index if resolver is enabled 523 523 - setup_did_index(&mut manager, config.enable_resolver, config.verbose, progress_callback_factory)?; 530 530 + setup_did_index( 531 531 + &mut manager, 532 532 + config.enable_resolver, 533 533 + config.verbose, 534 534 + progress_callback_factory, 535 535 + )?; 524 536 525 537 // Test handle resolver on startup if resolver is enabled 526 538 if config.enable_resolver { ··· 551 563 let mut resolver_tasks = JoinSet::new(); 552 564 553 565 // Start sync loop if enabled 554 554 - setup_sync_loop(Arc::clone(&manager), &config, &server_runtime, &mut background_tasks); 566 566 + setup_sync_loop( 567 567 + Arc::clone(&manager), 568 568 + &config, 569 569 + &server_runtime, 570 570 + &mut background_tasks, 571 571 + ); 555 572 556 573 // Start handle resolver keep-alive ping task if resolver is enabled 557 574 setup_resolver_ping_loop(&manager, &config, &server_runtime, &mut resolver_tasks); ··· 575 592 .context("Server error")?; 576 593 577 594 // Use common shutdown cleanup handler 578 578 - server_runtime.wait_for_shutdown_cleanup( 579 579 - "Server", 580 580 - Some(&mut resolver_tasks), 581 581 - Some(&mut background_tasks), 582 582 - ).await; 583 583 - 595 595 + server_runtime 596 596 + .wait_for_shutdown_cleanup( 597 597 + "Server", 598 598 + Some(&mut resolver_tasks), 599 599 + Some(&mut background_tasks), 600 600 + ) 601 601 + .await; 602 602 + 584 603 Ok(()) 585 604 } 586 605 587 606 /// Display server startup information 588 607 fn display_server_info(manager: &BundleManager, addr: &str, config: &StartupConfig) { 589 608 use crate::server::get_ascii_art_banner; 590 590 - 609 609 + 591 610 // Print ASCII art banner 592 592 - eprintln!( 593 593 - "{}", 594 594 - get_ascii_art_banner(env!("CARGO_PKG_VERSION")) 595 595 - ); 611 611 + eprintln!("{}", get_ascii_art_banner(env!("CARGO_PKG_VERSION"))); 596 612 eprintln!("{} HTTP server started", constants::BINARY_NAME); 597 597 - eprintln!( 598 598 - " Directory: {}", 599 599 - manager.directory().display() 600 600 - ); 613 613 + eprintln!(" Directory: {}", manager.directory().display()); 601 614 eprintln!(" Listening: http://{}", addr); 602 615 603 616 if config.sync { ··· 634 647 let index = manager.get_index(); 635 648 eprintln!(" Bundles: {} available", index.bundles.len()); 636 649 } 637 637 -

+21 -18

src/server/utils.rs

reviewed

··· 1 1 // Utility functions for validation and common operations 2 2 3 3 - use axum::http::{HeaderMap, HeaderValue, Uri}; 4 3 use crate::constants; 4 4 + use axum::http::{HeaderMap, HeaderValue, Uri}; 5 5 6 6 /// Check if a path is a common browser file that should be ignored 7 7 pub fn is_common_browser_file(path: &str) -> bool { ··· 55 55 56 56 // Must not have invalid characters 57 57 for c in input.chars() { 58 58 - if !(c.is_ascii_lowercase() || c.is_ascii_uppercase() || c.is_ascii_digit() || c == '.' || c == '-') { 58 58 + if !(c.is_ascii_lowercase() 59 59 + || c.is_ascii_uppercase() 60 60 + || c.is_ascii_digit() 61 61 + || c == '.' 62 62 + || c == '-') 63 63 + { 59 64 return false; 60 65 } 61 66 } ··· 117 122 /// Extract base URL from request headers and URI 118 123 pub fn extract_base_url(headers: &HeaderMap, uri: &Uri) -> String { 119 124 if let Some(host_str) = headers.get("host").and_then(|h| h.to_str().ok()) { 120 120 - // Check if request is HTTPS (from X-Forwarded-Proto or X-Forwarded-Ssl) 121 121 - let is_https = headers 122 122 - .get("x-forwarded-proto") 125 125 + // Check if request is HTTPS (from X-Forwarded-Proto or X-Forwarded-Ssl) 126 126 + let is_https = headers 127 127 + .get("x-forwarded-proto") 128 128 + .and_then(|v| v.to_str().ok()) 129 129 + .map(|s| s == "https") 130 130 + .unwrap_or(false) 131 131 + || headers 132 132 + .get("x-forwarded-ssl") 123 133 .and_then(|v| v.to_str().ok()) 124 124 - .map(|s| s == "https") 125 125 - .unwrap_or(false) 126 126 - || headers 127 127 - .get("x-forwarded-ssl") 128 128 - .and_then(|v| v.to_str().ok()) 129 129 - .map(|s| s == "on") 130 130 - .unwrap_or(false); 134 134 + .map(|s| s == "on") 135 135 + .unwrap_or(false); 131 136 132 132 - let scheme = if is_https { "https" } else { "http" }; 133 133 - return format!("{}://{}", scheme, host_str); 137 137 + let scheme = if is_https { "https" } else { "http" }; 138 138 + return format!("{}://{}", scheme, host_str); 134 139 } 135 135 - 140 140 + 136 141 if let Some(authority) = uri.authority() { 137 142 format!("http://{}", authority) 138 143 } else { ··· 155 160 pub fn parse_duration(s: &str) -> anyhow::Result<tokio::time::Duration> { 156 161 use anyhow::Context; 157 162 use tokio::time::Duration; 158 158 - 163 163 + 159 164 // Simple parser: "60s", "5m", "1h" 160 165 let s = s.trim(); 161 166 if let Some(stripped) = s.strip_suffix('s') { ··· 173 178 Ok(Duration::from_secs(secs)) 174 179 } 175 180 } 176 176 - 177 177 -

+4 -2

src/server/websocket.rs

reviewed

··· 79 79 80 80 // Stream existing bundles 81 81 if !bundles.is_empty() { 82 82 - let (start_bundle_num, start_position) = crate::constants::global_to_bundle_position(start_cursor); 82 82 + let (start_bundle_num, start_position) = 83 83 + crate::constants::global_to_bundle_position(start_cursor); 83 84 let start_bundle_idx = (start_bundle_num - 1) as usize; 84 85 85 86 if start_bundle_idx < bundles.len() { ··· 125 126 126 127 if bundles.len() > last_bundle_count { 127 128 let new_bundle_count = bundles.len() - last_bundle_count; 128 128 - current_record += crate::constants::total_operations_from_bundles(new_bundle_count as u32); 129 129 + current_record += 130 130 + crate::constants::total_operations_from_bundles(new_bundle_count as u32); 129 131 last_bundle_count = bundles.len(); 130 132 last_seen_mempool_count = 0; 131 133 }

+35 -18

src/sync.rs

reviewed

··· 1 1 - 2 1 //! PLC synchronization: events and logger, boundary-CID deduplication, one-shot and continuous modes, and robust error/backoff handling 3 2 4 3 // Sync module - PLC directory synchronization ··· 22 21 nullified: Option<Value>, 23 22 #[serde(rename = "createdAt")] 24 23 created_at: String, 25 25 - 26 24 27 25 #[serde(skip)] 28 26 pub raw_json: Option<String>, ··· 181 179 // Allow the sync logger to accept multiple arguments for detailed bundle info 182 180 // (Removed workaround method; use allow attribute on trait method instead) 183 181 184 184 - 185 182 fn on_caught_up( 186 183 &self, 187 184 next_bundle: u32, ··· 256 253 257 254 fn on_sync_start(&self, interval: Duration) { 258 255 eprintln!("[Sync] Starting initial sync..."); 259 259 - if let Some(verbose) = &self.verbose && *verbose.lock().unwrap() { 256 256 + if let Some(verbose) = &self.verbose 257 257 + && *verbose.lock().unwrap() 258 258 + { 260 259 eprintln!("[Sync] Sync loop interval: {:?}", interval); 261 260 } 262 261 } ··· 468 467 let segments_compacted = delta_segments_before.saturating_sub(delta_segments_after); 469 468 eprintln!( 470 469 "[Sync] ✓ Index compacted | segments: {} → {} ({} removed) | index: {}ms", 471 471 - delta_segments_before, 472 472 - delta_segments_after, 473 473 - segments_compacted, 474 474 - index_ms 470 470 + delta_segments_before, delta_segments_after, segments_compacted, index_ms 475 471 ); 476 472 } 477 473 } ··· 482 478 loop { 483 479 // Check for shutdown if configured 484 480 if let Some(ref shutdown_rx) = self.config.shutdown_rx 485 485 - && *shutdown_rx.borrow() { 486 486 - break; 487 487 - } 481 481 + && *shutdown_rx.borrow() 482 482 + { 483 483 + break; 484 484 + } 488 485 489 486 // Get stats before sync to track compaction 490 487 let stats_before = self.manager.get_did_index_stats(); ··· 525 522 }); 526 523 527 524 // Show compaction message if index was compacted 528 528 - self.show_compaction_if_needed(did_index_compacted, delta_segments_before, index_ms); 525 525 + self.show_compaction_if_needed( 526 526 + did_index_compacted, 527 527 + delta_segments_before, 528 528 + index_ms, 529 529 + ); 529 530 530 531 // Check if we've reached the limit 531 531 - if let Some(max) = max_bundles && synced >= max { 532 532 + if let Some(max) = max_bundles 533 533 + && synced >= max 534 534 + { 532 535 break; 533 536 } 534 537 } ··· 593 596 594 597 loop { 595 598 // Check for shutdown before starting sync 596 596 - if let Some(ref shutdown_rx) = self.config.shutdown_rx && *shutdown_rx.borrow() { 599 599 + if let Some(ref shutdown_rx) = self.config.shutdown_rx 600 600 + && *shutdown_rx.borrow() 601 601 + { 597 602 if self.config.verbose { 598 603 eprintln!("[Sync] Shutdown requested, stopping..."); 599 604 } ··· 647 652 }); 648 653 649 654 // Show compaction message if index was compacted 650 650 - self.show_compaction_if_needed(did_index_compacted, delta_segments_before, index_ms); 655 655 + self.show_compaction_if_needed( 656 656 + did_index_compacted, 657 657 + delta_segments_before, 658 658 + index_ms, 659 659 + ); 651 660 652 661 // Check max bundles limit 653 662 if self.config.max_bundles > 0 ··· 663 672 } 664 673 665 674 // Check for shutdown before sleeping 666 666 - if let Some(ref shutdown_rx) = self.config.shutdown_rx && *shutdown_rx.borrow() { 675 675 + if let Some(ref shutdown_rx) = self.config.shutdown_rx 676 676 + && *shutdown_rx.borrow() 677 677 + { 667 678 if self.config.verbose { 668 679 eprintln!("[Sync] Shutdown requested, stopping..."); 669 680 } ··· 700 711 fetch_duration_ms, 701 712 }) => { 702 713 // Check for shutdown 703 703 - if let Some(ref shutdown_rx) = self.config.shutdown_rx && *shutdown_rx.borrow() { 714 714 + if let Some(ref shutdown_rx) = self.config.shutdown_rx 715 715 + && *shutdown_rx.borrow() 716 716 + { 704 717 if self.config.verbose { 705 718 eprintln!("[Sync] Shutdown requested, stopping..."); 706 719 } ··· 985 998 986 999 #[test] 987 1000 fn test_get_boundary_cids_single() { 988 988 - let ops = vec![create_operation_helper("did:plc:test", Some("cid1"), "2024-01-01T00:00:00Z")]; 1001 1001 + let ops = vec![create_operation_helper( 1002 1002 + "did:plc:test", 1003 1003 + Some("cid1"), 1004 1004 + "2024-01-01T00:00:00Z", 1005 1005 + )]; 989 1006 let cids = get_boundary_cids(&ops); 990 1007 assert_eq!(cids.len(), 1); 991 1008 assert!(cids.contains("cid1"));

+144 -24

src/verification.rs

reviewed

··· 375 375 use crate::manager::{ChainVerifySpec, VerifySpec}; 376 376 use tempfile::TempDir; 377 377 378 378 - fn create_test_bundle_metadata(bundle_num: u32, parent: &str, cursor: &str, end_time: &str) -> BundleMetadata { 378 378 + fn create_test_bundle_metadata( 379 379 + bundle_num: u32, 380 380 + parent: &str, 381 381 + cursor: &str, 382 382 + end_time: &str, 383 383 + ) -> BundleMetadata { 379 384 BundleMetadata { 380 385 bundle_number: bundle_num, 381 386 start_time: "2024-01-01T00:00:00Z".to_string(), ··· 462 467 updated_at: "2024-01-01T00:00:00Z".to_string(), 463 468 total_size_bytes: 1000, 464 469 total_uncompressed_size_bytes: 2000, 465 465 - bundles: vec![create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z")], 470 470 + bundles: vec![create_test_bundle_metadata( 471 471 + 1, 472 472 + "", 473 473 + "", 474 474 + "2024-01-01T01:00:00Z", 475 475 + )], 466 476 }; 467 477 468 478 let spec = ChainVerifySpec { ··· 491 501 total_uncompressed_size_bytes: 6000, 492 502 bundles: vec![ 493 503 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), // end_time = "2024-01-01T01:00:00Z" 494 494 - create_test_bundle_metadata(2, "hash1", "2024-01-01T01:00:00Z", "2024-01-01T02:00:00Z"), // cursor = bundle 1's end_time 495 495 - create_test_bundle_metadata(3, "hash2", "2024-01-01T02:00:00Z", "2024-01-01T03:00:00Z"), // cursor = bundle 2's end_time 504 504 + create_test_bundle_metadata( 505 505 + 2, 506 506 + "hash1", 507 507 + "2024-01-01T01:00:00Z", 508 508 + "2024-01-01T02:00:00Z", 509 509 + ), // cursor = bundle 1's end_time 510 510 + create_test_bundle_metadata( 511 511 + 3, 512 512 + "hash2", 513 513 + "2024-01-01T02:00:00Z", 514 514 + "2024-01-01T03:00:00Z", 515 515 + ), // cursor = bundle 2's end_time 496 516 ], 497 517 }; 498 518 ··· 520 540 total_uncompressed_size_bytes: 4000, 521 541 bundles: vec![ 522 542 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), 523 523 - create_test_bundle_metadata(2, "wrong_hash", "2024-01-01T01:00:00Z", "2024-01-01T02:00:00Z"), // Wrong parent hash 543 543 + create_test_bundle_metadata( 544 544 + 2, 545 545 + "wrong_hash", 546 546 + "2024-01-01T01:00:00Z", 547 547 + "2024-01-01T02:00:00Z", 548 548 + ), // Wrong parent hash 524 549 ], 525 550 }; 526 551 ··· 534 559 assert!(!result.valid); 535 560 assert_eq!(result.bundles_checked, 2); 536 561 assert!(!result.errors.is_empty()); 537 537 - assert!(result.errors.iter().any(|(_, msg)| msg.contains("Parent hash mismatch"))); 562 562 + assert!( 563 563 + result 564 564 + .errors 565 565 + .iter() 566 566 + .any(|(_, msg)| msg.contains("Parent hash mismatch")) 567 567 + ); 538 568 } 539 569 540 570 #[test] ··· 563 593 assert!(!result.valid); 564 594 assert_eq!(result.bundles_checked, 2); 565 595 assert!(!result.errors.is_empty()); 566 566 - assert!(result.errors.iter().any(|(_, msg)| msg.contains("Cursor mismatch"))); 596 596 + assert!( 597 597 + result 598 598 + .errors 599 599 + .iter() 600 600 + .any(|(_, msg)| msg.contains("Cursor mismatch")) 601 601 + ); 567 602 } 568 603 569 604 #[test] ··· 576 611 updated_at: "2024-01-01T00:00:00Z".to_string(), 577 612 total_size_bytes: 1000, 578 613 total_uncompressed_size_bytes: 2000, 579 579 - bundles: vec![create_test_bundle_metadata(1, "", "should_be_empty", "2024-01-01T01:00:00Z")], // First bundle has non-empty cursor 614 614 + bundles: vec![create_test_bundle_metadata( 615 615 + 1, 616 616 + "", 617 617 + "should_be_empty", 618 618 + "2024-01-01T01:00:00Z", 619 619 + )], // First bundle has non-empty cursor 580 620 }; 581 621 582 622 let spec = ChainVerifySpec { ··· 589 629 assert!(!result.valid); 590 630 assert_eq!(result.bundles_checked, 1); 591 631 assert!(!result.errors.is_empty()); 592 592 - assert!(result.errors.iter().any(|(_, msg)| msg.contains("Cursor should be empty"))); 632 632 + assert!( 633 633 + result 634 634 + .errors 635 635 + .iter() 636 636 + .any(|(_, msg)| msg.contains("Cursor should be empty")) 637 637 + ); 593 638 } 594 639 595 640 #[test] ··· 604 649 total_uncompressed_size_bytes: 4000, 605 650 bundles: vec![ 606 651 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), 607 607 - create_test_bundle_metadata(3, "hash2", "2024-01-01T02:00:00Z", "2024-01-01T03:00:00Z"), // Missing bundle 2 652 652 + create_test_bundle_metadata( 653 653 + 3, 654 654 + "hash2", 655 655 + "2024-01-01T02:00:00Z", 656 656 + "2024-01-01T03:00:00Z", 657 657 + ), // Missing bundle 2 608 658 ], 609 659 }; 610 660 ··· 619 669 assert_eq!(result.bundles_checked, 2); // Only bundles 1 and 3 checked 620 670 assert!(!result.errors.is_empty()); 621 671 assert!(result.errors.iter().any(|(num, _)| *num == 2)); 622 622 - assert!(result.errors.iter().any(|(_, msg)| msg.contains("not in index"))); 672 672 + assert!( 673 673 + result 674 674 + .errors 675 675 + .iter() 676 676 + .any(|(_, msg)| msg.contains("not in index")) 677 677 + ); 623 678 } 624 679 625 680 #[test] ··· 632 687 updated_at: "2024-01-01T00:00:00Z".to_string(), 633 688 total_size_bytes: 1000, 634 689 total_uncompressed_size_bytes: 2000, 635 635 - bundles: vec![create_test_bundle_metadata(2, "hash1", "2024-01-01T01:00:00Z", "2024-01-01T02:00:00Z")], // Missing bundle 1 690 690 + bundles: vec![create_test_bundle_metadata( 691 691 + 2, 692 692 + "hash1", 693 693 + "2024-01-01T01:00:00Z", 694 694 + "2024-01-01T02:00:00Z", 695 695 + )], // Missing bundle 1 636 696 }; 637 697 638 698 let spec = ChainVerifySpec { ··· 645 705 assert!(!result.valid); 646 706 assert_eq!(result.bundles_checked, 1); // Only bundle 2 checked 647 707 assert!(!result.errors.is_empty()); 648 648 - assert!(result.errors.iter().any(|(_, msg)| msg.contains("Previous bundle 1 not found"))); 708 708 + assert!( 709 709 + result 710 710 + .errors 711 711 + .iter() 712 712 + .any(|(_, msg)| msg.contains("Previous bundle 1 not found")) 713 713 + ); 649 714 } 650 715 651 716 #[test] ··· 661 726 total_uncompressed_size_bytes: 10000, 662 727 bundles: vec![ 663 728 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), // end_time = "2024-01-01T01:00:00Z" 664 664 - create_test_bundle_metadata(2, "hash1", "2024-01-01T01:00:00Z", "2024-01-01T02:00:00Z"), // cursor = bundle 1's end_time 665 665 - create_test_bundle_metadata(3, "hash2", "2024-01-01T02:00:00Z", "2024-01-01T03:00:00Z"), // cursor = bundle 2's end_time 666 666 - create_test_bundle_metadata(4, "hash3", "2024-01-01T03:00:00Z", "2024-01-01T04:00:00Z"), // cursor = bundle 3's end_time 667 667 - create_test_bundle_metadata(5, "hash4", "2024-01-01T04:00:00Z", "2024-01-01T05:00:00Z"), // cursor = bundle 4's end_time 729 729 + create_test_bundle_metadata( 730 730 + 2, 731 731 + "hash1", 732 732 + "2024-01-01T01:00:00Z", 733 733 + "2024-01-01T02:00:00Z", 734 734 + ), // cursor = bundle 1's end_time 735 735 + create_test_bundle_metadata( 736 736 + 3, 737 737 + "hash2", 738 738 + "2024-01-01T02:00:00Z", 739 739 + "2024-01-01T03:00:00Z", 740 740 + ), // cursor = bundle 2's end_time 741 741 + create_test_bundle_metadata( 742 742 + 4, 743 743 + "hash3", 744 744 + "2024-01-01T03:00:00Z", 745 745 + "2024-01-01T04:00:00Z", 746 746 + ), // cursor = bundle 3's end_time 747 747 + create_test_bundle_metadata( 748 748 + 5, 749 749 + "hash4", 750 750 + "2024-01-01T04:00:00Z", 751 751 + "2024-01-01T05:00:00Z", 752 752 + ), // cursor = bundle 4's end_time 668 753 ], 669 754 }; 670 755 ··· 692 777 total_uncompressed_size_bytes: 4000, 693 778 bundles: vec![ 694 779 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), 695 695 - create_test_bundle_metadata(2, "wrong_hash", "wrong_cursor", "2024-01-01T02:00:00Z"), // Wrong but won't be checked 780 780 + create_test_bundle_metadata( 781 781 + 2, 782 782 + "wrong_hash", 783 783 + "wrong_cursor", 784 784 + "2024-01-01T02:00:00Z", 785 785 + ), // Wrong but won't be checked 696 786 ], 697 787 }; 698 788 ··· 721 811 total_uncompressed_size_bytes: 6000, 722 812 bundles: vec![ 723 813 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), // end_time = "2024-01-01T01:00:00Z" 724 724 - create_test_bundle_metadata(2, "hash1", "2024-01-01T01:00:00Z", "2024-01-01T02:00:00Z"), // cursor = bundle 1's end_time 725 725 - create_test_bundle_metadata(3, "hash2", "2024-01-01T02:00:00Z", "2024-01-01T03:00:00Z"), // cursor = bundle 2's end_time 814 814 + create_test_bundle_metadata( 815 815 + 2, 816 816 + "hash1", 817 817 + "2024-01-01T01:00:00Z", 818 818 + "2024-01-01T02:00:00Z", 819 819 + ), // cursor = bundle 1's end_time 820 820 + create_test_bundle_metadata( 821 821 + 3, 822 822 + "hash2", 823 823 + "2024-01-01T02:00:00Z", 824 824 + "2024-01-01T03:00:00Z", 825 825 + ), // cursor = bundle 2's end_time 726 826 ], 727 827 }; 728 828 ··· 750 850 total_uncompressed_size_bytes: 10000, 751 851 bundles: vec![ 752 852 create_test_bundle_metadata(1, "", "", "2024-01-01T01:00:00Z"), // end_time = "2024-01-01T01:00:00Z" 753 753 - create_test_bundle_metadata(2, "hash1", "2024-01-01T01:00:00Z", "2024-01-01T02:00:00Z"), // cursor = bundle 1's end_time 754 754 - create_test_bundle_metadata(3, "hash2", "2024-01-01T02:00:00Z", "2024-01-01T03:00:00Z"), // cursor = bundle 2's end_time 755 755 - create_test_bundle_metadata(4, "hash3", "2024-01-01T03:00:00Z", "2024-01-01T04:00:00Z"), // cursor = bundle 3's end_time 756 756 - create_test_bundle_metadata(5, "hash4", "2024-01-01T04:00:00Z", "2024-01-01T05:00:00Z"), // cursor = bundle 4's end_time 853 853 + create_test_bundle_metadata( 854 854 + 2, 855 855 + "hash1", 856 856 + "2024-01-01T01:00:00Z", 857 857 + "2024-01-01T02:00:00Z", 858 858 + ), // cursor = bundle 1's end_time 859 859 + create_test_bundle_metadata( 860 860 + 3, 861 861 + "hash2", 862 862 + "2024-01-01T02:00:00Z", 863 863 + "2024-01-01T03:00:00Z", 864 864 + ), // cursor = bundle 2's end_time 865 865 + create_test_bundle_metadata( 866 866 + 4, 867 867 + "hash3", 868 868 + "2024-01-01T03:00:00Z", 869 869 + "2024-01-01T04:00:00Z", 870 870 + ), // cursor = bundle 3's end_time 871 871 + create_test_bundle_metadata( 872 872 + 5, 873 873 + "hash4", 874 874 + "2024-01-01T04:00:00Z", 875 875 + "2024-01-01T05:00:00Z", 876 876 + ), // cursor = bundle 4's end_time 757 877 ], 758 878 }; 759 879

+1 -2

tests/common/mod.rs

reviewed

··· 1 1 - 2 1 use anyhow::Result; 3 2 use std::io::Write; 4 3 use std::path::PathBuf; 5 4 use std::sync::Arc; 6 5 use tempfile::TempDir; 7 6 7 7 + use plcbundle::BundleManager; 8 8 use plcbundle::index::{BundleMetadata, Index}; 9 9 use plcbundle::operations::Operation; 10 10 - use plcbundle::BundleManager; 11 10 12 11 pub fn setup_manager(dir: &PathBuf) -> Result<BundleManager> { 13 12 plcbundle::Index::init(dir, "http://localhost:1234".to_string(), true)?;

+6 -2

tests/manager.rs

reviewed

··· 28 28 assert!(raw.contains("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa")); 29 29 30 30 // Build DID index so DID lookups work 31 31 - manager.batch_update_did_index_async(1, manager.get_last_bundle()).await?; 31 31 + manager 32 32 + .batch_update_did_index_async(1, manager.get_last_bundle()) 33 33 + .await?; 32 34 33 35 // Query DID operations and resolve DID 34 36 let did = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; ··· 125 127 common::setup_manager(&dir2_path)?; 126 128 common::add_dummy_bundle(&dir2_path)?; 127 129 let manager2 = plcbundle::BundleManager::new(dir2_path.clone(), ())?; 128 128 - manager2.batch_update_did_index_async(1, manager2.get_last_bundle()).await?; 130 130 + manager2 131 131 + .batch_update_did_index_async(1, manager2.get_last_bundle()) 132 132 + .await?; 129 133 130 134 // Verify we can query DID operations from the newly built index 131 135 let first_did = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";

+33 -8

tests/server.rs

reviewed

··· 57 57 let manager = plcbundle::BundleManager::new(dir_path, ())?; 58 58 let manager = Arc::new(manager); 59 59 // Build DID index so the resolver can find operations in bundles 60 60 - manager.batch_update_did_index_async(1, manager.get_last_bundle()).await?; 60 60 + manager 61 61 + .batch_update_did_index_async(1, manager.get_last_bundle()) 62 62 + .await?; 61 63 let port = 3032; 62 64 let server_handle = common::start_test_server(Arc::clone(&manager), port).await?; 63 65 ··· 66 68 67 69 // Test DID document endpoint (first DID from dummy bundle) 68 70 let first_did = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; 69 69 - let res = client.get(format!("{}/{}", base_url, first_did)).send().await?; 71 71 + let res = client 72 72 + .get(format!("{}/{}", base_url, first_did)) 73 73 + .send() 74 74 + .await?; 70 75 let status = res.status(); 71 76 let header_x_request_type = res 72 77 .headers() ··· 88 93 assert_eq!(json["id"], first_did); 89 94 90 95 // Test DID data endpoint 91 91 - let res = client.get(format!("{}/did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/data", base_url)).send().await?; 96 96 + let res = client 97 97 + .get(format!( 98 98 + "{}/did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/data", 99 99 + base_url 100 100 + )) 101 101 + .send() 102 102 + .await?; 92 103 let status = res.status(); 93 104 let body_text = res.text().await?; 94 105 if !status.is_success() { ··· 98 109 assert_eq!(json["did"], "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"); 99 110 100 111 // Test DID audit log endpoint 101 101 - let res = client.get(format!("{}/did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/log/audit", base_url)).send().await?; 112 112 + let res = client 113 113 + .get(format!( 114 114 + "{}/did:plc:aaaaaaaaaaaaaaaaaaaaaaaa/log/audit", 115 115 + base_url 116 116 + )) 117 117 + .send() 118 118 + .await?; 102 119 let status = res.status(); 103 120 let body_text = res.text().await?; 104 121 if !status.is_success() { ··· 107 124 let json: serde_json::Value = serde_json::from_str(&body_text)?; 108 125 assert!(json.is_array()); 109 126 assert!(!json.as_array().unwrap().is_empty()); 110 110 - assert_eq!(json.as_array().unwrap()[0]["did"], "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"); 127 127 + assert_eq!( 128 128 + json.as_array().unwrap()[0]["did"], 129 129 + "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" 130 130 + ); 111 131 112 132 server_handle.abort(); 113 133 Ok(()) ··· 128 148 let manager = plcbundle::BundleManager::new(dir_path, ())?; 129 149 let manager = Arc::new(manager); 130 150 // Ensure DID index is available for data/op lookups 131 131 - manager.batch_update_did_index_async(1, manager.get_last_bundle()).await?; 151 151 + manager 152 152 + .batch_update_did_index_async(1, manager.get_last_bundle()) 153 153 + .await?; 132 154 let port = 3031; 133 155 let server_handle = common::start_test_server(Arc::clone(&manager), port).await?; 134 156 ··· 136 158 let base_url = format!("http://127.0.0.1:{}", port); 137 159 138 160 // Test index.json endpoint 139 139 - let res = client.get(format!("{}/index.json", base_url)).send().await?; 161 161 + let res = client 162 162 + .get(format!("{}/index.json", base_url)) 163 163 + .send() 164 164 + .await?; 140 165 println!("Testing endpoint: /index.json, status: {}", res.status()); 141 166 assert!(res.status().is_success()); 142 167 let json: serde_json::Value = res.json().await?; ··· 177 202 178 203 server_handle.abort(); 179 204 Ok(()) 180 180 - } 205 205 + }