+1

Cargo.lock

··· 1362 1362 "mlf-lexicon-fetcher", 1363 1363 "serde", 1364 1364 "serde_json", 1365 1365 + "tempfile", 1365 1366 "tokio", 1366 1367 "toml", 1367 1368 ]

+5 -3

justfile

··· 42 42 @echo "\nRunning workspace resolution tests..." 43 43 cargo test -p mlf-integration-tests --test workspace_integration -- --nocapture 44 44 45 45 - # Run real-world lexicon tests (when implemented) 45 45 + # Run real-world round-trip tests (network-dependent, ignored by default) 46 46 test-real-world: 47 47 - @echo "\nRunning real-world lexicon tests..." 48 48 - cargo test -p mlf-integration-tests --test real_world_integration -- --nocapture 47 47 + @echo "\n🌐 Running real-world round-trip tests (fetches from network)..." 48 48 + @echo "This will download lexicons from: app.bsky.*, net.anisota.*, place.stream.*, pub.leaflet.*" 49 49 + @echo "" 50 50 + cargo test -p mlf-integration-tests --test real_world_roundtrip -- --ignored --nocapture 49 51 50 52 # Run all workspace tests (excluding problematic packages) 51 53 test-all:

+2

tests/.gitignore

··· 1 1 + # Real-world test artifacts (generated during test runs) 2 2 + real_world/roundtrip/diffs/

+5

tests/Cargo.toml

··· 17 17 serde = { version = "1.0", features = ["derive"] } 18 18 toml = "0.8" 19 19 tokio = { version = "1", features = ["full"] } 20 20 + tempfile = "3.8" 20 21 21 22 [dev-dependencies] 22 23 # Any additional test dependencies ··· 32 33 [[test]] 33 34 name = "lexicon_fetcher_integration" 34 35 path = "lexicon_fetcher_integration.rs" 36 36 + 37 37 + [[test]] 38 38 + name = "real_world_roundtrip" 39 39 + path = "real_world/roundtrip.rs"

+13 -12

tests/README.md

··· 34 34 ## Current Status 35 35 36 36 ### ✅ Implemented 37 37 - - **mlf-lang/tests/lang/** - 17 tests for parsing and validation 38 38 - - **tests/codegen/lexicon/** - 4 tests for lexicon generation 37 37 + - **mlf-lang/tests/lang/** - 21 tests for parsing and validation 38 38 + - **tests/codegen/lexicon/** - 10 tests for lexicon generation 39 39 + - **tests/real_world_roundtrip** - Round-trip test (JSON → MLF → JSON) with real lexicons 39 40 40 41 ### 🚧 Planned 41 42 ··· 80 81 - **workspace/precedence** - Resolution order (local > home > std) 81 82 - **workspace/sibling_files** - Multi-file modules 82 83 83 83 - #### Real-World Tests 84 84 - - **real_world/bsky** - Full app.bsky.* lexicons 85 85 - - **real_world/place_stream** - Full place.stream.* lexicons 86 86 - - **real_world/atproto** - Full com.atproto.* lexicons 87 87 - - **real_world/bidirectional** - Roundtrip MLF ↔ JSON 84 84 + #### Real-World Tests ✅ 85 85 + - **real_world/** - Tests using real lexicons from production networks 86 86 + - **roundtrip** - Round-trip test: JSON → MLF → JSON 87 87 + - Fetches real lexicons (app.bsky.*, net.anisota.*, place.stream.*, pub.leaflet.*) 88 88 + - Validates accurate conversion both ways 89 89 + - Writes diff files to `real_world/roundtrip/diffs/` (gitignored) 90 90 + - Run with: `just test-real-world` 91 91 + - See `real_world/README.md` for details 88 92 89 93 ## Running Tests 90 94 ··· 100 104 just test-codegen # Codegen tests (4 tests) 101 105 just test-validation # Validation tests (12 tests) 102 106 103 103 - # Future test categories 104 104 - just test-cli # CLI integration tests 105 105 - just test-diagnostics # Error message tests 106 106 - just test-workspace # Multi-file resolution tests 107 107 - just test-real-world # Full lexicon suites 107 107 + # Network-dependent tests (run explicitly) 108 108 + just test-real-world # Round-trip test: fetch real lexicons, convert MLF→JSON, verify 108 109 109 110 # Other useful commands 110 111 just test-all # All workspace tests (includes unit tests)

+58

tests/real_world/README.md

··· 1 1 + # Real-World Tests 2 2 + 3 3 + Tests that fetch and validate real lexicons from production networks. 4 4 + 5 5 + ## Tests 6 6 + 7 7 + ### `roundtrip.rs` - Round-Trip Test 8 8 + 9 9 + Validates that MLF can accurately convert lexicons: JSON → MLF → JSON 10 10 + 11 11 + **What it does:** 12 12 + 1. Fetches real lexicons from production networks: 13 13 + - `app.bsky.actor.*`, `app.bsky.feed.*`, `app.bsky.graph.*` 14 14 + - `net.anisota.*` 15 15 + - `place.stream.*` 16 16 + - `pub.leaflet.*` 17 17 + 2. Converts downloaded JSON to MLF (automatic during fetch) 18 18 + 3. Generates JSON back from MLF files 19 19 + 4. Compares original vs regenerated JSON 20 20 + 21 21 + **Running:** 22 22 + ```bash 23 23 + # Using just 24 24 + just test-real-world 25 25 + 26 26 + # Using cargo 27 27 + cargo test -p mlf-integration-tests --test real_world_roundtrip -- --ignored --nocapture 28 28 + ``` 29 29 + 30 30 + **Network-dependent:** This test fetches from real networks, so it: 31 31 + - Is marked `#[ignore]` by default 32 32 + - Requires internet connectivity 33 33 + - Takes 30-60 seconds to run 34 34 + 35 35 + **Diff files:** When differences are found, the test writes three files per lexicon to `roundtrip/diffs/`: 36 36 + - `{nsid}.original.json` - The original fetched JSON 37 37 + - `{nsid}.generated.json` - The regenerated JSON from MLF 38 38 + - `{nsid}.diff` - Unified diff output (`diff -u`) 39 39 + 40 40 + Files are organized into subdirectories: 41 41 + - `diffs/acceptable/` - Acceptable differences (field ordering, `$type` fields) 42 42 + - `diffs/failure/` - Structural differences that indicate bugs 43 43 + 44 44 + These files are gitignored but persisted locally for review. 45 45 + 46 46 + ## Adding New Tests 47 47 + 48 48 + To add more real-world test sources: 49 49 + 50 50 + 1. Edit `TEST_SOURCES` in `roundtrip.rs` 51 51 + 2. Ensure the NSID has published DNS TXT records 52 52 + 3. Test with `mlf fetch <nsid>` first to verify 53 53 + 54 54 + ## Notes 55 55 + 56 56 + - These tests validate the core MLF workflow end-to-end 57 57 + - Failures indicate bugs in either JSON→MLF or MLF→JSON conversion 58 58 + - Review diff files to diagnose what changed

+505

tests/real_world/roundtrip.rs

··· 1 1 + // Real-world round-trip tests: JSON → MLF → JSON 2 2 + // 3 3 + // These tests fetch real lexicons from the network, convert them to MLF, 4 4 + // then generate JSON back and verify the round-trip is accurate. 5 5 + // 6 6 + // Run with: cargo test --test real_world_roundtrip -- --ignored --nocapture 7 7 + 8 8 + use std::collections::HashSet; 9 9 + use std::fs; 10 10 + use std::path::{Path, PathBuf}; 11 11 + use std::process::Command; 12 12 + use tempfile::TempDir; 13 13 + 14 14 + /// Real-world lexicon sources to test 15 15 + /// These use specific namespaces that have DNS TXT records published 16 16 + const TEST_SOURCES: &[&str] = &[ 17 17 + // Bluesky - use specific namespaces since top-level doesn't have TXT record 18 18 + "app.bsky.actor.*", 19 19 + "app.bsky.feed.*", 20 20 + "app.bsky.graph.*", 21 21 + // Other networks 22 22 + "net.anisota.*", 23 23 + "place.stream.*", 24 24 + "pub.leaflet.*", 25 25 + ]; 26 26 + 27 27 + #[test] 28 28 + #[ignore] // Network-dependent test, run explicitly with --ignored 29 29 + fn test_real_world_roundtrip() { 30 30 + println!("\n🌐 Real-World Round-Trip Test"); 31 31 + println!("=============================\n"); 32 32 + 33 33 + // Create temp directory for test workspace 34 34 + let temp_dir = TempDir::new().expect("Failed to create temp directory"); 35 35 + let workspace_path = temp_dir.path(); 36 36 + 37 37 + println!("📁 Test workspace: {}\n", workspace_path.display()); 38 38 + 39 39 + // Step 1: Initialize MLF project 40 40 + println!("1️⃣ Initializing MLF project..."); 41 41 + init_mlf_project(workspace_path).expect("Failed to initialize project"); 42 42 + 43 43 + // Step 2: Fetch real lexicons 44 44 + println!("\n2️⃣ Fetching real lexicons from network..."); 45 45 + for source in TEST_SOURCES { 46 46 + println!(" Fetching: {}", source); 47 47 + fetch_lexicons(workspace_path, source).expect(&format!("Failed to fetch {}", source)); 48 48 + } 49 49 + 50 50 + // Step 3: Copy MLF files to standard lexicons directory 51 51 + println!("\n3️⃣ Copying MLF files to standard lexicons directory..."); 52 52 + let source_mlf_dir = workspace_path.join(".mlf/lexicons/mlf"); 53 53 + let lexicons_dir = workspace_path.join("lexicons"); 54 54 + copy_mlf_files(&source_mlf_dir, &lexicons_dir).expect("Failed to copy MLF files"); 55 55 + 56 56 + // Step 4: Generate JSON from MLF 57 57 + println!("\n4️⃣ Generating JSON from MLF files..."); 58 58 + let output_dir = workspace_path.join("generated-lexicons"); 59 59 + generate_json_from_mlf(workspace_path, &output_dir).expect("Failed to generate JSON"); 60 60 + 61 61 + // Step 5: Compare original vs regenerated JSON 62 62 + println!("\n5️⃣ Comparing original vs regenerated JSON..."); 63 63 + let original_dir = workspace_path.join(".mlf/lexicons/json"); 64 64 + 65 65 + // Write diffs to tests/real_world/roundtrip/diffs/ (persisted, gitignored) 66 66 + let diffs_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) 67 67 + .join("real_world/roundtrip/diffs"); 68 68 + 69 69 + let stats = compare_json_files(&original_dir, &output_dir, &diffs_dir) 70 70 + .expect("Failed to compare JSON files"); 71 71 + 72 72 + // Step 6: Report results 73 73 + println!("\n📊 Round-Trip Test Results"); 74 74 + println!("==========================="); 75 75 + println!("Total lexicons tested: {}", stats.total); 76 76 + println!("Perfect matches: {}", stats.perfect_matches); 77 77 + println!("Acceptable differences: {}", stats.acceptable_diffs); 78 78 + println!("Failures: {}", stats.failures); 79 79 + 80 80 + if !stats.failed_lexicons.is_empty() { 81 81 + println!("\n❌ Failed lexicons:"); 82 82 + for (nsid, reason) in &stats.failed_lexicons { 83 83 + println!(" - {}: {}", nsid, reason); 84 84 + } 85 85 + } 86 86 + 87 87 + if stats.acceptable_diffs > 0 || stats.failures > 0 { 88 88 + println!("\n📁 Diff files written to: {}", diffs_dir.display()); 89 89 + println!(" Review these files to see what changed between original and regenerated JSON"); 90 90 + } 91 91 + 92 92 + // Assert that we have no failures 93 93 + assert_eq!( 94 94 + stats.failures, 0, 95 95 + "Round-trip test failed for {} lexicon(s). Check diff files in {}", 96 96 + stats.failures, 97 97 + diffs_dir.display() 98 98 + ); 99 99 + 100 100 + println!("\n✅ All round-trip tests passed!"); 101 101 + } 102 102 + 103 103 + /// Initialize an MLF project with mlf.toml 104 104 + fn init_mlf_project(workspace_path: &Path) -> Result<(), String> { 105 105 + // Create mlf.toml 106 106 + let mlf_toml = r#" 107 107 + [package] 108 108 + name = "roundtrip-test" 109 109 + version = "0.1.0" 110 110 + 111 111 + [dependencies] 112 112 + dependencies = [] 113 113 + allow_transitive_deps = true 114 114 + optimize_transitive_fetches = false 115 115 + "#; 116 116 + 117 117 + fs::write(workspace_path.join("mlf.toml"), mlf_toml) 118 118 + .map_err(|e| format!("Failed to write mlf.toml: {}", e))?; 119 119 + 120 120 + Ok(()) 121 121 + } 122 122 + 123 123 + /// Fetch lexicons using `mlf fetch` 124 124 + fn fetch_lexicons(workspace_path: &Path, nsid_pattern: &str) -> Result<(), String> { 125 125 + let output = Command::new("mlf") 126 126 + .arg("fetch") 127 127 + .arg(nsid_pattern) 128 128 + .current_dir(workspace_path) 129 129 + .output() 130 130 + .map_err(|e| format!("Failed to execute mlf fetch: {}", e))?; 131 131 + 132 132 + if !output.status.success() { 133 133 + return Err(format!( 134 134 + "mlf fetch failed:\n{}", 135 135 + String::from_utf8_lossy(&output.stderr) 136 136 + )); 137 137 + } 138 138 + 139 139 + Ok(()) 140 140 + } 141 141 + 142 142 + /// Copy MLF files from .mlf/lexicons/mlf to lexicons/ 143 143 + fn copy_mlf_files(source_dir: &Path, dest_dir: &Path) -> Result<(), String> { 144 144 + if !source_dir.exists() { 145 145 + return Err(format!("Source directory not found: {}", source_dir.display())); 146 146 + } 147 147 + 148 148 + fn copy_recursive(src: &Path, dst: &Path) -> std::io::Result<()> { 149 149 + fs::create_dir_all(dst)?; 150 150 + for entry in fs::read_dir(src)? { 151 151 + let entry = entry?; 152 152 + let src_path = entry.path(); 153 153 + let dst_path = dst.join(entry.file_name()); 154 154 + 155 155 + if src_path.is_dir() { 156 156 + copy_recursive(&src_path, &dst_path)?; 157 157 + } else { 158 158 + fs::copy(&src_path, &dst_path)?; 159 159 + } 160 160 + } 161 161 + Ok(()) 162 162 + } 163 163 + 164 164 + copy_recursive(source_dir, dest_dir) 165 165 + .map_err(|e| format!("Failed to copy MLF files: {}", e))?; 166 166 + 167 167 + let mlf_count = find_mlf_files(dest_dir)?.len(); 168 168 + println!(" Copied {} MLF files", mlf_count); 169 169 + 170 170 + Ok(()) 171 171 + } 172 172 + 173 173 + /// Generate JSON from MLF files using `mlf generate lexicon` 174 174 + fn generate_json_from_mlf(workspace_dir: &Path, output_dir: &Path) -> Result<(), String> { 175 175 + let lexicons_dir = workspace_dir.join("lexicons"); 176 176 + 177 177 + if !lexicons_dir.exists() { 178 178 + return Err(format!("Lexicons directory not found: {}", lexicons_dir.display())); 179 179 + } 180 180 + 181 181 + // Create output directory 182 182 + fs::create_dir_all(output_dir) 183 183 + .map_err(|e| format!("Failed to create output directory: {}", e))?; 184 184 + 185 185 + // Generate all JSON files at once by passing the lexicons directory 186 186 + // This allows proper dependency resolution between MLF files 187 187 + println!(" Generating JSON files..."); 188 188 + let output = Command::new("mlf") 189 189 + .arg("generate") 190 190 + .arg("lexicon") 191 191 + .arg("-i") 192 192 + .arg("lexicons") 193 193 + .arg("-o") 194 194 + .arg(output_dir) 195 195 + .current_dir(workspace_dir) 196 196 + .output() 197 197 + .map_err(|e| format!("Failed to execute mlf generate: {}", e))?; 198 198 + 199 199 + if !output.status.success() { 200 200 + return Err(format!( 201 201 + "mlf generate lexicon failed:\n{}", 202 202 + String::from_utf8_lossy(&output.stderr) 203 203 + )); 204 204 + } 205 205 + 206 206 + println!(" Generated JSON successfully"); 207 207 + 208 208 + Ok(()) 209 209 + } 210 210 + 211 211 + /// Find all .mlf files recursively 212 212 + fn find_mlf_files(dir: &Path) -> Result<Vec<PathBuf>, String> { 213 213 + let mut mlf_files = Vec::new(); 214 214 + 215 215 + fn walk_dir(dir: &Path, files: &mut Vec<PathBuf>) -> std::io::Result<()> { 216 216 + if dir.is_dir() { 217 217 + for entry in fs::read_dir(dir)? { 218 218 + let entry = entry?; 219 219 + let path = entry.path(); 220 220 + if path.is_dir() { 221 221 + walk_dir(&path, files)?; 222 222 + } else if path.extension().and_then(|s| s.to_str()) == Some("mlf") { 223 223 + files.push(path); 224 224 + } 225 225 + } 226 226 + } 227 227 + Ok(()) 228 228 + } 229 229 + 230 230 + walk_dir(dir, &mut mlf_files).map_err(|e| format!("Failed to walk directory: {}", e))?; 231 231 + Ok(mlf_files) 232 232 + } 233 233 + 234 234 + #[derive(Debug)] 235 235 + struct ComparisonStats { 236 236 + total: usize, 237 237 + perfect_matches: usize, 238 238 + acceptable_diffs: usize, 239 239 + failures: usize, 240 240 + failed_lexicons: Vec<(String, String)>, 241 241 + } 242 242 + 243 243 + /// Compare original JSON with regenerated JSON 244 244 + fn compare_json_files( 245 245 + original_dir: &Path, 246 246 + generated_dir: &Path, 247 247 + diffs_dir: &Path, 248 248 + ) -> Result<ComparisonStats, String> { 249 249 + // Create diffs directory 250 250 + fs::create_dir_all(diffs_dir) 251 251 + .map_err(|e| format!("Failed to create diffs directory: {}", e))?; 252 252 + let mut stats = ComparisonStats { 253 253 + total: 0, 254 254 + perfect_matches: 0, 255 255 + acceptable_diffs: 0, 256 256 + failures: 0, 257 257 + failed_lexicons: Vec::new(), 258 258 + }; 259 259 + 260 260 + // Find all JSON files in original directory 261 261 + let original_files = find_json_files(original_dir)?; 262 262 + stats.total = original_files.len(); 263 263 + 264 264 + println!(" Comparing {} lexicon files...", stats.total); 265 265 + 266 266 + for original_file in original_files { 267 267 + let relative_path = original_file 268 268 + .strip_prefix(original_dir) 269 269 + .map_err(|e| format!("Failed to strip prefix: {}", e))?; 270 270 + 271 271 + let generated_file = generated_dir.join(relative_path); 272 272 + 273 273 + // Extract NSID from path for reporting 274 274 + let nsid = relative_path 275 275 + .with_extension("") 276 276 + .to_str() 277 277 + .unwrap() 278 278 + .replace(std::path::MAIN_SEPARATOR, "."); 279 279 + 280 280 + if !generated_file.exists() { 281 281 + stats.failures += 1; 282 282 + stats.failed_lexicons 283 283 + .push((nsid.clone(), "Generated file not found".to_string())); 284 284 + eprintln!(" ✗ {}: Generated file not found", nsid); 285 285 + continue; 286 286 + } 287 287 + 288 288 + // Read and parse JSON files 289 289 + let original_json = fs::read_to_string(&original_file) 290 290 + .map_err(|e| format!("Failed to read original: {}", e))?; 291 291 + let generated_json = fs::read_to_string(&generated_file) 292 292 + .map_err(|e| format!("Failed to read generated: {}", e))?; 293 293 + 294 294 + let original: serde_json::Value = serde_json::from_str(&original_json) 295 295 + .map_err(|e| format!("Failed to parse original JSON: {}", e))?; 296 296 + let generated: serde_json::Value = serde_json::from_str(&generated_json) 297 297 + .map_err(|e| format!("Failed to parse generated JSON: {}", e))?; 298 298 + 299 299 + // Compare with allowed differences 300 300 + match compare_lexicon_json(&original, &generated) { 301 301 + ComparisonResult::Perfect => { 302 302 + stats.perfect_matches += 1; 303 303 + println!(" ✓ {} (perfect match)", nsid); 304 304 + } 305 305 + ComparisonResult::AcceptableDifferences(diffs) => { 306 306 + stats.acceptable_diffs += 1; 307 307 + println!(" ✓ {} (acceptable diffs: {})", nsid, diffs.join(", ")); 308 308 + 309 309 + // Write diff file for acceptable differences 310 310 + write_diff_file(diffs_dir, &nsid, &original_json, &generated_json, "acceptable") 311 311 + .unwrap_or_else(|e| eprintln!("Warning: Failed to write diff: {}", e)); 312 312 + } 313 313 + ComparisonResult::Failure(reason) => { 314 314 + stats.failures += 1; 315 315 + stats.failed_lexicons.push((nsid.clone(), reason.clone())); 316 316 + eprintln!(" ✗ {}: {}", nsid, reason); 317 317 + 318 318 + // Write diff file for failures 319 319 + write_diff_file(diffs_dir, &nsid, &original_json, &generated_json, "failure") 320 320 + .unwrap_or_else(|e| eprintln!("Warning: Failed to write diff: {}", e)); 321 321 + } 322 322 + } 323 323 + } 324 324 + 325 325 + Ok(stats) 326 326 + } 327 327 + 328 328 + /// Find all JSON files recursively 329 329 + fn find_json_files(dir: &Path) -> Result<Vec<PathBuf>, String> { 330 330 + let mut json_files = Vec::new(); 331 331 + 332 332 + fn walk_dir(dir: &Path, files: &mut Vec<PathBuf>) -> std::io::Result<()> { 333 333 + if dir.is_dir() { 334 334 + for entry in fs::read_dir(dir)? { 335 335 + let entry = entry?; 336 336 + let path = entry.path(); 337 337 + if path.is_dir() { 338 338 + walk_dir(&path, files)?; 339 339 + } else if path.extension().and_then(|s| s.to_str()) == Some("json") { 340 340 + files.push(path); 341 341 + } 342 342 + } 343 343 + } 344 344 + Ok(()) 345 345 + } 346 346 + 347 347 + walk_dir(dir, &mut json_files).map_err(|e| format!("Failed to walk directory: {}", e))?; 348 348 + Ok(json_files) 349 349 + } 350 350 + 351 351 + #[derive(Debug)] 352 352 + enum ComparisonResult { 353 353 + Perfect, 354 354 + AcceptableDifferences(Vec<String>), 355 355 + Failure(String), 356 356 + } 357 357 + 358 358 + /// Compare two lexicon JSON objects, allowing certain acceptable differences 359 359 + fn compare_lexicon_json( 360 360 + original: &serde_json::Value, 361 361 + generated: &serde_json::Value, 362 362 + ) -> ComparisonResult { 363 363 + let mut acceptable_diffs = Vec::new(); 364 364 + 365 365 + // Strip $type fields (these are often added/removed) 366 366 + let original_stripped = strip_dollar_type(original); 367 367 + let generated_stripped = strip_dollar_type(generated); 368 368 + 369 369 + // Check if they're identical after stripping $type 370 370 + if original_stripped == generated_stripped { 371 371 + return ComparisonResult::Perfect; 372 372 + } 373 373 + 374 374 + // Allow $type differences 375 375 + if has_only_dollar_type_diff(&original_stripped, &generated_stripped) { 376 376 + acceptable_diffs.push("$type fields".to_string()); 377 377 + } 378 378 + 379 379 + // Check for field ordering differences (same fields, different order) 380 380 + if has_only_ordering_diff(&original_stripped, &generated_stripped) { 381 381 + acceptable_diffs.push("field ordering".to_string()); 382 382 + return ComparisonResult::AcceptableDifferences(acceptable_diffs); 383 383 + } 384 384 + 385 385 + // If we have acceptable diffs, return them 386 386 + if !acceptable_diffs.is_empty() { 387 387 + return ComparisonResult::AcceptableDifferences(acceptable_diffs); 388 388 + } 389 389 + 390 390 + // Otherwise, it's a failure 391 391 + ComparisonResult::Failure(format!( 392 392 + "Structural differences detected" 393 393 + )) 394 394 + } 395 395 + 396 396 + /// Recursively strip $type fields from JSON 397 397 + fn strip_dollar_type(value: &serde_json::Value) -> serde_json::Value { 398 398 + match value { 399 399 + serde_json::Value::Object(map) => { 400 400 + let mut new_map = serde_json::Map::new(); 401 401 + for (k, v) in map { 402 402 + if k != "$type" { 403 403 + new_map.insert(k.clone(), strip_dollar_type(v)); 404 404 + } 405 405 + } 406 406 + serde_json::Value::Object(new_map) 407 407 + } 408 408 + serde_json::Value::Array(arr) => { 409 409 + serde_json::Value::Array(arr.iter().map(strip_dollar_type).collect()) 410 410 + } 411 411 + _ => value.clone(), 412 412 + } 413 413 + } 414 414 + 415 415 + /// Check if the only difference is $type fields 416 416 + fn has_only_dollar_type_diff(v1: &serde_json::Value, v2: &serde_json::Value) -> bool { 417 417 + // After stripping $type, they should be equal 418 418 + v1 == v2 419 419 + } 420 420 + 421 421 + /// Write diff files showing differences between original and generated JSON 422 422 + fn write_diff_file( 423 423 + diffs_dir: &Path, 424 424 + nsid: &str, 425 425 + original_json: &str, 426 426 + generated_json: &str, 427 427 + diff_type: &str, 428 428 + ) -> Result<(), String> { 429 429 + // Create subdirectory based on diff type 430 430 + let type_dir = diffs_dir.join(diff_type); 431 431 + fs::create_dir_all(&type_dir) 432 432 + .map_err(|e| format!("Failed to create diff type directory: {}", e))?; 433 433 + 434 434 + // Create base filename from NSID 435 435 + let base_filename = nsid.replace('.', "_"); 436 436 + 437 437 + // Write original JSON 438 438 + let original_path = type_dir.join(format!("{}.original.json", base_filename)); 439 439 + fs::write(&original_path, original_json) 440 440 + .map_err(|e| format!("Failed to write original JSON: {}", e))?; 441 441 + 442 442 + // Write generated JSON 443 443 + let generated_path = type_dir.join(format!("{}.generated.json", base_filename)); 444 444 + fs::write(&generated_path, generated_json) 445 445 + .map_err(|e| format!("Failed to write generated JSON: {}", e))?; 446 446 + 447 447 + // Run diff command and save output 448 448 + let diff_path = type_dir.join(format!("{}.diff", base_filename)); 449 449 + let diff_output = Command::new("diff") 450 450 + .arg("-u") 451 451 + .arg(&original_path) 452 452 + .arg(&generated_path) 453 453 + .output() 454 454 + .map_err(|e| format!("Failed to run diff command: {}", e))?; 455 455 + 456 456 + // diff returns exit code 1 when files differ, which is expected 457 457 + // Only error if exit code is 2+ (indicates an error running diff) 458 458 + if diff_output.status.code() == Some(2) { 459 459 + return Err(format!("diff command error: {}", String::from_utf8_lossy(&diff_output.stderr))); 460 460 + } 461 461 + 462 462 + // Write diff output 463 463 + fs::write(&diff_path, &diff_output.stdout) 464 464 + .map_err(|e| format!("Failed to write diff output: {}", e))?; 465 465 + 466 466 + Ok(()) 467 467 + } 468 468 + 469 469 + /// Check if the only difference is field ordering in objects 470 470 + fn has_only_ordering_diff(v1: &serde_json::Value, v2: &serde_json::Value) -> bool { 471 471 + match (v1, v2) { 472 472 + (serde_json::Value::Object(map1), serde_json::Value::Object(map2)) => { 473 473 + // Check if they have the same keys 474 474 + let keys1: HashSet<_> = map1.keys().collect(); 475 475 + let keys2: HashSet<_> = map2.keys().collect(); 476 476 + 477 477 + if keys1 != keys2 { 478 478 + return false; 479 479 + } 480 480 + 481 481 + // Check if all values match (recursively) 482 482 + for key in keys1 { 483 483 + let val1 = &map1[key]; 484 484 + let val2 = &map2[key]; 485 485 + 486 486 + if !has_only_ordering_diff(val1, val2) && val1 != val2 { 487 487 + return false; 488 488 + } 489 489 + } 490 490 + 491 491 + true 492 492 + } 493 493 + (serde_json::Value::Array(arr1), serde_json::Value::Array(arr2)) => { 494 494 + // Arrays must match exactly (order matters) 495 495 + if arr1.len() != arr2.len() { 496 496 + return false; 497 497 + } 498 498 + 499 499 + arr1.iter() 500 500 + .zip(arr2.iter()) 501 501 + .all(|(v1, v2)| has_only_ordering_diff(v1, v2) || v1 == v2) 502 502 + } 503 503 + _ => v1 == v2, 504 504 + } 505 505 + }

+2

tests/real_world/roundtrip/.gitignore

··· 1 1 + # Diff files generated by round-trip tests 2 2 + diffs/