+5 -1

mlf-lang/src/ast.rs

··· 62 62 pub span: Span, 63 63 } 64 64 65 65 - /// Annotation (e.g., @deprecated, @since(1, 0, 0)) 65 65 + /// Annotation (e.g., @deprecated, @rust:deprecated, @rust,typescript:deprecated) 66 66 #[derive(Debug, Clone, PartialEq)] 67 67 pub struct Annotation { 68 68 + /// Generator selectors (empty for bare annotations visible to all generators) 69 69 + pub selectors: Vec<Ident>, 70 70 + /// Annotation name 68 71 pub name: Ident, 72 72 + /// Annotation arguments 69 73 pub args: Vec<AnnotationArg>, 70 74 pub span: Span, 71 75 }

+155 -1

mlf-lang/src/parser.rs

··· 230 230 231 231 fn parse_annotation(&mut self) -> Result<Annotation, ParseError> { 232 232 let start = self.expect(LexToken::At)?; 233 233 - let name = self.parse_ident()?; 233 233 + let first_ident = self.parse_ident()?; 234 234 + 235 235 + // Check if this is a generator selector or bare annotation 236 236 + let (selectors, name) = if matches!(self.current().token, LexToken::Comma) || matches!(self.current().token, LexToken::Colon) { 237 237 + // Generator selector syntax: @rust:deprecated or @rust,typescript:deprecated 238 238 + let mut selectors = alloc::vec![first_ident]; 239 239 + 240 240 + // Parse comma-separated selectors 241 241 + while matches!(self.current().token, LexToken::Comma) { 242 242 + self.advance(); // consume comma 243 243 + selectors.push(self.parse_ident()?); 244 244 + } 245 245 + 246 246 + // Now expect and consume the colon 247 247 + self.expect(LexToken::Colon)?; 248 248 + 249 249 + // Parse the annotation name 250 250 + let name = self.parse_ident()?; 251 251 + (selectors, name) 252 252 + } else { 253 253 + // Bare annotation: @deprecated 254 254 + (alloc::vec![], first_ident) 255 255 + }; 234 256 235 257 let mut args = Vec::new(); 236 258 ··· 257 279 }; 258 280 259 281 Ok(Annotation { 282 282 + selectors, 260 283 name, 261 284 args, 262 285 span: Span::new(start.start, end), ··· 1462 1485 assert_eq!(r.annotations[0].args.len(), 2); 1463 1486 } 1464 1487 _ => panic!("Expected record"), 1488 1488 + } 1489 1489 + } 1490 1490 + 1491 1491 + #[test] 1492 1492 + fn test_parse_annotation_bare() { 1493 1493 + let input = "@deprecated\nquery foo();"; 1494 1494 + let result = parse_lexicon(input); 1495 1495 + assert!(result.is_ok()); 1496 1496 + let lexicon = result.unwrap(); 1497 1497 + assert_eq!(lexicon.items.len(), 1); 1498 1498 + match &lexicon.items[0] { 1499 1499 + Item::Query(q) => { 1500 1500 + assert_eq!(q.annotations.len(), 1); 1501 1501 + assert_eq!(q.annotations[0].selectors.len(), 0); 1502 1502 + assert_eq!(q.annotations[0].name.name, "deprecated"); 1503 1503 + } 1504 1504 + _ => panic!("Expected query"), 1505 1505 + } 1506 1506 + } 1507 1507 + 1508 1508 + #[test] 1509 1509 + fn test_parse_annotation_single_selector() { 1510 1510 + let input = "@rust:deprecated\nquery bar();"; 1511 1511 + let result = parse_lexicon(input); 1512 1512 + assert!(result.is_ok()); 1513 1513 + let lexicon = result.unwrap(); 1514 1514 + assert_eq!(lexicon.items.len(), 1); 1515 1515 + match &lexicon.items[0] { 1516 1516 + Item::Query(q) => { 1517 1517 + assert_eq!(q.annotations.len(), 1); 1518 1518 + assert_eq!(q.annotations[0].selectors.len(), 1); 1519 1519 + assert_eq!(q.annotations[0].selectors[0].name, "rust"); 1520 1520 + assert_eq!(q.annotations[0].name.name, "deprecated"); 1521 1521 + } 1522 1522 + _ => panic!("Expected query"), 1523 1523 + } 1524 1524 + } 1525 1525 + 1526 1526 + #[test] 1527 1527 + fn test_parse_annotation_multiple_selectors() { 1528 1528 + let input = "@rust,typescript:deprecated\nquery baz();"; 1529 1529 + let result = parse_lexicon(input); 1530 1530 + assert!(result.is_ok()); 1531 1531 + let lexicon = result.unwrap(); 1532 1532 + assert_eq!(lexicon.items.len(), 1); 1533 1533 + match &lexicon.items[0] { 1534 1534 + Item::Query(q) => { 1535 1535 + assert_eq!(q.annotations.len(), 1); 1536 1536 + assert_eq!(q.annotations[0].selectors.len(), 2); 1537 1537 + assert_eq!(q.annotations[0].selectors[0].name, "rust"); 1538 1538 + assert_eq!(q.annotations[0].selectors[1].name, "typescript"); 1539 1539 + assert_eq!(q.annotations[0].name.name, "deprecated"); 1540 1540 + } 1541 1541 + _ => panic!("Expected query"), 1542 1542 + } 1543 1543 + } 1544 1544 + 1545 1545 + #[test] 1546 1546 + fn test_parse_annotation_multiple_separate() { 1547 1547 + let input = "@rust:deprecated\n@typescript:deprecated\nquery qux();"; 1548 1548 + let result = parse_lexicon(input); 1549 1549 + assert!(result.is_ok()); 1550 1550 + let lexicon = result.unwrap(); 1551 1551 + assert_eq!(lexicon.items.len(), 1); 1552 1552 + match &lexicon.items[0] { 1553 1553 + Item::Query(q) => { 1554 1554 + assert_eq!(q.annotations.len(), 2); 1555 1555 + assert_eq!(q.annotations[0].selectors[0].name, "rust"); 1556 1556 + assert_eq!(q.annotations[0].name.name, "deprecated"); 1557 1557 + assert_eq!(q.annotations[1].selectors[0].name, "typescript"); 1558 1558 + assert_eq!(q.annotations[1].name.name, "deprecated"); 1559 1559 + } 1560 1560 + _ => panic!("Expected query"), 1561 1561 + } 1562 1562 + } 1563 1563 + 1564 1564 + #[test] 1565 1565 + fn test_parse_annotation_bare_with_args() { 1566 1566 + let input = "@foo(1)\n@bar(\"blah\", true)\n@baz(thing: 1, thang: 2)\nquery test();"; 1567 1567 + let result = parse_lexicon(input); 1568 1568 + assert!(result.is_ok()); 1569 1569 + let lexicon = result.unwrap(); 1570 1570 + assert_eq!(lexicon.items.len(), 1); 1571 1571 + match &lexicon.items[0] { 1572 1572 + Item::Query(q) => { 1573 1573 + assert_eq!(q.annotations.len(), 3); 1574 1574 + // @foo(1) 1575 1575 + assert_eq!(q.annotations[0].selectors.len(), 0); 1576 1576 + assert_eq!(q.annotations[0].name.name, "foo"); 1577 1577 + assert_eq!(q.annotations[0].args.len(), 1); 1578 1578 + // @bar("blah", true) 1579 1579 + assert_eq!(q.annotations[1].selectors.len(), 0); 1580 1580 + assert_eq!(q.annotations[1].name.name, "bar"); 1581 1581 + assert_eq!(q.annotations[1].args.len(), 2); 1582 1582 + // @baz(thing: 1, thang: 2) 1583 1583 + assert_eq!(q.annotations[2].selectors.len(), 0); 1584 1584 + assert_eq!(q.annotations[2].name.name, "baz"); 1585 1585 + assert_eq!(q.annotations[2].args.len(), 2); 1586 1586 + } 1587 1587 + _ => panic!("Expected query"), 1588 1588 + } 1589 1589 + } 1590 1590 + 1591 1591 + #[test] 1592 1592 + fn test_parse_annotation_selectors_with_args() { 1593 1593 + let input = "@rust:derive(\"Debug, Clone\")\n@typescript:export(name: \"CustomName\")\n@rust,typescript:since(1, 2, 0)\nquery test();"; 1594 1594 + let result = parse_lexicon(input); 1595 1595 + assert!(result.is_ok()); 1596 1596 + let lexicon = result.unwrap(); 1597 1597 + assert_eq!(lexicon.items.len(), 1); 1598 1598 + match &lexicon.items[0] { 1599 1599 + Item::Query(q) => { 1600 1600 + assert_eq!(q.annotations.len(), 3); 1601 1601 + // @rust:derive("Debug, Clone") 1602 1602 + assert_eq!(q.annotations[0].selectors.len(), 1); 1603 1603 + assert_eq!(q.annotations[0].selectors[0].name, "rust"); 1604 1604 + assert_eq!(q.annotations[0].name.name, "derive"); 1605 1605 + assert_eq!(q.annotations[0].args.len(), 1); 1606 1606 + // @typescript:export(name: "CustomName") 1607 1607 + assert_eq!(q.annotations[1].selectors.len(), 1); 1608 1608 + assert_eq!(q.annotations[1].selectors[0].name, "typescript"); 1609 1609 + assert_eq!(q.annotations[1].name.name, "export"); 1610 1610 + assert_eq!(q.annotations[1].args.len(), 1); 1611 1611 + // @rust,typescript:since(1, 2, 0) 1612 1612 + assert_eq!(q.annotations[2].selectors.len(), 2); 1613 1613 + assert_eq!(q.annotations[2].selectors[0].name, "rust"); 1614 1614 + assert_eq!(q.annotations[2].selectors[1].name, "typescript"); 1615 1615 + assert_eq!(q.annotations[2].name.name, "since"); 1616 1616 + assert_eq!(q.annotations[2].args.len(), 3); 1617 1617 + } 1618 1618 + _ => panic!("Expected query"), 1465 1619 } 1466 1620 } 1467 1621

+3 -207

website/content/docs/cli/07-fetch.md

··· 105 105 106 106 ## How It Works 107 107 108 108 - The fetch command follows the ATProto lexicon discovery protocol: 109 109 - 110 110 - 1. **DNS Lookup** - Queries `_lexicon.<reversed-authority>` TXT record 111 111 - 2. **DID Resolution** - Resolves the DID to a PDS endpoint 112 112 - 3. **Fetch Records** - Queries `com.atproto.repo.listRecords` for lexicon schemas 113 113 - 4. **Save & Convert** - Saves JSON and converts to MLF format 114 114 - 5. **Update Lockfile** - Records NSIDs, DIDs, checksums, and dependencies 108 108 + The fetch command uses ATProto's lexicon discovery protocol to locate and download lexicons via DNS lookups and DID resolution, then converts them to MLF format and updates the lockfile. 115 109 116 110 ## Examples 117 111 ··· 208 202 2. Adds `"com.example.forum.*"` to the dependencies array in `mlf.toml` 209 203 3. Creates `mlf.toml` if it doesn't exist 210 204 211 211 - ## Storage Structure 212 205 213 213 - Fetched lexicons are stored in `.mlf/lexicons/`: 214 214 - 215 215 - ``` 216 216 - .mlf/ 217 217 - ├── .gitignore # Auto-generated 218 218 - └── lexicons/ 219 219 - ├── json/ # Original JSON lexicons 220 220 - │ ├── com/ 221 221 - │ │ └── example/ 222 222 - │ │ ├── forum/ 223 223 - │ │ │ ├── post.json 224 224 - │ │ │ └── thread.json 225 225 - │ │ └── social/ 226 226 - │ │ └── profile.json 227 227 - │ └── app/ 228 228 - │ └── bsky/ 229 229 - │ └── feed/ 230 230 - │ └── post.json 231 231 - └── mlf/ # Converted MLF format 232 232 - ├── com/ 233 233 - │ └── example/ 234 234 - │ ├── forum/ 235 235 - │ │ ├── post.mlf 236 236 - │ │ └── thread.mlf 237 237 - │ └── social/ 238 238 - │ └── profile.mlf 239 239 - └── app/ 240 240 - └── bsky/ 241 241 - └── feed/ 242 242 - └── post.mlf 243 243 - ``` 244 244 - 245 245 - **Note:** The lockfile (`mlf-lock.toml`) lives at the project root, sibling to `mlf.toml`. 246 246 - 247 247 - ## DNS Resolution 248 248 - 249 249 - For an NSID like `com.example.forum.post`: 250 250 - 251 251 - 1. Extract authority: `com.example` (first 2 segments) 252 252 - 2. Reverse for DNS: `example.com` 253 253 - 3. Query TXT record: `_lexicon.example.com` 254 254 - 4. Parse `did=did:web:...` or `did=did:plc:...` 255 255 - 256 256 - **Example DNS record:** 257 257 - ``` 258 258 - _lexicon.example.com. 300 IN TXT "did=did:web:example.com" 259 259 - ``` 260 260 - 261 261 - ## DID Resolution 262 262 - 263 263 - ### did:web 264 264 - 265 265 - For `did:web:example.com`, the PDS is `https://example.com` 266 266 - 267 267 - ### did:plc 268 268 - 269 269 - For `did:plc:abc123...`, query `https://plc.directory/did:plc:abc123...` to get the PDS endpoint from the DID document. 270 270 - 271 271 - ## Fetched Lexicons in Your Code 272 272 - 273 273 - Once fetched, lexicons in `.mlf/lexicons/mlf/` are automatically available for: 274 274 - 275 275 - ### Type References 276 276 - 277 277 - ```mlf 278 278 - use com.example.forum.post; 279 279 - 280 280 - record reply { 281 281 - post!: com.example.forum.post, 282 282 - text!: string, 283 283 - } 284 284 - ``` 285 285 - 286 286 - ### Code Generation 287 287 - 288 288 - ```bash 289 289 - mlf generate code -g typescript -i my-lexicon.mlf -o src/types/ 290 290 - ``` 291 291 - 292 292 - The generator can resolve references to fetched lexicons. 293 293 - 294 294 - ### Validation 295 295 - 296 296 - ```bash 297 297 - mlf check my-lexicon.mlf 298 298 - ``` 299 299 - 300 300 - The check command loads fetched lexicons for type resolution. 301 301 - 302 302 - ## Working Without mlf.toml 303 303 - 304 304 - If you don't have an `mlf.toml`, the fetch command will offer to create one: 305 305 - 306 306 - ```bash 307 307 - $ mlf fetch com.example.forum.* 308 308 - No mlf.toml found in current or parent directories. 309 309 - Would you like to create one in the current directory? (y/n) 310 310 - y 311 311 - Created mlf.toml in /path/to/current/dir 312 312 - ... 313 313 - ``` 314 314 - 315 315 - ## Error Handling 316 316 - 317 317 - ### DNS Errors 318 318 - 319 319 - ``` 320 320 - ✗ DNS lookup failed: No TXT record found for _lexicon.example.com 321 321 - ``` 322 322 - 323 323 - **Causes:** 324 324 - - Domain doesn't have a lexicon TXT record 325 325 - - DNS propagation delay 326 326 - - Network issues 327 327 - 328 328 - ### DID Resolution Errors 329 329 - 330 330 - ``` 331 331 - ✗ Failed to resolve DID: No PDS endpoint found in DID document 332 332 - ``` 333 333 - 334 334 - **Causes:** 335 335 - - Invalid DID format 336 336 - - PLC directory unreachable 337 337 - - DID document missing PDS service 338 338 - 339 339 - ### No Records Found 340 340 - 341 341 - ``` 342 342 - ✗ No lexicons matched pattern: com.example.forum.* 343 343 - ``` 344 344 - 345 345 - **Causes:** 346 346 - - No lexicons exist matching the pattern 347 347 - - Wrong NSID (typo) 348 348 - - PDS doesn't support lexicon publishing 349 349 - 350 350 - ### Invalid NSID Format 351 351 - 352 352 - ``` 353 353 - ✗ NSID must have at least 2 segments or use wildcard: com 354 354 - ``` 355 355 - 356 356 - **Solution:** 357 357 - - Use a specific NSID: `com.example.forum.post` 358 358 - - Or use a wildcard: `com.example.forum.*` 359 359 - 360 360 - ### Checksum Mismatch (--locked mode) 361 361 - 362 362 - ``` 363 363 - ✗ Checksum mismatch for place.stream.facet: expected sha256:abc123, got sha256:def456 364 364 - ``` 365 365 - 366 366 - **Causes:** 367 367 - - Lexicon was updated on the server 368 368 - - Lock file is out of date 369 369 - 370 370 - **Solution:** 371 371 - ```bash 372 372 - mlf fetch --update # Update lockfile with new checksums 373 373 - ``` 374 206 375 207 ## Best Practices 376 208 377 209 1. **Commit lockfile** - Always commit `mlf-lock.toml` to version control 378 210 2. **Use --locked in CI** - Ensures reproducible builds in CI/CD pipelines 379 379 - 3. **Fetch before work** - Always fetch dependencies before coding 380 380 - 4. **Use --save** - Keep `mlf.toml` up to date with dependencies 381 381 - 5. **Don't commit `.mlf/`** - Let each developer fetch independently 382 382 - 6. **Check DNS** - Verify TXT records before fetching 383 383 - 7. **Update explicitly** - Use `mlf fetch --update` when you want latest versions 211 211 + 3. **Don't commit `.mlf/`** - Let each developer fetch independently 212 212 + 4. **Update explicitly** - Use `mlf fetch --update` when you want latest versions 384 213 385 214 ## Comparison with npm/cargo 386 215 ··· 396 225 | Lock file | `package-lock.json` | `Cargo.lock` | `mlf-lock.toml` | 397 226 | Cache | `node_modules/` | `~/.cargo/` | `.mlf/` | 398 227 399 399 - ## Troubleshooting 400 400 - 401 401 - ### Network Issues 402 402 - 403 403 - ```bash 404 404 - # Check DNS resolution 405 405 - dig TXT _lexicon.example.com 406 406 - 407 407 - # Test DID resolution 408 408 - curl https://plc.directory/did:plc:abc123 409 409 - ``` 410 410 - 411 411 - ### Invalid NSID Format 412 412 - 413 413 - Make sure you're using the correct format: 414 414 - - ✓ `com.example.forum.post` (specific lexicon) 415 415 - - ✓ `com.example.forum.*` (wildcard) 416 416 - - ✓ `app.bsky.feed.*` (real-world wildcard) 417 417 - - ✗ `com` (must have at least 2 segments) 418 418 - 419 419 - ### Permission Errors 420 420 - 421 421 - Ensure you have write permissions for the project directory to create `.mlf/` and `mlf-lock.toml`. 422 422 - 423 423 - ### Conflicting Flags 424 424 - 425 425 - ``` 426 426 - ✗ Cannot use --update and --locked together 427 427 - ``` 428 428 - 429 429 - Choose one mode: 430 430 - - Use `--update` to get latest versions 431 431 - - Use `--locked` for strict reproducible builds

+40 -52

website/content/docs/language-guide/11-annotations.md

··· 37 37 38 38 ```mlf 39 39 @validate(min: 0, max: 100, strict: true) 40 40 - @codegen(language: "rust", derive: "Debug, Clone") 40 40 + @cache(ttl: 3600, strategy: "lru") 41 41 record example { 42 42 field: integer, 43 43 } ··· 72 72 } 73 73 ``` 74 74 75 75 - ## MLF Annotations vs Generator Annotations 75 75 + ## Annotation Semantics 76 76 77 77 - MLF distinguishes between two categories: 77 77 + Annotations in MLF are interpreted by whatever consumes them - whether that's the MLF compiler itself or external code generators. 78 78 79 79 - ### 1. MLF Annotations 79 79 + ### Bare Annotations 80 80 81 81 - Built into the MLF language and affect compilation/validation. These are **bare annotations** without any namespace prefix: 82 82 - 83 83 - **`@main`** - Marks an item as the main definition when there's ambiguity: 81 81 + **Bare annotations** (without generator selectors) are visible to **all generators** and each can interpret them as needed: 84 82 85 83 ```mlf 86 86 - // File: com/example/thread.mlf 87 87 - @main 88 88 - record thread { 89 89 - title!: string, 90 90 - } 91 91 - 92 92 - // This def shares the same name but is not main 93 93 - def type thread = { 94 94 - id!: string, 95 95 - viewCount!: integer, 96 96 - } 84 84 + // Visible to all generators - each interprets as appropriate 85 85 + @deprecated 86 86 + query foo(); 97 87 ``` 98 88 99 99 - See [Important Info](/docs/language-guide/important-info/#the-main-definition) for more details on the `@main` annotation. 89 89 + MLF itself recognizes the **`@main` annotation** for resolving naming conflicts. See [Important Info](/docs/language-guide/important-info/#the-main-definition) for details. 100 90 101 101 - ### 2. Generator Annotations 91 91 + ### Generator Selectors 102 92 103 103 - Used by code generators and external tools. These have no effect on MLF compilation and **must** be namespaced with the generator name: 93 93 + To target **specific generators**, use the generator selector syntax with a colon: 104 94 105 95 ```mlf 106 106 - @rust:derive("Debug, Clone, Serialize") 107 107 - @typescript:export 108 108 - @go:tag(json: "custom_name") 109 109 - record example { 110 110 - field: string, 111 111 - } 96 96 + // Only for rust generator 97 97 + @rust:deprecated 98 98 + query bar(); 99 99 + 100 100 + // Only for typescript generator 101 101 + @typescript:deprecated 102 102 + query baz(); 112 103 ``` 113 104 114 114 - **Generator namespacing rules:** 115 115 - - All generator annotations must have a namespace prefix (e.g., `@rust:foo`) 116 116 - - Use `@all:annotation` to apply an annotation to all generators 117 117 - - Bare annotations (without `:`) are reserved for MLF itself 105 105 + ### Multiple Generator Selectors 118 106 119 119 - **Common generator namespaces:** 120 120 - - `@rust:*` - Rust code generator annotations 121 121 - - `@typescript:*` - TypeScript code generator annotations 122 122 - - `@go:*` - Go code generator annotations 123 123 - - `@python:*` - Python code generator annotations 124 124 - - `@all:*` - Applies to all generators 107 107 + You can apply an annotation to multiple specific generators using comma-separated selectors: 125 108 126 126 - ## Custom Annotations 109 109 + ```mlf 110 110 + // For both rust AND typescript 111 111 + @rust,typescript:deprecated 112 112 + query qux(); 113 113 + ``` 127 114 128 128 - You can define your own annotations for custom tooling: 115 115 + Alternatively, you can write separate annotations: 129 116 130 117 ```mlf 131 131 - @myapp:cache(ttl: 3600) 132 132 - @myapp:permission("read:public") 133 133 - query getProfile(actor!: Did): profile; 134 134 - 135 135 - @myapp:audit_log 136 136 - @myapp:rate_limit(requests: 100, window: 60) 137 137 - procedure updateProfile(data!: profile): unit; 118 118 + // Equivalent to above 119 119 + @rust:deprecated 120 120 + @typescript:deprecated 121 121 + query qux(); 138 122 ``` 139 123 140 140 - The interpretation is entirely up to your tooling. 124 124 + **Common generator selectors:** 125 125 + - `@rust:*` - Rust code generator annotations 126 126 + - `@typescript:*` - TypeScript code generator annotations 127 127 + - `@go:*` - Go code generator annotations 141 128 142 129 ## Annotation Processing 143 130 ··· 153 140 154 141 ## Best Practices 155 142 156 156 - 1. **Always namespace generator annotations** - Use `@generator:name` for all generator-specific annotations 157 157 - 2. **Use `@all:` for cross-generator annotations** - When an annotation should apply to all generators 158 158 - 3. **Document custom annotations** - Keep a registry of annotations your project uses 159 159 - 4. **Be consistent** - Use the same annotation patterns across your codebase 160 160 - 5. **Don't overuse** - Annotations should augment, not replace, good design 143 143 + 1. **Use bare annotations for universal concepts** - Use `@deprecated` without selectors when you want all generators to see it 144 144 + 2. **Use generator selectors for specific tooling** - Use `@rust:derive` or `@typescript:export` when targeting one generator 145 145 + 3. **Group with comma selectors when appropriate** - Use `@rust,typescript:deprecated` to apply the same annotation to multiple generators 146 146 + 4. **Document custom annotations** - Keep a registry of annotations your project uses 147 147 + 5. **Be consistent** - Use the same annotation patterns across your codebase 148 148 + 6. **Don't overuse** - Annotations should augment, not replace, good design 161 149 162 150 ## What's Next? 163 151