An experimental TypeSpec syntax for Lexicon

[New feature] Externals #2

closed opened by danabra.mov targeting main from external

A step towards https://tangled.org/@danabra.mov/typelex/issues/5.

This adds support for Externals, which are namespaces with @external decorator.

@external
namespace com.atproto.label.defs {
  model Label { }
}

@external
namespace app.bsky.graph.defs {
  model StarterPackViewBasic { }
  model ListViewBasic { }
}

They do NOT get emitted as JSON. They only exist to represent external Lexicons for which you have only JSON.

Models inside @external namespaces must be empty. However, they're allowed to be @tokens which we need to know to disambiguate unions and knownValues. (In the future, it's possible we won't rely on knowing that, but for now we do.)

I've converted the playground to use externals properly. I will convert tests in a follow-up.

Labels

None yet.

assignee

None yet.

Participants 1
AT URI
at://did:plc:fpruhuo22xkm5o7ttr2ktxdo/sh.tangled.repo.pull/3m2siinlkdr22
+1736 -100
Diff #1
+13
packages/emitter/lib/decorators.tsp
··· 159 159 * ``` 160 160 */ 161 161 extern dec errors(target: unknown, ...errors: unknown[]); 162 + 163 + /** 164 + * Marks a namespace as external, preventing it from emitting JSON output. 165 + * This decorator can only be applied to namespaces. 166 + * Useful for importing definitions from other lexicons without re-emitting them. 167 + * 168 + * @example 169 + * ```typespec 170 + * @external 171 + * namespace com.atproto.repo.defs; 172 + * ``` 173 + */ 174 + extern dec external(target: Namespace);
+23
packages/emitter/src/decorators.ts
··· 24 24 const inlineKey = Symbol("inline"); 25 25 const maxBytesKey = Symbol("maxBytes"); 26 26 const minBytesKey = Symbol("minBytes"); 27 + const externalKey = Symbol("external"); 27 28 28 29 /** 29 30 * @maxBytes decorator for maximum length of bytes type ··· 293 294 294 295 export function isReadOnly(program: Program, target: Type): boolean { 295 296 return program.stateSet(readOnlyKey).has(target); 297 + } 298 + 299 + /** 300 + * @external decorator for marking a namespace as external 301 + * External namespaces are skipped during emission and don't produce JSON files 302 + */ 303 + export function $external(context: DecoratorContext, target: Type) { 304 + if (target.kind !== "Namespace") { 305 + context.program.reportDiagnostic({ 306 + code: "external-not-on-namespace", 307 + severity: "error", 308 + message: "@external decorator can only be applied to namespaces", 309 + target: target, 310 + }); 311 + return; 312 + } 313 + 314 + context.program.stateSet(externalKey).add(target); 315 + } 316 + 317 + export function isExternal(program: Program, target: Type): boolean { 318 + return program.stateSet(externalKey).has(target); 296 319 }
+1382
packages/emitter/src/emitter.ts
··· 67 67 isInline, 68 68 getMaxBytes, 69 69 getMinBytes, 70 + isExternal, 70 71 } from "./decorators.js"; 71 72 72 73 export interface EmitterOptions { ··· 121 122 return; 122 123 } 123 124 125 + // Skip external namespaces - they don't emit JSON files 126 + if (isExternal(this.program, ns)) { 127 + // Validate that all models in external namespaces are empty (stub-only) 128 + for (const [_, model] of ns.models) { 129 + if (model.properties && model.properties.size > 0) { 130 + this.program.reportDiagnostic({ 131 + code: "external-model-not-empty", 132 + severity: "error", 133 + message: `Models in @external namespaces must be empty stubs. Model '${model.name}' in namespace '${fullName}' has properties.`, 134 + target: model, 135 + }); 136 + } 137 + } 138 + return; 139 + } 140 + 124 141 // Check for TypeSpec enum syntax and throw error 125 142 if (ns.enums && ns.enums.size > 0) { 126 143 for (const [_, enumType] of ns.enums) { 144 + 145 + 146 + 147 + 148 + 149 + 150 + 151 + 152 + 153 + 154 + 155 + 156 + 157 + 158 + 159 + 160 + 161 + 162 + 163 + 164 + 165 + 166 + 167 + 168 + 169 + 170 + 171 + 172 + 173 + 174 + 175 + 176 + 177 + 178 + 179 + 180 + 181 + 182 + 183 + 184 + 185 + 186 + 187 + 188 + 189 + 190 + 191 + 192 + 193 + 194 + 195 + 196 + 197 + 198 + 199 + 200 + 201 + 202 + 203 + 204 + 205 + 206 + 207 + 208 + 209 + 210 + 211 + 212 + 213 + 214 + 215 + 216 + 217 + 218 + 219 + 220 + 221 + 222 + 223 + 224 + 225 + 226 + 227 + 228 + 229 + 230 + 231 + 232 + 233 + 234 + 235 + 236 + 237 + 238 + 239 + 240 + 241 + 242 + 243 + 244 + 245 + 246 + 247 + 248 + 249 + 250 + 251 + 252 + 253 + 254 + 255 + 256 + 257 + 258 + 259 + 260 + 261 + 262 + 263 + 264 + 265 + 266 + 267 + 268 + 269 + 270 + 271 + 272 + 273 + 274 + 275 + 276 + 277 + 278 + 279 + 280 + 281 + 282 + 283 + 284 + 285 + 286 + 287 + 288 + 289 + 290 + 291 + 292 + 293 + 294 + 295 + 296 + 297 + 298 + 299 + 300 + 301 + 302 + 303 + 304 + 305 + 306 + 307 + 308 + 309 + 310 + 311 + 312 + 313 + 314 + 315 + 316 + 317 + 318 + 319 + 320 + 321 + 322 + 323 + 324 + 325 + 326 + 327 + 328 + 329 + 330 + 331 + 332 + 333 + 334 + 335 + 336 + 337 + 338 + 339 + 340 + 341 + 342 + 343 + 344 + 345 + 346 + 347 + 348 + 349 + 350 + 351 + 352 + 353 + 354 + 355 + 356 + 357 + 358 + 359 + 360 + 361 + 362 + 363 + 364 + 365 + 366 + 367 + 368 + 369 + 370 + 371 + 372 + 373 + 374 + 375 + 376 + 377 + 378 + 379 + 380 + 381 + 382 + 383 + 384 + 385 + 386 + 387 + 388 + 389 + 390 + 391 + 392 + 393 + 394 + 395 + 396 + 397 + 398 + 399 + 400 + 401 + 402 + 403 + 404 + 405 + 406 + 407 + 408 + 409 + 410 + 411 + 412 + 413 + 414 + 415 + 416 + 417 + 418 + 419 + 420 + 421 + 422 + 423 + 424 + 425 + 426 + 427 + 428 + 429 + 430 + 431 + 432 + 433 + 434 + 435 + 436 + 437 + 438 + 439 + 440 + 441 + 442 + 443 + 444 + 445 + 446 + 447 + 448 + 449 + 450 + 451 + 452 + 453 + 454 + 455 + 456 + 457 + 458 + 459 + 460 + 461 + 462 + 463 + 464 + 465 + 466 + 467 + 468 + 469 + 470 + 471 + 472 + 473 + 474 + 475 + 476 + 477 + 478 + 479 + 480 + 481 + 482 + 483 + 484 + 485 + 486 + 487 + 488 + 489 + 490 + 491 + 492 + 493 + 494 + 495 + 496 + 497 + 498 + 499 + 500 + 501 + 502 + 503 + 504 + 505 + 506 + 507 + 508 + 509 + 510 + 511 + 512 + 513 + 514 + 515 + 516 + 517 + 518 + 519 + 520 + 521 + 522 + 523 + 524 + 525 + 526 + 527 + 528 + 529 + 530 + 531 + 532 + 533 + 534 + 535 + 536 + 537 + 538 + 539 + 540 + 541 + 542 + 543 + 544 + 545 + 546 + 547 + 548 + 549 + 550 + 551 + 552 + 553 + 554 + 555 + 556 + // Model reference union (including empty union with unknown) 557 + if (variants.unionRefs.length > 0 || variants.hasUnknown) { 558 + if ( 559 + variants.stringLiterals.length > 0 || 560 + variants.knownValueRefs.length > 0 561 + ) { 562 + this.program.reportDiagnostic({ 563 + code: "union-mixed-refs-literals", 564 + severity: "error", 565 + 566 + 567 + 568 + 569 + 570 + 571 + 572 + 573 + 574 + 575 + 576 + 577 + 578 + 579 + 580 + 581 + 582 + 583 + 584 + 585 + 586 + 587 + 588 + 589 + 590 + 591 + 592 + 593 + 594 + 595 + 596 + 597 + 598 + 599 + 600 + 601 + 602 + 603 + 604 + 605 + 606 + 607 + 608 + 609 + 610 + 611 + 612 + 613 + 614 + 615 + 616 + 617 + 618 + 619 + 620 + 621 + 622 + 623 + 624 + 625 + 626 + 627 + 628 + 629 + 630 + 631 + 632 + 633 + 634 + 635 + 636 + 637 + 638 + 639 + 640 + 641 + 642 + 643 + 644 + 645 + 646 + 647 + 648 + 649 + 650 + 651 + 652 + 653 + 654 + 655 + 656 + 657 + 658 + 659 + 660 + 661 + 662 + 663 + 664 + 665 + 666 + 667 + 668 + 669 + 670 + 671 + 672 + 673 + 674 + 675 + 676 + 677 + 678 + 679 + 680 + 681 + 682 + 683 + 684 + 685 + 686 + 687 + 688 + 689 + 690 + 691 + 692 + 693 + 694 + 695 + 696 + 697 + 698 + 699 + 700 + 701 + 702 + 703 + 704 + 705 + 706 + 707 + 708 + 709 + 710 + 711 + 712 + 713 + 714 + 715 + 716 + 717 + 718 + 719 + 720 + 721 + 722 + 723 + 724 + 725 + 726 + 727 + 728 + 729 + 730 + 731 + 732 + 733 + 734 + 735 + 736 + 737 + 738 + 739 + 740 + 741 + 742 + 743 + 744 + 745 + 746 + 747 + 748 + 749 + 750 + 751 + 752 + 753 + 754 + 755 + 756 + 757 + 758 + 759 + 760 + 761 + 762 + 763 + 764 + 765 + 766 + 767 + 768 + 769 + 770 + 771 + 772 + 773 + 774 + 775 + 776 + 777 + 778 + 779 + 780 + 781 + 782 + 783 + 784 + 785 + 786 + 787 + 788 + 789 + 790 + 791 + 792 + 793 + 794 + 795 + 796 + 797 + 798 + 799 + 800 + 801 + 802 + 803 + 804 + 805 + 806 + 807 + 808 + 809 + 810 + 811 + 812 + 813 + 814 + 815 + 816 + 817 + 818 + 819 + 820 + 821 + 822 + 823 + 824 + 825 + 826 + 827 + 828 + 829 + 830 + 831 + 832 + 833 + 834 + 835 + 836 + 837 + 838 + 839 + 840 + 841 + 842 + 843 + 844 + 845 + 846 + 847 + 848 + 849 + 850 + 851 + 852 + 853 + 854 + 855 + 856 + 857 + 858 + 859 + 860 + 861 + 862 + 863 + 864 + 865 + 866 + 867 + 868 + 869 + 870 + 871 + 872 + 873 + 874 + 875 + 876 + 877 + 878 + 879 + 880 + 881 + 882 + 883 + 884 + 885 + 886 + 887 + 888 + 889 + 890 + 891 + 892 + 893 + 894 + 895 + 896 + 897 + 898 + 899 + 900 + 901 + 902 + 903 + 904 + 905 + 906 + 907 + 908 + 909 + 910 + 911 + 912 + 913 + 914 + 915 + 916 + 917 + 918 + 919 + 920 + 921 + 922 + 923 + 924 + 925 + 926 + 927 + 928 + 929 + 930 + 931 + 932 + 933 + 934 + 935 + 936 + 937 + 938 + 939 + 940 + 941 + 942 + 943 + 944 + 945 + 946 + 947 + 948 + 949 + 950 + 951 + 952 + 953 + 954 + 955 + 956 + 957 + 958 + 959 + 960 + 961 + 962 + 963 + 964 + 965 + 966 + 967 + 968 + 969 + 970 + 971 + 972 + 973 + 974 + 975 + 976 + 977 + 978 + 979 + 980 + 981 + 982 + 983 + 984 + 985 + 986 + 987 + 988 + 989 + 990 + 991 + 992 + 993 + 994 + 995 + 996 + 997 + 998 + 999 + 1000 + 1001 + 1002 + 1003 + 1004 + 1005 + 1006 + 1007 + 1008 + 1009 + 1010 + 1011 + 1012 + 1013 + 1014 + 1015 + 1016 + 1017 + 1018 + 1019 + 1020 + 1021 + 1022 + 1023 + 1024 + 1025 + 1026 + 1027 + 1028 + 1029 + 1030 + 1031 + 1032 + 1033 + 1034 + 1035 + 1036 + 1037 + 1038 + 1039 + 1040 + 1041 + 1042 + 1043 + 1044 + 1045 + 1046 + 1047 + 1048 + 1049 + 1050 + 1051 + 1052 + 1053 + 1054 + 1055 + 1056 + 1057 + 1058 + 1059 + 1060 + 1061 + 1062 + 1063 + 1064 + 1065 + 1066 + 1067 + 1068 + 1069 + 1070 + 1071 + 1072 + 1073 + 1074 + 1075 + 1076 + 1077 + 1078 + 1079 + 1080 + 1081 + 1082 + 1083 + 1084 + 1085 + 1086 + 1087 + 1088 + 1089 + 1090 + 1091 + 1092 + 1093 + 1094 + 1095 + 1096 + 1097 + 1098 + 1099 + 1100 + 1101 + 1102 + 1103 + 1104 + 1105 + 1106 + 1107 + 1108 + 1109 + 1110 + 1111 + 1112 + 1113 + 1114 + 1115 + 1116 + 1117 + 1118 + 1119 + 1120 + 1121 + 1122 + 1123 + 1124 + 1125 + 1126 + 1127 + 1128 + 1129 + 1130 + 1131 + 1132 + 1133 + 1134 + 1135 + 1136 + 1137 + 1138 + 1139 + 1140 + 1141 + 1142 + 1143 + 1144 + 1145 + 1146 + 1147 + 1148 + 1149 + 1150 + 1151 + 1152 + 1153 + 1154 + 1155 + 1156 + 1157 + 1158 + 1159 + 1160 + 1161 + 1162 + 1163 + 1164 + 1165 + 1166 + 1167 + 1168 + 1169 + 1170 + 1171 + 1172 + 1173 + 1174 + 1175 + 1176 + 1177 + 1178 + 1179 + 1180 + 1181 + 1182 + 1183 + 1184 + 1185 + 1186 + 1187 + 1188 + 1189 + 1190 + 1191 + 1192 + 1193 + 1194 + 1195 + 1196 + 1197 + 1198 + 1199 + 1200 + 1201 + 1202 + 1203 + 1204 + 1205 + 1206 + 1207 + 1208 + 1209 + 1210 + 1211 + 1212 + 1213 + 1214 + 1215 + 1216 + 1217 + 1218 + 1219 + 1220 + 1221 + 1222 + 1223 + 1224 + 1225 + 1226 + 1227 + 1228 + 1229 + 1230 + 1231 + 1232 + 1233 + 1234 + 1235 + 1236 + 1237 + 1238 + 1239 + 1240 + 1241 + 1242 + 1243 + 1244 + 1245 + 1246 + 1247 + 1248 + 1249 + 1250 + 1251 + 1252 + 1253 + 1254 + 1255 + 1256 + 1257 + 1258 + 1259 + 1260 + 1261 + 1262 + 1263 + 1264 + 1265 + 1266 + 1267 + 1268 + 1269 + 1270 + 1271 + 1272 + 1273 + 1274 + 1275 + 1276 + 1277 + 1278 + 1279 + 1280 + 1281 + 1282 + 1283 + 1284 + 1285 + 1286 + 1287 + 1288 + 1289 + 1290 + 1291 + 1292 + 1293 + 1294 + 1295 + 1296 + 1297 + 1298 + 1299 + 1300 + 1301 + 1302 + 1303 + 1304 + 1305 + 1306 + 1307 + 1308 + 1309 + 1310 + 1311 + 1312 + 1313 + 1314 + 1315 + 1316 + 1317 + 1318 + 1319 + 1320 + 1321 + 1322 + 1323 + 1324 + 1325 + 1326 + 1327 + 1328 + 1329 + 1330 + 1331 + 1332 + 1333 + 1334 + 1335 + 1336 + 1337 + 1338 + 1339 + 1340 + 1341 + 1342 + 1343 + 1344 + 1345 + 1346 + 1347 + 1348 + 1349 + 1350 + 1351 + 1352 + 1353 + 1354 + 1355 + 1356 + 1357 + 1358 + 1359 + 1360 + 1361 + 1362 + 1363 + 1364 + 1365 + 1366 + 1367 + 1368 + 1369 + 1370 + 1371 + 1372 + 1373 + 1374 + 1375 + 1376 + 1377 + 1378 + 1379 + 1380 + 1381 + 1382 + 1383 + 1384 + 1385 + 1386 + 1387 + 1388 + 1389 + 1390 + 1391 + 1392 + 1393 + 1394 + 1395 + 1396 + 1397 + 1398 + 1399 + 1400 + 1401 + 1402 + 1403 + 1404 + 1405 + 1406 + 1407 + 1408 + 1409 + 1410 + 1411 + 1412 + 1413 + 1414 + 1415 + 1416 + 1417 + 1418 + 1419 + 1420 + 1421 + 1422 + 1423 + 1424 + 1425 + 1426 + 1427 + 1428 + 1429 + 1430 + 1431 + 1432 + 1433 + 1434 + 1435 + 1436 + 1437 + 1438 + 1439 + 1440 + 1441 + 1442 + 1443 + 1444 + 1445 + 1446 + 1447 + 1448 + 1449 + 1450 + 1451 + 1452 + 1453 + 1454 + 1455 + 1456 + 1457 + 1458 + 1459 + 1460 + 1461 + 1462 + 1463 + 1464 + 1465 + 1466 + 1467 + 1468 + 1469 + 1470 + 1471 + 1472 + 1473 + 1474 + 1475 + 1476 + 1477 + 1478 + 1479 + 1480 + 1481 + 1482 + 1483 + 1484 + 1485 + 1486 + 1487 + 1488 + 1489 + 1490 + 1491 + 1492 + 1493 + 1494 + 1495 + 1496 + 1497 + model: Model, 1498 + fullyQualified = false, 1499 + ): string | null { 1500 + return this.getReference( 1501 + model, 1502 + model.name, 1503 + model.namespace, 1504 + fullyQualified, 1505 + ); 1506 + } 1507 + 1508 + private getUnionReference(union: Union): string | null {
+2
packages/emitter/src/tsp-index.ts
··· 14 14 $inline, 15 15 $maxBytes, 16 16 $minBytes, 17 + $external, 17 18 } from "./decorators.js"; 18 19 19 20 /** @internal */ ··· 34 35 inline: $inline, 35 36 maxBytes: $maxBytes, 36 37 minBytes: $minBytes, 38 + external: $external, 37 39 }, 38 40 };
+23 -1
packages/emitter/test/spec.test.ts
··· 106 106 assert.deepStrictEqual(actual, expected); 107 107 }); 108 108 } else { 109 - it.skip(`TODO: ${expectedPath} (add ${inputPath})`, function () {}); 109 + it(`should emit ${expectedPath}`, function () { 110 + assert.fail( 111 + `Expected output file ${expectedPath} has no corresponding input file ${inputPath}. ` + 112 + `Either add the input file or remove the expected output.` 113 + ); 114 + }); 110 115 } 111 116 } 117 + 118 + // Check for unexpected emitted files 119 + it("should not emit unexpected files", function () { 120 + const emittedFiles = Object.keys(emitResult.files).filter(f => f.endsWith(".json")); 121 + const expectedPaths = Object.keys(expectedFiles) 122 + .filter(f => f.endsWith(".json")) 123 + .map(normalizePathToPosix); 124 + 125 + const unexpected = emittedFiles.filter(f => !expectedPaths.includes(f)); 126 + 127 + if (unexpected.length > 0) { 128 + assert.fail( 129 + `Unexpected files were emitted: ${unexpected.join(", ")}. ` + 130 + `Either add expected output files or ensure these should not be emitted.` 131 + ); 132 + } 133 + }); 112 134 }); 113 135 } 114 136 });
+21
packages/emitter/test/spec/basic/output/com/example/other.json
··· 1 + { 2 + "lexicon": 1, 3 + "id": "com.example.other", 4 + "defs": { 5 + "main": { 6 + "type": "object", 7 + "properties": {} 8 + }, 9 + "someDef": { 10 + "type": "object", 11 + "required": [ 12 + "value" 13 + ], 14 + "properties": { 15 + "value": { 16 + "type": "string" 17 + } 18 + } 19 + } 20 + } 21 + }
+8
packages/emitter/test/spec/external/input/test/external.tsp
··· 1 + import "@typelex/emitter"; 2 + 3 + @external 4 + namespace test.external { 5 + model Main { } 6 + 7 + model AlsoNotEmitted { } 8 + }
+7
packages/emitter/test/spec/external/input/test/normal.tsp
··· 1 + import "@typelex/emitter"; 2 + 3 + namespace test.normal { 4 + model Main { 5 + name?: string; 6 + } 7 + }
+14
packages/emitter/test/spec/external/output/test/normal.json
··· 1 + { 2 + "lexicon": 1, 3 + "id": "test.normal", 4 + "defs": { 5 + "main": { 6 + "type": "object", 7 + "properties": { 8 + "name": { 9 + "type": "string" 10 + } 11 + } 12 + } 13 + } 14 + }
+26 -5
DOCS.md
··· 236 236 237 237 This works across files too—just remember to `import` the file with the definition. 238 238 239 - ### External Stubs 239 + ### External References 240 240 241 - If you don't have TypeSpec definitions for external Lexicons, you can stub them out: 241 + If you don't have TypeSpec definitions for external Lexicons, you can stub them out using the `@external` decorator: 242 242 243 243 ```typescript 244 244 import "@typelex/emitter"; ··· 249 249 } 250 250 } 251 251 252 - // Empty stub (like .d.ts in TypeScript) 252 + // External stub (like .d.ts in TypeScript) 253 + @external 253 254 namespace com.atproto.label.defs { 254 255 model SelfLabels { } 256 + @token model SomeToken { } // use @token for tokens 255 257 } 256 258 ``` 257 259 258 - You could collect stubs in one file and import them: 260 + The `@external` decorator tells the emitter to skip JSON output for that namespace. This is useful when referencing definitions from other Lexicons that you don't want to re-emit. 259 261 262 + You could collect external stubs in one file and import them: 263 + 260 264 ```typescript 261 265 import "@typelex/emitter"; 262 266 ··· 268 272 } 269 273 ``` 270 274 271 - You'll want to replace the stubbed lexicons in the output folder with their real JSON before running codegen. 275 + Then in `atproto-stubs.tsp`: 276 + 277 + ```typescript 278 + import "@typelex/emitter"; 279 + 280 + @external 281 + namespace com.atproto.label.defs { 282 + model SelfLabels { } 283 + } 284 + 285 + @external 286 + namespace com.atproto.repo.defs { 287 + model StrongRef { } 288 + } 289 + // ... more stubs 290 + ``` 291 + 292 + You'll want to ensure the real JSON for external Lexicons is available before running codegen. 272 293 273 294 ### Inline Models 274 295
+66 -3
packages/playground/samples/build.js
··· 1 1 // @ts-check 2 - import { writeFileSync, mkdirSync } from "fs"; 2 + import { writeFileSync, mkdirSync, readFileSync } from "fs"; 3 3 import { dirname, resolve, join } from "path"; 4 4 import { fileURLToPath } from "url"; 5 + import { deepStrictEqual } from "assert"; 5 6 import { lexicons, bundleLexicon } from "./index.js"; 7 + import { createTestHost, findTestPackageRoot, resolveVirtualPath } from "@typespec/compiler/testing"; 6 8 7 9 const __dirname = dirname(fileURLToPath(import.meta.url)); 8 10 const outputDir = resolve(__dirname, "dist"); 11 + const pkgRoot = await findTestPackageRoot(import.meta.url); 12 + 13 + // TypeSpec library setup for testing 14 + const TypelexTestLibrary = { 15 + name: "@typelex/emitter", 16 + packageRoot: pkgRoot.replace("/playground", "/emitter"), 17 + files: [ 18 + { realDir: "", pattern: "package.json", virtualPath: "./node_modules/@typelex/emitter" }, 19 + { realDir: "dist", pattern: "**/*.js", virtualPath: "./node_modules/@typelex/emitter/dist" }, 20 + { realDir: "lib/", pattern: "*.tsp", virtualPath: "./node_modules/@typelex/emitter/lib" }, 21 + ], 22 + }; 9 23 10 24 // Create output directory 11 25 mkdirSync(outputDir, { recursive: true }); 12 26 13 - // Write each bundled lexicon to disk 27 + // Write each bundled lexicon to disk and verify it compiles correctly 14 28 const samplesList = {}; 15 29 16 30 for (const [namespace, lexicon] of lexicons) { ··· 20 34 21 35 writeFileSync(filepath, bundled); 22 36 37 + const host = await createTestHost({ libraries: [TypelexTestLibrary] }); 38 + host.addTypeSpecFile("main.tsp", bundled); 39 + 40 + const baseOutputPath = resolveVirtualPath("test-output/"); 41 + const [, diagnostics] = await host.compileAndDiagnose("main.tsp", { 42 + outputDir: baseOutputPath, 43 + noEmit: false, 44 + emit: ["@typelex/emitter"], 45 + }); 46 + 47 + if (diagnostics.length > 0) { 48 + console.error(`❌ ${namespace}: Compilation errors`); 49 + diagnostics.forEach(d => console.error(` ${d.message}`)); 50 + process.exit(1); 51 + } 52 + 53 + // Get emitted JSON 54 + const outputFiles = [...host.fs.entries()] 55 + .filter(([name]) => name.startsWith(baseOutputPath)) 56 + .map(([name, value]) => { 57 + let relativePath = name.replace(baseOutputPath, ""); 58 + if (relativePath.startsWith("@typelex/emitter/")) { 59 + relativePath = relativePath.replace("@typelex/emitter/", ""); 60 + } 61 + return [relativePath, value]; 62 + }); 63 + 64 + const expectedJsonPath = namespace.replace(/\./g, "/") + ".json"; 65 + const emittedJson = outputFiles.find(([path]) => path === expectedJsonPath); 66 + 67 + if (!emittedJson) { 68 + console.error(`❌ ${namespace}: No JSON output found (expected ${expectedJsonPath})`); 69 + process.exit(1); 70 + } 71 + 72 + // Compare with expected JSON 73 + const expectedJsonFile = join( 74 + pkgRoot.replace("/playground", "/emitter"), 75 + "test/integration", 76 + lexicon.suite, 77 + "output", 78 + lexicon.file.replace(".tsp", ".json") 79 + ); 80 + 81 + const expectedJson = JSON.parse(readFileSync(expectedJsonFile, "utf-8")); 82 + const actualJson = JSON.parse(emittedJson[1]); 83 + 84 + deepStrictEqual(actualJson, expectedJson); 85 + 23 86 samplesList[namespace] = { 24 87 filename: `samples/dist/${filename}`, 25 88 preferredEmitter: "@typelex/emitter", ··· 30 93 const samplesIndex = `export default ${JSON.stringify(samplesList, null, 2)};`; 31 94 writeFileSync(join(outputDir, "samples.js"), samplesIndex); 32 95 33 - console.log(`Wrote ${Object.keys(samplesList).length} bundled samples to disk`); 96 + console.log(`\n✅ ${lexicons.size} samples verified successfully`);
+151 -91
packages/playground/samples/index.js
··· 5 5 6 6 const __dirname = dirname(fileURLToPath(import.meta.url)); 7 7 8 - // Get all tsp files 9 - function getAllTspFiles(dir, baseDir = dir) { 8 + // Get all tsp and json files 9 + function getAllFiles(dir, baseDir = dir) { 10 10 const files = []; 11 11 const entries = readdirSync(dir); 12 12 ··· 15 15 const stat = statSync(fullPath); 16 16 17 17 if (stat.isDirectory()) { 18 - files.push(...getAllTspFiles(fullPath, baseDir)); 19 - } else if (entry.endsWith(".tsp")) { 18 + files.push(...getAllFiles(fullPath, baseDir)); 19 + } else if (entry.endsWith(".tsp") || entry.endsWith(".json")) { 20 20 files.push(relative(baseDir, fullPath)); 21 21 } 22 22 } ··· 24 24 return files.sort(); 25 25 } 26 26 27 - // Extract dependencies from a file 28 - function extractDependencies(content) { 29 - const deps = new Set(); 30 - // Match namespace references like "com.atproto.label.defs.Label" or "com.atproto.repo.strongRef.Main" 31 - // Pattern: word.word.word... followed by dot and identifier starting with capital letter 32 - const pattern = 33 - /\b([a-z]+(?:\.[a-z]+)+(?:\.[a-z][a-zA-Z]*)*)\.[A-Z][a-zA-Z]*/g; 34 - const withoutDeclaration = content.replace(/namespace\s+[a-z.]+\s*\{/, ""); 35 - 36 - const matches = withoutDeclaration.matchAll(pattern); 37 - for (const match of matches) { 38 - deps.add(match[1]); 27 + // Extract all refs from JSON (recursively search for strings with #) 28 + function extractRefsFromJson(obj, refs = new Map()) { 29 + if (typeof obj === "string") { 30 + // Match pattern like "foo.bar#baz" or "foo.barCamel#baz" (must have # to be a ref) 31 + const match = obj.match(/^([a-z][a-zA-Z.]+)#([a-z][a-zA-Z]*)$/); 32 + if (match) { 33 + const ns = match[1]; 34 + const def = match[2]; 35 + const modelName = def.charAt(0).toUpperCase() + def.slice(1); 36 + if (!refs.has(ns)) { 37 + refs.set(ns, new Set()); 38 + } 39 + refs.get(ns).add(modelName); 40 + } else { 41 + // Also match plain namespace refs like "foo.bar.baz" or "foo.bar.bazCamel" (must have at least 2 dots) 42 + const nsMatch = obj.match(/^([a-z][a-zA-Z]*(?:\.[a-z][a-zA-Z]*){2,})$/); 43 + if (nsMatch) { 44 + const ns = nsMatch[1]; 45 + if (!refs.has(ns)) { 46 + refs.set(ns, new Set()); 47 + } 48 + refs.get(ns).add("Main"); 49 + } 50 + } 51 + } else if (Array.isArray(obj)) { 52 + for (const item of obj) { 53 + extractRefsFromJson(item, refs); 54 + } 55 + } else if (obj && typeof obj === "object") { 56 + for (const value of Object.values(obj)) { 57 + extractRefsFromJson(value, refs); 58 + } 39 59 } 40 - 41 - return Array.from(deps); 60 + return refs; 42 61 } 43 62 44 - const atprotoInputDir = join( 45 - __dirname, 46 - "../../emitter/test/integration/atproto/input", 47 - ); 48 - const lexiconExamplesDir = join( 49 - __dirname, 50 - "../../emitter/test/integration/lexicon-examples/input", 51 - ); 52 - 53 - const atprotoFiles = getAllTspFiles(atprotoInputDir); 54 - const lexiconExampleFiles = getAllTspFiles(lexiconExamplesDir); 55 - 56 - // Build dependency graph 57 - const lexicons = new Map(); // namespace -> { file, content, deps } 58 - 59 - // Process atproto files 60 - for (const file of atprotoFiles) { 61 - const fullPath = join(atprotoInputDir, file); 62 - const content = readFileSync(fullPath, "utf-8"); 63 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 64 - const deps = extractDependencies(content); 65 - 66 - lexicons.set(namespace, { file: `atproto/${file}`, content, deps }); 67 - } 63 + const integrationDir = join(__dirname, "../../emitter/test/integration"); 68 64 69 - // Process lexicon-examples files 70 - for (const file of lexiconExampleFiles) { 71 - const fullPath = join(lexiconExamplesDir, file); 72 - const content = readFileSync(fullPath, "utf-8"); 73 - const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 74 - const deps = extractDependencies(content); 65 + // Get all test suite directories 66 + const testSuites = readdirSync(integrationDir).filter((name) => { 67 + const fullPath = join(integrationDir, name); 68 + return statSync(fullPath).isDirectory() && !name.startsWith("."); 69 + }); 75 70 76 - lexicons.set(namespace, { file: `examples/${file}`, content, deps }); 77 - } 71 + // Build lexicons with refs extracted from JSON 72 + const lexicons = new Map(); // namespace -> { file, content, refs, suite } 78 73 79 - // Recursively collect all dependencies (topological sort) 80 - function collectDependencies( 81 - namespace, 82 - collected = new Set(), 83 - visiting = new Set(), 84 - ) { 85 - if (collected.has(namespace)) return; 86 - if (visiting.has(namespace)) return; // circular dependency 74 + // Process all test suites 75 + for (const suite of testSuites) { 76 + const inputDir = join(integrationDir, suite, "input"); 77 + const outputDir = join(integrationDir, suite, "output"); 87 78 88 - const lexicon = lexicons.get(namespace); 89 - if (!lexicon) return; 79 + const inputFiles = getAllFiles(inputDir).filter((f) => f.endsWith(".tsp")); 80 + 81 + for (const file of inputFiles) { 82 + const fullPath = join(inputDir, file); 83 + const content = readFileSync(fullPath, "utf-8"); 84 + const namespace = file.replace(/\.tsp$/, "").replace(/\//g, "."); 90 85 91 - visiting.add(namespace); 86 + // Find corresponding JSON output 87 + const jsonFile = file.replace(/\.tsp$/, ".json"); 88 + const jsonPath = join(outputDir, jsonFile); 89 + const jsonContent = readFileSync(jsonPath, "utf-8"); 90 + const jsonData = JSON.parse(jsonContent); 91 + const refs = extractRefsFromJson(jsonData); 92 92 93 - // First collect all dependencies 94 - for (const dep of lexicon.deps) { 95 - collectDependencies(dep, collected, visiting); 93 + lexicons.set(namespace, { file, content, refs, suite }); 96 94 } 95 + } 97 96 98 - visiting.delete(namespace); 99 - collected.add(namespace); 97 + // TypeSpec reserved keywords that need escaping 98 + const TYPESPEC_KEYWORDS = new Set([ 99 + "record", 100 + "pub", 101 + "interface", 102 + "model", 103 + "namespace", 104 + "op", 105 + "import", 106 + "export", 107 + "using", 108 + "alias", 109 + "enum", 110 + "union", 111 + "scalar", 112 + "extends", 113 + ]); 114 + 115 + // Escape a namespace part if it's a reserved keyword 116 + function escapeNamespacePart(part) { 117 + return TYPESPEC_KEYWORDS.has(part) ? `\`${part}\`` : part; 100 118 } 101 119 102 - // Bundle a lexicon with all its dependencies 103 - function bundleLexicon(namespace) { 104 - const collected = new Set(); 105 - collectDependencies(namespace, collected); 120 + // Escape a full namespace path 121 + function escapeNamespace(namespace) { 122 + return namespace.split(".").map(escapeNamespacePart).join("."); 123 + } 106 124 107 - // Put the main lexicon FIRST, then its dependencies 108 - const mainLexicon = lexicons.get(namespace); 109 - const deps = Array.from(collected).filter((ns) => ns !== namespace); 125 + // Get the JSON for a lexicon to check its definitions 126 + function getLexiconJson(namespace) { 127 + const lexicon = lexicons.get(namespace); 128 + if (!lexicon) return null; 129 + 130 + const jsonPath = join( 131 + integrationDir, 132 + lexicon.suite, 133 + "output", 134 + lexicon.file.replace(".tsp", ".json"), 135 + ); 136 + 137 + try { 138 + return JSON.parse(readFileSync(jsonPath, "utf-8")); 139 + } catch { 140 + return null; 141 + } 142 + } 110 143 111 - let bundled = 'import "@typelex/emitter";\n\n'; 144 + // Check if a definition in JSON is a token 145 + function isToken(lexiconJson, defName) { 146 + if (!lexiconJson || !lexiconJson.defs) return false; 147 + const def = lexiconJson.defs[defName]; 148 + return def && def.type === "token"; 149 + } 112 150 113 - // Main lexicon first (so it shows in the playground) 114 - if (mainLexicon) { 115 - const contentWithoutImport = mainLexicon.content.replace( 116 - /^import "@typelex\/emitter";\s*\n/, 117 - "", 118 - ); 119 - bundled += `// ${mainLexicon.file}\n${contentWithoutImport}\n`; 120 - } 151 + // Bundle a lexicon with stubs for referenced types (from JSON) 152 + function bundleLexicon(namespace) { 153 + const mainLexicon = lexicons.get(namespace); 154 + if (!mainLexicon) return ""; 155 + 156 + let bundled = mainLexicon.content; 157 + 158 + // Add stubs from refs extracted from JSON output (excluding self-references) 159 + if (mainLexicon.refs.size > 0) { 160 + let hasExternalRefs = false; 161 + for (const [ns] of mainLexicon.refs) { 162 + if (ns !== namespace) { 163 + hasExternalRefs = true; 164 + break; 165 + } 166 + } 121 167 122 - // Then dependencies 123 - for (const ns of deps) { 124 - const lexicon = lexicons.get(ns); 125 - if (!lexicon) continue; 168 + if (hasExternalRefs) { 169 + bundled += "\n// --- Externals ---\n"; 170 + } 126 171 127 - const contentWithoutImport = lexicon.content.replace( 128 - /^import "@typelex\/emitter";\s*\n/, 129 - "", 130 - ); 131 - bundled += `// ${lexicon.file}\n${contentWithoutImport}\n`; 172 + for (const [ns, models] of mainLexicon.refs) { 173 + // Skip if this is the current namespace 174 + if (ns === namespace) continue; 175 + 176 + // Get the JSON for this referenced namespace to check for tokens 177 + const refJson = getLexiconJson(ns); 178 + 179 + const escapedNs = escapeNamespace(ns); 180 + bundled += `\n@external\nnamespace ${escapedNs} {\n`; 181 + for (const model of models) { 182 + // Check if this definition exists in the JSON and is a token 183 + const defName = model.charAt(0).toLowerCase() + model.slice(1); 184 + if (refJson && isToken(refJson, defName)) { 185 + bundled += ` @token model ${model} { }\n`; 186 + } else { 187 + bundled += ` model ${model} { }\n`; 188 + } 189 + } 190 + bundled += `}\n`; 191 + } 132 192 } 133 193 134 194 return bundled;

History

2 rounds 0 comments
sign up or login to add to the discussion
5 commits
expand
add @external decorator
add test for @external
document @external
replacing bundling with externals in playground
enforce empty @externals
1/1 failed
expand
expand 0 comments
closed without merging
5 commits
expand
add @external decorator
add test for @external
document @external
replacing bundling with externals in playground
enforce empty @externals
1/1 success
expand
expand 0 comments