+96 -35

bin/main.ml

··· 318 318 let doc = "Path to vendor cache (overrides config and UNPAC_VENDOR_CACHE env var)." in 319 319 Arg.(value & opt (some string) None & info ["cache"] ~docv:"PATH" ~doc) 320 320 in 321 321 - let run () pkg name_opt version_opt branch_opt solve cli_cache = 321 321 + let fast_arg = 322 322 + let doc = "Fast mode: skip history rewriting (useful for large repos like dune)." in 323 323 + Arg.(value & flag & info ["fast"] ~doc) 324 324 + in 325 325 + let run () pkg name_opt version_opt branch_opt solve cli_cache fast = 326 326 + let preserve_history = not fast in 322 327 with_root @@ fun ~env:_ ~fs ~proc_mgr ~root -> 323 328 let config = load_config root in 324 329 let cache = resolve_cache ~proc_mgr ~fs ~config ~cli_cache in ··· 383 388 url; 384 389 branch = None; 385 390 } in 386 386 - match Unpac_opam.Opam.add_package ~proc_mgr ~root ?cache info with 391 391 + match Unpac_opam.Opam.add_package ~preserve_history ~proc_mgr ~root ?cache info with 387 392 | Unpac.Backend.Added { name = pkg_name; sha } -> 388 393 Format.printf "Added %s (%s)@." pkg_name (String.sub sha 0 7); 389 394 if List.length g.packages > 1 then ··· 442 447 url; 443 448 branch = branch_opt; 444 449 } in 445 445 - match Unpac_opam.Opam.add_package ~proc_mgr ~root ?cache info with 450 450 + match Unpac_opam.Opam.add_package ~preserve_history ~proc_mgr ~root ?cache info with 446 451 | Unpac.Backend.Added { name = pkg_name; sha } -> 447 452 Format.printf "Added %s (%s)@." pkg_name (String.sub sha 0 7); 448 453 Format.printf "@.Next steps:@."; ··· 456 461 end 457 462 in 458 463 let info = Cmd.info "add" ~doc in 459 459 - Cmd.v info Term.(const run $ logging_term $ pkg_arg $ name_arg $ version_arg $ branch_arg $ solve_arg $ cache_arg) 464 464 + Cmd.v info Term.(const run $ logging_term $ pkg_arg $ name_arg $ version_arg $ branch_arg $ solve_arg $ cache_arg $ fast_arg) 460 465 461 466 (* Opam list command *) 462 467 let opam_list_cmd = ··· 475 480 476 481 (* Opam edit command *) 477 482 let opam_edit_cmd = 478 478 - let doc = "Open a package's patches worktree for editing." in 483 483 + let doc = "Open a package's patches worktree for editing. \ 484 484 + Also creates a vendor worktree for reference." in 479 485 let pkg_arg = 480 486 let doc = "Package name to edit." in 481 487 Arg.(required & pos 0 (some string) None & info [] ~docv:"PACKAGE" ~doc) ··· 488 494 Format.eprintf "Package '%s' is not vendored@." pkg; 489 495 exit 1 490 496 end; 491 491 - (* Ensure patches worktree exists *) 497 497 + (* Ensure both patches and vendor worktrees exist *) 492 498 Unpac.Worktree.ensure ~proc_mgr root (Unpac.Worktree.Opam_patches pkg); 493 493 - let wt_path = Unpac.Worktree.path root (Unpac.Worktree.Opam_patches pkg) in 494 494 - let path_str = snd wt_path in 499 499 + Unpac.Worktree.ensure ~proc_mgr root (Unpac.Worktree.Opam_vendor pkg); 500 500 + let patches_path = snd (Unpac.Worktree.path root (Unpac.Worktree.Opam_patches pkg)) in 501 501 + let vendor_path = snd (Unpac.Worktree.path root (Unpac.Worktree.Opam_vendor pkg)) in 495 502 Format.printf "Editing %s@." pkg; 496 496 - Format.printf "Worktree: %s@." path_str; 497 503 Format.printf "@."; 498 498 - Format.printf "Make your changes, then:@."; 499 499 - Format.printf " cd %s@." path_str; 504 504 + Format.printf "Worktrees created:@."; 505 505 + Format.printf " patches: %s (make changes here)@." patches_path; 506 506 + Format.printf " vendor: %s (original for reference)@." vendor_path; 507 507 + Format.printf "@."; 508 508 + Format.printf "Make your changes in the patches worktree, then:@."; 509 509 + Format.printf " cd %s@." patches_path; 500 510 Format.printf " git add -A && git commit -m 'your message'@."; 501 511 Format.printf "@."; 502 512 Format.printf "When done: unpac opam done %s@." pkg ··· 506 516 507 517 (* Opam done command *) 508 518 let opam_done_cmd = 509 509 - let doc = "Close a package's patches worktree after editing." in 519 519 + let doc = "Close a package's patches and vendor worktrees after editing." in 510 520 let pkg_arg = 511 521 let doc = "Package name." in 512 522 Arg.(required & pos 0 (some string) None & info [] ~docv:"PACKAGE" ~doc) 513 523 in 514 524 let run () pkg = 515 525 with_root @@ fun ~env:_ ~fs:_ ~proc_mgr ~root -> 516 516 - let kind = Unpac.Worktree.Opam_patches pkg in 517 517 - if not (Unpac.Worktree.exists root kind) then begin 526 526 + let patches_kind = Unpac.Worktree.Opam_patches pkg in 527 527 + let vendor_kind = Unpac.Worktree.Opam_vendor pkg in 528 528 + if not (Unpac.Worktree.exists root patches_kind) then begin 518 529 Format.eprintf "No editing session for '%s'@." pkg; 519 530 exit 1 520 531 end; 521 521 - (* Check for uncommitted changes *) 522 522 - let wt_path = Unpac.Worktree.path root kind in 532 532 + (* Check for uncommitted changes in patches worktree *) 533 533 + let wt_path = Unpac.Worktree.path root patches_kind in 523 534 let status = Unpac.Git.run_exn ~proc_mgr ~cwd:wt_path ["status"; "--porcelain"] in 524 535 if String.trim status <> "" then begin 525 536 Format.eprintf "Warning: uncommitted changes in %s@." pkg; 526 537 Format.eprintf "Commit or discard them before closing.@."; 527 538 exit 1 528 539 end; 529 529 - (* Remove worktree *) 530 530 - Unpac.Worktree.remove ~proc_mgr root kind; 540 540 + (* Remove both worktrees *) 541 541 + Unpac.Worktree.remove ~proc_mgr root patches_kind; 542 542 + if Unpac.Worktree.exists root vendor_kind then 543 543 + Unpac.Worktree.remove ~proc_mgr root vendor_kind; 531 544 Format.printf "Closed editing session for %s@." pkg; 532 545 Format.printf "@.Next steps:@."; 533 546 Format.printf " unpac opam diff %s # view your changes@." pkg; ··· 563 576 564 577 (* Opam merge command *) 565 578 let opam_merge_cmd = 566 566 - let doc = "Merge a vendored opam package into a project." in 567 567 - let pkg_arg = 568 568 - let doc = "Package name to merge." in 569 569 - Arg.(required & pos 0 (some string) None & info [] ~docv:"PACKAGE" ~doc) 579 579 + let doc = "Merge vendored opam packages into a project. \ 580 580 + Use --all to merge all vendored packages." in 581 581 + let args = 582 582 + let doc = "With --all: PROJECT. Without --all: PACKAGE PROJECT." in 583 583 + Arg.(value & pos_all string [] & info [] ~docv:"ARGS" ~doc) 570 584 in 571 571 - let project_arg = 572 572 - let doc = "Target project name." in 573 573 - Arg.(required & pos 1 (some string) None & info [] ~docv:"PROJECT" ~doc) 585 585 + let all_flag = 586 586 + let doc = "Merge all vendored packages into the project." in 587 587 + Arg.(value & flag & info ["all"] ~doc) 574 588 in 575 575 - let run () pkg project = 589 589 + let run () args all = 576 590 with_root @@ fun ~env:_ ~fs:_ ~proc_mgr ~root -> 577 577 - let patches_branch = Unpac_opam.Opam.patches_branch pkg in 578 578 - match Unpac.Backend.merge_to_project ~proc_mgr ~root ~project ~patches_branch with 579 579 - | Ok () -> 580 580 - Format.printf "Merged %s into project %s@." pkg project; 581 581 - Format.printf "@.Next: Build your project in project/%s@." project 582 582 - | Error (`Conflict files) -> 583 583 - Format.eprintf "Merge conflict in %s:@." pkg; 584 584 - List.iter (Format.eprintf " %s@.") files; 591 591 + (* Parse arguments based on --all flag *) 592 592 + let pkg, project = if all then 593 593 + match args with 594 594 + | [project] -> None, project 595 595 + | _ -> 596 596 + Format.eprintf "Usage: unpac opam merge --all PROJECT@."; 597 597 + exit 1 598 598 + else 599 599 + match args with 600 600 + | [pkg; project] -> Some pkg, project 601 601 + | _ -> 602 602 + Format.eprintf "Usage: unpac opam merge PACKAGE PROJECT@."; 603 603 + exit 1 604 604 + in 605 605 + let merge_one pkg = 606 606 + let patches_branch = Unpac_opam.Opam.patches_branch pkg in 607 607 + match Unpac.Backend.merge_to_project ~proc_mgr ~root ~project ~patches_branch with 608 608 + | Ok () -> 609 609 + Format.printf "Merged %s into project %s@." pkg project; 610 610 + true 611 611 + | Error (`Conflict files) -> 612 612 + Format.eprintf "Merge conflict in %s:@." pkg; 613 613 + List.iter (Format.eprintf " %s@.") files; 614 614 + false 615 615 + in 616 616 + if all then begin 617 617 + (* Merge all vendored packages *) 618 618 + let packages = Unpac_opam.Opam.list_packages ~proc_mgr ~root in 619 619 + if packages = [] then begin 620 620 + Format.eprintf "No vendored packages to merge.@."; 621 621 + exit 1 622 622 + end; 623 623 + Format.printf "Merging %d packages into project %s...@." (List.length packages) project; 624 624 + let (successes, failures) = List.fold_left (fun (s, f) pkg -> 625 625 + if merge_one pkg then (s + 1, f) else (s, f + 1) 626 626 + ) (0, 0) packages in 627 627 + Format.printf "@.Done: %d merged" successes; 628 628 + if failures > 0 then Format.printf ", %d had conflicts" failures; 629 629 + Format.printf "@."; 630 630 + if failures > 0 then begin 585 631 Format.eprintf "Resolve conflicts in project/%s and commit.@." project; 586 632 exit 1 633 633 + end else 634 634 + Format.printf "Next: Build your project in project/%s@." project 635 635 + end else begin 636 636 + match pkg with 637 637 + | Some pkg -> 638 638 + if merge_one pkg then 639 639 + Format.printf "@.Next: Build your project in project/%s@." project 640 640 + else begin 641 641 + Format.eprintf "Resolve conflicts in project/%s and commit.@." project; 642 642 + exit 1 643 643 + end 644 644 + | None -> 645 645 + Format.eprintf "Error: Either provide a package name or use --all@."; 646 646 + exit 1 647 647 + end 587 648 in 588 649 let info = Cmd.info "merge" ~doc in 589 589 - Cmd.v info Term.(const run $ logging_term $ pkg_arg $ project_arg) 650 650 + Cmd.v info Term.(const run $ logging_term $ args $ all_flag) 590 651 591 652 (* Opam info command *) 592 653 let opam_info_cmd =

+61

lib/git.ml

··· 395 395 let clean_fd ~proc_mgr ~cwd = 396 396 Log.debug (fun m -> m "Cleaning untracked files"); 397 397 run_exn ~proc_mgr ~cwd ["clean"; "-fd"] |> ignore 398 398 + 399 399 + let filter_branch_to_subdirectory ~proc_mgr ~cwd ~branch ~subdirectory = 400 400 + Log.info (fun m -> m "Rewriting history of %s into subdirectory %s..." branch subdirectory); 401 401 + (* Use filter-branch with index-filter to rewrite all paths into subdirectory. 402 402 + This preserves full history with paths prefixed. 403 403 + 404 404 + For bare repositories, we need to create a temporary worktree, run filter-branch 405 405 + there, and then update the branch in the bare repo. *) 406 406 + 407 407 + (* Create a unique temporary worktree name using the branch name *) 408 408 + let safe_branch = String.map (fun c -> if c = '/' then '-' else c) branch in 409 409 + let temp_wt_name = ".filter-tmp-" ^ safe_branch in 410 410 + let temp_wt_relpath = "../" ^ temp_wt_name in 411 411 + 412 412 + (* Construct the worktree path - cwd is (fs, path_string), so we go up one level *) 413 413 + let fs = fst cwd in 414 414 + let git_path = snd cwd in 415 415 + let parent_path = Filename.dirname git_path in 416 416 + let temp_wt_path = Filename.concat parent_path temp_wt_name in 417 417 + let temp_wt : path = (fs, temp_wt_path) in 418 418 + 419 419 + (* Remove any existing temp worktree *) 420 420 + ignore (run ~proc_mgr ~cwd ["worktree"; "remove"; "-f"; temp_wt_relpath]); 421 421 + 422 422 + (* Create worktree for the branch *) 423 423 + run_exn ~proc_mgr ~cwd ["worktree"; "add"; temp_wt_relpath; branch] |> ignore; 424 424 + 425 425 + (* Run filter-branch in the worktree *) 426 426 + let script = Printf.sprintf 427 427 + {|git ls-files -s | sed "s,\t,&%s/," | GIT_INDEX_FILE=$GIT_INDEX_FILE.new git update-index --index-info && mv "$GIT_INDEX_FILE.new" "$GIT_INDEX_FILE"|} 428 428 + subdirectory 429 429 + in 430 430 + 431 431 + (* Set environment to suppress the warning *) 432 432 + let old_env = try Some (Unix.getenv "FILTER_BRANCH_SQUELCH_WARNING") with Not_found -> None in 433 433 + Unix.putenv "FILTER_BRANCH_SQUELCH_WARNING" "1"; 434 434 + 435 435 + let result = run ~proc_mgr ~cwd:temp_wt [ 436 436 + "filter-branch"; "-f"; 437 437 + "--index-filter"; script; 438 438 + "--"; "HEAD" 439 439 + ] in 440 440 + 441 441 + (* Restore environment *) 442 442 + (match old_env with 443 443 + | Some v -> Unix.putenv "FILTER_BRANCH_SQUELCH_WARNING" v 444 444 + | None -> (try Unix.putenv "FILTER_BRANCH_SQUELCH_WARNING" "" with _ -> ())); 445 445 + 446 446 + (* Handle result: get the new SHA, cleanup worktree, then update branch *) 447 447 + (match result with 448 448 + | Ok _ -> 449 449 + (* Get the new HEAD SHA from the worktree BEFORE removing it *) 450 450 + let new_sha = run_exn ~proc_mgr ~cwd:temp_wt ["rev-parse"; "HEAD"] |> string_trim in 451 451 + (* Cleanup temporary worktree first (must do this before updating branch) *) 452 452 + ignore (run ~proc_mgr ~cwd ["worktree"; "remove"; "-f"; temp_wt_relpath]); 453 453 + (* Now update the branch in the bare repo *) 454 454 + run_exn ~proc_mgr ~cwd ["branch"; "-f"; branch; new_sha] |> ignore 455 455 + | Error e -> 456 456 + (* Cleanup and re-raise *) 457 457 + ignore (run ~proc_mgr ~cwd ["worktree"; "remove"; "-f"; temp_wt_relpath]); 458 458 + raise (err e))

+10

lib/git.mli

··· 344 344 cwd:path -> 345 345 unit 346 346 (** [clean_fd] removes untracked files and directories. *) 347 347 + 348 348 + val filter_branch_to_subdirectory : 349 349 + proc_mgr:proc_mgr -> 350 350 + cwd:path -> 351 351 + branch:string -> 352 352 + subdirectory:string -> 353 353 + unit 354 354 + (** [filter_branch_to_subdirectory ~proc_mgr ~cwd ~branch ~subdirectory] 355 355 + rewrites the history of [branch] so all files are moved into [subdirectory]. 356 356 + This preserves full commit history with paths prefixed. *)

+16 -1

lib/init.ml

··· 20 20 let project_dune = {|(vendored_dirs vendor) 21 21 |} 22 22 23 23 + let project_gitignore = {|_build/ 24 24 + *.install 25 25 + |} 26 26 + 27 27 + let vendor_dune = {|(vendored_dirs opam) 28 28 + |} 29 29 + 23 30 (** Initialize a new unpac project at the given path. *) 24 31 let init ~proc_mgr ~fs path = 25 32 (* Convert relative paths to absolute *) ··· 110 117 Eio.Path.(project_path / "dune") 111 118 project_dune; 112 119 113 113 - (* Create empty vendor directory structure *) 120 120 + Eio.Path.save ~create:(`Or_truncate 0o644) 121 121 + Eio.Path.(project_path / ".gitignore") 122 122 + project_gitignore; 123 123 + 124 124 + (* Create vendor directory structure with dune file *) 114 125 Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 115 126 Eio.Path.(project_path / "vendor" / "opam"); 127 127 + 128 128 + Eio.Path.save ~create:(`Or_truncate 0o644) 129 129 + Eio.Path.(project_path / "vendor" / "dune") 130 130 + vendor_dune; 116 131 117 132 (* Commit template *) 118 133 Git.run_exn ~proc_mgr ~cwd:project_path ["add"; "-A"] |> ignore;

+46 -26

lib/opam/opam.ml

··· 2 2 3 3 Implements vendoring of opam packages using the three-tier branch model: 4 4 - opam/upstream/<pkg> - pristine upstream code 5 5 - - opam/vendor/<pkg> - orphan branch with vendor/opam/<pkg>/ prefix 6 6 - - opam/patches/<pkg> - local modifications *) 5 5 + - opam/vendor/<pkg> - upstream history rewritten with vendor/opam/<pkg>/ prefix 6 6 + - opam/patches/<pkg> - local modifications 7 7 + 8 8 + The vendor branch preserves full git history from upstream, with all paths 9 9 + rewritten to be under vendor/opam/<pkg>/. This allows git blame/log to work 10 10 + correctly on vendored files. *) 7 11 8 12 module Worktree = Unpac.Worktree 9 13 module Git = Unpac.Git ··· 62 66 end 63 67 ) 64 68 65 65 - let add_package ~proc_mgr ~root ?cache (info : Backend.package_info) = 69 69 + let add_package ?(preserve_history=true) ~proc_mgr ~root ?cache (info : Backend.package_info) = 66 70 let pkg = info.name in 67 71 let git = Worktree.git_dir root in 68 72 ··· 71 75 if Worktree.branch_exists ~proc_mgr root (patches_kind pkg) then 72 76 Backend.Already_exists pkg 73 77 else begin 74 74 - (* Step 1: Create upstream branch and fetch *) 75 75 - let upstream_wt = Worktree.path root (upstream_kind pkg) in 76 76 - 77 78 (* Rewrite URL for known mirrors *) 78 79 let url = Git_repo_lookup.rewrite_url info.url in 79 80 ··· 100 101 remote ^ "/" ^ branch 101 102 in 102 103 103 103 - (* Create upstream branch *) 104 104 + (* Step 1: Create upstream branch from fetched ref *) 104 105 Git.branch_force ~proc_mgr ~cwd:git 105 106 ~name:(upstream_branch pkg) ~point:ref_point; 106 107 107 107 - (* Create upstream worktree temporarily *) 108 108 - Worktree.ensure ~proc_mgr root (upstream_kind pkg); 108 108 + let vendor_sha = 109 109 + if preserve_history then begin 110 110 + (* Step 2a: Create vendor branch from upstream (preserves history) *) 111 111 + Git.branch_force ~proc_mgr ~cwd:git 112 112 + ~name:(vendor_branch pkg) ~point:(upstream_branch pkg); 109 113 110 110 - (* Step 2: Create vendor branch (orphan) with prefix *) 111 111 - Worktree.ensure_orphan ~proc_mgr root (vendor_kind pkg); 112 112 - let vendor_wt = Worktree.path root (vendor_kind pkg) in 114 114 + (* Rewrite vendor branch history to move all files into vendor/opam/<pkg>/ *) 115 115 + Git.filter_branch_to_subdirectory ~proc_mgr ~cwd:git 116 116 + ~branch:(vendor_branch pkg) 117 117 + ~subdirectory:(vendor_path pkg); 113 118 114 114 - (* Copy files with vendor/opam/<pkg>/ prefix *) 115 115 - copy_with_prefix 116 116 - ~src_dir:upstream_wt 117 117 - ~dst_dir:vendor_wt 118 118 - ~prefix:(vendor_path pkg); 119 119 + (* Get the vendor SHA after rewriting *) 120 120 + match Git.rev_parse ~proc_mgr ~cwd:git (vendor_branch pkg) with 121 121 + | Some sha -> sha 122 122 + | None -> failwith "Vendor branch not found after filter-branch" 123 123 + end else begin 124 124 + (* Step 2b: Fast mode - create orphan vendor branch without history *) 125 125 + Worktree.ensure ~proc_mgr root (upstream_kind pkg); 126 126 + let upstream_wt = Worktree.path root (upstream_kind pkg) in 119 127 120 120 - (* Commit vendor branch *) 121 121 - Git.add_all ~proc_mgr ~cwd:vendor_wt; 122 122 - Git.commit ~proc_mgr ~cwd:vendor_wt 123 123 - ~message:(Printf.sprintf "Vendor %s" pkg); 128 128 + Worktree.ensure_orphan ~proc_mgr root (vendor_kind pkg); 129 129 + let vendor_wt = Worktree.path root (vendor_kind pkg) in 124 130 125 125 - let vendor_sha = Git.current_head ~proc_mgr ~cwd:vendor_wt in 131 131 + (* Copy files with vendor/opam/<pkg>/ prefix *) 132 132 + copy_with_prefix 133 133 + ~src_dir:upstream_wt 134 134 + ~dst_dir:vendor_wt 135 135 + ~prefix:(vendor_path pkg); 136 136 + 137 137 + (* Commit vendor branch *) 138 138 + Git.add_all ~proc_mgr ~cwd:vendor_wt; 139 139 + Git.commit ~proc_mgr ~cwd:vendor_wt 140 140 + ~message:(Printf.sprintf "Vendor %s" pkg); 141 141 + 142 142 + let sha = Git.current_head ~proc_mgr ~cwd:vendor_wt in 143 143 + 144 144 + (* Cleanup worktrees *) 145 145 + Worktree.remove ~proc_mgr root (upstream_kind pkg); 146 146 + Worktree.remove ~proc_mgr root (vendor_kind pkg); 147 147 + sha 148 148 + end 149 149 + in 126 150 127 151 (* Step 3: Create patches branch from vendor *) 128 152 Git.branch_create ~proc_mgr ~cwd:git 129 153 ~name:(patches_branch pkg) 130 154 ~start_point:(vendor_branch pkg); 131 131 - 132 132 - (* Cleanup worktrees *) 133 133 - Worktree.remove ~proc_mgr root (upstream_kind pkg); 134 134 - Worktree.remove ~proc_mgr root (vendor_kind pkg); 135 155 136 156 Backend.Added { name = pkg; sha = vendor_sha } 137 157 end

+5 -1

lib/opam/opam.mli

··· 31 31 (** {1 Package Operations} *) 32 32 33 33 val add_package : 34 34 + ?preserve_history:bool -> 34 35 proc_mgr:Unpac.Git.proc_mgr -> 35 36 root:Unpac.Worktree.root -> 36 37 ?cache:Unpac.Vendor_cache.t -> ··· 39 40 (** [add_package ~proc_mgr ~root ?cache info] vendors a single package. 40 41 41 42 1. Fetches upstream into opam/upstream/<pkg> (via cache if provided) 42 42 - 2. Creates opam/vendor/<pkg> orphan with vendor/opam/<pkg>/ prefix 43 43 + 2. Creates opam/vendor/<pkg> with vendor/opam/<pkg>/ prefix 43 44 3. Creates opam/patches/<pkg> from vendor 44 45 46 46 + @param preserve_history If true (default), rewrites git history to preserve 47 47 + full commit history in the vendor branch. Set to false for faster 48 48 + vendoring without history (useful for large repositories). 45 49 @param cache Optional vendor cache for shared fetches across projects. *) 46 50 47 51 val update_package :