Generate flake.nix from module options. Discussions: https://oeiuwq.zulipchat.com/join/nqp26cd4kngon6mo3ncgnuap/
dendrix.oeiuwq.com/Dendritic.html
dendritic
nix
inputs
npins (#71)
test npins
authored by
GitHub
c9d91bd7
8a118dae
+217
-124
+37
.github/workflows/flake-check.yaml
+37
.github/workflows/flake-check.yaml
···
39
39
if: ${{ matrix.bootstrap == 'unflake' }}
40
40
- run: cat bootstrap/npins/sources.json
41
41
if: ${{ matrix.bootstrap == 'npins' }}
42
+
- name: Assert bootstrap npins transitive discovery (flake-parts -> nixpkgs-lib)
43
+
if: ${{ matrix.bootstrap == 'npins' }}
44
+
run: |
45
+
jq -e '.pins | has("flake-parts")' bootstrap/npins/sources.json
46
+
jq -e '.pins | has("nixpkgs-lib")' bootstrap/npins/sources.json
42
47
template:
43
48
name: Check template ${{matrix.template}}
44
49
needs: [find-templates]
···
84
89
sed -i 's/# flake-file = import/flake-file = import/' default.nix
85
90
echo "{ inputs, ... }: { npins.pkgs = import inputs.nixpkgs {}; }" | tee modules/pkgs.nix
86
91
nix-shell . -A npins.env --run write-npins
92
+
npins-transitive:
93
+
name: Check npins transitive discovery
94
+
runs-on: ubuntu-latest
95
+
env:
96
+
NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz"
97
+
steps:
98
+
- uses: actions/checkout@v4
99
+
- uses: wimpysworld/nothing-but-nix@main
100
+
- uses: cachix/install-nix-action@v31
101
+
- uses: DeterminateSystems/magic-nix-cache-action@main
102
+
- name: Run write-npins with neomacs (has transitive deps)
103
+
run: |
104
+
mkdir test-out
105
+
nix-shell ./default.nix -A flake-file.sh \
106
+
--run write-npins \
107
+
--arg modules ./modules/npins-transitive-test.nix \
108
+
--argstr outdir test-out
109
+
- name: Assert declared inputs are pinned
110
+
run: |
111
+
jq -e '.pins | has("neomacs")' test-out/npins/sources.json
112
+
jq -e '.pins | has("nixpkgs")' test-out/npins/sources.json
113
+
- name: Assert transitive deps are auto-discovered
114
+
run: |
115
+
jq -e '.pins | has("crane")' test-out/npins/sources.json
116
+
jq -e '.pins | has("rust-overlay")' test-out/npins/sources.json
117
+
jq -e '.pins | has("nix-wpe-webkit")' test-out/npins/sources.json
118
+
- name: Assert dedup-by-name (nixpkgs stays as Channel, not re-pinned from neomacs dep)
119
+
run: |
120
+
jq -e '.pins.nixpkgs.type == "Channel"' test-out/npins/sources.json
121
+
- name: Show pinned sources
122
+
if: always()
123
+
run: cat test-out/npins/sources.json
87
124
unflake:
88
125
name: Check unflake
89
126
runs-on: ubuntu-latest
+71
dev/modules/unit-tests/npins.nix
+71
dev/modules/unit-tests/npins.nix
···
1
+
{ lib, ... }:
2
+
let
3
+
esc = lib.escapeShellArg;
4
+
5
+
# Mirrors pinnableInputs from modules/npins.nix
6
+
pinnableInputs = inputs: lib.filterAttrs (_: v: v.url or "" != "") inputs;
7
+
8
+
# Mirrors queueSeed from modules/npins.nix
9
+
queueSeed =
10
+
pinnable:
11
+
let
12
+
lines = lib.mapAttrsToList (
13
+
name: input: " printf '%s\\t%s\\n' ${esc name} ${esc (input.url or "")}"
14
+
) pinnable;
15
+
in
16
+
"{\n" + lib.concatStringsSep "\n" lines + "\n} >> \"$QUEUE_FILE\"";
17
+
18
+
tests.npins."pinnableInputs excludes empty-url entries" = {
19
+
expr = pinnableInputs {
20
+
foo.url = "github:owner/foo";
21
+
bar.url = "";
22
+
baz = { };
23
+
};
24
+
expected = {
25
+
foo.url = "github:owner/foo";
26
+
};
27
+
};
28
+
29
+
tests.npins."pinnableInputs keeps all non-empty urls" = {
30
+
expr = lib.attrNames (pinnableInputs {
31
+
a.url = "github:a/a";
32
+
b.url = "github:b/b";
33
+
c.url = "";
34
+
});
35
+
expected = [
36
+
"a"
37
+
"b"
38
+
];
39
+
};
40
+
41
+
tests.npins."pinnableInputs is empty on no-url inputs" = {
42
+
expr = pinnableInputs { foo.follows = "bar"; };
43
+
expected = { };
44
+
};
45
+
46
+
tests.npins."queueSeed contains name and url for each pinnable input" = {
47
+
expr = queueSeed { foo.url = "github:owner/foo"; };
48
+
expected = "{\n printf '%s\\t%s\\n' 'foo' 'github:owner/foo'\n} >> \"$QUEUE_FILE\"";
49
+
};
50
+
51
+
tests.npins."queueSeed wraps all printfs in one redirect block" = {
52
+
expr =
53
+
let
54
+
seed = queueSeed {
55
+
a.url = "github:a/a";
56
+
b.url = "github:b/b";
57
+
};
58
+
in
59
+
lib.hasPrefix "{" seed && lib.hasSuffix ">> \"$QUEUE_FILE\"" seed;
60
+
expected = true;
61
+
};
62
+
63
+
tests.npins."queueSeed is empty block on no pinnable inputs" = {
64
+
expr = queueSeed { };
65
+
expected = "{\n\n} >> \"$QUEUE_FILE\"";
66
+
};
67
+
68
+
in
69
+
{
70
+
flake = { inherit tests; };
71
+
}
+13
modules/npins-transitive-test.nix
+13
modules/npins-transitive-test.nix
···
1
+
{ lib, ... }:
2
+
{
3
+
# Declares neomacs with nixpkgs-follows so nixpkgs is not re-pinned separately.
4
+
# Transitive discovery must find crane, rust-overlay, nix-wpe-webkit
5
+
# from neomacs's own flake.nix at runtime.
6
+
flake-file.inputs = {
7
+
nixpkgs.url = lib.mkDefault "https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.xz";
8
+
neomacs = {
9
+
url = "github:eval-exec/neomacs";
10
+
inputs.nixpkgs.follows = "nixpkgs";
11
+
};
12
+
};
13
+
}
+96
-124
modules/npins.nix
+96
-124
modules/npins.nix
···
1
1
{ lib, config, ... }:
2
2
let
3
3
inherit (config) flake-file;
4
-
5
4
inherit (import ./../dev/modules/_lib lib) inputsExpr;
6
5
7
6
inputs = inputsExpr flake-file.inputs;
8
-
9
-
parseFlakeUrl =
10
-
url:
11
-
let
12
-
parts = lib.splitString ":" url;
13
-
scheme = lib.head parts;
14
-
rest = lib.concatStringsSep ":" (lib.tail parts);
15
-
in
16
-
if scheme == "github" then
17
-
parseGithub rest
18
-
else if scheme == "gitlab" then
19
-
parseGitlab rest
20
-
else if isChannelUrl url then
21
-
parseChannel url
22
-
else if lib.hasPrefix "https://" url || lib.hasPrefix "http://" url then
23
-
{
24
-
type = "tarball";
25
-
inherit url;
26
-
}
27
-
else
28
-
{
29
-
type = "git";
30
-
inherit url;
31
-
};
32
-
33
-
parseGithub =
34
-
rest:
35
-
let
36
-
segments = lib.splitString "/" rest;
37
-
owner = lib.elemAt segments 0;
38
-
repo = lib.elemAt segments 1;
39
-
ref = if lib.length segments > 2 then lib.elemAt segments 2 else null;
40
-
in
41
-
{
42
-
type = "github";
43
-
inherit owner repo ref;
44
-
};
45
-
46
-
parseGitlab =
47
-
rest:
48
-
let
49
-
segments = lib.splitString "/" rest;
50
-
owner = lib.elemAt segments 0;
51
-
repo = lib.elemAt segments 1;
52
-
ref = if lib.length segments > 2 then lib.elemAt segments 2 else null;
53
-
in
54
-
{
55
-
type = "gitlab";
56
-
inherit owner repo ref;
57
-
};
58
-
59
-
isChannelUrl =
60
-
url:
61
-
lib.hasPrefix "https://channels.nixos.org/" url || lib.hasPrefix "https://releases.nixos.org/" url;
62
-
63
-
parseChannel =
64
-
url:
65
-
let
66
-
path = lib.removePrefix "https://channels.nixos.org/" url;
67
-
channel = lib.head (lib.splitString "/" path);
68
-
in
69
-
{
70
-
type = "channel";
71
-
inherit channel;
72
-
};
73
-
74
-
branchFlag =
75
-
parsed:
76
-
if parsed ? ref && parsed.ref != null then " -b ${lib.escapeShellArg parsed.ref}" else " -b main";
77
-
78
7
esc = lib.escapeShellArg;
79
8
80
-
npinsAddCmd =
81
-
name: parsed:
82
-
if parsed.type == "github" then
83
-
"npins add github --name ${esc name}${branchFlag parsed} ${esc parsed.owner} ${esc parsed.repo}"
84
-
else if parsed.type == "gitlab" then
85
-
"npins add gitlab --name ${esc name}${branchFlag parsed} ${esc parsed.owner} ${esc parsed.repo}"
86
-
else if parsed.type == "channel" then
87
-
"npins add channel --name ${esc name} ${esc parsed.channel}"
88
-
else if parsed.type == "tarball" then
89
-
"npins add tarball --name ${esc name} ${esc parsed.url}"
90
-
else
91
-
"npins add git --name ${esc name} ${esc parsed.url}";
92
-
93
-
hasFollows = sub: sub ? follows && sub.follows != null;
94
-
95
-
transitiveInputs =
96
-
name: input:
97
-
let
98
-
subs = input.inputs or { };
99
-
nonFollowed = lib.filterAttrs (_: sub: !(hasFollows sub)) subs;
100
-
in
101
-
lib.mapAttrs' (sub: _: lib.nameValuePair "${name}/${sub}" sub) nonFollowed;
102
-
103
-
collectTransitive = lib.foldlAttrs (
104
-
acc: name: input:
105
-
acc // (transitiveInputs name input)
106
-
) { };
107
-
108
9
pinnableInputs = lib.filterAttrs (_: v: v.url or "" != "") inputs;
109
10
110
-
allTransitive = collectTransitive inputs;
111
-
112
-
pinnableTransitive = lib.filterAttrs (_: v: v.url or "" != "") allTransitive;
113
-
114
-
allPins = pinnableInputs // pinnableTransitive;
115
-
116
-
addIfMissing =
117
-
name: input:
11
+
# Seed the runtime queue with one tab-separated "name\turl" line per declared input.
12
+
queueSeed =
118
13
let
119
-
cmd = npinsAddCmd name (parseFlakeUrl (input.url or ""));
14
+
lines = lib.mapAttrsToList (
15
+
name: input: " printf '%s\\t%s\\n' ${esc name} ${esc (input.url or "")}"
16
+
) pinnableInputs;
120
17
in
121
-
''
122
-
if ! jq -e --arg n ${esc name} '.pins | has($n)' npins/sources.json >/dev/null 2>&1; then
123
-
${cmd} || (printf "\ncommand FAILED:\n ${cmd}" >&2 && exit 1)
124
-
fi
125
-
'';
126
-
127
-
addCommands = lib.concatStringsSep "\n" (lib.mapAttrsToList addIfMissing allPins);
128
-
129
-
pinNames = lib.concatStringsSep " " (lib.attrNames allPins);
18
+
"{\n" + lib.concatStringsSep "\n" lines + "\n} >> \"$QUEUE_FILE\"";
130
19
131
20
write-npins =
132
21
pkgs:
···
135
24
runtimeInputs = [
136
25
pkgs.npins
137
26
pkgs.jq
27
+
pkgs.curl
28
+
pkgs.nix
138
29
];
139
30
text = ''
140
31
cd ${flake-file.intoPath}
141
32
npins init --bare 2>/dev/null || true
142
-
${addCommands}
143
-
wanted="${pinNames}"
33
+
34
+
SEEN_FILE=$(mktemp)
35
+
QUEUE_FILE=$(mktemp)
36
+
trap 'rm -f "$SEEN_FILE" "$QUEUE_FILE"' EXIT
37
+
38
+
# Add a pin by its flake-style URL (github:o/r, gitlab:o/r, channel URL, etc.)
39
+
npins_add_url() {
40
+
local name="$1" url="$2" spec owner repo ref channel
41
+
case "$url" in
42
+
github:*)
43
+
spec="''${url#github:}" owner="''${spec%%/*}" spec="''${spec#*/}"
44
+
repo="''${spec%%/*}" ref="''${spec#*/}"
45
+
if [ "$ref" != "$repo" ]; then
46
+
npins add github --name "$name" -b "$ref" "$owner" "$repo"
47
+
else
48
+
# No explicit ref: prefer a release tag, fall back to common branches.
49
+
npins add github --name "$name" "$owner" "$repo" 2>/dev/null \
50
+
|| npins add github --name "$name" -b main "$owner" "$repo" 2>/dev/null \
51
+
|| npins add github --name "$name" -b master "$owner" "$repo"
52
+
fi ;;
53
+
gitlab:*)
54
+
spec="''${url#gitlab:}" owner="''${spec%%/*}" spec="''${spec#*/}"
55
+
repo="''${spec%%/*}" ref="''${spec#*/}"
56
+
if [ "$ref" != "$repo" ]; then
57
+
npins add gitlab --name "$name" -b "$ref" "$owner" "$repo"
58
+
else
59
+
npins add gitlab --name "$name" "$owner" "$repo" 2>/dev/null \
60
+
|| npins add gitlab --name "$name" -b main "$owner" "$repo" 2>/dev/null \
61
+
|| npins add gitlab --name "$name" -b master "$owner" "$repo"
62
+
fi ;;
63
+
https://channels.nixos.org/*|https://releases.nixos.org/*)
64
+
channel=$(printf '%s' "$url" | sed 's|https://[^/]*/||;s|/.*||')
65
+
npins add channel --name "$name" "$channel" ;;
66
+
https://*|http://*)
67
+
npins add tarball --name "$name" "$url" ;;
68
+
*)
69
+
npins add git --name "$name" "$url" ;;
70
+
esac
71
+
}
72
+
73
+
# Fetch a github input's flake.nix and append its deps to QUEUE_FILE.
74
+
discover_transitive() {
75
+
local name="$1" url="$2" spec owner repo ref
76
+
[[ "$url" != github:* ]] && return 0
77
+
spec="''${url#github:}" owner="''${spec%%/*}" spec="''${spec#*/}"
78
+
repo="''${spec%%/*}" ref="''${spec#*/}"
79
+
[ "$ref" = "$repo" ] && ref="HEAD"
80
+
81
+
local flake_tmp nix_tmp
82
+
flake_tmp=$(mktemp --suffix=.nix)
83
+
nix_tmp=$(mktemp --suffix=.nix)
84
+
85
+
curl -sf "https://raw.githubusercontent.com/''${owner}/''${repo}/''${ref}/flake.nix" \
86
+
> "$flake_tmp" || { rm -f "$flake_tmp" "$nix_tmp"; return 0; }
87
+
88
+
# Write a nix expression that extracts just the inputs URLs (no network at eval time).
89
+
printf 'let f = import %s; norm = v: if builtins.isString v then v else v.url or ""; in builtins.mapAttrs (_: norm) (f.inputs or {})\n' \
90
+
"$flake_tmp" > "$nix_tmp"
91
+
92
+
nix-instantiate --eval --strict --json "$nix_tmp" 2>/dev/null \
93
+
| jq -r 'to_entries[] | select(.value != "") | [.key, .value] | @tsv' \
94
+
>> "$QUEUE_FILE" || true
95
+
96
+
rm -f "$flake_tmp" "$nix_tmp"
97
+
}
98
+
99
+
# Seed the BFS queue with all declared inputs.
100
+
${queueSeed}
101
+
102
+
# BFS: process queue items until none remain unvisited.
103
+
while true; do
104
+
name="" url=""
105
+
while IFS=$'\t' read -r qname qurl; do
106
+
if ! grep -qxF "$qname" "$SEEN_FILE" 2>/dev/null; then
107
+
name="$qname" url="$qurl"
108
+
break
109
+
fi
110
+
done < "$QUEUE_FILE"
111
+
[ -z "$name" ] && break
112
+
printf '%s\n' "$name" >> "$SEEN_FILE"
113
+
if ! jq -e --arg n "$name" '.pins | has($n)' npins/sources.json >/dev/null 2>&1; then
114
+
npins_add_url "$name" "$url" || true
115
+
fi
116
+
discover_transitive "$name" "$url"
117
+
done
118
+
119
+
# Remove any pins that were not reachable in this run.
144
120
if [ -f npins/sources.json ]; then
145
121
for existing in $(jq -r '.pins | keys[]' npins/sources.json); do
146
-
keep=false
147
-
for w in $wanted; do
148
-
if [ "$existing" = "$w" ]; then keep=true; break; fi
149
-
done
150
-
if [ "$keep" = false ]; then
122
+
if ! grep -qxF "$existing" "$SEEN_FILE"; then
151
123
npins remove "$existing"
152
124
fi
153
125
done