+42 -1

Makefile

··· 30 30 # Updates: build-spindle → push-spindle → update-spindle (restarts where already active) 31 31 SPINDLE_CORE ?= /tmp/tangled-core 32 32 33 33 - .PHONY: build-spindle push-spindle update-spindle start-spindle logs-spindle 33 33 + .PHONY: build-spindle push-spindle update-spindle start-spindle logs-spindle setup-openbao logs-openbao 34 34 35 35 build-spindle: 36 36 @test -d "$(SPINDLE_CORE)" || { echo "error: tangled core not found at $(SPINDLE_CORE)"; echo "Clone: git clone git@tangled.org:tangled.org/core.git $(SPINDLE_CORE)"; exit 1; } ··· 81 81 echo "==> Streaming logs from $$NODE_NAME ($$NODE_IP)" && \ 82 82 ssh -p 22222 -o StrictHostKeyChecking=no -i keypair/id_ed25519_homelab root@$$NODE_IP \ 83 83 'SPINDLE_UID=$$(id -u spindle) && runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/$$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$$SPINDLE_UID/bus journalctl --user -u spindle.service -f --no-pager' 84 84 + 85 85 + logs-openbao: 86 86 + kubectl logs -n openbao statefulset/openbao -f 87 87 + 88 88 + # OpenBao — one-time init + AppRole setup for Spindle secrets 89 89 + # Initializes OpenBao, creates KV engine + AppRole, deploys proxy credentials to spindle leader node 90 90 + setup-openbao: 91 91 + @echo "==> Initializing OpenBao..." 92 92 + @INIT_OUTPUT=$$(kubectl exec -n openbao openbao-0 -- bao operator init -key-shares=1 -key-threshold=1 -format=json 2>/dev/null) && \ 93 93 + UNSEAL_KEY=$$(echo "$$INIT_OUTPUT" | python3 -c "import sys,json; print(json.load(sys.stdin)['unseal_keys_b64'][0])") && \ 94 94 + ROOT_TOKEN=$$(echo "$$INIT_OUTPUT" | python3 -c "import sys,json; print(json.load(sys.stdin)['root_token'])") && \ 95 95 + echo "==> Unsealing..." && \ 96 96 + kubectl exec -n openbao openbao-0 -- bao operator unseal "$$UNSEAL_KEY" >/dev/null && \ 97 97 + echo "==> Creating unseal Secret..." && \ 98 98 + kubectl create secret generic openbao-unseal -n openbao --from-literal=unseal-key="$$UNSEAL_KEY" --dry-run=client -o yaml | kubectl apply -f - && \ 99 99 + echo "==> Configuring KV engine + AppRole..." && \ 100 100 + kubectl exec -n openbao openbao-0 -- sh -c "export BAO_TOKEN=$$ROOT_TOKEN && \ 101 101 + bao secrets enable -path=spindle -version=2 kv && \ 102 102 + bao policy write spindle-policy /openbao/config/spindle-policy.hcl && \ 103 103 + bao auth enable approle && \ 104 104 + bao write auth/approle/role/spindle token_policies=spindle-policy token_ttl=1h token_max_ttl=4h bind_secret_id=true secret_id_ttl=0 secret_id_num_uses=0" && \ 105 105 + ROLE_ID=$$(kubectl exec -n openbao openbao-0 -- sh -c "BAO_TOKEN=$$ROOT_TOKEN bao read -field=role_id auth/approle/role/spindle/role-id") && \ 106 106 + SECRET_ID=$$(kubectl exec -n openbao openbao-0 -- sh -c "BAO_TOKEN=$$ROOT_TOKEN bao write -f -field=secret_id auth/approle/role/spindle/secret-id") && \ 107 107 + echo "==> Deploying proxy credentials to spindle leader..." && \ 108 108 + NODE_NAME=$$(kubectl get configmap spindle-leader -o jsonpath='{.data.node}' 2>/dev/null) && \ 109 109 + NODE_IP=$$(kubectl get node $$NODE_NAME -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}' 2>/dev/null) && \ 110 110 + test -n "$$NODE_IP" || { echo "error: could not find spindle leader node — run make start-spindle first"; exit 1; } && \ 111 111 + ssh -p 22222 -o StrictHostKeyChecking=no -i keypair/id_ed25519_homelab root@$$NODE_IP " \ 112 112 + printf '%s' '$$ROLE_ID' > /home/spindle/.openbao/role-id && \ 113 113 + printf '%s' '$$SECRET_ID' > /home/spindle/.openbao/secret-id && \ 114 114 + chmod 600 /home/spindle/.openbao/role-id /home/spindle/.openbao/secret-id && \ 115 115 + chown spindle:spindle /home/spindle/.openbao/role-id /home/spindle/.openbao/secret-id && \ 116 116 + SPINDLE_UID=\$$(id -u spindle) && \ 117 117 + runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/\$$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/\$$SPINDLE_UID/bus systemctl --user restart openbao-proxy.service && \ 118 118 + runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/\$$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/\$$SPINDLE_UID/bus systemctl --user restart spindle.service \ 119 119 + " && \ 120 120 + echo "" && \ 121 121 + echo "==> Done. Save this root token somewhere safe:" && \ 122 122 + echo " $$ROOT_TOKEN" && \ 123 123 + echo "" && \ 124 124 + echo "OpenBao initialized, AppRole configured, proxy credentials deployed."

+6

k8s/kustomization.yaml

··· 10 10 - registry 11 11 - alerting 12 12 - opake 13 13 + - openbao 13 14 14 15 generatorOptions: 15 16 disableNameSuffixHash: true ··· 67 68 type: Opaque 68 69 files: 69 70 - password=postgres/postgres-password.secret 71 71 + - name: hetzner-api-token 72 72 + namespace: cert-manager 73 73 + type: Opaque 74 74 + files: 75 75 + - token=shared/hetzner-dns-token.secret 70 76 - name: juicefs-secret 71 77 namespace: juicefs 72 78 type: Opaque

+17

k8s/openbao/kustomization.yaml

··· 1 1 + apiVersion: kustomize.config.k8s.io/v1beta1 2 2 + kind: Kustomization 3 3 + namespace: openbao 4 4 + 5 5 + resources: 6 6 + - namespace.yaml 7 7 + - statefulset.yaml 8 8 + - service.yaml 9 9 + 10 10 + configMapGenerator: 11 11 + - name: openbao-config 12 12 + namespace: openbao 13 13 + files: 14 14 + - server.hcl 15 15 + - spindle-policy.hcl 16 16 + options: 17 17 + disableNameSuffixHash: true

+4

k8s/openbao/namespace.yaml

··· 1 1 + apiVersion: v1 2 2 + kind: Namespace 3 3 + metadata: 4 4 + name: openbao

+11

k8s/openbao/server.hcl

··· 1 1 + storage "file" { 2 2 + path = "/openbao/data" 3 3 + } 4 4 + 5 5 + listener "tcp" { 6 6 + address = "0.0.0.0:8200" 7 7 + tls_disable = true 8 8 + } 9 9 + 10 10 + disable_mlock = true 11 11 + ui = false

+14

k8s/openbao/service.yaml

··· 1 1 + apiVersion: v1 2 2 + kind: Service 3 3 + metadata: 4 4 + name: openbao 5 5 + namespace: openbao 6 6 + spec: 7 7 + type: NodePort 8 8 + selector: 9 9 + app: openbao 10 10 + ports: 11 11 + - name: api 12 12 + port: 8200 13 13 + targetPort: 8200 14 14 + nodePort: 30820

+15

k8s/openbao/spindle-policy.hcl

··· 1 1 + path "spindle/data/*" { 2 2 + capabilities = ["create", "read", "update", "delete"] 3 3 + } 4 4 + 5 5 + path "spindle/metadata/*" { 6 6 + capabilities = ["list", "read", "delete", "update"] 7 7 + } 8 8 + 9 9 + path "spindle/" { 10 10 + capabilities = ["list"] 11 11 + } 12 12 + 13 13 + path "auth/token/lookup-self" { 14 14 + capabilities = ["read"] 15 15 + }

+89

k8s/openbao/statefulset.yaml

··· 1 1 + apiVersion: apps/v1 2 2 + kind: StatefulSet 3 3 + metadata: 4 4 + name: openbao 5 5 + namespace: openbao 6 6 + spec: 7 7 + serviceName: openbao 8 8 + replicas: 1 9 9 + selector: 10 10 + matchLabels: 11 11 + app: openbao 12 12 + template: 13 13 + metadata: 14 14 + labels: 15 15 + app: openbao 16 16 + spec: 17 17 + securityContext: 18 18 + runAsUser: 100 19 19 + runAsGroup: 1000 20 20 + fsGroup: 1000 21 21 + containers: 22 22 + - name: openbao 23 23 + image: openbao/openbao:2.5.1 24 24 + args: ["server", "-config=/openbao/config/server.hcl"] 25 25 + ports: 26 26 + - containerPort: 8200 27 27 + name: api 28 28 + env: 29 29 + - name: BAO_ADDR 30 30 + value: "http://127.0.0.1:8200" 31 31 + - name: SKIP_SETCAP 32 32 + value: "true" 33 33 + resources: 34 34 + requests: 35 35 + cpu: 50m 36 36 + memory: 64Mi 37 37 + limits: 38 38 + cpu: 200m 39 39 + memory: 256Mi 40 40 + readinessProbe: 41 41 + httpGet: 42 42 + path: /v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 43 43 + port: 8200 44 44 + initialDelaySeconds: 5 45 45 + periodSeconds: 10 46 46 + livenessProbe: 47 47 + httpGet: 48 48 + path: /v1/sys/health?standbyok=true&uninitcode=200&sealedcode=200 49 49 + port: 8200 50 50 + initialDelaySeconds: 10 51 51 + periodSeconds: 30 52 52 + lifecycle: 53 53 + postStart: 54 54 + exec: 55 55 + command: 56 56 + - /bin/sh 57 57 + - -c 58 58 + - | 59 59 + sleep 3 60 60 + if [ -f /openbao/credentials/unseal-key ]; then 61 61 + bao operator unseal "$(cat /openbao/credentials/unseal-key)" 2>/dev/null || true 62 62 + fi 63 63 + volumeMounts: 64 64 + - name: data 65 65 + mountPath: /openbao/data 66 66 + - name: config 67 67 + mountPath: /openbao/config 68 68 + readOnly: true 69 69 + - name: unseal-key 70 70 + mountPath: /openbao/credentials 71 71 + readOnly: true 72 72 + volumes: 73 73 + - name: config 74 74 + configMap: 75 75 + name: openbao-config 76 76 + - name: unseal-key 77 77 + secret: 78 78 + secretName: openbao-unseal 79 79 + optional: true 80 80 + volumeClaimTemplates: 81 81 + - metadata: 82 82 + name: data 83 83 + spec: 84 84 + accessModes: 85 85 + - ReadWriteOnce 86 86 + storageClassName: juicefs-sc 87 87 + resources: 88 88 + requests: 89 89 + storage: 1Gi

+14 -3

kube.tf

··· 113 113 "grep -q '^spindle:' /etc/subgid || usermod --add-subgids 100000-165535 spindle", 114 114 "loginctl enable-linger spindle", 115 115 116 116 - # Directories: logs, systemd unit, podman config 116 116 + # Directories: logs, data, systemd unit, podman config, openbao proxy 117 117 "mkdir -p /var/log/spindle && chown spindle:spindle /var/log/spindle", 118 118 + "mkdir -p /var/lib/spindle/logs && chown -R spindle:spindle /var/lib/spindle", 118 119 "mkdir -p /home/spindle/.config/systemd/user && chown -R spindle:spindle /home/spindle/.config", 120 120 + "mkdir -p /home/spindle/.openbao && chown spindle:spindle /home/spindle/.openbao", 119 121 120 122 # Disable SELinux labels in rootless containers — kernel 6.19 + SELinux prevents 121 123 # mprotect in user namespaces, causing RELRO failures in musl-based containers. ··· 126 128 # Fix SELinux contexts on home dir (provisioning runs as root, labels end up wrong) 127 129 "restorecon -R /home/spindle", 128 130 129 129 - # Pull spindle binary from Zot OCI image 130 130 - "podman pull zot.sans-self.org/infra/spindle:latest && CID=$(podman create zot.sans-self.org/infra/spindle:latest) && podman cp $CID:/spindle /usr/local/bin/spindle && podman cp $CID:/spindle.service /home/spindle/.config/systemd/user/spindle.service && podman rm $CID && chmod 755 /usr/local/bin/spindle && chown -R spindle:spindle /home/spindle/.config || echo 'WARN: spindle image not available, run make setup-spindle after cluster is ready'", 131 131 + # Pull spindle + openbao proxy from Zot OCI image 132 132 + "podman pull zot.sans-self.org/infra/spindle:latest && CID=$(podman create zot.sans-self.org/infra/spindle:latest) && podman cp $CID:/spindle /usr/local/bin/spindle && podman cp $CID:/bao /usr/local/bin/bao && podman cp $CID:/spindle.service /home/spindle/.config/systemd/user/spindle.service && podman cp $CID:/openbao-proxy.service /home/spindle/.config/systemd/user/openbao-proxy.service && podman cp $CID:/openbao-proxy.hcl /home/spindle/.openbao/proxy.hcl && podman rm $CID && chmod 755 /usr/local/bin/spindle /usr/local/bin/bao && chown -R spindle:spindle /home/spindle/.config /home/spindle/.openbao || echo 'WARN: spindle image not available, run make update-spindle after cluster is ready'", 131 133 132 134 # Enable podman socket for rootless container API 133 135 "sleep 2 && SPINDLE_UID=$(id -u spindle) && runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$SPINDLE_UID/bus systemctl --user daemon-reload && runuser -u spindle -- env XDG_RUNTIME_DIR=/run/user/$SPINDLE_UID DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$SPINDLE_UID/bus systemctl --user enable --now podman.socket || true", ··· 157 159 chart = "juicefs-csi-driver" 158 160 version = "0.31.2" 159 161 values = [file("juicefs-csi-values.yaml")] 162 162 + 163 163 + depends_on = [module.kube-hetzner] 164 164 + } 165 165 + 166 166 + resource "helm_release" "cert_manager_webhook_hetzner" { 167 167 + name = "cert-manager-webhook-hetzner" 168 168 + namespace = "cert-manager" 169 169 + repository = "https://charts.hetzner.cloud" 170 170 + chart = "cert-manager-webhook-hetzner" 160 171 161 172 depends_on = [module.kube-hetzner] 162 173 }

+5

spindle/Containerfile

··· 1 1 + FROM openbao/openbao:2.5.1 AS openbao 2 2 + 1 3 FROM busybox:1.37 2 4 COPY spindle /spindle 3 5 COPY spindle.service /spindle.service 6 6 + COPY --from=openbao /bin/bao /bao 7 7 + COPY openbao-proxy.service /openbao-proxy.service 8 8 + COPY openbao-proxy.hcl /openbao-proxy.hcl

+2

spindle/healthcheck-cronjob.yaml

··· 149 149 XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 150 150 runuser -u spindle -- systemctl --user enable --now podman.socket 151 151 XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 152 152 + runuser -u spindle -- systemctl --user enable --now openbao-proxy.service 153 153 + XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus \ 152 154 runuser -u spindle -- systemctl --user enable --now spindle.service 153 155 ' 154 156

+34

spindle/openbao-proxy.hcl

··· 1 1 + vault { 2 2 + address = "http://127.0.0.1:30820" 3 3 + } 4 4 + 5 5 + auto_auth { 6 6 + method "approle" { 7 7 + mount_path = "auth/approle" 8 8 + config = { 9 9 + role_id_file_path = "/home/spindle/.openbao/role-id" 10 10 + secret_id_file_path = "/home/spindle/.openbao/secret-id" 11 11 + remove_secret_id_file_after_reading = false 12 12 + } 13 13 + } 14 14 + 15 15 + sink "file" { 16 16 + config = { 17 17 + path = "/home/spindle/.openbao/token" 18 18 + mode = 0640 19 19 + } 20 20 + } 21 21 + } 22 22 + 23 23 + listener "tcp" { 24 24 + address = "127.0.0.1:8201" 25 25 + tls_disable = true 26 26 + } 27 27 + 28 28 + api_proxy { 29 29 + use_auto_auth_token = true 30 30 + } 31 31 + 32 32 + cache { 33 33 + use_auto_auth_token = true 34 34 + }

+12

spindle/openbao-proxy.service

··· 1 1 + [Unit] 2 2 + Description=OpenBao Proxy (Spindle secrets) 3 3 + After=podman.socket 4 4 + 5 5 + [Service] 6 6 + Type=simple 7 7 + ExecStart=/usr/local/bin/bao proxy -config=/home/spindle/.openbao/proxy.hcl 8 8 + Restart=on-failure 9 9 + RestartSec=5 10 10 + 11 11 + [Install] 12 12 + WantedBy=default.target

+5 -1

spindle/spindle.service

··· 1 1 [Unit] 2 2 Description=Spindle CI Runner 3 3 - After=podman.socket 3 3 + After=openbao-proxy.service podman.socket 4 4 + Requires=openbao-proxy.service 4 5 5 6 [Service] 6 7 Type=simple ··· 11 12 Environment=SPINDLE_SERVER_OWNER=did:plc:wydyrngmxbcsqdvhmd7whmye 12 13 Environment=SPINDLE_PIPELINES_LOG_DIR=/var/lib/spindle/logs 13 14 Environment=SPINDLE_PIPELINES_WORKFLOW_TIMEOUT=10m 15 15 + Environment=SPINDLE_SERVER_SECRETS_PROVIDER=openbao 16 16 + Environment=SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=http://127.0.0.1:8201 17 17 + Environment=SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=spindle 14 18 ExecStart=/usr/local/bin/spindle 15 19 Restart=on-failure 16 20 RestartSec=5

+2 -1

spindle/start-job.yaml

··· 27 27 export XDG_RUNTIME_DIR=/run/user/$uid 28 28 export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus 29 29 runuser -u spindle -- systemctl --user enable --now podman.socket 30 30 + runuser -u spindle -- systemctl --user enable --now openbao-proxy.service 30 31 runuser -u spindle -- systemctl --user enable --now spindle.service 31 31 - echo "Spindle started" 32 32 + echo "OpenBao proxy + Spindle started" 32 33 ' 33 34 restartPolicy: Never 34 35 backoffLimit: 1

+19 -2

spindle/update-job.yaml

··· 27 27 privileged: true 28 28 command: ["/bin/sh", "-c"] 29 29 args: 30 30 - - cp /spindle /host-bin/spindle && cp /spindle.service /host-service/spindle.service 30 30 + - | 31 31 + cp /spindle /host-bin/spindle 32 32 + cp /bao /host-bin/bao 33 33 + cp /spindle.service /host-service/spindle.service 34 34 + cp /openbao-proxy.service /host-service/openbao-proxy.service 35 35 + cp /openbao-proxy.hcl /host-openbao/proxy.hcl 31 36 volumeMounts: 32 37 - name: host-bin 33 38 mountPath: /host-bin 34 39 - name: host-service 35 40 mountPath: /host-service 41 41 + - name: host-openbao 42 42 + mountPath: /host-openbao 36 43 containers: 37 44 - name: deploy 38 45 image: alpine:3.21 ··· 43 50 - | 44 51 apk add --no-cache --quiet util-linux >/dev/null 2>&1 45 52 nsenter -t 1 -m -u -i -n -- bash -c ' 46 46 - chmod 755 /usr/local/bin/spindle 53 53 + chmod 755 /usr/local/bin/spindle /usr/local/bin/bao 47 54 chown -R spindle:spindle /home/spindle/.config 55 55 + chown spindle:spindle /home/spindle/.openbao /home/spindle/.openbao/proxy.hcl 56 56 + mkdir -p /var/lib/spindle/logs && chown -R spindle:spindle /var/lib/spindle 48 57 uid=$(id -u spindle 2>/dev/null) || { echo "spindle user missing"; exit 0; } 49 58 export XDG_RUNTIME_DIR=/run/user/$uid 50 59 export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus 51 60 runuser -u spindle -- systemctl --user daemon-reload 61 61 + if runuser -u spindle -- systemctl --user is-active --quiet openbao-proxy.service 2>/dev/null; then 62 62 + runuser -u spindle -- systemctl --user restart openbao-proxy.service 63 63 + echo "OpenBao proxy restarted" 64 64 + fi 52 65 if runuser -u spindle -- systemctl --user is-active --quiet spindle.service 2>/dev/null; then 53 66 runuser -u spindle -- systemctl --user restart spindle.service 54 67 echo "Spindle restarted" ··· 64 77 - name: host-service 65 78 hostPath: 66 79 path: /home/spindle/.config/systemd/user 80 80 + type: DirectoryOrCreate 81 81 + - name: host-openbao 82 82 + hostPath: 83 83 + path: /home/spindle/.openbao 67 84 type: DirectoryOrCreate 68 85 restartPolicy: Never 69 86 backoffLimit: 3