#!/usr/bin/env bash # Shared helpers for the cargo-cache actions. Sourced, never executed # directly — every caller does: # # . "$(dirname "${BASH_SOURCE[0]}")/cache-lib.sh" # # Nothing here reads the environment implicitly; every function takes what it # needs as an argument, so the selftests can drive them against scratch # directories without a CI context. # --------------------------------------------------------------------------- # Cache keys # --------------------------------------------------------------------------- # Maps a raw git ref to a filesystem-safe, collision-resistant directory # component. # # Two properties matter and neither is free: # # Determinism — the same raw ref must always produce the same key, or a # branch's second run lands in a different directory from its first and the # whole cache is pointless. # # Collision resistance — `tr -c 'A-Za-z0-9._-' '-'` maps every disallowed # byte to the same `-`, so `feat/foo` and `feat-foo` sanitise identically # and would share one directory: two unrelated branches interleaving # fingerprints in one tree, which is the exact cross-branch-contamination # hazard this whole scheme exists to close, reopened through the sanitiser. # An 8-hex-char prefix of the SHA-1 of the *raw* (pre-sanitisation) ref is # appended so distinct refs always get distinct keys regardless of what # sanitisation or truncation did to the readable part. # # The readable part is capped at 48 characters so a long branch name can't # approach filesystem path-length limits; the hash suffix is what keeps two # refs sharing a 48-char prefix apart. cache_key() { local raw="$1" slug hash if [ -z "$raw" ]; then echo "cache_key: refusing to key an empty ref" >&2 return 1 fi slug=$(printf '%s' "$raw" | tr -c 'A-Za-z0-9._-' '-' | sed 's/-\{2,\}/-/g; s/^-//; s/-$//') hash=$(printf '%s' "$raw" | sha1sum | cut -c1-8) printf '%s-%s' "${slug:0:48}" "$hash" } target_dir_for() { printf '%s/target-%s' "$1" "$2"; } snapshot_dir_for() { printf '%s/snapshot-%s' "$1" "$2"; } # --------------------------------------------------------------------------- # Disk accounting # --------------------------------------------------------------------------- usage_kb() { if [ -d "$1" ]; then du -sk "$1" 2>/dev/null | awk '{print $1}'; else echo 0; fi } usage_gb() { awk "BEGIN { printf \"%.1f\", $(usage_kb "$1") / 1024 / 1024 }" } # df -kP: portable POSIX one-line-per-fs output; columns are 1k-blocks total, # used, available, capacity%, mounted-on. Prints " ". # # CACHE_DF_OVERRIDE exists for the selftests: a scratch tmpdir on the test # host's real filesystem won't sit below an arbitrary threshold on demand, and # the eviction passes are precisely what needs testing under pressure. read_df() { if [ -n "${CACHE_DF_OVERRIDE:-}" ]; then printf '%s\n' "$CACHE_DF_OVERRIDE"; return; fi df -kP "$1" | awk 'NR==2 {print $2, $4}' } report_df() { local label="$1" free_kb="$2" total_kb="$3" awk -v l="$label" -v f="$free_kb" -v t="$total_kb" \ 'BEGIN { printf "%s %.1f GB free / %.0f GB (%.1f%%)", l, f/1048576, t/1048576, (f*100)/t }' } # Appends to the Actions job summary, which is read on green runs — unlike a # ::warning:: buried in a log nobody opens. No-op outside Actions so the # scripts still run standalone under the selftests. summary_line() { [ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf '%s\n' "$1" >> "$GITHUB_STEP_SUMMARY" return 0 } # --------------------------------------------------------------------------- # Hardlink cloning, and the part that makes it sound # --------------------------------------------------------------------------- # Cargo's own target-dir lock files (.cargo-lock, .cargo-build-lock, # .cargo-artifact-lock) are zero-byte files it opens and flock(2)s IN PLACE # for the duration of a build — it never truncates-and-renames them the way it # does real artifacts. `cp -al` leaves the clone's copy hardlinked to the same # inode as the source's, and flock() contention is inode-based, not # path-based, so a build in the clone and a build in the source would # serialize on one mutex. # # The glob deliberately reaches past the three observed names so a lock file # added by a future Cargo version is swept too; nothing else in a target dir # is named `.cargo-*lock*`. Cargo recreates whichever it needs as a fresh, # unshared inode the next time it opens the directory, at no cost. strip_cargo_locks() { find "$1" -type f -name '.cargo-*lock*' -delete 2>/dev/null || true return 0 } # Replaces a subtree with a real (non-hardlinked) copy of itself, in place. # # Staged through a sibling temp path and swapped with `mv -T` rather than # copied over the original file-by-file: the copy is a fresh tree of fresh # inodes, so nothing in it can alias the source it was cloned from. Callers # only ever run this against a staging directory nothing else can see yet # (see hardlink_clone_into's contract), so the brief window where the path is # absent is not observable. unshare_subtree() { local d="$1" tmp [ -d "$d" ] || return 0 tmp="${d}.unshare.$$" rm -rf "$tmp" cp -a "$d" "$tmp" rm -rf "$d" mv -T "$tmp" "$d" } _unshare_files() { # `-links +1` restricts the work to files that are actually shared, which # makes this idempotent and near-free on an already-unshared tree. find "$@" -links +1 -print0 2>/dev/null | xargs -0 -r -n 64 bash -c 'for f; do cp -p -- "$f" "$f.unshare.$$" && mv -f -- "$f.unshare.$$" "$f"; done' _ return 0 } # THE load-bearing function of this whole design. # # A hardlink clone is only safe if every write the clone's build performs # lands on a NEW inode, leaving the source's data untouched. That is true for # compilation artifacts — rustc and the linker replace `deps/*.rlib`, # `*.rmeta`, and binaries rather than truncating them in place — and it is # NOT true for the metadata Cargo and build scripts write with a plain # truncating write. Measured directly (Linux, ext4, cargo 1.9x nightly: # `cp -al` a warm target dir, change a source file, build in the clone, diff # the source) the following files in the SOURCE were mutated through the # shared inode: # # /.fingerprint//dep- (only under # CARGO_UNSTABLE_CHECKSUM_FRESHNESS, # where this file carries the # per-source blake3 checksums) # /build//output, root-output (Cargo build-script metadata) # /build//out/** (whatever the build script # writes into OUT_DIR — build # scripts overwhelmingly use a # plain fs::write) # /deps/*.d, /*.d (Cargo's post-processed # dep-info) # # The checksum-freshness case is not a cosmetic one. Reproduced end to end: # branch B clones base's cache, builds its own content, and thereby rewrites # base's `dep-` to describe B's sources while base's cache still holds # the artifact built from base's sources. When B merges and base next builds, # Cargo reads that dep file, finds the checksums match the (now merged) # sources, reports `Fresh`, and reuses a binary built from the PRE-merge code. # That is silent stale-artifact reuse — a wrong answer, not a slow one. # # So: hardlink the artifacts (the GB), real-copy the metadata (the MB). # Measured on a 6.9 GB Bevy workspace target dir, the unshared set is # .fingerprint 22 MB + build/ 237 MB + a handful of dep-info files — about # 3.7% of the tree, against 100% for a plain `cp -a`. # # `incremental/` is deliberately left shared: rustc writes each incremental # session to a fresh `s-*-working` directory and finalises it with a rename, # and garbage-collects old sessions by unlinking directory entries — neither # of which mutates a shared inode. CI should still set CARGO_INCREMENTAL=0, # for size rather than correctness. unshare_mutable_paths() { local root="$1" d [ -d "$root" ] || return 0 # The list is materialised in full before anything is replaced: each # replacement deletes and recreates a directory, and a live `find` walk over # a tree being mutated underneath it is a needless hazard. `-prune` keeps a # match's own contents out of the list. local -a dirs=() mapfile -t dirs < <(find "$root" -type d \( -name .fingerprint -o -name build \) -prune -print 2>/dev/null) for d in "${dirs[@]}"; do [ -n "$d" ] || continue unshare_subtree "$d" done _unshare_files "$root" -type f -name '*.d' _unshare_files "$root" -maxdepth 3 -type f -name '.rustc_info.json' return 0 } # Hardlink-clones SRC to a staging path, sanitises it, and publishes it to DST # with a single atomic rename. # # The staging + rename is what closes the concurrent-seed race structurally # rather than by runner topology: a second job sharing this cache key either # sees DST absent (and stages its own clone, losing the rename harmlessly) or # sees it complete. There is no observable half-populated state, because a # directory rename is atomic and DST is never written through. # # Returns 0 if this caller's clone won the rename, 1 if another caller got # there first (the staging copy is discarded; DST is already valid). hardlink_clone_into() { local src="$1" dst="$2" tag="$3" tmp parent parent=$(dirname "$dst") tmp="${parent}/.stage-${tag}" rm -rf "$tmp" cp -al "$src" "$tmp" strip_cargo_locks "$tmp" rm -f "$tmp"/.cache-last-used "$tmp"/.ci-lock-* 2>/dev/null || true unshare_mutable_paths "$tmp" if mv -T "$tmp" "$dst" 2>/dev/null; then return 0 fi rm -rf "$tmp" return 1 }