gitdan-actions/scripts/cache-lib.sh

#!/usr/bin/env bash
# Shared helpers for the cargo-cache actions. Sourced, never executed
# directly — every caller does:
#
#   . "$(dirname "${BASH_SOURCE[0]}")/cache-lib.sh"
#
# Nothing here reads the environment implicitly; every function takes what it
# needs as an argument, so the selftests can drive them against scratch
# directories without a CI context.

# ---------------------------------------------------------------------------
# Cache keys
# ---------------------------------------------------------------------------

# Maps a raw git ref to a filesystem-safe, collision-resistant directory
# component.
#
# Two properties matter and neither is free:
#
#   Determinism — the same raw ref must always produce the same key, or a
#   branch's second run lands in a different directory from its first and the
#   whole cache is pointless.
#
#   Collision resistance — `tr -c 'A-Za-z0-9._-' '-'` maps every disallowed
#   byte to the same `-`, so `feat/foo` and `feat-foo` sanitise identically
#   and would share one directory: two unrelated branches interleaving
#   fingerprints in one tree, which is the exact cross-branch-contamination
#   hazard this whole scheme exists to close, reopened through the sanitiser.
#   An 8-hex-char prefix of the SHA-1 of the *raw* (pre-sanitisation) ref is
#   appended so distinct refs always get distinct keys regardless of what
#   sanitisation or truncation did to the readable part.
#
# The readable part is capped at 48 characters so a long branch name can't
# approach filesystem path-length limits; the hash suffix is what keeps two
# refs sharing a 48-char prefix apart.
cache_key() {
  local raw="$1" slug hash
  if [ -z "$raw" ]; then
    echo "cache_key: refusing to key an empty ref" >&2
    return 1
  fi
  slug=$(printf '%s' "$raw" | tr -c 'A-Za-z0-9._-' '-' | sed 's/-\{2,\}/-/g; s/^-//; s/-$//')
  hash=$(printf '%s' "$raw" | sha1sum | cut -c1-8)
  printf '%s-%s' "${slug:0:48}" "$hash"
}

target_dir_for() { printf '%s/target-%s' "$1" "$2"; }
snapshot_dir_for() { printf '%s/snapshot-%s' "$1" "$2"; }

# ---------------------------------------------------------------------------
# Disk accounting
# ---------------------------------------------------------------------------

usage_kb() {
  if [ -d "$1" ]; then du -sk "$1" 2>/dev/null | awk '{print $1}'; else echo 0; fi
}

usage_gb() {
  awk "BEGIN { printf \"%.1f\", $(usage_kb "$1") / 1024 / 1024 }"
}

# df -kP: portable POSIX one-line-per-fs output; columns are 1k-blocks total,
# used, available, capacity%, mounted-on. Prints "<total_kb> <free_kb>".
#
# CACHE_DF_OVERRIDE exists for the selftests: a scratch tmpdir on the test
# host's real filesystem won't sit below an arbitrary threshold on demand, and
# the eviction passes are precisely what needs testing under pressure.
read_df() {
  if [ -n "${CACHE_DF_OVERRIDE:-}" ]; then printf '%s\n' "$CACHE_DF_OVERRIDE"; return; fi
  df -kP "$1" | awk 'NR==2 {print $2, $4}'
}

report_df() {
  local label="$1" free_kb="$2" total_kb="$3"
  awk -v l="$label" -v f="$free_kb" -v t="$total_kb" \
    'BEGIN { printf "%s %.1f GB free / %.0f GB (%.1f%%)", l, f/1048576, t/1048576, (f*100)/t }'
}

# Appends to the Actions job summary, which is read on green runs — unlike a
# ::warning:: buried in a log nobody opens. No-op outside Actions so the
# scripts still run standalone under the selftests.
summary_line() {
  [ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf '%s\n' "$1" >> "$GITHUB_STEP_SUMMARY"
  return 0
}

# ---------------------------------------------------------------------------
# Hardlink cloning, and the part that makes it sound
# ---------------------------------------------------------------------------

# Cargo's own target-dir lock files (.cargo-lock, .cargo-build-lock,
# .cargo-artifact-lock) are zero-byte files it opens and flock(2)s IN PLACE
# for the duration of a build — it never truncates-and-renames them the way it
# does real artifacts. `cp -al` leaves the clone's copy hardlinked to the same
# inode as the source's, and flock() contention is inode-based, not
# path-based, so a build in the clone and a build in the source would
# serialize on one mutex.
#
# The glob deliberately reaches past the three observed names so a lock file
# added by a future Cargo version is swept too; nothing else in a target dir
# is named `.cargo-*lock*`. Cargo recreates whichever it needs as a fresh,
# unshared inode the next time it opens the directory, at no cost.
strip_cargo_locks() {
  find "$1" -type f -name '.cargo-*lock*' -delete 2>/dev/null || true
  return 0
}

# Replaces a subtree with a real (non-hardlinked) copy of itself, in place.
#
# Staged through a sibling temp path and swapped with `mv -T` rather than
# copied over the original file-by-file: the copy is a fresh tree of fresh
# inodes, so nothing in it can alias the source it was cloned from. Callers
# only ever run this against a staging directory nothing else can see yet
# (see hardlink_clone_into's contract), so the brief window where the path is
# absent is not observable.
unshare_subtree() {
  local d="$1" tmp
  [ -d "$d" ] || return 0
  tmp="${d}.unshare.$$"
  rm -rf "$tmp"
  cp -a "$d" "$tmp"
  rm -rf "$d"
  mv -T "$tmp" "$d"
}

_unshare_files() {
  # `-links +1` restricts the work to files that are actually shared, which
  # makes this idempotent and near-free on an already-unshared tree.
  find "$@" -links +1 -print0 2>/dev/null |
    xargs -0 -r -n 64 bash -c 'for f; do cp -p -- "$f" "$f.unshare.$$" && mv -f -- "$f.unshare.$$" "$f"; done' _
  return 0
}

# THE load-bearing function of this whole design.
#
# A hardlink clone is only safe if every write the clone's build performs
# lands on a NEW inode, leaving the source's data untouched. That is true for
# compilation artifacts — rustc and the linker replace `deps/*.rlib`,
# `*.rmeta`, and binaries rather than truncating them in place — and it is
# NOT true for the metadata Cargo and build scripts write with a plain
# truncating write. Measured directly (Linux, ext4, cargo 1.9x nightly:
# `cp -al` a warm target dir, change a source file, build in the clone, diff
# the source) the following files in the SOURCE were mutated through the
# shared inode:
#
#   <profile>/.fingerprint/<unit>/dep-<target>   (only under
#                                                CARGO_UNSTABLE_CHECKSUM_FRESHNESS,
#                                                where this file carries the
#                                                per-source blake3 checksums)
#   <profile>/build/<pkg>/output, root-output   (Cargo build-script metadata)
#   <profile>/build/<pkg>/out/**                (whatever the build script
#                                                writes into OUT_DIR — build
#                                                scripts overwhelmingly use a
#                                                plain fs::write)
#   <profile>/deps/*.d, <profile>/*.d           (Cargo's post-processed
#                                                dep-info)
#
# The checksum-freshness case is not a cosmetic one. Reproduced end to end:
# branch B clones base's cache, builds its own content, and thereby rewrites
# base's `dep-<target>` to describe B's sources while base's cache still holds
# the artifact built from base's sources. When B merges and base next builds,
# Cargo reads that dep file, finds the checksums match the (now merged)
# sources, reports `Fresh`, and reuses a binary built from the PRE-merge code.
# That is silent stale-artifact reuse — a wrong answer, not a slow one.
#
# So: hardlink the artifacts (the GB), real-copy the metadata (the MB).
# Measured on a 6.9 GB Bevy workspace target dir, the unshared set is
# .fingerprint 22 MB + build/ 237 MB + a handful of dep-info files — about
# 3.7% of the tree, against 100% for a plain `cp -a`.
#
# `incremental/` is deliberately left shared: rustc writes each incremental
# session to a fresh `s-*-working` directory and finalises it with a rename,
# and garbage-collects old sessions by unlinking directory entries — neither
# of which mutates a shared inode. CI should still set CARGO_INCREMENTAL=0,
# for size rather than correctness.
unshare_mutable_paths() {
  local root="$1" d
  [ -d "$root" ] || return 0
  # The list is materialised in full before anything is replaced: each
  # replacement deletes and recreates a directory, and a live `find` walk over
  # a tree being mutated underneath it is a needless hazard. `-prune` keeps a
  # match's own contents out of the list.
  local -a dirs=()
  mapfile -t dirs < <(find "$root" -type d \( -name .fingerprint -o -name build \) -prune -print 2>/dev/null)
  for d in "${dirs[@]}"; do
    [ -n "$d" ] || continue
    unshare_subtree "$d"
  done
  _unshare_files "$root" -type f -name '*.d'
  _unshare_files "$root" -maxdepth 3 -type f -name '.rustc_info.json'
  return 0
}

# Hardlink-clones SRC to a staging path, sanitises it, and publishes it to DST
# with a single atomic rename.
#
# The staging + rename is what closes the concurrent-seed race structurally
# rather than by runner topology: a second job sharing this cache key either
# sees DST absent (and stages its own clone, losing the rename harmlessly) or
# sees it complete. There is no observable half-populated state, because a
# directory rename is atomic and DST is never written through.
#
# Returns 0 if this caller's clone won the rename, 1 if another caller got
# there first (the staging copy is discarded; DST is already valid).
hardlink_clone_into() {
  local src="$1" dst="$2" tag="$3" tmp parent
  parent=$(dirname "$dst")
  tmp="${parent}/.stage-${tag}"
  rm -rf "$tmp"
  cp -al "$src" "$tmp"
  strip_cargo_locks "$tmp"
  rm -f "$tmp"/.cache-last-used "$tmp"/.ci-lock-* 2>/dev/null || true
  unshare_mutable_paths "$tmp"
  if mv -T "$tmp" "$dst" 2>/dev/null; then
    return 0
  fi
  rm -rf "$tmp"
  return 1
}