gitdan-actions/scripts/cache-lib.sh

#!/usr/bin/env bash
# Shared helpers for the cargo-cache actions. Sourced, never executed
# directly — every caller does:
#
#   . "$(dirname "${BASH_SOURCE[0]}")/cache-lib.sh"
#
# Nothing here reads the environment implicitly; every function takes what it
# needs as an argument, so the selftests can drive them against scratch
# directories without a CI context.

# ---------------------------------------------------------------------------
# CROSS-REPO CONTRACT: the dot-prefixed names left in a cache root
# ---------------------------------------------------------------------------
#
# The volume these scripts write into is also swept by a host-level arbiter
# that runs outside any job and outside this repo: daniel/gitdan's
# `scripts/ci-cache-reclaim.sh`. It reclaims the dot-prefixed trees a killed
# job strands here, and it reads the reader markers below to decide whether one
# of those trees is still live. Its `LEFTOVER NAMING CONTRACT` block is the
# canonical description of the arrangement; what follows is the producing
# side's half — which names this repo creates, and what changing one obliges.
#
# Created directly under a cache root:
#
#   .stage-<tag>              cache-lib.sh, hardlink_clone_into(): the tree a
#                             clone is built in before the atomic rename that
#                             gives it its real name. <tag> is unique per job
#                             per run, so a job killed before the rename
#                             strands a whole hardlink clone under a name no
#                             later run reuses. Nothing in this repo sweeps it.
#   .publish-new-<tag>        publish-snapshot.sh: the staged snapshot, between
#                             its clone and the swap. Strands the same way.
#   .publish-old-<key>-<tag>  publish-snapshot.sh: the rotated-away generation,
#                             kept while a reader still holds it and swept by
#                             the next publish of the same key — if there ever
#                             is one. A merged or renamed branch never
#                             publishes again, and its last one stays.
#   .evicting-<name>-<pid>    prune-cache.sh, evict_dir(): a cache renamed
#                             aside so the decision to unlink it can be retaken
#                             after the rename. Swept at the start of every
#                             prune pass, so it only strands when this repo's
#                             workflow stops running at all.
#   .reading-<source>-<tag>   cache-lib.sh, reader_lock_acquire(): NOT garbage.
#                             It is the live-reader signal the arbiter reads,
#                             and the one shape it must never delete — removing
#                             one clears the way to unlink a tree out from
#                             under an in-flight walk, which is the silent
#                             truncation this whole interlock exists to
#                             prevent.
#
# THE RULE, which the arbiter states as its own: no new dot-prefixed entry
# under a cache root without a matching prefix in that script. ADDING a shape
# counts exactly as much as renaming one, because that script enumerates by
# explicit prefix rather than by dotglob — deliberately, since a dotglob would
# pull reader markers into the candidate stream alongside the trees they
# protect. A shape it has not been told about is not handled conservatively, it
# is invisible: an unreclaimed staging tree is a full clone of a multi-GB
# target dir on the one volume whose entire problem is disk.
#
# The two `.publish-*` names predate the contract and were outside it when this
# block was written — the rule catching an uncovered shape on its first
# application. `.publish-new-` is the one that mattered: it is tagged per job
# per run exactly as `.stage-` is, so a publisher killed before the swap
# strands a tree under a name no later run of that script matches, which is
# precisely the shape only the arbiter can reach. Bringing both in is tracked
# as daniel/gitdan#30.
#
# The two lists are meant to be the same length: the five names above, and the
# prefix constants that script DECLARES — not the subset it enumerates as
# reclaim candidates, which is smaller because `.reading-` is read and never
# swept. A shape here without a constant there is one side having changed
# without telling the other, and it is cheapest to notice by counting.
#
# The two staleness constants the arbiter mirrors are part of the same
# contract, and that half has a direction to it — see CACHE_READ_STALE_SECONDS
# below and STALE_LOCK_SECONDS in prune-cache.sh.

# ---------------------------------------------------------------------------
# Cache keys
# ---------------------------------------------------------------------------

# Maps a raw git ref to a filesystem-safe, collision-resistant directory
# component.
#
# Two properties matter and neither is free:
#
#   Determinism — the same raw ref must always produce the same key, or a
#   branch's second run lands in a different directory from its first and the
#   whole cache is pointless.
#
#   Collision resistance — `tr -c 'A-Za-z0-9._-' '-'` maps every disallowed
#   byte to the same `-`, so `feat/foo` and `feat-foo` sanitise identically
#   and would share one directory: two unrelated branches interleaving
#   fingerprints in one tree, which is the exact cross-branch-contamination
#   hazard this whole scheme exists to close, reopened through the sanitiser.
#   An 8-hex-char prefix of the SHA-1 of the *raw* (pre-sanitisation) ref is
#   appended so distinct refs always get distinct keys regardless of what
#   sanitisation or truncation did to the readable part.
#
# The readable part is capped at 48 characters so a long branch name can't
# approach filesystem path-length limits; the hash suffix is what keeps two
# refs sharing a 48-char prefix apart.
cache_key() {
  local raw="$1" slug hash
  if [ -z "$raw" ]; then
    echo "cache_key: refusing to key an empty ref" >&2
    return 1
  fi
  slug=$(printf '%s' "$raw" | tr -c 'A-Za-z0-9._-' '-' | sed 's/-\{2,\}/-/g; s/^-//; s/-$//')
  hash=$(printf '%s' "$raw" | sha1sum | cut -c1-8)
  printf '%s-%s' "${slug:0:48}" "$hash"
}

target_dir_for() { printf '%s/target-%s' "$1" "$2"; }
snapshot_dir_for() { printf '%s/snapshot-%s' "$1" "$2"; }

# ---------------------------------------------------------------------------
# Disk accounting
# ---------------------------------------------------------------------------

# Always prints a number. A directory we cannot read measures as 0 rather than
# as the empty string, which would otherwise be spliced into usage_gb's awk
# program and make it a syntax error at the exact moment something is already
# going wrong.
usage_kb() {
  local kb=""
  [ -d "$1" ] && kb=$(du -sk "$1" 2>/dev/null | awk '{print $1}')
  printf '%s' "${kb:-0}"
  return 0
}

usage_gb() {
  awk "BEGIN { printf \"%.1f\", $(usage_kb "$1") / 1024 / 1024 }"
}

# df -kP: portable POSIX one-line-per-fs output; columns are 1k-blocks total,
# used, available, capacity%, mounted-on. Prints "<total_kb> <free_kb>".
#
# CACHE_DF_OVERRIDE exists for the selftests: a scratch tmpdir on the test
# host's real filesystem won't sit below an arbitrary threshold on demand, and
# the eviction passes are precisely what needs testing under pressure.
read_df() {
  if [ -n "${CACHE_DF_OVERRIDE:-}" ]; then printf '%s\n' "$CACHE_DF_OVERRIDE"; return; fi
  df -kP "$1" | awk 'NR==2 {print $2, $4}'
}

report_df() {
  local label="$1" free_kb="$2" total_kb="$3"
  awk -v l="$label" -v f="$free_kb" -v t="$total_kb" \
    'BEGIN { printf "%s %.1f GB free / %.0f GB (%.1f%%)", l, f/1048576, t/1048576, (f*100)/t }'
}

# Appends to the Actions job summary, which is read on green runs — unlike a
# ::warning:: buried in a log nobody opens. No-op outside Actions so the
# scripts still run standalone under the selftests.
summary_line() {
  [ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf '%s\n' "$1" >> "$GITHUB_STEP_SUMMARY"
  return 0
}

# ---------------------------------------------------------------------------
# Hardlink cloning, and the part that makes it sound
# ---------------------------------------------------------------------------

# Cargo's own target-dir lock files (.cargo-lock, .cargo-build-lock,
# .cargo-artifact-lock) are zero-byte files it opens and flock(2)s IN PLACE
# for the duration of a build — it never truncates-and-renames them the way it
# does real artifacts. `cp -al` leaves the clone's copy hardlinked to the same
# inode as the source's, and flock() contention is inode-based, not
# path-based, so a build in the clone and a build in the source would
# serialize on one mutex.
#
# The glob deliberately reaches past the three observed names so a lock file
# added by a future Cargo version is swept too; nothing else in a target dir
# is named `.cargo-*lock*`. Cargo recreates whichever it needs as a fresh,
# unshared inode the next time it opens the directory, at no cost.
strip_cargo_locks() {
  find "$1" -type f -name '.cargo-*lock*' -delete 2>/dev/null || true
  return 0
}

# Replaces a subtree with a real (non-hardlinked) copy of itself, in place.
#
# Staged through a sibling temp path and swapped with `mv -T` rather than
# copied over the original file-by-file: the copy is a fresh tree of fresh
# inodes, so nothing in it can alias the source it was cloned from. Callers
# only ever run this against a staging directory nothing else can see yet
# (see hardlink_clone_into's contract), so the brief window where the path is
# absent is not observable.
#
# Returns non-zero if the copy or either rename failed. That status is
# load-bearing: a failed unshare leaves the staging tree still aliasing its
# source, which is the exact corruption `unshare_mutable_paths` exists to
# prevent, so it must abort the clone rather than be swallowed.
unshare_subtree() {
  local d="$1" tmp
  [ -d "$d" ] || return 0
  tmp="${d}.unshare.$$"
  rm -rf "$tmp"
  cp -a "$d" "$tmp" || { rm -rf "$tmp"; return 1; }
  rm -rf "$d" || { rm -rf "$tmp"; return 1; }
  mv -T "$tmp" "$d" || return 1
  return 0
}

_unshare_files() {
  # `-links +1` restricts the work to files that are actually shared, which
  # makes this idempotent and near-free on an already-unshared tree.
  #
  # The inner shell propagates a failure of any individual copy-and-rename out
  # through xargs (which exits 123 if any invocation exits 1-125), so a
  # partially-unshared tree is reported rather than silently accepted.
  find "$@" -links +1 -print0 2>/dev/null |
    xargs -0 -r -n 64 bash -c 'rc=0; for f; do cp -p -- "$f" "$f.unshare.$$" && mv -f -- "$f.unshare.$$" "$f" || rc=1; done; exit $rc' _
}

# THE load-bearing function of this whole design.
#
# A hardlink clone is only safe if every write the clone's build performs
# lands on a NEW inode, leaving the source's data untouched. That is true for
# compilation artifacts — rustc and the linker replace `deps/*.rlib`,
# `*.rmeta`, and binaries rather than truncating them in place — and it is
# NOT true for the metadata Cargo and build scripts write with a plain
# truncating write. Measured directly (Linux, ext4, cargo 1.9x nightly:
# `cp -al` a warm target dir, change a source file, build in the clone, diff
# the source) the following files in the SOURCE were mutated through the
# shared inode:
#
#   <profile>/.fingerprint/<unit>/dep-<target>   (only under
#                                                CARGO_UNSTABLE_CHECKSUM_FRESHNESS,
#                                                where this file carries the
#                                                per-source blake3 checksums)
#   <profile>/build/<pkg>/output, root-output   (Cargo build-script metadata)
#   <profile>/build/<pkg>/out/**                (whatever the build script
#                                                writes into OUT_DIR — build
#                                                scripts overwhelmingly use a
#                                                plain fs::write)
#   <profile>/deps/*.d, <profile>/*.d           (Cargo's post-processed
#                                                dep-info)
#
# The checksum-freshness case is not a cosmetic one. Reproduced end to end:
# branch B clones base's cache, builds its own content, and thereby rewrites
# base's `dep-<target>` to describe B's sources while base's cache still holds
# the artifact built from base's sources. When B merges and base next builds,
# Cargo reads that dep file, finds the checksums match the (now merged)
# sources, reports `Fresh`, and reuses a binary built from the PRE-merge code.
# That is silent stale-artifact reuse — a wrong answer, not a slow one.
#
# So: hardlink the artifacts (the GB), real-copy the metadata (the MB).
# Measured on a 6.9 GB Bevy workspace target dir, the unshared set is
# .fingerprint 22 MB + build/ 237 MB + a handful of dep-info files — about
# 3.7% of the tree, against 100% for a plain `cp -a`.
#
# `incremental/` is deliberately left shared: rustc writes each incremental
# session to a fresh `s-*-working` directory and finalises it with a rename,
# and garbage-collects old sessions by unlinking directory entries — neither
# of which mutates a shared inode. CI should still set CARGO_INCREMENTAL=0,
# for size rather than correctness.
unshare_mutable_paths() {
  local root="$1" d
  [ -d "$root" ] || return 0
  # The list is materialised in full before anything is replaced: each
  # replacement deletes and recreates a directory, and a live `find` walk over
  # a tree being mutated underneath it is a needless hazard. `-prune` keeps a
  # match's own contents out of the list.
  local -a dirs=()
  mapfile -t dirs < <(find "$root" -type d \( -name .fingerprint -o -name build \) -prune -print 2>/dev/null)
  for d in "${dirs[@]}"; do
    [ -n "$d" ] || continue
    unshare_subtree "$d" || {
      echo "::error::unshare_mutable_paths: failed to unshare ${d}" >&2
      return 1
    }
  done
  _unshare_files "$root" -type f -name '*.d' || {
    echo "::error::unshare_mutable_paths: failed to unshare dep-info files under ${root}" >&2
    return 1
  }
  _unshare_files "$root" -maxdepth 3 -type f -name '.rustc_info.json' || {
    echo "::error::unshare_mutable_paths: failed to unshare .rustc_info.json under ${root}" >&2
    return 1
  }
  return 0
}

# ---------------------------------------------------------------------------
# Reader markers: the consume side's half of the seed-vs-republish interlock
# ---------------------------------------------------------------------------
#
# A hardlink clone reads its source over many seconds. The publish side
# rotates a snapshot with two renames and then unlinks the generation it
# rotated away — and unlinking entries out from under an in-flight directory
# walk is what produces a SILENTLY truncated clone: `cp -al` reports the
# entries it manages to stat, and simply never sees a subdirectory that was
# unlinked before it read the parent's listing. Exit status alone does not
# catch that case.
#
# So the two sides interlock through a marker file, and the ordering is what
# makes it sound rather than probabilistic:
#
#   Consumer:  create .reading-<snap>-<tag>  ->  stat <snap>  ->  cp -al
#   Publisher: mv <snap> aside  ->  mv new into place  ->  scan for markers
#              ->  unlink the rotated-away generation
#
# If a consumer's `stat` resolved to the OLD generation, that stat happened
# before the publisher's first rename, so its marker — created strictly
# earlier still — was already on disk before the publisher's scan, which
# happens strictly after that rename. The publisher therefore cannot miss it.
# A consumer that creates its marker after the scan necessarily resolves the
# path to the NEW generation, which is not the one being unlinked.
#
# The wait is bounded (CACHE_READ_GRACE_SECONDS). Exceeding it does not force
# the unlink: reclamation of that generation is DEFERRED to a later publish
# instead. The residual is therefore disk, not correctness.
CACHE_READ_GRACE_SECONDS="${CACHE_READ_GRACE_SECONDS:-300}"
# A marker older than this belongs to a job the runner killed before it could
# clean up. Honouring one forever would let a crashed job pin an entire
# snapshot generation on disk permanently.
#
# RAISING THIS IS A CROSS-REPO CHANGE, and the drift is not symmetric.
# daniel/gitdan's ci-cache-reclaim.sh mirrors this value as
# CI_CACHE_READER_STALE_SECONDS (and CI_CACHE_LEFTOVER_MIN_AGE_SECONDS beside
# it), and its copies must be GREATER THAN OR EQUAL TO this one. Raise this for
# longer jobs while that one stays at 7200 and the arbiter reads a marker whose
# owner still considers it live as stale, then deletes the tree under an
# in-flight clone; its minimum-age guard does not back-stop that, because a
# clone holding a three-hour-old marker has a staging tree roughly three hours
# old too, so both of its guards pass. Raise theirs first. Lowering this one
# needs no coordination at all: the arbiter then defers a reclamation this side
# would already have permitted, which costs disk and not correctness.
CACHE_READ_STALE_SECONDS="${CACHE_READ_STALE_SECONDS:-7200}"

# `.reading-<source>-<tag>` is a contract name, not a private one: the
# host-level arbiter reads these to tell a live clone from an abandoned one,
# and never deletes one. See the cross-repo contract at the top of this file
# before changing the spelling.
reader_marker_path() { printf '%s/.reading-%s-%s' "$1" "$2" "$3"; }

reader_lock_acquire() {
  date +%s > "$(reader_marker_path "$1" "$2" "$3")" 2>/dev/null || true
  return 0
}

reader_lock_release() {
  rm -f "$(reader_marker_path "$1" "$2" "$3")" 2>/dev/null || true
  return 0
}

# Prints the number of live readers of <source-name> under <marker-root>, and
# sweeps markers past the staleness threshold as it goes.
live_reader_count() {
  local root="$1" name="$2" now marker age n=0
  now=$(date +%s)
  for marker in "$root"/.reading-"$name"-*; do
    [ -e "$marker" ] || continue
    age=$(( now - $(stat -c '%Y' "$marker" 2>/dev/null || echo "$now") ))
    if [ "$age" -lt "$CACHE_READ_STALE_SECONDS" ]; then
      n=$((n + 1))
    else
      echo "readers: sweeping stale marker $(basename "$marker") (${age}s old > ${CACHE_READ_STALE_SECONDS}s)" >&2
      rm -f "$marker" 2>/dev/null || true
    fi
  done
  printf '%s' "$n"
  return 0
}

# Blocks until nothing is reading <source-name>, or until the grace period
# expires. Returns 0 when drained, 1 on timeout — the caller decides what to
# do with a timeout, and in this codebase that decision is always "defer the
# unlink", never "unlink anyway".
wait_for_readers() {
  local root="$1" name="$2" grace="${3:-$CACHE_READ_GRACE_SECONDS}" deadline n waited=0
  deadline=$(( $(date +%s) + grace ))
  while :; do
    n=$(live_reader_count "$root" "$name")
    [ "$n" -eq 0 ] && {
      [ "$waited" -gt 0 ] && echo "readers: ${name} drained after ${waited}s"
      return 0
    }
    if [ "$(date +%s)" -ge "$deadline" ]; then
      echo "readers: ${n} job(s) still reading ${name} after ${grace}s" >&2
      return 1
    fi
    [ "$waited" = 0 ] && echo "readers: waiting for ${n} in-flight clone(s) of ${name} (grace ${grace}s)"
    sleep 1
    waited=$((waited + 1))
  done
}

# ---------------------------------------------------------------------------
# The clone itself
# ---------------------------------------------------------------------------

# Number of times a torn clone is retried before the caller is failed. A tear
# means the source changed identity or lost entries mid-walk, which is a
# transient condition by definition — the publisher that caused it has already
# put a complete new generation at the same path — so one retry almost always
# suffices; the rest are headroom.
CACHE_CLONE_ATTEMPTS="${CACHE_CLONE_ATTEMPTS:-4}"

_tree_entries() {
  local n
  n=$(find "$1" -mindepth 1 2>/dev/null | wc -l) || n=0
  printf '%s' "$n"
  return 0
}

_dir_inode() {
  stat -c '%i' "$1" 2>/dev/null || printf 'missing'
  return 0
}

write_cache_lock() {
  local dir="$1" id="$2"
  [ -n "$id" ] || return 0
  [ -d "$dir" ] || return 0
  date +%s > "${dir}/.ci-lock-${id}" 2>/dev/null || true
  return 0
}

# Hardlink-clones SRC to a staging path, sanitises it, and publishes it to DST
# with a single atomic rename.
#
#   hardlink_clone_into <src> <dst> <tag> [lock-id]
#
# The staging + rename is what closes the concurrent-seed race structurally
# rather than by runner topology: a second job sharing this cache key either
# sees DST absent (and stages its own clone, losing the rename harmlessly) or
# sees it complete. There is no observable half-populated state, because a
# directory rename is atomic and DST is never written through.
#
# The rename is necessary but NOT sufficient, and that gap is what this
# function's retry loop closes. An atomic rename of a TRUNCATED tree publishes
# a truncated tree atomically. Three things can truncate one:
#
#   * `cp -al` failing partway (a source entry vanished after readdir listed
#     it) — caught by checking its exit status, which is why that status is
#     read into a variable here rather than left to an ambient `set -e` the
#     CALL SITES suppress anyway by invoking this function as a condition;
#   * `cp -al` succeeding while having silently never seen a subtree that was
#     unlinked before it read the parent's listing — caught only by the entry
#     count, since there is no error to report;
#   * the source being replaced wholesale mid-walk, so the clone splices two
#     generations — caught by comparing the source directory's inode before
#     and after.
#
# All three are verified on every attempt and a failing one restarts the
# clone; a tree that fails the last attempt is deleted and reported, never
# renamed into place. Combined with the reader marker (held across the copy,
# which is what stops the publish side unlinking underneath it in the first
# place), a partial tree cannot reach DST.
#
# `lock-id`, when given, writes this job's cache lock INTO the staging tree so
# the directory already carries it the instant it appears under its final
# name. Acquiring the lock after the rename would leave a freshly seeded
# directory momentarily unlocked and therefore evictable by a concurrent job's
# prune pass.
#
# Returns 0 if this caller's clone won the rename, 1 if another caller got
# there first (the staging copy is discarded; DST is already valid), and 2 if
# the source could not be cloned consistently at all.
hardlink_clone_into() {
  local src="$1" dst="$2" tag="$3" lock_id="${4:-}"
  local parent tmp src_name attempt cp_rc n_before n_after i_before i_after

  if [ ! -d "$src" ]; then
    echo "::error::clone: source ${src} does not exist" >&2
    return 2
  fi

  parent=$(dirname "$dst")
  src_name=$(basename "$src")
  # `.stage-<tag>` is a contract name (see the top of this file): a job killed
  # between the copy below and the rename at the end strands this tree, and the
  # only thing that ever reclaims one is the host-level arbiter, by this exact
  # prefix.
  tmp="${parent}/.stage-${tag}"

  attempt=1
  while : ; do
    rm -rf "$tmp"
    # Marker first, then the identity read, then the copy — see the ordering
    # proof in the reader-marker section above; swapping the first two lines
    # is what would reintroduce the race.
    reader_lock_acquire "$parent" "$src_name" "$tag"
    i_before=$(_dir_inode "$src")
    n_before=$(_tree_entries "$src")
    cp_rc=0
    cp -al "$src" "$tmp" || cp_rc=$?
    n_after=$(_tree_entries "$tmp")
    i_after=$(_dir_inode "$src")
    reader_lock_release "$parent" "$src_name" "$tag"

    if [ "$cp_rc" -eq 0 ] && [ "$i_before" != missing ] && [ "$i_before" = "$i_after" ] \
       && [ "$n_after" -eq "$n_before" ]; then
      break
    fi

    echo "::warning::clone: attempt ${attempt}/${CACHE_CLONE_ATTEMPTS} of ${src_name} was torn (cp rc=${cp_rc}, ${n_after}/${n_before} entries, source inode ${i_before} -> ${i_after}) — discarding and retrying" >&2
    rm -rf "$tmp"
    if [ "$attempt" -ge "$CACHE_CLONE_ATTEMPTS" ]; then
      echo "::error::clone: ${src} could not be read consistently in ${CACHE_CLONE_ATTEMPTS} attempts — refusing to publish a partial tree at ${dst}" >&2
      return 2
    fi
    attempt=$((attempt + 1))
    sleep 1
  done

  strip_cargo_locks "$tmp"
  rm -f "$tmp"/.cache-last-used "$tmp"/.ci-lock-* 2>/dev/null || true
  if ! unshare_mutable_paths "$tmp"; then
    echo "::error::clone: could not privately own the mutable paths of ${dst} — discarding the staging tree rather than publishing one that aliases ${src}" >&2
    rm -rf "$tmp"
    return 2
  fi
  write_cache_lock "$tmp" "$lock_id"

  if mv -T "$tmp" "$dst" 2>/dev/null; then
    return 0
  fi
  rm -rf "$tmp"
  return 1
}