gitdan-actions/scripts/publish-snapshot.sh

#!/usr/bin/env bash
# Publish side: atomically republish this reference branch's live target dir
# as the immutable snapshot other branches seed from.
#
# Usage: publish-snapshot.sh <own-key> <cache-root> <tag>
#
# Run only after the build has already succeeded, and only on a branch that is
# a publisher (see the cargo-cache-publish action, which makes that decision).
# A red build must never overwrite a known-good snapshot: the caller's step
# ordering is the whole gate here, so keep this step last.
#
# The swap is two renames, not one, because POSIX rename() can only replace an
# EMPTY directory and a snapshot from a prior publish is not one:
#
#   1. stage the new snapshot at .stage-<tag>   (copy time is off every
#      consumer's hot path — nothing reads a staging path);
#   2. rename the current snapshot aside to .publish-old-<tag>, if present;
#   3. rename the staged snapshot into place.
#
# Step 3 is a single atomic rename onto a path now guaranteed absent, so it
# can never partially overwrite a live snapshot. A consumer landing in the
# window between 2 and 3 sees the snapshot as absent and falls through to its
# own cold-start path — a safe degrade that self-heals on its next run, not
# corruption.
#
# `rm -rf` on the old snapshot removes directory entries only, so a consumer
# that has ALREADY FINISHED cloning from it keeps every inode alive through
# its own links. That is the easy half, and on its own it is not enough: a
# consumer still WALKING the old generation has its entries unlinked out from
# under it, and `cp -al` does not report a subtree that was removed before it
# read the parent's listing. That is a silently truncated clone — the failure
# mode this script's own selftest (scenario 8) reproduces against the
# unguarded version.
#
# So the unlink is interlocked with the consume side rather than
# unconditional: after the swap, this script waits for every in-flight reader
# of this snapshot to drain (see the reader-marker ordering proof in
# cache-lib.sh) and only then reclaims the rotated-away generation. If the
# grace period expires first, reclamation is DEFERRED — the directory is left
# under `.publish-old-<key>-<tag>` and swept by a later publish once no reader
# holds it. The residual of a very slow consumer is therefore one extra
# generation of directory entries on disk, never a torn clone.
set -euo pipefail
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/cache-lib.sh"

OWN_KEY="${1:?usage: publish-snapshot.sh <own-key> <cache-root> <tag>}"
ROOT="${2:?}"; TAG="${3:?}"

SRC=$(target_dir_for "$ROOT" "$OWN_KEY")
DST=$(snapshot_dir_for "$ROOT" "$OWN_KEY")
# Keyed by cache key as well as tag, so a deferred generation can be matched
# back to the snapshot whose readers must drain before it is safe to reclaim.
OLD="${ROOT}/.publish-old-${OWN_KEY}-${TAG}"
SNAP_NAME=$(basename "$DST")
GRACE="${CACHE_READ_GRACE_SECONDS}"

if [ ! -d "$SRC" ]; then
  echo "publish: no target dir at ${SRC} — nothing to snapshot"
  exit 0
fi

# Leftovers from a run cancelled mid-swap. Scoped to this job's own tag so a
# concurrently running job's staging directory is never touched.
rm -rf "${ROOT}/.stage-${TAG}" "$OLD"

# Deferred reclamation from an earlier publish of THIS snapshot whose readers
# had not drained in time. Safe to sweep now only if nothing is reading the
# snapshot at all: a reader holds the snapshot path, not the deferred name, so
# "no readers of snapshot-<key>" is the condition that makes every deferred
# generation of it unreachable. Over-conservative by design — a reader of the
# CURRENT generation also defers the sweep to the next publish, which costs a
# directory listing, not correctness.
if [ "$(live_reader_count "$ROOT" "$SNAP_NAME")" -eq 0 ]; then
  for stale_old in "${ROOT}/.publish-old-${OWN_KEY}-"*; do
    [ -d "$stale_old" ] || continue
    echo "publish: reclaiming deferred snapshot generation $(basename "$stale_old")"
    rm -rf "$stale_old"
  done
fi

start=$(date +%s)
# The staged snapshot is hardlinked to SRC's artifacts and holds its OWN copy
# of every file Cargo rewrites in place (unshare_mutable_paths, called inside
# hardlink_clone_into). Without that, this branch's NEXT build would mutate
# the snapshot it just published — the same aliasing hazard the consume side
# closes, pointing the other way. The staging path is not the final name, so
# `hardlink_clone_into`'s rename lands on DST only after OLD is out of the way.
TMP_DST="${ROOT}/.publish-new-${TAG}"
rm -rf "$TMP_DST"
hardlink_clone_into "$SRC" "$TMP_DST" "$TAG" || {
  echo "::error::publish: failed to stage a snapshot of ${SRC}"
  exit 1
}

if [ -d "$DST" ]; then mv -T "$DST" "$OLD"; fi
mv -T "$TMP_DST" "$DST"

# The scan below happens strictly after the rename above, which is what makes
# it impossible for a consumer holding the OLD generation to be missed: such a
# consumer resolved the path before that rename, and published its marker
# before that. See cache-lib.sh's reader-marker section.
if [ -d "$OLD" ]; then
  if wait_for_readers "$ROOT" "$SNAP_NAME" "$GRACE"; then
    rm -rf "$OLD"
  else
    echo "::warning::publish: a consumer is still cloning the previous ${SNAP_NAME} after ${GRACE}s — deferring reclamation of $(basename "$OLD") rather than unlinking a tree being read"
    summary_line "- deferred reclaiming the previous \`${SNAP_NAME}\` generation (a consumer is still cloning it); it will be swept by a later publish"
  fi
fi

echo "publish: ${DST} ($(usage_gb "$DST") GB) published in $(( $(date +%s) - start ))s"
summary_line "- published cache snapshot \`$(basename "$DST")\` ($(usage_gb "$DST") GB)"