pyenv-binary: install published CPython packages (#3567)

plugins/pyenv-binary/README.md

@@ -19,6 +19,20 @@ by the `xz` package).
 
 ## Commands
 
+### `pyenv binary install <version>`
+
+Installs a matching published package from pyenv.github.io under `<version>`.
+The command selects by OS, architecture and distribution; on macOS, an archive
+built on an older major release can also be used. Available packages and their
+definition checksums are recorded in `share/pyenv-binary/versions`. Add an entry
+there when publishing a new package. The definition checks its system-library
+requirements when an `ldconfig` cache is available; otherwise, it warns and
+proceeds.
+
+```sh
+pyenv binary install 3.14.7
+```
+
 ### `pyenv binary package [-v|--verbose] <version>[:<entry>] --archive-base-url <url>`
 
 Installs `<version>` from source under a separate name, packages that install
@@ -55,12 +69,13 @@ pyenv binary package-name 3.12.7
 # 3.12.7-debian-12-x86_64
 ```
 
-### `pyenv binary save <version> [<output-dir>] [--name <name>]`
+### `pyenv binary save <version> [<output-dir>] [--name <name>] [--source-version <version>]`
 
 Packs an installed version into `<version>-<platform>.tar.xz` (relative paths)
 and writes `<version>-<platform>.meta` describing the build platform (OS, arch,
 distro and libc version) and the system libraries the build links against. Use
-`--name` to set a different base name for both files.
+`--name` to set a different base name for both files. `--source-version` records
+the source version when the installed version has a different name.
 
 ```sh
 pyenv binary save 3.12.7 ./dist
@@ -87,11 +102,12 @@ pyenv binary generate-installer ./dist/3.12.7-linux-x86_64.meta \
 pyenv install 3.12.7-linux-x86_64
 ```
 
-### `pyenv binary relocate <prefix>`
+### `pyenv binary relocate <prefix> [<build-prefix>]`
 
 Rewrites the rpaths of a Python tree unpacked into `<prefix>` so the interpreter
 and its extension modules load the bundled libraries from there rather than from
-the path the archive was built at. Uses `patchelf`. The generated definition
+the path the archive was built at. Linux and FreeBSD use `patchelf`; macOS
-calls this; you rarely run it by hand.
+requires the original `<build-prefix>`. The generated definition calls this;
+you rarely run it by hand.
 
-Relocation is implemented for Linux; macOS is not wired up yet.
+Relocation is implemented for Linux, FreeBSD and macOS.

plugins/pyenv-binary/libexec/pyenv-binary-install

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+#
+# Summary: Install a published binary package
+#
+# Usage: pyenv binary install <version>
+#
+#   Selects a compatible package from pyenv.github.io and installs it under
+#   <version>. Ordinary `pyenv install' is unchanged.
+#
+set -e
+[ -n "$PYENV_DEBUG" ] && set -x
+
+if [ "$1" = "--complete" ]; then
+  exit
+fi
+
+if [ "$#" -ne 1 ]; then
+  pyenv-help --usage binary-install >&2
+  exit 1
+fi
+
+case "$1" in
+*/* | .. | . | *[[:cntrl:]]* )
+  echo "pyenv-binary: invalid version name \`$1'" >&2
+  exit 1
+  ;;
+esac
+
+requested="$1"
+os="$(uname -s)"
+arch="$(uname -m)"
+
+case "$os" in
+Linux )
+  if [ -r /etc/os-release ]; then
+    distro="$(. /etc/os-release && printf '%s %s' "${ID:-}" "${VERSION_ID:-}")"
+  elif type -p lsb_release >/dev/null; then
+    distro="$(lsb_release -si) $(lsb_release -sr)"
+  fi
+  distro="$(printf '%s' "$distro" | tr '[:upper:]' '[:lower:]')"
+  ;;
+Darwin )
+  distro="macos $(sw_vers -productVersion)"
+  ;;
+esac
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+base_url="https://pyenv.github.io/pythons/binaries"
+manifest_dir="$(dirname "${BASH_SOURCE[0]}")/../share/pyenv-binary/versions"
+case "$requested" in
+*.*.* ) manifests=( "$manifest_dir/${requested%.*}" ) ;;
+*.* ) manifests=( "$manifest_dir/$requested" ) ;;
+* ) manifests=( "$manifest_dir/$requested".* ) ;;
+esac
+
+matches="$tmpdir/matches"
+: > "$matches"
+for manifest in "${manifests[@]}"; do
+  [ -f "$manifest" ] || continue
+  while IFS=$'\t' read -r source_version name target_os target_arch target_distro sha256; do
+    case "$source_version" in
+    "$requested" | "$requested".* ) ;;
+    * ) continue ;;
+    esac
+    [[ "$source_version" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]] || continue
+    [ "$target_os" = "$os" ] && [ "$target_arch" = "$arch" ] || continue
+    case "$name" in
+    "" | . | .. | *[!A-Za-z0-9._-]* ) continue ;;
+    esac
+    case "$os" in
+    Linux ) [ "$(printf '%s' "$target_distro" | tr '[:upper:]' '[:lower:]')" = "$distro" ] || continue ;;
+    Darwin )
+      target_version="${target_distro#macos }"
+      [ "$target_version" != "$target_distro" ] || continue
+      host_version="${distro#macos }"
+      [[ "$target_version" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]] || continue
+      [[ "$host_version" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]] || continue
+      IFS=. read -r target_major target_minor target_patch <<< "$target_version"
+      IFS=. read -r host_major host_minor host_patch <<< "$host_version"
+      target_patch="${target_patch:-0}"
+      host_patch="${host_patch:-0}"
+      [ "$host_major" -ge "$target_major" ] || continue
+      if [ "$host_major" -eq "$target_major" ]; then
+        [ "$host_minor" -ge "$target_minor" ] || continue
+        if [ "$host_minor" -eq "$target_minor" ]; then
+          [ "$host_patch" -ge "$target_patch" ] || continue
+        fi
+      fi
+      ;;
+    * ) continue ;;
+    esac
+    printf '%s\t%s\t%s\n' "$source_version" "$name" "$sha256" >> "$matches"
+  done < "$manifest"
+done
+
+selection="$(sort -t. -k1,1nr -k2,2nr -k3,3nr "$matches" | head -n1)"
+IFS=$'\t' read -r version entry expected_sha <<< "$selection"
+if [ -z "$entry" ]; then
+  echo "pyenv-binary: no binary available for $requested on $os/$arch" >&2
+  exit 1
+fi
+if ! [[ "$expected_sha" =~ ^[0-9a-fA-F]{64}$ ]]; then
+  echo "pyenv-binary: invalid checksum for $entry" >&2
+  exit 1
+fi
+expected_sha="$(printf '%s' "$expected_sha" | tr '[:upper:]' '[:lower:]')"
+
+curl -fsSL -o "$tmpdir/$entry" "$base_url/$entry"
+if command -v sha256sum >/dev/null 2>&1; then
+  actual_sha="$(sha256sum "$tmpdir/$entry")"; actual_sha="${actual_sha%% *}"
+elif command -v shasum >/dev/null 2>&1; then
+  actual_sha="$(shasum -a 256 "$tmpdir/$entry")"; actual_sha="${actual_sha%% *}"
+elif command -v openssl >/dev/null 2>&1; then
+  actual_sha="$(openssl dgst -sha256 "$tmpdir/$entry")"; actual_sha="${actual_sha##* }"
+else
+  echo "pyenv-binary: need sha256sum, shasum or openssl to verify $entry" >&2
+  exit 1
+fi
+if [ "$actual_sha" != "$expected_sha" ]; then
+  echo "pyenv-binary: checksum mismatch for $entry" >&2
+  exit 1
+fi
+pyenv-install "$tmpdir/$entry:$version"

plugins/pyenv-binary/libexec/pyenv-binary-package

@@ -77,7 +77,9 @@ esac
 
 # `pyenv install' puts a `<version>:<alias>' build under versions/<alias>.
 pyenv-install ${verbose:+--verbose} "$spec"
-pyenv-binary-save "$entry" "$PWD" --name "$entry"
+source_version="$(pyenv-latest -f -k "${spec%%:*}")"
+source_version="${source_version##*/}"
+pyenv-binary-save "$entry" "$PWD" --name "$entry" --source-version "$source_version"
 
 pyenv-binary-generate-installer "${entry}.meta" \
   --archive-url "${archive_base_url%/}/${entry}.tar.xz" -o "$entry"

plugins/pyenv-binary/libexec/pyenv-binary-save

@@ -2,7 +2,7 @@
 #
 # Summary: Save an installed Python version as a relocatable archive
 #
-# Usage: pyenv binary save <version> [<output-dir>] [--name <name>]
+# Usage: pyenv binary save <version> [<output-dir>] [--name <name>] [--source-version <version>]
 #
 #   Packs an installed version into a relocatable .tar.xz (relative paths) and
 #   writes a metadata file listing the build platform and the system libraries
@@ -12,6 +12,7 @@
 #   <output-dir>   Where to write the archive and metadata (default: `.').
 #   --name <name>  Use <name> as the archive and metadata base name instead of
 #                  <version>-<platform>.
+#   --source-version <version>  Record the version used to build a renamed install.
 #
 set -e
 [ -n "$PYENV_DEBUG" ] && set -x
@@ -19,18 +20,23 @@ set -e
 # Provide pyenv completions
 if [ "$1" = "--complete" ]; then
   echo --name
+  echo --source-version
   exec pyenv-versions --bare
 fi
 
 version=""
 output_dir=""
 package_name=""
+source_version=""
 
 while [ $# -gt 0 ]; do
   case "$1" in
   --name )
     [ $# -ge 2 ] && [ -n "$2" ] || { echo "pyenv-binary: --name needs a value" >&2; exit 1; }
     package_name="$2"; shift 2 ;;
+  --source-version )
+    [ $# -ge 2 ] && [ -n "$2" ] && [ "${2#-}" = "$2" ] || { echo "pyenv-binary: --source-version needs a value" >&2; exit 1; }
+    source_version="$2"; shift 2 ;;
   -* )
     echo "pyenv-binary: unknown option \`$1'" >&2; exit 1 ;;
   * )
@@ -49,7 +55,7 @@ done
 output_dir="${output_dir:-$PWD}"
 
 if [ -z "$version" ]; then
-  echo "Usage: pyenv binary save <version> [<output-dir>] [--name <name>]" >&2
+  echo "Usage: pyenv binary save <version> [<output-dir>] [--name <name>] [--source-version <version>]" >&2
   exit 1
 fi
 
@@ -69,6 +75,14 @@ case "$package_name" in
   ;;
 esac
 
+source_version="${source_version:-$version}"
+case "$source_version" in
+*/* | .. | . | *[[:cntrl:]]* )
+  echo "pyenv-binary: invalid source version \`${source_version}'" >&2
+  exit 1
+  ;;
+esac
+
 prefix="${PYENV_ROOT}/versions/${version}"
 if [ ! -d "${prefix}/bin" ]; then
   echo "pyenv-binary: version \`${version}' is not installed" >&2
@@ -142,6 +156,7 @@ tar -C "$(dirname "$prefix")" -cJf "${output_dir}/${archive}" "$(basename "$pref
 {
   echo "# pyenv-binary metadata"
   echo "version=${version}"
+  echo "source_version=${source_version}"
   echo "os=${os}"
   echo "arch=${arch}"
   echo "platform=${platform}"

plugins/pyenv-binary/share/pyenv-binary/versions/3.14

@@ -0,0 +1,3 @@
+source_version	entry	os	arch	distro	sha256
+3.14.7	3.14.7-macos-15-arm64	Darwin	arm64	macos 15.7.9	8a8ff86b520c3b5da00b49e04a5345b7a28f2f9e8da5a9ae43b8ac309e8f3e55
+3.14.7	3.14.7-ubuntu-24.04-x86_64	Linux	x86_64	ubuntu 24.04	e63dbcb981ec9ba83a2bc9b4d8b369620cec4aa45394559d719c5606ee9ff718

plugins/pyenv-binary/test/install.bats

@@ -0,0 +1,144 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+host_distro() {
+  if [ -r /etc/os-release ]; then
+    . /etc/os-release
+    printf '%s %s' "$ID" "$VERSION_ID" | tr '[:upper:]' '[:lower:]'
+  else
+    printf '%s %s' "$(lsb_release -si)" "$(lsb_release -sr)" | tr '[:upper:]' '[:lower:]'
+  fi
+}
+
+use_manifest() {
+  local plugin="${BATS_TEST_TMPDIR}/plugin"
+  mkdir -p "$plugin/libexec" "$plugin/share/pyenv-binary/versions"
+  cp "${BATS_TEST_DIRNAME}/../libexec/pyenv-binary-install" "$plugin/libexec/"
+  PATH="$plugin/libexec:$PATH"
+  printf 'source_version\tentry\tos\tarch\tdistro\tsha256\n' > "$plugin/share/pyenv-binary/versions/3.14"
+  definition_sha="4c4ed1afbfdaa1e4c3bf7bbb82d730cecb7e384da91eea4f3cc093fd545524d6"
+}
+
+stub_downloads() {
+  create_stub curl <<'STUB'
+output=""
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+  -o ) output="$2"; shift ;;
+  http* ) url="$1" ;;
+  esac
+  shift
+done
+case "$url" in
+*/3.14.7-ubuntu-24.04-x86_64 | */3.14.7-macos-15-arm64 | */3.14.10-ubuntu-24.04-x86_64 ) printf definition > "$output" ;;
+* ) exit 1 ;;
+esac
+STUB
+}
+
+@test "rejects a modified published definition" {
+  create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 24.04;; esac'
+  use_manifest
+  printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%064d\n' \
+    "$(host_distro)" 0 >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  stub_downloads
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+  create_stub pyenv-install 'echo installed'
+
+  run pyenv-binary-install 3.14.7
+  assert_failure
+  [[ "$output" == *'checksum mismatch'* ]]
+}
+
+@test "completion does not offer source-only versions" {
+  create_stub pyenv-install 'echo 3.14.7'
+
+  run pyenv-binary-install --complete
+  assert_success ""
+}
+
+@test "installs a matching published binary with an uppercase checksum" {
+  create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 24.04;; esac'
+  use_manifest
+  local uppercase_sha="$(printf '%s' "$definition_sha" | tr '[:lower:]' '[:upper:]')"
+  printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \
+    "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \
+    "$(host_distro | tr '[:lower:]' '[:upper:]')" "$uppercase_sha" \
+    >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  stub_downloads
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+  create_stub pyenv-install <<'STUB'
+definition="${1%:*}"
+printf '%s:%s\n' "${definition##*/}" "${1##*:}"
+cat "$definition"
+STUB
+
+  run pyenv-binary-install 3.14
+  assert_success "3.14.7-ubuntu-24.04-x86_64:3.14.7
+definition"
+}
+
+@test "a version prefix selects the latest published binary" {
+  create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 24.04;; esac'
+  use_manifest
+  printf '3.14.10\t3.14.10-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \
+    "$(host_distro)" "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \
+    "$(host_distro)" "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  stub_downloads
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+  create_stub pyenv-install 'echo "${1##*:}"'
+
+  run pyenv-binary-install 3.14
+  assert_success "3.14.10"
+}
+
+@test "fails when no published binary matches the host" {
+  use_manifest
+  printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \
+    "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+
+  run pyenv-binary-install 3.14.7
+  assert_failure "pyenv-binary: no binary available for 3.14.7 on Linux/x86_64"
+}
+
+@test "does not select a package for another Linux release" {
+  create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 22.04;; esac'
+  use_manifest
+  local other_release="ubuntu 24.04"
+  [ "$(host_distro)" = "$other_release" ] && other_release="ubuntu 22.04"
+  printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \
+    "$other_release" "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+
+  run pyenv-binary-install 3.14.7
+  assert_failure "pyenv-binary: no binary available for 3.14.7 on Linux/x86_64"
+}
+
+@test "uses a macOS binary built on an older release" {
+  use_manifest
+  printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \
+    "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  stub_downloads
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_stub sw_vers 'echo 26.6.2'
+  create_stub pyenv-install 'echo "${1##*:}"'
+
+  run pyenv-binary-install 3.14.7
+  assert_success "3.14.7"
+}
+
+@test "rejects a macOS binary built on a newer patch release" {
+  use_manifest
+  printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \
+    "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14"
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_stub sw_vers 'echo 15.7.8'
+  create_stub pyenv-install 'echo installed'
+
+  run pyenv-binary-install 3.14.7
+  assert_failure "pyenv-binary: no binary available for 3.14.7 on Darwin/arm64"
+}

plugins/pyenv-binary/test/package.bats

@@ -68,10 +68,24 @@ pyenv-install --list --bare
   assert [ -d "${PYENV_ROOT}/versions/3.12.7-debian-12-x86_64" ]
   assert [ -f "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64.tar.xz" ]
   assert [ -f "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64.meta" ]
+  run grep '^source_version=' "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64.meta"
+  assert_success "source_version=3.12.7"
   run grep '^ARCHIVE_URL=' "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64"
   assert_success "ARCHIVE_URL=http://example.com/binaries/3.12.7-debian-12-x86_64.tar.xz"
 }
 
+@test "records the definition name when packaging from a file" {
+  stub_build_environment
+  cd "${BATS_TEST_TMPDIR}"
+  printf 'install_package "foo"\n' > custom-python
+
+  run pyenv-binary-package "${BATS_TEST_TMPDIR}/custom-python:custom-build" \
+    --archive-base-url http://example.com/binaries
+  assert_success
+  run grep '^source_version=' custom-build.meta
+  assert_success "source_version=custom-python"
+}
+
 @test "rejects an entry name containing a slash" {
   run pyenv-binary-package "3.12.7:foo/bar" --archive-base-url http://x/b
   assert_failure "pyenv-binary: invalid entry name \`foo/bar'"
@@ -84,6 +98,7 @@ pyenv-install --list --bare
 
 @test "packages on macOS" {
   create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_stub pyenv-latest 'echo 3.12.7'
   create_stub pyenv-install 'echo install'
   create_stub pyenv-binary-save 'echo save'
   create_stub pyenv-binary-generate-installer 'echo generate-installer'

plugins/pyenv-binary/test/save.bats

@@ -12,7 +12,7 @@ platform() {
 
 @test "fails with no version given" {
   run pyenv-binary-save
-  assert_failure "Usage: pyenv binary save <version> [<output-dir>] [--name <name>]"
+  assert_failure "Usage: pyenv binary save <version> [<output-dir>] [--name <name>] [--source-version <version>]"
 }
 
 @test "fails for a version that is not installed" {
@@ -55,6 +55,28 @@ platform() {
   assert_success "archive=custom.tar.xz"
 }
 
+@test "records the source version for a renamed build" {
+  create_version "3.12.7-ubuntu-24.04-x86_64"
+  local out="${BATS_TEST_TMPDIR}/dist"
+
+  run pyenv-binary-save "3.12.7-ubuntu-24.04-x86_64" "$out" \
+    --name "3.12.7-ubuntu-24.04-x86_64" --source-version "3.12.7"
+  assert_success
+  run grep '^source_version=' "${out}/3.12.7-ubuntu-24.04-x86_64.meta"
+  assert_success "source_version=3.12.7"
+}
+
+@test "rejects an option as the source version" {
+  run pyenv-binary-save "3.12.7" --source-version --name custom
+  assert_failure "pyenv-binary: --source-version needs a value"
+}
+
+@test "rejects control characters in the default source version" {
+  run pyenv-binary-save $'3.12.7\nsource_version=other'
+  assert_failure
+  [[ "$output" == "pyenv-binary: invalid source version"* ]]
+}
+
 @test "fails when --name has no value" {
   run pyenv-binary-save "3.12.7" --name
   assert_failure "pyenv-binary: --name needs a value"