From 513364609d0e56b919bcc44163614b04fa239621 Mon Sep 17 00:00:00 2001 From: Ayush <63203492+macayu17@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:15:13 +0530 Subject: [PATCH] pyenv-binary: install published CPython packages (#3567) --- plugins/pyenv-binary/README.md | 28 +++- .../pyenv-binary/libexec/pyenv-binary-install | 124 +++++++++++++++ .../pyenv-binary/libexec/pyenv-binary-package | 4 +- .../pyenv-binary/libexec/pyenv-binary-save | 19 ++- .../share/pyenv-binary/versions/3.14 | 3 + plugins/pyenv-binary/test/install.bats | 144 ++++++++++++++++++ plugins/pyenv-binary/test/package.bats | 15 ++ plugins/pyenv-binary/test/save.bats | 24 ++- 8 files changed, 351 insertions(+), 10 deletions(-) create mode 100755 plugins/pyenv-binary/libexec/pyenv-binary-install create mode 100644 plugins/pyenv-binary/share/pyenv-binary/versions/3.14 create mode 100644 plugins/pyenv-binary/test/install.bats diff --git a/plugins/pyenv-binary/README.md b/plugins/pyenv-binary/README.md index e88537f4..14004dd5 100644 --- a/plugins/pyenv-binary/README.md +++ b/plugins/pyenv-binary/README.md @@ -19,6 +19,20 @@ by the `xz` package). ## Commands +### `pyenv binary install ` + +Installs a matching published package from pyenv.github.io under ``. +The command selects by OS, architecture and distribution; on macOS, an archive +built on an older major release can also be used. Available packages and their +definition checksums are recorded in `share/pyenv-binary/versions`. Add an entry +there when publishing a new package. The definition checks its system-library +requirements when an `ldconfig` cache is available; otherwise, it warns and +proceeds. + +```sh +pyenv binary install 3.14.7 +``` + ### `pyenv binary package [-v|--verbose] [:] --archive-base-url ` Installs `` from source under a separate name, packages that install @@ -55,12 +69,13 @@ pyenv binary package-name 3.12.7 # 3.12.7-debian-12-x86_64 ``` -### `pyenv binary save [] [--name ]` +### `pyenv binary save [] [--name ] [--source-version ]` Packs an installed version into `-.tar.xz` (relative paths) and writes `-.meta` describing the build platform (OS, arch, distro and libc version) and the system libraries the build links against. Use -`--name` to set a different base name for both files. +`--name` to set a different base name for both files. `--source-version` records +the source version when the installed version has a different name. ```sh pyenv binary save 3.12.7 ./dist @@ -87,11 +102,12 @@ pyenv binary generate-installer ./dist/3.12.7-linux-x86_64.meta \ pyenv install 3.12.7-linux-x86_64 ``` -### `pyenv binary relocate ` +### `pyenv binary relocate []` Rewrites the rpaths of a Python tree unpacked into `` so the interpreter and its extension modules load the bundled libraries from there rather than from -the path the archive was built at. Uses `patchelf`. The generated definition -calls this; you rarely run it by hand. +the path the archive was built at. Linux and FreeBSD use `patchelf`; macOS +requires the original ``. The generated definition calls this; +you rarely run it by hand. -Relocation is implemented for Linux; macOS is not wired up yet. +Relocation is implemented for Linux, FreeBSD and macOS. diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-install b/plugins/pyenv-binary/libexec/pyenv-binary-install new file mode 100755 index 00000000..80c73b87 --- /dev/null +++ b/plugins/pyenv-binary/libexec/pyenv-binary-install @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# +# Summary: Install a published binary package +# +# Usage: pyenv binary install +# +# Selects a compatible package from pyenv.github.io and installs it under +# . Ordinary `pyenv install' is unchanged. +# +set -e +[ -n "$PYENV_DEBUG" ] && set -x + +if [ "$1" = "--complete" ]; then + exit +fi + +if [ "$#" -ne 1 ]; then + pyenv-help --usage binary-install >&2 + exit 1 +fi + +case "$1" in +*/* | .. | . | *[[:cntrl:]]* ) + echo "pyenv-binary: invalid version name \`$1'" >&2 + exit 1 + ;; +esac + +requested="$1" +os="$(uname -s)" +arch="$(uname -m)" + +case "$os" in +Linux ) + if [ -r /etc/os-release ]; then + distro="$(. /etc/os-release && printf '%s %s' "${ID:-}" "${VERSION_ID:-}")" + elif type -p lsb_release >/dev/null; then + distro="$(lsb_release -si) $(lsb_release -sr)" + fi + distro="$(printf '%s' "$distro" | tr '[:upper:]' '[:lower:]')" + ;; +Darwin ) + distro="macos $(sw_vers -productVersion)" + ;; +esac + +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT +base_url="https://pyenv.github.io/pythons/binaries" +manifest_dir="$(dirname "${BASH_SOURCE[0]}")/../share/pyenv-binary/versions" +case "$requested" in +*.*.* ) manifests=( "$manifest_dir/${requested%.*}" ) ;; +*.* ) manifests=( "$manifest_dir/$requested" ) ;; +* ) manifests=( "$manifest_dir/$requested".* ) ;; +esac + +matches="$tmpdir/matches" +: > "$matches" +for manifest in "${manifests[@]}"; do + [ -f "$manifest" ] || continue + while IFS=$'\t' read -r source_version name target_os target_arch target_distro sha256; do + case "$source_version" in + "$requested" | "$requested".* ) ;; + * ) continue ;; + esac + [[ "$source_version" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]] || continue + [ "$target_os" = "$os" ] && [ "$target_arch" = "$arch" ] || continue + case "$name" in + "" | . | .. | *[!A-Za-z0-9._-]* ) continue ;; + esac + case "$os" in + Linux ) [ "$(printf '%s' "$target_distro" | tr '[:upper:]' '[:lower:]')" = "$distro" ] || continue ;; + Darwin ) + target_version="${target_distro#macos }" + [ "$target_version" != "$target_distro" ] || continue + host_version="${distro#macos }" + [[ "$target_version" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]] || continue + [[ "$host_version" =~ ^[0-9]+(\.[0-9]+){1,2}$ ]] || continue + IFS=. read -r target_major target_minor target_patch <<< "$target_version" + IFS=. read -r host_major host_minor host_patch <<< "$host_version" + target_patch="${target_patch:-0}" + host_patch="${host_patch:-0}" + [ "$host_major" -ge "$target_major" ] || continue + if [ "$host_major" -eq "$target_major" ]; then + [ "$host_minor" -ge "$target_minor" ] || continue + if [ "$host_minor" -eq "$target_minor" ]; then + [ "$host_patch" -ge "$target_patch" ] || continue + fi + fi + ;; + * ) continue ;; + esac + printf '%s\t%s\t%s\n' "$source_version" "$name" "$sha256" >> "$matches" + done < "$manifest" +done + +selection="$(sort -t. -k1,1nr -k2,2nr -k3,3nr "$matches" | head -n1)" +IFS=$'\t' read -r version entry expected_sha <<< "$selection" +if [ -z "$entry" ]; then + echo "pyenv-binary: no binary available for $requested on $os/$arch" >&2 + exit 1 +fi +if ! [[ "$expected_sha" =~ ^[0-9a-fA-F]{64}$ ]]; then + echo "pyenv-binary: invalid checksum for $entry" >&2 + exit 1 +fi +expected_sha="$(printf '%s' "$expected_sha" | tr '[:upper:]' '[:lower:]')" + +curl -fsSL -o "$tmpdir/$entry" "$base_url/$entry" +if command -v sha256sum >/dev/null 2>&1; then + actual_sha="$(sha256sum "$tmpdir/$entry")"; actual_sha="${actual_sha%% *}" +elif command -v shasum >/dev/null 2>&1; then + actual_sha="$(shasum -a 256 "$tmpdir/$entry")"; actual_sha="${actual_sha%% *}" +elif command -v openssl >/dev/null 2>&1; then + actual_sha="$(openssl dgst -sha256 "$tmpdir/$entry")"; actual_sha="${actual_sha##* }" +else + echo "pyenv-binary: need sha256sum, shasum or openssl to verify $entry" >&2 + exit 1 +fi +if [ "$actual_sha" != "$expected_sha" ]; then + echo "pyenv-binary: checksum mismatch for $entry" >&2 + exit 1 +fi +pyenv-install "$tmpdir/$entry:$version" diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-package b/plugins/pyenv-binary/libexec/pyenv-binary-package index 77479f2a..e8b698e1 100755 --- a/plugins/pyenv-binary/libexec/pyenv-binary-package +++ b/plugins/pyenv-binary/libexec/pyenv-binary-package @@ -77,7 +77,9 @@ esac # `pyenv install' puts a `:' build under versions/. pyenv-install ${verbose:+--verbose} "$spec" -pyenv-binary-save "$entry" "$PWD" --name "$entry" +source_version="$(pyenv-latest -f -k "${spec%%:*}")" +source_version="${source_version##*/}" +pyenv-binary-save "$entry" "$PWD" --name "$entry" --source-version "$source_version" pyenv-binary-generate-installer "${entry}.meta" \ --archive-url "${archive_base_url%/}/${entry}.tar.xz" -o "$entry" diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-save b/plugins/pyenv-binary/libexec/pyenv-binary-save index 444c02cc..2793ea26 100755 --- a/plugins/pyenv-binary/libexec/pyenv-binary-save +++ b/plugins/pyenv-binary/libexec/pyenv-binary-save @@ -2,7 +2,7 @@ # # Summary: Save an installed Python version as a relocatable archive # -# Usage: pyenv binary save [] [--name ] +# Usage: pyenv binary save [] [--name ] [--source-version ] # # Packs an installed version into a relocatable .tar.xz (relative paths) and # writes a metadata file listing the build platform and the system libraries @@ -12,6 +12,7 @@ # Where to write the archive and metadata (default: `.'). # --name Use as the archive and metadata base name instead of # -. +# --source-version Record the version used to build a renamed install. # set -e [ -n "$PYENV_DEBUG" ] && set -x @@ -19,18 +20,23 @@ set -e # Provide pyenv completions if [ "$1" = "--complete" ]; then echo --name + echo --source-version exec pyenv-versions --bare fi version="" output_dir="" package_name="" +source_version="" while [ $# -gt 0 ]; do case "$1" in --name ) [ $# -ge 2 ] && [ -n "$2" ] || { echo "pyenv-binary: --name needs a value" >&2; exit 1; } package_name="$2"; shift 2 ;; + --source-version ) + [ $# -ge 2 ] && [ -n "$2" ] && [ "${2#-}" = "$2" ] || { echo "pyenv-binary: --source-version needs a value" >&2; exit 1; } + source_version="$2"; shift 2 ;; -* ) echo "pyenv-binary: unknown option \`$1'" >&2; exit 1 ;; * ) @@ -49,7 +55,7 @@ done output_dir="${output_dir:-$PWD}" if [ -z "$version" ]; then - echo "Usage: pyenv binary save [] [--name ]" >&2 + echo "Usage: pyenv binary save [] [--name ] [--source-version ]" >&2 exit 1 fi @@ -69,6 +75,14 @@ case "$package_name" in ;; esac +source_version="${source_version:-$version}" +case "$source_version" in +*/* | .. | . | *[[:cntrl:]]* ) + echo "pyenv-binary: invalid source version \`${source_version}'" >&2 + exit 1 + ;; +esac + prefix="${PYENV_ROOT}/versions/${version}" if [ ! -d "${prefix}/bin" ]; then echo "pyenv-binary: version \`${version}' is not installed" >&2 @@ -142,6 +156,7 @@ tar -C "$(dirname "$prefix")" -cJf "${output_dir}/${archive}" "$(basename "$pref { echo "# pyenv-binary metadata" echo "version=${version}" + echo "source_version=${source_version}" echo "os=${os}" echo "arch=${arch}" echo "platform=${platform}" diff --git a/plugins/pyenv-binary/share/pyenv-binary/versions/3.14 b/plugins/pyenv-binary/share/pyenv-binary/versions/3.14 new file mode 100644 index 00000000..bac10ce1 --- /dev/null +++ b/plugins/pyenv-binary/share/pyenv-binary/versions/3.14 @@ -0,0 +1,3 @@ +source_version entry os arch distro sha256 +3.14.7 3.14.7-macos-15-arm64 Darwin arm64 macos 15.7.9 8a8ff86b520c3b5da00b49e04a5345b7a28f2f9e8da5a9ae43b8ac309e8f3e55 +3.14.7 3.14.7-ubuntu-24.04-x86_64 Linux x86_64 ubuntu 24.04 e63dbcb981ec9ba83a2bc9b4d8b369620cec4aa45394559d719c5606ee9ff718 diff --git a/plugins/pyenv-binary/test/install.bats b/plugins/pyenv-binary/test/install.bats new file mode 100644 index 00000000..367e7b4f --- /dev/null +++ b/plugins/pyenv-binary/test/install.bats @@ -0,0 +1,144 @@ +#!/usr/bin/env bats + +load test_helper + +host_distro() { + if [ -r /etc/os-release ]; then + . /etc/os-release + printf '%s %s' "$ID" "$VERSION_ID" | tr '[:upper:]' '[:lower:]' + else + printf '%s %s' "$(lsb_release -si)" "$(lsb_release -sr)" | tr '[:upper:]' '[:lower:]' + fi +} + +use_manifest() { + local plugin="${BATS_TEST_TMPDIR}/plugin" + mkdir -p "$plugin/libexec" "$plugin/share/pyenv-binary/versions" + cp "${BATS_TEST_DIRNAME}/../libexec/pyenv-binary-install" "$plugin/libexec/" + PATH="$plugin/libexec:$PATH" + printf 'source_version\tentry\tos\tarch\tdistro\tsha256\n' > "$plugin/share/pyenv-binary/versions/3.14" + definition_sha="4c4ed1afbfdaa1e4c3bf7bbb82d730cecb7e384da91eea4f3cc093fd545524d6" +} + +stub_downloads() { + create_stub curl <<'STUB' +output="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o ) output="$2"; shift ;; + http* ) url="$1" ;; + esac + shift +done +case "$url" in +*/3.14.7-ubuntu-24.04-x86_64 | */3.14.7-macos-15-arm64 | */3.14.10-ubuntu-24.04-x86_64 ) printf definition > "$output" ;; +* ) exit 1 ;; +esac +STUB +} + +@test "rejects a modified published definition" { + create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 24.04;; esac' + use_manifest + printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%064d\n' \ + "$(host_distro)" 0 >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + stub_downloads + create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac' + create_stub pyenv-install 'echo installed' + + run pyenv-binary-install 3.14.7 + assert_failure + [[ "$output" == *'checksum mismatch'* ]] +} + +@test "completion does not offer source-only versions" { + create_stub pyenv-install 'echo 3.14.7' + + run pyenv-binary-install --complete + assert_success "" +} + +@test "installs a matching published binary with an uppercase checksum" { + create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 24.04;; esac' + use_manifest + local uppercase_sha="$(printf '%s' "$definition_sha" | tr '[:lower:]' '[:upper:]')" + printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \ + "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \ + "$(host_distro | tr '[:lower:]' '[:upper:]')" "$uppercase_sha" \ + >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + stub_downloads + create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac' + create_stub pyenv-install <<'STUB' +definition="${1%:*}" +printf '%s:%s\n' "${definition##*/}" "${1##*:}" +cat "$definition" +STUB + + run pyenv-binary-install 3.14 + assert_success "3.14.7-ubuntu-24.04-x86_64:3.14.7 +definition" +} + +@test "a version prefix selects the latest published binary" { + create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 24.04;; esac' + use_manifest + printf '3.14.10\t3.14.10-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \ + "$(host_distro)" "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \ + "$(host_distro)" "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + stub_downloads + create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac' + create_stub pyenv-install 'echo "${1##*:}"' + + run pyenv-binary-install 3.14 + assert_success "3.14.10" +} + +@test "fails when no published binary matches the host" { + use_manifest + printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \ + "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac' + + run pyenv-binary-install 3.14.7 + assert_failure "pyenv-binary: no binary available for 3.14.7 on Linux/x86_64" +} + +@test "does not select a package for another Linux release" { + create_stub lsb_release 'case "$1" in -si) echo Ubuntu;; -sr) echo 22.04;; esac' + use_manifest + local other_release="ubuntu 24.04" + [ "$(host_distro)" = "$other_release" ] && other_release="ubuntu 22.04" + printf '3.14.7\t3.14.7-ubuntu-24.04-x86_64\tLinux\tx86_64\t%s\t%s\n' \ + "$other_release" "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac' + + run pyenv-binary-install 3.14.7 + assert_failure "pyenv-binary: no binary available for 3.14.7 on Linux/x86_64" +} + +@test "uses a macOS binary built on an older release" { + use_manifest + printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \ + "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + stub_downloads + create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_stub sw_vers 'echo 26.6.2' + create_stub pyenv-install 'echo "${1##*:}"' + + run pyenv-binary-install 3.14.7 + assert_success "3.14.7" +} + +@test "rejects a macOS binary built on a newer patch release" { + use_manifest + printf '3.14.7\t3.14.7-macos-15-arm64\tDarwin\tarm64\tmacos 15.7.9\t%s\n' \ + "$definition_sha" >> "$BATS_TEST_TMPDIR/plugin/share/pyenv-binary/versions/3.14" + create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_stub sw_vers 'echo 15.7.8' + create_stub pyenv-install 'echo installed' + + run pyenv-binary-install 3.14.7 + assert_failure "pyenv-binary: no binary available for 3.14.7 on Darwin/arm64" +} diff --git a/plugins/pyenv-binary/test/package.bats b/plugins/pyenv-binary/test/package.bats index 40e875a9..46f85a2e 100644 --- a/plugins/pyenv-binary/test/package.bats +++ b/plugins/pyenv-binary/test/package.bats @@ -68,10 +68,24 @@ pyenv-install --list --bare assert [ -d "${PYENV_ROOT}/versions/3.12.7-debian-12-x86_64" ] assert [ -f "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64.tar.xz" ] assert [ -f "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64.meta" ] + run grep '^source_version=' "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64.meta" + assert_success "source_version=3.12.7" run grep '^ARCHIVE_URL=' "${BATS_TEST_TMPDIR}/3.12.7-debian-12-x86_64" assert_success "ARCHIVE_URL=http://example.com/binaries/3.12.7-debian-12-x86_64.tar.xz" } +@test "records the definition name when packaging from a file" { + stub_build_environment + cd "${BATS_TEST_TMPDIR}" + printf 'install_package "foo"\n' > custom-python + + run pyenv-binary-package "${BATS_TEST_TMPDIR}/custom-python:custom-build" \ + --archive-base-url http://example.com/binaries + assert_success + run grep '^source_version=' custom-build.meta + assert_success "source_version=custom-python" +} + @test "rejects an entry name containing a slash" { run pyenv-binary-package "3.12.7:foo/bar" --archive-base-url http://x/b assert_failure "pyenv-binary: invalid entry name \`foo/bar'" @@ -84,6 +98,7 @@ pyenv-install --list --bare @test "packages on macOS" { create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_stub pyenv-latest 'echo 3.12.7' create_stub pyenv-install 'echo install' create_stub pyenv-binary-save 'echo save' create_stub pyenv-binary-generate-installer 'echo generate-installer' diff --git a/plugins/pyenv-binary/test/save.bats b/plugins/pyenv-binary/test/save.bats index 62e0fe83..d9090ec0 100644 --- a/plugins/pyenv-binary/test/save.bats +++ b/plugins/pyenv-binary/test/save.bats @@ -12,7 +12,7 @@ platform() { @test "fails with no version given" { run pyenv-binary-save - assert_failure "Usage: pyenv binary save [] [--name ]" + assert_failure "Usage: pyenv binary save [] [--name ] [--source-version ]" } @test "fails for a version that is not installed" { @@ -55,6 +55,28 @@ platform() { assert_success "archive=custom.tar.xz" } +@test "records the source version for a renamed build" { + create_version "3.12.7-ubuntu-24.04-x86_64" + local out="${BATS_TEST_TMPDIR}/dist" + + run pyenv-binary-save "3.12.7-ubuntu-24.04-x86_64" "$out" \ + --name "3.12.7-ubuntu-24.04-x86_64" --source-version "3.12.7" + assert_success + run grep '^source_version=' "${out}/3.12.7-ubuntu-24.04-x86_64.meta" + assert_success "source_version=3.12.7" +} + +@test "rejects an option as the source version" { + run pyenv-binary-save "3.12.7" --source-version --name custom + assert_failure "pyenv-binary: --source-version needs a value" +} + +@test "rejects control characters in the default source version" { + run pyenv-binary-save $'3.12.7\nsource_version=other' + assert_failure + [[ "$output" == "pyenv-binary: invalid source version"* ]] +} + @test "fails when --name has no value" { run pyenv-binary-save "3.12.7" --name assert_failure "pyenv-binary: --name needs a value"