Merge pull request #3488 from macayu17/feat/2334-pyenv-binary-installer

pyenv-binary: add the `generate-installer` subcommand

.gitignore

@@ -11,3 +11,4 @@
 /default-packages
 .idea
 *.un~
+*.swp

Makefile

@@ -1,16 +1,21 @@
 TEST_BATS_VERSION = v1.10.0
 TEST_BASH_VERSIONS = 3.2.57 4.1.17
+
 TEST_UNIT_DOCKER_PREFIX = test-unit-docker
 TEST_UNIT_DOCKER_TARGETS = $(foreach bash,$(TEST_BASH_VERSIONS),$(addsuffix -$(bash),$(TEST_UNIT_DOCKER_PREFIX)) $(addsuffix -gnu-$(bash),$(TEST_UNIT_DOCKER_PREFIX)))
-TEST_PLUGIN_DOCKER_PREFIX = test-plugin-docker
-TEST_PLUGIN_DOCKER_TARGETS = $(foreach bash,$(TEST_BASH_VERSIONS),$(addsuffix -$(bash),$(TEST_PLUGIN_DOCKER_PREFIX)) $(addsuffix -gnu-$(bash),$(TEST_PLUGIN_DOCKER_PREFIX)))
+
+TEST_PYTHON_BUILD_DOCKER_PREFIX = test-python-build-docker
+TEST_PYTHON_BUILD_DOCKER_TARGETS = $(foreach bash,$(TEST_BASH_VERSIONS),$(addsuffix -$(bash),$(TEST_PYTHON_BUILD_DOCKER_PREFIX)) $(addsuffix -gnu-$(bash),$(TEST_PYTHON_BUILD_DOCKER_PREFIX)))
+
+TEST_BINARY_DOCKER_PREFIX = test-binary-docker
+TEST_BINARY_DOCKER_TARGETS = $(foreach bash,$(TEST_BASH_VERSIONS),$(addsuffix -$(bash),$(TEST_BINARY_DOCKER_PREFIX)) $(addsuffix -gnu-$(bash),$(TEST_BINARY_DOCKER_PREFIX)))
+
 TEST_BATS_IMAGE_PREFIX = test-pyenv-docker-image
 TEST_BATS_IMAGE_TARGETS = $(foreach bash,$(TEST_BASH_VERSIONS),$(addsuffix -$(bash),$(TEST_BATS_IMAGE_PREFIX)) $(addsuffix -gnu-$(bash),$(TEST_BATS_IMAGE_PREFIX)))
 
-.PHONY:
-test-docker: $(TEST_UNIT_DOCKER_PREFIX) $(TEST_PLUGIN_DOCKER_PREFIX)
+.PHONY: test-docker
+test-docker: $(TEST_UNIT_DOCKER_PREFIX) $(TEST_PYTHON_BUILD_DOCKER_PREFIX) $(TEST_BINARY_DOCKER_PREFIX)
 
-# Run all unit test under bats docker
 .PHONY: $(TEST_UNIT_DOCKER_PREFIX)
 $(TEST_UNIT_DOCKER_PREFIX): $(TEST_UNIT_DOCKER_TARGETS)
 
@@ -31,23 +36,22 @@ $(TEST_UNIT_DOCKER_TARGETS): $(TEST_UNIT_DOCKER_PREFIX)-% : $(TEST_BATS_IMAGE_PR
 		-u "$$(id -u $$(whoami)):$$(id -g $$(whoami))" \
 		$${BATS_TEST_FILTER:+-e BATS_TEST_FILTER="$${BATS_TEST_FILTER}"} \
 		$${BATS_FILE_FILTER:+-e BATS_FILE_FILTER="$${BATS_FILE_FILTER}"} \
-	        $${CI+-e CI="$${CI}"} \
-	        $(INTERACTIVE) \
+		$${CI+-e CI="$${CI}"} \
+		$(INTERACTIVE) \
 		$(DOCKER_IMAGE):$(DOCKER_TAG) \
 		test/run
 
-# Run all plugin test under bats docker
-.PHONY: $(TEST_PLUGIN_DOCKER_PREFIX)
-$(TEST_PLUGIN_DOCKER_PREFIX): $(TEST_PLUGIN_DOCKER_TARGETS)
+.PHONY: $(TEST_PYTHON_BUILD_DOCKER_PREFIX)
+$(TEST_PYTHON_BUILD_DOCKER_PREFIX): $(TEST_PYTHON_BUILD_DOCKER_TARGETS)
 
 # Run each plugin test under bats docker
-.PHONY: $(TEST_PLUGIN_DOCKER_TARGETS)
-$(TEST_PLUGIN_DOCKER_TARGETS): DOCKER_IMAGE = $(TEST_BATS_IMAGE_PREFIX)
-$(TEST_PLUGIN_DOCKER_TARGETS): GNU = $(if $(findstring -gnu-,$@),True,False)
-$(TEST_PLUGIN_DOCKER_TARGETS): BASH = $(filter $(TEST_BASH_VERSIONS),$(subst -, ,$@))
-$(TEST_PLUGIN_DOCKER_TARGETS): DOCKER_TAG = bash-$(BASH)-gnu-$(GNU)
-$(TEST_PLUGIN_DOCKER_TARGETS): INTERACTIVE = $(if $(findstring true,$(CI)),,-ti)
-$(TEST_PLUGIN_DOCKER_TARGETS): $(TEST_PLUGIN_DOCKER_PREFIX)-% : $(TEST_BATS_IMAGE_PREFIX)-%
+.PHONY: $(TEST_PYTHON_BUILD_DOCKER_TARGETS)
+$(TEST_PYTHON_BUILD_DOCKER_TARGETS): DOCKER_IMAGE = $(TEST_BATS_IMAGE_PREFIX)
+$(TEST_PYTHON_BUILD_DOCKER_TARGETS): GNU = $(if $(findstring -gnu-,$@),True,False)
+$(TEST_PYTHON_BUILD_DOCKER_TARGETS): BASH = $(filter $(TEST_BASH_VERSIONS),$(subst -, ,$@))
+$(TEST_PYTHON_BUILD_DOCKER_TARGETS): DOCKER_TAG = bash-$(BASH)-gnu-$(GNU)
+$(TEST_PYTHON_BUILD_DOCKER_TARGETS): INTERACTIVE = $(if $(findstring true,$(CI)),,-ti)
+$(TEST_PYTHON_BUILD_DOCKER_TARGETS): $(TEST_PYTHON_BUILD_DOCKER_PREFIX)-% : $(TEST_BATS_IMAGE_PREFIX)-%
 	$(info Running test with docker image '$(DOCKER_IMAGE):$(DOCKER_TAG)')
 	docker run \
 		--init \
@@ -55,11 +59,33 @@ $(TEST_PLUGIN_DOCKER_TARGETS): $(TEST_PLUGIN_DOCKER_PREFIX)-% : $(TEST_BATS_IMAG
 		-v /etc/passwd:/etc/passwd:ro \
 		-v /etc/group:/etc/group:ro \
 		-u "$$(id -u $$(whoami)):$$(id -g $$(whoami))" \
-	        $${CI+-e CI="$${CI}"} \
-	        $(INTERACTIVE) \
+		$${CI+-e CI="$${CI}"} \
+		$(INTERACTIVE) \
 		$(DOCKER_IMAGE):$(DOCKER_TAG) \
 		bats $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} plugins/python-build/test/$${BATS_FILE_FILTER}
 
+.PHONY: $(TEST_BINARY_DOCKER_PREFIX)
+$(TEST_BINARY_DOCKER_PREFIX): $(TEST_BINARY_DOCKER_TARGETS)
+
+.PHONY: $(TEST_BINARY_DOCKER_TARGETS)
+$(TEST_BINARY_DOCKER_TARGETS): DOCKER_IMAGE = $(TEST_BATS_IMAGE_PREFIX)
+$(TEST_BINARY_DOCKER_TARGETS): GNU = $(if $(findstring -gnu-,$@),True,False)
+$(TEST_BINARY_DOCKER_TARGETS): BASH = $(filter $(TEST_BASH_VERSIONS),$(subst -, ,$@))
+$(TEST_BINARY_DOCKER_TARGETS): DOCKER_TAG = bash-$(BASH)-gnu-$(GNU)
+$(TEST_BINARY_DOCKER_TARGETS): INTERACTIVE = $(if $(findstring true,$(CI)),,-ti)
+$(TEST_BINARY_DOCKER_TARGETS): $(TEST_BINARY_DOCKER_PREFIX)-% : $(TEST_BATS_IMAGE_PREFIX)-%
+	$(info Running test with docker image '$(DOCKER_IMAGE):$(DOCKER_TAG)')
+	docker run \
+		--init \
+		-v $(PWD):/code:ro \
+		-v /etc/passwd:/etc/passwd:ro \
+		-v /etc/group:/etc/group:ro \
+		-u "$$(id -u $$(whoami)):$$(id -g $$(whoami))" \
+		$${CI+-e CI="$${CI}"} \
+		$(INTERACTIVE) \
+		$(DOCKER_IMAGE):$(DOCKER_TAG) \
+		bats $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} plugins/pyenv-binary/test/$${BATS_FILE_FILTER}
+
 # Build all images needed for bats under docker
 .PHONY: $(TEST_BATS_IMAGE_PREFIX)
 $(TEST_BATS_IMAGE_PREFIX): $(TEST_BATS_IMAGE_TARGETS)
@@ -71,7 +97,7 @@ $(TEST_BATS_IMAGE_TARGETS): GNU = $(if $(findstring -gnu-,$@),True,False)
 $(TEST_BATS_IMAGE_TARGETS): BASH = $(filter $(TEST_BASH_VERSIONS),$(subst -, ,$@))
 $(TEST_BATS_IMAGE_TARGETS): DOCKER_TAG = bash-$(BASH)-gnu-$(GNU)
 $(TEST_BATS_IMAGE_TARGETS):
-	$(info Building docker image '$(DOCKER_IMAGE):$(DOCKER_TAG)')
+	if [ -z "$$(docker images -q '$(DOCKER_IMAGE):$(DOCKER_TAG)')" ]]; then \
 	docker build \
 		--quiet \
 		-f "$(PWD)/test/Dockerfile" \
@@ -79,41 +105,32 @@ $(TEST_BATS_IMAGE_TARGETS):
 		--build-arg BASH="$(BASH)" \
 		--build-arg BATS_VERSION="$(TEST_BATS_VERSION)" \
 		-t $(DOCKER_IMAGE):$(DOCKER_TAG) \
-		./
+		./ ; \
+	fi
 
-.PHONY: test test-build test-unit test-plugin
+.PHONY: test test-unit test-python-build test-binary
 
 # Do not pass in user flags to build tests.
 unexport PYTHON_CFLAGS
 unexport PYTHON_CONFIGURE_OPTS
 
-test: test-unit test-plugin
+test: test-unit test-python-build test-binary
 
 test-unit: bats
 	PATH="./bats/bin:$$PATH" test/run
 	
-test-plugin: bats
+test-python-build: bats
 	cd plugins/python-build && $(PWD)/bats/bin/bats $${CI:+--tap} $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} test/$${BATS_FILE_FILTER}
 
-PYTHON_BUILD_ROOT := $(CURDIR)/plugins/python-build
-PYTHON_BUILD_OPTS ?= --verbose
-PYTHON_BUILD_VERSION ?= 3.8-dev
-PYTHON_BUILD_TEST_PREFIX ?= $(PYTHON_BUILD_ROOT)/test/build/tmp/dist
-
-test-build:
-	$(RM) -r $(PYTHON_BUILD_TEST_PREFIX)
-	$(PYTHON_BUILD_ROOT)/bin/python-build $(PYTHON_BUILD_OPTS) $(PYTHON_BUILD_VERSION) $(PYTHON_BUILD_TEST_PREFIX)
-	[ -e $(PYTHON_BUILD_TEST_PREFIX)/bin/python ]
-	$(PYTHON_BUILD_TEST_PREFIX)/bin/python -V
-	[ -e $(PYTHON_BUILD_TEST_PREFIX)/bin/pip ]
-	$(PYTHON_BUILD_TEST_PREFIX)/bin/pip -V
+test-binary: bats
+	cd plugins/pyenv-binary && $(PWD)/bats/bin/bats $${CI:+--tap} $${BATS_TEST_FILTER:+--filter "$${BATS_TEST_FILTER}"} test/$${BATS_FILE_FILTER}
 
 .SECONDARY: bats-$(TEST_BATS_VERSION)
 bats-$(TEST_BATS_VERSION):
-	rm -rf bats
-	ln -sf bats-$(TEST_BATS_VERSION) bats
 	git clone --depth 1 --branch $(TEST_BATS_VERSION) https://github.com/bats-core/bats-core.git bats-$(TEST_BATS_VERSION)
 
 .PHONY: bats
 bats: bats-$(TEST_BATS_VERSION)
-	ln -sf bats-$(TEST_BATS_VERSION) bats
+	if [ \( ! -L bats \) -o \( "x$$(readlink bats)" != "xbats-$(TEST_BATS_VERSION)" \) ]; then \
+		rm -rf bats; ln -s bats-$(TEST_BATS_VERSION) bats; \
+	fi

README.md

@@ -837,13 +837,10 @@ e.g. `~/.pyenv --install bash`.
 ## Development
 
 The pyenv source code is [hosted on
-GitHub](https://github.com/pyenv/pyenv).  It's clean, modular,
-and easy to understand, even if you're not a shell hacker.
+GitHub](https://github.com/pyenv/pyenv).
 
-Tests are executed using [Bats](https://github.com/bats-core/bats-core):
-
-    bats test
-    bats/test/<file>.bats
+Tests are executed using [Bats](https://github.com/bats-core/bats-core).
+See the [tests README](test/README.md) for details.
 
 
 ### Contributing

plugins/pyenv-binary/README.md

@@ -25,3 +25,33 @@ distro and libc version) and the system libraries the build links against.
 ```sh
 pyenv binary save 3.12.7 ./dist
 ```
+
+### `pyenv binary generate-installer <metadata-file> --archive-url <url> [-o <output>]`
+
+Reads a `.meta` file and emits a python-build definition. Drop it into
+python-build's definition directory and `pyenv install <name>` installs the
+archive like any other version. The archive location is a parameter, so you can
+host it anywhere (it does not have to be a pyenv location); the archive itself
+must sit next to the `.meta` file so its checksum can be baked into the
+definition.
+
+The definition refuses to install on a different OS/architecture, or an older
+glibc, than the archive was built for, and checks that the system libraries it
+needs are present.
+
+```sh
+pyenv binary generate-installer ./dist/3.12.7-linux-x86_64.meta \
+  --archive-url https://example.com/3.12.7-linux-x86_64.tar.gz \
+  -o "$(pyenv root)/plugins/python-build/share/python-build/3.12.7-linux-x86_64"
+
+pyenv install 3.12.7-linux-x86_64
+```
+
+### `pyenv binary relocate <prefix>`
+
+Rewrites the rpaths of a Python tree unpacked into `<prefix>` so the interpreter
+and its extension modules load the bundled libraries from there rather than from
+the path the archive was built at. Uses `patchelf`. The generated definition
+calls this; you rarely run it by hand.
+
+Relocation is implemented for Linux; macOS is not wired up yet.

plugins/pyenv-binary/bin/pyenv-binary

@@ -16,6 +16,11 @@ set -e
 
 libexec="${BASH_SOURCE%/*}/../libexec"
 
+# The pyenv launcher only puts a plugin's bin on PATH, but `pyenv-help' looks a
+# command up there. Add our libexec so the subcommands, and the help text they
+# print, can be found.
+export PATH="${libexec}:${PATH}"
+
 list_commands() {
   local path
   for path in "$libexec"/pyenv-binary-*; do
@@ -54,8 +59,7 @@ if [ ! -x "$command_path" ]; then
 fi
 
 if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
-  # `pyenv help' looks the command up on PATH; the subcommands live in libexec.
-  PATH="${libexec}:${PATH}" exec pyenv-help "binary-${subcommand}"
+  exec pyenv-help "binary-${subcommand}"
 fi
 
 exec "$command_path" "$@"

plugins/pyenv-binary/libexec/pyenv-binary-generate-installer

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+#
+# Summary: Generate a python-build definition for a saved binary archive
+#
+# Usage: pyenv binary generate-installer <metadata-file> --archive-url <url> [-o <output>]
+#
+#   Reads a metadata file written by `pyenv binary save' and emits a python-build
+#   definition. Dropped into python-build's definition directory, it installs the
+#   archive like any other version: `pyenv install' downloads it from <url>,
+#   checks the platform and the required system libraries, unpacks it, then
+#   rewrites rpaths so the copy runs from its prefix.
+#
+#   The archive must sit next to the metadata file so its checksum can be baked
+#   into the definition; `pyenv binary save' writes the two together.
+#
+#   <metadata-file>       A .meta file written by `pyenv binary save'.
+#                         The corresponsing binary package written by `pyenv binary save'
+#                         needs to be present alongside it.
+#   --archive-url <url>   The URL for the resulting installation script to download
+#                         the package from.
+#   -o <output>           Write the definition here (default: stdout).
+#
+set -e
+[ -n "$PYENV_DEBUG" ] && set -x
+
+# Provide pyenv completions
+if [ "$1" = "--complete" ]; then
+  echo --archive-url
+  echo -o
+  exit
+fi
+
+metadata=""
+archive_url=""
+output=""
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+  --archive-url )
+    [ $# -ge 2 ] || { echo "pyenv-binary: --archive-url needs a value" >&2; exit 1; }
+    archive_url="$2"; shift 2 ;;
+  -o )
+    [ $# -ge 2 ] || { echo "pyenv-binary: -o needs a value" >&2; exit 1; }
+    output="$2"; shift 2 ;;
+  -* )
+    echo "pyenv-binary: unknown option \`$1'" >&2; exit 1 ;;
+  * )
+    [ -z "$metadata" ] || { echo "pyenv-binary: unexpected argument \`$1'" >&2; exit 1; }
+    metadata="$1"; shift ;;
+  esac
+done
+
+if [ -z "$metadata" ] || [ -z "$archive_url" ]; then
+  pyenv-help --usage binary-generate-installer >&2
+  exit 1
+fi
+
+version="" os="" arch="" distro="" libc="" archive=""
+deps=""
+while IFS='=' read -r key value; do
+  case "$key" in
+  version ) version="$value" ;;
+  os ) os="$value" ;;
+  arch ) arch="$value" ;;
+  distro ) distro="$value" ;;
+  libc ) libc="$value" ;;
+  archive ) archive="$value" ;;
+  dep ) deps="${deps:+$deps }$value" ;;
+  esac
+done < "$metadata"
+
+# Fail here if the metadata is incomplete, rather than bake a blank into the
+# definition where a missing arch would give a confusing platform error.
+for field in version os arch archive; do
+  if [ -z "${!field}" ]; then
+    echo "pyenv-binary: metadata is missing \`${field}'" >&2
+    exit 1
+  fi
+done
+
+# macOS relocation needs install_name_tool and a different rpath scheme, so it is
+# not supported yet. Other systems relocate with patchelf.
+if [ "$os" = "Darwin" ]; then
+  echo "pyenv-binary: macOS archives are not supported yet" >&2
+  exit 1
+fi
+
+# The glibc floor only applies on Linux; require it there so the definition can
+# tell whether the target is new enough to run the binaries.
+if [ "$os" = "Linux" ] && [ -z "$libc" ]; then
+  echo "pyenv-binary: metadata is missing \`libc'" >&2
+  exit 1
+fi
+
+# python-build downloads the archive itself, so the checksum has to be baked in.
+# The archive sits next to the metadata `save' wrote it alongside.
+archive_path="$(dirname "$metadata")/${archive}"
+if [ ! -r "$archive_path" ]; then
+  echo "pyenv-binary: cannot read the archive \`${archive_path}' to checksum it" >&2
+  exit 1
+fi
+if command -v sha256sum >/dev/null 2>&1; then
+  sha256="$(sha256sum "$archive_path")"; sha256="${sha256%% *}"
+elif command -v shasum >/dev/null 2>&1; then
+  sha256="$(shasum -a 256 "$archive_path")"; sha256="${sha256%% *}"
+elif command -v openssl >/dev/null 2>&1; then
+  sha256="$(openssl dgst -sha256 "$archive_path")"; sha256="${sha256##* }"
+else
+  echo "pyenv-binary: need sha256sum, shasum or openssl to checksum the archive" >&2
+  exit 1
+fi
+
+emit() {
+  # The values known now go in as %q-quoted assignments so a stray quote or space
+  # in the URL or metadata cannot break the definition or inject into it. The body
+  # below is a quoted here-doc, verbatim.
+  {
+    printf '# python-build definition for prebuilt Python %s (%s/%s%s).\n' \
+      "$version" "$os" "$arch" "${distro:+, $distro}"
+    echo "# Generated by \`pyenv binary generate-installer'."
+    echo
+    printf 'EXPECT_OS=%q\n' "$os"
+    printf 'EXPECT_ARCH=%q\n' "$arch"
+    printf 'EXPECT_LIBC=%q\n' "$libc"
+    printf 'VERSION=%q\n' "$version"
+    printf 'ARCHIVE_URL=%q\n' "$archive_url"
+    printf 'SHA256=%q\n' "$sha256"
+    printf 'DEPS=%q\n' "$deps"
+  }
+  cat <<'EOF'
+
+os="$(uname -s)"
+arch="$(uname -m)"
+if [ "$os" != "$EXPECT_OS" ] || [ "$arch" != "$EXPECT_ARCH" ]; then
+  echo "pyenv-binary: this archive is for ${EXPECT_OS}/${EXPECT_ARCH}, not ${os}/${arch}" >&2
+  exit 1
+fi
+
+# Refuse a target whose glibc is older than the one the archive was built on;
+# the dynamic loader would reject the binaries.
+case "$EXPECT_LIBC" in
+"glibc "* )
+  build_libc="${EXPECT_LIBC#glibc }"
+  target_libc="$(getconf GNU_LIBC_VERSION 2>/dev/null || true)"
+  case "$target_libc" in
+  "glibc "* )
+    target_libc="${target_libc#glibc }"
+    older="$(printf '%s\n%s\n' "$build_libc" "$target_libc" | sort -V | head -n1)"
+    if [ "$older" = "$target_libc" ] && [ "$target_libc" != "$build_libc" ]; then
+      echo "pyenv-binary: archive needs glibc ${build_libc} or newer, but this system has ${target_libc}" >&2
+      exit 1
+    fi
+    ;;
+  esac
+  ;;
+esac
+
+# Required system libraries must already be present on the target. ldconfig -p
+# lists each library as "<soname> (...) => <path>"; match the soname column
+# exactly so a longer name like libc.so.6.1 does not satisfy libc.so.6.
+if command -v ldconfig &>/dev/null && cache="$(ldconfig -p)"; then
+  missing=""
+  for dep in $DEPS; do
+    printf '%s\n' "$cache" | awk -v d="$dep" '$1 == d { found = 1 } END { exit !found }' \
+      || missing="${missing} ${dep}"
+  done
+  if [ -n "$missing" ]; then
+    echo "pyenv-binary: missing required system libraries:${missing}" >&2
+    exit 1
+  fi
+elif [ -n "$DEPS" ]; then
+  echo "pyenv-binary: cannot check required system libraries (ldconfig with -p not found)" >&2
+fi
+
+build_package_relocate() {
+  pyenv binary relocate "$PREFIX_PATH"
+}
+
+install_package "Python-${VERSION}-binary" "${ARCHIVE_URL}#${SHA256}" copy relocate
+
+EOF
+}
+
+if [ -n "$output" ]; then
+  emit > "$output"
+  echo "Wrote definition to ${output}"
+else
+  emit
+fi

plugins/pyenv-binary/libexec/pyenv-binary-relocate

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Summary: Rewrite an unpacked Python's rpaths so it runs from its prefix
+#
+# Usage: pyenv binary relocate <prefix>
+#
+#   Rewrites the rpaths of a Python tree that was unpacked into <prefix> so the
+#   interpreter and its extension modules load the bundled libraries from their
+#   new location rather than the path it was built at. Requires patchelf.
+#
+#   The definition that `pyenv binary generate-installer' produces calls this
+#   after `install_package ... copy' has laid the tree down.
+#
+set -e
+[ -n "$PYENV_DEBUG" ] && set -x
+
+if [ "$1" = "--complete" ]; then
+  exit
+fi
+
+prefix="$1"
+if [ -z "$prefix" ]; then
+  pyenv-help --usage binary-relocate >&2
+  exit 1
+fi
+
+# patchelf can add an rpath and write one of any length; chrpath can only shorten
+# an existing one, which is not enough to relocate every build, so require it.
+if ! command -v patchelf >/dev/null 2>&1; then
+  echo "pyenv-binary: need patchelf to relocate the binary" >&2
+  exit 1
+fi
+
+# The interpreter loads libpython from ../lib. Patch every ELF binary in bin/ so
+# this holds whatever the interpreter is named (python3, python2.7, ...); the
+# scripts alongside it (pip, idle) are not ELF, so `--print-rpath' fails on them
+# and they are skipped. A tree with no interpreter at all did not unpack the way
+# we expect, so treat that as an error rather than quietly relocating nothing.
+found=0
+for file in "$prefix"/bin/*; do
+  [[ -f $file && -x $file && ! -L $file ]] || continue
+  patchelf --print-rpath "$file" >/dev/null 2>&1 || continue
+  patchelf --set-rpath "$prefix/lib" "$file"
+  found=1
+done
+if [ "$found" -eq 0 ]; then
+  echo "pyenv-binary: found no interpreter to relocate under \`${prefix}/bin'" >&2
+  exit 1
+fi
+
+# The extension modules CPython built are the only other objects with an rpath
+# into the old prefix, so point them at lib/ as well. Anything a wheel installed
+# carries an rpath of its own, often into a bundled library directory beside it,
+# and would stop loading if we overwrote it. Each match is an ELF object we expect
+# to patch, so let a failure stop us rather than swallow it.
+while IFS= read -r so; do
+  patchelf --set-rpath "$prefix/lib" "$so"
+done < <(find "$prefix" -path '*/lib-dynload/*.so')

plugins/pyenv-binary/libexec/pyenv-binary-save

@@ -88,7 +88,7 @@ mkdir -p "$output_dir"
 archive="${version}-${platform}.tar.gz"
 metadata="${version}-${platform}.meta"
 
-tar -C "$prefix" -czf "${output_dir}/${archive}" .
+tar -C "$(dirname "$prefix")" -czf "${output_dir}/${archive}" "$(basename "$prefix")"
 
 {
   echo "# pyenv-binary metadata"

plugins/pyenv-binary/test/generate-installer.bats

@@ -0,0 +1,123 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+create_meta() {
+  local os="${1-Linux}"
+  local arch="${2-x86_64}"
+  local libc="${3-glibc 2.17}"
+  local archive="${BATS_TEST_TMPDIR}/3.12.7.tar.gz"
+  local meta="${BATS_TEST_TMPDIR}/sample.meta"
+
+  rm -rf "${BATS_TEST_TMPDIR}/archive"
+  mkdir -p "${BATS_TEST_TMPDIR}/archive/3.12.7/bin"
+  printf '#!/bin/sh\n' > "${BATS_TEST_TMPDIR}/archive/3.12.7/bin/python"
+  chmod +x "${BATS_TEST_TMPDIR}/archive/3.12.7/bin/python"
+  tar -C "${BATS_TEST_TMPDIR}/archive" -czf "$archive" 3.12.7
+
+  {
+    echo "version=3.12.7"
+    echo "os=${os}"
+    echo "arch=${arch}"
+    [ -z "$libc" ] || echo "libc=${libc}"
+    echo "archive=${archive##*/}"
+  } > "$meta"
+  echo "$meta"
+}
+
+@test "completion lists the options" {
+  run pyenv-binary-generate-installer --complete
+  assert_success "--archive-url
+-o"
+}
+
+@test "fails with no arguments" {
+  create_stub pyenv-help "echo usage"
+  run pyenv-binary-generate-installer
+  assert_failure "usage"
+}
+
+@test "fails without an archive url" {
+  create_stub pyenv-help "echo usage"
+  run pyenv-binary-generate-installer "$(create_meta)"
+  assert_failure "usage"
+}
+
+@test "fails when --archive-url has no value" {
+  run pyenv-binary-generate-installer "$(create_meta)" --archive-url
+  assert_failure "pyenv-binary: --archive-url needs a value"
+}
+
+@test "rejects a second positional argument" {
+  run pyenv-binary-generate-installer "$(create_meta)" extra --archive-url http://x/a.tar.gz
+  assert_failure "pyenv-binary: unexpected argument \`extra'"
+}
+
+@test "fails for a metadata file that cannot be read" {
+  run pyenv-binary-generate-installer /no/such.meta --archive-url http://x/a.tar.gz
+  assert_failure
+}
+
+@test "refuses macOS metadata" {
+  run pyenv-binary-generate-installer "$(create_meta Darwin arm64 '')" \
+    --archive-url http://x/a.tar.gz
+  assert_failure "pyenv-binary: macOS archives are not supported yet"
+}
+
+@test "fails when required metadata is missing" {
+  local field meta
+  for field in version os arch archive; do
+    meta="$(create_meta)"
+    #cannot use -i: GNU sed requires -i[suf], BSD sed required -i <suf>
+    sed "/^${field}=/d" "$meta" > "${meta}.tmp"; mv "${meta}"{.tmp,}
+
+    run pyenv-binary-generate-installer "$meta" --archive-url http://x/a.tar.gz
+    assert_failure "pyenv-binary: metadata is missing \`${field}'"
+  done
+}
+
+@test "fails when Linux metadata is missing libc" {
+  run pyenv-binary-generate-installer "$(create_meta Linux x86_64 '')" \
+    --archive-url http://x/a.tar.gz
+  assert_failure "pyenv-binary: metadata is missing \`libc'"
+}
+
+@test "refuses a host whose glibc is older than the archive" {
+  local out="${BATS_TEST_TMPDIR}/definition"
+  pyenv-binary-generate-installer "$(create_meta Linux x86_64 'glibc 99.0')" \
+    --archive-url http://example.com/a.tar.gz -o "$out"
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+  create_stub getconf 'echo "glibc 2.31"'
+
+  run bash "$out"
+  assert_failure "pyenv-binary: archive needs glibc 99.0 or newer, but this system has 2.31"
+}
+
+@test "fails when the archive is not beside the metadata" {
+  local meta="$(create_meta)"
+  rm "${BATS_TEST_TMPDIR}/3.12.7.tar.gz"
+
+  run pyenv-binary-generate-installer "$meta" --archive-url http://x/a.tar.gz
+  assert_failure "pyenv-binary: cannot read the archive \`${BATS_TEST_TMPDIR}/3.12.7.tar.gz' to checksum it"
+}
+
+@test "the generated definition is installable with python-build" {
+  ldconfig -p &>/dev/null || skip "ldconfig with -p is not present"
+  local archive="${BATS_TEST_TMPDIR}/3.12.7.tar.gz"
+  local cache="${BATS_TEST_TMPDIR}/cache"
+  local definition="${BATS_TEST_TMPDIR}/definition"
+  local prefix="${BATS_TEST_TMPDIR}/install"
+  pyenv-binary-generate-installer "$(create_meta)" \
+    --archive-url http://example.com/3.12.7.tar.gz -o "$definition"
+  mkdir -p "$cache"
+  cp "$archive" "${cache}/Python-3.12.7-binary.tar.gz"
+  create_stub pyenv 'echo "pyenv $*"'
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
+
+  PYTHON_BUILD_CACHE_PATH="$cache" run \
+    "${BATS_TEST_DIRNAME}/../../python-build/bin/python-build" "$definition" "$prefix"
+  assert_success
+  assert_line "pyenv binary relocate ${prefix}"
+  assert_line "Installed Python-3.12.7-binary to ${prefix}"
+  assert [ -x "${prefix}/bin/python" ]
+}

plugins/pyenv-binary/test/relocate.bats

@@ -0,0 +1,72 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+_setup() {
+  create_stub pyenv-help "echo usage"
+}
+
+stub_patchelf() {
+  create_path_executable patchelf <<STUB
+if [ "\$1" = "--print-rpath" ]; then
+  exit 0
+fi
+echo "\$*" >> "${BATS_TEST_TMPDIR}/patchelf.log"
+STUB
+}
+
+create_interpreter() {
+  mkdir -p "${BATS_TEST_TMPDIR}/prefix/bin"
+  printf '#!/bin/sh\n' > "${BATS_TEST_TMPDIR}/prefix/bin/python2.7"
+  chmod +x "${BATS_TEST_TMPDIR}/prefix/bin/python2.7"
+}
+
+@test "completion produces nothing" {
+  run pyenv-binary-relocate --complete
+  assert_success ""
+}
+
+@test "fails without a prefix" {
+  run pyenv-binary-relocate
+  assert_failure "usage"
+}
+
+@test "fails when patchelf is not available" {
+  PATH="$(path_without patchelf)" run pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix"
+  assert_failure "pyenv-binary: need patchelf to relocate the binary"
+}
+
+@test "fails when the prefix has no interpreter" {
+  create_path_executable patchelf "exit 0"
+  mkdir -p "${BATS_TEST_TMPDIR}/prefix"
+
+  run pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix"
+  assert_failure "pyenv-binary: found no interpreter to relocate under \`${BATS_TEST_TMPDIR}/prefix/bin'"
+}
+
+@test "relocates an executable interpreter" {
+  stub_patchelf
+  create_interpreter
+
+  run pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix"
+  assert_success
+  run cat "${BATS_TEST_TMPDIR}/patchelf.log"
+  assert_output "--set-rpath ${BATS_TEST_TMPDIR}/prefix/lib ${BATS_TEST_TMPDIR}/prefix/bin/python2.7"
+}
+
+@test "relocates the extension modules but not what a wheel installed" {
+  local lib="${BATS_TEST_TMPDIR}/prefix/lib/python3.12"
+  stub_patchelf
+  create_interpreter
+  mkdir -p "${lib}/lib-dynload" "${lib}/site-packages/numpy"
+  touch "${lib}/lib-dynload/_ssl.cpython-312-x86_64-linux-gnu.so"
+  # A wheel points its extensions at the libraries it bundles alongside them, so
+  # its rpath is its own business and must survive relocation.
+  touch "${lib}/site-packages/numpy/_multiarray.so"
+
+  run pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix"
+  assert_success
+  run cat "${BATS_TEST_TMPDIR}/patchelf.log"
+  assert_line "--set-rpath ${BATS_TEST_TMPDIR}/prefix/lib ${lib}/lib-dynload/_ssl.cpython-312-x86_64-linux-gnu.so"
+  refute_line "--set-rpath ${BATS_TEST_TMPDIR}/prefix/lib ${lib}/site-packages/numpy/_multiarray.so"
+}

plugins/pyenv-binary/test/save.bats

@@ -33,40 +33,39 @@ platform() {
 @test "packages an installed version" {
   create_version "3.12.7"
   local out="${BATS_TEST_TMPDIR}/dist"
+  local archive="${out}/3.12.7-$(platform).tar.gz"
+
   run pyenv-binary-save "3.12.7" "$out"
   assert_success "Saved 3.12.7-$(platform).tar.gz and 3.12.7-$(platform).meta to $out"
-  assert [ -f "${out}/3.12.7-$(platform).tar.gz" ]
+  assert [ -f "$archive" ]
   assert [ -f "${out}/3.12.7-$(platform).meta" ]
+  run tar -tzf "$archive"
+  assert_success
+  assert_line 0 "3.12.7/"
 }
 
 @test "records the platform in the metadata" {
   create_version "3.12.7"
   local out="${BATS_TEST_TMPDIR}/dist"
   pyenv-binary-save "3.12.7" "$out" >/dev/null
+
   run cat "${out}/3.12.7-$(platform).meta"
   assert_success
-  assert_output_contains "version=3.12.7"
-  assert_output_contains "platform=$(platform)"
-  assert_output_contains "archive=3.12.7-$(platform).tar.gz"
-}
-
-# Put an executable on PATH that stands in for a system tool during the test.
-stub() {
-  local name="$1"
-  local dir="${BATS_TEST_TMPDIR}/stubs"
-  mkdir -p "$dir"
-  { echo "#!/usr/bin/env bash"; cat -; } > "${dir}/${name}"
-  chmod +x "${dir}/${name}"
-  export PATH="${dir}:$PATH"
+  assert_line "version=3.12.7"
+  assert_line "platform=$(platform)"
+  assert_line "archive=3.12.7-$(platform).tar.gz"
 }
 
 @test "records only the libraries ldd resolves outside the prefix" {
   create_version "3.12.7"
   touch "${PYENV_ROOT}/versions/3.12.7/bin/python3.12"
-
-  # A realistic ldd listing: the vdso and the loader have no `=>' mapping,
-  # libpython resolves inside the prefix, and libc/libm are external.
-  stub ldd <<'STUB'
+  create_path_executable uname <<'STUB'
+case "$1" in
+  -s) echo Linux ;;
+  -m) echo x86_64 ;;
+esac
+STUB
+  create_path_executable ldd <<'STUB'
 prefix="${PYENV_ROOT}/versions/3.12.7"
 cat <<EOF
 	linux-vdso.so.1 (0x00007ffd1adfe000)
@@ -79,7 +78,6 @@ STUB
 
   run pyenv-binary-save "3.12.7" "${BATS_TEST_TMPDIR}/dist"
   assert_success
-
   run grep '^dep=' "${BATS_TEST_TMPDIR}/dist/"*.meta
   assert_output "dep=libc.so.6
 dep=libm.so.6"
@@ -88,18 +86,13 @@ dep=libm.so.6"
 @test "records only the libraries otool resolves outside the prefix" {
   create_version "3.12.7"
   touch "${PYENV_ROOT}/versions/3.12.7/bin/python3.12"
-
-  # Take the macOS path by faking the platform, then feed a realistic otool
-  # listing: the first line names the file, @rpath and in-prefix entries are
-  # bundled, and libSystem is the one external dependency.
-  stub uname <<'STUB'
+  create_path_executable uname <<'STUB'
 case "$1" in
   -s) echo Darwin ;;
   -m) echo arm64 ;;
-  *) exec /usr/bin/uname "$@" ;;
 esac
 STUB
-  stub otool <<'STUB'
+  create_path_executable otool <<'STUB'
 prefix="${PYENV_ROOT}/versions/3.12.7"
 cat <<EOF
 ${2}:
@@ -111,7 +104,6 @@ STUB
 
   run pyenv-binary-save "3.12.7" "${BATS_TEST_TMPDIR}/dist"
   assert_success
-
   run grep '^dep=' "${BATS_TEST_TMPDIR}/dist/"*.meta
   assert_output "dep=/usr/lib/libSystem.B.dylib"
 }

plugins/pyenv-binary/test/test_helper.bash

@@ -1,68 +0,0 @@
-setup() {
-  export PYENV_ROOT="${BATS_TEST_TMPDIR}/root"
-  PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
-  PATH="${BATS_TEST_DIRNAME}/../libexec:$PATH"
-  export PATH
-
-  # If test specific setup exist, run it
-  if [[ $(type -t _setup) == function ]]; then
-    _setup
-  fi
-}
-
-flunk() {
-  { if [ "$#" -eq 0 ]; then cat -
-    else echo "$@"
-    fi
-  } | sed "s:${BATS_TEST_TMPDIR}:\${BATS_TEST_TMPDIR}:g" >&2
-  return 1
-}
-
-assert() {
-  if ! "$@"; then
-    flunk "failed: $@"
-  fi
-}
-
-assert_success() {
-  if [ "$status" -ne 0 ]; then
-    { echo "command failed with exit status $status"
-      echo "output: $output"
-    } | flunk
-  elif [ "$#" -gt 0 ]; then
-    assert_output "$1"
-  fi
-}
-
-assert_failure() {
-  if [ "$status" -eq 0 ]; then
-    flunk "expected failed exit status"
-  elif [ "$#" -gt 0 ]; then
-    assert_output "$1"
-  fi
-}
-
-assert_equal() {
-  if [ "$1" != "$2" ]; then
-    { echo "expected: $1"
-      echo "actual:   $2"
-    } | flunk
-  fi
-}
-
-assert_output() {
-  local expected
-  if [ $# -eq 0 ]; then expected="$(cat -)"
-  else expected="$1"
-  fi
-  assert_equal "$expected" "$output"
-}
-
-assert_output_contains() {
-  local expected="$1"
-  echo "$output" | grep -F "$expected" >/dev/null || {
-    { echo "expected output to contain: $expected"
-      echo "actual: $output"
-    } | flunk
-  }
-}

plugins/pyenv-binary/test/test_helper.bash

@@ -0,0 +1 @@
+../../../test/test_helper.bash

test/README.md

@@ -15,62 +15,25 @@ Under the hood, `pyenv` test suites use `bats` as a test framework and are run o
 - `test`
   - Run the whole test suite on the local host
 - `test-docker`
-  - Run the whole test suite on docker
-    - Some volumes are used in read-only mode
+  - Run the whole test suite in Docker (in all environments)
 - `test-unit`
-  - Run the unit test
-- `test-plugin`
-  - Run the plugin test
-- `test-unit-docker-[BASH_VERSION]`
-  - Run the unit test under **official** bash docker container (alpine/busybox) with the specified bash version if present is in the `Makefile`
-  - Some volumes are used in read-only mode
-- `test-unit-docker-gnu-[BASH_VERSION]`
-  - Run the unit test under **official** bash docker container (alpine/busybox), completed by **GNU Tools**, with the specified bash version if present is in the `Makefile`
-  - Some volumes are used in read-only mode
-- `test-plugin-docker-[BASH_VERSION]`
-  - Run the plugin test under **official** bash docker container (alpine/busybox), completed by **GNU Tools**, with the specified bash version if present is in the `Makefile`
-  - Some volumes are used in read-only mode
-- `test-plugin-docker-gnu-[BASH_VERSION]`
-  - Run the plugin test under **official** bash docker container (alpine/busybox), completed by **GNU Tools**, with the specified bash version if present is in the `Makefile`
-  - Some volumes are used in read-only mode
+  - Run core tests
+- `test-python-build`
+  - Run Python-Build tests
+- `test-binary`
+  - Run Pyenv-Binary tests
+- `test-*-docker`
+  - Run the corresponding test suite in Docker
+- `test-*-docker-[BASH_VERSION]`, `test-*-docker-gnu-[BASH_VERSION]`
+  - Run the corresponding test suite in the official Bash Docker container (alpine/busybox)
+    against either Busybox tools or GNU tools, with the specified Bash version
+    among those listed in the `Makefile`
 
 ## Targeting specific test / test file
 
- By setting some environment variables, it is possible to filtering which test and/or test file who will be tested with bats
+ By setting some environment variables, it is possible to filter which test and/or test file will be run
 
 - `BATS_FILE_FILTER`
-
-  - Run test only with the specified file
-
+  - Run tests from the specified file
 - `BATS_TEST_FILTER`
-  - Run test only who corresponding to the filter provided
-
-
-### Examples
-
-```bash
-    $ BATS_TEST_FILTER=".*installed.*" BATS_FILE_FILTER="build.bats" make test-plugin-docker-gnu-3.2.57
-    build.bats
-     ✓ yaml is installed for python
-     ✓ homebrew is used in Linux if Pyenv is installed with Homebrew
-     ✓ homebrew is not used in Linux if Pyenv is not installed with Homebrew
-    
-    3 tests, 0 failures
-    
-    $ BATS_TEST_FILTER=".*installed.*" BATS_FILE_FILTER="build.bats" make test-plugin
-    build.bats
-     ✓ yaml is installed for python
-     ✓ homebrew is used in Linux if Pyenv is installed with Homebrew
-     ✓ homebrew is not used in Linux if Pyenv is not installed with Homebrew
-    
-    3 tests, 0 failures
-```
-
-
-
-## Writing test
-
-To be reproducible, each test use/should use its own `TMPDIR` .  
-It's achieved by using the environment variable `BATS_TEST_TMPDIR` provided by bats that is automatically deleted at the end of each test. More info [here](https://bats-core.readthedocs.io/en/stable/writing-tests.html#special-variables)
-
-Another variable who could be used to source some file who need to be tested is `BATS_TEST_DIRNAME` who point to the directory in which the bats test file is located.
+  - Run tests with the names corresponding to the filter