pyenv-binary: support relocating macOS archives (#3528)

plugins/pyenv-binary/libexec/pyenv-binary-generate-installer

@@ -55,7 +55,7 @@ if [ -z "$metadata" ] || [ -z "$archive_url" ]; then
   exit 1
 fi
 
-version="" os="" arch="" distro="" libc="" archive=""
+version="" os="" arch="" distro="" libc="" build_prefix="" archive=""
 deps=""
 while IFS='=' read -r key value; do
   case "$key" in
@@ -64,6 +64,7 @@ while IFS='=' read -r key value; do
   arch ) arch="$value" ;;
   distro ) distro="$value" ;;
   libc ) libc="$value" ;;
+  build_prefix ) build_prefix="$value" ;;
   archive ) archive="$value" ;;
   dep ) deps="${deps:+$deps }$value" ;;
   esac
@@ -78,10 +79,9 @@ for field in version os arch archive; do
   fi
 done
 
-# macOS relocation needs install_name_tool and a different rpath scheme, so it is
+# Mach-O load commands contain the prefix where Python was built.
-# not supported yet. Other systems relocate with patchelf.
+if [ "$os" = "Darwin" ] && [ -z "$build_prefix" ]; then
-if [ "$os" = "Darwin" ]; then
+  echo "pyenv-binary: metadata is missing \`build_prefix'" >&2
-  echo "pyenv-binary: macOS archives are not supported yet" >&2
   exit 1
 fi
 
@@ -122,6 +122,7 @@ emit() {
     printf 'EXPECT_OS=%q\n' "$os"
     printf 'EXPECT_ARCH=%q\n' "$arch"
     printf 'EXPECT_LIBC=%q\n' "$libc"
+    printf 'BUILD_PREFIX=%q\n' "$build_prefix"
     printf 'VERSION=%q\n' "$version"
     printf 'ARCHIVE_URL=%q\n' "$archive_url"
     printf 'SHA256=%q\n' "$sha256"
@@ -185,13 +186,29 @@ elif [ -n "$DEPS" ]; then
   echo "pyenv-binary: cannot check required system libraries (ldconfig cache unavailable)" >&2
 fi
 
+case "$os" in
+Darwin )
+  for tool in otool install_name_tool; do
+    if ! command -v "$tool" >/dev/null 2>&1; then
+      echo "pyenv-binary: need ${tool} to relocate the binary" >&2
+      exit 1
+    fi
+  done
+  ;;
+* )
   if ! command -v patchelf >/dev/null 2>&1; then
     echo "pyenv-binary: need patchelf to relocate the binary" >&2
     exit 1
   fi
+  ;;
+esac
 
 build_package_relocate() {
+  if [ "$os" = "Darwin" ]; then
+    pyenv binary relocate "$PREFIX_PATH" "$BUILD_PREFIX"
+  else
     pyenv binary relocate "$PREFIX_PATH"
+  fi
 }
 
 install_package "Python-${VERSION}-binary" "${ARCHIVE_URL}#${SHA256}" copy relocate

plugins/pyenv-binary/libexec/pyenv-binary-package

@@ -75,15 +75,6 @@ latest )
   ;;
 esac
 
-os="$(uname -s)"
-
-# `generate-installer' refuses a macOS archive, so the last step here cannot
-# succeed on one. Give up before compiling rather than after it.
-if [ "$os" = "Darwin" ]; then
-  echo "pyenv-binary: macOS archives are not supported yet" >&2
-  exit 1
-fi
-
 # `pyenv install' puts a `<version>:<alias>' build under versions/<alias>.
 pyenv-install ${verbose:+--verbose} "$spec"
 pyenv-binary-save "$entry" "$PWD" --name "$entry"

plugins/pyenv-binary/libexec/pyenv-binary-relocate

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 #
-# Summary: Rewrite an unpacked Python's rpaths so it runs from its prefix
+# Summary: Rewrite an unpacked Python's library paths so it runs from its prefix
 #
-# Usage: pyenv binary relocate <prefix>
+# Usage: pyenv binary relocate <prefix> [<build-prefix>]
 #
-#   Rewrites the rpaths of a Python tree that was unpacked into <prefix> so the
+#   Rewrites the library paths of a Python tree that was unpacked into <prefix>
-#   interpreter and its extension modules load the bundled libraries from their
+#   so the interpreter and its extension modules load the bundled libraries from
-#   new location rather than the path it was built at. Requires patchelf.
+#   their new location rather than the path it was built at. Requires patchelf on
+#   Linux and FreeBSD, or otool and install_name_tool on macOS.
 #
 #   The definition that `pyenv binary generate-installer' produces calls this
 #   after `install_package ... copy' has laid the tree down.
@@ -24,6 +25,84 @@ if [ -z "$prefix" ]; then
   exit 1
 fi
 
+if [ "$(uname -s)" = "Darwin" ]; then
+  build_prefix="${2%/}"
+  if [ -z "$build_prefix" ]; then
+    echo "pyenv-binary: need the original build prefix to relocate a macOS binary" >&2
+    exit 1
+  fi
+  for tool in otool install_name_tool; do
+    if ! command -v "$tool" >/dev/null 2>&1; then
+      echo "pyenv-binary: need ${tool} to relocate the binary" >&2
+      exit 1
+    fi
+  done
+
+  relocate_macho() {
+    local file="$1" new_rpath="$2"
+    local load_commands install_ids rpaths dependency install_id rpath replacement
+
+    load_commands="$(otool -L "$file")"
+    install_ids="$(otool -D "$file" 2>/dev/null || true)"
+    rpaths="$(otool -l "$file")"
+    install_id="$(printf '%s\n' "$install_ids" | sed -n '2s/^[[:space:]]*//p')"
+
+    while IFS= read -r dependency; do
+      case "$dependency" in
+      "$build_prefix"/lib/* )
+        if [ "$dependency" != "$install_id" ]; then
+          replacement="@rpath/${dependency#"$build_prefix"/lib/}"
+          install_name_tool -change "$dependency" "$replacement" "$file"
+        fi
+        ;;
+      esac
+    done < <(printf '%s\n' "$load_commands" | tail -n +2 | sed -E \
+      's/^[[:space:]]*//; s/[[:space:]]+\(compatibility version.*$//')
+
+    case "$install_id" in
+    "$build_prefix"/lib/* )
+      replacement="@rpath/${install_id#"$build_prefix"/lib/}"
+      install_name_tool -id "$replacement" "$file"
+      ;;
+    esac
+
+    while IFS= read -r rpath; do
+      if [ "$rpath" = "$build_prefix/lib" ]; then
+        install_name_tool -rpath "$rpath" "$new_rpath" "$file"
+      fi
+    done < <(printf '%s\n' "$rpaths" | awk '
+      /cmd LC_RPATH/ { found = 1; next }
+      found && /^[[:space:]]*path / {
+        sub(/^[[:space:]]*path /, "")
+        sub(/ \(offset [0-9]+\)$/, "")
+        print
+        found = 0
+      }
+    ')
+  }
+
+  found=0
+  for file in "$prefix"/bin/*; do
+    [[ -f $file && -x $file && ! -L $file ]] || continue
+    otool -L "$file" >/dev/null 2>&1 || continue
+    relocate_macho "$file" '@executable_path/../lib'
+    found=1
+  done
+  if [ "$found" -eq 0 ]; then
+    echo "pyenv-binary: found no interpreter to relocate under \`${prefix}/bin'" >&2
+    exit 1
+  fi
+
+  for file in "$prefix"/lib/*.dylib; do
+    [[ -f $file && ! -L $file ]] || continue
+    relocate_macho "$file" '@loader_path'
+  done
+  while IFS= read -r so; do
+    relocate_macho "$so" '@loader_path/../..'
+  done < <(find "$prefix" -type f -path '*/lib/python*/lib-dynload/*.so')
+  exit
+fi
+
 # patchelf can add an rpath and write one of any length; chrpath can only shorten
 # an existing one, which is not enough to relocate every build, so require it.
 if ! command -v patchelf >/dev/null 2>&1; then

plugins/pyenv-binary/libexec/pyenv-binary-save

@@ -147,6 +147,7 @@ tar -C "$(dirname "$prefix")" -czf "${output_dir}/${archive}" "$(basename "$pref
   echo "platform=${platform}"
   [ -n "$distro" ] && echo "distro=${distro}"
   [ -n "$libc" ] && echo "libc=${libc}"
+  echo "build_prefix=${prefix}"
   echo "archive=${archive}"
   system_deps | while IFS= read -r dep; do
     [ -n "$dep" ] && echo "dep=${dep}"

plugins/pyenv-binary/test/generate-installer.bats

@@ -6,6 +6,7 @@ create_meta() {
   local os="${1-Linux}"
   local arch="${2-x86_64}"
   local libc="${3-glibc 2.17}"
+  local build_prefix="${4-}"
   local archive="${BATS_TEST_TMPDIR}/3.12.7.tar.gz"
   local meta="${BATS_TEST_TMPDIR}/sample.meta"
 
@@ -20,6 +21,7 @@ create_meta() {
     echo "os=${os}"
     echo "arch=${arch}"
     [ -z "$libc" ] || echo "libc=${libc}"
+    [ -z "$build_prefix" ] || echo "build_prefix=${build_prefix}"
     echo "archive=${archive##*/}"
   } > "$meta"
   echo "$meta"
@@ -58,10 +60,20 @@ create_meta() {
   assert_failure
 }
 
-@test "refuses macOS metadata" {
+@test "fails when macOS metadata is missing the build prefix" {
   run pyenv-binary-generate-installer "$(create_meta Darwin arm64 '')" \
     --archive-url http://x/a.tar.gz
-  assert_failure "pyenv-binary: macOS archives are not supported yet"
+  assert_failure "pyenv-binary: metadata is missing \`build_prefix'"
+}
+
+@test "quotes the macOS build prefix in the generated definition" {
+  local out="${BATS_TEST_TMPDIR}/definition"
+  pyenv-binary-generate-installer \
+    "$(create_meta Darwin arm64 '' '/build prefix/$name')" \
+    --archive-url http://example.com/a.tar.gz -o "$out"
+
+  run grep '^BUILD_PREFIX=' "$out"
+  assert_success 'BUILD_PREFIX=/build\ prefix/\$name'
 }
 
 @test "fails when required metadata is missing" {
@@ -117,6 +129,58 @@ create_meta() {
   assert_failure "pyenv-binary: need patchelf to relocate the binary"
 }
 
+@test "checks for otool before installing a macOS archive" {
+  local out="${BATS_TEST_TMPDIR}/definition"
+  pyenv-binary-generate-installer \
+    "$(create_meta Darwin arm64 '' /build/3.12.7)" \
+    --archive-url http://example.com/a.tar.gz -o "$out"
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_path_executable install_name_tool true
+
+  PATH="$(path_without otool)" run bash "$out"
+  assert_failure "pyenv-binary: need otool to relocate the binary"
+}
+
+@test "checks for install_name_tool before installing a macOS archive" {
+  local out="${BATS_TEST_TMPDIR}/definition"
+  pyenv-binary-generate-installer \
+    "$(create_meta Darwin arm64 '' /build/3.12.7)" \
+    --archive-url http://example.com/a.tar.gz -o "$out"
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_path_executable otool true
+
+  PATH="$(path_without install_name_tool)" run bash "$out"
+  assert_failure "pyenv-binary: need install_name_tool to relocate the binary"
+}
+
+@test "the generated macOS definition preserves both relocation arguments" {
+  local archive="${BATS_TEST_TMPDIR}/3.12.7.tar.gz"
+  local cache="${BATS_TEST_TMPDIR}/cache"
+  local definition="${BATS_TEST_TMPDIR}/definition"
+  local prefix="${BATS_TEST_TMPDIR}/install"
+  pyenv-binary-generate-installer \
+    "$(create_meta Darwin arm64 '' '/build prefix/$name')" \
+    --archive-url http://example.com/3.12.7.tar.gz -o "$definition"
+  mkdir -p "$cache"
+  cp "$archive" "${cache}/Python-3.12.7-binary.tar.gz"
+  create_stub pyenv 'printf "argc=%s\narg1=%s\narg2=%s\narg3=%s\narg4=%s\n" "$#" "$1" "$2" "$3" "$4"'
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_stub sw_vers 'echo 15.5'
+  create_stub ldconfig true
+  create_path_executable otool true
+  create_path_executable install_name_tool true
+
+  PYTHON_BUILD_CACHE_PATH="$cache" run \
+    "${BATS_TEST_DIRNAME}/../../python-build/bin/python-build" "$definition" "$prefix"
+  assert_success
+  assert_line "argc=4"
+  assert_line "arg1=binary"
+  assert_line "arg2=relocate"
+  assert_line "arg3=${prefix}"
+  assert_line 'arg4=/build prefix/$name'
+  assert_line "Installed Python-3.12.7-binary to ${prefix}"
+}
+
 @test "fails when the archive is not beside the metadata" {
   local meta="$(create_meta)"
   rm "${BATS_TEST_TMPDIR}/3.12.7.tar.gz"

plugins/pyenv-binary/test/package.bats

@@ -82,10 +82,16 @@ pyenv-install --list --bare
   assert_failure "pyenv-binary: \`latest' cannot be used as an entry name"
 }
 
-@test "refuses to package on macOS before compiling anything" {
+@test "packages on macOS" {
   create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_stub pyenv-install 'echo install'
+  create_stub pyenv-binary-save 'echo save'
+  create_stub pyenv-binary-generate-installer 'echo generate-installer'
+
   run pyenv-binary-package 3.12.7:3.12.7-test --archive-base-url http://x/b
-  assert_failure "pyenv-binary: macOS archives are not supported yet"
+  assert_success "install
+save
+generate-installer"
 }
 
 @test "writes the archive, metadata and definition under the entry name (integration)" {

plugins/pyenv-binary/test/relocate.bats

@@ -4,6 +4,7 @@ load test_helper
 
 _setup() {
   create_stub pyenv-help "echo usage"
+  create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'
 }
 
 stub_patchelf() {
@@ -21,6 +22,87 @@ create_interpreter() {
   chmod +x "${BATS_TEST_TMPDIR}/prefix/bin/python2.7"
 }
 
+create_macos_tree() {
+  local prefix="${BATS_TEST_TMPDIR}/prefix"
+  local lib="${prefix}/lib/python3.12"
+  mkdir -p "${prefix}/bin" "${lib}/lib-dynload" "${lib}/site-packages/numpy"
+  printf '#!/bin/sh\n' > "${prefix}/bin/python3.12"
+  chmod +x "${prefix}/bin/python3.12"
+  touch "${prefix}/lib/libpython3.12.dylib"
+  touch "${lib}/lib-dynload/_ssl.cpython-312-darwin.so"
+  touch "${lib}/site-packages/numpy/_multiarray.so"
+}
+
+stub_macos_tools() {
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_path_executable install_name_tool \
+    'echo "$*" >> "${BATS_TEST_TMPDIR}/install-name-tool.log"'
+  create_path_executable otool <<'STUB'
+file="${!#}"
+echo "$*" >> "${BATS_TEST_TMPDIR}/otool.log"
+old="/build prefix/3.12.7"
+case "$1" in
+-L )
+  echo "${file}:"
+  case "$file" in
+  */bin/python3.12 )
+    if [ -n "$OTOOL_RELOCATED" ]; then
+      echo "    @rpath/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)"
+    else
+      echo "    ${old}/lib/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)"
+    fi
+    echo "    ${old}-other/lib/libother.dylib (compatibility version 1.0.0, current version 1.0.0)"
+    echo "    /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1.0.0)"
+    echo "    /opt/homebrew/lib/libintl.8.dylib (compatibility version 1.0.0, current version 1.0.0)"
+    echo "    @loader_path/liblocal.dylib (compatibility version 1.0.0, current version 1.0.0)"
+    ;;
+  */lib/libpython3.12.dylib )
+    if [ -n "$OTOOL_RELOCATED" ]; then
+      echo "    @rpath/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)"
+    else
+      echo "    ${old}/lib/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)"
+    fi
+    ;;
+  esac
+  ;;
+-D )
+  case "$file" in
+  */lib/libpython3.12.dylib )
+    echo "${file}:"
+    if [ -n "$OTOOL_RELOCATED" ]; then
+      echo '@rpath/libpython3.12.dylib'
+    else
+      echo "${old}/lib/libpython3.12.dylib"
+    fi
+    ;;
+  * )
+    exit 1
+    ;;
+  esac
+  ;;
+-l )
+  if [ -n "$OTOOL_RELOCATED" ]; then
+    case "$file" in
+    */bin/* ) rpath='@executable_path/../lib' ;;
+    */lib/*.dylib ) rpath='@loader_path' ;;
+    * ) rpath='@loader_path/../..' ;;
+    esac
+  else
+    rpath="${old}/lib"
+  fi
+  cat <<EOF
+          cmd LC_RPATH
+         path ${rpath} (offset 12)
+          cmd LC_RPATH
+         path /opt/homebrew/lib (offset 12)
+          cmd LC_RPATH
+         path @loader_path/vendor (offset 12)
+EOF
+  ;;
+esac
+STUB
+}
+
 @test "completion produces nothing" {
   run pyenv-binary-relocate --complete
   assert_success ""
@@ -36,6 +118,31 @@ create_interpreter() {
   assert_failure "pyenv-binary: need patchelf to relocate the binary"
 }
 
+@test "fails without the original build prefix on macOS" {
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+
+  run pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix"
+  assert_failure "pyenv-binary: need the original build prefix to relocate a macOS binary"
+}
+
+@test "fails when otool is not available on macOS" {
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_path_executable install_name_tool true
+
+  PATH="$(path_without otool)" run \
+    pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix" /build/3.12.7
+  assert_failure "pyenv-binary: need otool to relocate the binary"
+}
+
+@test "fails when install_name_tool is not available on macOS" {
+  create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac'
+  create_path_executable otool true
+
+  PATH="$(path_without install_name_tool)" run \
+    pyenv-binary-relocate "${BATS_TEST_TMPDIR}/prefix" /build/3.12.7
+  assert_failure "pyenv-binary: need install_name_tool to relocate the binary"
+}
+
 @test "fails when the prefix has no interpreter" {
   create_path_executable patchelf "exit 0"
   mkdir -p "${BATS_TEST_TMPDIR}/prefix"
@@ -70,3 +177,45 @@ create_interpreter() {
   assert_line "--set-rpath ${BATS_TEST_TMPDIR}/prefix/lib ${lib}/lib-dynload/_ssl.cpython-312-x86_64-linux-gnu.so"
   refute_line "--set-rpath ${BATS_TEST_TMPDIR}/prefix/lib ${lib}/site-packages/numpy/_multiarray.so"
 }
+
+@test "relocates macOS load commands without changing unrelated entries" {
+  local prefix="${BATS_TEST_TMPDIR}/prefix"
+  local old="/build prefix/3.12.7"
+  local lib="${prefix}/lib/python3.12"
+  create_macos_tree
+  stub_macos_tools
+
+  run pyenv-binary-relocate "$prefix" "$old"
+  assert_success
+  run cat "${BATS_TEST_TMPDIR}/install-name-tool.log"
+  assert_output <<EOF
+-change ${old}/lib/libpython3.12.dylib @rpath/libpython3.12.dylib ${prefix}/bin/python3.12
+-rpath ${old}/lib @executable_path/../lib ${prefix}/bin/python3.12
+-id @rpath/libpython3.12.dylib ${prefix}/lib/libpython3.12.dylib
+-rpath ${old}/lib @loader_path ${prefix}/lib/libpython3.12.dylib
+-rpath ${old}/lib @loader_path/../.. ${lib}/lib-dynload/_ssl.cpython-312-darwin.so
+EOF
+  run grep -F "${lib}/site-packages/numpy/_multiarray.so" "${BATS_TEST_TMPDIR}/otool.log"
+  assert_failure
+}
+
+@test "skips already relocated macOS load commands" {
+  local prefix="${BATS_TEST_TMPDIR}/prefix"
+  create_macos_tree
+  stub_macos_tools
+  export OTOOL_RELOCATED=1
+
+  run pyenv-binary-relocate "$prefix" "/build prefix/3.12.7"
+  assert_success
+  assert [ ! -e "${BATS_TEST_TMPDIR}/install-name-tool.log" ]
+}
+
+@test "fails when install_name_tool cannot modify a selected file" {
+  local prefix="${BATS_TEST_TMPDIR}/prefix"
+  create_macos_tree
+  stub_macos_tools
+  create_path_executable install_name_tool 'exit 1'
+
+  run pyenv-binary-relocate "$prefix" "/build prefix/3.12.7"
+  assert_failure ""
+}

plugins/pyenv-binary/test/save.bats

@@ -79,6 +79,15 @@ platform() {
   assert_line "archive=3.12.7-$(platform).tar.gz"
 }
 
+@test "records the original installation prefix" {
+  create_version "3.12.7"
+  local out="${BATS_TEST_TMPDIR}/dist"
+  pyenv-binary-save "3.12.7" "$out" >/dev/null
+
+  run grep '^build_prefix=' "${out}/3.12.7-$(platform).meta"
+  assert_success "build_prefix=${PYENV_ROOT}/versions/3.12.7"
+}
+
 @test "fails when readelf is not available" {
   create_version "3.12.7"
   create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'