pyenv/plugins/python-build/scripts/add_pypy.py

#!/usr/bin/env python3
"""Rudimentary script to add PyPY releases automagically.
Requires requests-html.

Checks the PyPy download archives for new versions,
then writes a build script for any which do not exist locally,
saving it to plugins/python-build/share/python-build.

Written and tested in *CPython* 3.9.6. PyPy3 compatibility is not
guaranteed, but it should work.
"""
from pathlib import Path
import argparse
import logging
import re
import sys

from requests_html import HTMLSession

# https://downloads.python.org/pypy/versions.json doesn't contain
# checksums, so using checksums.html instead.
# More convenient than calculating them from downloads.
PYPY_REPO = "https://downloads.python.org/pypy/"
PYPY_CHECKSUMS = "https://pypy.org/checksums.html"

here = Path(__file__).resolve()
OUT_DIR: Path = here.parent.parent / "share" / "python-build"

EXCLUDED_RELEASES = []  # none for now

# Note: Remember to escape/unescape curly braces when editing the bash script
SCRIPT_BIN = """
VERSION='{version_pypy}'
PYVER='{version_python}'

# https://www.pypy.org/checksums.html
aarch64_hash={checksum_aarch64}
linux32_hash={checksum_linux32}
linux64_hash={checksum_linux64}
osarm64_hash={checksum_osarm64}
osx64_hash={checksum_osx64}

### end of manual settings - following lines same for every download

function err_no_binary {{
    local archmsg="${{1}}"
    local ver="pypy${{PYVER}}-v${{VERSION}}-src"
    local url="https://downloads.python.org/pypy/${{ver}}.tar.{file_ext}"
    {{ echo
      colorize 1 "ERROR"
      echo ": The binary distribution of PyPy is not available for ${{archmsg}}."
      echo "try '${{url}}' to build from source."
      echo
    }} >&2
    exit 1
}}

function pypy_pkg_data {{
    # pypy architecture tag
    local ARCH="${{1}}"

    # defaults
    local cmd='install_package'  # use {file_ext}
    local pkg="${{ARCH}}" # assume matches
    local ext='tar.{file_ext}'
    local hash='' # undefined

    # select the hash, fix pkg if not match ARCH
    case "${{ARCH}}" in
    'linux-aarch64' )
      hash="${{aarch64_hash}}"
      pkg='aarch64'
      ;;
    'linux' )
      hash="${{linux32_hash}}"
      pkg='linux32'
      ;;
    'linux64' )
      hash="${{linux64_hash}}"
      ;;
    'osarm64' )
      hash="${{osarm64_hash}}"
      pkg='macos_arm64'
      ;;
    'osx64' )
      if require_osx_version "10.15"; then
        hash="${{osx64_hash}}"
        pkg='macos_x86_64'
      else
        err_no_binary "${{ARCH}}, OS X < 10.15"
      fi
      ;;
    * )
      err_no_binary "${{ARCH}}"
      ;;
    esac

    local basever="pypy${{PYVER}}-v${{VERSION}}"
    local baseurl="https://downloads.python.org/pypy/${{basever}}"

    # result - command, package dir, url+hash
    echo "${{cmd}}" "${{basever}}-${{pkg}}" "${{baseurl}}-${{pkg}}.${{ext}}#${{hash}}"
}}

# determine command, package directory, url+hash
declare -a pd="$(pypy_pkg_data "$(pypy_architecture 2>/dev/null || true)")"

# install
${{pd[0]}} "${{pd[1]}}" "${{pd[2]}}" 'pypy' "verify_py${{PYVER//./}}" '{ensurepip}'
""".lstrip()

SCRIPT_SRC = """
VERSION='{version_pypy}'
PYVER='{version_python}'

# https://www.pypy.org/checksums.html
hash={checksum_src}

### end of manual settings - following lines same for every download

ver="pypy${{PYVER}}-v${{VERSION}}-src"
url="https://downloads.python.org/pypy/${{ver}}.tar.{file_ext}"

prefer_openssl11
install_package "openssl-1.1.1f" "https://www.openssl.org/source/openssl-1.1.1f.tar.gz#186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35" mac_openssl --if has_broken_mac_openssl
install_package "${{ver}}" "${{url}}#${{hash}}" 'pypy_builder' "verify_py${{PYVER//./}}" '{ensurepip}'
""".lstrip()


args = None
logger = logging.getLogger(__name__)


def main() -> bool:
    html_session = HTMLSession()
    response = html_session.get(PYPY_CHECKSUMS)
    page = response.html
    checksum_blocks = page.find("pre.literal-block")
    for csum_block in checksum_blocks:
        python_version, pypy_version = parse_pypy_ver(csum_block.text)
        py_it = f"pypy{python_version}-{pypy_version}"
        if py_it not in EXCLUDED_RELEASES:
            vbin_path = OUT_DIR.joinpath(py_it)
            vsrc_path = OUT_DIR.joinpath(py_it+"-src")

            kwargs = None
            for ps in ((vbin_path, SCRIPT_BIN), (vsrc_path, SCRIPT_SRC)):
                if not ps[0].is_file():
                    kwargs = kwargs or parse_checksums_ext(csum_block.text)
                    code = ps[1].format(
                            version_pypy=pypy_version,
                            version_python=python_version,
                            ensurepip="ensurepip" if not python_version == "2.7" else "ensurepip_lt21",
                            **kwargs  # which are:
                                      # checksum_aarch64
                                      # checksum_linux32
                                      # checksum_linux64
                                      # checksum_osarm64
                                      # checksum_osx64
                                      # checksum_src
                                      # file_ext
                        )
                    if not args.dry_run:
                        with open(file=ps[0], mode="w", encoding="utf-8") as f:
                            f.write(code)
                            f.close()
                    else:
                        print(f"---- DRY RUN SCRIPT FOR {ps[0]} ----")
                        print(code)


def parse_pypy_ver(csum_block:str) -> tuple[str,str]:
    """Gets the PyPy name from the checksum block."""
    match = re.findall(r"pypy([0-9]\.[0-9]+)\-v([0-9]\.[0-9]\.[0-9]+)", csum_block)
    match = set(match)
    if len(match) == 1:
        return match.pop()
    return ""


def parse_checksums_ext(csum_block:str) -> dict:
    """Gets the checksums and ext for each platform (excluding win64) from the checksum block."""
    chksm_ex = {
        r"aarch64": "",
        r"linux32": "",
        r"linux64": "",
        r"macos_arm64": "",
        r"macos_x86_64": "",
        r"src": "",
    }
    ext = set()
    for ex in chksm_ex:
        match = re.findall(r"([a-z0-9]+) pypy[0-9]\.[0-9]+\-v[0-9]\.[0-9]\.[0-9]+-"+ex+r".tar.(gz|bz2)", csum_block)
        chksm_ex[ex] = match[0][0]
        ext.add(match[0][1])
    # if both gz and bz2, prefer bz2
    ext = ext.pop() if len(ext) == 1 else "bz2"
    return {
        "checksum_aarch64": chksm_ex["aarch64"],
        "checksum_linux32": chksm_ex["linux32"],
        "checksum_linux64": chksm_ex["linux64"],
        "checksum_osarm64": chksm_ex["macos_arm64"],
        "checksum_osx64": chksm_ex["macos_x86_64"],
        "checksum_src": chksm_ex["src"],
        "file_ext": ext
    }


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "-d", "--dry-run", action="store_true",
        help="Do not write scripts, just report them to stdout",
    )
    args = parser.parse_args()
    #sys.excepthook seems to have no effect in Github Actions
    try:
        sys.exit(main())
    except Exception:
        logging.exception("Unhandled exception occurred")
        sys.exit(2)