python-build: Add add_pypy.py for pypy versions (#3566)

* python-build: add `add_pypy.py` for pypy versions

couldn't help myself ;P

python-build; add_pypy.py: ensure ensurepip is correct

python-build; add_pypy.py: check file extension

python-build; add_pypy.py: update os x version check to 10.15

python-build; add_pypy.py: remove outdated comment

python-build; add_pypy.py: cleanup comment

* Update add_pypy.py

* Remove comments irrelevant to reading the code

plugins/python-build/scripts/add_pypy.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""Rudimentary script to add PyPY releases automagically.
+Requires requests-html.
+
+Checks the PyPy download archives for new versions,
+then writes a build script for any which do not exist locally,
+saving it to plugins/python-build/share/python-build.
+
+Written and tested in *CPython* 3.9.6. PyPy3 compatibility is not
+guaranteed, but it should work.
+"""
+from pathlib import Path
+import argparse
+import logging
+import re
+import sys
+
+from requests_html import HTMLSession
+
+# https://downloads.python.org/pypy/versions.json doesn't contain
+# checksums, so using checksums.html instead.
+# More convenient than calculating them from downloads.
+PYPY_REPO = "https://downloads.python.org/pypy/"
+PYPY_CHECKSUMS = "https://pypy.org/checksums.html"
+
+here = Path(__file__).resolve()
+OUT_DIR: Path = here.parent.parent / "share" / "python-build"
+
+EXCLUDED_RELEASES = []  # none for now
+
+# Note: Remember to escape/unescape curly braces when editing the bash script
+SCRIPT_BIN = """
+VERSION='{version_pypy}'
+PYVER='{version_python}'
+
+# https://www.pypy.org/checksums.html
+aarch64_hash={checksum_aarch64}
+linux32_hash={checksum_linux32}
+linux64_hash={checksum_linux64}
+osarm64_hash={checksum_osarm64}
+osx64_hash={checksum_osx64}
+
+### end of manual settings - following lines same for every download
+
+function err_no_binary {{
+    local archmsg="${{1}}"
+    local ver="pypy${{PYVER}}-v${{VERSION}}-src"
+    local url="https://downloads.python.org/pypy/${{ver}}.tar.{file_ext}"
+    {{ echo
+      colorize 1 "ERROR"
+      echo ": The binary distribution of PyPy is not available for ${{archmsg}}."
+      echo "try '${{url}}' to build from source."
+      echo
+    }} >&2
+    exit 1
+}}
+
+function pypy_pkg_data {{
+    # pypy architecture tag
+    local ARCH="${{1}}"
+
+    # defaults
+    local cmd='install_package'  # use {file_ext}
+    local pkg="${{ARCH}}" # assume matches
+    local ext='tar.{file_ext}'
+    local hash='' # undefined
+
+    # select the hash, fix pkg if not match ARCH
+    case "${{ARCH}}" in
+    'linux-aarch64' )
+      hash="${{aarch64_hash}}"
+      pkg='aarch64'
+      ;;
+    'linux' )
+      hash="${{linux32_hash}}"
+      pkg='linux32'
+      ;;
+    'linux64' )
+      hash="${{linux64_hash}}"
+      ;;
+    'osarm64' )
+      hash="${{osarm64_hash}}"
+      pkg='macos_arm64'
+      ;;
+    'osx64' )
+      if require_osx_version "10.15"; then
+        hash="${{osx64_hash}}"
+        pkg='macos_x86_64'
+      else
+        err_no_binary "${{ARCH}}, OS X < 10.15"
+      fi
+      ;;
+    * )
+      err_no_binary "${{ARCH}}"
+      ;;
+    esac
+
+    local basever="pypy${{PYVER}}-v${{VERSION}}"
+    local baseurl="https://downloads.python.org/pypy/${{basever}}"
+
+    # result - command, package dir, url+hash
+    echo "${{cmd}}" "${{basever}}-${{pkg}}" "${{baseurl}}-${{pkg}}.${{ext}}#${{hash}}"
+}}
+
+# determine command, package directory, url+hash
+declare -a pd="$(pypy_pkg_data "$(pypy_architecture 2>/dev/null || true)")"
+
+# install
+${{pd[0]}} "${{pd[1]}}" "${{pd[2]}}" 'pypy' "verify_py${{PYVER//./}}" '{ensurepip}'
+""".lstrip()
+
+SCRIPT_SRC = """
+VERSION='{version_pypy}'
+PYVER='{version_python}'
+
+# https://www.pypy.org/checksums.html
+hash={checksum_src}
+
+### end of manual settings - following lines same for every download
+
+ver="pypy${{PYVER}}-v${{VERSION}}-src"
+url="https://downloads.python.org/pypy/${{ver}}.tar.{file_ext}"
+
+prefer_openssl11
+install_package "openssl-1.1.1f" "https://www.openssl.org/source/openssl-1.1.1f.tar.gz#186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35" mac_openssl --if has_broken_mac_openssl
+install_package "${{ver}}" "${{url}}#${{hash}}" 'pypy_builder' "verify_py${{PYVER//./}}" '{ensurepip}'
+""".lstrip()
+
+
+args = None
+logger = logging.getLogger(__name__)
+
+
+def main() -> bool:
+    html_session = HTMLSession()
+    response = html_session.get(PYPY_CHECKSUMS)
+    page = response.html
+    checksum_blocks = page.find("pre.literal-block")
+    for csum_block in checksum_blocks:
+        python_version, pypy_version = parse_pypy_ver(csum_block.text)
+        py_it = f"pypy{python_version}-{pypy_version}"
+        if py_it not in EXCLUDED_RELEASES:
+            vbin_path = OUT_DIR.joinpath(py_it)
+            vsrc_path = OUT_DIR.joinpath(py_it+"-src")
+
+            kwargs = None
+            for ps in ((vbin_path, SCRIPT_BIN), (vsrc_path, SCRIPT_SRC)):
+                if not ps[0].is_file():
+                    kwargs = kwargs or parse_checksums_ext(csum_block.text)
+                    code = ps[1].format(
+                            version_pypy=pypy_version,
+                            version_python=python_version,
+                            ensurepip="ensurepip" if not python_version == "2.7" else "ensurepip_lt21",
+                            **kwargs  # which are:
+                                      # checksum_aarch64
+                                      # checksum_linux32
+                                      # checksum_linux64
+                                      # checksum_osarm64
+                                      # checksum_osx64
+                                      # checksum_src
+                                      # file_ext
+                        )
+                    if not args.dry_run:
+                        with open(file=ps[0], mode="w", encoding="utf-8") as f:
+                            f.write(code)
+                            f.close()
+                    else:
+                        print(f"---- DRY RUN SCRIPT FOR {ps[0]} ----")
+                        print(code)
+
+
+def parse_pypy_ver(csum_block:str) -> tuple[str,str]:
+    """Gets the PyPy name from the checksum block."""
+    match = re.findall(r"pypy([0-9]\.[0-9]+)\-v([0-9]\.[0-9]\.[0-9]+)", csum_block)
+    match = set(match)
+    if len(match) == 1:
+        return match.pop()
+    return ""
+
+
+def parse_checksums_ext(csum_block:str) -> dict:
+    """Gets the checksums and ext for each platform (excluding win64) from the checksum block."""
+    chksm_ex = {
+        r"aarch64": "",
+        r"linux32": "",
+        r"linux64": "",
+        r"macos_arm64": "",
+        r"macos_x86_64": "",
+        r"src": "",
+    }
+    ext = set()
+    for ex in chksm_ex:
+        match = re.findall(r"([a-z0-9]+) pypy[0-9]\.[0-9]+\-v[0-9]\.[0-9]\.[0-9]+-"+ex+r".tar.(gz|bz2)", csum_block)
+        chksm_ex[ex] = match[0][0]
+        ext.add(match[0][1])
+    # if both gz and bz2, prefer bz2
+    ext = ext.pop() if len(ext) == 1 else "bz2"
+    return {
+        "checksum_aarch64": chksm_ex["aarch64"],
+        "checksum_linux32": chksm_ex["linux32"],
+        "checksum_linux64": chksm_ex["linux64"],
+        "checksum_osarm64": chksm_ex["macos_arm64"],
+        "checksum_osx64": chksm_ex["macos_x86_64"],
+        "checksum_src": chksm_ex["src"],
+        "file_ext": ext
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-d", "--dry-run", action="store_true",
+        help="Do not write scripts, just report them to stdout",
+    )
+    args = parser.parse_args()
+    #sys.excepthook seems to have no effect in Github Actions
+    try:
+        sys.exit(main())
+    except Exception:
+        logging.exception("Unhandled exception occurred")
+        sys.exit(2)