From 4b77d83d9d3dc3d6cb8ecb48bb3c9fabfd9e8724 Mon Sep 17 00:00:00 2001 From: native-api Date: Thu, 25 Jun 2026 09:48:57 +0300 Subject: [PATCH] CI: add_version enhancements (#3480) * Default 30 seems enough for our needs * comments on critical knowledge * Cleaner handling of t thunk patches * Fix an error when adding a release for which there's also a prerelease * Workaround excepthook not working in Github Actions * requirements.txt workaround is only needed for old Python --- plugins/python-build/scripts/add_cpython.py | 53 ++++++++++++------- plugins/python-build/scripts/requirements.txt | 2 +- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/plugins/python-build/scripts/add_cpython.py b/plugins/python-build/scripts/add_cpython.py index fc507f5c..cedc6294 100755 --- a/plugins/python-build/scripts/add_cpython.py +++ b/plugins/python-build/scripts/add_cpython.py @@ -31,13 +31,6 @@ import requests_html import sortedcontainers import tqdm -#CI uses exit code 1 as a signal that no new version is found -#so have to produce a different exit code on an exception -def _excepthook(type,value,traceback): - logging.error("Unhandled exception occured",exc_info=(type,value,traceback)) - sys.exit(2) -sys.excepthook = _excepthook - logger = logging.getLogger(__name__) CUTOFF_VERSION=packaging.version.Version('3.10') @@ -176,6 +169,8 @@ def handle_version_patches( logger.info(f"Copying patches from {previous_version} to {version}") shutil.copytree(previous_patches, new_patches) + # Subdir rename as a separate step from upper dir copying/moving + # in case there are patches for dependency packages as well previous_package_patches = new_patches / f"Python-{previous_version}" new_package_patches = new_patches / f"Python-{version}" if is_prerelease_upgrade: @@ -187,10 +182,9 @@ def handle_version_patches( else: previous_package_patches.rename(new_package_patches) - previous_t_patches = patches_dir / f"{previous_version}t" - if previous_t_patches.exists() or previous_t_patches.is_symlink(): - if is_prerelease_upgrade: - previous_t_patches.unlink() + if uses_t_thunks(previous_version) and is_prerelease_upgrade: + (patches_dir / f"{previous_version}t").unlink(missing_ok=True) + if uses_t_thunks(version): (patches_dir / f"{version}t").symlink_to( str(version), target_is_directory=True ) @@ -225,7 +219,7 @@ def cleanup_prerelease_upgrade( def handle_t_thunks(version, previous_version, is_prerelease_upgrade): - if (version.major, version.minor) < (3, 13): + if not uses_t_thunks(version): return # an old thunk may have older version-specific code @@ -244,6 +238,10 @@ def handle_t_thunks(version, previous_version, is_prerelease_upgrade): thunk_path.write_text(T_THUNK, encoding='utf-8') +def uses_t_thunks(version: packaging.version.Version) -> bool: + return (version.major, version.minor) >= (3, 13) + + Arguments: argparse.Namespace def main(): @@ -262,14 +260,20 @@ def main(): # So until we know the release is out, its directory is a potential prerelease directory. # Normally, prereleases are only made for initial releases (x.y.0) -- # but rarely, they may make them for other releases (e.g. 3.14.5). - for release in (v for v in frozenset(VersionDirectory.available.keys()) #refining changes the + for release in (v for v in frozenset(VersionDirectory.available.keys()) #refining alters the #corresponding directory key #which breaks iteration + #over the directory -- #so have to iterate over a copy if v not in VersionDirectory.existing): VersionDirectory.available.get_store_available_source_downloads(release, True) del release + # Excluding versions for which there already are PRs. + # This will prevent us from using advanced features of + # peter-evans/create-pull-request Github Action + # like updating a PR and closing a superseded PR + # but we don't really need them as of this writing. versions_to_add = sorted( VersionDirectory.available.keys() - VersionDirectory.existing.keys() @@ -293,9 +297,8 @@ def get_pending_versions() -> typing.Set[packaging.version.Version]: pending_versions = set() for line in ls_remote.splitlines(): - match = AUTO_ADD_VERSION_REF_RE.fullmatch(line) - if not match: - raise ValueError(f"Unexpected git ls-remote output: {line!r}") + if not (match := AUTO_ADD_VERSION_REF_RE.fullmatch(line)): + raise ValueError(f"Unexpected git ls-remote output line: {line!r}") pending_versions.update( packaging.version.Version(version) for version in match.group("versions").split("_") @@ -453,8 +456,9 @@ class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo, download_version = packaging.version.Version(m.group("version")) if download_version != version: if not refine_mode: - raise ValueError(f"Unexpectedly found a download {name} ({download_version}) " + logger.warning(f"Ignoring download {name} ({download_version}) " f"for {version} at page {entry.download_page_url}") + continue entry_to_fill = additional_versions_found.get_or_create( download_version, download_page_url=entry.download_page_url @@ -467,7 +471,10 @@ class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo, m.group("extension"), m.group('package'), url )) - if not exact_download_found: + # XXX: Exact download not found in non-refine mode never happens now + # 'cuz we first call the function in refine mode. + # Decide what's best to do if it starts to after a logic change. + if not exact_download_found and refine_mode: actual_version = max(additional_versions_found.keys()) logger.debug(f"Refining available version {version} to {actual_version}") del self[version] @@ -533,7 +540,7 @@ class OpenSSLVersionsDirectory(KeyedList[_OpenSSLVersionInfo, packaging.version. if matching: return max(matching, key=lambda release: release.version) - url = "https://api.github.com/repos/openssl/openssl/releases?per_page=100" + url = "https://api.github.com/repos/openssl/openssl/releases" while url: response = requests.get(url, timeout=30) response.raise_for_status() @@ -657,6 +664,7 @@ class DownloadPage: if session is None: session = requests_html.HTMLSession() response = session.get(url, timeout=30) + response.raise_for_status() page = response.html table = page.find("pre", first=True) # some GNU mirrors format entries as a table @@ -736,4 +744,9 @@ class Url: if __name__ == "__main__": - sys.exit(main()) + #sys.excepthook seems to have no effect in Github Actions + try: + sys.exit(main()) + except Exception: + logging.exception("Unhandled exception occured") + sys.exit(2) diff --git a/plugins/python-build/scripts/requirements.txt b/plugins/python-build/scripts/requirements.txt index 07d4f730..2b64919d 100644 --- a/plugins/python-build/scripts/requirements.txt +++ b/plugins/python-build/scripts/requirements.txt @@ -1,6 +1,6 @@ more_itertools requests-html -fake_useragent<2 +fake_useragent<2; python_version < "3.9" lxml[html_clean] packaging requests