From d9182d6edc5ab978c586a602cba3236e2d7b2a37 Mon Sep 17 00:00:00 2001 From: native-api Date: Fri, 9 Jan 2026 09:52:09 +0300 Subject: [PATCH] CI: automatically check for CPython releases (#3388) * .gitignore local venv * Add alt GNU mirror support with * Fix UnboundLocalError when no micro 0 releases * Cutoff for existing vetsions as well * add missing dependency more_itertools * workaround fake_useragent 2.0.0 falsely declaring 3.8 compatibility --- .github/workflows/add_version.yml | 45 ++++++++++++++++++ plugins/python-build/scripts/.gitignore | 1 + plugins/python-build/scripts/add_cpython.py | 46 ++++++++++++++----- plugins/python-build/scripts/requirements.txt | 2 + 4 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/add_version.yml create mode 100644 plugins/python-build/scripts/.gitignore diff --git a/.github/workflows/add_version.yml b/.github/workflows/add_version.yml new file mode 100644 index 00000000..c4b31efb --- /dev/null +++ b/.github/workflows/add_version.yml @@ -0,0 +1,45 @@ +name: Add versions + +on: + - workflow_dispatch + - schedule: + # Every N hours + - cron: '* */8 * * *' + +permissions: + contents: write + pull-requests: write + +jobs: + add_cpython: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: 3 + cache: 'pip' + cache-dependency-path: plugins/python-build/scripts/requirements.txt + - run: pip install -r plugins/python-build/scripts/requirements.txt + + - name: check for a release + run: | + python plugins/python-build/scripts/add_cpython.py --verbose >added_versions.lst && rc=$? || rc=$? + echo "rc=$rc" >> $GITHUB_ENV + - name: set PR properties + shell: python + run: | + import os + import sys + versions=[l.rstrip() for l in open("added_versions.lst")] + with open(os.environ['GITHUB_ENV'],'a') as f: + f.write(f'branch_name=auto_add_version/{"_".join(versions)}\n') + f.write(f'pr_name=Add {", ".join(versions)}\n') + os.remove("added_versions.lst") + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v8 + if: env.rc == 0 + with: + branch: ${{ env.branch_name }} + title: ${{ env.pr_name }} diff --git a/plugins/python-build/scripts/.gitignore b/plugins/python-build/scripts/.gitignore new file mode 100644 index 00000000..f9606a37 --- /dev/null +++ b/plugins/python-build/scripts/.gitignore @@ -0,0 +1 @@ +/venv diff --git a/plugins/python-build/scripts/add_cpython.py b/plugins/python-build/scripts/add_cpython.py index a21ae661..44551bc3 100755 --- a/plugins/python-build/scripts/add_cpython.py +++ b/plugins/python-build/scripts/add_cpython.py @@ -30,8 +30,6 @@ import tqdm logger = logging.getLogger(__name__) -REPO = "https://www.python.org/ftp/python/" - CUTOFF_VERSION=packaging.version.Version('3.9') EXCLUDED_VERSIONS= { packaging.version.Version("3.9.3") #recalled upstream @@ -126,6 +124,7 @@ def add_version(version: packaging.version.Version): handle_t_thunks(version, previous_version, is_prerelease_upgrade) + print(version) return True @@ -170,7 +169,7 @@ def main(): if v.micro == 0 and v not in VersionDirectory.existing): # may actually be a prerelease VersionDirectory.available.get_store_available_source_downloads(initial_release, True) - del initial_release + del initial_release versions_to_add = sorted(VersionDirectory.available.keys() - VersionDirectory.existing.keys()) @@ -294,12 +293,16 @@ class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo, super().__init__(seq) self._session = session - def populate(self, url=REPO, pattern=r'^\d+'): + def populate(self): """ Fetch remote versions """ logger.info("Fetching available CPython versions") - for name, url in DownloadPage.enum_download_entries(url, pattern, self._session): + for name, url in DownloadPage.enum_download_entries( + "https://www.python.org/ftp/python/", + r'^(\d+.*)/$', self._session, + make_name= lambda m: m.group(1) + ): v = packaging.version.Version(name) if v < CUTOFF_VERSION or v in EXCLUDED_VERSIONS: continue @@ -341,11 +344,13 @@ class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo, )) if not exact_download_found: + actual_version = max(additional_versions_found.keys()) + logger.debug(f"Refining available version {version} to {actual_version}") del self[version] self.append( additional_versions_found[ - max(additional_versions_found.keys()) + actual_version ]) @@ -368,6 +373,8 @@ class CPythonExistingScriptsDirectory(KeyedList[_CPythonExistingScriptInfo, pack continue try: v = packaging.version.Version(entry_name) + if v < CUTOFF_VERSION: + continue # branch tip scrpts are different from release scripts and thus unusable as a pattern if v.dev is not None: continue @@ -488,19 +495,34 @@ class DownloadPage: url: str @classmethod - def enum_download_entries(cls, url, pattern, session=None) -> typing.Generator[_DownloadPageEntry, None, None]: + def enum_download_entries(cls, url, pattern, session=None, + make_name = lambda m: m.string ) \ + -> typing.Generator[_DownloadPageEntry, None, None]: + """ + Enum download entries in a standard Apache directory page + (incl. CPython download page https://www.python.org/ftp/python/) + or a GNU mirror directory page + (https://ftpmirror.gnu.org// destinations) + """ if session is None: session = requests_html.HTMLSession() response = session.get(url) page = response.html table = page.find("pre", first=True) - # the first entry is ".." - links = table.find("a")[1:] + # some GNU mirrors format entries as a table + # (e.g. https://mirrors.ibiblio.org/gnu/readline/) + if table is None: + table = page.find("table", first=True) + links = table.find("a") for link in links: - name = link.text.rstrip('/') - if not re.match(pattern, name): + href = link.attrs['href'] + # CPython entries are directories + name = link.text + # skip directory entries + if not (m:=re.match(pattern, name)): continue - yield cls._DownloadPageEntry(name, urllib.parse.urljoin(response.url, link.attrs['href'])) + name = make_name(m) + yield cls._DownloadPageEntry(name, urllib.parse.urljoin(response.url, href)) class Re: diff --git a/plugins/python-build/scripts/requirements.txt b/plugins/python-build/scripts/requirements.txt index d7bf7606..c79b3e29 100644 --- a/plugins/python-build/scripts/requirements.txt +++ b/plugins/python-build/scripts/requirements.txt @@ -1,4 +1,6 @@ +more_itertools requests-html +fake_useragent<2 lxml[html_clean] packaging requests