CI: automatically check for CPython releases (#3388)

* .gitignore local venv
* Add alt GNU mirror support with <table>
* Fix UnboundLocalError when no micro 0 releases
* Cutoff for existing vetsions as well
* add missing dependency more_itertools
* workaround fake_useragent 2.0.0 falsely declaring 3.8 compatibility

.github/workflows/add_version.yml

@@ -0,0 +1,45 @@
+name: Add versions
+
+on:
+  - workflow_dispatch
+  - schedule:
+    # Every N hours
+    - cron: '* */8 * * *'
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  add_cpython:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6     
+        with:
+          python-version: 3
+          cache: 'pip'
+          cache-dependency-path: plugins/python-build/scripts/requirements.txt
+      - run: pip install -r plugins/python-build/scripts/requirements.txt
+      
+      - name: check for a release
+        run: |
+          python plugins/python-build/scripts/add_cpython.py --verbose >added_versions.lst && rc=$? || rc=$?
+          echo "rc=$rc" >> $GITHUB_ENV
+      - name: set PR properties
+        shell: python
+        run: |
+          import os
+          import sys
+          versions=[l.rstrip() for l in open("added_versions.lst")]
+          with open(os.environ['GITHUB_ENV'],'a') as f:
+            f.write(f'branch_name=auto_add_version/{"_".join(versions)}\n')
+            f.write(f'pr_name=Add {", ".join(versions)}\n')
+          os.remove("added_versions.lst")
+      
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v8
+        if: env.rc == 0
+        with:
+          branch: ${{ env.branch_name }}
+          title: ${{ env.pr_name }}

plugins/python-build/scripts/.gitignore

@@ -0,0 +1 @@
+/venv

plugins/python-build/scripts/add_cpython.py

@@ -30,8 +30,6 @@ import tqdm
 
 logger = logging.getLogger(__name__)
 
-REPO = "https://www.python.org/ftp/python/"
-
 CUTOFF_VERSION=packaging.version.Version('3.9')
 EXCLUDED_VERSIONS= {
     packaging.version.Version("3.9.3")  #recalled upstream
@@ -126,6 +124,7 @@ def add_version(version: packaging.version.Version):
 
     handle_t_thunks(version, previous_version, is_prerelease_upgrade)
 
+    print(version)
     return True
 
 
@@ -170,7 +169,7 @@ def main():
                             if v.micro == 0 and v not in VersionDirectory.existing):
         # may actually be a prerelease
         VersionDirectory.available.get_store_available_source_downloads(initial_release, True)
-    del initial_release
+        del initial_release
 
     versions_to_add = sorted(VersionDirectory.available.keys() - VersionDirectory.existing.keys())
 
@@ -294,12 +293,16 @@ class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo,
         super().__init__(seq)
         self._session = session
 
-    def populate(self, url=REPO, pattern=r'^\d+'):
+    def populate(self):
         """
         Fetch remote versions
         """
         logger.info("Fetching available CPython versions")
-        for name, url in DownloadPage.enum_download_entries(url, pattern, self._session):
+        for name, url in DownloadPage.enum_download_entries(
+                "https://www.python.org/ftp/python/",
+                r'^(\d+.*)/$', self._session,
+                make_name= lambda m: m.group(1)
+        ):
             v = packaging.version.Version(name)
             if v < CUTOFF_VERSION or v in EXCLUDED_VERSIONS:
                 continue
@@ -341,11 +344,13 @@ class CPythonAvailableVersionsDirectory(KeyedList[_CPythonAvailableVersionInfo,
             ))
 
         if not exact_download_found:
+            actual_version = max(additional_versions_found.keys())
+            logger.debug(f"Refining available version {version} to {actual_version}")
             del self[version]
 
             self.append(
                 additional_versions_found[
-                    max(additional_versions_found.keys())
+                    actual_version
                 ])
 
 
@@ -368,6 +373,8 @@ class CPythonExistingScriptsDirectory(KeyedList[_CPythonExistingScriptInfo, pack
                 continue
             try:
                 v = packaging.version.Version(entry_name)
+                if v < CUTOFF_VERSION:
+                    continue
                 # branch tip scrpts are different from release scripts and thus unusable as a pattern
                 if v.dev is not None:
                     continue
@@ -488,19 +495,34 @@ class DownloadPage:
         url: str
 
     @classmethod
-    def enum_download_entries(cls, url, pattern, session=None) -> typing.Generator[_DownloadPageEntry, None, None]:
+    def enum_download_entries(cls, url, pattern, session=None,
+                              make_name = lambda m: m.string ) \
+            -> typing.Generator[_DownloadPageEntry, None, None]:
+        """
+        Enum download entries in a standard Apache directory page
+        (incl. CPython download page https://www.python.org/ftp/python/)
+        or a GNU mirror directory page
+        (https://ftpmirror.gnu.org/<package>/ destinations)
+        """
         if session is None:
             session = requests_html.HTMLSession()
         response = session.get(url)
         page = response.html
         table = page.find("pre", first=True)
-        # the first entry is ".."
-        links = table.find("a")[1:]
+        # some GNU mirrors format entries as a table
+        # (e.g. https://mirrors.ibiblio.org/gnu/readline/)
+        if table is None:
+            table = page.find("table", first=True)
+        links = table.find("a")
         for link in links:
-            name = link.text.rstrip('/')
-            if not re.match(pattern, name):
+            href = link.attrs['href']
+            # CPython entries are directories
+            name = link.text
+            # skip directory entries
+            if not (m:=re.match(pattern, name)):
                 continue
-            yield cls._DownloadPageEntry(name, urllib.parse.urljoin(response.url, link.attrs['href']))
+            name = make_name(m)
+            yield cls._DownloadPageEntry(name, urllib.parse.urljoin(response.url, href))
 
 
 class Re:

plugins/python-build/scripts/requirements.txt

@@ -1,4 +1,6 @@
+more_itertools
 requests-html
+fake_useragent<2
 lxml[html_clean]
 packaging
 requests