diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c87dfcf3..6123425e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,17 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.5 + hooks: + - id: ruff + args: [--exit-non-zero-on-fix] + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-merge-conflict + - id: check-toml - id: check-yaml exclude: windows-release/azure-pipelines.yml - id: debug-statements diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000..91079503 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,23 @@ +fix = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle errors + "F", # pyflakes errors + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "RUF100", # unused noqa (yesqa) + "UP", # pyupgrade + "W", # pycodestyle warnings + "YTT", # flake8-2020 +] +ignore = [ + "E203", # Whitespace before ':' + "E221", # Multiple spaces before operator + "E226", # Missing whitespace around arithmetic operator + "E241", # Multiple spaces after ',' + "E501", # Line too long +] diff --git a/add-to-pydotorg.py b/add-to-pydotorg.py index c45b56f7..e76447f4 100755 --- a/add-to-pydotorg.py +++ b/add-to-pydotorg.py @@ -22,19 +22,18 @@ Georg Brandl, March 2014. """ -from __future__ import print_function import hashlib import json import os -from os import path import re -import sys -import time import subprocess +import sys +from os import path import requests + # Copied from release.py def error(*msgs): print('**ERROR**', file=sys.stderr) @@ -138,7 +137,7 @@ def slug_for(release): ('-' + release[len(base_version(release)):] if release[len(base_version(release)):] else '') def sigfile_for(release, rfile): - return download_root + '%s/%s.asc' % (release, rfile) + return download_root + f'{release}/{rfile}.asc' def md5sum_for(release, rfile): return hashlib.md5(open(ftp_root + base_version(release) + '/' + rfile, 'rb').read()).hexdigest() @@ -164,33 +163,33 @@ def minor_version_tuple(release): def build_file_dict(release, rfile, rel_pk, file_desc, os_pk, add_download, add_desc): """Return a dictionary with all needed fields for a ReleaseFile object.""" - d = dict( - name = file_desc, - slug = slug_for(release) + '-' + make_slug(file_desc)[:40], - os = '/api/v1/downloads/os/%s/' % os_pk, - release = '/api/v1/downloads/release/%s/' % rel_pk, - description = add_desc, - is_source = os_pk == 3, - url = download_root + '%s/%s' % (base_version(release), rfile), - md5_sum = md5sum_for(release, rfile), - filesize = filesize_for(release, rfile), - download_button = add_download, - ) + d = { + 'name': file_desc, + 'slug': slug_for(release) + '-' + make_slug(file_desc)[:40], + 'os': '/api/v1/downloads/os/%s/' % os_pk, + 'release': '/api/v1/downloads/release/%s/' % rel_pk, + 'description': add_desc, + 'is_source': os_pk == 3, + 'url': download_root + f'{base_version(release)}/{rfile}', + 'md5_sum': md5sum_for(release, rfile), + 'filesize': filesize_for(release, rfile), + 'download_button': add_download, + } # Upload GPG signature - if os.path.exists(ftp_root + "%s/%s.asc" % (base_version(release), rfile)): + if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.asc"): d["gpg_signature_file"] = sigfile_for(base_version(release), rfile) # Upload Sigstore signature - if os.path.exists(ftp_root + "%s/%s.sig" % (base_version(release), rfile)): - d["sigstore_signature_file"] = download_root + '%s/%s.sig' % (base_version(release), rfile) + if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sig"): + d["sigstore_signature_file"] = download_root + f'{base_version(release)}/{rfile}.sig' # Upload Sigstore certificate - if os.path.exists(ftp_root + "%s/%s.crt" % (base_version(release), rfile)): - d["sigstore_cert_file"] = download_root + '%s/%s.crt' % (base_version(release), rfile) + if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.crt"): + d["sigstore_cert_file"] = download_root + f'{base_version(release)}/{rfile}.crt' # Upload Sigstore bundle - if os.path.exists(ftp_root + "%s/%s.sigstore" % (base_version(release), rfile)): - d["sigstore_bundle_file"] = download_root + '%s/%s.sigstore' % (base_version(release), rfile) + if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sigstore"): + d["sigstore_bundle_file"] = download_root + f'{base_version(release)}/{rfile}.sigstore' # Upload SPDX SBOM file - if os.path.exists(ftp_root + "%s/%s.spdx.json" % (base_version(release), rfile)): - d["sbom_spdx2_file"] = download_root + '%s/%s.spdx.json' % (base_version(release), rfile) + if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.spdx.json"): + d["sbom_spdx2_file"] = download_root + f'{base_version(release)}/{rfile}.spdx.json' return d @@ -206,19 +205,19 @@ def list_files(release): if rfile.startswith(prefix): break else: - print(' File %s/%s has wrong prefix' % (reldir, rfile)) + print(f' File {reldir}/{rfile} has wrong prefix') continue if rfile.endswith('.chm'): if rfile[:-4] != 'python' + release.replace('.', ''): - print(' File %s/%s has a different version' % (reldir, rfile)) + print(f' File {reldir}/{rfile} has a different version') continue else: try: prefix, rest = rfile.split('-', 1) - except: + except: # noqa: E722 prefix, rest = rfile, '' if not rest.startswith((release + '-', release + '.')): - print(' File %s/%s has a different version' % (reldir, rfile)) + print(f' File {reldir}/{rfile} has a different version') continue for rx, info in get_file_descriptions(release): if rx.search(rfile): @@ -226,16 +225,16 @@ def list_files(release): yield rfile, file_desc, os_pk, add_download, add_desc break else: - print(' File %s/%s not recognized' % (reldir, rfile)) + print(f' File {reldir}/{rfile} not recognized') continue def query_object(objtype, **params): """Find an API object by query parameters.""" uri = base_url + 'downloads/%s/' % objtype - uri += '?' + '&'.join('%s=%s' % v for v in params.items()) + uri += '?' + '&'.join(f'{k}={v}' for k, v in params.items()) resp = requests.get(uri, headers=headers) if resp.status_code != 200 or not json.loads(resp.text)['objects']: - raise RuntimeError('no object for %s params=%r' % (objtype, params)) + raise RuntimeError(f'no object for {objtype} params={params!r}') obj = json.loads(resp.text)['objects'][0] return int(obj['resource_uri'].strip('/').split('/')[-1]) @@ -248,9 +247,9 @@ def post_object(objtype, datadict): info = json.loads(resp.text) print(info.get('error_message', 'No error message.')) print(info.get('traceback', '')) - except: + except: # noqa: E722 pass - print('Creating %s failed: %s' % (objtype, resp.status_code)) + print(f'Creating {objtype} failed: {resp.status_code}') return -1 newloc = resp.headers['Location'] pk = int(newloc.strip('/').split('/')[-1]) @@ -258,7 +257,7 @@ def post_object(objtype, datadict): def sign_release_files_with_sigstore(release, release_files): filenames = [ - ftp_root + "%s/%s" % (base_version(release), rfile) + ftp_root + f"{base_version(release)}/{rfile}" for rfile, file_desc, os_pk, add_download, add_desc in release_files ] diff --git a/release.py b/release.py index 71807002..115e5a00 100755 --- a/release.py +++ b/release.py @@ -12,12 +12,11 @@ import optparse import os import re -import readline # noqa +import readline # noqa: F401 import shutil import subprocess import sys import tempfile - from contextlib import contextmanager COMMASPACE = ', ' @@ -72,7 +71,7 @@ def chdir_to_repo_root(): def test_first_line(filename, test): if not os.path.exists(filename): return False - with open(filename, "rt") as f: + with open(filename) as f: lines = f.read().split('\n') if not (lines and test(lines[0])): return False @@ -168,12 +167,12 @@ def tweak_patchlevel(tag, done=False): /* Version as a string */ #define PY_VERSION \t\"{tag.text}{plus}"'''.strip() - level_def = dict( - a = 'PY_RELEASE_LEVEL_ALPHA', - b = 'PY_RELEASE_LEVEL_BETA', - rc = 'PY_RELEASE_LEVEL_GAMMA', - f = 'PY_RELEASE_LEVEL_FINAL', - )[tag.level] + level_def = { + 'a': 'PY_RELEASE_LEVEL_ALPHA', + 'b': 'PY_RELEASE_LEVEL_BETA', + 'rc': 'PY_RELEASE_LEVEL_GAMMA', + 'f': 'PY_RELEASE_LEVEL_FINAL', + }[tag.level] new_constants = template.format(tag=tag, level_def=level_def, plus=done and '+' or '') if tag.as_tuple() >= (3, 7, 0, 'a', 3): @@ -254,7 +253,7 @@ def tarball(source, clamp_mtime): # Sorts the entries in the tarball by name. "--sort=name", # Sets a maximum 'modified time' of entries in tarball. - "--mtime=%s" % (clamp_mtime,), "--clamp-mtime", + f"--mtime={clamp_mtime}", "--clamp-mtime", # Sets the owner uid and gid to 0. "--owner=0", "--group=0", "--numeric-owner", # Omits process ID, file access, and status change times. @@ -370,7 +369,7 @@ def build_docs(): with tempfile.TemporaryDirectory() as venv: run_cmd(['python3', '-m', 'venv', venv]) pip = os.path.join(venv, 'bin', 'pip') - run_cmd([pip, 'install', '-r' 'Doc/requirements.txt']) + run_cmd([pip, 'install', '-r', 'Doc/requirements.txt']) sphinx_build = os.path.join(venv, 'bin', 'sphinx-build') blurb = os.path.join(venv, 'bin', 'blurb') with pushd('Doc'): @@ -389,11 +388,11 @@ def scp(from_loc, to_loc): scp('src', '/data/python-releases/%s' % tag.nickname) print("Upload doc tarballs") scp('docs', '/data/python-releases/doc/%s' % tag.nickname) - print("* Now change the permissions on the tarballs so they are " \ - "writable by the webmaster group. *") + print("* Now change the permissions on the tarballs so they are " + "writable by the webmaster group. *") -class Tag(object): +class Tag: def __init__(self, tag_name): # if tag is ".", use current directory name as tag @@ -424,13 +423,13 @@ def __init__(self, tag_name): self.text = self.normalized() if self.level != "f": self.text += self.level + str(self.serial) - self.basic_version = '%s.%s' % (self.major, self.minor) + self.basic_version = f'{self.major}.{self.minor}' def __str__(self): return self.text def normalized(self): - return "{}.{}.{}".format(self.major, self.minor, self.patch) + return f"{self.major}.{self.minor}.{self.patch}" @property def branch(self): @@ -467,7 +466,7 @@ def committed_at(self): # Fetch the epoch of the tagged commit for build reproducibility. proc = subprocess.run(["git", "log", self.gitname, "-1", "--pretty=%ct"], stdout=subprocess.PIPE) if proc.returncode != 0: - error("Couldn't fetch the epoch of tag %s" % (self.gitname,)) + error(f"Couldn't fetch the epoch of tag {self.gitname}") return datetime.datetime.fromtimestamp(int(proc.stdout.decode().strip()), tz=datetime.timezone.utc) diff --git a/run_release.py b/run_release.py index 5b96b69d..60131dfe 100755 --- a/run_release.py +++ b/run_release.py @@ -11,7 +11,6 @@ import contextlib import functools import getpass -import itertools import json import os import pathlib @@ -19,13 +18,12 @@ import shelve import shutil import subprocess -import tempfile +import sys import time import urllib.request -import sys from dataclasses import dataclass from shelve import DbfilenameShelf -from typing import Callable, Iterator, List, Optional, final +from typing import Callable, Iterator, List, Optional import aiohttp import gnupg @@ -34,8 +32,8 @@ from alive_progress import alive_bar import release as release_mod -from buildbotapi import BuildBotAPI import sbom +from buildbotapi import BuildBotAPI API_KEY_REGEXP = re.compile(r"(?P\w+):(?P\w+)") @@ -443,7 +441,7 @@ def run_autoconf(db: DbfilenameShelf) -> None: def check_pyspecific(db): with open( - db["git_repo"] / "Doc" / "tools" / "extensions" / "pyspecific.py", "r" + db["git_repo"] / "Doc" / "tools" / "extensions" / "pyspecific.py" ) as pyspecific: for line in pyspecific: if "SOURCE_URI =" in line: @@ -557,20 +555,20 @@ def put_dir(self, source, target, progress=None): for item in os.listdir(source): if os.path.isfile(os.path.join(source, item)): progress.text(item) - self.put(os.path.join(source, item), "%s/%s" % (target, item)) + self.put(os.path.join(source, item), f"{target}/{item}") progress() else: - self.mkdir("%s/%s" % (target, item), ignore_existing=True) + self.mkdir(f"{target}/{item}", ignore_existing=True) self.put_dir( os.path.join(source, item), - "%s/%s" % (target, item), + f"{target}/{item}", progress=progress, ) def mkdir(self, path, mode=511, ignore_existing=False): try: super().mkdir(path, mode) - except IOError: + except OSError: if ignore_existing: pass else: diff --git a/sbom.py b/sbom.py index b2c26da4..432060a9 100644 --- a/sbom.py +++ b/sbom.py @@ -21,9 +21,9 @@ import subprocess import sys import tarfile +import typing import zipfile from urllib.request import urlopen -import typing def spdx_id(value: str) -> str: diff --git a/size.py b/size.py index d00aaca6..6b4473c3 100755 --- a/size.py +++ b/size.py @@ -2,16 +2,16 @@ """Dump the sizes for the given files, suitable for pasting into content.ht""" +import hashlib import os import sys -import hashlib DOT = '.' # For consistency with historical use. -sort_order = dict((ext, i) for i, ext in enumerate( - ('tgz', 'tar.bz2', 'tar.xz', 'pdb.zip', 'amd64.msi', 'msi', 'chm', 'dmg'))) +sort_order = {ext: i for i, ext in enumerate( + ('tgz', 'tar.bz2', 'tar.xz', 'pdb.zip', 'amd64.msi', 'msi', 'chm', 'dmg'))} def ignore(filename): @@ -35,4 +35,4 @@ def key(filename): with open(filename, 'rb') as fp: md5.update(fp.read()) size = os.stat(filename).st_size - print(' {} {:8} {}'.format(md5.hexdigest(), size, filename)) + print(f' {md5.hexdigest()} {size:8} {filename}') diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 7cce88b6..1b621e19 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -1,7 +1,7 @@ -import pathlib +import hashlib import json +import pathlib import random -import hashlib import re import unittest.mock diff --git a/windows-release/purge.py b/windows-release/purge.py index c5f2c10c..18d214fa 100644 --- a/windows-release/purge.py +++ b/windows-release/purge.py @@ -9,8 +9,7 @@ import re import sys - -from urllib.request import * +from urllib.request import Request, urlopen VERSION_RE = re.compile(r'(\d+\.\d+\.\d+)([A-Za-z_]+\d+)?$') @@ -24,7 +23,7 @@ print('Missing version argument. Expected something like "3.5.1rc1"') sys.exit(1) -URL = "https://www.python.org/ftp/python/{}/".format(m.group(1)) +URL = f"https://www.python.org/ftp/python/{m.group(1)}/" REL = m.group(2) or '' FILES = [ @@ -64,18 +63,18 @@ "Windows8-RT-KB2999226-x86.msu", ] PATHS = [ - "python-{}.exe".format(m.group(0)), - "python-{}-webinstall.exe".format(m.group(0)), - "python-{}-amd64.exe".format(m.group(0)), - "python-{}-amd64-webinstall.exe".format(m.group(0)), - "python-{}-arm64.exe".format(m.group(0)), - "python-{}-arm64-webinstall.exe".format(m.group(0)), - "python-{}-embed-amd64.zip".format(m.group(0)), - "python-{}-embed-win32.zip".format(m.group(0)), - "python-{}-embed-arm64.zip".format(m.group(0)), - *["win32{}/{}".format(REL, f) for f in FILES], - *["amd64{}/{}".format(REL, f) for f in FILES], - *["arm64{}/{}".format(REL, f) for f in FILES], + f"python-{m.group(0)}.exe", + f"python-{m.group(0)}-webinstall.exe", + f"python-{m.group(0)}-amd64.exe", + f"python-{m.group(0)}-amd64-webinstall.exe", + f"python-{m.group(0)}-arm64.exe", + f"python-{m.group(0)}-arm64-webinstall.exe", + f"python-{m.group(0)}-embed-amd64.zip", + f"python-{m.group(0)}-embed-win32.zip", + f"python-{m.group(0)}-embed-arm64.zip", + *[f"win32{REL}/{f}" for f in FILES], + *[f"amd64{REL}/{f}" for f in FILES], + *[f"arm64{REL}/{f}" for f in FILES], ] PATHS = PATHS + [p + ".asc" for p in PATHS]