From 4cad49dc1a7e15335e401ed39ae42c98b06391a7 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 3 May 2020 02:04:54 +0100 Subject: [PATCH 1/8] python3Packages.PyGithub: 1.47 -> 1.51 --- pkgs/development/python-modules/pyGithub/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/pyGithub/default.nix b/pkgs/development/python-modules/pyGithub/default.nix index 2102554face..cf0784024f1 100644 --- a/pkgs/development/python-modules/pyGithub/default.nix +++ b/pkgs/development/python-modules/pyGithub/default.nix @@ -12,14 +12,14 @@ buildPythonPackage rec { pname = "PyGithub"; - version = "1.47"; + version = "1.51"; disabled = !isPy3k; src = fetchFromGitHub { owner = "PyGithub"; repo = "PyGithub"; rev = "v${version}"; - sha256 = "0zvp1gib2lryw698vxkbdv40n3lsmdlhwp7vdcg41dqqa5nfryhn"; + hash = "sha256-8uQCFiw1ByPOX8ZRUlSLYPIibjmd19r/JtTnmQdz5cM="; }; checkInputs = [ httpretty parameterized pytestCheckHook ]; From 83c4ac2eb3ac2aacfc917e96b1a12091889fdfae Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 26 Apr 2020 19:28:24 +0100 Subject: [PATCH 2/8] linux/update-hardened.py: reformat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit $ isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses … $ black --line-length=80 … (per the black documentation) --- .../linux/kernel/update-hardened.py | 138 ++++++++++-------- 1 file changed, 80 insertions(+), 58 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/update-hardened.py b/pkgs/os-specific/linux/kernel/update-hardened.py index 7f6949653af..49a6228fb97 100755 --- a/pkgs/os-specific/linux/kernel/update-hardened.py +++ b/pkgs/os-specific/linux/kernel/update-hardened.py @@ -3,60 +3,68 @@ # This is automatically called by ./update.sh. -import re import json -import sys import os.path -from glob import glob +import re import subprocess +import sys +from glob import glob from tempfile import TemporaryDirectory from github import Github HERE = os.path.dirname(os.path.realpath(__file__)) -HARDENED_GITHUB_REPO = 'anthraxx/linux-hardened' -HARDENED_TRUSTED_KEY = os.path.join(HERE, 'anthraxx.asc') -HARDENED_PATCHES_PATH = os.path.join(HERE, 'hardened-patches.json') +HARDENED_GITHUB_REPO = "anthraxx/linux-hardened" +HARDENED_TRUSTED_KEY = os.path.join(HERE, "anthraxx.asc") +HARDENED_PATCHES_PATH = os.path.join(HERE, "hardened-patches.json") MIN_KERNEL_VERSION = [4, 14] + def run(*args, **kwargs): try: return subprocess.run( - args, **kwargs, - check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + args, + **kwargs, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) except subprocess.CalledProcessError as err: print( - f'error: `{err.cmd}` failed unexpectedly\n' - f'status code: {err.returncode}\n' + f"error: `{err.cmd}` failed unexpectedly\n" + f"status code: {err.returncode}\n" f'stdout:\n{err.stdout.decode("utf-8").strip()}\n' f'stderr:\n{err.stderr.decode("utf-8").strip()}', file=sys.stderr, ) sys.exit(1) + def nix_prefetch_url(url): - output = run('nix-prefetch-url', '--print-path', url).stdout - return output.decode('utf-8').strip().split('\n') + output = run("nix-prefetch-url", "--print-path", url).stdout + return output.decode("utf-8").strip().split("\n") + def verify_openpgp_signature(*, name, trusted_key, sig_path, data_path): - with TemporaryDirectory(suffix='.nixpkgs-gnupg-home') as gnupg_home: - run('gpg', '--homedir', gnupg_home, '--import', trusted_key) - keyring = os.path.join(gnupg_home, 'pubring.kbx') + with TemporaryDirectory(suffix=".nixpkgs-gnupg-home") as gnupg_home: + run("gpg", "--homedir", gnupg_home, "--import", trusted_key) + keyring = os.path.join(gnupg_home, "pubring.kbx") try: subprocess.run( - ('gpgv', '--keyring', keyring, sig_path, data_path), - check=True, stderr=subprocess.PIPE, + ("gpgv", "--keyring", keyring, sig_path, data_path), + check=True, + stderr=subprocess.PIPE, ) return True except subprocess.CalledProcessError as err: print( - f'error: signature for {name} failed to verify!', + f"error: signature for {name} failed to verify!", file=sys.stderr, ) - print(err.stderr.decode('utf-8'), file=sys.stderr, end='') + print(err.stderr.decode("utf-8"), file=sys.stderr, end="") return False + def fetch_patch(*, name, release): def find_asset(filename): try: @@ -68,12 +76,12 @@ def fetch_patch(*, name, release): except StopIteration: raise KeyError(filename) - patch_filename = f'{name}.patch' + patch_filename = f"{name}.patch" try: patch_url = find_asset(patch_filename) - sig_url = find_asset(patch_filename + '.sig') + sig_url = find_asset(patch_filename + ".sig") except KeyError: - print(f'error: {patch_filename}{{,.sig}} not present', file=sys.stderr) + print(f"error: {patch_filename}{{,.sig}} not present", file=sys.stderr) return None sha256, patch_path = nix_prefetch_url(patch_url) @@ -88,59 +96,71 @@ def fetch_patch(*, name, release): return None return { - 'name': patch_filename, - 'url': patch_url, - 'sha256': sha256, + "name": patch_filename, + "url": patch_url, + "sha256": sha256, } + def parse_version(version_str): version = [] - for component in version_str.split('.'): + for component in version_str.split("."): try: version.append(int(component)) except ValueError: version.append(component) return version + def version_string(version): - return '.'.join(str(component) for component in version) + return ".".join(str(component) for component in version) + def major_kernel_version_key(kernel_version): return version_string(kernel_version[:-1]) + def commit_patches(*, kernel_key, message): - with open(HARDENED_PATCHES_PATH + '.new', 'w') as new_patches_file: + with open(HARDENED_PATCHES_PATH + ".new", "w") as new_patches_file: json.dump(patches, new_patches_file, indent=4, sort_keys=True) - new_patches_file.write('\n') - os.rename(HARDENED_PATCHES_PATH + '.new', HARDENED_PATCHES_PATH) - message = f'linux/hardened-patches/{kernel_key}: {message}' + new_patches_file.write("\n") + os.rename(HARDENED_PATCHES_PATH + ".new", HARDENED_PATCHES_PATH) + message = f"linux/hardened-patches/{kernel_key}: {message}" print(message) - if os.environ.get('COMMIT'): + if os.environ.get("COMMIT"): run( - 'git', '-C', HERE, 'commit', f'--message={message}', - 'hardened-patches.json', + "git", + "-C", + HERE, + "commit", + f"--message={message}", + "hardened-patches.json", ) + # Load the existing patches. with open(HARDENED_PATCHES_PATH) as patches_file: patches = json.load(patches_file) -NIX_VERSION_RE = re.compile(r''' - \s* version \s* = - \s* " (?P [^"]*) " - \s* ; \s* \n -''', re.VERBOSE) +NIX_VERSION_RE = re.compile( + r""" + \s* version \s* = + \s* " (?P [^"]*) " + \s* ; \s* \n + """, + re.VERBOSE, +) # Get the set of currently packaged kernel versions. kernel_versions = {} for filename in os.listdir(HERE): - filename_match = re.fullmatch(r'linux-(\d+)\.(\d+)\.nix', filename) + filename_match = re.fullmatch(r"linux-(\d+)\.(\d+)\.nix", filename) if filename_match: with open(os.path.join(HERE, filename)) as nix_file: for nix_line in nix_file: match = NIX_VERSION_RE.fullmatch(nix_line) if match: - kernel_version = parse_version(match.group('version')) + kernel_version = parse_version(match.group("version")) if kernel_version < MIN_KERNEL_VERSION: continue kernel_key = major_kernel_version_key(kernel_version) @@ -148,9 +168,9 @@ for filename in os.listdir(HERE): # Remove patches for unpackaged kernel versions. for kernel_key in sorted(patches.keys() - kernel_versions.keys()): - commit_patches(kernel_key=kernel_key, message='remove') + commit_patches(kernel_key=kernel_key, message="remove") -g = Github(os.environ.get('GITHUB_TOKEN')) +g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo(HARDENED_GITHUB_REPO) failures = False @@ -171,8 +191,8 @@ for release in repo.get_releases(): continue release_info = { - 'version': version, - 'release': release, + "version": version, + "release": release, } if kernel_version == packaged_kernel_version: @@ -182,22 +202,24 @@ for release in repo.get_releases(): # skipping patches for kernels newer than the packaged one. if kernel_version > packaged_kernel_version: continue - elif (kernel_key not in releases or - releases[kernel_key]['version'] < version): + elif ( + kernel_key not in releases + or releases[kernel_key]["version"] < version + ): releases[kernel_key] = release_info # Update hardened-patches.json for each release. for kernel_key, release_info in releases.items(): - release = release_info['release'] - version = release_info['version'] + release = release_info["release"] + version = release_info["version"] version_str = release.tag_name - name = f'linux-hardened-{version_str}' + name = f"linux-hardened-{version_str}" try: - old_filename = patches[kernel_key]['name'] - old_version_str = (old_filename - .replace('linux-hardened-', '') - .replace('.patch', '')) + old_filename = patches[kernel_key]["name"] + old_version_str = old_filename.replace("linux-hardened-", "").replace( + ".patch", "" + ) old_version = parse_version(old_version_str) update = old_version < version except KeyError: @@ -211,17 +233,17 @@ for kernel_key, release_info in releases.items(): else: patches[kernel_key] = patch if old_version: - message = f'{old_version_str} -> {version_str}' + message = f"{old_version_str} -> {version_str}" else: - message = f'init at {version_str}' + message = f"init at {version_str}" commit_patches(kernel_key=kernel_key, message=message) missing_kernel_versions = kernel_versions.keys() - patches.keys() if missing_kernel_versions: print( - f'warning: no patches for kernel versions ' + - ', '.join(missing_kernel_versions), + f"warning: no patches for kernel versions " + + ", ".join(missing_kernel_versions), file=sys.stderr, ) From abe4bef033a8d6b1a82c84d2cd71f50a1624a389 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 26 Apr 2020 04:06:11 +0100 Subject: [PATCH 3/8] linux/update-hardened.py: use pathlib --- .../linux/kernel/update-hardened.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/update-hardened.py b/pkgs/os-specific/linux/kernel/update-hardened.py index 49a6228fb97..bc9110578d6 100755 --- a/pkgs/os-specific/linux/kernel/update-hardened.py +++ b/pkgs/os-specific/linux/kernel/update-hardened.py @@ -4,19 +4,19 @@ # This is automatically called by ./update.sh. import json -import os.path +import os import re import subprocess import sys -from glob import glob +from pathlib import Path from tempfile import TemporaryDirectory from github import Github -HERE = os.path.dirname(os.path.realpath(__file__)) +HERE = Path(__file__).resolve().parent HARDENED_GITHUB_REPO = "anthraxx/linux-hardened" -HARDENED_TRUSTED_KEY = os.path.join(HERE, "anthraxx.asc") -HARDENED_PATCHES_PATH = os.path.join(HERE, "hardened-patches.json") +HARDENED_TRUSTED_KEY = HERE / "anthraxx.asc" +HARDENED_PATCHES_PATH = HERE / "hardened-patches.json" MIN_KERNEL_VERSION = [4, 14] @@ -42,13 +42,15 @@ def run(*args, **kwargs): def nix_prefetch_url(url): output = run("nix-prefetch-url", "--print-path", url).stdout - return output.decode("utf-8").strip().split("\n") + sha256, path = output.decode("utf-8").strip().split("\n") + return sha256, Path(path) def verify_openpgp_signature(*, name, trusted_key, sig_path, data_path): - with TemporaryDirectory(suffix=".nixpkgs-gnupg-home") as gnupg_home: + with TemporaryDirectory(suffix=".nixpkgs-gnupg-home") as gnupg_home_str: + gnupg_home = Path(gnupg_home_str) run("gpg", "--homedir", gnupg_home, "--import", trusted_key) - keyring = os.path.join(gnupg_home, "pubring.kbx") + keyring = gnupg_home / "pubring.kbx" try: subprocess.run( ("gpgv", "--keyring", keyring, sig_path, data_path), @@ -121,10 +123,11 @@ def major_kernel_version_key(kernel_version): def commit_patches(*, kernel_key, message): - with open(HARDENED_PATCHES_PATH + ".new", "w") as new_patches_file: + new_patches_path = HARDENED_PATCHES_PATH.with_suffix(".new") + with open(new_patches_path, "w") as new_patches_file: json.dump(patches, new_patches_file, indent=4, sort_keys=True) new_patches_file.write("\n") - os.rename(HARDENED_PATCHES_PATH + ".new", HARDENED_PATCHES_PATH) + os.rename(new_patches_path, HARDENED_PATCHES_PATH) message = f"linux/hardened-patches/{kernel_key}: {message}" print(message) if os.environ.get("COMMIT"): @@ -156,7 +159,7 @@ kernel_versions = {} for filename in os.listdir(HERE): filename_match = re.fullmatch(r"linux-(\d+)\.(\d+)\.nix", filename) if filename_match: - with open(os.path.join(HERE, filename)) as nix_file: + with open(HERE / filename) as nix_file: for nix_line in nix_file: match = NIX_VERSION_RE.fullmatch(nix_line) if match: From d6fe0a4e2dc2711480f87fe8c9fa9b66323e4c25 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 26 Apr 2020 18:19:02 +0100 Subject: [PATCH 4/8] linux/hardened: move files into directory --- lib/kernel.nix | 2 +- .../linux/kernel/{ => hardened}/anthraxx.asc | 0 .../{hardened-config.nix => hardened/config.nix} | 0 .../patches.json} | 0 .../kernel/{ => hardened}/tag-hardened.patch | 0 .../{update-hardened.py => hardened/update.py} | 16 +++++++++------- pkgs/os-specific/linux/kernel/patches.nix | 4 ++-- pkgs/os-specific/linux/kernel/update.sh | 2 +- pkgs/top-level/all-packages.nix | 2 +- 9 files changed, 14 insertions(+), 12 deletions(-) rename pkgs/os-specific/linux/kernel/{ => hardened}/anthraxx.asc (100%) rename pkgs/os-specific/linux/kernel/{hardened-config.nix => hardened/config.nix} (100%) rename pkgs/os-specific/linux/kernel/{hardened-patches.json => hardened/patches.json} (100%) rename pkgs/os-specific/linux/kernel/{ => hardened}/tag-hardened.patch (100%) rename pkgs/os-specific/linux/kernel/{update-hardened.py => hardened/update.py} (94%) diff --git a/lib/kernel.nix b/lib/kernel.nix index 2ce19f8cb68..8045a228d05 100644 --- a/lib/kernel.nix +++ b/lib/kernel.nix @@ -14,7 +14,7 @@ with lib; freeform = x: { freeform = x; }; /* - Common patterns/legacy used in common-config/hardened-config.nix + Common patterns/legacy used in common-config/hardened/config.nix */ whenHelpers = version: { whenAtLeast = ver: mkIf (versionAtLeast version ver); diff --git a/pkgs/os-specific/linux/kernel/anthraxx.asc b/pkgs/os-specific/linux/kernel/hardened/anthraxx.asc similarity index 100% rename from pkgs/os-specific/linux/kernel/anthraxx.asc rename to pkgs/os-specific/linux/kernel/hardened/anthraxx.asc diff --git a/pkgs/os-specific/linux/kernel/hardened-config.nix b/pkgs/os-specific/linux/kernel/hardened/config.nix similarity index 100% rename from pkgs/os-specific/linux/kernel/hardened-config.nix rename to pkgs/os-specific/linux/kernel/hardened/config.nix diff --git a/pkgs/os-specific/linux/kernel/hardened-patches.json b/pkgs/os-specific/linux/kernel/hardened/patches.json similarity index 100% rename from pkgs/os-specific/linux/kernel/hardened-patches.json rename to pkgs/os-specific/linux/kernel/hardened/patches.json diff --git a/pkgs/os-specific/linux/kernel/tag-hardened.patch b/pkgs/os-specific/linux/kernel/hardened/tag-hardened.patch similarity index 100% rename from pkgs/os-specific/linux/kernel/tag-hardened.patch rename to pkgs/os-specific/linux/kernel/hardened/tag-hardened.patch diff --git a/pkgs/os-specific/linux/kernel/update-hardened.py b/pkgs/os-specific/linux/kernel/hardened/update.py similarity index 94% rename from pkgs/os-specific/linux/kernel/update-hardened.py rename to pkgs/os-specific/linux/kernel/hardened/update.py index bc9110578d6..1ef5acd3eb0 100755 --- a/pkgs/os-specific/linux/kernel/update-hardened.py +++ b/pkgs/os-specific/linux/kernel/hardened/update.py @@ -1,7 +1,7 @@ #! /usr/bin/env nix-shell #! nix-shell -i python -p "python3.withPackages (ps: [ps.PyGithub])" git gnupg -# This is automatically called by ./update.sh. +# This is automatically called by ../update.sh. import json import os @@ -14,9 +14,11 @@ from tempfile import TemporaryDirectory from github import Github HERE = Path(__file__).resolve().parent +NIXPKGS_KERNEL_PATH = HERE.parent +NIXPKGS_PATH = HERE.parents[4] HARDENED_GITHUB_REPO = "anthraxx/linux-hardened" HARDENED_TRUSTED_KEY = HERE / "anthraxx.asc" -HARDENED_PATCHES_PATH = HERE / "hardened-patches.json" +HARDENED_PATCHES_PATH = HERE / "patches.json" MIN_KERNEL_VERSION = [4, 14] @@ -128,16 +130,16 @@ def commit_patches(*, kernel_key, message): json.dump(patches, new_patches_file, indent=4, sort_keys=True) new_patches_file.write("\n") os.rename(new_patches_path, HARDENED_PATCHES_PATH) - message = f"linux/hardened-patches/{kernel_key}: {message}" + message = f"linux/hardened/patches/{kernel_key}: {message}" print(message) if os.environ.get("COMMIT"): run( "git", "-C", - HERE, + NIXPKGS_PATH, "commit", f"--message={message}", - "hardened-patches.json", + HARDENED_PATCHES_PATH, ) @@ -156,10 +158,10 @@ NIX_VERSION_RE = re.compile( # Get the set of currently packaged kernel versions. kernel_versions = {} -for filename in os.listdir(HERE): +for filename in os.listdir(NIXPKGS_KERNEL_PATH): filename_match = re.fullmatch(r"linux-(\d+)\.(\d+)\.nix", filename) if filename_match: - with open(HERE / filename) as nix_file: + with open(NIXPKGS_KERNEL_PATH / filename) as nix_file: for nix_line in nix_file: match = NIX_VERSION_RE.fullmatch(nix_line) if match: diff --git a/pkgs/os-specific/linux/kernel/patches.nix b/pkgs/os-specific/linux/kernel/patches.nix index 1c4af8c32a6..8ce1ac2b587 100644 --- a/pkgs/os-specific/linux/kernel/patches.nix +++ b/pkgs/os-specific/linux/kernel/patches.nix @@ -35,7 +35,7 @@ tag_hardened = { name = "tag-hardened"; - patch = ./tag-hardened.patch; + patch = ./hardened/tag-hardened.patch; }; hardened = let @@ -43,7 +43,7 @@ name = lib.removeSuffix ".patch" src.name; patch = fetchurl src; }; - patches = builtins.fromJSON (builtins.readFile ./hardened-patches.json); + patches = builtins.fromJSON (builtins.readFile ./hardened/patches.json); in lib.mapAttrs mkPatch patches; # https://bugzilla.kernel.org/show_bug.cgi?id=197591#c6 diff --git a/pkgs/os-specific/linux/kernel/update.sh b/pkgs/os-specific/linux/kernel/update.sh index c483661b6f5..55fdce06c97 100755 --- a/pkgs/os-specific/linux/kernel/update.sh +++ b/pkgs/os-specific/linux/kernel/update.sh @@ -62,4 +62,4 @@ done COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-libre.sh # Update linux-hardened -COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-hardened.py +COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/hardened/update.py diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 349bc242ee1..f6419394b2d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -17014,7 +17014,7 @@ in # Hardened linux hardenedLinuxPackagesFor = kernel: linuxPackagesFor (kernel.override { - structuredExtraConfig = import ../os-specific/linux/kernel/hardened-config.nix { + structuredExtraConfig = import ../os-specific/linux/kernel/hardened/config.nix { inherit stdenv; inherit (kernel) version; }; From e77d174fcdba9c12468553c895ce235e9ea6bb77 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 26 Apr 2020 04:06:11 +0100 Subject: [PATCH 5/8] linux/hardened/update.py: add type annotations --- .../linux/kernel/hardened/update.py | 86 ++++++++++++------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/hardened/update.py b/pkgs/os-specific/linux/kernel/hardened/update.py index 1ef5acd3eb0..116dd616891 100755 --- a/pkgs/os-specific/linux/kernel/hardened/update.py +++ b/pkgs/os-specific/linux/kernel/hardened/update.py @@ -1,17 +1,44 @@ #! /usr/bin/env nix-shell -#! nix-shell -i python -p "python3.withPackages (ps: [ps.PyGithub])" git gnupg +#! nix-shell -i python -p "python38.withPackages (ps: [ps.PyGithub])" git gnupg # This is automatically called by ../update.sh. +from __future__ import annotations + import json import os import re import subprocess import sys +from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory +from typing import ( + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypedDict, + Union, +) from github import Github +from github.GitRelease import GitRelease + +VersionComponent = Union[int, str] +Version = List[VersionComponent] + + +Patch = TypedDict("Patch", {"name": str, "url": str, "sha256": str}) + + +@dataclass +class ReleaseInfo: + version: Version + release: GitRelease + HERE = Path(__file__).resolve().parent NIXPKGS_KERNEL_PATH = HERE.parent @@ -19,17 +46,13 @@ NIXPKGS_PATH = HERE.parents[4] HARDENED_GITHUB_REPO = "anthraxx/linux-hardened" HARDENED_TRUSTED_KEY = HERE / "anthraxx.asc" HARDENED_PATCHES_PATH = HERE / "patches.json" -MIN_KERNEL_VERSION = [4, 14] +MIN_KERNEL_VERSION: Version = [4, 14] -def run(*args, **kwargs): +def run(*args: Union[str, Path]) -> subprocess.CompletedProcess[bytes]: try: return subprocess.run( - args, - **kwargs, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + args, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) except subprocess.CalledProcessError as err: print( @@ -42,13 +65,15 @@ def run(*args, **kwargs): sys.exit(1) -def nix_prefetch_url(url): +def nix_prefetch_url(url: str) -> Tuple[str, Path]: output = run("nix-prefetch-url", "--print-path", url).stdout sha256, path = output.decode("utf-8").strip().split("\n") return sha256, Path(path) -def verify_openpgp_signature(*, name, trusted_key, sig_path, data_path): +def verify_openpgp_signature( + *, name: str, trusted_key: Path, sig_path: Path, data_path: Path, +) -> bool: with TemporaryDirectory(suffix=".nixpkgs-gnupg-home") as gnupg_home_str: gnupg_home = Path(gnupg_home_str) run("gpg", "--homedir", gnupg_home, "--import", trusted_key) @@ -69,14 +94,15 @@ def verify_openpgp_signature(*, name, trusted_key, sig_path, data_path): return False -def fetch_patch(*, name, release): - def find_asset(filename): +def fetch_patch(*, name: str, release: GitRelease) -> Optional[Patch]: + def find_asset(filename: str) -> str: try: - return next( + it: Iterator[str] = ( asset.browser_download_url for asset in release.get_assets() if asset.name == filename ) + return next(it) except StopIteration: raise KeyError(filename) @@ -99,15 +125,11 @@ def fetch_patch(*, name, release): if not sig_ok: return None - return { - "name": patch_filename, - "url": patch_url, - "sha256": sha256, - } + return Patch(name=patch_filename, url=patch_url, sha256=sha256) -def parse_version(version_str): - version = [] +def parse_version(version_str: str) -> Version: + version: Version = [] for component in version_str.split("."): try: version.append(int(component)) @@ -116,15 +138,15 @@ def parse_version(version_str): return version -def version_string(version): +def version_string(version: Version) -> str: return ".".join(str(component) for component in version) -def major_kernel_version_key(kernel_version): +def major_kernel_version_key(kernel_version: Version) -> str: return version_string(kernel_version[:-1]) -def commit_patches(*, kernel_key, message): +def commit_patches(*, kernel_key: str, message: str) -> None: new_patches_path = HARDENED_PATCHES_PATH.with_suffix(".new") with open(new_patches_path, "w") as new_patches_file: json.dump(patches, new_patches_file, indent=4, sort_keys=True) @@ -144,6 +166,7 @@ def commit_patches(*, kernel_key, message): # Load the existing patches. +patches: Dict[str, Patch] with open(HARDENED_PATCHES_PATH) as patches_file: patches = json.load(patches_file) @@ -177,7 +200,6 @@ for kernel_key in sorted(patches.keys() - kernel_versions.keys()): g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo(HARDENED_GITHUB_REPO) - failures = False # Match each kernel version with the best patch version. @@ -195,10 +217,7 @@ for release in repo.get_releases(): except KeyError: continue - release_info = { - "version": version, - "release": release, - } + release_info = ReleaseInfo(version=version, release=release) if kernel_version == packaged_kernel_version: releases[kernel_key] = release_info @@ -208,18 +227,20 @@ for release in repo.get_releases(): if kernel_version > packaged_kernel_version: continue elif ( - kernel_key not in releases - or releases[kernel_key]["version"] < version + kernel_key not in releases or releases[kernel_key].version < version ): releases[kernel_key] = release_info # Update hardened-patches.json for each release. for kernel_key, release_info in releases.items(): - release = release_info["release"] - version = release_info["version"] + release = release_info.release + version = release_info.version version_str = release.tag_name name = f"linux-hardened-{version_str}" + old_version: Optional[Version] = None + old_version_str: Optional[str] = None + update: bool try: old_filename = patches[kernel_key]["name"] old_version_str = old_filename.replace("linux-hardened-", "").replace( @@ -229,7 +250,6 @@ for kernel_key, release_info in releases.items(): update = old_version < version except KeyError: update = True - old_version = None if update: patch = fetch_patch(name=name, release=release) From 88486c4e767cb92b0a51d9c41f4e40121c6e3bf1 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 26 Apr 2020 18:12:14 +0100 Subject: [PATCH 6/8] linux/hardened/update.py: get versions with nix(1) --- .../linux/kernel/hardened/update.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/hardened/update.py b/pkgs/os-specific/linux/kernel/hardened/update.py index 116dd616891..7960f1264f8 100755 --- a/pkgs/os-specific/linux/kernel/hardened/update.py +++ b/pkgs/os-specific/linux/kernel/hardened/update.py @@ -170,29 +170,24 @@ patches: Dict[str, Patch] with open(HARDENED_PATCHES_PATH) as patches_file: patches = json.load(patches_file) -NIX_VERSION_RE = re.compile( - r""" - \s* version \s* = - \s* " (?P [^"]*) " - \s* ; \s* \n - """, - re.VERBOSE, -) - # Get the set of currently packaged kernel versions. kernel_versions = {} for filename in os.listdir(NIXPKGS_KERNEL_PATH): filename_match = re.fullmatch(r"linux-(\d+)\.(\d+)\.nix", filename) if filename_match: - with open(NIXPKGS_KERNEL_PATH / filename) as nix_file: - for nix_line in nix_file: - match = NIX_VERSION_RE.fullmatch(nix_line) - if match: - kernel_version = parse_version(match.group("version")) - if kernel_version < MIN_KERNEL_VERSION: - continue - kernel_key = major_kernel_version_key(kernel_version) - kernel_versions[kernel_key] = kernel_version + nix_version_expr = f""" + with import {NIXPKGS_PATH} {{}}; + (callPackage {NIXPKGS_KERNEL_PATH / filename} {{}}).version + """ + kernel_version = parse_version( + run( + "nix", "eval", "--impure", "--raw", "--expr", nix_version_expr, + ).stdout.decode("utf-8") + ) + if kernel_version < MIN_KERNEL_VERSION: + continue + kernel_key = major_kernel_version_key(kernel_version) + kernel_versions[kernel_key] = kernel_version # Remove patches for unpackaged kernel versions. for kernel_key in sorted(patches.keys() - kernel_versions.keys()): From b2ad58536c7ed0f0a0d5434d63067011f531e0c5 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 26 Apr 2020 20:09:37 +0100 Subject: [PATCH 7/8] linux/hardened/update.py: commit updates in order --- pkgs/os-specific/linux/kernel/hardened/update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/os-specific/linux/kernel/hardened/update.py b/pkgs/os-specific/linux/kernel/hardened/update.py index 7960f1264f8..be955efbb53 100755 --- a/pkgs/os-specific/linux/kernel/hardened/update.py +++ b/pkgs/os-specific/linux/kernel/hardened/update.py @@ -227,7 +227,8 @@ for release in repo.get_releases(): releases[kernel_key] = release_info # Update hardened-patches.json for each release. -for kernel_key, release_info in releases.items(): +for kernel_key in sorted(releases.keys()): + release_info = releases[kernel_key] release = release_info.release version = release_info.version version_str = release.tag_name From 5a5a2d0342ee6610f14a7024c6b01ab6261749f9 Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 3 May 2020 12:54:11 +0100 Subject: [PATCH 8/8] linux/hardened/update.py: pass encoding to subprocess --- .../os-specific/linux/kernel/hardened/update.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/hardened/update.py b/pkgs/os-specific/linux/kernel/hardened/update.py index be955efbb53..3958c85fe20 100755 --- a/pkgs/os-specific/linux/kernel/hardened/update.py +++ b/pkgs/os-specific/linux/kernel/hardened/update.py @@ -52,14 +52,18 @@ MIN_KERNEL_VERSION: Version = [4, 14] def run(*args: Union[str, Path]) -> subprocess.CompletedProcess[bytes]: try: return subprocess.run( - args, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + args, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", ) except subprocess.CalledProcessError as err: print( f"error: `{err.cmd}` failed unexpectedly\n" f"status code: {err.returncode}\n" - f'stdout:\n{err.stdout.decode("utf-8").strip()}\n' - f'stderr:\n{err.stderr.decode("utf-8").strip()}', + f"stdout:\n{err.stdout.strip()}\n" + f"stderr:\n{err.stderr.strip()}", file=sys.stderr, ) sys.exit(1) @@ -67,7 +71,7 @@ def run(*args: Union[str, Path]) -> subprocess.CompletedProcess[bytes]: def nix_prefetch_url(url: str) -> Tuple[str, Path]: output = run("nix-prefetch-url", "--print-path", url).stdout - sha256, path = output.decode("utf-8").strip().split("\n") + sha256, path = output.strip().split("\n") return sha256, Path(path) @@ -83,6 +87,7 @@ def verify_openpgp_signature( ("gpgv", "--keyring", keyring, sig_path, data_path), check=True, stderr=subprocess.PIPE, + encoding="utf-8", ) return True except subprocess.CalledProcessError as err: @@ -90,7 +95,7 @@ def verify_openpgp_signature( f"error: signature for {name} failed to verify!", file=sys.stderr, ) - print(err.stderr.decode("utf-8"), file=sys.stderr, end="") + print(err.stderr, file=sys.stderr, end="") return False @@ -182,7 +187,7 @@ for filename in os.listdir(NIXPKGS_KERNEL_PATH): kernel_version = parse_version( run( "nix", "eval", "--impure", "--raw", "--expr", nix_version_expr, - ).stdout.decode("utf-8") + ).stdout ) if kernel_version < MIN_KERNEL_VERSION: continue