From f2f939df4b45e08640ee40243b3a61a5b0e9b3f2 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Thu, 5 Sep 2024 15:48:00 -0700 Subject: [PATCH] [ci] make checks faster Why === * GitHub actions is taking way to long to build all our modules. * Instead, let's just eval them and save the building for CD What changed === * use `nix eval .#bundle` isntead of the build_change_modules script * Remove some unushed scripts Test plan === * CI passes in less than 15 minutes --- scripts/build_changed_modules.py | 82 -------------------------------- scripts/cache_nix_build.sh | 12 ----- scripts/ci_check.sh | 2 +- scripts/convert_100doc.sh | 46 ------------------ 4 files changed, 1 insertion(+), 141 deletions(-) delete mode 100644 scripts/build_changed_modules.py delete mode 100755 scripts/cache_nix_build.sh delete mode 100644 scripts/convert_100doc.sh diff --git a/scripts/build_changed_modules.py b/scripts/build_changed_modules.py deleted file mode 100644 index 928f4035..00000000 --- a/scripts/build_changed_modules.py +++ /dev/null @@ -1,82 +0,0 @@ -# This script test builds the updated modules against an upstream branch for a pull request. - -import json -import os -import subprocess -import argparse -import re -from textwrap import dedent - -nix_store_path_pattern = re.compile(r'/nix/store/([0-9a-z]+)') - -# returns a set of module versions based on a map of module ID -> Nix store path -# a module version is a 2-tuple: (module ID, sha from Nix store path) -# example: ('python-3.10', 'f7mgxpvxl4vdkijdamfv94nwciv2laas') -def convert_module_map_to_verisons(module_map): - versions = set() - for module_id, path in module_map.items(): - match = nix_store_path_pattern.match(path) - sha = match[1] - versions.add((module_id, sha)) - return versions - -def get_module_versions(): - args = ['nix', 'eval', '.#modules', '--json'] - print(" ".join(args)) - output = subprocess.check_output(args) - return convert_module_map_to_verisons(json.loads(str(output, 'UTF-8'))) - -def build_module(module_id): - args = ['nix', 'build', '-L', '.#modules."%s"' % module_id, '--print-out-paths'] - print(" ".join(args)) - output = subprocess.check_output(args) - return str(output, 'UTF-8').strip() - -def get_upstream_module_versions(branch): - args = ['nix', 'eval', 'github:replit/nixmodules/%s#modules' % branch, '--json'] - print(" ".join(args)) - output = subprocess.check_output(args) - return convert_module_map_to_verisons(json.loads(str(output, 'UTF-8'))) - -def nix_collect_garbage(): - args = ['nix-collect-garbage'] - print(" ".join(args)) - subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - -def verify_no_existing_modules_removed(upstream_module_versions, current_module_versions): - upstream_modules = {version[0] for version in upstream_module_versions} - modules = {version[0] for version in current_module_versions} - diff = upstream_modules - modules - assert len(diff) == 0, dedent(f"""\ - module(s) deleted: {repr(diff)} - You may want to add new entries to ./pkgs/upgrade-map - """) - -def main(): - parser = argparse.ArgumentParser( - prog='build_changed_modules', - description='builds the modules that were changed from an upstream branch', - ) - - parser.add_argument('upstream_branch') - - args = parser.parse_args() - - upstream_module_versions = get_upstream_module_versions(args.upstream_branch) - current_module_versions = get_module_versions() - - verify_no_existing_modules_removed(upstream_module_versions, current_module_versions) - - new_module_versions = current_module_versions - upstream_module_versions - if len(new_module_versions) == 0: - print('Nothing changed') - else: - print('modules to build:', [version[0] for version in new_module_versions]) - for version in new_module_versions: - module_id, _ = version - actual_path = build_module(module_id) - print('%s ok' % module_id) - nix_collect_garbage() - -if __name__ == '__main__': - main() diff --git a/scripts/cache_nix_build.sh b/scripts/cache_nix_build.sh deleted file mode 100755 index fa3f86e7..00000000 --- a/scripts/cache_nix_build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# Builds and uploads a nix package and its run-time and build-time dependencies to our Nix cahce. - -set -eo pipefail - -nix build $@ - -nix-store -qR --include-outputs $(nix-store -qd $(nix path-info $@)) \ - | grep -v '\.drv$' \ - | xargs nix copy --to https://nix-build-cache.replit.com?secret-key=./nix_build_cache_signing_key - - diff --git a/scripts/ci_check.sh b/scripts/ci_check.sh index 3b851dac..83bac663 100755 --- a/scripts/ci_check.sh +++ b/scripts/ci_check.sh @@ -17,4 +17,4 @@ nix eval "${NIX_FLAGS[@]}" .#modules --json nix develop "${NIX_FLAGS[@]}" --command echo Hello, world -nix-shell -p python312 --command 'python scripts/build_changed_modules.py main' +nix eval .#bundle diff --git a/scripts/convert_100doc.sh b/scripts/convert_100doc.sh deleted file mode 100644 index b0b754a6..00000000 --- a/scripts/convert_100doc.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -set -xe - -rm -rf "${REPL_HOME}/replit.nix" "${REPL_HOME}/poetry.lock" "${REPL_HOME}/.upm" "${REPL_HOME}/venv" "${REPL_HOME}/.config" "${REPL_HOME}/.cache" - -mkdir -p "${REPL_HOME}/.pythonlibs" - -cat << EOF > "${REPL_HOME}/.replit" -entrypoint = "main.py" -run = "python main.py" -modules = ["python-3.10:v18-20230807-322e88b"] - -[nix] -channel = "stable-23_11" - -[unitTest] -language = "python3" - -[gitHubImport] -requiredFiles = [".replit", "replit.nix"] -EOF - -cat << EOF > "${REPL_HOME}/pyproject.toml" -[tool.poetry] -name = "python-template" -version = "0.1.0" -description = "" -authors = ["Your Name "] - -[tool.poetry.dependencies] -python = ">=3.10.0,<3.12" - -[tool.pyright] -# https://github.com/microsoft/pyright/blob/main/docs/configuration.md -useLibraryCodeForTypes = true -exclude = [".cache"] - -[tool.ruff] -# https://beta.ruff.rs/docs/configuration/ -select = ['E', 'W', 'F', 'I', 'B', 'C4', 'ARG', 'SIM'] -ignore = ['W291', 'W292', 'W293'] - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" -EOF