From b0042e72d846116a928674fb42f6bbaea7271d54 Mon Sep 17 00:00:00 2001 From: Asaf Flescher Date: Fri, 29 Mar 2024 11:46:33 -0400 Subject: [PATCH 1/3] Client should now respect the maxBatchSize setting from the server Also added to log output how many requests used Bytestream vs CAS --- cmd/casload/main.go | 7 ++++++- pkg/load/generate.go | 20 +++++++++++++++----- pkg/load/read.go | 19 ++++++++++++++----- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/cmd/casload/main.go b/cmd/casload/main.go index 3a9fac3..f4d7cf9 100644 --- a/cmd/casload/main.go +++ b/cmd/casload/main.go @@ -122,6 +122,11 @@ func main() { } defer cs.Close() + var finalMaxBatchBlobSize = *maxBatchBlobSize + if cs.maxBatchBlobSize > 0 && cs.maxBatchBlobSize < finalMaxBatchBlobSize { + finalMaxBatchBlobSize = cs.maxBatchBlobSize + } + actionContext := load.ActionContext{ InstanceName: instanceName, CasClient: cs.casClient, @@ -129,7 +134,7 @@ func main() { Ctx: ctx, KnownDigests: make(map[string]bool), WriteChunkSize: *writeChunkSize, - MaxBatchBlobSize: *maxBatchBlobSize, + MaxBatchBlobSize: finalMaxBatchBlobSize, } for _, action := range actions { diff --git a/pkg/load/generate.go b/pkg/load/generate.go index 2b3f6e1..0046e7a 100644 --- a/pkg/load/generate.go +++ b/pkg/load/generate.go @@ -39,10 +39,12 @@ func (g *generateAction) String() string { } type generateResult struct { - startTime time.Time - endTime time.Time - success int - errors int + startTime time.Time + endTime time.Time + success int + errors int + byteStreamWrites int + blobWrites int } type generateWorkItem struct { @@ -180,6 +182,12 @@ func (g *generateAction) RunAction(actionContext *ActionContext) error { } elapsedTimes[i] = r.elapsed + if int64(r.blobSize) < actionContext.MaxBatchBlobSize { + result.blobWrites += 1 + } else { + result.byteStreamWrites += 1 + } + if i%100 == 0 { log.Debugf("progress: %d / %d", i, g.numRequests) } @@ -189,12 +197,14 @@ func (g *generateAction) RunAction(actionContext *ActionContext) error { close(resultChan) - fmt.Printf("program: %s\n startTime: %s\n endTime: %s\n success: %d\n errors: %d\n", + fmt.Printf("program: %s\n startTime: %s\n endTime: %s\n success: %d\n errors: %d\n byteStreamWrites: %d\n blobWrites: %d\n", g.String(), result.startTime.String(), result.endTime.String(), result.success, result.errors, + result.byteStreamWrites, + result.blobWrites, ) stats.PrintTimingStats(elapsedTimes) diff --git a/pkg/load/read.go b/pkg/load/read.go index 46c36c1..cb05eb8 100644 --- a/pkg/load/read.go +++ b/pkg/load/read.go @@ -41,10 +41,12 @@ func (self *readAction) String() string { } type readResult struct { - startTime time.Time - endTime time.Time - success int - errors int + startTime time.Time + endTime time.Time + success int + errors int + byteStreamReads int + blobReads int } type readWorkItem struct { @@ -208,6 +210,11 @@ func (self *readAction) RunAction(actionContext *ActionContext) error { }).Error("request error") } elapsedTimes[i] = r.elapsed + if int64(r.digest.SizeBytes) < actionContext.MaxBatchBlobSize { + result.blobReads += 1 + } else { + result.byteStreamReads += 1 + } if i%100 == 0 { log.Debugf("progress: %d / %d", i, len(workItems)) @@ -218,12 +225,14 @@ func (self *readAction) RunAction(actionContext *ActionContext) error { close(resultChan) - fmt.Printf("program: %s\n startTime: %s\n endTime: %s\n success: %d\n errors: %d\n", + fmt.Printf("program: %s\n startTime: %s\n endTime: %s\n success: %d\n errors: %d\n byteStreamReads: %d\n blobReads: %d\n", self.String(), result.startTime.String(), result.endTime.String(), result.success, result.errors, + result.byteStreamReads, + result.blobReads, ) stats.PrintTimingStats(elapsedTimes) From 0c9d7d2f842867b837ed7623cbad194068f66f78 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Mon, 8 Apr 2024 22:15:05 -0400 Subject: [PATCH 2/3] Update Pants version. (#95) Use scie-pants, and update CI to recent actions versions. --- .github/workflows/build.yaml | 27 +-- pants | 389 ----------------------------------- pants.toml | 6 +- 3 files changed, 19 insertions(+), 403 deletions(-) delete mode 100755 pants diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2ce5031..5bd25a5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,33 +14,36 @@ jobs: name: Build and test runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.19' - - name: Cache pants - uses: actions/cache@v2 + - uses: actions/setup-python@v5 with: - key: ${{ runner.os }}-${{ hashFiles('pants*toml') }}-v2 - path: | - ~/.cache/pants/setup + python-version: '3.11' + - name: Initialize Pants + uses: pantsbuild/actions/init-pants@v8 + with: + # cache0 makes it easy to bust the cache if needed + gha-cache-key: cache0- + named-caches-hash: ${{ hashFiles('go.mod') }} - name: Set env vars run: | echo 'PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml"]' >> ${GITHUB_ENV} - name: Pants Bootstrap - run: ./pants version + run: pants version - name: show toolchain token if: github.event_name == 'push' - run: ./pants auth-token-info --auth-token-info-verbose + run: pants auth-token-info --auth-token-info-verbose - name: Lint & check run: | - ./pants lint check :: + pants lint check :: - name: Tests run: | - ./pants test :: + pants test :: - name: Build smoketest & casloader docker images run: | - ./pants package cmd/:: + pants package cmd/:: # Keeping those here since the publish workflow wasn't ported to use pants. - name: Build smoketest docker image uses: docker/build-push-action@v3 diff --git a/pants b/pants deleted file mode 100755 index 467e836..0000000 --- a/pants +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# =============================== NOTE =============================== -# This ./pants bootstrap script comes from the pantsbuild/setup -# project. It is intended to be checked into your code repository so -# that other developers have the same setup. -# -# Learn more here: https://www.pantsbuild.org/docs/installation -# ==================================================================== - -set -eou pipefail - -# NOTE: To use an unreleased version of Pants from the pantsbuild/pants main branch, -# locate the main branch SHA, set PANTS_SHA= in the environment, and run this script as usual. -# -# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version - -PYTHON_BIN_NAME="${PYTHON:-unspecified}" - -# Set this to specify a non-standard location for this script to read the Pants version from. -# NB: This will *not* cause Pants itself to use this location as a config file. -# You can use PANTS_CONFIG_FILES or --pants-config-files to do so. -PANTS_TOML=${PANTS_TOML:-pants.toml} - -PANTS_BIN_NAME="${PANTS_BIN_NAME:-$0}" - -PANTS_SETUP_CACHE="${PANTS_SETUP_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}" -# If given a relative path, we fix it to be absolute. -if [[ "$PANTS_SETUP_CACHE" != /* ]]; then - PANTS_SETUP_CACHE="${PWD}/${PANTS_SETUP_CACHE}" -fi - -PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)" - -_PEX_VERSION=2.1.62 -_PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${_PEX_VERSION}/pex" -_PEX_EXPECTED_SHA256="56668b1ca147bd63141e586ffee97c7cc51ce8e6eac6c9b7a4bf1215b94396e5" - -VIRTUALENV_VERSION=20.4.7 -VIRTUALENV_REQUIREMENTS=$( -cat << EOF -virtualenv==${VIRTUALENV_VERSION} --hash sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76 -filelock==3.0.12 --hash sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836 -six==1.16.0 --hash sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 -distlib==0.3.2 --hash sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c -appdirs==1.4.4 --hash sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 -importlib-resources==5.1.4; python_version < "3.7" --hash sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351 -importlib-metadata==4.5.0; python_version < "3.8" --hash sha256:833b26fb89d5de469b24a390e9df088d4e52e4ba33b01dc5e0e4f41b81a16c00 -zipp==3.4.1; python_version < "3.10" --hash sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098 -typing-extensions==3.10.0.0; python_version < "3.8" --hash sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 -EOF -) - -COLOR_RED="\x1b[31m" -COLOR_GREEN="\x1b[32m" -COLOR_YELLOW="\x1b[33m" -COLOR_RESET="\x1b[0m" - -function log() { - echo -e "$@" 1>&2 -} - -function die() { - (($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}" - exit 1 -} - -function green() { - (($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}" -} - -function warn() { - (($# > 0)) && log "${COLOR_YELLOW}$*${COLOR_RESET}" -} - -function tempdir { - mkdir -p "$1" - mktemp -d "$1"/pants.XXXXXX -} - -function get_exe_path_or_die { - local exe="$1" - if ! command -v "${exe}"; then - die "Could not find ${exe}. Please ensure ${exe} is on your PATH." - fi -} - -function get_pants_config_string_value { - local config_key="$1" - local optional_space="[[:space:]]*" - local prefix="^${config_key}${optional_space}=${optional_space}" - local raw_value - raw_value="$(sed -ne "/${prefix}/ s|${prefix}||p" "${PANTS_TOML}")" - local optional_suffix="${optional_space}(#.*)?$" - echo "${raw_value}" \ - | sed -E \ - -e "s|^'([^']*)'${optional_suffix}|\1|" \ - -e 's|^"([^"]*)"'"${optional_suffix}"'$|\1|' \ - && return 0 - return 0 -} - -function get_python_major_minor_version { - local python_exe="$1" - "$python_exe" </dev/null 2>&1; then - continue - fi - if [[ -n "$(check_python_exe_compatible_version "${interpreter_path}")" ]]; then - echo "${interpreter_path}" && return 0 - fi - done -} - -function determine_python_exe { - local pants_version="$1" - set_supported_python_versions "${pants_version}" - local requirement_str="For \`pants_version = \"${pants_version}\"\`, Pants requires Python ${supported_message} to run." - - local python_exe - if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then - python_exe="$(get_exe_path_or_die "${PYTHON_BIN_NAME}")" || exit 1 - if [[ -z "$(check_python_exe_compatible_version "${python_exe}")" ]]; then - die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}" - fi - else - python_exe="$(determine_default_python_exe)" - if [[ -z "${python_exe}" ]]; then - die "No valid Python interpreter found. ${requirement_str} Please check that a valid interpreter is installed and on your \$PATH." - fi - fi - echo "${python_exe}" -} - -function compute_sha256 { - local python="$1" - local path="$2" - - "$python" <&2 || exit 1 - fi - echo "${bootstrapped}" -} - -function scrub_PEX_env_vars { - # Ensure the virtualenv PEX runs as shrink-wrapped. - # See: https://github.com/pantsbuild/setup/issues/105 - if [[ -n "${!PEX_@}" ]]; then - warn "Scrubbing ${!PEX_@}" - unset "${!PEX_@}" - fi -} - -function bootstrap_virtualenv { - local python="$1" - local bootstrapped="${PANTS_BOOTSTRAP}/virtualenv-${VIRTUALENV_VERSION}/virtualenv.pex" - if [[ ! -f "${bootstrapped}" ]]; then - ( - green "Creating the virtualenv PEX." - pex_path="$(bootstrap_pex "${python}")" || exit 1 - mkdir -p "${PANTS_BOOTSTRAP}" - local staging_dir - staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") - cd "${staging_dir}" - echo "${VIRTUALENV_REQUIREMENTS}" > requirements.txt - ( - scrub_PEX_env_vars - "${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex - ) - mkdir -p "$(dirname "${bootstrapped}")" - mv -f "${staging_dir}/virtualenv.pex" "${bootstrapped}" - rm -rf "${staging_dir}" - ) 1>&2 || exit 1 - fi - echo "${bootstrapped}" -} - -function find_links_url { - local pants_version="$1" - local pants_sha="$2" - echo -n "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/${pants_sha}/${pants_version/+/%2B}/index.html" -} - -function get_version_for_sha { - local sha="$1" - - # Retrieve the Pants version associated with this commit. - local pants_version - pants_version="$(curl --proto "=https" \ - --tlsv1.2 \ - --fail \ - --silent \ - --location \ - "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")" - - # Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`, - # where the XXXXXXXX is the first 8 characters of the SHA. - echo "${pants_version}+git${sha:0:8}" -} - -function bootstrap_pants { - local pants_version="$1" - local python="$2" - local pants_sha="${3:-}" - - local pants_requirement="pantsbuild.pants==${pants_version}" - local maybe_find_links - if [[ -z "${pants_sha}" ]]; then - maybe_find_links="" - else - maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")" - fi - local python_major_minor_version - python_major_minor_version="$(get_python_major_minor_version "${python}")" - local target_folder_name="${pants_version}_py${python_major_minor_version}" - local bootstrapped="${PANTS_BOOTSTRAP}/${target_folder_name}" - - if [[ ! -d "${bootstrapped}" ]]; then - ( - green "Bootstrapping Pants using ${python}" - local staging_dir - staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") - local virtualenv_path - virtualenv_path="$(bootstrap_virtualenv "${python}")" || exit 1 - green "Installing ${pants_requirement} into a virtual environment at ${bootstrapped}" - ( - scrub_PEX_env_vars - # shellcheck disable=SC2086 - "${python}" "${virtualenv_path}" --quiet --no-download "${staging_dir}/install" && \ - # Grab the latest pip, but don't advance setuptools past 58 which drops support for the - # `setup` kwarg `use_2to3` which Pants 1.x sdist dependencies (pystache) use. - "${staging_dir}/install/bin/pip" install --quiet -U pip "setuptools<58" && \ - "${staging_dir}/install/bin/pip" install ${maybe_find_links} --quiet --progress-bar off "${pants_requirement}" - ) && \ - ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \ - mv "${staging_dir}/${target_folder_name}" "${bootstrapped}" && \ - green "New virtual environment successfully created at ${bootstrapped}." - ) 1>&2 || exit 1 - fi - echo "${bootstrapped}" -} - -# Ensure we operate from the context of the ./pants buildroot. -cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" -pants_version="$(determine_pants_version)" -python="$(determine_python_exe "${pants_version}")" -pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")" || exit 1 - -pants_python="${pants_dir}/bin/python" -pants_binary="${pants_dir}/bin/pants" -pants_extra_args="" -if [[ -n "${PANTS_SHA:-}" ]]; then - pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")" -fi - -# shellcheck disable=SC2086 -exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \ - --pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@" \ No newline at end of file diff --git a/pants.toml b/pants.toml index 5a54359..f8b9e38 100644 --- a/pants.toml +++ b/pants.toml @@ -1,5 +1,5 @@ [GLOBAL] -pants_version = "2.15.1" +pants_version = "2.19.1" backend_packages.add = [ "pants.backend.build_files.fmt.black", "pants.backend.experimental.go", @@ -17,10 +17,12 @@ root_patterns = ["/"] [anonymous-telemetry] enabled = false # anonymous-telemetry is being deprecated from pants - [golang] minimum_expected_version = "1.19" +[python] +interpreter_constraints = ['==3.11.*'] + [docker.registries.dockerhub] address = "registry.hub.docker.com" default = true From bdca1b75ab73b4de93767020f6fea5939a000f02 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 9 Apr 2024 13:43:33 -0400 Subject: [PATCH 3/3] More CI bitrot fixes (#96) --- .github/workflows/build.yaml | 3 --- .github/workflows/publish.yaml | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5bd25a5..133676e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -32,9 +32,6 @@ jobs: echo 'PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml"]' >> ${GITHUB_ENV} - name: Pants Bootstrap run: pants version - - name: show toolchain token - if: github.event_name == 'push' - run: pants auth-token-info --auth-token-info-verbose - name: Lint & check run: | pants lint check :: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 311037c..dc6800f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -7,15 +7,15 @@ jobs: publish-dockerhub: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - name: Build & Publish nightly smoketest id: docker_build_smoketest - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: build-args: APP_NAME=smoketest tags: toolchainlabs/remote-api-tools-smoketest:nightly @@ -24,7 +24,7 @@ jobs: run: echo ${{ steps.docker_build_smoketest.outputs.digest }} - name: Build & Publish nightly casload id: docker_build_casload - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: build-args: APP_NAME=casload tags: toolchainlabs/remote-api-tools-casload:nightly