ci/repo: Add deb fetcher

Signed-off-by: Ryan Northey <[email protected]>

.github/workflows/envoy.yml

@@ -20,6 +20,37 @@ jobs:
     if: github.repository_owner == 'envoyproxy'
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+    - id: versions
+      run: |
+        bazel build --config=ci //debs:minor_versions
+        VERSION_HASH="$(cat bazel-bin/debs/minor_versions.txt | sha256sum | cut -d' ' -f1)"
+        echo "hash=${VERSION_HASH}" >> "$GITHUB_OUTPUT"
+    - uses: actions/cache/restore@v4
+      with:
+        key: ${{ steps.versions.outputs.hash }}
+        path: /tmp/repository
     - run: |
-        bazel build --config=ci //:envoy_versions
-        cat bazel-bin/envoy_versions.json | jq '.'
+        mkdir -p /tmp/repository/v1.28.3
+        ls /tmp/repository | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | sort -u > debs/custom-excludes.txt
+        echo $GITHUB_TOKEN > debs/token.txt
+        bazel run \
+            --config=ci \
+            --//debs:excludes=//debs:custom-excludes.txt \
+            --//debs:token=//debs:token.txt \
+            --//tools/tarball:target=//debs \
+            //tools/tarball:unpack \
+            /tmp/repository
+        tar tf bazel-bin/debs/debs.tar.gz
+        ls -alh bazel-bin/debs/debs.tar.gz
+        ORIGINAL=$(cat debs/custom-excludes.txt | wc -l)
+        UPDATED=$(ls /tmp/repository | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | wc -l)
+        if [[ "$ORIGINAL" != "$UPDATED" ]]; then
+           echo "changed=1" >> $GITHUB_OUTPUT
+        fi
+        rm -rf debs/token.txt
+      id: update
+    - uses: actions/cache/save@v4
+      if: ${{ steps.update.outputs.changed }}
+      with:
+        key: ${{ steps.versions.outputs.hash }}
+        path: /tmp/repository

BUILD

@@ -1,6 +1,12 @@
 load("@bazel_gazelle//:def.bzl", "gazelle")
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@base_pip3//:requirements.bzl", "requirement")
+
+exports_files([
+    "custom-excludes.txt",
+])
 
 # gazelle:prefix github.com/aptly-dev/aptly
 gazelle(name = "gazelle")
@@ -22,26 +28,3 @@ gazelle(
     command = "update-repos",
     data = [":go.mod"],
 )
-
-jq(
-    name = "envoy_versions",
-    srcs = ["@envoy_repo//:project"],
-    out = "envoy_versions.json",
-    filter = """
-    .releases as $releases
-    | {
-      latest_releases: [
-        .stable_versions[]
-        | . as $minor
-        | {
-            version: $minor,
-            releases: (
-              $releases
-              | map(select(startswith("v" + $minor + ".")))
-            )
-          }
-      ]
-    }
-    """,
-    visibility = ["//visibility:public"],
-)

debs/BUILD

@@ -0,0 +1,146 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@base_pip3//:requirements.bzl", "requirement")
+load("@envoy//tools/base:envoy_python.bzl", "envoy_entry_point")
+
+exports_files([
+    "custom-excludes.txt",
+    "token.txt",
+])
+
+jq(
+    name = "envoy_versions",
+    srcs = ["@envoy_repo//:project"],
+    out = "envoy_versions.json",
+    filter = """
+    .releases as $releases
+    | {
+      latest_releases: [
+        .stable_versions[]
+        | . as $minor
+        | {
+            version: $minor,
+            releases: (
+              $releases
+              | map(select(startswith("v" + $minor + ".")))
+            )
+          }
+      ]
+    }
+    """,
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "minor_versions",
+    srcs = [":envoy_versions"],
+    out = "minor_versions.txt",
+    filter = """
+    .latest_releases[] | .version
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "patch_versions",
+    srcs = [":envoy_versions"],
+    out = "patch_versions.txt",
+    filter = """
+    .latest_releases[] | .releases[]
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "deb_checksum_downloads",
+    srcs = [":envoy_versions"],
+    out = "deb_checksum_downloads.txt",
+    filter = """
+    reduce .latest_releases[].releases[] as $item ({};
+      .[$item] = {"signature": "[email protected]"})
+    | with_entries(
+        {"key": "https://github.com/envoyproxy/envoy/releases/download/\\(.key)/checksums.txt.asc",
+         "value": .value})
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+envoy_entry_point(
+    name = "fetch",
+    pkg = "envoy.base.utils",
+    script = "envoy.fetch",
+    init_data = [],
+)
+
+genrule(
+    name = "published_checksums",
+    outs = ["published_checksums.txt"],
+    cmd = """
+    $(location :fetch) $(location :deb_checksum_downloads) --output=json > $@
+    """,
+    tools = [
+        ":fetch",
+        ":deb_checksum_downloads",
+    ],
+)
+
+jq(
+    name = "debs_downloads",
+    srcs = [":published_checksums"],
+    out = "debs_downloads.json",
+    filter = """
+    with_entries(
+      .key as $key
+        | .value as $value
+        | ($key | capture("v(?<version>[0-9.]+)") | .version) as $version
+        | {key: ("https://github.com/envoyproxy/envoy/releases/download/v\\($version)/debs.tar.gz"),
+           value: {
+             "path": $version,
+             "checksum": (
+               $value
+               | split("\n")
+               | map(select(endswith("debs.tar.gz")))
+               | first
+               | split(" ")
+               | .[0]
+             )}})
+    """,
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "empty",
+    outs = ["empty.txt"],
+    cmd = """
+    touch $@
+    """,
+)
+
+label_flag(
+    name = "excludes",
+    build_setting_default = ":empty",
+)
+
+label_flag(
+    name = "token",
+    build_setting_default = ":empty",
+)
+
+genrule(
+    name = "debs",
+    outs = ["debs.tar.gz"],
+    cmd = """
+    $(location :fetch) $(location :debs_downloads) -vdebug --excludes=$(location :excludes) --token-path=$(location :token) --extract-downloads --output-path=$@
+    """,
+    tools = [
+        ":fetch",
+        ":debs_downloads",
+        ":excludes",
+        ":token",
+    ],
+)

setup.bzl

@@ -1,6 +1,11 @@
 load("@rules_proto//proto:setup.bzl", "rules_proto_setup")
 load("@rules_python//python:repositories.bzl", "py_repositories")
+# load("@envoy//bazel:api_binding.bzl", "envoy_api_binding")
+# load("@envoy//bazel:repositories.bzl", "envoy_build_config")
+
 
 def setup():
     py_repositories()
     rules_proto_setup()
+    # envoy_api_binding()
+    # envoy_build_config(name = "envoy_build_config")

tools/tarball/BUILD

@@ -0,0 +1,7 @@
+load("@envoy_toolshed//tarball:macros.bzl", "unpacker")
+
+licenses(["notice"])  # Apache 2
+
+unpacker(
+    name = "unpack",
+)

versions.bzl

@@ -40,8 +40,8 @@ VERSIONS = {
     "envoy": {
         "type": "github_archive",
         "repo": "envoyproxy/envoy",
-        "version": "fea66c359069991e88bdfa4e0f2883c90cc39aef",
-        "sha256": "96294f4b491c676b650ddeb07c7986ec7e48b5ae5a75c4adebbb9a58741a2fb1",
+        "version": "4108a96e215897d47cfeb3578486f0578333c1bc",
+        "sha256": "62606a0dbe73d32edfde176faaf1325f5f7a1531d1d40e12f4d6d85b75bc6b2b",
         "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
         "strip_prefix": "envoy-{version}",
     },
@@ -63,6 +63,14 @@ VERSIONS = {
             "https://github.com/{repo}/releases/download/v{version}/rules_go-v{version}.zip",
         ],
     },
+    "rules_foreign_cc": {
+        "type": "github_archive",
+        "repo": "bazelbuild/rules_foreign_cc",
+        "version": "0.11.1",
+        "sha256": "4b33d62cf109bcccf286b30ed7121129cc34cf4f4ed9d8a11f38d9108f40ba74",
+        "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
+        "strip_prefix": "rules_foreign_cc-{version}",
+    },
     "rules_pkg": {
         "type": "github_archive",
         "repo": "bazelbuild/rules_pkg",