Merge pull request #2551 from crazy-max/metadata-warnings-2

build: opt to set progress warnings in response

.github/workflows/e2e.yml

@@ -84,6 +84,8 @@ jobs:
             endpoint: tcp://localhost:1234
           - driver: docker-container
             metadata-provenance: max
+          - driver: docker-container
+            metadata-warnings: true
         exclude:
           - driver: docker
             multi-node: mnode-true
@@ -134,6 +136,9 @@ jobs:
           if [ -n "${{ matrix.metadata-provenance }}" ]; then
             echo "BUILDX_METADATA_PROVENANCE=${{ matrix.metadata-provenance }}" >> $GITHUB_ENV
           fi
+          if [ -n "${{ matrix.metadata-warnings }}" ]; then
+            echo "BUILDX_METADATA_WARNINGS=${{ matrix.metadata-warnings }}" >> $GITHUB_ENV
+          fi
       -
         name: Install k3s
         if: matrix.driver == 'kubernetes'

commands/bake.go

@@ -22,6 +22,7 @@ import (
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli/command"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/pkg/errors"
@@ -130,15 +131,30 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
 		return err
 	}
 
+	var resp map[string]*client.SolveResponse
+
 	defer func() {
 		if printer != nil {
 			err1 := printer.Wait()
 			if err == nil {
 				err = err1
 			}
-			if err == nil && progressMode != progressui.QuietMode && progressMode != progressui.RawJSONMode {
+			if err != nil {
+				return
+			}
+			if progressMode != progressui.QuietMode && progressMode != progressui.RawJSONMode {
 				desktop.PrintBuildDetails(os.Stderr, printer.BuildRefs(), term)
 			}
+			if resp != nil && len(in.metadataFile) > 0 {
+				dt := make(map[string]interface{})
+				for t, r := range resp {
+					dt[t] = decodeExporterResponse(r.ExporterResponse)
+				}
+				if warnings := printer.Warnings(); len(warnings) > 0 && confutil.MetadataWarningsEnabled() {
+					dt["buildx.build.warnings"] = warnings
+				}
+				err = writeMetadataFile(in.metadataFile, dt)
+			}
 		}
 	}()
 
@@ -229,22 +245,12 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
 		return err
 	}
 
-	resp, err := build.Build(ctx, nodes, bo, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer)
+	resp, err = build.Build(ctx, nodes, bo, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer)
 	if err != nil {
 		return wrapBuildError(err, true)
 	}
 
-	if len(in.metadataFile) > 0 {
-		dt := make(map[string]interface{})
-		for t, r := range resp {
-			dt[t] = decodeExporterResponse(r.ExporterResponse)
-		}
-		if err := writeMetadataFile(in.metadataFile, dt); err != nil {
-			return err
-		}
-	}
-
-	return err
+	return
 }
 
 func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {

commands/build.go

@@ -374,7 +374,11 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
 			return err
 		}
 	} else if options.metadataFile != "" {
-		if err := writeMetadataFile(options.metadataFile, decodeExporterResponse(resp.ExporterResponse)); err != nil {
+		dt := decodeExporterResponse(resp.ExporterResponse)
+		if warnings := printer.Warnings(); len(warnings) > 0 && confutil.MetadataWarningsEnabled() {
+			dt["buildx.build.warnings"] = warnings
+		}
+		if err := writeMetadataFile(options.metadataFile, dt); err != nil {
 			return err
 		}
 	}

docs/reference/buildx_bake.md

@@ -119,6 +119,7 @@ $ cat metadata.json
 
 ```json
 {
+  "buildx.build.warnings": {},
   "db": {
     "buildx.build.provenance": {},
     "buildx.build.ref": "mybuilder/mybuilder0/0fjb6ubs52xx3vygf6fgdl611",
@@ -161,6 +162,12 @@ $ cat metadata.json
 > * `max` sets full provenance.
 > * `disabled`, `false` or `0` does not set any provenance.
 
+> **Note**
+> 
+> Build warnings (`buildx.build.warnings`) are not included by default. Set the
+> `BUILDX_METADATA_WARNINGS` environment variable to `1` or `true` to
+> include them.
+
 ### <a name="no-cache"></a> Don't use cache when building the image (--no-cache)
 
 Same as `build --no-cache`. Don't use cache when building the image.

docs/reference/buildx_build.md

@@ -330,6 +330,7 @@ $ cat metadata.json
 {
   "buildx.build.provenance": {},
   "buildx.build.ref": "mybuilder/mybuilder0/0fjb6ubs52xx3vygf6fgdl611",
+  "buildx.build.warnings": {},
   "containerimage.config.digest": "sha256:2937f66a9722f7f4a2df583de2f8cb97fc9196059a410e7f00072fc918930e66",
   "containerimage.descriptor": {
     "annotations": {
@@ -353,6 +354,12 @@ $ cat metadata.json
 > * `max` sets full provenance.
 > * `disabled`, `false` or `0` does not set any provenance.
 
+> **Note**
+> 
+> Build warnings (`buildx.build.warnings`) are not included by default. Set the
+> `BUILDX_METADATA_WARNINGS` environment variable to `1` or `true` to
+> include them.
+
 ### <a name="no-cache-filter"></a> Ignore build cache for specific stages (--no-cache-filter)
 
 The `--no-cache-filter` lets you specify one or more stages of a multi-stage

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992
 	github.com/hashicorp/hcl/v2 v2.20.1
 	github.com/in-toto/in-toto-golang v0.5.0
+	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/moby/buildkit v0.14.1
 	github.com/moby/sys/mountinfo v0.7.1
 	github.com/moby/sys/signal v0.7.0

go.sum

@@ -295,6 +295,8 @@ github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
+github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=

hack/test-driver

@@ -109,21 +109,21 @@ buildxCmd inspect --bootstrap --builder="${builderName}"
 
 # create dockerfile
 cat > "${dockerfile}" <<EOL
-FROM busybox as build
+fRom busybox as build
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 RUN echo "I am running on \$BUILDPLATFORM, building for \$TARGETPLATFORM" > /log
 
-FROM busybox AS log
+FROM busybox As log
 COPY --from=build /log /log
 RUN cat /log
 RUN uname -a
 
-FROM busybox AS hello
+FROm busybox AS hello
 RUN echo hello > /hello
 
 FROM scratch
-COPY --from=log /log /log
+CoPY --from=log /log /log
 COPY --from=hello /hello /hello
 EOL
 

tests/bake.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/containerd/continuity/fs/fstest"
 	"github.com/docker/buildx/util/gitutil"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/identity"
 	provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
 	"github.com/moby/buildkit/util/contentutil"
@@ -42,7 +43,9 @@ var bakeTests = []func(t *testing.T, sb integration.Sandbox){
 	testBakeEmpty,
 	testBakeShmSize,
 	testBakeUlimits,
-	testBakeMetadata,
+	testBakeMetadataProvenance,
+	testBakeMetadataWarnings,
+	testBakeMetadataWarningsDedup,
 	testBakeMultiExporters,
 	testBakeLoadPush,
 }
@@ -633,19 +636,22 @@ target "default" {
 	require.Contains(t, string(dt), `1024`)
 }
 
-func testBakeMetadata(t *testing.T, sb integration.Sandbox) {
+func testBakeMetadataProvenance(t *testing.T, sb integration.Sandbox) {
+	t.Run("default", func(t *testing.T) {
+		bakeMetadataProvenance(t, sb, "")
+	})
 	t.Run("max", func(t *testing.T) {
-		bakeMetadata(t, sb, "max")
+		bakeMetadataProvenance(t, sb, "max")
 	})
 	t.Run("min", func(t *testing.T) {
-		bakeMetadata(t, sb, "min")
+		bakeMetadataProvenance(t, sb, "min")
 	})
 	t.Run("disabled", func(t *testing.T) {
-		bakeMetadata(t, sb, "disabled")
+		bakeMetadataProvenance(t, sb, "disabled")
 	})
 }
 
-func bakeMetadata(t *testing.T, sb integration.Sandbox, metadataMode string) {
+func bakeMetadataProvenance(t *testing.T, sb integration.Sandbox, metadataMode string) {
 	dockerfile := []byte(`
 FROM scratch
 COPY foo /foo
@@ -676,7 +682,7 @@ target "default" {
 		withEnv("BUILDX_METADATA_PROVENANCE="+metadataMode),
 	)
 	out, err := cmd.CombinedOutput()
-	require.NoError(t, err, out)
+	require.NoError(t, err, string(out))
 
 	dt, err := os.ReadFile(filepath.Join(dirDest, "md.json"))
 	require.NoError(t, err)
@@ -706,6 +712,130 @@ target "default" {
 	require.Equal(t, provenancetypes.BuildKitBuildType, prv.BuildType)
 }
 
+func testBakeMetadataWarnings(t *testing.T, sb integration.Sandbox) {
+	t.Run("default", func(t *testing.T) {
+		bakeMetadataWarnings(t, sb, "")
+	})
+	t.Run("true", func(t *testing.T) {
+		bakeMetadataWarnings(t, sb, "true")
+	})
+	t.Run("false", func(t *testing.T) {
+		bakeMetadataWarnings(t, sb, "false")
+	})
+}
+
+func bakeMetadataWarnings(t *testing.T, sb integration.Sandbox, mode string) {
+	dockerfile := []byte(`
+frOM busybox as base
+cOpy Dockerfile .
+from scratch
+COPy --from=base \
+  /Dockerfile \
+  /
+	`)
+	bakefile := []byte(`
+target "default" {
+}
+`)
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+
+	dirDest := t.TempDir()
+
+	cmd := buildxCmd(
+		sb,
+		withDir(dir),
+		withArgs("bake", "--metadata-file", filepath.Join(dirDest, "md.json"), "--set", "*.output=type=cacheonly"),
+		withEnv("BUILDX_METADATA_WARNINGS="+mode),
+	)
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+
+	dt, err := os.ReadFile(filepath.Join(dirDest, "md.json"))
+	require.NoError(t, err)
+
+	type mdT struct {
+		BuildWarnings []client.VertexWarning `json:"buildx.build.warnings"`
+		Default       struct {
+			BuildRef string `json:"buildx.build.ref"`
+		} `json:"default"`
+	}
+	var md mdT
+	err = json.Unmarshal(dt, &md)
+	require.NoError(t, err, string(dt))
+
+	require.NotEmpty(t, md.Default.BuildRef, string(dt))
+	if mode == "" || mode == "false" {
+		require.Empty(t, md.BuildWarnings, string(dt))
+		return
+	}
+
+	skipNoCompatBuildKit(t, sb, ">= 0.14.0-0", "lint")
+	require.Len(t, md.BuildWarnings, 3, string(dt))
+}
+
+func testBakeMetadataWarningsDedup(t *testing.T, sb integration.Sandbox) {
+	dockerfile := []byte(`
+frOM busybox as base
+cOpy Dockerfile .
+from scratch
+COPy --from=base \
+  /Dockerfile \
+  /
+	`)
+	bakefile := []byte(`
+group "default" {
+  targets = ["base", "def"]
+}
+target "base" {
+  target = "base"
+}
+target "def" {
+}
+`)
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+
+	dirDest := t.TempDir()
+
+	cmd := buildxCmd(
+		sb,
+		withDir(dir),
+		withArgs("bake", "--metadata-file", filepath.Join(dirDest, "md.json"), "--set", "*.output=type=cacheonly"),
+		withEnv("BUILDX_METADATA_WARNINGS=true"),
+	)
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+
+	dt, err := os.ReadFile(filepath.Join(dirDest, "md.json"))
+	require.NoError(t, err)
+
+	type mdT struct {
+		BuildWarnings []client.VertexWarning `json:"buildx.build.warnings"`
+		Base          struct {
+			BuildRef string `json:"buildx.build.ref"`
+		} `json:"base"`
+		Def struct {
+			BuildRef string `json:"buildx.build.ref"`
+		} `json:"def"`
+	}
+	var md mdT
+	err = json.Unmarshal(dt, &md)
+	require.NoError(t, err, string(dt))
+
+	require.NotEmpty(t, md.Base.BuildRef, string(dt))
+	require.NotEmpty(t, md.Def.BuildRef, string(dt))
+
+	skipNoCompatBuildKit(t, sb, ">= 0.14.0-0", "lint")
+	require.Len(t, md.BuildWarnings, 3, string(dt))
+}
+
 func testBakeMultiExporters(t *testing.T, sb integration.Sandbox) {
 	if !isDockerContainerWorker(sb) {
 		t.Skip("only testing with docker-container worker")