Merge pull request #1362 from buildpacks/Update-1336

Overwrite Existing Buildpacks
Signed-off-by: David Freilich <[email protected]>

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/Microsoft/hcsshim v0.8.23 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
 	github.com/containerd/containerd v1.5.8 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.10.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
@@ -49,6 +50,7 @@ require (
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -74,6 +76,7 @@ require (
 	github.com/src-d/gcfg v1.4.0 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect

go.sum

@@ -187,6 +187,7 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1
 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
+github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
@@ -405,6 +406,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -932,6 +934,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

internal/builder/builder.go

@@ -31,7 +31,8 @@ import (
 const (
 	packName = "Pack CLI"
 
-	cnbDir = "/cnb"
+	cnbDir        = "/cnb"
+	buildpacksDir = "/cnb/buildpacks"
 
 	orderPath          = "/cnb/order.toml"
 	stackPath          = "/cnb/stack.toml"
@@ -314,7 +315,7 @@ func (b *Builder) Save(logger logging.Logger, creatorMetadata CreatorMetadata) e
 		return errors.Wrapf(err, "getting label %s", dist.BuildpackLayersLabel)
 	}
 
-	err = addBuildpacks(logger, tmpDir, b.image, b.additionalBuildpacks, bpLayers)
+	err = b.addBuildpacks(logger, tmpDir, b.image, b.additionalBuildpacks, bpLayers)
 	if err != nil {
 		return err
 	}
@@ -381,7 +382,7 @@ func (b *Builder) Save(logger logging.Logger, creatorMetadata CreatorMetadata) e
 
 // Helpers
 
-func addBuildpacks(logger logging.Logger, tmpDir string, image imgutil.Image, additionalBuildpacks []buildpack.Buildpack, bpLayers dist.BuildpackLayers) error {
+func (b *Builder) addBuildpacks(logger logging.Logger, tmpDir string, image imgutil.Image, additionalBuildpacks []buildpack.Buildpack, bpLayers dist.BuildpackLayers) error {
 	type buildpackToAdd struct {
 		tarPath   string
 		diffID    string
@@ -417,6 +418,15 @@ func addBuildpacks(logger logging.Logger, tmpDir string, image imgutil.Image, ad
 			if existingBPInfo.LayerDiffID == diffID.String() {
 				logger.Debugf("Buildpack %s already exists on builder with same contents, skipping...", style.Symbol(bpInfo.FullName()))
 				continue
+			} else {
+				whiteoutsTar, err := b.whiteoutLayer(tmpDir, i, bpInfo)
+				if err != nil {
+					return err
+				}
+
+				if err := image.AddLayer(whiteoutsTar); err != nil {
+					return errors.Wrap(err, "adding whiteout layer tar")
+				}
 			}
 
 			logger.Debugf(BuildpackOnBuilderMessage, style.Symbol(bpInfo.FullName()), style.Symbol(existingBPInfo.LayerDiffID), style.Symbol(diffID.String()))
@@ -788,3 +798,35 @@ func (b *Builder) envLayer(dest string, env map[string]string) (string, error) {
 
 	return fh.Name(), nil
 }
+
+func (b *Builder) whiteoutLayer(tmpDir string, i int, bpInfo dist.BuildpackInfo) (string, error) {
+	bpWhiteoutsTmpDir := filepath.Join(tmpDir, strconv.Itoa(i)+"_whiteouts")
+	if err := os.MkdirAll(bpWhiteoutsTmpDir, os.ModePerm); err != nil {
+		return "", errors.Wrap(err, "creating buildpack whiteouts temp dir")
+	}
+
+	fh, err := os.Create(filepath.Join(bpWhiteoutsTmpDir, "whiteouts.tar"))
+	if err != nil {
+		return "", err
+	}
+	defer fh.Close()
+
+	lw := b.layerWriterFactory.NewWriter(fh)
+	defer lw.Close()
+
+	if err := lw.WriteHeader(&tar.Header{
+		Name: path.Join(buildpacksDir, strings.ReplaceAll(bpInfo.ID, "/", "_"), fmt.Sprintf(".wh.%s", bpInfo.Version)),
+		Size: int64(0),
+		Mode: 0644,
+	}); err != nil {
+		return "", err
+	}
+	if _, err := lw.Write([]byte("")); err != nil {
+		return "", errors.Wrapf(err,
+			"creating whiteout layers' tarfile for buildpack %s",
+			style.Symbol(bpInfo.FullName()),
+		)
+	}
+
+	return fh.Name(), nil
+}

internal/builder/builder_test.go

@@ -658,6 +658,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
 
 				when("duplicated buildpack, has different contents", func() {
 					var bp1v1Alt buildpack.Buildpack
+					var bp1v1AltWithNewContent buildpack.Buildpack
 					it.Before(func() {
 						var err error
 						bp1v1Alt, err = ifakes.NewFakeBuildpack(dist.BuildpackDescriptor{
@@ -671,6 +672,45 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
 								Mixins: []string{"mixinX", "mixinY"},
 							}},
 						}, 0644, ifakes.WithExtraBuildpackContents("coolbeans", "a file cool as beans"))
+
+						h.AssertNil(t, err)
+
+						bp1v1AltWithNewContent, err = ifakes.NewFakeBuildpack(dist.BuildpackDescriptor{
+							API: api.MustParse("0.2"),
+							Info: dist.BuildpackInfo{
+								ID:      "buildpack-1-id",
+								Version: "buildpack-1-version-1",
+							},
+							Stacks: []dist.Stack{{
+								ID:     "some.stack.id",
+								Mixins: []string{"mixinX", "mixinY"},
+							}},
+						}, 0644, ifakes.WithExtraBuildpackContents("coolwatermelon", "a file cool as watermelon"))
+
+						h.AssertNil(t, err)
+					})
+
+					it("uses the whiteout layers", func() {
+						logger := logging.NewLogWithWriters(&outBuf, &outBuf, logging.WithVerbose())
+
+						subject.AddBuildpack(bp1v1Alt)
+						subject.AddBuildpack(bp1v1AltWithNewContent)
+
+						err := subject.Save(logger, builder.CreatorMetadata{})
+						h.AssertNil(t, err)
+
+						h.AssertEq(t, baseImage.IsSaved(), true)
+
+						oldPath := filepath.Join("/cnb", "buildpacks", "buildpack-1-id", "buildpack-1-version-1", "coolbeans")
+						layer, err := baseImage.FindLayerWithPath(oldPath)
+
+						h.AssertEq(t, layer, "")
+						h.AssertError(t, err, fmt.Sprintf("could not find '%s' in any layer", oldPath))
+
+						newPath := filepath.Join("/cnb", "buildpacks", "buildpack-1-id", "buildpack-1-version-1", "coolwatermelon")
+						layer, err = baseImage.FindLayerWithPath(newPath)
+
+						h.AssertNotEq(t, layer, "")
 						h.AssertNil(t, err)
 					})