From 34363a810dac1a1e43584a271b44c7df048ffd32 Mon Sep 17 00:00:00 2001 From: Tetsuya Kikuchi <97105818+t-kikuc@users.noreply.github.com> Date: Wed, 7 Aug 2024 08:54:47 +0900 Subject: [PATCH] Support Plan Preview for Lambda (#5046) --- .../cmd/planpreview/planpreview_test.go | 12 +- pkg/app/piped/planpreview/builder.go | 3 +- pkg/app/piped/planpreview/lambdadiff.go | 124 ++++++++++++++++++ .../piped/platformprovider/lambda/cache.go | 68 ++++++++++ pkg/app/piped/platformprovider/lambda/diff.go | 80 +++++++++++ .../platformprovider/lambda/diff_test.go | 91 +++++++++++++ .../lambda/testdata/new_function.yaml | 12 ++ .../lambda/testdata/old_function.yaml | 12 ++ 8 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 pkg/app/piped/planpreview/lambdadiff.go create mode 100644 pkg/app/piped/platformprovider/lambda/cache.go create mode 100644 pkg/app/piped/platformprovider/lambda/diff.go create mode 100644 pkg/app/piped/platformprovider/lambda/diff_test.go create mode 100644 pkg/app/piped/platformprovider/lambda/testdata/new_function.yaml create mode 100644 pkg/app/piped/platformprovider/lambda/testdata/old_function.yaml diff --git a/pkg/app/pipectl/cmd/planpreview/planpreview_test.go b/pkg/app/pipectl/cmd/planpreview/planpreview_test.go index 51530323ab..9360101110 100644 --- a/pkg/app/pipectl/cmd/planpreview/planpreview_test.go +++ b/pkg/app/pipectl/cmd/planpreview/planpreview_test.go @@ -172,6 +172,13 @@ NOTE: An error occurred while building plan-preview for applications of the foll ApplicationKind: model.ApplicationKind_ECS, Error: "wrong application configuration", }, + { + ApplicationId: "app-6", + ApplicationName: "app-6", + ApplicationUrl: "https://pipecd.dev/app-6", + ApplicationKind: model.ApplicationKind_LAMBDA, + Error: "wrong application configuration", + }, }, }, { @@ -203,7 +210,7 @@ changes-1 changes-2 ---DETAILS_END--- -NOTE: An error occurred while building plan-preview for the following 3 applications: +NOTE: An error occurred while building plan-preview for the following 4 applications: 1. app: app-3, env: env-3, kind: TERRAFORM reason: wrong application configuration @@ -214,6 +221,9 @@ NOTE: An error occurred while building plan-preview for the following 3 applicat 3. app: app-5, kind: ECS reason: wrong application configuration +4. app: app-6, kind: LAMBDA + reason: wrong application configuration + NOTE: An error occurred while building plan-preview for applications of the following 2 Pipeds: 1. piped: piped-name-1 (piped-1) diff --git a/pkg/app/piped/planpreview/builder.go b/pkg/app/piped/planpreview/builder.go index 315e8f30a6..a36776a060 100644 --- a/pkg/app/piped/planpreview/builder.go +++ b/pkg/app/piped/planpreview/builder.go @@ -251,8 +251,9 @@ func (b *builder) buildApp(ctx context.Context, worker int, command string, app dr, err = b.cloudrundiff(ctx, app, targetDSP, preCommit, &buf) case model.ApplicationKind_ECS: dr, err = b.ecsdiff(ctx, app, targetDSP, preCommit, &buf) + case model.ApplicationKind_LAMBDA: + dr, err = b.lambdadiff(ctx, app, targetDSP, preCommit, &buf) default: - // TODO: Calculating planpreview's diff for other application kinds. dr = &diffResult{ summary: fmt.Sprintf("%s application is not implemented yet (coming soon)", app.Kind.String()), } diff --git a/pkg/app/piped/planpreview/lambdadiff.go b/pkg/app/piped/planpreview/lambdadiff.go new file mode 100644 index 0000000000..3d80fce576 --- /dev/null +++ b/pkg/app/piped/planpreview/lambdadiff.go @@ -0,0 +1,124 @@ +// Copyright 2024 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package planpreview + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/pipe-cd/pipecd/pkg/app/piped/deploysource" + provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/lambda" + "github.com/pipe-cd/pipecd/pkg/diff" + "github.com/pipe-cd/pipecd/pkg/model" +) + +func (b *builder) lambdadiff( + ctx context.Context, + app *model.Application, + targetDSP deploysource.Provider, + lastCommit string, + buf *bytes.Buffer, +) (*diffResult, error) { + var ( + oldManifest, newManifest provider.FunctionManifest + err error + ) + + newManifest, err = b.loadFunctionManifest(ctx, *app, targetDSP) + if err != nil { + fmt.Fprintf(buf, "failed to load lambda manifest at the head commit (%v)\n", err) + return nil, err + } + + if lastCommit == "" { + fmt.Fprintf(buf, "failed to find the commit of the last successful deployment") + return nil, fmt.Errorf("cannot get the old manifest without the last successful deployment") + } + + runningDSP := deploysource.NewProvider( + b.workingDir, + deploysource.NewGitSourceCloner(b.gitClient, b.repoCfg, "running", lastCommit), + *app.GitPath, + b.secretDecrypter, + ) + + oldManifest, err = b.loadFunctionManifest(ctx, *app, runningDSP) + if err != nil { + fmt.Fprintf(buf, "failed to load lambda manifest at the running commit (%v)\n", err) + return nil, err + } + + result, err := provider.Diff( + oldManifest, + newManifest, + diff.WithEquateEmpty(), + diff.WithCompareNumberAndNumericString(), + ) + if err != nil { + fmt.Fprintf(buf, "failed to compare manifest (%v)\n", err) + return nil, err + } + + if result.NoChange() { + fmt.Fprintln(buf, "No changes were detected") + return &diffResult{ + summary: "No changes were detected", + noChange: true, + }, nil + } + + details := result.Render(provider.DiffRenderOptions{ + UseDiffCommand: true, + }) + fmt.Fprintf(buf, "--- Last Deploy\n+++ Head Commit\n\n%s\n", details) + + return &diffResult{ + summary: fmt.Sprintf("%d changes were detected", len(result.Diff.Nodes())), + }, nil +} + +func (b *builder) loadFunctionManifest(ctx context.Context, app model.Application, dsp deploysource.Provider) (provider.FunctionManifest, error) { + commit := dsp.Revision() + cache := provider.FunctionManifestCache{ + AppID: app.Id, + Cache: b.appManifestsCache, + Logger: b.logger, + } + + manifest, ok := cache.Get(commit) + if ok { + return manifest, nil + } + + ds, err := dsp.Get(ctx, io.Discard) + if err != nil { + return provider.FunctionManifest{}, err + } + + appCfg := ds.ApplicationConfig.LambdaApplicationSpec + if appCfg == nil { + return provider.FunctionManifest{}, fmt.Errorf("malformed application configuration file") + } + + manifest, err = provider.LoadFunctionManifest(ds.AppDir, appCfg.Input.FunctionManifestFile) + if err != nil { + return provider.FunctionManifest{}, err + } + + cache.Put(commit, manifest) + return manifest, nil +} diff --git a/pkg/app/piped/platformprovider/lambda/cache.go b/pkg/app/piped/platformprovider/lambda/cache.go new file mode 100644 index 0000000000..f98cc544f4 --- /dev/null +++ b/pkg/app/piped/platformprovider/lambda/cache.go @@ -0,0 +1,68 @@ +// Copyright 2024 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lambda + +import ( + "errors" + "fmt" + + "go.uber.org/zap" + + "github.com/pipe-cd/pipecd/pkg/cache" +) + +type FunctionManifestCache struct { + AppID string + Cache cache.Cache + Logger *zap.Logger +} + +func (c FunctionManifestCache) Get(commit string) (FunctionManifest, bool) { + key := manifestCacheKey(c.AppID, commit) + item, err := c.Cache.Get(key) + if err == nil { + return item.(FunctionManifest), true + } + + if errors.Is(err, cache.ErrNotFound) { + c.Logger.Info("function manifest wes not found in cache", + zap.String("app-id", c.AppID), + zap.String("commit-hash", commit), + ) + return FunctionManifest{}, false + } + + c.Logger.Error("failed while retrieving function manifest from cache", + zap.String("app-id", c.AppID), + zap.String("commit-hash", commit), + zap.Error(err), + ) + return FunctionManifest{}, false +} + +func (c FunctionManifestCache) Put(commit string, sm FunctionManifest) { + key := manifestCacheKey(c.AppID, commit) + if err := c.Cache.Put(key, sm); err != nil { + c.Logger.Error("failed while putting function manifest into cache", + zap.String("app-id", c.AppID), + zap.String("commit-hash", commit), + zap.Error(err), + ) + } +} + +func manifestCacheKey(appID, commit string) string { + return fmt.Sprintf("%s/%s", appID, commit) +} diff --git a/pkg/app/piped/platformprovider/lambda/diff.go b/pkg/app/piped/platformprovider/lambda/diff.go new file mode 100644 index 0000000000..81861da941 --- /dev/null +++ b/pkg/app/piped/platformprovider/lambda/diff.go @@ -0,0 +1,80 @@ +// Copyright 2024 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lambda + +import ( + "fmt" + "strings" + + "github.com/pipe-cd/pipecd/pkg/diff" +) + +const ( + diffCommand = "diff" +) + +type DiffResult struct { + Diff *diff.Result + Old FunctionManifest + New FunctionManifest +} + +func (d *DiffResult) NoChange() bool { + return len(d.Diff.Nodes()) == 0 +} + +func Diff(old, new FunctionManifest, opts ...diff.Option) (*DiffResult, error) { + d, err := diff.DiffStructureds(old, new, opts...) + if err != nil { + return nil, err + } + + if !d.HasDiff() { + return &DiffResult{Diff: d}, nil + } + + ret := &DiffResult{ + Old: old, + New: new, + Diff: d, + } + return ret, nil +} + +type DiffRenderOptions struct { + // If true, use "diff" command to render. + UseDiffCommand bool +} + +func (d *DiffResult) Render(opt DiffRenderOptions) string { + var b strings.Builder + opts := []diff.RenderOption{ + diff.WithLeftPadding(1), + } + renderer := diff.NewRenderer(opts...) + if !opt.UseDiffCommand { + b.WriteString(renderer.Render(d.Diff.Nodes())) + } else { + d, err := diff.RenderByCommand(diffCommand, d.Old, d.New) + if err != nil { + b.WriteString(fmt.Sprintf("An error occurred while rendering diff (%v)", err)) + } else { + b.Write(d) + } + } + b.WriteString("\n") + + return b.String() +} diff --git a/pkg/app/piped/platformprovider/lambda/diff_test.go b/pkg/app/piped/platformprovider/lambda/diff_test.go new file mode 100644 index 0000000000..477faecb4f --- /dev/null +++ b/pkg/app/piped/platformprovider/lambda/diff_test.go @@ -0,0 +1,91 @@ +// Copyright 2024 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lambda + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDiff(t *testing.T) { + t.Parallel() + + old, err := loadFunctionManifest("testdata/old_function.yaml") + require.NoError(t, err) + + new, err := loadFunctionManifest("testdata/new_function.yaml") + require.NoError(t, err) + + // Expect to have change. + diff, err := Diff(old, new) + require.NoError(t, err) + noChange := diff.NoChange() + require.False(t, noChange) + + // Expect no change. + diff, err = Diff(old, old) + require.NoError(t, err) + noChange = diff.NoChange() + require.True(t, noChange) +} + +func TestDiffResult_Render(t *testing.T) { + old, err := loadFunctionManifest("testdata/old_function.yaml") + require.NoError(t, err) + + new, err := loadFunctionManifest("testdata/new_function.yaml") + require.NoError(t, err) + + result, err := Diff(old, new) + require.NoError(t, err) + + // Not using diff command + opt := DiffRenderOptions{} + actual := result.Render(opt) + expected := ` spec: + environments: + #spec.environments.FOO +- FOO: bar ++ FOO: bar2 + + #spec.image +- image: ecr.ap-northeast-1.amazonaws.com/lambda-test:v0.0.1 ++ image: ecr.ap-northeast-1.amazonaws.com/lambda-test:v0.0.2 + + +` + + require.Equal(t, expected, actual) + + // Use diff command + opt = DiffRenderOptions{UseDiffCommand: true} + actual = result.Render(opt) + expected = `@@ -2,9 +2,9 @@ + kind: LambdaFunction + spec: + environments: +- FOO: bar ++ FOO: bar2 + handler: "" +- image: ecr.ap-northeast-1.amazonaws.com/lambda-test:v0.0.1 ++ image: ecr.ap-northeast-1.amazonaws.com/lambda-test:v0.0.2 + memory: 512 + name: TestFunction + role: arn:aws:iam:region:account-id:role/lambda-role +` + + require.Equal(t, expected, actual) +} diff --git a/pkg/app/piped/platformprovider/lambda/testdata/new_function.yaml b/pkg/app/piped/platformprovider/lambda/testdata/new_function.yaml new file mode 100644 index 0000000000..32910a6b9f --- /dev/null +++ b/pkg/app/piped/platformprovider/lambda/testdata/new_function.yaml @@ -0,0 +1,12 @@ +apiVersion: pipecd.dev/v1beta1 +kind: LambdaFunction +spec: + name: TestFunction + role: arn:aws:iam:region:account-id:role/lambda-role + image: ecr.ap-northeast-1.amazonaws.com/lambda-test:v0.0.2 + memory: 512 + timeout: 30 + environments: + FOO: bar2 + tags: + app: test diff --git a/pkg/app/piped/platformprovider/lambda/testdata/old_function.yaml b/pkg/app/piped/platformprovider/lambda/testdata/old_function.yaml new file mode 100644 index 0000000000..1afe310599 --- /dev/null +++ b/pkg/app/piped/platformprovider/lambda/testdata/old_function.yaml @@ -0,0 +1,12 @@ +apiVersion: pipecd.dev/v1beta1 +kind: LambdaFunction +spec: + name: TestFunction + role: arn:aws:iam:region:account-id:role/lambda-role + image: ecr.ap-northeast-1.amazonaws.com/lambda-test:v0.0.1 + memory: 512 + timeout: 30 + environments: + FOO: bar + tags: + app: test