v1.3 release

Merge changes for v1.3 release

Makefile

@@ -21,7 +21,7 @@ compile-lambda-linux-all:
 	make ARCH=old compile-lambda-linux
 
 compile-with-docker:
-	docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.14 make ARCH=${ARCH} compile-lambda-linux
+	docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.17 make ARCH=${ARCH} compile-lambda-linux
 
 compile-lambda-linux:
 	CGO_ENABLED=0 GOOS=linux GOARCH=${GO_ARCH_${ARCH}} go build -ldflags "${RELEASE_BUILD_LINKER_FLAGS}" -o ${DESTINATION_${ARCH}} ./cmd/aws-lambda-rie

cmd/aws-lambda-rie/main.go

@@ -21,7 +21,8 @@ const (
 )
 
 type options struct {
-	LogLevel string `long:"log-level" default:"info" description:"log level"`
+	LogLevel           string `long:"log-level" default:"info" description:"log level"`
+	InitCachingEnabled bool   `long:"enable-init-caching" description:"Enable support for Init Caching"`
 }
 
 func main() {
@@ -32,7 +33,11 @@ func main() {
 	rapidcore.SetLogLevel(opts.LogLevel)
 
 	bootstrap, handler := getBootstrap(args, opts)
-	sandbox := rapidcore.NewSandboxBuilder(bootstrap).AddShutdownFunc(context.CancelFunc(func() { os.Exit(0) })).SetExtensionsFlag(true)
+	sandbox := rapidcore.
+		NewSandboxBuilder(bootstrap).
+		AddShutdownFunc(context.CancelFunc(func() { os.Exit(0) })).
+		SetExtensionsFlag(true).
+		SetInitCachingFlag(opts.InitCachingEnabled)
 
 	if len(handler) > 0 {
 		sandbox.SetHandler(handler)
@@ -72,7 +77,7 @@ func getBootstrap(args []string, opts options) (*rapidcore.Bootstrap, string) {
 			fmt.Sprintf("%s/bootstrap", currentWorkingDir),
 		}
 
-		if !isBootstrapFileExist(bootstrapLookupCmd[0])  {
+		if !isBootstrapFileExist(bootstrapLookupCmd[0]) {
 			var bootstrapCmdCandidates = []string{
 				optBootstrap,
 				runtimeBootstrap,

go.mod

@@ -1,16 +1,24 @@
 module go.amzn.com
 
-go 1.13
+go 1.17
 
 require (
 	github.com/aws/aws-lambda-go v1.20.0
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/render v1.0.1
 	github.com/google/uuid v1.1.2
 	github.com/jessevdk/go-flags v1.4.0
-	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.6.0
-	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.6.1
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 )
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	golang.org/x/net v0.0.0-20211111160137-58aab5ef257a // indirect
+	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
+)

go.sum

@@ -14,9 +14,8 @@ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -26,16 +25,24 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
 github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+golang.org/x/net v0.0.0-20211111160137-58aab5ef257a h1:c83jeVQW0KGKNaKBRfelNYNHaev+qawl9yaA825s8XE=
+golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

lambda/agents/agent.go

@@ -25,13 +25,12 @@ type ExternalAgentProcess struct {
 }
 
 // NewExternalAgentProcess returns a new external agent process
-func NewExternalAgentProcess(path string, env []string, logWriter io.Writer) ExternalAgentProcess {
+func NewExternalAgentProcess(path string, env []string, stdoutWriter io.Writer, stderrWriter io.Writer) ExternalAgentProcess {
 	command := exec.Command(path)
 	command.Env = env
 
-	w := NewNewlineSplitWriter(logWriter)
-	command.Stdout = w
-	command.Stderr = w
+	command.Stdout = NewNewlineSplitWriter(stdoutWriter)
+	command.Stderr = NewNewlineSplitWriter(stderrWriter)
 	command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 
 	return ExternalAgentProcess{

lambda/agents/agent_test.go

@@ -204,30 +204,76 @@ func TestFindAgentMixed(t *testing.T) {
 // Test our ability to start agents
 func TestAgentStart(t *testing.T) {
 	assert := assert.New(t)
-	agent := NewExternalAgentProcess("../testdata/agents/bash_true.sh", []string{}, &mockWriter{})
+	agent := NewExternalAgentProcess("../testdata/agents/bash_true.sh", []string{}, &mockWriter{}, &mockWriter{})
 	assert.Nil(agent.Start())
 	assert.Nil(agent.Wait())
 }
 
 // Test that execution of invalid agents is correctly reported
 func TestInvalidAgentStart(t *testing.T) {
 	assert := assert.New(t)
-	agent := NewExternalAgentProcess("/bin/none", []string{}, &mockWriter{})
+	agent := NewExternalAgentProcess("/bin/none", []string{}, &mockWriter{}, &mockWriter{})
 	assert.True(os.IsNotExist(agent.Start()))
 }
 
-// Test that execution of invalid agents is correctly reported
-func TestAgentTelemetry(t *testing.T) {
+func TestAgentStdoutWriter(t *testing.T) {
+	// Given
+	assert := assert.New(t)
+
+	stdout := &mockWriter{}
+	stderr := &mockWriter{}
+	expectedStdout := "stdout line 1\nstdout line 2\nstdout line 3\n"
+	expectedStderr := ""
+
+	agent := NewExternalAgentProcess("../testdata/agents/bash_stdout.sh", []string{}, stdout, stderr)
+
+	// When
+	assert.NoError(agent.Start())
+	assert.NoError(agent.Wait())
+
+	// Then
+	assert.Equal(expectedStdout, string(bytes.Join(stdout.bytesReceived, []byte(""))))
+	assert.Equal(expectedStderr, string(bytes.Join(stderr.bytesReceived, []byte(""))))
+}
+
+func TestAgentStderrWriter(t *testing.T) {
+	// Given
 	assert := assert.New(t)
-	buffer := &mockWriter{}
 
-	agent := NewExternalAgentProcess("../testdata/agents/bash_echo.sh", []string{}, buffer)
+	stdout := &mockWriter{}
+	stderr := &mockWriter{}
+	expectedStdout := ""
+	expectedStderr := "stderr line 1\nstderr line 2\nstderr line 3\n"
+
+	agent := NewExternalAgentProcess("../testdata/agents/bash_stderr.sh", []string{}, stdout, stderr)
+
+	// When
+	assert.NoError(agent.Start())
+	assert.NoError(agent.Wait())
+
+	// Then
+	assert.Equal(expectedStdout, string(bytes.Join(stdout.bytesReceived, []byte(""))))
+	assert.Equal(expectedStderr, string(bytes.Join(stderr.bytesReceived, []byte(""))))
+}
+
+func TestAgentStdoutAndStderrSeperateWriters(t *testing.T) {
+	// Given
+	assert := assert.New(t)
+
+	stdout := &mockWriter{}
+	stderr := &mockWriter{}
+	expectedStdout := "stdout line 1\nstdout line 2\nstdout line 3\n"
+	expectedStderr := "stderr line 1\nstderr line 2\nstderr line 3\n"
+
+	agent := NewExternalAgentProcess("../testdata/agents/bash_stdout_and_stderr.sh", []string{}, stdout, stderr)
 
+	// When
 	assert.NoError(agent.Start())
 	assert.NoError(agent.Wait())
 
-	message := "hello world\n|barbaz\n|hello world\n|barbaz2"
-	assert.Equal(message, string(bytes.Join(buffer.bytesReceived, []byte("|"))))
+	// Then
+	assert.Equal(expectedStdout, string(bytes.Join(stdout.bytesReceived, []byte(""))))
+	assert.Equal(expectedStderr, string(bytes.Join(stderr.bytesReceived, []byte(""))))
 }
 
 type mockWriter struct {

lambda/appctx/appctxutil.go

@@ -5,11 +5,10 @@ package appctx
 
 import (
 	"context"
-	"net/http"
-	"strings"
-
 	"go.amzn.com/lambda/fatalerror"
 	"go.amzn.com/lambda/interop"
+	"net/http"
+	"strings"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -24,6 +23,9 @@ type ReqCtxKey int
 // context object into request context.
 const ReqCtxApplicationContextKey ReqCtxKey = iota
 
+// MaxRuntimeReleaseLength Max length for user agent string.
+const MaxRuntimeReleaseLength = 128
+
 // FromRequest retrieves application context from the request context.
 func FromRequest(request *http.Request) ApplicationContext {
 	return request.Context().Value(ReqCtxApplicationContextKey).(ApplicationContext)
@@ -39,24 +41,78 @@ func GetRuntimeRelease(appCtx ApplicationContext) string {
 	return appCtx.GetOrDefault(AppCtxRuntimeReleaseKey, "").(string)
 }
 
-// UpdateAppCtxWithRuntimeRelease extracts runtime release info from user agent header and put it into appCtx.
+// GetUserAgentFromRequest Returns the first token -seperated by a space-
+// from request header 'User-Agent'.
+func GetUserAgentFromRequest(request *http.Request) string {
+	runtimeRelease := ""
+	userAgent := request.Header.Get("User-Agent")
+	// Split around spaces and use only the first token.
+	if fields := strings.Fields(userAgent); len(fields) > 0 && len(fields[0]) > 0 {
+		runtimeRelease = fields[0]
+	}
+	return runtimeRelease
+}
+
+// CreateRuntimeReleaseFromRequest Gets runtime features from request header
+// 'Lambda-Runtime-Features', and append it to the given runtime release.
+func CreateRuntimeReleaseFromRequest(request *http.Request, runtimeRelease string) string {
+	lambdaRuntimeFeaturesHeader := request.Header.Get("Lambda-Runtime-Features")
+
+	// "(", ")" are not valid token characters, and potentially could invalidate runtime_release
+	lambdaRuntimeFeaturesHeader = strings.ReplaceAll(lambdaRuntimeFeaturesHeader, "(", "")
+	lambdaRuntimeFeaturesHeader = strings.ReplaceAll(lambdaRuntimeFeaturesHeader, ")", "")
+
+	numberOfAppendedFeatures := 0
+	// Available length is a maximum length available for runtime features (including delimiters). From maximal runtime
+	// release length we subtract what we already have plus 3 additional bytes for a space and a pair of brackets for
+	// list of runtime features that is added later.
+	runtimeReleaseLength := len(runtimeRelease)
+	if runtimeReleaseLength == 0 {
+		runtimeReleaseLength = len("Unknown")
+	}
+	availableLength := MaxRuntimeReleaseLength - runtimeReleaseLength - 3
+	var lambdaRuntimeFeatures []string
+
+	for _, feature := range strings.Fields(lambdaRuntimeFeaturesHeader) {
+		featureLength := len(feature)
+		// If featureLength <= availableLength - numberOfAppendedFeatures
+		// (where numberOfAppendedFeatures is equal to number of delimiters needed).
+		if featureLength <= availableLength-numberOfAppendedFeatures {
+			availableLength -= featureLength
+			lambdaRuntimeFeatures = append(lambdaRuntimeFeatures, feature)
+			numberOfAppendedFeatures++
+		}
+	}
+	// Append valid features to runtime release.
+	if len(lambdaRuntimeFeatures) > 0 {
+		if runtimeRelease == "" {
+			runtimeRelease = "Unknown"
+		}
+		runtimeRelease += " (" + strings.Join(lambdaRuntimeFeatures, " ") + ")"
+	}
+
+	return runtimeRelease
+}
+
+// UpdateAppCtxWithRuntimeRelease extracts runtime release info from user agent & lambda runtime features
+// headers and update it into appCtx.
 // Sample UA:
 // Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0
 func UpdateAppCtxWithRuntimeRelease(request *http.Request, appCtx ApplicationContext) bool {
-	// If appCtx has runtime release value already, skip updating for consistency.
-	if len(GetRuntimeRelease(appCtx)) > 0 {
-		return false
-	}
-
-	userAgent := request.Header.Get("User-Agent")
-	if len(userAgent) == 0 {
+	// If appCtx has runtime release value already, just append the runtime features.
+	if appCtxRuntimeRelease := GetRuntimeRelease(appCtx); len(appCtxRuntimeRelease) > 0 {
+		// if the runtime features are not appended before append them, otherwise ignore
+		if runtimeReleaseWithFeatures := CreateRuntimeReleaseFromRequest(request, appCtxRuntimeRelease); len(runtimeReleaseWithFeatures) > len(appCtxRuntimeRelease) &&
+			appCtxRuntimeRelease[len(appCtxRuntimeRelease)-1] != ')' {
+			appCtx.Store(AppCtxRuntimeReleaseKey, runtimeReleaseWithFeatures)
+			return true
+		}
 		return false
 	}
-
-	// Split around spaces and use only the first token.
-	if fields := strings.Fields(userAgent); len(fields) > 0 && len(fields[0]) > 0 {
-		appCtx.Store(AppCtxRuntimeReleaseKey,
-			fields[0])
+	// If appCtx doesn't have runtime release value, update it with user agent and runtime features.
+	if runtimeReleaseWithFeatures := CreateRuntimeReleaseFromRequest(request,
+		GetUserAgentFromRequest(request)); runtimeReleaseWithFeatures != "" {
+		appCtx.Store(AppCtxRuntimeReleaseKey, runtimeReleaseWithFeatures)
 		return true
 	}
 	return false