awslabs · tuladp · Jun 21, 2024 · Jun 23, 2024 · Jun 24, 2024 · Jun 24, 2024
diff --git a/deploy.sh b/deploy.sh
@@ -257,7 +257,7 @@ devops_account () {
     template_protection "$STACK_NAME" "$REGION" "$DEVOPS_AWS_PROFILE"
     rm -Rf "$DIRNAME"/output
 
-    declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main")
+    declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main" "sdlf-stage-lambda" "sdlf-stage-glue")
     if "$MONITORING"
     then
         REPOSITORIES+=("sdlf-monitoring")
@@ -272,15 +272,19 @@ devops_account () {
             GITLAB_NAMESPACE_ID=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/NamespaceId --query "Parameter.Value" --output text)
             GITLAB_GROUP_NAME=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --name /SDLF/GitLab/SdlfGitLabGroup --query "Parameter.Value" --output text)
 
+            GITLAB_HOST_NAME=gitlab.ssh.covestro.com
+
             echo "Creating $REPOSITORY repository in GitLab ..."
-            curl --request POST --header "PRIVATE-TOKEN: $GITLAB_ACCESSTOKEN" \
+            curl --insecure --request POST --header "PRIVATE-TOKEN: $GITLAB_ACCESSTOKEN" \
                 --header "Content-Type: application/json" \
                 --data "{\"name\": \"$REPOSITORY\", \"description\": \"$REPOSITORY\", \"path\": \"$REPOSITORY\", \"namespace_id\": \"$GITLAB_NAMESPACE_ID\", \"initialize_with_readme\": false}" \
                 --url "${GITLAB_URL}api/v4/projects/"
 
 
             GITLAB_REPOSITORY_URL="https://aws:$GITLAB_ACCESSTOKEN@${GITLAB_URL#https://}${GITLAB_GROUP_NAME}/$REPOSITORY.git"
+            GITLAB_SSH_URI=git@${GITLAB_HOST_NAME}:${GITLAB_GROUP_NAME}/$REPOSITORY.git
 
+            echo "Origin for repo is $GITLAB_SSH_URI"
             if [ "$REPOSITORY" = "sdlf-main" ]
             then
                 mkdir sdlf-main
@@ -289,8 +293,9 @@ devops_account () {
             pushd "$REPOSITORY" || exit
             if [ ! -d .git ] # if .git exists, deploy.sh has likely been run before - do not try to push the base repositories
             then
-                git init
-                git remote add origin "$GITLAB_REPOSITORY_URL" || exit 1
+                git init --initial-branch=main
+                git remote rename origin old-origin
+                git remote add origin "$GITLAB_SSH_URI" || exit 1
                 git add .
                 git commit -m "initial commit"
                 git push origin main || exit 1
@@ -306,7 +311,7 @@ devops_account () {
     done
 
     aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" s3api put-object --bucket "$ARTIFACTS_BUCKET" --key sam-translate.py --body "$DIRNAME"/sdlf-cicd/sam-translate.py
-    curl -L -O --output-dir "$DIRNAME"/sdlf-cicd/ https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
+    curl -L -O --insecure --output-dir "$DIRNAME"/sdlf-cicd/ https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
     aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" s3api put-object --bucket "$ARTIFACTS_BUCKET" --key aws-sam-cli-linux-x86_64.zip --body "$DIRNAME"/sdlf-cicd/aws-sam-cli-linux-x86_64.zip
     rm "$DIRNAME"/sdlf-cicd/aws-sam-cli-linux-x86_64.zip
 

diff --git a/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py b/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py
@@ -2,6 +2,7 @@
 import logging
 import os
 import zipfile
+import ssl
 from io import BytesIO
 from tempfile import mkdtemp
 from urllib.request import HTTPError, Request, URLError, urlopen
@@ -182,13 +183,13 @@ def create_team_repository_cicd_stack(domain, team_name, template_body_url, clou
     }
     json_data = json.dumps(data).encode('utf-8')
     req = Request(url, data=json_data, headers=headers, method='POST')
-
+    unverified_context = ssl._create_unverified_context()
     try:
-        with urlopen(req) as response:
+        with urlopen(req, context=unverified_context) as response:
             response_body = response.read().decode('utf-8')
             logger.info(response_body)
     except HTTPError as e:
-        logger.error(f"HTTP error occurred: {e.code} {e.reason}")
+        logger.warn(f"HTTP error occurred: {e.code} {e.reason}. Most likely the repository {repository} already exists")
     except URLError as e:
         logger.error(f"URL error occurred: {e.reason}")
 
@@ -318,11 +319,21 @@ def create_team_pipeline_cicd_stack(
                     "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}",
                     "UsePreviousValue": False,
                 },
+                {
+                    "ParameterKey": "pStageLambdaRepository",
+                    "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}",
+                    "UsePreviousValue": False,
+                },
                 {
                     "ParameterKey": "pStageBRepository",
                     "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}",
                     "UsePreviousValue": False,
                 },
+                {
+                    "ParameterKey": "pStageGlueRepository",
+                    "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}",
+                    "UsePreviousValue": False,
+                },
                 {
                     "ParameterKey": "pDatasetRepository",
                     "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}",
@@ -390,11 +401,21 @@ def create_team_pipeline_cicd_stack(
                         "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}",
                         "UsePreviousValue": False,
                     },
+                    {
+                        "ParameterKey": "pStageLambdaRepository",
+                        "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}",
+                        "UsePreviousValue": False,
+                    },                    
                     {
                         "ParameterKey": "pStageBRepository",
                         "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}",
                         "UsePreviousValue": False,
                     },
+                    {
+                        "ParameterKey": "pStageGlueRepository",
+                        "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}",
+                        "UsePreviousValue": False,
+                    },                    
                     {
                         "ParameterKey": "pDatasetRepository",
                         "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}",
@@ -697,4 +718,4 @@ def lambda_handler(event, context):
         raise
 
     codepipeline.put_job_success_result(jobId=event["CodePipeline.job"]["id"])
-    return "Success"
+    return "Success"
diff --git a/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml b/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml
@@ -90,24 +90,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
         - PolicyName: sdlf-cicd-build-stages-cfn-modules
           PolicyDocument:
@@ -127,7 +120,7 @@ Resources:
                   - codecommit:GetUploadArchiveStatus
                   - codecommit:CancelUploadArchive
                 Resource:
-                  - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStagesRepositoriesPrefix}*
+                  - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:*:${pStagesRepositoriesPrefix}*
 
   rBuildCloudformationModuleStage:
     Type: AWS::CodeBuild::Project

diff --git a/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml b/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml
@@ -87,24 +87,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
   rGlueJobPackage:

diff --git a/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml b/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml
@@ -83,24 +83,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
   rBuildLambdaLayersPackage:

diff --git a/sdlf-cicd/template-cicd-domain-roles.yaml b/sdlf-cicd/template-cicd-domain-roles.yaml
@@ -427,12 +427,6 @@ Resources:
                   - lambda:CreateFunction
                   - lambda:UpdateFunctionConfiguration
                 Resource: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:sdlf-*
-                Condition: !If
-                  - RunInVpc
-                  - StringEquals:
-                      "lambda:VpcIds":
-                        - "{{resolve:ssm:/SDLF/VPC/VpcId}}"
-                  - !Ref "AWS::NoValue"
               - Effect: Allow
                 Action:
                   - lambda:AddPermission

diff --git a/sdlf-cicd/template-cicd-domain-team-role.yaml b/sdlf-cicd/template-cicd-domain-team-role.yaml
@@ -293,12 +293,6 @@ Resources:
               - lambda:CreateFunction
               - lambda:UpdateFunctionConfiguration
             Resource: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:sdlf-${pTeamName}-*
-            Condition: !If
-              - RunInVpc
-              - StringEquals:
-                  "lambda:VpcIds":
-                    - "{{resolve:ssm:/SDLF/VPC/VpcId}}"
-              - !Ref "AWS::NoValue"
           - Effect: Allow
             Action:
               - lambda:AddPermission

diff --git a/sdlf-cicd/template-cicd-sdlf-pipelines.yaml b/sdlf-cicd/template-cicd-sdlf-pipelines.yaml
@@ -664,24 +664,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
   rCloudFormationPackageCodeBuildProject:
@@ -792,24 +785,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
 

diff --git a/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml b/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml
@@ -8,7 +8,7 @@ Parameters:
     Default: /SDLF/KMS/CICDKeyId
   pSdlfGitLabGroup:
     Type: String
-    Default: datamesh
+    Default: covestro-analytics-platform/datamesh
   pCicdRepository:
     Type: String
     Default: sdlf-cicd
@@ -27,9 +27,15 @@ Parameters:
   pStageARepository:
     Type: String
     Default: sdlf-stageA
+  pStageLambdaRepository:
+    Type: String
+    Default: sdlf-stage-lambda
   pStageBRepository:
     Type: String
     Default: sdlf-stageB
+  pStageGlueRepository:
+    Type: String
+    Default: sdlf-stage-glue
   pDatalakeLibraryRepository:
     Type: String
     Default: sdlf-datalakeLibrary
@@ -114,6 +120,14 @@ Resources:
       Value: !Ref pStageARepository # !GetAtt rStageAGitLab.Name
       Description: Name of the StageA repository
 
+  rStageLambdaGitLabSsm:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: /SDLF/GitLab/StageLambdaGitLab
+      Type: String
+      Value: !Ref pStageLambdaRepository # !GetAtt rStageLambdaGitLab.Name
+      Description: Name of the Stage-Lambda repository
+
   rStageBGitLabSsm:
     Type: AWS::SSM::Parameter
     Properties:
@@ -122,6 +136,14 @@ Resources:
       Value: !Ref pStageBRepository # !GetAtt rStageBGitLab.Name
       Description: Name of the StageB repository
 
+  rStageGlueGitLabSsm:
+    Type: AWS::SSM::Parameter
+    Properties:
+      Name: /SDLF/GitLab/StageGlueGitLab
+      Type: String
+      Value: !Ref pStageGlueRepository # !GetAtt rStageGlueGitLab.Name
+      Description: Name of the Stage-Glue repository
+
   rDatalakeLibraryGitLabSsm:
     Type: AWS::SSM::Parameter
     Properties:
@@ -159,4 +181,4 @@ Outputs:
   # workaround {{resolve:ssm:}} not returning an array that can be used directly in VpcConfig blocks
   oKmsKey:
     Description: CICD KMS Key
-    Value: !Ref pKMSKey
+    Value: !Ref pKMSKey