Automated patching for arclight using CodePipeline

infrastructure/cinco/config/dags-deploy.yaml

@@ -2,11 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-dags-deploy
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
   CodeBuildPolicies:
     - arn:aws:iam::777968769372:policy/cinco-dags-sync
   TriggerFilterGroups:

infrastructure/cinco/config/patching-lambda-codebuild.yaml

@@ -0,0 +1,48 @@
+template:
+  path: codebuild.j2
+  type: file
+parameters:
+  Namespace: cinco-trigger-ecs-patching-pipeline
+
+sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
+  ECRRepositoryName: cinco-trigger-ecs-patching-pipeline
+  BuildEnvironment:
+    - SUBNET_IDS: {{ var.stage.SUBNET_IDS }}
+  TriggerFilterGroups:
+    - - Type: EVENT
+        Pattern: PUSH
+      - Type: HEAD_REF
+        Pattern: ^refs/heads/main$
+        ExcludeMatchedPattern: false
+      - Type: FILE_PATH
+        Pattern: "infrastructure/lambdas/trigger-ecs-patching-pipeline/*"
+        ExcludeMatchedPattern: false
+  build_spec: |-
+    !Sub >-
+              version: 0.2
+
+              phases:
+                pre_build:
+                  commands:
+                    - cd $CODEBUILD_SRC_DIR/infrastructure/lambdas/trigger-ecs-patching-pipeline
+                build:
+                  commands:
+                    # login to ECR public so we can pull lambda image
+                    - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+                    - docker build --platform linux/amd64 -t ${Namespace} .
+                    # login to ECR private so we can push new image
+                    - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com
+                    - docker tag ${Namespace}:latest ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/${Namespace}:latest
+                    - docker push ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/${Namespace}:latest
+                    # update lambda function with new image
+                    - aws lambda update-function-code --function-name cinco-trigger-ecs-patching-pipeline-lambda-function --image-uri ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/${Namespace}:latest
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - lambda:GetLayerVersion
+                  - lambda:UpdateFunctionCode
+                Resource: "*"

infrastructure/cinco/config/patching-lambda-function.yaml

@@ -0,0 +1,31 @@
+template:
+  path: lambda-function.j2
+  type: file
+parameters:
+  Namespace: cinco-trigger-ecs-patching-pipeline
+  VpcId: {{ var.prd.VPC_ID }}
+  FunctionDescription: Trigger patching pipeline for an ECS Service
+  ECRRepositoryName: cinco-trigger-ecs-patching-pipeline
+  SubnetId1: subnet-08a31a6d2aff9ac32
+  SubnetId2: subnet-09e65806b80ebad6b
+  Timeout: '20'
+
+sceptre_user_data:
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:ListServiceDeployments
+                  - ecs:DescribeServiceRevisions
+                Resource:
+                  - !Sub arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:service/*
+                  - !Sub arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:service-revision/*
+              - Effect: Allow
+                Action:
+                  - ecs:DescribeTaskDefinition
+                Resource:
+                  - '*'
+              - Effect: Allow
+                Action:
+                  - codepipeline:StartPipelineExecution
+                Resource:
+                  - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:*

infrastructure/cinco/config/prd/arclight/build.yaml

@@ -2,20 +2,15 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-arclight-prd
-  Branch: ''
   ECRRepositoryURI: 777968769372.dkr.ecr.us-west-2.amazonaws.com/cinco-arclight
 
 sceptre_user_data:
+  Source:
+    Type: CODEPIPELINE
   BuildEnvironment:
     - SUBNET_IDS: {{ var.prd.SUBNET_IDS }}
     - SLACK_HOOK: {{ var.prd.SLACK_HOOK }}
-  TriggerFilterGroups:
-  # A build is triggered if any filter group evaluates to true, which
-  # occurs when all the filters in the group evaluate to true.
-    - - Type: EVENT
-        Pattern: RELEASED
   build_spec: |-
     !Sub >-
               version: 0.2
@@ -26,7 +21,8 @@ sceptre_user_data:
                     python: 3.12
                 build:
                   commands:
-                    - TAG=`git describe --tags --abbrev=0`
+                    - VERSION=`git describe --tags --abbrev=0`
+                    - TAG="$VERSION--`date +"%Y%m%d%H%M%S"`"
                     - REPO="$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com"
                     - NEW_IMAGE="$REPO/cinco-arclight:$TAG"
                     - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $REPO
@@ -39,13 +35,19 @@ sceptre_user_data:
                     - docker cp arclight-tail-dev-null:/rails/public/assets ./public_assets
                     - aws s3 cp --recursive ./public_assets s3://cinco-prd/arclight_assets/assets
                     - docker stop arclight-tail-dev-null
-                    - TASK_FAMILY="cinco-arclight-prd"
-                    - TASK_DEFINITION=$(aws ecs describe-task-definition --task-definition "$TASK_FAMILY")
-                    - NEW_TASK_DEFINITION=$(echo $TASK_DEFINITION | jq --arg IMAGE "$NEW_IMAGE" '.taskDefinition | .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) | del(.registeredAt) | del(.registeredBy)')
-                    - REGISTERED_DEFINITION=$(aws ecs register-task-definition --cli-input-json "$NEW_TASK_DEFINITION")
-                    - TD_REVISION=$(echo $REGISTERED_DEFINITION | jq -r '.taskDefinition.revision')
-                    - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/message_slack.py --application arclight --cinco_version $TAG --td_family_revisions "$TASK_FAMILY:$TD_REVISION"
-                    - aws ecs update-service --cluster cinco-prd --service cinco-arclight-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
+                    - printf '[{"name":"cinco-arclight-prd-container","imageUri":"%s"}]' ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/cinco-arclight:$TAG > imagedefinitions.json
+
+                    ## move to codepipeline deploy phase:
+                    # - TASK_FAMILY="cinco-arclight-prd"
+                    # - TASK_DEFINITION=$(aws ecs describe-task-definition --task-definition "$TASK_FAMILY")
+                    # - NEW_TASK_DEFINITION=$(echo $TASK_DEFINITION | jq --arg IMAGE "$NEW_IMAGE" '.taskDefinition | .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) | del(.registeredAt) | del(.registeredBy)')
+                    # - REGISTERED_DEFINITION=$(aws ecs register-task-definition --cli-input-json "$NEW_TASK_DEFINITION")
+                    # - TD_REVISION=$(echo $REGISTERED_DEFINITION | jq -r '.taskDefinition.revision')
+                    # - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/message_slack.py --application arclight --cinco_version $TAG --td_family_revisions "$TASK_FAMILY:$TD_REVISION"
+                    # - aws ecs update-service --cluster cinco-prd --service cinco-arclight-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
+              artifacts:
+                files:
+                  - imagedefinitions.json
   additional_policy_statements: |-
     - Effect: Allow
                 Action:
@@ -56,3 +58,4 @@ sceptre_user_data:
                   - s3:GetBucketLocation
                 Resource:
                   - !Sub arn:aws:s3:::cinco-prd*
+                  - arn:aws:s3:::cinco-arclight-prd-pipeline-artifacts*

infrastructure/cinco/config/prd/arclight/code-pipeline.yaml

@@ -0,0 +1,22 @@
+template:
+  path: code-pipeline.yaml.j2
+  type: file
+parameters:
+  Namespace: cinco-arclight-prd
+  SourceRepositoryId: ucldc/cinco
+  ChangeDetectionBranchName: main
+  CodeConnectionArn: arn:aws:codeconnections:us-west-2:777968769372:connection/a757f2ea-6fdf-4153-bf40-6a2c7e9dd96d
+  CodeBuildRoleName: !stack_output_external cinco-prd-arclight-build::CodeBuildRole
+  CodeBuildProjectName: cinco-arclight-prd
+  EcsClusterName: cinco-prd
+  EcsServiceName: cinco-arclight-prd-service
+
+sceptre_user_data:
+  # there is no native release event for CodePipeline, so we trigger on
+  # tags starting with 'v' for version
+  GitPushFilter:
+    - Tags:
+        Includes:
+          - v*
+  CodeBuildEnvironmentVariables: '[{"name":"CURRENT_ENVIRONMENT","value":"Production","type":"PLAINTEXT"},
+                              {"name":"UseParamStore","value":"CURRENT_ENVIRONMENT","type":"PARAMETER_STORE"}]'

infrastructure/cinco/config/prd/arclight/patching-scheduler.yaml

@@ -0,0 +1,10 @@
+template:
+  path: lambda-scheduler.yaml
+  type: file
+parameters:
+  Namespace: cinco-arclight-prd-patching
+  TargetLambdaArn: !stack_output_external cinco-patching-lambda-function::LambdaFunctionArn
+  Description: Scheduler for cinco arclight prd regular patching
+  # Wednesdays at 10:30am
+  ScheduleExpression: "cron(30 10 ? * 4 *)"
+  Input: '{"pipeline": "cinco-arclight-prd-pipeline", "cluster": "cinco-prd", "service": "cinco-arclight-prd-service", "gitrepo": "ucldc/cinco"}'

infrastructure/cinco/config/prd/cincoctrl/build.yaml

@@ -2,20 +2,21 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-ctrl-prd
-  Branch: ''
   ECRRepositoryURI: 777968769372.dkr.ecr.us-west-2.amazonaws.com/cinco-ctrl
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
   BuildEnvironment:
-    - SUBNET_IDS: {{ var.prd.SUBNET_IDS }}
-    - SLACK_HOOK: {{ var.prd.SLACK_HOOK }}
+  - SUBNET_IDS: {{ var.prd.SUBNET_IDS }}
+  - SLACK_HOOK: {{ var.prd.SLACK_HOOK }}
   TriggerFilterGroups:
   # A build is triggered if any filter group evaluates to true, which
   # occurs when all the filters in the group evaluate to true.
-    - - Type: EVENT
-        Pattern: RELEASED
+  - - Type: EVENT
+      Pattern: RELEASED
   build_spec: |-
     !Sub >-
               version: 0.2
@@ -49,3 +50,12 @@ sceptre_user_data:
                     - echo "Forcing a new deployment of the service to apply changes"
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/message_slack.py --application cincoctrl --cinco_version $TAG --td_family_revisions "$TASK_FAMILY:$TD_REVISION"
                     - aws ecs update-service --cluster cinco-prd --service cinco-ctrl-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+                Resource: "*"

infrastructure/cinco/config/prd/solr/build.yaml

@@ -2,12 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-solr-prd
-  Branch: ''
   ECRRepositoryURI: !stack_output stage/solr/build.yaml::ECRRepository
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
   TriggerFilterGroups:
   # A build is triggered if any filter group evaluates to true, which
   # occurs when all the filters in the group evaluate to true.
@@ -71,3 +72,13 @@ sceptre_user_data:
                     # - aws ecs update-service --cluster cinco-prd --service cinco-solr-follower-2-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
                     ###### Message Slack #####
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/message_slack.py --application solr --cinco_version $TAG --td_family_revisions $LEADER $FOLLOWER_1 $FOLLOWER_2
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+
+                Resource: "*"

infrastructure/cinco/config/stage/arclight/build.yaml

@@ -2,23 +2,14 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-arclight-stage
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: CODEPIPELINE
   ECRRepositoryName: cinco-arclight
   BuildEnvironment:
     - SUBNET_IDS: {{ var.stage.SUBNET_IDS }}
-  TriggerFilterGroups:
-    - - Type: EVENT
-        Pattern: PUSH
-      - Type: HEAD_REF
-        Pattern: ^refs/heads/main$
-        ExcludeMatchedPattern: false
-      - Type: FILE_PATH
-        Pattern: arclight/.*
-        ExcludeMatchedPattern: false
   build_spec: |-
     !Sub >-
               version: 0.2
@@ -36,7 +27,10 @@ sceptre_user_data:
                     - docker cp arclight-tail-dev-null:/rails/public/assets ./public_assets
                     - aws s3 cp --recursive ./public_assets s3://cinco-stage/arclight_assets/assets
                     - docker stop arclight-tail-dev-null
-                    - aws ecs update-service --cluster cinco-stage --service cinco-arclight-stage-service --force-new-deployment
+                    - printf '[{"name":"cinco-arclight-stage-container","imageUri":"%s"}]' ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/cinco-arclight:latest > imagedefinitions.json
+              artifacts:
+                files:
+                  - imagedefinitions.json
   additional_policy_statements: |-
     - Effect: Allow
                 Action:
@@ -47,3 +41,4 @@ sceptre_user_data:
                   - s3:GetBucketLocation
                 Resource:
                   - !Sub arn:aws:s3:::cinco-stage*
+                  - arn:aws:s3:::cinco-arclight-stage-pipeline-artifacts*

infrastructure/cinco/config/stage/arclight/code-pipeline.yaml

@@ -0,0 +1,21 @@
+template:
+  path: code-pipeline.yaml.j2
+  type: file
+parameters:
+  Namespace: cinco-arclight-stage
+  SourceRepositoryId: ucldc/cinco
+  ChangeDetectionBranchName: main
+  CodeConnectionArn: arn:aws:codeconnections:us-west-2:777968769372:connection/a757f2ea-6fdf-4153-bf40-6a2c7e9dd96d
+  CodeBuildRoleName: !stack_output_external cinco-stage-arclight-build::CodeBuildRole
+  CodeBuildProjectName: cinco-arclight-stage
+  EcsClusterName: cinco-stage
+  EcsServiceName: cinco-arclight-stage-service
+
+sceptre_user_data:
+  GitPushFilter:
+    - Branches:
+        Includes:
+          - main
+      FilePaths:
+        Includes:
+          - arclight/**

infrastructure/cinco/config/stage/arclight/patching-scheduler.yaml

@@ -0,0 +1,10 @@
+template:
+  path: lambda-scheduler.yaml
+  type: file
+parameters:
+  Namespace: cinco-arclight-stage-patching
+  TargetLambdaArn: !stack_output_external cinco-patching-lambda-function::LambdaFunctionArn
+  Description: Scheduler for cinco arclight stage regular patching
+  # Tuesdays at 10:30am
+  ScheduleExpression: "cron(30 10 ? * 3 *)"
+  Input: '{"pipeline": "cinco-arclight-stage-pipeline", "cluster": "cinco-stage", "service": "cinco-arclight-stage-service", "gitrepo": "ucldc/cinco"}'

infrastructure/cinco/config/stage/cincoctrl/build.yaml

@@ -2,11 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-ctrl-stage
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
   ECRRepositoryName: cinco-ctrl
   BuildEnvironment:
     - SUBNET_IDS: {{ var.stage.SUBNET_IDS }}
@@ -39,3 +41,13 @@ sceptre_user_data:
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/ecs_manage.py collectstatic --no-input
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/ecs_manage.py migrate --no-input
                     - aws ecs update-service --cluster cinco-stage --service cinco-ctrl-stage-service --force-new-deployment
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+
+                Resource: "*"