add GitHub action workflow (#3)

Co-authored-by: HarshCasper <[email protected]>

.env

@@ -0,0 +1,12 @@
+AWS_DEFAULT_REGION=us-east-1
+AWS_ENDPOINT_URL=http://localhost:4566
+
+REDSHIFT_MASTER_USER=dwh_user
+REDSHIFT_MASTER_PASSWORD=123456789Test
+REDSHIFT_DB_NAME=streaming_db
+REDSHIFT_CLUSTER_NAME=redshift-cluster
+REDSHIFT_TABLE_NAME=user_health_data
+
+KINESIS_STREAM_NAME=kinesis-stream
+
+S3_BUCKET_NAME=firehose-raw-data

.github/workflows/main.yaml

@@ -0,0 +1,90 @@
+name: Deploy on LocalStack 
+
+on:
+  push:
+    paths-ignore:
+      - 'README.md'
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  AWS_DEFAULT_REGION: us-east-1
+  AWS_ENDPOINT_URL: http://localhost:4566
+  AWS_ACCESS_KEY_ID: test
+  AWS_SECRET_ACCESS_KEY: test
+
+jobs:
+  cdk:
+    name: Setup infrastructure using CDK
+    runs-on: ubuntu-latest
+    environment: default-env
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Load .env file
+        uses: xom9ikk/dotenv@v2
+        with:
+          path: ./
+          mode: ""
+          load-mode: strict
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Install CDK
+        run: |
+          npm install -g aws-cdk-local aws-cdk
+          cdklocal --version
+
+      - name: Set up Python 3.11
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@main
+        with:
+            image-tag: ${{ inputs.release-tag || 'latest'}}
+            use-pro: 'true'
+            install-awslocal: 'true'
+            configuration: DEBUG=1
+        env:
+            LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
+
+      - name: Deploy using CDK
+        run: |
+            make deploy-localstack
+
+      - name: List the resources
+        run: |
+            make list-resources-localstack
+
+      - name: Run pytests
+        run: |
+            make test
+
+      - name: Generate a Diagnostic Report
+        if: failure()
+        run: |
+          curl -s localhost:4566/_localstack/diagnose | gzip -cf > diagnose.json.gz
+
+      - name: Upload the Diagnostic Report
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: diagnose.json.gz
+          path: ./diagnose.json.gz
+

.gitignore

@@ -120,7 +120,6 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
 env/
 venv/

Makefile

@@ -6,10 +6,17 @@ deploy-localstack:
 	cdklocal synth
 	@echo "Deploy redshift stack"
 	cdklocal deploy KinesisFirehoseRedshiftStack1 --require-approval never
-	source .venv/bin/activate &&  python utils/prepare_redshift.py
+	python utils/prepare_redshift.py
 	@echo "Deploy firehose stack"
 	cdklocal deploy KinesisFirehoseRedshiftStack2 --require-approval never
 
+list-resources-localstack:
+	@echo "List resources"
+	awslocal s3 ls
+	awslocal kinesis list-streams
+	awslocal firehose list-delivery-streams
+	awslocal redshift describe-clusters
+
 deploy-aws:
 	@echo "Preparing deployment"
 	cdk bootstrap
@@ -27,4 +34,4 @@ start-localstack:
 	@docker ps -f "name=localstack" | grep localstack > /dev/null || (echo "Starting localstack..." && localstack start)
 
 test:
-	source .venv/bin/activate && pytest -v
+	pytest -v

README.md

@@ -148,6 +148,7 @@ The same is true for the AWS deployment, you can manually clean up the Redshift
 
 ## Github Actions CI tests
 The github actions workflow defined in `.github/workflows/main.yaml` will install the required dependencies, start a LocalStack containerdeploy the infrastructure aginast LocalStack and run the test.
+To set up the workflow, you need to create an environment and set the variables and secrets from you `.env` file.
 The workflow will run on every push to the main branch.
 
 # Interact with the deployed resources