chore: bring staging changes to production

.travis.yml

@@ -1,5 +1,5 @@
-sudo: required
-language: minimal
+os: linux
+language: shell
 
 git:
   depth: 2
@@ -10,29 +10,38 @@ branches:
   - devel
 
 services:
-  - docker
+- docker
 
 env:
   global:
-    - IMAGE_REPO=gcr.io/dd-decaf-cfbf6/design-storage
-    - IMAGE_TAG=${TRAVIS_BRANCH}
+  - IMAGE=gcr.io/dd-decaf-cfbf6/design-storage
+  - BRANCH=${TRAVIS_BRANCH}
+  - BUILD_COMMIT=${TRAVIS_COMMIT}
+  - SHORT_COMMIT=${TRAVIS_COMMIT:0:7}
+  - BUILD_DATE=$(date -u +%Y-%m-%d)
+  - BUILD_TAG=${BRANCH}_${BUILD_DATE}_${SHORT_COMMIT}
+  - POSTGRES_PASS=secret
+
+before_install:
+- make setup
 
 install:
-  - docker build -t ${IMAGE_REPO}:${TRAVIS_COMMIT::12} -t ${IMAGE_REPO}:${TRAVIS_BRANCH} .
-  - make setup
+- make build
+- make build-travis
+- make databases
+- make start
 
 script:
-  - make flake8
-  - make isort
-  - make license
-  - make safety
-  - make test-travis
+- make style
+- make safety
+# Run the tests and report coverage (see https://docs.codecov.io/docs/testing-with-docker).
+- docker-compose exec -e ENVIRONMENT=testing web pytest --cov=design_storage --cov-report=term --cov-report=xml
+- bash <(curl -s https://codecov.io/bash)
 
 before_deploy:
-  - ./scripts/install_gcloud.sh
-  - ./scripts/install_kubectl.sh
-  - docker push ${IMAGE_REPO}:${TRAVIS_COMMIT::12}
-  - docker push ${IMAGE_REPO}:${TRAVIS_BRANCH}
+- ./scripts/install_gcloud.sh
+- ./scripts/install_kubectl.sh
+- if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then make push; fi
 
 deploy:
   provider: script

Dockerfile

@@ -1,39 +1,47 @@
-# Copyright 2018 Novo Nordisk Foundation Center for Biosustainability, DTU.
+# Copyright (c) 2018-2020 Novo Nordisk Foundation Center for Biosustainability,
+# Technical University of Denmark.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
+#    https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM dddecaf/postgres-base:master
+ARG BASE_TAG=alpine
 
-ENV APP_USER=giraffe
+FROM dddecaf/postgres-base:${BASE_TAG}
 
-ARG UID=1000
-ARG GID=1000
+ARG BASE_TAG=alpine
+ARG BUILD_COMMIT
+
+LABEL dk.dtu.biosustain.design-storage.alpine.vendor="Novo Nordisk Foundation \
+Center for Biosustainability, Technical University of Denmark"
+LABEL maintainer="[email protected]"
+LABEL dk.dtu.biosustain.design-storage.alpine.build.base-tag="${BASE_TAG}"
+LABEL dk.dtu.biosustain.design-storage.alpine.build.commit="${BUILD_COMMIT}"
 
 ARG CWD="/app"
 
 ENV PYTHONPATH="${CWD}/src"
 
-RUN addgroup -S -g "${GID}" "${APP_USER}" && \
-    adduser -S -H -u "${UID}" -G "${APP_USER}" "${APP_USER}"
-
 WORKDIR "${CWD}"
 
-COPY requirements ./requirements
+COPY requirements ./requirements/
+
+RUN set -eux \
+    && pip install -r requirements/requirements.txt \
+    && rm -rf /root/.cache/pip
+
+COPY . ./
 
-RUN apk add --no-cache build-base && \
-    pip-sync requirements/requirements.txt && \
-    apk del build-base
+RUN chown -R "${APP_USER}:${APP_USER}" .
 
-COPY . "${CWD}/"
+EXPOSE 8000
 
-RUN chown -R "${APP_USER}:${APP_USER}" "${CWD}"
+CMD ["gunicorn", "-c", "gunicorn.py", "design_storage.wsgi:app"]

LATEST_BASE_TAG

@@ -0,0 +1 @@
+ alpine_2020-05-26_97a4608

Makefile

@@ -1,19 +1,77 @@
-.PHONY: setup network build start qa style safety test test-travis flake8 \
-    isort isort-save license stop clean logs
-SHELL:=/bin/bash
+.PHONY: setup lock own build push start qa style safety test qc stop clean logs
 
-#################################################################################
-# COMMANDS                                                                      #
-#################################################################################
+################################################################################
+# Variables                                                                    #
+################################################################################
 
-## Run all initialization targets.
-setup: network databases
+IMAGE ?= gcr.io/dd-decaf-cfbf6/design-storage
+BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
+BUILD_COMMIT ?= $(shell git rev-parse HEAD)
+SHORT_COMMIT ?= $(shell git rev-parse --short HEAD)
+BUILD_DATE ?= $(shell date -u +%Y-%m-%d)
+BUILD_TAG ?= ${BRANCH}_${BUILD_DATE}_${SHORT_COMMIT}
+
+################################################################################
+# Commands                                                                     #
+################################################################################
 
 ## Create the docker bridge network if necessary.
 network:
 	docker network inspect DD-DeCaF >/dev/null 2>&1 || \
 		docker network create DD-DeCaF
 
+## Run all initialization targets.
+setup: network
+
+## Generate the compiled requirements files.
+lock:
+	docker pull dddecaf/tag-spy:latest
+	$(eval LATEST_BASE_TAG := $(shell docker run --rm dddecaf/tag-spy:latest tag-spy dddecaf/postgres-base alpine))
+	$(file >LATEST_BASE_TAG, $(LATEST_BASE_TAG))
+	$(eval COMPILER_TAG := $(subst alpine,alpine-compiler,$(LATEST_BASE_TAG)))
+	$(info ************************************************************)
+	$(info * Compiling service dependencies on the basis of:)
+	$(info * dddecaf/postgres-base:$(COMPILER_TAG))
+	$(info ************************************************************)
+	docker pull dddecaf/postgres-base:$(COMPILER_TAG)
+	docker run --rm --mount \
+		"source=$(CURDIR)/requirements,target=/opt/requirements,type=bind" \
+		dddecaf/postgres-base:$(COMPILER_TAG) \
+		pip-compile --allow-unsafe --verbose --generate-hashes --upgrade \
+		/opt/requirements/requirements.in
+
+## Change file ownership from root to local user.
+own:
+	sudo chown "$(shell id --user --name):$(shell id --group --name)" .
+
+## Build the Docker image for deployment.
+build-travis:
+	$(eval LATEST_BASE_TAG := $(shell cat LATEST_BASE_TAG))
+	$(info ************************************************************)
+	$(info * Building the service on the basis of:)
+	$(info * dddecaf/postgres-base:$(LATEST_BASE_TAG))
+	$(info * Today is $(shell date -u +%Y-%m-%d).)
+	$(info * Please re-run `make lock` if you want to check for and)
+	$(info * depend on a later version.)
+	$(info ************************************************************)
+	docker pull dddecaf/postgres-base:$(LATEST_BASE_TAG)
+	docker build \
+		--build-arg BASE_TAG=$(LATEST_BASE_TAG) \
+		--build-arg BUILD_COMMIT=$(BUILD_COMMIT) \
+		--tag $(IMAGE):$(BRANCH) \
+		--tag $(IMAGE):$(BUILD_TAG) \
+		.
+
+## Build local docker images.
+build:
+	$(eval LATEST_BASE_TAG := $(shell cat LATEST_BASE_TAG))
+	BASE_TAG=$(LATEST_BASE_TAG) docker-compose build
+
+## Push local Docker images to their registries.
+push:
+	docker push $(IMAGE):$(BRANCH)
+	docker push $(IMAGE):$(BUILD_TAG)
+
 ## Create databases.
 databases:
 	docker-compose up -d postgres
@@ -23,62 +81,52 @@ databases:
 	docker-compose run --rm web flask db upgrade
 	docker-compose stop
 
-## Build local docker images.
-build:
-	docker-compose build
-
-## Recompile requirements and store pinned dependencies with hashes.
-pip-compile:
-	docker run --rm -v `pwd`/requirements:/build dddecaf/postgres-base:compiler \
-		pip-compile --upgrade --generate-hashes \
-		--output-file /build/requirements.txt /build/requirements.in
-
 ## Start all services in the background.
 start:
 	docker-compose up --force-recreate -d
 
-## Run all QA targets.
-qa: style safety test
-
-## Run all style related targets.
-style: flake8 isort license
-
-## Run flake8.
-flake8:
-	docker-compose run --rm web \
-		flake8 src/design_storage tests
+## Apply all quality assurance (QA) tools.
+qa:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		isort --recursive src tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		black src tests
 
-## Check Python package import order.
 isort:
-	docker-compose run --rm web \
-		isort --check-only --recursive src/design_storage tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		isort --check-only --diff --recursive src tests
 
-## Sort imports and write changes to files.
-isort-save:
-	docker-compose run --rm web \
-		isort --recursive src/design_storage tests
+black:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		black --check --diff src tests
+
+flake8:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		flake8 src tests
 
-## Verify source code license headers.
 license:
-	./scripts/verify_license_headers.sh src/design_storage tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		./scripts/verify_license_headers.sh src tests
+
+## Run all style checks.
+style: isort black flake8 license
 
-## Check for known vulnerabilities in python dependencies.
+## Check installed dependencies for vulnerabilities.
 safety:
-	docker-compose run --rm web safety check
+	docker-compose exec -e ENVIRONMENT=testing web \
+		safety check --full-report
 
-## Run the tests.
+## Run the test suite.
 test:
-	docker-compose run --rm -e ENVIRONMENT=testing web \
-		pytest --cov=src/design_storage
-
-## Run the tests and report coverage (see https://docs.codecov.io/docs/testing-with-docker).
-shared := /tmp/coverage
-test-travis:
-	mkdir --parents "$(shared)"
-	docker-compose run --rm -e ENVIRONMENT=testing -v "$(shared):$(shared)" \
-		web pytest --cov-report "xml:$(shared)/coverage.xml" --cov-report term \
-		--cov=src/design_storage
-	bash <(curl -s https://codecov.io/bash) -f "$(shared)/coverage.xml"
+	docker-compose exec -e ENVIRONMENT=testing web \
+		pytest --cov=design_storage --cov-report=term
+
+## Run all quality control (QC) tools.
+qc: style safety test
+
+## Check the gunicorn configuration.
+gunicorn:
+	docker-compose run --rm web gunicorn --check-config -c gunicorn.py design_storage.wsgi:app
 
 ## Stop all services.
 stop:

deployment/production/deployment.yml

@@ -56,6 +56,11 @@ spec:
         - name: IAM_API
           value: "http://iam-production/iam"
         command: ["flask", "db", "upgrade"]
+        resources:
+          requests:
+            cpu: "1m"
+          limits:
+            cpu: "2000m"
       containers:
       - name: web
         image: gcr.io/dd-decaf-cfbf6/design-storage:master
@@ -99,14 +104,13 @@ spec:
               key: POSTGRES_PASS
         - name: IAM_API
           value: "http://iam-production/iam"
-        command: ["gunicorn", "-c", "gunicorn.py", "design_storage.wsgi:app"]
         readinessProbe:
           httpGet:
             path: /design-storage/healthz
             port: 8000
           initialDelaySeconds: 5
         resources:
           requests:
-            cpu: "10m"
+            cpu: "1m"
           limits:
             cpu: "2000m"

deployment/staging/deployment.yml

@@ -56,6 +56,11 @@ spec:
         - name: IAM_API
           value: "http://iam-staging/iam"
         command: ["flask", "db", "upgrade"]
+        resources:
+          requests:
+            cpu: "1m"
+          limits:
+            cpu: "2000m"
       containers:
       - name: web
         image: gcr.io/dd-decaf-cfbf6/design-storage:devel
@@ -99,14 +104,13 @@ spec:
               key: POSTGRES_PASS
         - name: IAM_API
           value: "http://iam-staging/iam"
-        command: ["gunicorn", "-c", "gunicorn.py", "design_storage.wsgi:app"]
         readinessProbe:
           httpGet:
             path: /design-storage/healthz
             port: 8000
           initialDelaySeconds: 5
         resources:
           requests:
-            cpu: "10m"
+            cpu: "1m"
           limits:
             cpu: "2000m"

docker-compose.yml

@@ -4,7 +4,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    image: gcr.io/dd-decaf-cfbf6/design-storage:${IMAGE_TAG:-latest}
+      args:
+        - BASE_TAG=${BASE_TAG:-alpine}
+        - BUILD_COMMIT=${BUILD_COMMIT:-unknown}
+    image: gcr.io/dd-decaf-cfbf6/design-storage:${BUILD_TAG:-latest}
     networks:
       default:
       DD-DeCaF:
@@ -28,12 +31,13 @@ services:
       - POSTGRES_USERNAME=${POSTGRES_USERNAME:-postgres}
       - POSTGRES_PASS=${POSTGRES_PASS}
       - IAM_API=${IAM_API:-https://api-staging.dd-decaf.eu/iam}
-    command: gunicorn -c gunicorn.py design_storage.wsgi:app
 
   postgres:
     image: postgres:9.6-alpine
     networks:
       - default
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASS}
 
 networks:
   DD-DeCaF:

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 80
+python-version = ['py36']