From c6241375a033f6f09012cc4711ffbd6065ad36d3 Mon Sep 17 00:00:00 2001
From: David Neudorfer <david@davidneudorfer.com>
Date: Tue, 24 Jan 2017 21:32:11 -0800
Subject: [PATCH] adds docker files

---
 .dockerignore                        |  2 +
 .travis.yml                          | 49 +++++++++-----
 Dockerfile.app                       | 39 +++++++++++
 Dockerfile.db                        | 19 ++++++
 docker-compose.production.yml        | 27 ++++++++
 docker-compose.staging.yml           | 27 ++++++++
 docker-compose.yml                   | 49 ++++++++++++++
 docker-entrypoint-initdb.d/setup.sql |  6 ++
 scripts/deploy.sh                    | 99 ++++++++++++++++++++++++++++
 startup-tm.sh                        |  4 ++
 wait-for-postgres.sh                 | 16 +++++
 11 files changed, 321 insertions(+), 16 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 Dockerfile.app
 create mode 100644 Dockerfile.db
 create mode 100644 docker-compose.production.yml
 create mode 100644 docker-compose.staging.yml
 create mode 100644 docker-compose.yml
 create mode 100644 docker-entrypoint-initdb.d/setup.sql
 create mode 100755 scripts/deploy.sh
 create mode 100755 startup-tm.sh
 create mode 100755 wait-for-postgres.sh

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..3978a0f7
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,2 @@
+.git
+.gitignore
diff --git a/.travis.yml b/.travis.yml
index cd21b01a..b1c16bc6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,32 +1,49 @@
+branches:
+  only:
+    - gh-pages
+    - /.*/
+
 language: python
 python:
 - 2.7
 
 cache:
   directories:
-    - $HOME/.cache/pip
-    - $HOME/build/hotosm/osm-tasking-manager2/env
+  - "$HOME/.cache/pip"
+  - "$HOME/build/hotosm/osm-tasking-manager2/env"
 
 sudo: false
 
 before_install:
-- deactivate
+  - deactivate
 
 install:
-- virtualenv env
-- env/bin/pip install --upgrade pip
-- env/bin/pip install -r requirements.txt
-- createuser www-data --no-superuser --no-createdb --no-createrole -U postgres
-- createdb -O www-data osmtm_tests -U postgres
-- psql -d osmtm_tests -c "CREATE EXTENSION postgis;" -U postgres
-- env/bin/pip install flake8
+  - virtualenv env
+  - env/bin/pip install --upgrade pip
+  - env/bin/pip install -r requirements.txt
+  - createuser www-data --no-superuser --no-createdb --no-createrole -U postgres
+  - createdb -O www-data osmtm_tests -U postgres
+  - psql -d osmtm_tests -c "CREATE EXTENSION postgis;" -U postgres
+  - env/bin/pip install flake8
 
 script:
-- echo [app:main] >> local.test.ini
-- echo sqlalchemy.url = postgresql://www-data:www-data@localhost/osmtm_tests >> local.test.ini
-- env/bin/nosetests
-- env/bin/flake8 osmtm --exclude=osmtm/static
+  - echo [app:main] >> local.test.ini
+  - echo sqlalchemy.url = postgresql://www-data:www-data@localhost/osmtm_tests >> local.test.ini
+  - env/bin/nosetests
+  - env/bin/flake8 osmtm --exclude=osmtm/static
 
 after_success:
-- env/bin/pip install coveralls
-- env/bin/coveralls
+  - env/bin/pip install coveralls
+  - env/bin/coveralls
+
+before_deploy:
+  - 'curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"'
+  - 'unzip awscli-bundle.zip'
+  - './awscli-bundle/install -b ~/bin/aws'
+  - 'export PATH=~/bin:$PATH'
+  - 'rm awscli-bundle.zip'
+  - 'rm -rf awscli-bundle/'
+
+deploy:
+  provider: script
+  script: scripts/deploy.sh
diff --git a/Dockerfile.app b/Dockerfile.app
new file mode 100644
index 00000000..32bc6dc9
--- /dev/null
+++ b/Dockerfile.app
@@ -0,0 +1,39 @@
+FROM python:2.7.12
+
+RUN apt-get update \
+    && apt-get install -y \
+    --no-install-recommends \
+        libgeos-dev \
+        software-properties-common \
+        python-software-properties \
+        wget \
+    && apt-get clean \
+    && apt-get autoremove \
+    && rm -r /var/lib/apt/lists/*
+
+RUN add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main"
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN apt-get update \
+    && apt-get install -y \
+    --no-install-recommends \
+        postgresql-client-common \
+        postgresql-client-9.6 \
+    && apt-get clean \
+    && apt-get autoremove \
+    && rm -r /var/lib/apt/lists/*
+
+ADD requirements.txt /opt/app/requirements.txt
+ADD setup.py /opt/app/setup.py
+ADD setup.cfg /opt/app/setup.cfg
+ADD README.md /opt/app/README.md
+ADD CHANGELOG.md /opt/app/CHANGELOG.md
+
+WORKDIR /opt/app
+
+RUN easy_install virtualenv
+RUN virtualenv --no-site-packages env
+RUN ./env/bin/pip install -r requirements.txt
+
+ADD . /opt/app
+
+CMD ./startup-tm.sh
diff --git a/Dockerfile.db b/Dockerfile.db
new file mode 100644
index 00000000..981c2c93
--- /dev/null
+++ b/Dockerfile.db
@@ -0,0 +1,19 @@
+FROM library/postgres
+
+RUN apt-get update \
+    && apt-get install -y \
+    --no-install-recommends \
+      postgis \
+      postgresql-9.6-postgis-2.3 \
+      postgresql-contrib-9.6 \
+      postgresql-9.6-postgis-scripts \
+      pgtop \
+    && apt-get clean \
+    && apt-get autoremove \
+    && rm -r /var/lib/apt/lists/*
+
+COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
+
+RUN dpkg-query -l postgis
+
+RUN ls -lAh /usr/share/postgresql/9.6/extension/postgis*
diff --git a/docker-compose.production.yml b/docker-compose.production.yml
new file mode 100644
index 00000000..30d2aba5
--- /dev/null
+++ b/docker-compose.production.yml
@@ -0,0 +1,27 @@
+version: '2'
+
+services:
+
+  app:
+    command: ["true"]
+
+# https://github.com/vincetse/docker-compose-zero-downtime-deployment
+  app_green:
+    container_name: app_green
+    ports:
+      - 3000:6543
+    environment:
+      - VIRTUAL_PORT=3000
+      - VIRTUAL_HOST=tasks.hotosm.org,tasks-green.hotosm.org
+    extends:
+      service: app
+
+  app_blue:
+    container_name: app_blue
+    ports:
+      - 3001:6543
+    environment:
+      - VIRTUAL_PORT=3001
+      - VIRTUAL_HOST=tasks.hotosm.org,tasks-blue.hotosm.org
+    extends:
+      service: app
diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml
new file mode 100644
index 00000000..49e491db
--- /dev/null
+++ b/docker-compose.staging.yml
@@ -0,0 +1,27 @@
+version: '2'
+
+services:
+
+  app:
+    command: ["true"]
+
+# https://github.com/vincetse/docker-compose-zero-downtime-deployment
+  app_green:
+    container_name: app_green
+    ports:
+      - 3000:6543
+    environment:
+      - VIRTUAL_PORT=3000
+      - VIRTUAL_HOST=tasks-staging.hotosm.org,tasks-staging-green.hotosm.org
+    extends:
+      service: app
+
+  app_blue:
+    container_name: app_blue
+    ports:
+      - 3001:6543
+    environment:
+      - VIRTUAL_PORT=3001
+      - VIRTUAL_HOST=tasks-staging.hotosm.org,tasks-staging-blue.hotosm.org
+    extends:
+      service: app
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..b47d3ce3
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,49 @@
+version: '2'
+
+services:
+# https://hub.docker.com/_/postgres/
+# https://store.docker.com/images/022689bf-dfd8-408f-9e1c-19acac32e57b
+
+  db:
+    build:
+      context: ./
+      dockerfile: Dockerfile.db
+    container_name: db
+    restart: always
+    volumes:
+      - postgres-db-volume:/data/postgres
+      - /tmp:/srv
+    cpu_shares: 8
+    cpuset: 0-8
+    networks:
+      - database_network
+
+  app:
+    build:
+      context: ./
+      dockerfile: Dockerfile.app
+    container_name: app
+    environment:
+      - VIRTUAL_HOST=tasks-staging.hotosm.org
+    restart: always
+    # https://docs.docker.com/compose/startup-order/
+    command: ["./wait-for-postgres.sh", "db", "./startup-tm.sh"]
+    networks:
+      - database_network
+      - code_network
+
+# cpuset, such as `docker run –cpuset-cpus`, allows users to specify
+# which CPUs to allow execution in. accepts a string that looks like
+# 0
+# 0,1
+# 0-2
+# cpu_shares
+
+volumes:
+  postgres-db-volume:
+
+networks:
+  code_network:
+    driver: bridge
+  database_network:
+    driver: bridge
diff --git a/docker-entrypoint-initdb.d/setup.sql b/docker-entrypoint-initdb.d/setup.sql
new file mode 100644
index 00000000..f92b1924
--- /dev/null
+++ b/docker-entrypoint-initdb.d/setup.sql
@@ -0,0 +1,6 @@
+CREATE USER "www-data" WITH PASSWORD 'password';
+ALTER USER "www-data" WITH NOSUPERUSER NOCREATEROLE NOCREATEDB;
+CREATE DATABASE osmtm OWNER "www-data" ENCODING 'UTF8' TEMPLATE template0;
+\connect osmtm
+CREATE EXTENSION postgis;
+SELECT postgis_full_version();
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
new file mode 100755
index 00000000..342fba89
--- /dev/null
+++ b/scripts/deploy.sh
@@ -0,0 +1,99 @@
+#!/bin/bash -eux
+
+DEPLOY_USER="mozart"
+HOST="62.210.100.219"
+SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i mozart_rsa -o ForwardAgent=yes -o IPQoS=throughput -tt"
+BASE_DIR="/srv/deploy/osm-tasking-manager2"
+
+# download mozarts key
+aws s3 cp s3://hotosm-secure/keys/mozart_rsa.enc mozart_rsa.enc
+# unencrypt mozarts key
+openssl aes-256-cbc -K $encrypted_e101044e37e0_key -iv $encrypted_e101044e37e0_iv \
+  -in mozart_rsa.enc -out mozart_rsa -d
+# fix permissions
+chmod 700 mozart_rsa
+
+# set environment variable based on branch or tag
+if [ ! -z $TRAVIS_TAG ]; then
+  echo "Deploying $TRAVIS_TAG to prod"
+  URL_PREFIX="tasks"
+  ENV="production"
+elif [ "$TRAVIS_BRANCH" == "develop" -o "$TRAVIS_PULL_REQUEST" != "false" ]; then
+  echo "Deploying develop branch to staging"
+  URL_PREFIX="tasks-staging"
+  ENV="staging"
+else
+  echo "Not a tag or develop branch. Doing nothing"
+  exit 0
+fi
+
+# rsync files to server and respective environment path
+rsync -arvz --progress . \
+ --exclude='.git/' \
+ -e "ssh -o StrictHostKeyChecking=no \
+         -o UserKnownHostsFile=/dev/null \
+         -i mozart_rsa" \
+$DEPLOY_USER@$HOST:$BASE_DIR-$ENV
+
+# find the live environment
+LIVE_COLOR=$(ssh $SSH_OPTS $DEPLOY_USER@$HOST \
+  "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \
+  -f docker-compose.yml \
+  -f docker-compose.$ENV.yml \
+  ps'" | awk '/app.*Up/ {print $1}')
+
+# set values for live color and deploy color
+if [[ $LIVE_COLOR == *"blue"* ]]; then
+  deploy="green"; live="blue"
+elif [[ $LIVE_COLOR == *"green"* ]]; then
+  deploy="blue"; live="green"
+else
+  echo "app not running"
+fi
+
+# echo build color
+echo "building app_$deploy"
+
+# build new color
+ssh $DEPLOY_USER@$HOST $SSH_OPTS \
+  "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \
+  -f docker-compose.yml \
+  -f docker-compose.$ENV.yml \
+  build app app_$deploy'"
+
+# echo deploy color
+echo "deploying app_$deploy"
+
+# deploy new color
+ssh $DEPLOY_USER@$HOST $SSH_OPTS \
+  "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \
+    -f docker-compose.yml \
+    -f docker-compose.$ENV.yml \
+    up -d \
+    --force-recreate app_$deploy'"
+
+# wait a little in case the service takes a bit to start
+sleep 5
+
+# test new server alt url
+until $(curl --output /dev/null --silent --head --fail http://$URL_PREFIX-$deploy.hotosm.org); do
+    printf '.'
+    sleep 2
+done
+
+# TODO
+# add timeout
+# add fail check to tear down new servers
+
+# stop old server
+ssh $DEPLOY_USER@$HOST $SSH_OPTS \
+  "sudo su -c 'cd $BASE_DIR-$ENV && docker-compose \
+    -f docker-compose.yml \
+    -f docker-compose.$ENV.yml \
+    stop app_$live'"
+
+# test main site
+until $(curl --output /dev/null --silent --head --fail http://$URL_PREFIX.hotosm.org); do
+    printf '.'
+    sleep 2
+done
diff --git a/startup-tm.sh b/startup-tm.sh
new file mode 100755
index 00000000..a69aff5d
--- /dev/null
+++ b/startup-tm.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# ./env/bin/initialize_osmtm_db
+./env/bin/pserve --reload development.ini
diff --git a/wait-for-postgres.sh b/wait-for-postgres.sh
new file mode 100755
index 00000000..5a817d49
--- /dev/null
+++ b/wait-for-postgres.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# wait-for-postgres.sh
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+until psql -h "$host" -U "postgres" -c '\l'; do
+  >&2 echo "Postgres is unavailable - sleeping"
+  sleep 1
+done
+
+>&2 echo "Postgres is up - executing command"
+exec $cmd