Merge pull request #145 from fako/docker

Docker

.dockerignore

@@ -0,0 +1,4 @@
+data/
+postgres/
+redis/
+venv/

.env

@@ -0,0 +1,5 @@
+POSTGRES_PASSWORD=33SebQd4Ugkqx4KjqanZDsWRXTR54M6k
+DJANGO_POSTGRES_PASSWORD=ZV35A5x89pbHuiYSrvqroPHKDnng7dRF
+DS_DATA_DIR=./data
+DJANGO_MODE=development
+INVOKE_DJANGO_DEBUG=1

.gitignore

@@ -3,18 +3,14 @@ datascope.iml
 .DS_store
 
 *.pyc
-*.db
-*.csv
-db-dump.*.json
-*.sql
 *.pkl
+celerybeat-schedule.db
 
-datagrowth/docs/_build/
+venv/
 
-datascope/settings.py
-datascope/bootstrap.py
-datascope/secrets.py
-datascope/logs/
-datascope/statics/
+src/datascope/logs/
+src/datascope/statics/
 
-data/
+/data
+/models
+secrets.bk.py

.travis.yml

@@ -1,15 +1,15 @@
 language: python
 python:
-  - "3.5"
   - "3.6"
 # command to install dependencies
 before_install:
   - sudo apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran
   - sudo sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf
   - sudo service postgresql restart
   - sleep 1
-install: "pip install -r datascope/requirements/production.txt"
+install: "pip install -r src/datascope/requirements/production.txt"
 # command to run tests
+before_script: cd src && python manage.py collectstatic --noinput
 script: python manage.py test --settings=datascope.settings_test
 services:
   - postgresql
@@ -22,3 +22,6 @@ addons:
 env:
   global:
   - PGPORT=5432
+  - DJANGO_CONTEXT=host
+  - INVOKE_POSTGRES_HOST=127.0.0.1
+  - INVOKE_DJANGO_DATABASE_USER=postgres

Dockerfile

@@ -0,0 +1,51 @@
+FROM python:3.6
+
+RUN apt-get update && \
+    apt-get install -y vim binutils libproj-dev gdal-bin gettext default-jdk
+
+# Setup directories, 3rd party models and users
+RUN mkdir -p /usr/etc/datascope
+COPY deploy/environments /usr/etc/datascope
+RUN mkdir -p /usr/etc/models
+COPY models /usr/etc/models
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+RUN useradd -ms /bin/bash app
+
+# Install Python dependencies and copy app
+RUN pip install --upgrade pip
+COPY src/datascope/requirements /usr/src/app/datascope/requirements
+RUN pip install --no-cache-dir -r datascope/requirements/production.txt
+COPY --chown=app:app src /usr/src/app
+
+# We setup spaCy models outside of the pip flow to prevent repetious downloads
+RUN python -m spacy link /usr/etc/models/spacy/en_core_web_lg-2.0.0/en_core_web_lg en_core_web_lg
+RUN python -m spacy link /usr/etc/models/spacy/nl_core_news_sm-2.0.0/nl_core_news_sm nl_core_news_sm
+
+# We're serving static files through Whitenoise
+# See: http://whitenoise.evans.io/en/stable/index.html#
+# If you doubt this decision then read the "infrequently asked question" section for details
+# Here we gather static files that get served through uWSGI
+# NB: runs with production settings
+RUN python manage.py collectstatic --noinput
+
+# We're switching user to a non-priviliged user
+# The Python packages directory and datagrowth package needs to belong to that user
+# for dynamic packaging (see entrypoint)
+RUN chown app:app /usr/local/lib/python3.6/site-packages
+RUN chown -R app:app /usr/local/lib/python3.6/site-packages/datagrowth*
+USER app:app
+
+# Compiling translations
+# NB: runs with production settings
+# NB: not enabled
+# RUN python manage.py compilemessages
+
+# Entrypoint handles some edge cases before running
+ENTRYPOINT ["/usr/src/app/entrypoint.sh"]
+
+# The default command is to start a uWSGI server
+CMD ["uwsgi", "--ini", "/usr/src/app/uwsgi.ini"]
+
+# EXPOSE port 8000 to allow communication to/from server
+EXPOSE 8000

Makefile

@@ -1,19 +1,7 @@
-now = $(shell date +"%Y-%m-%d")
-
 clean:
 	find . -type f -name "*.pyc" -delete;
 	find . -type d -name "__pycache__" -delete;
 
-deploy: clean
-	sudo service uwsgi restart
-	sudo service celeryd restart
-
-backup-db:
-	pg_dump -h localhost -U postgres datascope > data/datascope.postgres.sql
-
-import-db:
-	cat $(backup) | psql -h localhost -U postgres datascope
-
 backup-data:
 	# Syncing local data to a harddrive
 	# -z means use compression
@@ -24,10 +12,10 @@ backup-data:
 	rsync -zrthv --progress data /Volumes/Leo65/data/datascope
 
 start-celery:
-	celery -A datascope worker --loglevel=info -B
+	cd src && celery -A datascope worker --loglevel=info -B
 
 start-postgres:
 	psql -h localhost -U postgres -d postgres
 
 test:
-	./manage.py test --settings=datascope.settings_test $(filter)
+	cd src && ./manage.py test --settings=datascope.settings_test $(filter)

activate.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source venv/bin/activate
+export DJANGO_CONTEXT=host
+export $(cat .env | xargs)

build.py

@@ -0,0 +1,34 @@
+from invoke import task
+from src.datascope.version import get_project_version
+
+
+REPOSITORY = "eu.gcr.io/datascope-266618"
+
+
+@task()
+def container(ctx, version=None):
+
+    if not version:
+        version = get_project_version("src/package.json")
+
+    print(f"Building: {version}")
+    tag = f"{REPOSITORY}/datascope:{version}"
+    rsl = ctx.run(
+        f"docker build . -t {tag}",
+        echo=True
+    )
+
+    if rsl.failed:
+        print(rsl.stderr)
+        raise RuntimeError("Failed to build docker image")
+
+    print("Pushing")
+    rsl = ctx.run(
+        "docker push {}".format(tag),
+        echo=True,
+        pty=True
+    )
+
+    if rsl.failed:
+        print(rsl.stderr)
+        raise RuntimeError("Failed to push image tagged {}".format(tag))