diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 95d6841af..9e659135a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.6 +current_version = 2.1.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2024-02-15T16:29:01Z +search = {current_version} 2024-02-23T20:32:41Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2024-02-15T16:29:01Z +values = 2024-02-23T20:32:41Z [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/.gitignore b/.gitignore index 84d36d61b..128ede050 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ venv/ ## Testing .pytest_cache/ +*.log diff --git a/CHANGES.md b/CHANGES.md index 90a4071ea..ceca39fe7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,13 +15,49 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -## Fixes +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[2.1.0](https://github.com/bird-house/birdhouse-deploy/tree/2.1.0) (2024-02-23) +------------------------------------------------------------------------------------------------------------------ +## Changes +- Compose script utilities: + * Add `BIRDHOUSE_COLOR` option and various logging/messaging definitions in `birdhouse/scripts/logging.include.sh`. + * Replace all explicit color "logging" related `echo` in scripts by a utility `log {LEVEL} {message}` function + that employs variables `LOG_DEBUG`, `LOG_INFO`, `LOG_WARN`, `LOG_ERROR` and `LOG_CRITICAL` as applicable per + respective messages to report logging messages in a standard approach. + Colors can be disabled with `BIRDHOUSE_COLOR=0` and logging level can be set with `BIRDHOUSE_LOG_LEVEL={LEVEL}` + where all levels above or equal to the configured one will be displayed (default logging level is `INFO`). + * Unify all `birdhouse/scripts` utilities to employ the same `COMPOSE_DIR` variable (auto-resolved or explicitly set) + in order to include or source any relevant dependencies they might have within the `birdhouse-deploy` repository. + * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call. + This can be used to perform a "dry-run" of the command and validate that was is loaded is as expected, by inspecting + provided log messages. + * Replace older backtick (``` ` ```) executions by `$(...)` representation except for `eval` calls that require + them for backward compatibility of `sh` on some server instances. + * Modify the `sh -x` calls to scripts listed in `COMPONENT_PRE_COMPOSE_UP` and `COMPONENT_POST_COMPOSE_UP` to employ + the `-x` flag (showing commands) only when `BIRDHOUSE_LOG_LEVEL=DEBUG`. + +- Defaults: + * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS. + These variables use a special combination of `DELAYED_EVAL` and `OPTIONAL_VARS` definitions that can make use + of a variable formatted as `='${__DEFAULT__}'` that will print a warning messages indicating + that the default is employed, although *STRONGLY* recommended to be overridden. This allows a middle ground between + backward-compatible `env.local` while flagging potentially misused configurations. + +## Fixes +- Canarie-API: updated references + * Use the new `SERVER_[...]` variables. + * Replace the LICENSE URL of the server node pointing + at [Ouranosinc/pavics-sdi](https://github.com/Ouranosinc/pavics-sdi) instead + of intended [bird-house/birdhouse-deploy](https://github.com/bird-house/birdhouse-deploy). +- Magpie: ensure that the `MAGPIE_ADMIN_USERNAME` variable is respected + * When determining the `JUPYTERHUB_ADMIN_USERS` variable + * Double check that it is being respected everywhere else - env.local.example: fix `JUPYTERHUB_CONFIG_OVERRIDE` comment section `JUPYTERHUB_CONFIG_OVERRIDE` was disconnected from its sample code. - [2.0.6](https://github.com/bird-house/birdhouse-deploy/tree/2.0.6) (2024-02-15) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index c21836bfa..332c9177b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 2.0.6 +override APP_VERSION := 2.1.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 82436e237..ab9e92bd0 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.0.6.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.1.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/2.0.6...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/2.1.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-2.0.6-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-2.1.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/2.0.6 + :target: https://github.com/bird-house/birdhouse-deploy/tree/2.1.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index 652e5ffbe..b77a8711e 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -2.0.6 2024-02-15T16:29:01Z +2.1.0 2024-02-23T20:32:41Z diff --git a/birdhouse/README.rst b/birdhouse/README.rst index 3d9f8557a..d9f53f5fe 100644 --- a/birdhouse/README.rst +++ b/birdhouse/README.rst @@ -148,7 +148,7 @@ instructions below. Manual instructions: * Go to - ``https:///magpie/ui/login`` and login with the ``admin`` user. The password should be in ``env.local``. + ``https:///magpie/ui/login`` and login with the ``MAGPIE_ADMIN_USERNAME`` user. The password should be in ``env.local``. * Then go to ``https:///magpie/ui/users/add``. diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index 80f25bf73..a56e460ac 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -304,7 +304,7 @@ birdhouse-deploy software stack and the machine that it is running on. It is hig make these routes available to anyone who does not have proper access permissions. Add existing users to the ``monitoring`` group to allow them access to the various monitoring WebUI. -This way, we do not need to share the ``admin`` user account and do not have to add them to the +This way, we do not need to share the ``MAGPIE_ADMIN_USERNAME`` user account and do not have to add them to the ``administrators`` group, which would give them too much permissions. diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index 2a3aed472..642d68127 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -109,25 +109,25 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.0.6', - 'releaseTime': '2024-02-15T16:29:01Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', + 'version': '2.1.0', + 'releaseTime': '2024-02-23T20:32:41Z', + 'institution': '${SERVER_INSTITUTION}', + 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', 'category': 'Resource/Cloud Management', - 'tags': ['Climatology'] + 'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()], }, 'stats': { 'method': '.*', 'route': '(?!)' # this will be set by CANARIE_STATS_ROUTES (see below) }, 'redirect': { - 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', - 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md', - 'support': 'https://github.com/bird-house/birdhouse-deploy/issues', + 'doc': '${SERVER_DOCUMENTATION_URL}', + 'releasenotes': '${SERVER_RELEASE_NOTES_URL}', + 'support': '${SERVER_SUPPORT_URL}', 'source': 'https://github.com/bird-house/birdhouse-deploy', 'tryme': 'https://${PAVICS_FQDN_PUBLIC}', - 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', + 'licence': '${SERVER_LICENSE_URL}', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' }, 'monitoring': {} # filled in after processing everything, see end of script @@ -142,12 +142,12 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.0.6', - 'releaseTime': '2024-02-15T16:29:01Z', - 'institution': 'Ouranos', - 'researchSubject': 'Climatology', + 'version': '2.1.0', + 'releaseTime': '2024-02-23T20:32:41Z', + 'institution': '${SERVER_INSTITUTION}', + 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', - 'tags': ['Climatology', 'Cloud'] + 'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()], }, 'stats': { 'method': '.*', diff --git a/birdhouse/components/geoserver/pre-docker-compose-up b/birdhouse/components/geoserver/pre-docker-compose-up index 78cc13880..cd482ea2f 100755 --- a/birdhouse/components/geoserver/pre-docker-compose-up +++ b/birdhouse/components/geoserver/pre-docker-compose-up @@ -1,10 +1,17 @@ #!/bin/sh -THIS_FILE="`realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" -COMPOSE_DIR="$THIS_DIR/../.." +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" +COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}/..")}" + +if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/read-configs.include.sh" + + # resolve GEOSERVER_DATA_DIR + read_configs +fi if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then - echo "fix GeoServer data dir permission on first run only, when data dir do not exist yet." - FIRST_RUN_ONLY=1 "$COMPOSE_DIR"/deployment/fix-geoserver-data-dir-perm + log INFO "fix GeoServer data dir permission on first run only, when data dir do not exist yet." + FIRST_RUN_ONLY=1 "${COMPOSE_DIR}/deployment/fix-geoserver-data-dir-perm" fi diff --git a/birdhouse/components/jupyterhub/default.env b/birdhouse/components/jupyterhub/default.env index c7013f931..2a77689ad 100644 --- a/birdhouse/components/jupyterhub/default.env +++ b/birdhouse/components/jupyterhub/default.env @@ -68,9 +68,13 @@ export JUPYTERHUB_CRYPT_KEY= # JUPYTERHUB_CRYPT_KEY is set. export JUPYTERHUB_AUTHENTICATOR_REFRESH_AGE=60 +# Usernames that should be given admin access in jupyterhub +export JUPYTERHUB_ADMIN_USERS='{\"${MAGPIE_ADMIN_USERNAME}\"}' # python set syntax + export DELAYED_EVAL=" $DELAYED_EVAL JUPYTERHUB_USER_DATA_DIR + JUPYTERHUB_ADMIN_USERS " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here diff --git a/birdhouse/components/proxy/pre-docker-compose-up b/birdhouse/components/proxy/pre-docker-compose-up index 0d5acbe8c..698ecb289 100755 --- a/birdhouse/components/proxy/pre-docker-compose-up +++ b/birdhouse/components/proxy/pre-docker-compose-up @@ -3,12 +3,11 @@ # Create JSON files containing the version information, available services, and # enabled components. These files will be served by the nginx proxy as static files. -THIS_FILE="$(realpath "$0")" -THIS_DIR="$(dirname "$THIS_FILE")" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" mkdir -p "${THIS_DIR}/static" echo "${BIRDHOUSE_VERSION_JSON}" > "${THIS_DIR}/static/version.json" echo "${BIRDHOUSE_DEPLOY_SERVICES_JSON}" > "${THIS_DIR}/static/services.json" echo "${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}" > "${THIS_DIR}/static/components.json" - diff --git a/birdhouse/default.env b/birdhouse/default.env index 1db0e7f32..bec5cec1c 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -6,6 +6,13 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. +# Any default value that should be marked for security concern or recommended modificiation should +# use the '${__DEFAULT__{var}}' naming format. These can then be referenced in 'env.local.example' to +# avoid literal value duplication, and ensure they remain in sync. Also, those '${__DEFAULT__{var}}' +# definitions should *NOT* be exported to avoid unnecessary polution of the environment variables. +# Variables with format '${__DEFAULT__{var}}' will be flagged accordingly to their required/optional status +# (see also: 'check_default_vars' in 'birdhouse/read-configs.include.sh'). + export BASH_IMAGE="bash:5.1.4" # Root directory under which all data persistence should be nested under @@ -29,19 +36,86 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}' export DELAYED_EVAL=" $DELAYED_EVAL PAVICS_FQDN_PUBLIC + DOC_URL + SUPPORT_EMAIL + SSL_CERTIFICATE DATA_PERSIST_SHARED_ROOT WPS_OUTPUTS_DIR + SERVER_NAME + SERVER_DESCRIPTION + SERVER_INSTITUTION + SERVER_SUBJECT + SERVER_TAGS + SERVER_DOCUMENTATION_URL + SERVER_RELEASE_NOTES_URL + SERVER_SUPPORT_URL + SERVER_LICENSE_URL " - -export SERVER_NAME=PAVICS -export SERVER_DESCRIPTION=" +# Server Identification Details +# Following definitions should definitenly be updated. +# Previous defaults are defined for backward-compatibility. +# If not overridden explicitly by their non '__' prefixed variant, +# a WARN message will be displayed by pavics-compose. +__DEFAULT__SERVER_NAME="PAVICS" +__DEFAULT__SERVER_DESCRIPTION=" The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of climate analysis services served through Open Geospatial Consortium (OGC) protocols. These services include data access, processing and visualization. Both data and algorithms can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, or a custom web interface. " +__DEFAULT__SERVER_INSTITUTION="Ouranos" +__DEFAULT__SERVER_SUBJECT="Climatology" +# below can be a CSV list of tags +__DEFAULT__SERVER_TAGS="Climatology" +__DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html" +__DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md" +__DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues" +# NOTE: +# This value does not use the previously hard coded default. +# Previous default pointed at the wrong repository with a mismatching LICENSE file. +__DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE" +__DEFAULT__SUPPORT_EMAIL="helpdesk@example.com" +__DEFAULT__DOC_URL="https://www.example.com/" +__DEFAULT__PAVICS_FQDN="hostname.domainname" +__DEFAULT__SSL_CERTIFICATE="/path/to/ssl/cert.pem" + +# apply overrides or fallback above defaults with delayed evaluation +# exceptions for 'SUPPORT_EMAIL' and 'DOC_URL' using the old name for backward compatibility. +export SUPPORT_EMAIL='${__DEFAULT__SUPPORT_EMAIL}' +export DOC_URL='${__DEFAULT__DOC_URL}' +export SSL_CERTIFICATE='${__DEFAULT__SSL_CERTIFICATE}' +export SERVER_NAME='${__DEFAULT__SERVER_NAME}' +export SERVER_DESCRIPTION='${__DEFAULT__SERVER_DESCRIPTION}' +export SERVER_INSTITUTION='${__DEFAULT__SERVER_INSTITUTION}' +export SERVER_SUBJECT='${__DEFAULT__SERVER_SUBJECT}' +export SERVER_TAGS='${__DEFAULT__SERVER_TAGS}' +export SERVER_DOCUMENTATION_URL='${__DEFAULT__SERVER_DOCUMENTATION_URL}' +export SERVER_RELEASE_NOTES_URL='${__DEFAULT__SERVER_RELEASE_NOTES_URL}' +export SERVER_SUPPORT_URL='${__DEFAULT__SERVER_SUPPORT_URL}' +export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}' + +# Defaults for required variables recommended for override for security reasons. +# Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance. +# These values would be detected only if the instance was configured using a copy of 'env.local.example'. +__DEFAULT__MAGPIE_SECRET="itzaseekrit" +__DEFAULT__MAGPIE_ADMIN_USERNAME="admin" +__DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!" +__DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics" +__DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty" +__DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie" +__DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty" +__DEFAULT__GEOSERVER_ADMIN_USER="admingeo" +__DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass" +############################################################################# +# Deprecated vars (for components in the ./deprecated-components directory) +############################################################################# +__DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass" +__DEFAULT__CATALOG_USERNAME="admin-catalog" +__DEFAULT__CATALOG_PASSWORD="qwerty" +__DEFAULT__PHOENIX_PASSWORD="phoenix_pass" +__DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234" export DEFAULT_CONF_DIRS=' ./components/proxy diff --git a/birdhouse/deployment/certbotwrapper b/birdhouse/deployment/certbotwrapper index 4cc3769dc..34828d67c 100755 --- a/birdhouse/deployment/certbotwrapper +++ b/birdhouse/deployment/certbotwrapper @@ -46,8 +46,8 @@ certbotwrapper START_TIME=$START_TIME" set -x -THIS_FILE="`realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" SAVED_PWD="`pwd`" . "$THIS_DIR/../read-configs.include.sh" diff --git a/birdhouse/deployment/deploy.sh b/birdhouse/deployment/deploy.sh index 7159ae5e3..8df019bf3 100755 --- a/birdhouse/deployment/deploy.sh +++ b/birdhouse/deployment/deploy.sh @@ -53,9 +53,9 @@ # are re-read. docker-compose is not aware of any changes outside of the # docker-compose.yml file. -if [ ! -z "$AUTODEPLOY_SILENT" ]; then +if [ ! -z "${AUTODEPLOY_SILENT}" ]; then LOG_FILE="/var/log/PAVICS/autodeploy.log" - exec >>$LOG_FILE 2>&1 + exec >> "${LOG_FILE}" 2>&1 fi usage() { @@ -65,7 +65,7 @@ usage() { COMPOSE_DIR="$1" ENV_LOCAL_FILE="$2" -if [ -z "$COMPOSE_DIR" ]; then +if [ -z "${COMPOSE_DIR}" ]; then echo "ERROR: please provide path to PAVICS docker-compose dir." 1>&2 usage exit 2 @@ -73,23 +73,22 @@ else shift fi -if [ -z "$ENV_LOCAL_FILE" ]; then - ENV_LOCAL_FILE="$COMPOSE_DIR/env.local" +if [ -z "${ENV_LOCAL_FILE}" ]; then + ENV_LOCAL_FILE="${COMPOSE_DIR}/env.local" else shift fi -COMPOSE_DIR="`realpath "$COMPOSE_DIR"`" -REPO_ROOT="`realpath "$COMPOSE_DIR/.."`" +COMPOSE_DIR="$(realpath "${COMPOSE_DIR}")" -if [ ! -f "$COMPOSE_DIR/docker-compose.yml" -o \ - ! -f "$COMPOSE_DIR/pavics-compose.sh" ]; then - echo "ERROR: missing docker-compose.yml or pavics-compose.sh file in '$COMPOSE_DIR'" 1>&2 +if [ ! -f "${COMPOSE_DIR}/docker-compose.yml" ] || \ + [ ! -f "${COMPOSE_DIR}/pavics-compose.sh" ]; then + echo "ERROR: missing docker-compose.yml or pavics-compose.sh file in '${COMPOSE_DIR}'" 1>&2 exit 2 fi -if [ ! -f "$ENV_LOCAL_FILE" ]; then - echo "ERROR: env.local '$ENV_LOCAL_FILE' not found, please instantiate from '$COMPOSE_DIR/env.local.example'" 1>&2 +if [ ! -f "${ENV_LOCAL_FILE}" ]; then + echo "ERROR: env.local '${ENV_LOCAL_FILE}' not found, please instantiate from '${COMPOSE_DIR}/env.local.example'" 1>&2 exit 2 fi @@ -100,56 +99,56 @@ fi # Setup COMPOSE_DIR and PWD for sourcing env.local. # Prevent un-expected difference when this script is run inside autodeploy # container and manually from the host. -cd $COMPOSE_DIR +cd "${COMPOSE_DIR}" || exit -START_TIME="`date -Isecond`" -echo "deploy START_TIME=$START_TIME" +START_TIME="$(date -Isecond)" +echo "deploy START_TIME=${START_TIME}" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/read-configs.include.sh" # Read AUTODEPLOY_EXTRA_REPOS read_basic_configs_only set -x -for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do - if [ -d "$adir" ]; then - cd $adir +for adir in "${COMPOSE_DIR}" ${AUTODEPLOY_EXTRA_REPOS}; do + if [ -d "${adir}" ]; then + cd "${adir}" || exit # fail fast if unclean checkout - if [ ! -z "`git status -u --porcelain`" ]; then - echo "ERROR: unclean repo '$adir'" 1>&2 + if [ ! -z "$(git status -u --porcelain)" ]; then + echo "ERROR: unclean repo '${adir}'" 1>&2 exit 1 fi else - echo "WARNING: extra repo '$adir' do not exist" + echo "WARNING: extra repo '${adir}' do not exist" fi done -cd $COMPOSE_DIR +cd "${COMPOSE_DIR}" || exit read_basic_configs_only # stop all to force reload any changed config that are volume-mount into the containers ./pavics-compose.sh stop -for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do - if [ -d "$adir" ]; then - cd $adir +for adir in "${COMPOSE_DIR}" ${AUTODEPLOY_EXTRA_REPOS}; do + if [ -d "${adir}" ]; then + cd "${adir}" || exit - EXTRA_REPO="`git rev-parse --show-toplevel`" - DEPLOY_KEY="$AUTODEPLOY_DEPLOY_KEY_ROOT_DIR/`basename "$EXTRA_REPO"`_deploy_key" - DEFAULT_DEPLOY_KEY="$AUTODEPLOY_DEPLOY_KEY_ROOT_DIR/id_rsa_git_ssh_read_only" - if [ ! -e "$DEPLOY_KEY" -a -e "$DEFAULT_DEPLOY_KEY" ]; then - DEPLOY_KEY="$DEFAULT_DEPLOY_KEY" + EXTRA_REPO="$(git rev-parse --show-toplevel)" + DEPLOY_KEY="${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}/$(basename "${EXTRA_REPO}")_deploy_key" + DEFAULT_DEPLOY_KEY="${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}/id_rsa_git_ssh_read_only" + if [ ! -e "${DEPLOY_KEY}" ] && [ -e "${DEFAULT_DEPLOY_KEY}" ]; then + DEPLOY_KEY="${DEFAULT_DEPLOY_KEY}" fi export GIT_SSH_COMMAND="" # git ver 2.3+ - if [ -e "$DEPLOY_KEY" ]; then + if [ -e "${DEPLOY_KEY}" ]; then # override git ssh command for private repos only # # https://git-scm.com/docs/git-config#Documentation/git-config.txt-sshvariant - export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=$DEPLOY_KEY" + export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=${DEPLOY_KEY}" else unset GIT_SSH_COMMAND fi @@ -160,18 +159,18 @@ for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do # This runs as the root user so new/updated files will be owned by root after the git pull, this sets the # owner of the code to CODE_OWNERSHIP if set. CODE_OWNERSHIP should contain uids instead of usernames since # usernames within a docker container will not necessarily line up with those on the host system. - if [ -n "$CODE_OWNERSHIP" ]; then - chown -R "$CODE_OWNERSHIP" "$(git rev-parse --show-toplevel)" + if [ -n "${CODE_OWNERSHIP}" ]; then + chown -R "${CODE_OWNERSHIP}" "$(git rev-parse --show-toplevel)" fi else - echo "WARNING: extra repo '$adir' do not exist" + echo "WARNING: extra repo '${adir}' do not exist" fi done -cd $COMPOSE_DIR +cd "${COMPOSE_DIR}" || exit # reload again after git pull because this file could be changed by the pull -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/read-configs.include.sh" # reload again after default.env since env.local can override default.env # (ex: JUPYTERHUB_USER_DATA_DIR) @@ -183,8 +182,8 @@ read_basic_configs_only set +x echo " -deploy finished START_TIME=$START_TIME -deploy finished END_TIME=`date -Isecond`" +deploy finished START_TIME=${START_TIME} +deploy finished END_TIME=$(date -Isecond)" # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/birdhouse/deployment/fix-geoserver-data-dir-perm b/birdhouse/deployment/fix-geoserver-data-dir-perm index 644922674..f87dfa2d4 100755 --- a/birdhouse/deployment/fix-geoserver-data-dir-perm +++ b/birdhouse/deployment/fix-geoserver-data-dir-perm @@ -8,13 +8,11 @@ # global.xml will exist and this script will not execute. Without # FIRST_RUN_ONLY, this script will always execute. -THIS_FILE="`realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" +COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}")}" -# Go to repo root. -cd $THIS_DIR/../.. - -. birdhouse/read-configs.include.sh +. "${COMPOSE_DIR}/read-configs.include.sh" # Get BASH_IMAGE # Get GEOSERVER_DATA_DIR @@ -22,15 +20,15 @@ read_configs set -x -DATA_DIR="$GEOSERVER_DATA_DIR" +DATA_DIR="${GEOSERVER_DATA_DIR}" if [ -n "$1" ]; then DATA_DIR="$1"; shift fi docker run --rm --name fix-geoserver-data-dir-perm \ - --volume ${DATA_DIR}:/datadir \ + --volume "${DATA_DIR}":/datadir \ --env FIRST_RUN_ONLY \ - $BASH_IMAGE \ + "${BASH_IMAGE}" \ bash -xc 'if [ -z "$FIRST_RUN_ONLY" -o ! -f /datadir/global.xml ]; \ then chown -R 1000:10001 /datadir; else echo "No execute."; fi' diff --git a/birdhouse/deployment/fix-write-perm b/birdhouse/deployment/fix-write-perm index 49d2cd73d..00d73df55 100755 --- a/birdhouse/deployment/fix-write-perm +++ b/birdhouse/deployment/fix-write-perm @@ -37,26 +37,27 @@ # So the setfacl solution is the simplest, most portable/generic and most # localized (only the directories we need) solution. -THIS_FILE="`realpath "$0"`" -THIS_DIR="`dirname "$THIS_FILE"`" +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" +COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}")}" -# Go to repo root. -cd $THIS_DIR/../.. - -. birdhouse/read-configs.include.sh +. "${COMPOSE_DIR}/read-configs.include.sh" # Get GEOSERVER_DATA_DIR, JUPYTERHUB_USER_DATA_DIR, MAGPIE_PERSIST_DIR read_configs -DEFAULT_EXTRA_DATA_DIR="$GEOSERVER_DATA_DIR/ $JUPYTERHUB_USER_DATA_DIR/ \ - $MAGPIE_PERSIST_DIR/" +DEFAULT_EXTRA_DATA_DIR=" + ${GEOSERVER_DATA_DIR}/ + ${JUPYTERHUB_USER_DATA_DIR}/ \ + ${MAGPIE_PERSIST_DIR}/ \ +" EXTRA_DATA_DIR="" if [ -n "$FIX_WRITE_PERM_EXTRA" ]; then - EXTRA_DATA_DIR="$DEFAULT_EXTRA_DATA_DIR" + EXTRA_DATA_DIR="${DEFAULT_EXTRA_DATA_DIR}" fi set -x -sudo setfacl -Rdm u:$USER:rwX $PWD $AUTODEPLOY_EXTRA_REPOS $EXTRA_DATA_DIR "$@" # for future files -sudo setfacl -Rm u:$USER:rwX $PWD $AUTODEPLOY_EXTRA_REPOS $EXTRA_DATA_DIR "$@" # for existing files +sudo setfacl -Rdm "u:${USER}:rwX" "${PWD}" ${AUTODEPLOY_EXTRA_REPOS} ${EXTRA_DATA_DIR} "$@" # for future files +sudo setfacl -Rm "u:${USER}:rwX" "${PWD}" ${AUTODEPLOY_EXTRA_REPOS} ${EXTRA_DATA_DIR} "$@" # for existing files diff --git a/birdhouse/deployment/install-deploy-notebook b/birdhouse/deployment/install-deploy-notebook index baf148e57..2a64b33bb 100755 --- a/birdhouse/deployment/install-deploy-notebook +++ b/birdhouse/deployment/install-deploy-notebook @@ -21,21 +21,21 @@ if [ -z "$1" ]; then fi -REPO_ROOT="`realpath "$1"`"; shift # path to PAVICS checkout +REPO_ROOT="$(realpath "$1")"; shift # path to PAVICS checkout -if [ ! -e "$REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook" ]; then - echo "ERROR: bad/wrong pavics-checkout '$REPO_ROOT' " 1>&2 +if [ ! -e "${REPO_ROOT}/birdhouse/deployment/trigger-deploy-notebook" ]; then + echo "ERROR: bad/wrong pavics-checkout '${REPO_ROOT}' " 1>&2 usage exit 2 fi -. "$REPO_ROOT/birdhouse/read-configs.include.sh" +. "${REPO_ROOT}/birdhouse/read-configs.include.sh" # Get JUPYTERHUB_USER_DATA_DIR read_configs set -x -cat $REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook | envsubst '${JUPYTERHUB_USER_DATA_DIR}' | sudo tee $CRON_FILE -sudo chown root:root $CRON_FILE -sudo chmod 755 $CRON_FILE +cat "${REPO_ROOT}/birdhouse/deployment/trigger-deploy-notebook" | envsubst '${JUPYTERHUB_USER_DATA_DIR}' | sudo tee "${CRON_FILE}" +sudo chown root:root "${CRON_FILE}" +sudo chmod 755 "${CRON_FILE}" diff --git a/birdhouse/deployment/trigger-deploy-notebook b/birdhouse/deployment/trigger-deploy-notebook index bb052d3c0..c0da0aa70 100755 --- a/birdhouse/deployment/trigger-deploy-notebook +++ b/birdhouse/deployment/trigger-deploy-notebook @@ -12,6 +12,10 @@ # # Logs to /var/log/PAVICS/notebookdeploy.log, re-use existing logrotate. +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" +COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}")}" + LOG_FILE="/var/log/PAVICS/notebookdeploy.log" exec >>$LOG_FILE 2>&1 @@ -20,24 +24,21 @@ cleanup_on_exit() { set +x echo " notebookdeploy finished START_TIME=$START_TIME -notebookdeploy finished END_TIME=`date -Isecond`" +notebookdeploy finished END_TIME=$(date -Isecond)" } trap cleanup_on_exit EXIT -START_TIME="`date -Isecond`" +START_TIME="$(date -Isecond)" echo "========== notebookdeploy START_TIME=$START_TIME" # running script manually (not with cron) source env.local file. -if [ -z "$COMPOSE_DIR" ]; then - COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")" -fi - -if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then - . "$COMPOSE_DIR/read-configs.include.sh" +if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then + . "${COMPOSE_DIR}/read-configs.include.sh" # Get JUPYTERHUB_USER_DATA_DIR + # Get BASH_IMAGE read_configs fi @@ -45,31 +46,31 @@ set -x NOTEBOOK_DIR_MNT="/notebook_dir" TUTORIAL_NOTEBOOKS_DIR="tutorial-notebooks" -if [ -z "$WORKFLOW_TESTS_BRANCH" ]; then +if [ -z "${WORKFLOW_TESTS_BRANCH}" ]; then # when testing, override this WORKFLOW_TESTS_BRANCH="master" fi -if [ -z "$TMP_BASE_DIR" ]; then +if [ -z "${TMP_BASE_DIR}" ]; then TMP_BASE_DIR="/tmp" fi -TMPDIR="`mktemp -d -t notebookdeploy.XXXXXXXXXXXX -p $TMP_BASE_DIR`" +TMPDIR="$(mktemp -d -t notebookdeploy.XXXXXXXXXXXX -p "${TMP_BASE_DIR}")" -cd $TMPDIR -mkdir $TUTORIAL_NOTEBOOKS_DIR -cd $TUTORIAL_NOTEBOOKS_DIR +cd "${TMPDIR}" || exit +mkdir "${TUTORIAL_NOTEBOOKS_DIR}" +cd "${TUTORIAL_NOTEBOOKS_DIR}" || exit -wget --quiet https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/$WORKFLOW_TESTS_BRANCH/downloadrepos +wget --quiet "https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/${WORKFLOW_TESTS_BRANCH}/downloadrepos" chmod a+x downloadrepos -wget --quiet https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/$WORKFLOW_TESTS_BRANCH/default_build_params +wget --quiet "https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/${WORKFLOW_TESTS_BRANCH}/default_build_params" -wget --quiet https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/$WORKFLOW_TESTS_BRANCH/binder/reorg-notebooks +wget --quiet "https://raw.githubusercontent.com/Ouranosinc/PAVICS-e2e-workflow-tests/${WORKFLOW_TESTS_BRANCH}/binder/reorg-notebooks" chmod a+x reorg-notebooks # same sequence as Binder, same layout as Binder -wget --quiet --output-document - https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/archive/master.tar.gz | tar xz +wget --quiet --output-document - "https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/archive/master.tar.gz" | tar xz ./downloadrepos ./reorg-notebooks mv -v PAVICS-e2e-workflow-tests-master/notebooks/*.ipynb ./ @@ -77,30 +78,30 @@ rm -rf PAVICS-e2e-workflow-tests-master rm -rf downloadrepos default_build_params reorg-notebooks TMP_SCRIPT="$TMPDIR/deploy-notebook" -cat << __EOF__ > $TMP_SCRIPT +cat << __EOF__ > "${TMP_SCRIPT}" #!/bin/sh -x if [ -n "\`ls -A /$TUTORIAL_NOTEBOOKS_DIR/\`" ]; then - cd $NOTEBOOK_DIR_MNT - rm -rf $TUTORIAL_NOTEBOOKS_DIR/* - if [ ! -d $TUTORIAL_NOTEBOOKS_DIR ]; then - mkdir $TUTORIAL_NOTEBOOKS_DIR + cd "${NOTEBOOK_DIR_MNT}" + rm -rf ${TUTORIAL_NOTEBOOKS_DIR}/* + if [ ! -d "${TUTORIAL_NOTEBOOKS_DIR}" ]; then + mkdir "${TUTORIAL_NOTEBOOKS_DIR}" fi - cp -rv /$TUTORIAL_NOTEBOOKS_DIR/* $TUTORIAL_NOTEBOOKS_DIR + cp -rv /${TUTORIAL_NOTEBOOKS_DIR}/* "${TUTORIAL_NOTEBOOKS_DIR}" # make read-only - chown -R root:root $TUTORIAL_NOTEBOOKS_DIR + chown -R root:root "${TUTORIAL_NOTEBOOKS_DIR}" fi __EOF__ -chmod a+x $TMP_SCRIPT +chmod a+x "${TMP_SCRIPT}" docker run --rm \ --name deploy_tutorial_notebooks \ -u root \ - -v $TMP_SCRIPT:/deploy-notebook:ro \ - -v $TMPDIR/$TUTORIAL_NOTEBOOKS_DIR:/$TUTORIAL_NOTEBOOKS_DIR:ro \ - -v "$JUPYTERHUB_USER_DATA_DIR":$NOTEBOOK_DIR_MNT:rw \ + -v "${TMP_SCRIPT}":/deploy-notebook:ro \ + -v "${TMPDIR}/${TUTORIAL_NOTEBOOKS_DIR}":"/${TUTORIAL_NOTEBOOKS_DIR}":ro \ + -v "${JUPYTERHUB_USER_DATA_DIR}":"${NOTEBOOK_DIR_MNT}":rw \ --entrypoint /deploy-notebook \ - bash:5.1.4 + "${BASH_IMAGE}" # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/birdhouse/deployment/triggerdeploy.sh b/birdhouse/deployment/triggerdeploy.sh index 4ddce8122..82752dfb8 100755 --- a/birdhouse/deployment/triggerdeploy.sh +++ b/birdhouse/deployment/triggerdeploy.sh @@ -59,7 +59,7 @@ else shift fi -COMPOSE_DIR="`realpath "$COMPOSE_DIR"`" +COMPOSE_DIR="$(realpath "$COMPOSE_DIR")" if [ ! -f "$COMPOSE_DIR/docker-compose.yml" ]; then echo "ERROR: missing docker-compose.yml in '$COMPOSE_DIR'" 1>&2 @@ -78,12 +78,12 @@ cd $COMPOSE_DIR should_trigger() { - EXTRA_REPO="`git rev-parse --show-toplevel`" + EXTRA_REPO="$(git rev-parse --show-toplevel)" - DEPLOY_KEY="$AUTODEPLOY_DEPLOY_KEY_ROOT_DIR/`basename "$EXTRA_REPO"`_deploy_key" - DEFAULT_DEPLOY_KEY="$AUTODEPLOY_DEPLOY_KEY_ROOT_DIR/id_rsa_git_ssh_read_only" - if [ ! -e "$DEPLOY_KEY" -a -e "$DEFAULT_DEPLOY_KEY" ]; then - DEPLOY_KEY="$DEFAULT_DEPLOY_KEY" + DEPLOY_KEY="${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}/$(basename "$EXTRA_REPO")_deploy_key" + DEFAULT_DEPLOY_KEY="${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}/id_rsa_git_ssh_read_only" + if [ ! -e "$DEPLOY_KEY" ] && [ -e "${DEFAULT_DEPLOY_KEY}" ]; then + DEPLOY_KEY="${DEFAULT_DEPLOY_KEY}" fi DEPLOY_KEY_DISPLAY="" @@ -100,80 +100,84 @@ should_trigger() { # fetch remote branches, not affecting current checkout git fetch --prune --all - CURRENT_BRANCH="`git rev-parse --abbrev-ref HEAD`" + CURRENT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" # find out if we are behind and/or ahead remote tracking branch TMP_SAVE_FILE="/tmp/triggerdeploy-behindcnt.txt" - echo "BEHIND_CNT=-1" > $TMP_SAVE_FILE - echo "AHEAD_CNT=-1" >> $TMP_SAVE_FILE - echo "local=" >> $TMP_SAVE_FILE - echo "remote=" >> $TMP_SAVE_FILE + { + echo "BEHIND_CNT=-1" + echo "AHEAD_CNT=-1" + echo "local=" + echo "remote=" + } >> "${TMP_SAVE_FILE}" git for-each-ref --format="%(refname:short) %(upstream:short)" refs/heads | \ while read local remote do - [ -z "$remote" ] && continue - git show-branch remotes/$remote || continue - [ "$local" != "$CURRENT_BRANCH" ] && continue - BEHIND_CNT="`git rev-list --right-only --count ${local}...${remote}`" - AHEAD_CNT="`git rev-list --left-only --count ${local}...${remote}`" - echo "BEHIND_CNT=$BEHIND_CNT" >> $TMP_SAVE_FILE - echo "AHEAD_CNT=$AHEAD_CNT" >> $TMP_SAVE_FILE - echo "local=$local" >> $TMP_SAVE_FILE - echo "remote=$remote" >> $TMP_SAVE_FILE + [ -z "${remote}" ] && continue + git show-branch "remotes/${remote}" || continue + [ "${local}" != "${CURRENT_BRANCH}" ] && continue + BEHIND_CNT="$(git rev-list --right-only --count "${local}...${remote}")" + AHEAD_CNT="$(git rev-list --left-only --count "${local}...${remote}")" + { + echo "BEHIND_CNT=${BEHIND_CNT}" + echo "AHEAD_CNT=${AHEAD_CNT}" + echo "local=${local}" + echo "remote=${remote}" + } >> "${TMP_SAVE_FILE}" break done - . $TMP_SAVE_FILE - rm $TMP_SAVE_FILE + . "${TMP_SAVE_FILE}" + rm "${TMP_SAVE_FILE}" # If should trigger deploy given diff. - GIT_CHANGED_FILES="`git diff --name-status $local remotes/$remote`" + GIT_CHANGED_FILES="$(git diff --name-status "${local}" "remotes/${remote}")" TMP_SCRIPT="/tmp/repo-conditional-trigger" # get latest version of script on same branch - CURRENT_REMOTE_BRANCH="`git rev-parse --abbrev-ref --symbolic-full-name @{upstream}`" + CURRENT_REMOTE_BRANCH="$(git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}')" # Default to always trigger if checkout is behind to be backward-compatible and safe. CONDITIONAL_TRIGGER_EXIT_CODE=0 - git show $CURRENT_REMOTE_BRANCH:./autodeploy/conditional-trigger > $TMP_SCRIPT + git show "${CURRENT_REMOTE_BRANCH}":./autodeploy/conditional-trigger > "${TMP_SCRIPT}" if [ "$?" -eq 0 ]; then # Execute script only if it exists, script is optional. - chmod a+x $TMP_SCRIPT - GIT_CHANGED_FILES="$GIT_CHANGED_FILES" $TMP_SCRIPT + chmod a+x "${TMP_SCRIPT}" + GIT_CHANGED_FILES="$GIT_CHANGED_FILES" "${TMP_SCRIPT}" CONDITIONAL_TRIGGER_EXIT_CODE=$? fi - rm $TMP_SCRIPT + rm "${TMP_SCRIPT}" # trigger deploy if the behind count is greater than zero and ahead count is # less than or equal zero - if [ "$BEHIND_CNT" -gt 0 ]; then - if [ "$AHEAD_CNT" -le 0 ]; then - if [ $CONDITIONAL_TRIGGER_EXIT_CODE -eq 0 ]; then - echo "triggerdeploy: repo '$EXTRA_REPO' is behind will trigger deploy" + if [ "${BEHIND_CNT}" -gt 0 ]; then + if [ "${AHEAD_CNT}" -le 0 ]; then + if [ ${CONDITIONAL_TRIGGER_EXIT_CODE} -eq 0 ]; then + echo "triggerdeploy: repo '${EXTRA_REPO}' is behind will trigger deploy" return 0 else - echo "triggerdeploy: repo '$EXTRA_REPO' is behind but do not need to trigger deploy, just pull if clean checkout" + echo "triggerdeploy: repo '${EXTRA_REPO}' is behind but do not need to trigger deploy, just pull if clean checkout" # pull only if clean checkout to avoid overriding important changes - if [ -z "`git status -u --porcelain`" ]; then + if [ -z "$(git status -u --porcelain)" ]; then git pull else - echo "triggerdeploy: WARNING: unclean repo '$EXTRA_REPO', unable to pull" 1>&2 + echo "triggerdeploy: WARNING: unclean repo '${EXTRA_REPO}', unable to pull" 1>&2 fi return 1 fi else - echo "triggerdeploy: do nothing, repo '$EXTRA_REPO' is both behind and ahead (divergent), must be a devel repo" + echo "triggerdeploy: do nothing, repo '${EXTRA_REPO}' is both behind and ahead (divergent), must be a devel repo" return 1 fi else - echo "triggerdeploy: repo '$EXTRA_REPO' is up-to-date or ahead, no deploy needed" + echo "triggerdeploy: repo '${EXTRA_REPO}' is up-to-date or ahead, no deploy needed" return 1 fi } -START_TIME="`date -Isecond`" +START_TIME="$(date -Isecond)" echo "========== -triggerdeploy START_TIME=$START_TIME" +triggerdeploy START_TIME=${START_TIME}" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/read-configs.include.sh" # Read AUTODEPLOY_EXTRA_REPOS read_basic_configs_only @@ -181,40 +185,40 @@ read_basic_configs_only set -x SHOULD_TRIGGER="" -for adir in $COMPOSE_DIR $AUTODEPLOY_EXTRA_REPOS; do - if [ -d "$adir" ]; then - cd $adir +for adir in "${COMPOSE_DIR}" ${AUTODEPLOY_EXTRA_REPOS}; do + if [ -d "${adir}" ]; then + cd "${adir}" || exit if should_trigger; then SHOULD_TRIGGER=1 break fi else - echo "WARNING: extra repo '$adir' do not exist" + echo "WARNING: extra repo '${adir}' do not exist" fi done EXIT_CODE=0 -if [ -n "$SHOULD_TRIGGER" ]; then - cd $COMPOSE_DIR +if [ -n "${SHOULD_TRIGGER}" ]; then + cd "${COMPOSE_DIR}" || exit TMP_SCRIPT="/tmp/latestdeploy.sh" # get latest version of deploy script on same branch - CURRENT_REMOTE_BRANCH="`git rev-parse --abbrev-ref --symbolic-full-name @{upstream}`" - git show $CURRENT_REMOTE_BRANCH:./deployment/deploy.sh > $TMP_SCRIPT + CURRENT_REMOTE_BRANCH="$(git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}')" + git show "${CURRENT_REMOTE_BRANCH}":./deployment/deploy.sh > "${TMP_SCRIPT}" - chmod a+x $TMP_SCRIPT - $TMP_SCRIPT $COMPOSE_DIR $ENV_LOCAL_FILE + chmod a+x "${TMP_SCRIPT}" + $TMP_SCRIPT "${COMPOSE_DIR}" "${ENV_LOCAL_FILE}" EXIT_CODE=$? - rm $TMP_SCRIPT + rm "${TMP_SCRIPT}" fi set +x echo " -triggerdeploy finished START_TIME=$START_TIME -triggerdeploy finished END_TIME=`date -Isecond`" +triggerdeploy finished START_TIME=${START_TIME} +triggerdeploy finished END_TIME=$(date -Isecond)" -exit $EXIT_CODE +exit ${EXIT_CODE} # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 5f283dce0..1487ba44c 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -4,6 +4,12 @@ # # Do NOT use environment variables in here since when pavics-compose.sh runs # inside a container, the environment vars do not have the same value. +# +# Any default value that can pose a security concern or that are strongly +# recommended to be modified should use '__DEFAULT__{var}' definition, and +# have those default definitions defined in 'default.env'. This will ensure +# that these example values are flagged by the script if left unmodified +# (see also: 'check_default_vars' in 'birdhouse/read-configs.include.sh'). ############################################################################# # Override data persistence root directory @@ -13,24 +19,23 @@ # are "shared" between subdirectories). This means that the subdirectory structure is fixed. #export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' # otherwise use the value from 'default.env', must exist -export SSL_CERTIFICATE="/path/to/ssl/cert.pem" # *absolute* path to the nginx ssl certificate, path and key bundle -export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation -export DOC_URL="https://www.example.com/" # URL where /doc gets redirected -export MAGPIE_SECRET=itzaseekrit -export MAGPIE_ADMIN_USERNAME=admin +export SSL_CERTIFICATE="${__DEFAULT__SSL_CERTIFICATE}" # *absolute* path to the nginx ssl certificate, path and key bundle +export PAVICS_FQDN="${__DEFAULT__PAVICS_FQDN}" # Fully qualified domain name of this Pavics installation +export DOC_URL="${__DEFAULT__DOC_URL}" # URL where /doc gets redirected +export MAGPIE_SECRET="${__DEFAULT__MAGPIE_SECRET}" +export MAGPIE_ADMIN_USERNAME="${__DEFAULT__MAGPIE_ADMIN_USERNAME}" # Magpie now requires a password length of at least 12 characters # For initial bootstrap only, change in the Magpie Web UI after initial boostrap. -export MAGPIE_ADMIN_PASSWORD=qwertyqwerty! -export TWITCHER_PROTECTED_PATH=/twitcher/ows/proxy -export SUPPORT_EMAIL=helpdesk@example.com -export CMIP5_THREDDS_ROOT=birdhouse/CMIP5/CCCMA -export JUPYTERHUB_ADMIN_USERS="{'admin'}" # python set syntax -export POSTGRES_PAVICS_USERNAME=postgres-pavics -export POSTGRES_PAVICS_PASSWORD=postgres-qwerty -export POSTGRES_MAGPIE_USERNAME=postgres-magpie -export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty -export GEOSERVER_ADMIN_USER=admingeo -export GEOSERVER_ADMIN_PASSWORD=geoserverpass +export MAGPIE_ADMIN_PASSWORD="${__DEFAULT__MAGPIE_ADMIN_PASSWORD}" +export TWITCHER_PROTECTED_PATH="/twitcher/ows/proxy" +export SUPPORT_EMAIL="${__DEFAULT__SUPPORT_EMAIL}" +export CMIP5_THREDDS_ROOT="birdhouse/CMIP5/CCCMA" +export POSTGRES_PAVICS_USERNAME="${__DEFAULT__POSTGRES_PAVICS_USERNAME}" +export POSTGRES_PAVICS_PASSWORD="${__DEFAULT__POSTGRES_PAVICS_PASSWORD}" +export POSTGRES_MAGPIE_USERNAME="${__DEFAULT__POSTGRES_MAGPIE_USERNAME}" +export POSTGRES_MAGPIE_PASSWORD="${__DEFAULT__POSTGRES_MAGPIE_PASSWORD}" +export GEOSERVER_ADMIN_USER="${__DEFAULT__GEOSERVER_ADMIN_USER}" +export GEOSERVER_ADMIN_PASSWORD="${__DEFAULT__GEOSERVER_ADMIN_PASSWORD}" ############################################################################# # Optional vars @@ -424,6 +429,12 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass #c.Spawner.pre_spawn_hook = custom_create_dir_hook #" +# Usernames that should be given admin access in jupyterhub +# By default, only the MAGPIE_ADMIN_USERNAME user is given admin access. Update this variable only if you wish +# to give additional users admin access by default. +# Note that you can also give users admin access through the jupyterhub UI. +#export JUPYTERHUB_ADMIN_USERS='{\"${MAGPIE_ADMIN_USERNAME}\", \"othername\"}' # python set syntax + # Extra PyWPS config for **all** WPS services (currently only Flyingpigeon, Finch and Raven supported). # export EXTRA_PYWPS_CONFIG=" # [logging] @@ -576,9 +587,9 @@ export THREDDS_ADDITIONAL_CATALOG="" ############################################################################# # Deprecated vars (for components in the ./deprecated-components directory) ############################################################################# -export TOMCAT_NCWMS_PASSWORD=ncwmspass -export CATALOG_USERNAME=admin-catalog -export CATALOG_PASSWORD=qwerty -export CATALOG_THREDDS_SERVICE=thredds -export PHOENIX_PASSWORD=phoenix_pass -export PHOENIX_PASSWORD_HASH=sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234 +export TOMCAT_NCWMS_PASSWORD="${__DEFAULT__TOMCAT_NCWMS_PASSWORD}" +export CATALOG_USERNAME="${__DEFAULT__CATALOG_USERNAME}" +export CATALOG_PASSWORD="${__DEFAULT__CATALOG_PASSWORD}" +export CATALOG_THREDDS_SERVICE="thredds" +export PHOENIX_PASSWORD="${__DEFAULT__PHOENIX_PASSWORD}" +export PHOENIX_PASSWORD_HASH="${__DEFAULT__PHOENIX_PASSWORD_HASH}" diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 44636ea51..9d7e47c63 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -10,10 +10,6 @@ # * Try to keep the same behavior/code, inside and outside of the # autodeploy container to catch error early with the autodeploy. -YELLOW=$(tput setaf 3) -RED=$(tput setaf 1) -NORMAL=$(tput sgr0) - # list of all variables to be substituted in templates # some of these variables *could* employ provided values in 'default.env', # but they must ultimately be defined one way or another for the server to work @@ -31,83 +27,94 @@ VARS=' # when the value provided explicitly, it will be used instead of guessing it by inferred values from other variables OPTIONAL_VARS=' $PAVICS_FQDN_PUBLIC + $SSL_CERTIFICATE $EXTRA_PYWPS_CONFIG $SERVER_NAME $SERVER_DESCRIPTION + $SERVER_INSTITUTION + $SERVER_SUBJECT + $SERVER_TAGS + $SERVER_DOCUMENTATION_URL + $SERVER_RELEASE_NOTES_URL + $SERVER_SUPPORT_URL + $SERVER_LICENSE_URL ' # we switch to the real directory of the script, so it still works when used from $PATH # tip: ln -s /path/to/pavics-compose.sh ~/bin/ # Setup PWD for sourcing env.local. -cd $(dirname $(readlink -f $0 || realpath $0)) +cd "$(dirname "$(readlink -f "$0" || realpath "$0")")" || exit 1 # Setup COMPOSE_DIR for sourcing env.local. # Prevent un-expected difference when this script is run inside autodeploy # container and manually from the host. -COMPOSE_DIR="`pwd`" +COMPOSE_DIR="$(pwd)" -. "$COMPOSE_DIR/read-configs.include.sh" +. "${COMPOSE_DIR}/read-configs.include.sh" read_configs # this sets ALL_CONF_DIRS -. ./scripts/get-components-json.include.sh -. ./scripts/get-services-json.include.sh -. ./scripts/get-version-json.include.sh +. "${COMPOSE_DIR}/scripts/get-components-json.include.sh" +. "${COMPOSE_DIR}/scripts/get-services-json.include.sh" +. "${COMPOSE_DIR}/scripts/get-version-json.include.sh" -for i in ${VARS} -do - v="${i}" - if [ -z "`eval "echo ${v}"`" ] - then - echo "${RED}Error${NORMAL}: Required variable $v is not set. Check env.local file." - exit 1 - fi -done +check_required_vars || exit $? ## check fails when root access is required to access this file.. workaround possible by going through docker daemon... but # will add delay # if [ ! -f $SSL_CERTIFICATE ] # then -# echo "Error, SSL certificate file $SSL_CERTIFICATE is missing" +# log ERROR "SSL certificate file $SSL_CERTIFICATE is missing" # exit 1 # fi TIMEWAIT_REUSE=$(/sbin/sysctl -n net.ipv4.tcp_tw_reuse) -if [ ${TIMEWAIT_REUSE} -eq 0 ] +if [ "${TIMEWAIT_REUSE}" -eq 0 ] then - echo "${YELLOW}Warning:${NORMAL} the sysctl net.ipv4.tcp_tw_reuse is not enabled" - echo " It it suggested to set it to 1, otherwise the pavicscrawler may fail" + log WARN "the sysctl net.ipv4.tcp_tw_reuse is not enabled. " \ + "It it suggested to set it to 1, otherwise the pavicscrawler may fail." fi export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES="" -for adir in $AUTODEPLOY_EXTRA_REPOS; do +for adir in ${AUTODEPLOY_EXTRA_REPOS}; do # 4 spaces in front of '--volume' is important - AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES="$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES + AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES="${AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES} --volume ${adir}:${adir}:rw" done export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES # we apply all the templates -find $ALL_CONF_DIRS -name '*.template' | +find ${ALL_CONF_DIRS} -name '*.template' 2>/dev/null | while read FILE do DEST=${FILE%.template} - cat ${FILE} | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > ${DEST} + cat "${FILE}" | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > "${DEST}" done +SHELL_EXEC_FLAGS= +if [ "${BIRDHOUSE_LOG_LEVEL}" = "DEBUG" ]; then + SHELL_EXEC_FLAGS=-x +fi + +create_compose_conf_list # this sets COMPOSE_CONF_LIST +log INFO "Displaying resolved compose configurations:" +echo "COMPOSE_CONF_LIST=" +echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' + +if [ x"$1" = x"info" ]; then + log INFO "Stopping before execution of docker-compose command." + exit 0 +fi + if [ x"$1" = x"up" ]; then for adir in $ALL_CONF_DIRS; do COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up" if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then - echo "executing '$COMPONENT_PRE_COMPOSE_UP'" - sh -x "$COMPONENT_PRE_COMPOSE_UP" + log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'" + sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP" fi done fi -create_compose_conf_list # this sets COMPOSE_CONF_LIST -echo "COMPOSE_CONF_LIST=" -echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f' - # the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script PROXY_SECURE_PORT=443 HOSTNAME=${PAVICS_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ERR=$? @@ -116,7 +123,7 @@ ERR=$? type post-compose 2>&1 | grep 'post-compose is a function' > /dev/null if [ $? -eq 0 ] then - [ ${ERR} -gt 0 ] && { echo "Error occurred with docker-compose, not running post-compose"; exit $?; } + [ ${ERR} -gt 0 ] && { log ERROR "Error occurred with docker-compose, not running post-compose"; exit $?; } post-compose $* fi @@ -137,8 +144,8 @@ do for adir in $ALL_CONF_DIRS; do COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up" if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then - echo "executing '$COMPONENT_POST_COMPOSE_UP'" - sh -x "$COMPONENT_POST_COMPOSE_UP" + log INFO "Executing '$COMPONENT_POST_COMPOSE_UP'" + sh ${SHELL_EXEC_FLAGS} "$COMPONENT_POST_COMPOSE_UP" fi done diff --git a/birdhouse/read-configs.include.sh b/birdhouse/read-configs.include.sh index 6e9c56007..dd6a05bbc 100644 --- a/birdhouse/read-configs.include.sh +++ b/birdhouse/read-configs.include.sh @@ -16,7 +16,7 @@ # # # Source the script providing function read_configs. # # read_configs uses COMPOSE_DIR to find default.env and env.local. -# . $COMPOSE_DIR/read-configs.include.sh +# . ${COMPOSE_DIR}/read-configs.include.sh # # # Call function read_configs to read the various config files in the # # appropriate order and process delayed eval vars properly. @@ -24,11 +24,12 @@ # Derive COMPOSE_DIR from the most probable locations. -# This is NOT meant to be exhautive. +# This is NOT meant to be exhaustive. # Assume the checkout is named "birdhouse-deploy", which might NOT be true. # Caller of this file can simply set COMPOSE_DIR itself, this is the safest way. +# WARNING: cannot use 'log' calls within this function until the following logging script gets resolved and sourced. discover_compose_dir() { - if [ -z "$COMPOSE_DIR" ] || [ ! -e "$COMPOSE_DIR" ]; then + if [ -z "${COMPOSE_DIR}" ] || [ ! -e "${COMPOSE_DIR}" ]; then if [ -f "./pavics-compose.sh" ]; then # Current dir is COMPOSE_DIR COMPOSE_DIR="$(realpath .)" @@ -55,34 +56,48 @@ discover_compose_dir() { # Case of sub-subdir of sibling checkout at same level as birdhouse-deploy. COMPOSE_DIR="$(realpath "../../../birdhouse-deploy/birdhouse")" fi - echo "$COMPOSE_DIR" export COMPOSE_DIR fi + # Perform last-chance validation in case 'COMPOSE_DIR' was incorrectly set explicitly + # and that 'read-configs.include.sh' was sourced directly from an invalid location. + if [ ! -f "${COMPOSE_DIR}/pavics-compose.sh" ]; then + echo \ + "CRITICAL: [${COMPOSE_DIR}/pavics-compose.sh] not found," \ + "please set variable 'COMPOSE_DIR' to a valid location." \ + "Many features depend on this variable." 1>&2 + return 2 + fi } +# error out appropriately without closing shell according to 'sh