Merge branch 'develop' into 214700_fix_cycle_deletion_dev

src/.flake8 → .flake8

@@ -21,6 +21,7 @@ exclude =
     migrations,
     snapshots,
     __pypackages__,
+    frontend,
 
 ignore =
     # black formatting related

.github/helpers/dev.sh

@@ -17,10 +17,10 @@ else
         -n auto \
         --reruns 3 \
         --reruns-delay 1 \
-        --cov-report xml:coverage.xml \
+        --cov-report xml:test-coverage/coverage.xml \
         --randomly-seed=42 \
         --create-db \
-        /tests/unit/
+        ./tests/unit/
       ;;
     "lint")
       mkdir -p ./lint-results

.github/helpers/docker-compose.selenium.yml

@@ -4,11 +4,11 @@ volumes:
 services:
   backend:
     volumes:
-      - ../src/report/screenshot/:/code/screenshot/
-      - ../src/report/:/code/report/
+      - ../../tests/test-coverage:/code/test-coverage
+      - ../../tests/report/:/code/tests/selenium/output_data/report/
       - type: volume
         source: backend-web-app
-        target: /code/hct_mis_api/apps/web
+        target: /code/src/hct_mis_api/apps/web
         volume:
           nocopy: false
     depends_on:
@@ -31,7 +31,7 @@ services:
       - backend-web-app:/tmp/
     command: |
       sh -c "
-        cp -r ./hct_mis_api/apps/web/* /tmp/
+        cp -r /packages/__pypackages__/3.11/lib/hct_mis_api/apps/web/* /tmp/
       "
     restart: "no"
 

.github/helpers/docker-compose.tst.yml

@@ -3,7 +3,7 @@ services:
   backend:
     image: ${dev_backend_image}
     volumes:
-      - ../../src/test-coverage:/code/backend/test-coverage
+      - ../../tests/test-coverage:/code/test-coverage
       - ./dev.sh:/code/dev.sh
     depends_on:
       - db

.github/workflows/ci.yml

@@ -228,15 +228,14 @@ jobs:
 
       - name: Unit tests
         run: |
-          # a little hack to allow to have 1 compose file running 2 different test suites because --env-file didn't work as docs suggested it should
           dev_backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}-dev docker compose \
             -f ./.github/helpers/docker-compose.tst.yml \
             --profile unit \
             run backend ./dev.sh test
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
-          files: ./src/coverage.xml
+          files: ./tests/test-coverage/coverage.xml
           flags: unittests
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
@@ -265,22 +264,22 @@ jobs:
             -f ./.github/helpers/docker-compose.selenium.yml \
             run backend bash -c "
               waitforit -host=db -port=5432 -timeout=30
-              pytest -svvv $extra_options /tests/selenium --cov-report xml:./coverage.xml --html-report=./report/report.html --randomly-seed=42
+              pytest -svvv $extra_options ./tests/selenium --cov-report xml:test-coverage/coverage.xml --html-report=./tests/selenium/output_data/report/report.html --randomly-seed=42
             "
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         if: always()
         continue-on-error: true
         with:
           name: report
-          path: ./src/report/
+          path: ./tests/report/
           retention-days: 5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         if: always()
         continue-on-error: true
         with:
-          files: ./src/coverage.xml
+          files: ./test-coverage/coverage.xml
           flags: e2e
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
@@ -297,6 +296,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Run Trivy vulnerability scanner
+        continue-on-error: true # due to getting TOOMANYREQUESTS
         uses: aquasecurity/trivy-action@master
         with:
           image-ref: '${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:core-${{ github.sha }}'

.gitignore

@@ -101,6 +101,8 @@ ENV/
 env.bak/
 venv.bak/
 
+.pdm-python
+
 # Spyder project settings
 .spyderproject
 .spyproject
@@ -129,3 +131,5 @@ venv.bak/
 cov.json
 
 .vscode/
+output_data/
+.coverage

.pre-commit-config.yaml

@@ -1,42 +1,9 @@
 repos:
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        stages: [commit]
-  - repo: https://github.com/ambv/black
-    rev: 23.12.1
-    hooks:
-      - id: black
-        args: [--config=backend/pyproject.toml]
-        exclude: "migrations|snapshots"
-        stages: [commit]
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        args: [--config=backend/.flake8]
-        additional_dependencies: [flake8-bugbear==22.12.6]
-        stages: [ commit ]
-        exclude: /deployment/|/migrations/
-  # mypy precommit hook
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.982
-    hooks:
-      - id: mypy
-        verbose: true
-        args: [--config-file=backend/pyproject.toml, --follow-imports=skip]
-        pass_filenames: true
-        stages: [commit]
-        additional_dependencies: [
-          types-requests==2.28.11.15,
-          types-redis==4.5.1.4,
-          types-python-dateutil==2.8.19.10,
-          types-pytz==2022.7.1.2,
-          djangorestframework-stubs==1.9.1,
-          graphene-stubs==0.15,
-          django-stubs==1.15.0,
-          django-stubs-ext==0.7.0,
-          openpyxl-stubs==0.1.25
-        ]
-        exclude: /deployment/|/migrations/
+  - repo: local
+    hooks:
+      - id: Linters checks
+        name: Linters checks
+        entry: docker compose -f development_tools/compose.yml run --no-TTY -i --rm backend sh -c 'black . --check && isort . --check-only && flake8 . && mypy .'
+        language: system
+        pass_filenames: false
+        verbose: true

development_tools/compose.yml

@@ -25,11 +25,10 @@ services:
     ports:
       - "8080:8000"
     volumes:
-      - ../src:/code/
-      - ../tests:/tests/
+      - ../.:/code/
       - backend-data:/data
-      - ../src/pyproject.toml:/packages/pyproject.toml
-      - ../src/pdm.lock:/packages/pdm.lock
+      - ../pyproject.toml:/packages/pyproject.toml
+      - ../pdm.lock:/packages/pdm.lock
       - ipython_data_local:/root/.ipython
     command: "dev"
     depends_on:
@@ -125,9 +124,9 @@ services:
     env_file:
       - .env
     ports:
-      - "5433:5432"
+      - "5432:5432"
     healthcheck:
-      test: [ "CMD-SHELL", "su - postgres -c 'pg_isready -U postgres'" ]
+      test: [ "CMD-SHELL", "su - postgres -c 'pg_isready -h db -U postgres'" ]
       interval: 10s
       timeout: 10s
       retries: 5
@@ -138,8 +137,8 @@ services:
     profiles:
       - default
       - services
-    expose:
-      - "6379"
+    ports:
+      - "6379:6379"
     healthcheck:
       test: [ "CMD", "redis-cli", "ping" ]
       interval: 10s

development_tools/local_selenium_env.sh → development_tools/local_selenium_init.sh

@@ -1,3 +1,7 @@
+if [ "$0" = "$BASH_SOURCE" ]; then
+    echo "Please source this script."
+    exit 1
+fi
 export SECRET_KEY=SOME_KEY_HERE
 export DEBUG=true
 export ENV=dev
@@ -38,5 +42,14 @@ export CACHE_LOCATION=redis://localhost:6379/1
 export USE_DUMMY_EXCHANGE_RATES=yes
 export ELASTICSEARCH_HOST=http://localhost:9200
 export CELERY_TASK_ALWAYS_EAGER=true
-export DATA_VOLUME="./data1"
-export LIBRARY_PATHS=true
+export LIBRARY_PATHS=true
+SCRIPT_DIR=$(realpath "$(dirname $0)")
+MAIN_DIR=$(realpath $SCRIPT_DIR/..)
+echo "SCRIPT_DIR: $SCRIPT_DIR"
+export PYTHONPATH=$MAIN_DIR/src:$PYTHONPATH
+export OUTPUT_DATA_ROOT=$MAIN_DIR/tests/selenium/output_data
+export DATA_VOLUME=$OUTPUT_DATA_ROOT/data
+pushd  $MAIN_DIR/src/frontend
+yarn
+yarn build-for-backend
+popd

docker/Dockerfile

@@ -41,35 +41,34 @@ RUN apt-get update \
 
 ENV PDM_PACKAGES=/packages
 ENV CODE=/code
-ENV PDM_NO_SELF=True
 ENV PDM_PROJECT=$PDM_PACKAGES
 ENV PYPACKAGES=$PDM_PACKAGES/__pypackages__/3.11
 ENV PYTHONPYCACHEPREFIX=/tmp/pycache \
-    PYTHONPATH=$PYPACKAGES/lib:$CODE:$PYTHONPATH \
+    PYTHONPATH=$PYPACKAGES/lib:$PYTHONPATH \
     PATH=$PYPACKAGES/bin:$PATH \
     XDG_RUNTIME_DIR=/run/user/"${UID}"
-
+ENV DJANGO_SETTINGS_MODULE=hct_mis_api.config.settings
 WORKDIR $CODE
 
 COPY --from=waitforit /data/waitforit /usr/local/bin/waitforit
 
 # Dist builder image
-FROM base as builder
-
+FROM base as pdm
 RUN pip install --upgrade pip &&\
     pip install pdm==2.15.2 &&\
     pip install setuptools==71.1.0 &&\
     pdm config cache_dir /var/cache/pdm &&\
     pdm config python.use_venv false &&\
     pdm config venv.in_project true &&\
     pdm config check_update false
-
 WORKDIR $PDM_PACKAGES
-COPY src/pyproject.toml src/pdm.lock ./
-RUN pdm sync --prod --no-editable --no-self --no-isolation
+COPY README.md LICENSE pyproject.toml pdm.lock ./
+
+
+
 
 # Dev image
-FROM builder AS dev
+FROM pdm AS dev
 
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
@@ -83,26 +82,14 @@ RUN apt-get update \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+ENV PYTHONPATH=$CODE/src:$CODE/test/:$PYTHONPATH
 RUN pdm sync --no-editable --no-self --no-isolation
 
 WORKDIR $CODE
-COPY ./src/ ./
-COPY ./tests /tests
-
-
-
-COPY ./docker/entrypoint.sh /bin/
-ENTRYPOINT ["entrypoint.sh"]
-
-## Dist (backend only) image
-FROM base AS be-dist
-
-COPY ./src/ ./
-COPY --chown=hope:hope --from=builder $PDM_PACKAGES $PDM_PACKAGES
-COPY --chown=hope:hope --from=certs /data/psql-cert.crt /code/psql-cert.crt
-
-USER hope
-
+COPY ./src/ ./src/
+COPY ./tests ./tests
+COPY ./manage.py ./manage.py
+COPY .flake8 pyproject.toml pdm.lock ./
 COPY ./docker/entrypoint.sh /bin/
 ENTRYPOINT ["entrypoint.sh"]
 
@@ -117,7 +104,22 @@ RUN yarn install --frozen-lockfile --network-timeout 600000
 COPY ./src/frontend ./
 RUN NODE_ENV="production" NODE_OPTIONS="--max-old-space-size=4096" yarn build
 
+# Dist builder image
+FROM pdm as dist-builder
+COPY ./src/ ./src/
+COPY --chown=hope:hope --from=frontend-builder /fe-build/build $PDM_PACKAGES/src/hct_mis_api/apps/web/static/web
+RUN pdm sync --prod --no-editable --no-isolation
+
 
-FROM be-dist AS dist
+## Dist (backend only) image
+FROM base AS dist
 
-COPY --chown=hope:hope --from=frontend-builder /fe-build/build ./hct_mis_api/apps/web/static/web
+
+COPY ./src/gunicorn_config.py /conf/gunicorn_config.py
+COPY --chown=hope:hope --from=dist-builder $PDM_PACKAGES $PDM_PACKAGES
+COPY --chown=hope:hope --from=certs /data/psql-cert.crt /certs/psql-cert.crt
+
+USER hope
+
+COPY ./docker/entrypoint.sh /bin/
+ENTRYPOINT ["entrypoint.sh"]

docker/entrypoint.sh

@@ -7,7 +7,7 @@ wait_for_db() {
 }
 
 if [ $# -eq 0 ]; then
-    exec gunicorn hct_mis_api.wsgi -c /code/gunicorn_config.py
+    exec gunicorn hct_mis_api.wsgi -c /conf/gunicorn_config.py
 else
   case "$1" in
     "dev")