diff --git a/.github/workflows/ci_docs.yaml b/.github/workflows/ci_docs.yaml index f6b0f3bfee..039f282388 100644 --- a/.github/workflows/ci_docs.yaml +++ b/.github/workflows/ci_docs.yaml @@ -42,18 +42,14 @@ jobs: ${{ env.pythonLocation }} key: ubuntu-latest_python-3.11_${{ hashFiles('pyproject.toml', '*/pyproject.toml') }} - - name: Build docs and use-cases - # build is available in docs/hr/build - run: | - make hr-docs - - name: Install dependencies run: pip install .[docs] - - - name: Build inline api docs - # build is available in docs/hr/build/apidocs + + - name: Build docs run: | - make api-docs + # Build HR and API docs + make build-docs + - name: Deploy uses: peaceiris/actions-gh-pages@v3 diff --git a/Makefile b/Makefile index acae14c6e7..006c474458 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +##@ Release Management + # RELEASE_VERSION defines the project version for the operator. # Update this value when you upgrade the version of your project. # The general flow is VERSION -> make new_release -> GITHUB_ACTIONS -> {make docker_push, ...} @@ -46,38 +48,87 @@ new_release: ## Release a new version of SuperDuperDB git push --set-upstream origin release-$(RELEASE_VERSION) --tags +build-docs: ## Generate Docs and API + @echo "===> Generate docusaurus docs and blog-posts <===" + cd docs/hr && npm i --legacy-peer-deps && npm run build + cd ../.. + @echo "Build finished. The HTML pages are in docs/hr/build" + + @echo "===> Generate Sphinx HTML documentation, including API docs <===" + rm -rf docs/api/source/ + rm -rf docs/hr/build/apidocs + sphinx-apidoc -f -o docs/api/source superduperdb + sphinx-build -a docs/api docs/hr/build/apidocs + @echo "Build finished. The HTML pages are in docs/hr/build/apidocs" + + ##@ DevKit -devkit: ## Add essential development tools +install-devkit: ## Add essential development tools # Add pre-commit hooks to ensure that no strange stuff are being committed. # https://stackoverflow.com/questions/3462955/putting-git-hooks-into-a-repository - pip install pre-commit + python -m pip install pre-commit pre-commit autoupdate # Download tools for code quality testing - pip install .[quality] + python -m pip install .[quality] # Set git to continuously update submodules git config --global submodule.recurse true -##@ CI Doc Functions -api-docs: ## Generate Sphinx inline-API HTML documentation +lint-and-type-check: ## Perform code linting and type checking @echo "===> Generate Sphinx HTML documentation, including API docs <===" + # Code formatting rm -rf docs/api/source/ + black --check $(DIRECTORIES) rm -rf docs/hr/build/apidocs + # Linter and code formatting sphinx-apidoc -f -o docs/api/source superduperdb + ruff check $(DIRECTORIES) sphinx-build -a docs/api docs/hr/build/apidocs + # Static Typing Checker @echo "Build finished. The HTML pages are in docs/hr/build/apidocs" + mypy superduperdb + # Check for missing docstrings + interrogate superduperdb + # Check for unused dependencies + # deptry ./ + # Check for deadcode + # vulture ./ +fix-and-test: ## Lint the code before testing + # Code formatting + black $(DIRECTORIES) + # Linter and code formatting + ruff check --fix $(DIRECTORIES) + # Linting + mypy superduperdb + # Unit testing + pytest $(PYTEST_ARGUMENTS) + # Check for missing docstrings + interrogate superduperdb + + + +##@ Base Image Management + +# superduperdb/superduperdb is a minimal image contains only what is needed for the framework. +build_superduperdb: ## Build a minimal Docker image for general use + echo "===> build superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)" + docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \ + --build-arg BUILD_ENV="release" + + +push_superduperdb: ## Push the superduperdb/superduperdb:latest image + @echo "===> release superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)" + docker push superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) + + @echo "===> release superduperdb/superduperdb:latest" + docker tag superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) superduperdb/superduperdb:latest + docker push superduperdb/superduperdb:latest -hr-docs: ## Generate Docusaurus documentation and blog posts - @echo "===> Generate docusaurus docs and blog-posts <===" - cd docs/hr && npm i --legacy-peer-deps && npm run build - cd ../.. - @echo "Build finished. The HTML pages are in docs/hr/build" -##@ Testing Environments testenv_image: ## Build a sandbox image @echo "===> Build superduperdb/sandbox" @@ -85,6 +136,10 @@ testenv_image: ## Build a sandbox image --build-arg BUILD_ENV="sandbox" \ --build-arg SUPERDUPERDB_EXTRAS="dev" + + +##@ Testing Environments + testenv_init: ## Initialize a local Testing environment @echo "===> Ensure hostnames" @deploy/testenv/validate_hostnames.sh @@ -117,6 +172,7 @@ testdb_init: ## Initialize databases in Docker testdb_shutdown: ## Terminate Databases Containers cd deploy/databases/; docker compose down + ##@ CI Testing Functions unit-testing: ## Execute unit testing @@ -129,106 +185,9 @@ integration-testing: ## Execute integration testing # Run the test pytest $(PYTEST_ARGUMENTS) ./test/integration -fix-and-test: ## Lint the code before testing - # Code formatting - black $(DIRECTORIES) - # Linter and code formatting - ruff check --fix $(DIRECTORIES) - # Linting - mypy superduperdb - # Unit testing - pytest $(PYTEST_ARGUMENTS) - # Check for missing docstrings - interrogate superduperdb - # Check for unused dependencies - #deptry ./ - -test-and-fix: ## Test the code before linting - # Linting - mypy superduperdb - # Unit testing - pytest $(PYTEST_ARGUMENTS) - # Code formatting - black $(DIRECTORIES) - # Linter and code formatting - ruff check --fix $(DIRECTORIES) - # Check for missing docstrings - interrogate superduperdb - -lint-and-type-check: ## Perform code linting and type checking - # Linting - mypy superduperdb - # Code formatting - black --check $(DIRECTORIES) - # Linter and code formatting - ruff check $(DIRECTORIES) - # Check for missing docstrings - interrogate superduperdb - test_notebooks: ## Test notebooks (argument: NOTEBOOKS=) @echo "Notebook Path: $(NOTEBOOKS)" @if [ -n "${NOTEBOOKS}" ]; then \ pytest --nbval-lax ${NOTEBOOKS}; \ fi - - -##@ Development Sandbox Management - -run_sandbox-pr: ## Run a pull request in the sandbox (argument: PR_NUMBER=555) - @if [[ -z "${PR_NUMBER}" ]]; then echo "Usage: make run_sandbox-pr PR_NUMBER="; exit -1; fi - - @echo "===> Checkout Pull Request #"${PR_NUMBER}" <===" - - # checkout remote repo. - git clone --depth 1 --single-branch git@github.com:SuperDuperDB/superduperdb.git /tmp/superduperdb_pr_$(PR_NUMBER) - - # fetch specific pr - cd /tmp/superduperdb_pr_$(PR_NUMBER) && \ - git fetch --depth 1 origin pull/$(PR_NUMBER)/head:pr_branch && \ - git checkout pr_branch - - # mount pr to sandbox - docker run -p 8888:8888 -v /tmp/superduperdb_pr_$(PR_NUMBER):/home/superduper/pull_request superduperdb/sandbox - - # clean up the tmp directory - rm -rf /tmp/superduperdb_pr_$(PR_NUMBER) - - - -##@ Base Image Management - -# superduperdb/superduperdb is a minimal image contains only what is needed for the framework. -build_superduperdb: ## Build a minimal Docker image for general use - echo "===> build superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)" - docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \ - --build-arg BUILD_ENV="release" - - -push_superduperdb: ## Push the superduperdb/superduperdb:latest image - @echo "===> release superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)" - docker push superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) - - @echo "===> release superduperdb/superduperdb:latest" - docker tag superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) superduperdb/superduperdb:latest - docker push superduperdb/superduperdb:latest - - -##@ Demo Image Management - -# superduperdb/demo is a bloated image that contains everything we need to run the online demo. -build_demo: ## Build a feature-rich Docker image for demonstrations - echo "===> build superduperdb/demo:$(RELEASE_VERSION:v%=%)" - docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/demo:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \ - --build-arg BUILD_ENV="release" \ - --build-arg SUPERDUPERDB_EXTRAS="demo" - -push_demo: ## Push the superduperdb/demo:latest image - @echo "===> release superduperdb/demo:$(RELEASE_VERSION:v%=%) <===" - docker push superduperdb/demo:$(RELEASE_VERSION:v%=%) - - @echo "===> release superduperdb/demo:latest <===" - docker tag superduperdb/demo:$(RELEASE_VERSION:v%=%) superduperdb/demo:latest - docker push superduperdb/demo:latest - -