Skip to content

Commit

Permalink
Makefile update (#1755)
Browse files Browse the repository at this point in the history 
Signed-off-by: Fotis Nikolaidis <[email protected]>
  • Loading branch information
@fnikolai
fnikolai authored Feb 9, 2024
1 parent 0e3497c commit 413b3ab
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 117 deletions.
14 changes: 5 additions & 9 deletions .github/workflows/ci_docs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,14 @@ jobs:
${{ env.pythonLocation }}
key: ubuntu-latest_python-3.11_${{ hashFiles('pyproject.toml', '*/pyproject.toml') }}

- name: Build docs and use-cases
# build is available in docs/hr/build
run: |
make hr-docs
- name: Install dependencies
run: pip install .[docs]

- name: Build inline api docs
# build is available in docs/hr/build/apidocs

- name: Build docs
run: |
make api-docs
# Build HR and API docs
make build-docs
- name: Deploy
uses: peaceiris/actions-gh-pages@v3
Expand Down
175 changes: 67 additions & 108 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)


##@ Release Management

# RELEASE_VERSION defines the project version for the operator.
# Update this value when you upgrade the version of your project.
# The general flow is VERSION -> make new_release -> GITHUB_ACTIONS -> {make docker_push, ...}
Expand Down Expand Up @@ -46,45 +48,98 @@ new_release: ## Release a new version of SuperDuperDB
git push --set-upstream origin release-$(RELEASE_VERSION) --tags


build-docs: ## Generate Docs and API
@echo "===> Generate docusaurus docs and blog-posts <==="
cd docs/hr && npm i --legacy-peer-deps && npm run build
cd ../..
@echo "Build finished. The HTML pages are in docs/hr/build"

@echo "===> Generate Sphinx HTML documentation, including API docs <==="
rm -rf docs/api/source/
rm -rf docs/hr/build/apidocs
sphinx-apidoc -f -o docs/api/source superduperdb
sphinx-build -a docs/api docs/hr/build/apidocs
@echo "Build finished. The HTML pages are in docs/hr/build/apidocs"


##@ DevKit

devkit: ## Add essential development tools
install-devkit: ## Add essential development tools
# Add pre-commit hooks to ensure that no strange stuff are being committed.
# https://stackoverflow.com/questions/3462955/putting-git-hooks-into-a-repository
pip install pre-commit
python -m pip install pre-commit
pre-commit autoupdate

# Download tools for code quality testing
pip install .[quality]
python -m pip install .[quality]

# Set git to continuously update submodules
git config --global submodule.recurse true

##@ CI Doc Functions

api-docs: ## Generate Sphinx inline-API HTML documentation
lint-and-type-check: ## Perform code linting and type checking
@echo "===> Generate Sphinx HTML documentation, including API docs <==="
# Code formatting
rm -rf docs/api/source/
black --check $(DIRECTORIES)
rm -rf docs/hr/build/apidocs
# Linter and code formatting
sphinx-apidoc -f -o docs/api/source superduperdb
ruff check $(DIRECTORIES)
sphinx-build -a docs/api docs/hr/build/apidocs
# Static Typing Checker
@echo "Build finished. The HTML pages are in docs/hr/build/apidocs"
mypy superduperdb
# Check for missing docstrings
interrogate superduperdb
# Check for unused dependencies
# deptry ./
# Check for deadcode
# vulture ./

fix-and-test: ## Lint the code before testing
# Code formatting
black $(DIRECTORIES)
# Linter and code formatting
ruff check --fix $(DIRECTORIES)
# Linting
mypy superduperdb
# Unit testing
pytest $(PYTEST_ARGUMENTS)
# Check for missing docstrings
interrogate superduperdb



##@ Base Image Management

# superduperdb/superduperdb is a minimal image contains only what is needed for the framework.
build_superduperdb: ## Build a minimal Docker image for general use
echo "===> build superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)"
docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
--build-arg BUILD_ENV="release"


push_superduperdb: ## Push the superduperdb/superduperdb:latest image
@echo "===> release superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)"
docker push superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)

@echo "===> release superduperdb/superduperdb:latest"
docker tag superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) superduperdb/superduperdb:latest
docker push superduperdb/superduperdb:latest

hr-docs: ## Generate Docusaurus documentation and blog posts
@echo "===> Generate docusaurus docs and blog-posts <==="
cd docs/hr && npm i --legacy-peer-deps && npm run build
cd ../..
@echo "Build finished. The HTML pages are in docs/hr/build"

##@ Testing Environments

testenv_image: ## Build a sandbox image
@echo "===> Build superduperdb/sandbox"
docker build . -f deploy/images/superduperdb/Dockerfile -t superduperdb/sandbox --progress=plain \
--build-arg BUILD_ENV="sandbox" \
--build-arg SUPERDUPERDB_EXTRAS="dev"



##@ Testing Environments

testenv_init: ## Initialize a local Testing environment
@echo "===> Ensure hostnames"
@deploy/testenv/validate_hostnames.sh
Expand Down Expand Up @@ -117,6 +172,7 @@ testdb_init: ## Initialize databases in Docker
testdb_shutdown: ## Terminate Databases Containers
cd deploy/databases/; docker compose down


##@ CI Testing Functions

unit-testing: ## Execute unit testing
Expand All @@ -129,106 +185,9 @@ integration-testing: ## Execute integration testing
# Run the test
pytest $(PYTEST_ARGUMENTS) ./test/integration

fix-and-test: ## Lint the code before testing
# Code formatting
black $(DIRECTORIES)
# Linter and code formatting
ruff check --fix $(DIRECTORIES)
# Linting
mypy superduperdb
# Unit testing
pytest $(PYTEST_ARGUMENTS)
# Check for missing docstrings
interrogate superduperdb
# Check for unused dependencies
#deptry ./

test-and-fix: ## Test the code before linting
# Linting
mypy superduperdb
# Unit testing
pytest $(PYTEST_ARGUMENTS)
# Code formatting
black $(DIRECTORIES)
# Linter and code formatting
ruff check --fix $(DIRECTORIES)
# Check for missing docstrings
interrogate superduperdb

lint-and-type-check: ## Perform code linting and type checking
# Linting
mypy superduperdb
# Code formatting
black --check $(DIRECTORIES)
# Linter and code formatting
ruff check $(DIRECTORIES)
# Check for missing docstrings
interrogate superduperdb

test_notebooks: ## Test notebooks (argument: NOTEBOOKS=<test|dir>)
@echo "Notebook Path: $(NOTEBOOKS)"

@if [ -n "${NOTEBOOKS}" ]; then \
pytest --nbval-lax ${NOTEBOOKS}; \
fi


##@ Development Sandbox Management

run_sandbox-pr: ## Run a pull request in the sandbox (argument: PR_NUMBER=555)
@if [[ -z "${PR_NUMBER}" ]]; then echo "Usage: make run_sandbox-pr PR_NUMBER=<pull-request-number>"; exit -1; fi

@echo "===> Checkout Pull Request #"${PR_NUMBER}" <==="

# checkout remote repo.
git clone --depth 1 --single-branch [email protected]:SuperDuperDB/superduperdb.git /tmp/superduperdb_pr_$(PR_NUMBER)

# fetch specific pr
cd /tmp/superduperdb_pr_$(PR_NUMBER) && \
git fetch --depth 1 origin pull/$(PR_NUMBER)/head:pr_branch && \
git checkout pr_branch

# mount pr to sandbox
docker run -p 8888:8888 -v /tmp/superduperdb_pr_$(PR_NUMBER):/home/superduper/pull_request superduperdb/sandbox

# clean up the tmp directory
rm -rf /tmp/superduperdb_pr_$(PR_NUMBER)



##@ Base Image Management

# superduperdb/superduperdb is a minimal image contains only what is needed for the framework.
build_superduperdb: ## Build a minimal Docker image for general use
echo "===> build superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)"
docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
--build-arg BUILD_ENV="release"


push_superduperdb: ## Push the superduperdb/superduperdb:latest image
@echo "===> release superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)"
docker push superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)

@echo "===> release superduperdb/superduperdb:latest"
docker tag superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) superduperdb/superduperdb:latest
docker push superduperdb/superduperdb:latest


##@ Demo Image Management

# superduperdb/demo is a bloated image that contains everything we need to run the online demo.
build_demo: ## Build a feature-rich Docker image for demonstrations
echo "===> build superduperdb/demo:$(RELEASE_VERSION:v%=%)"
docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/demo:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
--build-arg BUILD_ENV="release" \
--build-arg SUPERDUPERDB_EXTRAS="demo"

push_demo: ## Push the superduperdb/demo:latest image
@echo "===> release superduperdb/demo:$(RELEASE_VERSION:v%=%) <==="
docker push superduperdb/demo:$(RELEASE_VERSION:v%=%)

@echo "===> release superduperdb/demo:latest <==="
docker tag superduperdb/demo:$(RELEASE_VERSION:v%=%) superduperdb/demo:latest
docker push superduperdb/demo:latest


0 comments on commit 413b3ab

Please sign in to comment.