Generate developer docs for Python API (#503)

Co-authored-by: Salomon Popp <[email protected]>

docs/docs/developer/api.md

@@ -0,0 +1,16 @@
+# Python API
+
+<!-- dprint-ignore-start -->
+
+::: kpops.api
+    options:
+      filters:
+        - "!^_"
+
+::: kpops.pipeline.Pipeline
+    options:
+      filters:
+        - "!^_"
+        - "!^model_config$"
+
+<!-- dprint-ignore-end -->

docs/mkdocs.yml

@@ -37,6 +37,22 @@ plugins:
       exclude:
         - resources/*
   - glightbox
+  - mkdocstrings:
+      handlers:
+        python:
+          paths: [..]
+          options:
+            docstring_style: sphinx
+            show_if_no_docstring: true
+            line_length: 60
+            separate_signature: true
+            show_signature_annotations: true
+            docstring_section_style: spacy
+            show_root_heading: true
+            # The options below are not yet publicly available,
+            # but would be a nice addition when released.
+            #show_inheritance_diagram: true # https://mkdocstrings.github.io/python/usage/configuration/general/?h=show_inheri#show_inheritance_diagram
+            #modernize_annotations: true # https://mkdocstrings.github.io/python/usage/configuration/signatures/#modernize_annotations
 
 extra:
   version:
@@ -120,6 +136,7 @@ nav:
               - GitHub Actions: user/references/ci-integration/github-actions.md
   - Developer Guide:
       - Getting Started: developer/getting-started.md
+      - API: developer/api.md
       - Contributing: developer/contributing.md
       - Code base:
           - Auto generation: developer/auto-generation.md

kpops/api/__init__.py

@@ -37,13 +37,24 @@ def generate(
     environment: str | None = None,
     verbose: bool = False,
 ) -> Pipeline:
+    """Generate enriched pipeline representation.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param dotenv: Paths to dotenv files.
+    :param config: Path to the dir containing config.yaml files.
+    :param steps: Set of steps (components) to apply the command on.
+    :param filter_type: Whether `steps` should include/exclude the steps.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :param verbose: Enable verbose printing.
+    :return: Generated `Pipeline` object.
+    """
     kpops_config = KpopsConfig.create(
         config,
         dotenv,
         environment,
         verbose,
     )
-    pipeline = create_pipeline(pipeline_path, kpops_config, environment)
+    pipeline = _create_pipeline(pipeline_path, kpops_config, environment)
     log.info(f"Picked up pipeline '{pipeline_path.parent.name}'")
     if steps:
         component_names = steps
@@ -68,6 +79,17 @@ def manifest(
     environment: str | None = None,
     verbose: bool = False,
 ) -> list[Resource]:
+    """Generate pipeline, return final resource representations for each step.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param dotenv: Paths to dotenv files.
+    :param config: Path to the dir containing config.yaml files.
+    :param steps: Set of steps (components) to apply the command on.
+    :param filter_type: Whether `steps` should include/exclude the steps.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :param verbose: Enable verbose printing.
+    :return: Resources.
+    """
     pipeline = kpops.generate(
         pipeline_path=pipeline_path,
         dotenv=dotenv,
@@ -95,6 +117,18 @@ def deploy(
     verbose: bool = True,
     parallel: bool = False,
 ):
+    """Deploy pipeline steps.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param dotenv: Paths to dotenv files.
+    :param config: Path to the dir containing config.yaml files.
+    :param steps: Set of steps (components) to apply the command on.
+    :param filter_type: Whether `steps` should include/exclude the steps.
+    :param dry_run: Whether to dry run the command or execute it.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :param verbose: Enable verbose printing.
+    :param parallel: Enable or disable parallel execution of pipeline steps.
+    """
     pipeline = kpops.generate(
         pipeline_path=pipeline_path,
         dotenv=dotenv,
@@ -131,6 +165,18 @@ def destroy(
     verbose: bool = True,
     parallel: bool = False,
 ):
+    """Destroy pipeline steps.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param dotenv: Paths to dotenv files.
+    :param config: Path to the dir containing config.yaml files.
+    :param steps: Set of steps (components) to apply the command on.
+    :param filter_type: Whether `steps` should include/exclude the steps.
+    :param dry_run: Whether to dry run the command or execute it.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :param verbose: Enable verbose printing.
+    :param parallel: Enable or disable parallel execution of pipeline steps.
+    """
     pipeline = kpops.generate(
         pipeline_path=pipeline_path,
         dotenv=dotenv,
@@ -169,6 +215,18 @@ def reset(
     verbose: bool = True,
     parallel: bool = False,
 ):
+    """Reset pipeline steps.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param dotenv: Paths to dotenv files.
+    :param config: Path to the dir containing config.yaml files.
+    :param steps: Set of steps (components) to apply the command on.
+    :param filter_type: Whether `steps` should include/exclude the steps.
+    :param dry_run: Whether to dry run the command or execute it.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :param verbose: Enable verbose printing.
+    :param parallel: Enable or disable parallel execution of pipeline steps.
+    """
     pipeline = kpops.generate(
         pipeline_path=pipeline_path,
         dotenv=dotenv,
@@ -206,6 +264,18 @@ def clean(
     verbose: bool = True,
     parallel: bool = False,
 ):
+    """Clean pipeline steps.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param dotenv: Paths to dotenv files.
+    :param config: Path to the dir containing config.yaml files.
+    :param steps: Set of steps (components) to apply the command on.
+    :param filter_type: Whether `steps` should include/exclude the steps.
+    :param dry_run: Whether to dry run the command or execute it.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :param verbose: Enable verbose printing.
+    :param parallel: Enable or disable parallel execution of pipeline steps.
+    """
     pipeline = kpops.generate(
         pipeline_path=pipeline_path,
         dotenv=dotenv,
@@ -236,6 +306,12 @@ def init(
     path: Path,
     config_include_opt: bool = False,
 ):
+    """Initiate a default empty project.
+
+    :param path: Directory in which the project should be initiated.
+    :param conf_incl_opt: Whether to include non-required settings
+        in the generated config file.
+    """
     if not path.exists():
         path.mkdir(parents=False)
     elif next(path.iterdir(), False):
@@ -244,22 +320,34 @@ def init(
     init_project(path, config_include_opt)
 
 
-def create_pipeline(
+def _create_pipeline(
     pipeline_path: Path,
     kpops_config: KpopsConfig,
     environment: str | None,
 ) -> Pipeline:
+    """Create pipeline.
+
+    :param pipeline_path: Path to pipeline definition yaml file.
+    :param config: KPOps Config.
+    :param environment: The environment to generate and deploy the pipeline to.
+    :return: Created `Pipeline` object.
+    """
     registry = Registry()
     if kpops_config.components_module:
         registry.find_components(kpops_config.components_module)
     registry.find_components("kpops.components")
 
-    handlers = setup_handlers(kpops_config)
+    handlers = _setup_handlers(kpops_config)
     parser = PipelineGenerator(kpops_config, registry, handlers)
     return parser.load_yaml(pipeline_path, environment)
 
 
-def setup_handlers(config: KpopsConfig) -> ComponentHandlers:
+def _setup_handlers(config: KpopsConfig) -> ComponentHandlers:
+    """Set up handlers for a component.
+
+    :param config: KPOps config.
+    :return: Handlers for a component.
+    """
     schema_handler = SchemaHandler.load_schema_handler(config)
     connector_handler = KafkaConnectHandler.from_kpops_config(config)
     proxy_wrapper = ProxyWrapper(config.kafka_rest)

pyproject.toml

@@ -62,6 +62,7 @@ mkdocs-material = "^9.0.0"
 mkdocs-glightbox = "^0.3.1"
 mkdocs-exclude-search = "^0.6.5"
 mike = "^1.1.2"
+mkdocstrings = { extras = ["python"], version = "^0.25.1" }
 
 [tool.poetry_bumpversion.file."kpops/__init__.py"]
 

tests/api/test_handlers.py

@@ -1,6 +1,6 @@
 from pytest_mock import MockerFixture
 
-from kpops.api import setup_handlers
+from kpops.api import _setup_handlers
 from kpops.component_handlers import ComponentHandlers
 from kpops.component_handlers.kafka_connect.kafka_connect_handler import (
     KafkaConnectHandler,
@@ -35,7 +35,7 @@ def test_set_up_handlers_with_no_schema_handler(mocker: MockerFixture):
         topic_handler=topic_handler,
     )
 
-    actual_handlers = setup_handlers(config)
+    actual_handlers = _setup_handlers(config)
 
     connector_handler_mock.from_kpops_config.assert_called_once_with(config)
 
@@ -72,7 +72,7 @@ def test_set_up_handlers_with_schema_handler(mocker: MockerFixture):
         topic_handler=topic_handler,
     )
 
-    actual_handlers = setup_handlers(config)
+    actual_handlers = _setup_handlers(config)
 
     schema_handler_mock.load_schema_handler.assert_called_once_with(config)