KPOps 6.0.0 (#496)

Co-authored-by: Salomon Popp <[email protected]>
bakdata · Jun 6, 2024 · 90d693e · 90d693e
1 parent 3d3d9c9
commit 90d693e
Show file tree

Hide file tree

Showing 68 changed files with 1,250 additions and 695 deletions.
diff --git a/docs/docs/resources/pipeline-config/config.yaml b/docs/docs/resources/pipeline-config/config.yaml
@@ -7,8 +7,6 @@ pipeline_base_dir: .
 # The Kafka brokers address.
 # REQUIRED
 kafka_brokers: "http://broker1:9092,http://broker2:9092"
-# The name of the defaults file and the prefix of the defaults environment file.
-defaults_filename_prefix: defaults
 # Configure the topic name variables you can use in the pipeline definition.
 topic_name_config: 
   # Configures the value for the variable ${output_topic_name}

diff --git a/docs/docs/resources/variables/cli_env_vars.env b/docs/docs/resources/variables/cli_env_vars.env
@@ -14,7 +14,8 @@ KPOPS_DOTENV_PATH # No default value, not required
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
 KPOPS_ENVIRONMENT # No default value, not required
-# Path to YAML with pipeline definition
-KPOPS_PIPELINE_PATH # No default value, required
+# Paths to dir containing 'pipeline.yaml' or files named
+# 'pipeline.yaml'.
+KPOPS_PIPELINE_PATHS # No default value, required
 # Comma separated list of steps to apply the command on
 KPOPS_PIPELINE_STEPS # No default value, not required
diff --git a/docs/docs/resources/variables/cli_env_vars.md b/docs/docs/resources/variables/cli_env_vars.md
@@ -5,5 +5,5 @@ These variables take precedence over the commands' flags. If a variable is set,
 |KPOPS_CONFIG_PATH   |.            |False   |Path to the dir containing config.yaml files                                                                                                                                |
 |KPOPS_DOTENV_PATH   |             |False   |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.                                         |
 |KPOPS_ENVIRONMENT   |             |False   |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|
-|KPOPS_PIPELINE_PATH |             |True    |Path to YAML with pipeline definition                                                                                                                                       |
+|KPOPS_PIPELINE_PATHS|             |True    |Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.                                                                                                     |
 |KPOPS_PIPELINE_STEPS|             |False   |Comma separated list of steps to apply the command on                                                                                                                       |
diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env
@@ -14,10 +14,6 @@ KPOPS_PIPELINE_BASE_DIR=.
 # kafka_brokers
 # The comma separated Kafka brokers address.
 KPOPS_KAFKA_BROKERS # No default value, required
-# defaults_filename_prefix
-# The name of the defaults file and the prefix of the defaults
-# environment file.
-KPOPS_DEFAULTS_FILENAME_PREFIX=defaults
 # topic_name_config.default_output_topic_name
 # Configures the value for the variable ${output_topic_name}
 KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME=${pipeline.name}-${component.name}

diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md
@@ -5,7 +5,6 @@ These variables take precedence over the settings in `config.yaml`. Variables ma
 |KPOPS_COMPONENTS_MODULE                           |                                        |False   |Custom Python module defining project-specific KPOps components                   |components_module                          |
 |KPOPS_PIPELINE_BASE_DIR                           |.                                       |False   |Base directory to the pipelines (default is current working directory)            |pipeline_base_dir                          |
 |KPOPS_KAFKA_BROKERS                               |                                        |True    |The comma separated Kafka brokers address.                                        |kafka_brokers                              |
-|KPOPS_DEFAULTS_FILENAME_PREFIX                    |defaults                                |False   |The name of the defaults file and the prefix of the defaults environment file.    |defaults_filename_prefix                   |
 |KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name}      |False   |Configures the value for the variable ${output_topic_name}                        |topic_name_config.default_output_topic_name|
 |KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False   |Configures the value for the variable ${error_topic_name}                         |topic_name_config.default_error_topic_name |
 |KPOPS_SCHEMA_REGISTRY__ENABLED                    |False                                   |False   |Whether the Schema Registry handler should be initialized.                        |schema_registry.enabled                    |

diff --git a/docs/docs/resources/variables/variable_substitution.yaml b/docs/docs/resources/variables/variable_substitution.yaml
@@ -4,8 +4,6 @@
       app_type: "${component.type}"
       app_name: "${component.name}"
       app_schedule: "${component.app.schedule}"
-      helm_release_name: ${component.helm_release_name}
-      helm_name_override: ${component.helm_name_override}
     commandLine:
       FAKE_ARG: "fake-arg-value"
     schedule: "30 3/8 * * *"

diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json
@@ -196,12 +196,6 @@
             "title": "Create Namespace",
             "type": "boolean"
         },
-        "defaults_filename_prefix": {
-            "default": "defaults",
-            "description": "The name of the defaults file and the prefix of the defaults environment file.",
-            "title": "Defaults Filename Prefix",
-            "type": "string"
-        },
         "helm_config": {
             "allOf": [
                 {

diff --git a/docs/docs/user/core-concepts/defaults.md b/docs/docs/user/core-concepts/defaults.md
@@ -44,13 +44,6 @@ It is important to note that `defaults_{environment}.yaml` overrides only the se
 
 <!-- dprint-ignore-end -->
 
-<!-- dprint-ignore-start -->
-
-!!! tip
-    `defaults` is the default value of `defaults_filename_prefix`.
-
-<!-- dprint-ignore-end -->
-
 ## Components
 
 <!-- When possible, automatically generate a list of all component-specific settings under each component. -->

diff --git a/docs/docs/user/migration-guide/v5-v6.md b/docs/docs/user/migration-guide/v5-v6.md
@@ -0,0 +1,16 @@
+# Migrate from V5 to V6
+
+## [Deploy multiple pipelines](https://github.com/bakdata/kpops/pull/487)
+
+KPOps can now deploy multiple pipelines in a single command. It is possible to pass one or many pipeline.yaml files or pass a directory with many pipeline.yaml files within it.
+
+The environment variable `KPOPS_PIPELINE_PATH` is changed to `KPOPS_PIPELINE_PATHS`.
+
+Read more:
+
+- [CLI Usage](https://bakdata.github.io/kpops/6.0/user/references/cli-commands/)
+- [Environment variables](https://bakdata.github.io/kpops/6.0/user/core-concepts/variables/environment_variables/)
+
+## [Separate KPOps API from the CLI](https://github.com/bakdata/kpops/pull/489)
+
+KPops Python API is now stable and separated from the CLI! 🎉
diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md
@@ -31,12 +31,12 @@ Clean pipeline steps
 **Usage**:
 
 ```console
-$ kpops clean [OPTIONS] PIPELINE_PATH
+$ kpops clean [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
@@ -57,12 +57,12 @@ Deploy pipeline steps
 **Usage**:
 
 ```console
-$ kpops deploy [OPTIONS] PIPELINE_PATH
+$ kpops deploy [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
@@ -83,12 +83,12 @@ Destroy pipeline steps
 **Usage**:
 
 ```console
-$ kpops destroy [OPTIONS] PIPELINE_PATH
+$ kpops destroy [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
@@ -109,18 +109,17 @@ Enrich pipeline steps with defaults. The enriched pipeline is used for all KPOps
 **Usage**:
 
 ```console
-$ kpops generate [OPTIONS] PIPELINE_PATH
+$ kpops generate [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
 * `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.  [env var: KPOPS_DOTENV_PATH]
 * `--config DIRECTORY`: Path to the dir containing config.yaml files  [env var: KPOPS_CONFIG_PATH; default: .]
-* `--output / --no-output`: Enable output printing  [default: output]
 * `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: FilterType.INCLUDE]
 * `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).   [env var: KPOPS_ENVIRONMENT]
@@ -153,18 +152,17 @@ In addition to generate, render final resource representation for each pipeline
 **Usage**:
 
 ```console
-$ kpops manifest [OPTIONS] PIPELINE_PATH
+$ kpops manifest [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 
 * `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.  [env var: KPOPS_DOTENV_PATH]
 * `--config DIRECTORY`: Path to the dir containing config.yaml files  [env var: KPOPS_CONFIG_PATH; default: .]
-* `--output / --no-output`: Enable output printing  [default: output]
 * `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
 * `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: FilterType.INCLUDE]
 * `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).   [env var: KPOPS_ENVIRONMENT]
@@ -178,12 +176,12 @@ Reset pipeline steps
 **Usage**:
 
 ```console
-$ kpops reset [OPTIONS] PIPELINE_PATH
+$ kpops reset [OPTIONS] PIPELINE_PATHS...
 ```
 
 **Arguments**:
 
-* `PIPELINE_PATH`: Path to YAML with pipeline definition  [env var: KPOPS_PIPELINE_PATH;required]
+* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
 
 **Options**:
 

diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -113,6 +113,7 @@ nav:
               - Migrate from v2 to v3: user/migration-guide/v2-v3.md
               - Migrate from v3 to v4: user/migration-guide/v3-v4.md
               - Migrate from v4 to v5: user/migration-guide/v4-v5.md
+              - Migrate from v5 to v6: user/migration-guide/v5-v6.md
           - CLI usage: user/references/cli-commands.md
           - Editor integration: user/references/editor-integration.md
           - CI integration:

diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py
@@ -8,7 +8,7 @@
 import yaml
 
 from hooks import ROOT
-from kpops.cli.registry import _find_classes
+from kpops.api.registry import _find_classes
 from kpops.components import KafkaConnector, PipelineComponent
 from kpops.utils.colorify import redify, yellowify
 from kpops.utils.pydantic import issubclass_patched

diff --git a/hooks/gen_schema.py b/hooks/gen_schema.py
@@ -5,8 +5,8 @@
 from pathlib import Path
 
 from hooks import ROOT
+from kpops.api.file_type import KpopsFileType
 from kpops.utils.gen_schema import (
-    SchemaScope,
     gen_config_schema,
     gen_defaults_schema,
     gen_pipeline_schema,
@@ -15,7 +15,7 @@
 PATH_TO_SCHEMA = ROOT / "docs/docs/schema"
 
 
-def gen_schema(scope: SchemaScope):
+def gen_schema(scope: KpopsFileType):
     """Generate the specified schema and save it to a file.
 
     The file is located in docs/docs/schema and is named ``<scope.value>.json``
@@ -24,16 +24,16 @@ def gen_schema(scope: SchemaScope):
     """
     with redirect_stdout(StringIO()) as f:
         match scope:
-            case SchemaScope.PIPELINE:
+            case KpopsFileType.PIPELINE:
                 gen_pipeline_schema()
-            case SchemaScope.DEFAULTS:
+            case KpopsFileType.DEFAULTS:
                 gen_defaults_schema()
-            case SchemaScope.CONFIG:
+            case KpopsFileType.CONFIG:
                 gen_config_schema()
         Path(PATH_TO_SCHEMA / f"{scope.value}.json").write_text(f.getvalue())
 
 
 if __name__ == "__main__":
-    gen_schema(SchemaScope.PIPELINE)
-    gen_schema(SchemaScope.DEFAULTS)
-    gen_schema(SchemaScope.CONFIG)
+    gen_schema(KpopsFileType.PIPELINE)
+    gen_schema(KpopsFileType.DEFAULTS)
+    gen_schema(KpopsFileType.CONFIG)
diff --git a/kpops/__init__.py b/kpops/__init__.py
@@ -1,7 +1,7 @@
 __version__ = "5.1.1"
 
 # export public API functions
-from kpops.cli.main import clean, deploy, destroy, generate, init, manifest, reset
+from kpops.api import clean, deploy, destroy, generate, init, manifest, reset
 
 __all__ = (
     "generate",