From 34b1c6419e2a0572afed5c223e51598b794d606e Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 18 Mar 2024 14:54:28 +0000 Subject: [PATCH 01/16] Add python 3.12 support for airflow Signed-off-by: Ankita Katiyar --- .github/workflows/kedro-airflow.yml | 4 ++-- kedro-airflow/RELEASE.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml index b04801f12..87aa5b0df 100644 --- a/.github/workflows/kedro-airflow.yml +++ b/.github/workflows/kedro-airflow.yml @@ -22,7 +22,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, windows-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-airflow @@ -40,7 +40,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/e2e-tests.yml with: plugin: kedro-airflow diff --git a/kedro-airflow/RELEASE.md b/kedro-airflow/RELEASE.md index fe75f751b..45aaab8a9 100755 --- a/kedro-airflow/RELEASE.md +++ b/kedro-airflow/RELEASE.md @@ -2,6 +2,7 @@ * Sort DAGs to make sure `kedro airflow create` is deterministic. * Option to group MemoryDatasets in the same Airflow task (breaking change for custom template via `--jinja-file`). * Include the environment name in the DAG file name when different from the default. +* Added support for Python 3.12 # Release 0.8.0 * Added support for Kedro 0.19.x From 72765bab067e7aaace2e2cc2d85edec8f1f5c801 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 20 Mar 2024 15:40:34 +0000 Subject: [PATCH 02/16] temp: update gitpod Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 70738f4c0..72b6436a5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 +image: gitpod/workspace-python-3.12:2023-04-20-16-32-37 tasks: From 700bf582b3c099e751c39341937b296c688ab87d Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 20 Mar 2024 15:42:43 +0000 Subject: [PATCH 03/16] Update gitpod Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 72b6436a5..873db577c 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.12:2023-04-20-16-32-37 +image: gitpod/workspace-python-3.12:2023-10-16-15-49-4 tasks: From 2673d0a7e8b923fecd444c3330a240f54fa678d1 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 20 Mar 2024 15:44:02 +0000 Subject: [PATCH 04/16] Update gitpod Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 873db577c..f096a7014 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.12:2023-10-16-15-49-4 +image: gitpod/workspace-python-3.12:2023-10-16-15-49-40 tasks: From 8ffe5171dc31e455694004cf8ced9293fbd8fac8 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 25 Mar 2024 11:56:01 +0100 Subject: [PATCH 05/16] Resolve path Signed-off-by: Ankita Katiyar --- kedro-airflow/features/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-airflow/features/environment.py b/kedro-airflow/features/environment.py index 8f87afd7f..d2e30a7db 100644 --- a/kedro-airflow/features/environment.py +++ b/kedro-airflow/features/environment.py @@ -51,7 +51,7 @@ def call(cmd, print_output=False): call([context.python, "-m", "pip", "install", "-U", "pip", "pip-tools"]) call([context.pip, "install", ".[test]"]) - context.temp_dir = Path(tempfile.mkdtemp()) + context.temp_dir = Path(tempfile.mkdtemp()).resolve() def after_scenario(context, scenario): From 76f34697982742853f273846965007a8a3c6ae71 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 25 Mar 2024 12:21:22 +0100 Subject: [PATCH 06/16] remove pins Signed-off-by: Ankita Katiyar --- kedro-airflow/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index 2efd3611e..247a2f796 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -28,8 +28,6 @@ test = [ "bandit", "behave", "black~=22.0", - "connexion<3.0.0", # TODO: Temporary fix, connexion has changed their API, but airflow hasn't caught up yet - "Flask-Session<0.6", # TODO: Temporary pin, 0.6 breaks apache-airflow "kedro-datasets", "pre-commit>=2.9.2", "pytest", From 26b56f8d341e8aba59cd8abb832f95f2d7e1cfa2 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 25 Mar 2024 12:27:25 +0100 Subject: [PATCH 07/16] Try bump apache Signed-off-by: Ankita Katiyar --- kedro-airflow/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index 247a2f796..df4c13a5a 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -24,7 +24,7 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues" [project.optional-dependencies] test = [ - "apache-airflow<3.0", + "apache-airflow=~2.8", "bandit", "behave", "black~=22.0", From 39ca2bc4981effda7e2fdf1ccd35130edcb97bfb Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 25 Mar 2024 12:28:29 +0100 Subject: [PATCH 08/16] Try bump apache Signed-off-by: Ankita Katiyar --- kedro-airflow/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index df4c13a5a..5ecd5e9ac 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -24,7 +24,7 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues" [project.optional-dependencies] test = [ - "apache-airflow=~2.8", + "apache-airflow~=2.8", "bandit", "behave", "black~=22.0", From 129e65580df326641623c8d8de3c7a16ff005b93 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 25 Mar 2024 12:34:31 +0100 Subject: [PATCH 09/16] Revert airflow pin Signed-off-by: Ankita Katiyar --- kedro-airflow/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index 5ecd5e9ac..247a2f796 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -24,7 +24,7 @@ Tracker = "https://github.com/kedro-org/kedro-plugins/issues" [project.optional-dependencies] test = [ - "apache-airflow~=2.8", + "apache-airflow<3.0", "bandit", "behave", "black~=22.0", From a955fd8bcd27e41d5f96abf5ff36a6b8bcb7ccd5 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Apr 2024 15:10:23 +0100 Subject: [PATCH 10/16] Revert extra changes Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- kedro-airflow/features/environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index f096a7014..70738f4c0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.12:2023-10-16-15-49-40 +image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 tasks: diff --git a/kedro-airflow/features/environment.py b/kedro-airflow/features/environment.py index d2e30a7db..8f87afd7f 100644 --- a/kedro-airflow/features/environment.py +++ b/kedro-airflow/features/environment.py @@ -51,7 +51,7 @@ def call(cmd, print_output=False): call([context.python, "-m", "pip", "install", "-U", "pip", "pip-tools"]) call([context.pip, "install", ".[test]"]) - context.temp_dir = Path(tempfile.mkdtemp()).resolve() + context.temp_dir = Path(tempfile.mkdtemp()) def after_scenario(context, scenario): From fd210507598947dc21def1658c3049862cb64657 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Apr 2024 15:30:05 +0100 Subject: [PATCH 11/16] update install kedro-datasets step Signed-off-by: Ankita Katiyar --- kedro-airflow/features/steps/cli_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro-airflow/features/steps/cli_steps.py b/kedro-airflow/features/steps/cli_steps.py index 759e70f2d..6cacd8d77 100644 --- a/kedro-airflow/features/steps/cli_steps.py +++ b/kedro-airflow/features/steps/cli_steps.py @@ -20,7 +20,7 @@ def init_airflow(context, home_dir): def install_kedro(context, version): """Execute Kedro command and check the status.""" if version == "latest": - cmd = [context.pip, "install", "-U", "kedro-datasets[PANDAS]"] + cmd = [context.pip, "install", "-U", "kedro-datasets[pandas-csvdataset]"] else: cmd = [context.pip, "install", f"kedro-datasets[PANDAS]=={version}"] res = run(cmd, env=context.env) From 41121c6b2153c6f76cf2a157a4ba171e7777a03f Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Apr 2024 15:31:00 +0100 Subject: [PATCH 12/16] try remove install kedro-datasets step Signed-off-by: Ankita Katiyar --- kedro-airflow/features/airflow.feature | 2 -- 1 file changed, 2 deletions(-) diff --git a/kedro-airflow/features/airflow.feature b/kedro-airflow/features/airflow.feature index b6e61b7b5..ed98e9c1a 100644 --- a/kedro-airflow/features/airflow.feature +++ b/kedro-airflow/features/airflow.feature @@ -4,7 +4,6 @@ Feature: Airflow Given I have initialized Airflow with home dir "airflow" Scenario: Print a list of tasks with latest Kedro - Given I have installed kedro version "latest" And I have prepared a config file And I have run a non-interactive kedro new And I have executed the kedro command "airflow create -t ../airflow/dags/" @@ -15,7 +14,6 @@ Feature: Airflow And I should get a message including "predict" Scenario: Run Airflow task locally with latest Kedro - Given I have installed kedro version "latest" And I have prepared a config file And I have run a non-interactive kedro new And I have executed the kedro command "airflow create -t ../airflow/dags/" From 2beaed0078fc80dbf2ef165e0abe4e050c620100 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Apr 2024 15:39:14 +0100 Subject: [PATCH 13/16] Install kedro from main Signed-off-by: Ankita Katiyar --- kedro-airflow/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index 247a2f796..eb60eb189 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -28,6 +28,7 @@ test = [ "bandit", "behave", "black~=22.0", + "kedro @ git+https://github.com/kedro-org/kedro@main", "kedro-datasets", "pre-commit>=2.9.2", "pytest", From e3d636263ca6ae538b7abcd873ae7779c5d2befb Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Apr 2024 15:43:16 +0100 Subject: [PATCH 14/16] Revert changes Signed-off-by: Ankita Katiyar --- kedro-airflow/features/airflow.feature | 2 ++ kedro-airflow/features/steps/cli_steps.py | 2 +- kedro-airflow/pyproject.toml | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kedro-airflow/features/airflow.feature b/kedro-airflow/features/airflow.feature index ed98e9c1a..b6e61b7b5 100644 --- a/kedro-airflow/features/airflow.feature +++ b/kedro-airflow/features/airflow.feature @@ -4,6 +4,7 @@ Feature: Airflow Given I have initialized Airflow with home dir "airflow" Scenario: Print a list of tasks with latest Kedro + Given I have installed kedro version "latest" And I have prepared a config file And I have run a non-interactive kedro new And I have executed the kedro command "airflow create -t ../airflow/dags/" @@ -14,6 +15,7 @@ Feature: Airflow And I should get a message including "predict" Scenario: Run Airflow task locally with latest Kedro + Given I have installed kedro version "latest" And I have prepared a config file And I have run a non-interactive kedro new And I have executed the kedro command "airflow create -t ../airflow/dags/" diff --git a/kedro-airflow/features/steps/cli_steps.py b/kedro-airflow/features/steps/cli_steps.py index 6cacd8d77..759e70f2d 100644 --- a/kedro-airflow/features/steps/cli_steps.py +++ b/kedro-airflow/features/steps/cli_steps.py @@ -20,7 +20,7 @@ def init_airflow(context, home_dir): def install_kedro(context, version): """Execute Kedro command and check the status.""" if version == "latest": - cmd = [context.pip, "install", "-U", "kedro-datasets[pandas-csvdataset]"] + cmd = [context.pip, "install", "-U", "kedro-datasets[PANDAS]"] else: cmd = [context.pip, "install", f"kedro-datasets[PANDAS]=={version}"] res = run(cmd, env=context.env) diff --git a/kedro-airflow/pyproject.toml b/kedro-airflow/pyproject.toml index eb60eb189..247a2f796 100644 --- a/kedro-airflow/pyproject.toml +++ b/kedro-airflow/pyproject.toml @@ -28,7 +28,6 @@ test = [ "bandit", "behave", "black~=22.0", - "kedro @ git+https://github.com/kedro-org/kedro@main", "kedro-datasets", "pre-commit>=2.9.2", "pytest", From 434b73c7e2b7effcf53d8609152b5b40dd4d8a65 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 15 Apr 2024 09:06:35 +0100 Subject: [PATCH 15/16] Use 3.12 gitpod Signed-off-by: Ankita Katiyar --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 70738f4c0..f096a7014 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 +image: gitpod/workspace-python-3.12:2023-10-16-15-49-40 tasks: From 36610f6b321b895fdd5db3883f011c60c01515b3 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:11:37 +0100 Subject: [PATCH 16/16] Update .gitpod.yml Signed-off-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index f096a7014..70738f4c0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart -image: gitpod/workspace-python-3.12:2023-10-16-15-49-40 +image: gitpod/workspace-python-3.10:2023-04-20-16-32-37 tasks: