From fbdebf4c5e2a101ee9e84e86bb51aa04b6306de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Ra=C4=8Dinsk=C3=BD?= Date: Mon, 26 Aug 2024 16:46:41 +0200 Subject: [PATCH 1/2] renamed plugins to cli reference added docs about the stack traces updated some dependencies explicit encoding, should prevent weird windows bugs --- docs/troubleshooting.md | 89 +++++++++++++++++++ mkdocs.yaml | 3 +- poetry.lock | 32 +++---- pyproject.toml | 5 +- .../pipelines/dataproc_simple/docker/pi.py | 2 +- .../kubeflow/upload_model_version.py | 2 +- src/wanna/components/loader.py | 2 +- src/wanna/core/services/docker.py | 10 +-- src/wanna/core/utils/config_loader.py | 2 +- src/wanna/core/utils/loaders.py | 2 +- src/wanna/core/utils/templates.py | 2 +- 11 files changed, 121 insertions(+), 30 deletions(-) create mode 100644 docs/troubleshooting.md diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..e52e143d --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,89 @@ +--- +title: Troubleshooting +summary: Troubleshooting and debugging +authors: + - Matěj Račinský +date: 2024-08-26 +--- + +# Troubleshooting + +## Stack traces +Wanna-ml CLI interface uses [typer](https://typer.tiangolo.com/) package and +[rich](https://rich.readthedocs.io/en/latest/) for showing help, stack traces etc. + +By default, the wanna-ml will show verbose stack trace containing all local variables which can simplify +the development, but can be too verbose sometimes, +see [the docs](https://typer.tiangolo.com/tutorial/exceptions/#exceptions-with-rich) for more details. + +The stack trace looks something like this: + +``` +│ │ timeout = None │ │ +│ │ transcoded_request = { │ │ +│ │ │ 'uri': '/compute/v1/projects/your-gcp-project-id/regions', │ │ +│ │ │ 'query_params': , │ │ +│ │ │ 'method': 'get' │ │ +│ │ } │ │ +│ │ uri = '/compute/v1/projects/your-gcp-project-id/regions' │ │ +│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +NotFound: 404 GET https://compute.googleapis.com/compute/v1/projects/your-gcp-project-id/regions: The resource 'projects/your-gcp-project-id' was not found +``` + +If you don't like this tabular stack trace, you can disable this behavior by setting environment variable + +```shell +export _TYPER_STANDARD_TRACEBACK=1 +``` +in shell, or + +```powershell +$Env:_TYPER_STANDARD_TRACEBACK=1 +``` +in powershell. Then, the regular stack trace will be shown, like this: +``` + File "C:\Projects\others\wanna-ml\src\wanna\core\utils\validators.py", line 29, in validate_region + available_regions = get_available_regions(project_id=values.get("project_id")) + File "C:\Projects\others\wanna-ml\src\wanna\core\utils\gcp.py", line 228, in get_available_regions + response = RegionsClient(credentials=get_credentials()).list(project=project_id) + File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\cloud\compute_v1\services\regions\client.py", line 874, in list + response = rpc( + File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\api_core\gapic_v1\method.py", line 131, in __call__ + return wrapped_func(*args, **kwargs) + File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\api_core\grpc_helpers.py", line 76, in error_remapped_callable + return callable_(*args, **kwargs) + File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\cloud\compute_v1\services\regions\transports\rest.py", line 392, in __call__ + raise core_exceptions.from_http_response(response) +google.api_core.exceptions.NotFound: 404 GET https://compute.googleapis.com/compute/v1/projects/your-gcp-project-id/regions: The resource 'projects/your-gcp-project-id' was not found +``` + +If you like the verbosity of the tabular stack trace, but it's too narrow, you can increase the width to an arbitrary number by setting the `COLUMNS` or `TERMINAL_WIDTH` environment variable, e.g.: + +```shell +export COLUMNS=150 +export TERMINAL_WIDTH=150 +``` +in shell, or +```powershell +$Env:COLUMNS=150 +$Env:TERMINAL_WIDTH=150 +``` + +Then, the stack trace will look like this: + +``` +│ │ self = _List( │ │ +│ │ │ _session=, │ │ +│ │ │ _host='https://compute.googleapis.com', │ │ +│ │ │ _interceptor= │ │ +│ │ ) │ │ +│ │ timeout = None │ │ +│ │ transcoded_request = {'uri': '/compute/v1/projects/your-gcp-project-id/regions', 'query_params': , 'method': 'get'} │ │ +│ │ uri = '/compute/v1/projects/your-gcp-project-id/regions' │ │ +│ ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +NotFound: 404 GET https://compute.googleapis.com/compute/v1/projects/your-gcp-project-id/regions: The resource 'projects/your-gcp-project-id' was not +found +``` diff --git a/mkdocs.yaml b/mkdocs.yaml index 75bc10c4..77e1f498 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -53,6 +53,7 @@ theme: nav: - Overview: 'index.md' - Installation: 'installation.md' + - Troubleshooting: 'troubleshooting.md' - Tutorial: - 'Get Started': 'tutorial/index.md' - 'WANNA Project': 'tutorial/project.md' @@ -63,7 +64,7 @@ nav: - 'WANNA Managed Notebook': 'tutorial/managed-notebook.md' - 'WANNA Job': 'tutorial/job.md' - 'WANNA Pipeline': 'tutorial/pipeline.md' - - Plugins: 'cli/commands.md' + - CLI Reference: 'cli/commands.md' plugins: - search diff --git a/poetry.lock b/poetry.lock index 3ad9063b..0fc0896d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "arrow" @@ -425,17 +425,17 @@ pytz = "*" [[package]] name = "dirhash" -version = "0.4.0" +version = "0.5.0" description = "Python module and CLI for hashing of file system directories." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "dirhash-0.4.0-py3-none-any.whl", hash = "sha256:faa77de66a9674b06d9009f248056f25e1ae52ec36997ddb91e07bbde99c69be"}, - {file = "dirhash-0.4.0.tar.gz", hash = "sha256:264120928ff712f8cdf7ccdbe797f3a78fa78d6a17d5ee88d757ff06d335cd31"}, + {file = "dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09"}, + {file = "dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386"}, ] [package.dependencies] -scantree = "*" +scantree = ">=0.0.4" [[package]] name = "distlib" @@ -2715,18 +2715,18 @@ files = [ [[package]] name = "scantree" -version = "0.0.1" +version = "0.0.4" description = "Flexible recursive directory iterator: scandir meets glob(\"**\", recursive=True)" optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "scantree-0.0.1.tar.gz", hash = "sha256:2a8b163de0e4b2f9e4f37f8caf3f0b265172bbf174111e1bebc7955581895b39"}, + {file = "scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174"}, + {file = "scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac"}, ] [package.dependencies] attrs = ">=18.0.0" -pathspec = ">=0.5.9" -six = ">=1.12.0" +pathspec = ">=0.10.1" [[package]] name = "scikit-learn" @@ -3054,13 +3054,13 @@ six = "*" [[package]] name = "typer" -version = "0.12.3" +version = "0.12.5" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" files = [ - {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, - {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, + {file = "typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b"}, + {file = "typer-0.12.5.tar.gz", hash = "sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722"}, ] [package.dependencies] @@ -3357,6 +3357,8 @@ python-versions = ">=3.8" files = [ {file = "xgboost-2.1.0-py3-none-macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64.whl", hash = "sha256:19d145eb847b070c32342b1bf2d7331c102783e07a484f8b13b7d759d707c6b0"}, {file = "xgboost-2.1.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:840a0c6e2119d8c8f260a5dace996ea064a267f62b301a25d7d452488a7ac860"}, + {file = "xgboost-2.1.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:73673c9bb85927db7fe2e3aed6df6d35dba708cfd6767cc63d4ea11dda2dede5"}, + {file = "xgboost-2.1.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:43b16205689249d7509daf7a6ab00ad0e6c570b3a9c263cb32b26e39d9477bb3"}, {file = "xgboost-2.1.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:cedc2e386e686795735448fd4597533acacc5ba6fb47dd910c204c468b80bb96"}, {file = "xgboost-2.1.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:b2a456eb0f3d3e8fd8ab37e44ac288292bf8ea8744c294be9fd88713d27af810"}, {file = "xgboost-2.1.0-py3-none-win_amd64.whl", hash = "sha256:74904b91c42524a6c32147fe5718569e78fb65911ff4499b053f81d0964514d4"}, @@ -3394,4 +3396,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.12.0" -content-hash = "ab5743c3bffb1da163a3850fe26400967f169dc9043119748fd3148c6fb92143" +content-hash = "1f8f86565c944c1ac5d044901d25072908898b98608fda503046794d12afe318" diff --git a/pyproject.toml b/pyproject.toml index d32f2924..c44d5a8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ gcloud-config-helper = "^0.3.1" case-converter = "^1.1.0" cookiecutter = "^2.6.0" cron-validator = "^1.0.8" -dirhash = "^0.4.0" +dirhash = "^0.5.0" email-validator = "^2.2.0" emoji = "^2.12.1" gitpython = "^3.1.43" @@ -58,13 +58,12 @@ pyyaml-include = "^1.3.2" PyYAML = "^6.0.1" smart-open = {extras = ["gcs"], version = "^7.0.4"} treelib = "^1.7.0" -typer = "0.12.3" +typer = "0.12.5" waiting = "^1.4.1" rich = "^13.7.1" scandir = [ {version = "^1.10.0", platform="win32" } ] -scantree = "0.0.1" pendulum = "^2.1.2" igittigitt = "^2.1.4" diff --git a/samples/pipelines/dataproc_simple/docker/pi.py b/samples/pipelines/dataproc_simple/docker/pi.py index 2ab94a2b..7b72a2c3 100644 --- a/samples/pipelines/dataproc_simple/docker/pi.py +++ b/samples/pipelines/dataproc_simple/docker/pi.py @@ -25,7 +25,7 @@ def kubeflow_output_dump(path: Union[Path, str], content: str): print(f"dumping output values to {path}") - with open(path, "w") as f: + with open(path, "w", encoding="utf-8") as f: f.write(content) diff --git a/src/wanna/components/kubeflow/upload_model_version.py b/src/wanna/components/kubeflow/upload_model_version.py index 6d08550c..afaf4e46 100644 --- a/src/wanna/components/kubeflow/upload_model_version.py +++ b/src/wanna/components/kubeflow/upload_model_version.py @@ -91,5 +91,5 @@ def upload_model_version( model.uri = vertex_uri_prefix + new_model_resource_name model.metadata = {"resourceName": new_model_resource_name} - with open(model_output_path, "w") as output_file: + with open(model_output_path, "w", encoding="utf-8") as output_file: output_file.write(json.dumps(model.__dict__)) diff --git a/src/wanna/components/loader.py b/src/wanna/components/loader.py index e8808d9b..b8b29303 100644 --- a/src/wanna/components/loader.py +++ b/src/wanna/components/loader.py @@ -7,7 +7,7 @@ def load_wanna_component(path: Union[Path, str]): - with open(str(path), "r") as f: + with open(str(path), "r", encoding="utf-8") as f: t = Template(f.read()) component = t.safe_substitute(os.environ) return comp.load_component_from_text(component) diff --git a/src/wanna/core/services/docker.py b/src/wanna/core/services/docker.py index a6ad123b..629f49ab 100644 --- a/src/wanna/core/services/docker.py +++ b/src/wanna/core/services/docker.py @@ -125,7 +125,7 @@ def _read_build_config( """ if os.path.isfile(config_path): - with open(config_path) as file: + with open(config_path, encoding="utf-8") as file: # Load workflow file build_config_dict = loaders.load_yaml(file, self.work_dir) build_config = DockerBuildConfigModel.parse_obj(build_config_dict) @@ -158,7 +158,7 @@ def _get_ignore_patterns(context_dir: Path) -> List[str]: ignore = [] if docker_ignore.exists(): - with open(docker_ignore, "r") as f: + with open(docker_ignore, "r", encoding="utf-8") as f: lines = f.readlines() ignore += [ ignore.rstrip() @@ -396,7 +396,7 @@ def _should_skip_build( context_dir_hash_match = False if cache_file.exists(): - with open(cache_file, "r") as f: + with open(cache_file, "r", encoding="utf-8") as f: old_hash = f.read().replace("\n", "") context_dir_hash_match = old_hash == sha256hash @@ -421,7 +421,7 @@ def _write_context_dir_checksum( cache_file = self._get_cache_path(hash_cache_dir) sha256hash = self._get_dirhash(context_dir, ignore_patterns) - with open(cache_file, "w") as f: + with open(cache_file, "w", encoding="utf-8") as f: f.write(sha256hash) def _build_image_on_gcp_cloud_build( @@ -651,7 +651,7 @@ def _jinja_render_dockerfile( docker_file_path = build_dir / Path(f"{image_model.name}.Dockerfile") - with open(docker_file_path, "w") as file: + with open(docker_file_path, "w", encoding="utf-8") as file: file.write(rendered) return docker_file_path diff --git a/src/wanna/core/utils/config_loader.py b/src/wanna/core/utils/config_loader.py index fd0616e3..e51feb5e 100644 --- a/src/wanna/core/utils/config_loader.py +++ b/src/wanna/core/utils/config_loader.py @@ -62,7 +62,7 @@ def load_config_from_yaml( """ with logger.user_spinner("Reading and validating wanna yaml config"): - with open(wanna_config_path) as file: + with open(wanna_config_path, encoding="utf-8") as file: # Load workflow file wanna_dict = loaders.load_yaml( file, pathlib.Path(wanna_config_path).parent.resolve() diff --git a/src/wanna/core/utils/loaders.py b/src/wanna/core/utils/loaders.py index 4ca8ea93..2c2229d4 100644 --- a/src/wanna/core/utils/loaders.py +++ b/src/wanna/core/utils/loaders.py @@ -39,5 +39,5 @@ def load_yaml_path(path: Path, context_dir: Path, **extras: Any) -> Dict[Any, An """ Convert a Path into a yaml dict """ - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: return load_yaml(f, context_dir, **extras) diff --git a/src/wanna/core/utils/templates.py b/src/wanna/core/utils/templates.py index 1d73cbf2..541ca56b 100644 --- a/src/wanna/core/utils/templates.py +++ b/src/wanna/core/utils/templates.py @@ -8,6 +8,6 @@ def render_template(source_path: Path, **kwargs) -> str: templates_dir = Path(os.path.dirname(sys.modules["wanna.core"].__file__)) / "templates" # type: ignore source_path = templates_dir / source_path - with open(source_path) as f: + with open(source_path, encoding="utf-8") as f: template = Template(f.read()) return template.render(**kwargs) From ffde8e10e0af7940c579f140d320bfd3fdb43a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Ra=C4=8Dinsk=C3=BD?= Date: Tue, 27 Aug 2024 09:21:07 +0200 Subject: [PATCH 2/2] removed scandir, because it's not used --- poetry.lock | 22 +--------------------- pyproject.toml | 3 --- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0fc0896d..439b1256 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2693,26 +2693,6 @@ files = [ {file = "ruff-0.4.10.tar.gz", hash = "sha256:3aa4f2bc388a30d346c56524f7cacca85945ba124945fe489952aadb6b5cd804"}, ] -[[package]] -name = "scandir" -version = "1.10.0" -description = "scandir, a better directory iterator and faster os.walk()" -optional = false -python-versions = "*" -files = [ - {file = "scandir-1.10.0-cp27-cp27m-win32.whl", hash = "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188"}, - {file = "scandir-1.10.0-cp27-cp27m-win_amd64.whl", hash = "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac"}, - {file = "scandir-1.10.0-cp34-cp34m-win32.whl", hash = "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f"}, - {file = "scandir-1.10.0-cp34-cp34m-win_amd64.whl", hash = "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e"}, - {file = "scandir-1.10.0-cp35-cp35m-win32.whl", hash = "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f"}, - {file = "scandir-1.10.0-cp35-cp35m-win_amd64.whl", hash = "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32"}, - {file = "scandir-1.10.0-cp36-cp36m-win32.whl", hash = "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022"}, - {file = "scandir-1.10.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4"}, - {file = "scandir-1.10.0-cp37-cp37m-win32.whl", hash = "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173"}, - {file = "scandir-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d"}, - {file = "scandir-1.10.0.tar.gz", hash = "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae"}, -] - [[package]] name = "scantree" version = "0.0.4" @@ -3396,4 +3376,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.12.0" -content-hash = "1f8f86565c944c1ac5d044901d25072908898b98608fda503046794d12afe318" +content-hash = "ce63cecb4a38c4412715770119c1d3d56303487bb13a0d64fc60d79cb85d08f6" diff --git a/pyproject.toml b/pyproject.toml index c44d5a8c..995bb0b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,9 +61,6 @@ treelib = "^1.7.0" typer = "0.12.5" waiting = "^1.4.1" rich = "^13.7.1" -scandir = [ - {version = "^1.10.0", platform="win32" } -] pendulum = "^2.1.2" igittigitt = "^2.1.4"