Add validate command (#480)

* Add command to detect invalid config files

* make

* Updated command description + improved output clarity

* updated the description

* Updated command to test more precisely

* Added validation to overrides

* Added non-zero exit code if error is found

* Fixed python scope bug

* Improved error messages

* Added smoke tests

* Switch pprint to print

* Moved validate_config to api for enhanced testing

* Moved validate config api command to config file

src/fixit/cli.py

@@ -14,7 +14,7 @@
 from fixit import __version__
 
 from .api import fixit_paths, print_result
-from .config import collect_rules, generate_config, parse_rule
+from .config import collect_rules, generate_config, parse_rule, validate_config
 from .ftypes import Config, LSPOptions, Options, OutputFormat, QualifiedRule, Tags
 from .rule import LintRule
 from .testing import generate_lint_rule_test_cases
@@ -349,3 +349,23 @@ def debug(ctx: click.Context, paths: Sequence[Path]) -> None:
             "disabled:",
             sorted(f"{rule()} ({reason})" for rule, reason in disabled.items()),
         )
+
+
+@main.command(name="validate-config")
+@click.pass_context
+@click.argument("path", nargs=1, type=click.Path(exists=True, path_type=Path))
+def validate_config_command(ctx: click.Context, path: Path) -> None:
+    """
+    validate the config provided
+    """
+    exceptions = validate_config(path)
+
+    try:
+        from rich import print as pprint
+    except ImportError:
+        from pprint import pprint  # type: ignore
+
+    if exceptions:
+        for e in exceptions:
+            pprint(e)
+        exit(-1)

src/fixit/config.py

@@ -592,3 +592,52 @@ def generate_config(
             config.output_template = options.output_template
 
     return config
+
+
+def validate_config(path: Path) -> List[str]:
+    """
+    Validate the config provided. The provided path is expected to be a valid toml
+    config file. Any exception found while parsing or importing will be added to a list
+    of exceptions that are returned.
+    """
+    exceptions: List[str] = []
+    try:
+        configs = read_configs([path])[0]
+
+        def validate_rules(rules: List[str], path: Path, context: str) -> None:
+            for rule in rules:
+                try:
+                    qualified_rule = parse_rule(rule, path, configs)
+                    try:
+                        for _ in find_rules(qualified_rule):
+                            pass
+                    except Exception as e:
+                        exceptions.append(
+                            f"Failed to import rule `{rule}` for {context}: {e.__class__.__name__}: {e}"
+                        )
+                except Exception as e:
+                    exceptions.append(
+                        f"Failed to parse rule `{rule}` for {context}: {e.__class__.__name__}: {e}"
+                    )
+
+        data = configs.data
+        validate_rules(data.get("enable", []), path, "global enable")
+        validate_rules(data.get("disable", []), path, "global disable")
+
+        for override in data.get("overrides", []):
+            override_path = Path(override.get("path", path))
+            validate_rules(
+                override.get("enable", []),
+                override_path,
+                f"override enable: `{override_path}`",
+            )
+            validate_rules(
+                override.get("disable", []),
+                override_path,
+                f"override disable: `{override_path}`",
+            )
+
+    except Exception as e:
+        exceptions.append(f"Invalid config: {e.__class__.__name__}: {e}")
+
+    return exceptions

src/fixit/tests/config.py

@@ -662,3 +662,72 @@ def test_format_output(self) -> None:
                 self.assertRegex(
                     result.output, r"file .*f_string\.py line \d+ rule UseFstring"
                 )
+
+    def test_validate_config(self) -> None:
+        with self.subTest("validate-config valid"):
+            with TemporaryDirectory() as td:
+                tdp = Path(td).resolve()
+                path = tdp / ".fixit.toml"
+                path.write_text(
+                    """
+                    [tool.fixit]
+                    disable = ["fixit.rules"]
+                    root = true
+                    """
+                )
+
+                results = config.validate_config(path)
+
+                self.assertEqual(results, [])
+
+        with self.subTest("validate-config invalid config"):
+            with TemporaryDirectory() as td:
+                tdp = Path(td).resolve()
+                path = tdp / ".fixit.toml"
+                path.write_text(
+                    """
+                    [tool.fixit]
+                    enable = ["fixit/rules:DeprecatedABCImport"]
+                    disable = ["fixit.rules"]
+                    root = true
+                    """
+                )
+
+                results = config.validate_config(path)
+
+                self.assertEqual(
+                    results,
+                    [
+                        "Failed to parse rule `fixit/rules:DeprecatedABCImport` for global enable: ConfigError: invalid rule name 'fixit/rules:DeprecatedABCImport'"
+                    ],
+                )
+
+        with self.subTest("validate-config multiple errors"):
+            with TemporaryDirectory() as td:
+                tdp = Path(td).resolve()
+                config_path = tdp / ".fixit.toml"
+                config_path.write_text(
+                    """
+                    [tool.fixit]
+                    enable = ["fixit/rules:DeprecatedABCImport"]
+                    disable = ["fixit.rules"]
+                    root = true
+
+                    [[tool.fixit.overrides]]
+                    path = "SUPER_REAL_PATH"
+                    enable = ["fixit.rules:DeprecatedABCImport_SUPER_REAL"]
+                    """
+                )
+
+                path = tdp / "file.py"
+                path.write_text("error")
+
+                results = config.validate_config(config_path)
+
+                self.assertEqual(
+                    results,
+                    [
+                        "Failed to parse rule `fixit/rules:DeprecatedABCImport` for global enable: ConfigError: invalid rule name 'fixit/rules:DeprecatedABCImport'",
+                        "Failed to import rule `fixit.rules:DeprecatedABCImport_SUPER_REAL` for override enable: `SUPER_REAL_PATH`: CollectionError: could not find rule fixit.rules:DeprecatedABCImport_SUPER_REAL",
+                    ],
+                )