Support command-line overrides

.pre-commit-config.yaml

@@ -18,6 +18,7 @@ repos:
           - "distlib~=0.3.5"
           - "pyproject-metadata~=0.7.1"
           - "tomli>=1.2.3,<3; python_version < '3.11'"
+          - "lark>=1.1.9,<2"
           - "click~=8.1.3"
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: "v18.1.2"

pyproject.toml

@@ -3,6 +3,7 @@ requires = [
     "distlib~=0.3.5",
     "pyproject-metadata~=0.7.1",
     "tomli>=1.2.3,<3; python_version < '3.11'",
+    "lark>=1.1.9,<2",
 ]
 build-backend = "py_build_cmake.build"
 backend-path = ["src"]
@@ -32,6 +33,7 @@ dependencies = [
     "distlib~=0.3.5",
     "pyproject-metadata~=0.7.1",
     "tomli>=1.2.3,<3; python_version < '3.11'",
+    "lark>=1.1.9,<2",
     "click~=8.1.3",
 ]
 dynamic = ["version", "description"]

src/py_build_cmake/config/cli_override.lark

@@ -0,0 +1,71 @@
+lines: _EOL* line* [_SOL]
+
+?line: [_SOL] option _EOL+
+
+?option: prepend
+       | prepend_path
+       | append
+       | append_path
+       | remove
+       | clear
+       | assign
+
+!?append:       keys OP_APPEND       value      -> full
+!?append_path:  keys OP_APPEND_PATH  value      -> full
+!?remove:       keys OP_REMOVE       value      -> full
+!?prepend:      keys OP_PREPEND      value      -> full
+!?prepend_path: keys OP_PREPEND_PATH value      -> full
+!?clear:        keys OP_CLEAR                   -> full
+!?assign:       keys OP_ASSIGN       value      -> full
+
+OP_APPEND.2:       "+="
+OP_APPEND_PATH.3:  "+=(path)"
+OP_PREPEND.2:      "=+"
+OP_PREPEND_PATH.3: "=+(path)"
+OP_REMOVE.2:       "-="
+OP_CLEAR.2:        "=!"
+OP_ASSIGN.1:       "="
+
+keys: key ("." key)* [_WS]
+
+?value     : object
+           | array
+           | NUMBER
+           | TRUE
+           | FALSE
+           | string
+
+?sub_value : object
+           | array
+           | NUMBER
+           | TRUE
+           | FALSE
+           | string_no_ws
+
+object   : "{" [pair ("," pair)*] "}"
+array    : "[" ( [_WS] sub_value [_WS] ("," [_WS] sub_value [_WS])* ["," [_WS]] )? "]"
+NUMBER.2 : /-?[0-9]+/
+TRUE.2   : "true" | "True"
+FALSE.2  : "false" | "False"
+pair     : [_WS] dict_key [_WS] "=" [_WS] sub_value [_WS]
+
+key          : ESCAPED_STRING               -> escaped_string
+             | UNQUOTED_KEY                 -> unquoted_string
+string       : ESCAPED_STRING               -> escaped_string
+             | UNQUOTED_STRING              -> unquoted_string
+string_no_ws : ESCAPED_STRING               -> escaped_string
+             | UNQUOTED_STRING_NO_WS        -> unquoted_string
+dict_key     : ESCAPED_STRING               -> escaped_string
+             | UNQUOTED_DICT_KEY            -> unquoted_string
+
+_WS: /[ \t\f\r\n]+/
+_SOL: /[ \t]+/
+_EOL: /[ \t\f\r]*\n/
+UNQUOTED_KEY:          /[a-zA-Z0-9_-]*[a-zA-Z0-9_]/
+UNQUOTED_STRING:       /[^"\[\]{}\r\n#]+/
+UNQUOTED_STRING_NO_WS: /[^"\[\]{},\r\n# \t\f]+/
+UNQUOTED_DICT_KEY:     /[^"\[\]{},\r\n# \t\f=]+/
+
+%import common.ESCAPED_STRING
+%import common.SH_COMMENT
+%ignore [_WS] SH_COMMENT

src/py_build_cmake/config/cli_override.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+from lark import Lark, Token, Transformer, v_args
+
+grammar = Path(__file__).with_suffix(".lark").read_text()
+
+
+@dataclass
+class CLIOption:
+    action: str
+    key: tuple[str, ...]
+    value: bool | str | list | dict | float | None
+
+
+class TreeToCLIOption(Transformer):
+    @v_args(inline=True)
+    def escaped_string(self, s: Token):
+        return s[1:-1].encode("raw_unicode_escape").decode("unicode_escape")
+
+    @v_args(inline=True)
+    def unquoted_string(self, s: Token):
+        return s.value
+
+    @v_args(inline=True)
+    def full(self, k, a, v=None):
+        return CLIOption(a.value, k, v)
+
+    lines = list
+    keys = tuple
+
+    array = list
+    pair = tuple
+    object = dict
+
+    NUMBER = int
+    TRUE = lambda self, _: True
+    FALSE = lambda self, _: False
+
+
+cli_parser = Lark(grammar, start="option", parser="lalr", transformer=TreeToCLIOption())
+file_parser = Lark(grammar, start="lines", parser="lalr", transformer=TreeToCLIOption())
+
+
+def parse_cli(s: str) -> CLIOption:
+    return cast(CLIOption, cli_parser.parse(s))
+
+
+def parse_file(s: str) -> list[CLIOption]:
+    return cast(list[CLIOption], file_parser.parse(s))

src/py_build_cmake/config/load.py

@@ -11,9 +11,11 @@
 
 import pyproject_metadata
 from distlib.util import normalize_name  # type: ignore[import-untyped]
+from lark import LarkError
 
 from .. import __version__
 from ..common import Config, ConfigError
+from .cli_override import CLIOption, parse_cli
 from .options.config_path import ConfPath
 from .options.config_reference import ConfigReference
 from .options.default import ConfigDefaulter
@@ -26,7 +28,7 @@
     get_options,
     get_tool_pbc_path,
 )
-from .options.value_reference import ValueReference
+from .options.value_reference import OverrideAction, OverrideActionEnum, ValueReference
 from .options.verify import ConfigVerifier
 from .quirks import config_quirks
 
@@ -44,8 +46,8 @@ def read_full_config(
     verbose: bool,
 ) -> Config:
     config_settings = config_settings or {}
-    overrides = parse_config_settings_overrides(config_settings, verbose)
-    cfg = read_config(pyproject_path, overrides)
+    overrides, cli_overrides = parse_config_settings_overrides(config_settings, verbose)
+    cfg = read_config(pyproject_path, overrides, cli_overrides)
     if verbose:
         print_config_verbose(cfg)
     return cfg
@@ -65,11 +67,26 @@ def get_as_list(key: str):
         return listify(config_settings.get(key) or [])
 
     keys = ["local", "cross"]
-    overrides = {key: get_as_list("--" + key) + get_as_list(key) for key in keys}
+    file_overrides = {key: get_as_list("--" + key) + get_as_list(key) for key in keys}
+    cli_overrides = (
+        get_as_list("-o")
+        + get_as_list("o")
+        + get_as_list("override")
+        + get_as_list("--override")
+    )
     if verbose:
-        print("Configuration settings for local and cross-compilation overrides:")
-        pprint(overrides)
-    return overrides
+        print("Configuration settings for local and cross-compilation file overrides:")
+        pprint(file_overrides)
+        print("Configuration settings command-line overrides:")
+        pprint(cli_overrides)
+    parsed_cli_overrides = []
+    for o in cli_overrides:
+        try:
+            parsed_cli_overrides.append(parse_cli(o))
+        except LarkError as e:
+            msg = f"Failed to parse command line override: {o}"
+            raise ConfigError(msg) from e
+    return file_overrides, parsed_cli_overrides
 
 
 def try_load_toml(path: Path):
@@ -87,7 +104,9 @@ def try_load_toml(path: Path):
 
 
 def read_config(
-    pyproject_path: str | Path, flag_overrides: dict[str, list[str]]
+    pyproject_path: str | Path,
+    flag_overrides: dict[str, list[str]],
+    cli_overrides: list[CLIOption],
 ) -> Config:
     # Load the pyproject.toml file
     pyproject_path = Path(pyproject_path)
@@ -109,14 +128,14 @@ def read_config(
     # File names mapping to the actual dict with the config
     config_files: dict[str, dict[str, Any]] = {
         "pyproject.toml": pyproject,
-        "<command-line>": {},  # FIXME: implement this
     }
 
     # Additional options for config_options
     overrides: dict[ConfPath, ConfPath] = {}
     # What to override
     extra_flag_paths = {"local": get_tool_pbc_path(), "cross": get_cross_path()}
 
+    # Files specified on the command line
     for flag, targetpath in extra_flag_paths.items():
         for path in map(Path, flag_overrides[flag]):
             if path.is_absolute():
@@ -128,9 +147,26 @@ def read_config(
                 config_files[fullpath.as_posix()] = config
                 overrides[ConfPath((fullpath.as_posix(),))] = targetpath
 
+    # Command-line overrides
+    for i, o in enumerate(cli_overrides):
+        overrides.update(add_cli_override(config_files, o, f"<cli:{i+1}>"))
+
     return process_config(pyproject_path, config_files, overrides)
 
 
+def add_cli_override(
+    config_files: dict[str, dict[str, Any]], opt: CLIOption, label: str
+):
+    overrides = {ConfPath.from_string(label): get_tool_pbc_path()}
+    o: dict = config_files.setdefault(label, {})
+    for k in opt.key[:-1]:
+        o = o.setdefault(k, {})
+    o[opt.key[-1]] = OverrideAction(
+        action=OverrideActionEnum(opt.action), values=opt.value
+    )
+    return overrides
+
+
 def process_config(
     pyproject_path: Path | PurePosixPath,
     config_files: dict[str, dict[str, Any]],

src/py_build_cmake/config/options/bool.py

@@ -1,6 +1,6 @@
 from ...common import ConfigError
 from .config_option import ConfigOption
-from .value_reference import ValueReference
+from .value_reference import OverrideActionEnum, ValueReference
 
 
 class BoolConfigOption(ConfigOption):
@@ -13,6 +13,10 @@ def override(self, old_value, new_value):
         return new_value.values
 
     def verify(self, values: ValueReference):
+        if values.action != OverrideActionEnum.Assign:
+            msg = f"Option {values.value_path} of type {self.get_typename()} "
+            msg += f"does not support operation {values.action.value}"
+            raise ConfigError(msg)
         if self.sub_options:
             msg = f"Type of {values.value_path} should be {bool}, "
             msg += f"not {dict}"

src/py_build_cmake/config/options/cmake_opt.py

@@ -7,7 +7,7 @@
 from ...common import ConfigError
 from .config_path import ConfPath
 from .dict import DictOfStrConfigOption
-from .value_reference import ValueReference
+from .value_reference import OverrideAction, OverrideActionEnum, ValueReference
 
 logger = logging.getLogger(__name__)
 
@@ -193,33 +193,52 @@ def override(self, old_value: ValueReference, new_value: ValueReference):
                 )
         return r
 
+    def _verify_cmake_list(self, x, pth: ConfPath):
+        if isinstance(x, list):
+            for v in x:
+                if not isinstance(v, str) and not isinstance(v, bool):
+                    msg = f"Type of values in list {pth} should be str or "
+                    msg += f"bool, not {type(v)}"
+                    raise ConfigError(msg)
+        elif not isinstance(x, str) and not isinstance(x, bool):
+            msg = f"Type of values in {pth} should be str or bool or a "
+            msg += f"list thereof, not {type(x)}"
+            raise ConfigError(msg)
+
+    def _verify_cmake_list_or_dict(self, x, pth: ConfPath):
+        if isinstance(x, dict):
+            for k, v in x.items():
+                if not isinstance(k, str):
+                    msg = f"Type of keys in {pth} should be str, "
+                    msg += f"not {type(k)}"
+                    raise ConfigError(msg)
+                self._verify_cmake_list(v, pth.join(k))
+        else:
+            self._verify_cmake_list(x, pth)
+
     def verify(self, values: ValueReference):
         """Checks the data types and keys of the values, and then converts
         them to a dictionary of CMakeOptions."""
 
-        def verify_cmake_list(x, pth: ConfPath):
-            if isinstance(x, list):
-                for v in x:
-                    if not isinstance(v, str) and not isinstance(v, bool):
-                        msg = f"Type of values in list {pth} should be str or "
-                        msg += f"bool, not {type(v)}"
-                        raise ConfigError(msg)
-            elif not isinstance(x, str) and not isinstance(x, bool):
-                msg = f"Type of values in {pth} should be str or bool or a "
-                msg += f"list thereof, not {type(x)}"
+        def convert_override_to_dict(x: OverrideAction, pth: ConfPath):
+            def raise_err():
+                msg = f"Option {pth} does not support operation "
+                msg += f"{x.action.value}"
                 raise ConfigError(msg)
 
-        def verify_cmake_list_or_dict(x, pth: ConfPath):
-            if isinstance(x, dict):
-                for k, v in x.items():
-                    if not isinstance(k, str):
-                        msg = f"Type of keys in {pth} should be str, "
-                        msg += f"not {type(k)}"
-                        raise ConfigError(msg)
-                    verify_cmake_list(v, pth.join(k))
-            else:
-                verify_cmake_list(x, pth)
+            return {
+                OverrideActionEnum.Assign: lambda: {"value": x.values},
+                OverrideActionEnum.Append: lambda: {"append": x.values},
+                OverrideActionEnum.Prepend: lambda: {"prepend": x.values},
+                OverrideActionEnum.Remove: lambda: {"-": x.values},
+                OverrideActionEnum.AppendPath: lambda: raise_err(),
+                OverrideActionEnum.PrependPath: lambda: raise_err(),
+            }[x.action]()
 
+        if values.action != OverrideActionEnum.Assign:
+            msg = f"Option {values.value_path} of type {self.get_typename()} "
+            msg += f"does not support operation {values.action.value}"
+            raise ConfigError(msg)
         if values.values is None:
             return None
         valdict = copy(values.values)
@@ -231,7 +250,10 @@ def verify_cmake_list_or_dict(x, pth: ConfPath):
             msg = f"Type of keys in {values.value_path} should be {str}"
             raise ConfigError(msg)
         for k, v in valdict.items():
-            verify_cmake_list_or_dict(v, values.value_path.join(k))
+            pth = values.value_path.join(k)
+            if isinstance(v, OverrideAction):
+                valdict[k] = convert_override_to_dict(v, pth)
+            self._verify_cmake_list_or_dict(valdict[k], pth)
 
         def check_dict_keys(d, pth):
             if not isinstance(d, dict):