From 04c124df44f4985005bf4bceb3897bd526f65552 Mon Sep 17 00:00:00 2001 From: Pieter Pas Date: Sun, 26 May 2024 15:30:27 +0200 Subject: [PATCH] Support command-line overrides --- .pre-commit-config.yaml | 1 + pyproject.toml | 2 + src/py_build_cmake/config/cli_override.lark | 71 +++++++++ src/py_build_cmake/config/cli_override.py | 53 +++++++ src/py_build_cmake/config/load.py | 54 +++++-- src/py_build_cmake/config/options/bool.py | 6 +- .../config/options/cmake_opt.py | 66 +++++--- .../config/options/config_option.py | 9 +- src/py_build_cmake/config/options/dict.py | 32 +++- src/py_build_cmake/config/options/enum.py | 6 +- src/py_build_cmake/config/options/int.py | 6 +- src/py_build_cmake/config/options/list.py | 52 +++++-- src/py_build_cmake/config/options/override.py | 26 ++-- src/py_build_cmake/config/options/path.py | 6 +- src/py_build_cmake/config/options/string.py | 11 +- .../config/options/value_reference.py | 59 ++++++- tests/test_cli_override.py | 69 ++++++++ tests/test_config_load.py | 147 +++++++++++++++++- tests/test_configoptions.py | 60 ++++++- 19 files changed, 657 insertions(+), 79 deletions(-) create mode 100644 src/py_build_cmake/config/cli_override.lark create mode 100644 src/py_build_cmake/config/cli_override.py create mode 100644 tests/test_cli_override.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e3191b..61661e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ repos: - "distlib~=0.3.5" - "pyproject-metadata~=0.7.1" - "tomli>=1.2.3,<3; python_version < '3.11'" + - "lark>=1.1.9,<2" - "click~=8.1.3" - repo: https://github.com/pre-commit/mirrors-clang-format rev: "v18.1.2" diff --git a/pyproject.toml b/pyproject.toml index 1baa29e..7708fdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ requires = [ "distlib~=0.3.5", "pyproject-metadata~=0.7.1", "tomli>=1.2.3,<3; python_version < '3.11'", + "lark>=1.1.9,<2", ] build-backend = "py_build_cmake.build" backend-path = ["src"] @@ -32,6 +33,7 @@ dependencies = [ "distlib~=0.3.5", "pyproject-metadata~=0.7.1", "tomli>=1.2.3,<3; python_version < '3.11'", + "lark>=1.1.9,<2", "click~=8.1.3", ] dynamic = ["version", "description"] diff --git a/src/py_build_cmake/config/cli_override.lark b/src/py_build_cmake/config/cli_override.lark new file mode 100644 index 0000000..a6fa193 --- /dev/null +++ b/src/py_build_cmake/config/cli_override.lark @@ -0,0 +1,71 @@ +lines: _EOL* line* [_SOL] + +?line: [_SOL] option _EOL+ + +?option: prepend + | prepend_path + | append + | append_path + | remove + | clear + | assign + +!?append: keys OP_APPEND value -> full +!?append_path: keys OP_APPEND_PATH value -> full +!?remove: keys OP_REMOVE value -> full +!?prepend: keys OP_PREPEND value -> full +!?prepend_path: keys OP_PREPEND_PATH value -> full +!?clear: keys OP_CLEAR -> full +!?assign: keys OP_ASSIGN value -> full + +OP_APPEND.2: "+=" +OP_APPEND_PATH.3: "+=(path)" +OP_PREPEND.2: "=+" +OP_PREPEND_PATH.3: "=+(path)" +OP_REMOVE.2: "-=" +OP_CLEAR.2: "=!" +OP_ASSIGN.1: "=" + +keys: key ("." key)* [_WS] + +?value : object + | array + | NUMBER + | TRUE + | FALSE + | string + +?sub_value : object + | array + | NUMBER + | TRUE + | FALSE + | string_no_ws + +object : "{" [pair ("," pair)*] "}" +array : "[" ( [_WS] sub_value [_WS] ("," [_WS] sub_value [_WS])* ["," [_WS]] )? "]" +NUMBER.2 : /-?[0-9]+/ +TRUE.2 : "true" | "True" +FALSE.2 : "false" | "False" +pair : [_WS] dict_key [_WS] "=" [_WS] sub_value [_WS] + +key : ESCAPED_STRING -> escaped_string + | UNQUOTED_KEY -> unquoted_string +string : ESCAPED_STRING -> escaped_string + | UNQUOTED_STRING -> unquoted_string +string_no_ws : ESCAPED_STRING -> escaped_string + | UNQUOTED_STRING_NO_WS -> unquoted_string +dict_key : ESCAPED_STRING -> escaped_string + | UNQUOTED_DICT_KEY -> unquoted_string + +_WS: /[ \t\f\r\n]+/ +_SOL: /[ \t]+/ +_EOL: /[ \t\f\r]*\n/ +UNQUOTED_KEY: /[a-zA-Z0-9_-]*[a-zA-Z0-9_]/ +UNQUOTED_STRING: /[^"\[\]{}\r\n#]+/ +UNQUOTED_STRING_NO_WS: /[^"\[\]{},\r\n# \t\f]+/ +UNQUOTED_DICT_KEY: /[^"\[\]{},\r\n# \t\f=]+/ + +%import common.ESCAPED_STRING +%import common.SH_COMMENT +%ignore [_WS] SH_COMMENT diff --git a/src/py_build_cmake/config/cli_override.py b/src/py_build_cmake/config/cli_override.py new file mode 100644 index 0000000..d5fad0c --- /dev/null +++ b/src/py_build_cmake/config/cli_override.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from lark import Lark, Token, Transformer, v_args + +grammar = Path(__file__).with_suffix(".lark").read_text() + + +@dataclass +class CLIOption: + action: str + key: tuple[str, ...] + value: bool | str | list | dict | float | None + + +class TreeToCLIOption(Transformer): + @v_args(inline=True) + def escaped_string(self, s: Token): + return s[1:-1].encode("raw_unicode_escape").decode("unicode_escape") + + @v_args(inline=True) + def unquoted_string(self, s: Token): + return s.value + + @v_args(inline=True) + def full(self, k, a, v=None): + return CLIOption(a.value, k, v) + + lines = list + keys = tuple + + array = list + pair = tuple + object = dict + + NUMBER = int + TRUE = lambda self, _: True + FALSE = lambda self, _: False + + +cli_parser = Lark(grammar, start="option", parser="lalr", transformer=TreeToCLIOption()) +file_parser = Lark(grammar, start="lines", parser="lalr", transformer=TreeToCLIOption()) + + +def parse_cli(s: str) -> CLIOption: + return cast(CLIOption, cli_parser.parse(s)) + + +def parse_file(s: str) -> list[CLIOption]: + return cast(list[CLIOption], file_parser.parse(s)) diff --git a/src/py_build_cmake/config/load.py b/src/py_build_cmake/config/load.py index 6cb89f0..7bc68ea 100644 --- a/src/py_build_cmake/config/load.py +++ b/src/py_build_cmake/config/load.py @@ -11,9 +11,11 @@ import pyproject_metadata from distlib.util import normalize_name # type: ignore[import-untyped] +from lark import LarkError from .. import __version__ from ..common import Config, ConfigError +from .cli_override import CLIOption, parse_cli from .options.config_path import ConfPath from .options.config_reference import ConfigReference from .options.default import ConfigDefaulter @@ -26,7 +28,7 @@ get_options, get_tool_pbc_path, ) -from .options.value_reference import ValueReference +from .options.value_reference import OverrideAction, OverrideActionEnum, ValueReference from .options.verify import ConfigVerifier from .quirks import config_quirks @@ -44,8 +46,8 @@ def read_full_config( verbose: bool, ) -> Config: config_settings = config_settings or {} - overrides = parse_config_settings_overrides(config_settings, verbose) - cfg = read_config(pyproject_path, overrides) + overrides, cli_overrides = parse_config_settings_overrides(config_settings, verbose) + cfg = read_config(pyproject_path, overrides, cli_overrides) if verbose: print_config_verbose(cfg) return cfg @@ -65,11 +67,26 @@ def get_as_list(key: str): return listify(config_settings.get(key) or []) keys = ["local", "cross"] - overrides = {key: get_as_list("--" + key) + get_as_list(key) for key in keys} + file_overrides = {key: get_as_list("--" + key) + get_as_list(key) for key in keys} + cli_overrides = ( + get_as_list("-o") + + get_as_list("o") + + get_as_list("override") + + get_as_list("--override") + ) if verbose: - print("Configuration settings for local and cross-compilation overrides:") - pprint(overrides) - return overrides + print("Configuration settings for local and cross-compilation file overrides:") + pprint(file_overrides) + print("Configuration settings command-line overrides:") + pprint(cli_overrides) + parsed_cli_overrides = [] + for o in cli_overrides: + try: + parsed_cli_overrides.append(parse_cli(o)) + except LarkError as e: + msg = f"Failed to parse command line override: {o}" + raise ConfigError(msg) from e + return file_overrides, parsed_cli_overrides def try_load_toml(path: Path): @@ -87,7 +104,9 @@ def try_load_toml(path: Path): def read_config( - pyproject_path: str | Path, flag_overrides: dict[str, list[str]] + pyproject_path: str | Path, + flag_overrides: dict[str, list[str]], + cli_overrides: list[CLIOption], ) -> Config: # Load the pyproject.toml file pyproject_path = Path(pyproject_path) @@ -109,7 +128,6 @@ def read_config( # File names mapping to the actual dict with the config config_files: dict[str, dict[str, Any]] = { "pyproject.toml": pyproject, - "": {}, # FIXME: implement this } # Additional options for config_options @@ -117,6 +135,7 @@ def read_config( # What to override extra_flag_paths = {"local": get_tool_pbc_path(), "cross": get_cross_path()} + # Files specified on the command line for flag, targetpath in extra_flag_paths.items(): for path in map(Path, flag_overrides[flag]): if path.is_absolute(): @@ -128,9 +147,26 @@ def read_config( config_files[fullpath.as_posix()] = config overrides[ConfPath((fullpath.as_posix(),))] = targetpath + # Command-line overrides + for i, o in enumerate(cli_overrides): + overrides.update(add_cli_override(config_files, o, f"")) + return process_config(pyproject_path, config_files, overrides) +def add_cli_override( + config_files: dict[str, dict[str, Any]], opt: CLIOption, label: str +): + overrides = {ConfPath.from_string(label): get_tool_pbc_path()} + o: dict = config_files.setdefault(label, {}) + for k in opt.key[:-1]: + o = o.setdefault(k, {}) + o[opt.key[-1]] = OverrideAction( + action=OverrideActionEnum(opt.action), values=opt.value + ) + return overrides + + def process_config( pyproject_path: Path | PurePosixPath, config_files: dict[str, dict[str, Any]], diff --git a/src/py_build_cmake/config/options/bool.py b/src/py_build_cmake/config/options/bool.py index 235bb7b..9ce40d1 100644 --- a/src/py_build_cmake/config/options/bool.py +++ b/src/py_build_cmake/config/options/bool.py @@ -1,6 +1,6 @@ from ...common import ConfigError from .config_option import ConfigOption -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference class BoolConfigOption(ConfigOption): @@ -13,6 +13,10 @@ def override(self, old_value, new_value): return new_value.values def verify(self, values: ValueReference): + if values.action != OverrideActionEnum.Assign: + msg = f"Option {values.value_path} of type {self.get_typename()} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) if self.sub_options: msg = f"Type of {values.value_path} should be {bool}, " msg += f"not {dict}" diff --git a/src/py_build_cmake/config/options/cmake_opt.py b/src/py_build_cmake/config/options/cmake_opt.py index 0f7e7e8..f8d5211 100644 --- a/src/py_build_cmake/config/options/cmake_opt.py +++ b/src/py_build_cmake/config/options/cmake_opt.py @@ -7,7 +7,7 @@ from ...common import ConfigError from .config_path import ConfPath from .dict import DictOfStrConfigOption -from .value_reference import ValueReference +from .value_reference import OverrideAction, OverrideActionEnum, ValueReference logger = logging.getLogger(__name__) @@ -193,33 +193,52 @@ def override(self, old_value: ValueReference, new_value: ValueReference): ) return r + def _verify_cmake_list(self, x, pth: ConfPath): + if isinstance(x, list): + for v in x: + if not isinstance(v, str) and not isinstance(v, bool): + msg = f"Type of values in list {pth} should be str or " + msg += f"bool, not {type(v)}" + raise ConfigError(msg) + elif not isinstance(x, str) and not isinstance(x, bool): + msg = f"Type of values in {pth} should be str or bool or a " + msg += f"list thereof, not {type(x)}" + raise ConfigError(msg) + + def _verify_cmake_list_or_dict(self, x, pth: ConfPath): + if isinstance(x, dict): + for k, v in x.items(): + if not isinstance(k, str): + msg = f"Type of keys in {pth} should be str, " + msg += f"not {type(k)}" + raise ConfigError(msg) + self._verify_cmake_list(v, pth.join(k)) + else: + self._verify_cmake_list(x, pth) + def verify(self, values: ValueReference): """Checks the data types and keys of the values, and then converts them to a dictionary of CMakeOptions.""" - def verify_cmake_list(x, pth: ConfPath): - if isinstance(x, list): - for v in x: - if not isinstance(v, str) and not isinstance(v, bool): - msg = f"Type of values in list {pth} should be str or " - msg += f"bool, not {type(v)}" - raise ConfigError(msg) - elif not isinstance(x, str) and not isinstance(x, bool): - msg = f"Type of values in {pth} should be str or bool or a " - msg += f"list thereof, not {type(x)}" + def convert_override_to_dict(x: OverrideAction, pth: ConfPath): + def raise_err(): + msg = f"Option {pth} does not support operation " + msg += f"{x.action.value}" raise ConfigError(msg) - def verify_cmake_list_or_dict(x, pth: ConfPath): - if isinstance(x, dict): - for k, v in x.items(): - if not isinstance(k, str): - msg = f"Type of keys in {pth} should be str, " - msg += f"not {type(k)}" - raise ConfigError(msg) - verify_cmake_list(v, pth.join(k)) - else: - verify_cmake_list(x, pth) + return { + OverrideActionEnum.Assign: lambda: {"value": x.values}, + OverrideActionEnum.Append: lambda: {"append": x.values}, + OverrideActionEnum.Prepend: lambda: {"prepend": x.values}, + OverrideActionEnum.Remove: lambda: {"-": x.values}, + OverrideActionEnum.AppendPath: lambda: raise_err(), + OverrideActionEnum.PrependPath: lambda: raise_err(), + }[x.action]() + if values.action != OverrideActionEnum.Assign: + msg = f"Option {values.value_path} of type {self.get_typename()} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) if values.values is None: return None valdict = copy(values.values) @@ -231,7 +250,10 @@ def verify_cmake_list_or_dict(x, pth: ConfPath): msg = f"Type of keys in {values.value_path} should be {str}" raise ConfigError(msg) for k, v in valdict.items(): - verify_cmake_list_or_dict(v, values.value_path.join(k)) + pth = values.value_path.join(k) + if isinstance(v, OverrideAction): + valdict[k] = convert_override_to_dict(v, pth) + self._verify_cmake_list_or_dict(valdict[k], pth) def check_dict_keys(d, pth): if not isinstance(d, dict): diff --git a/src/py_build_cmake/config/options/config_option.py b/src/py_build_cmake/config/options/config_option.py index e3b1d9a..c0f3b39 100644 --- a/src/py_build_cmake/config/options/config_option.py +++ b/src/py_build_cmake/config/options/config_option.py @@ -7,7 +7,7 @@ from ...common import ConfigError from .config_path import ConfPath from .default import DefaultValue, NoDefaultValue -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference class ConfigOption: @@ -46,6 +46,10 @@ def get_typename(self, md: bool = False) -> str | None: return None def verify(self, values: ValueReference) -> Any: + if values.action != OverrideActionEnum.Assign: + msg = f"Option {values.value_path} does not support " + msg += f"operation {values.action.value}" + raise ConfigError(msg) if not isinstance(values.values, dict): msg = f"Type of {values.value_path} should be 'dict', " msg += f"not {type(values.values)}" @@ -55,7 +59,8 @@ def verify(self, values: ValueReference) -> Any: for k in sorted(unknown_keys): suggested = get_close_matches(k, self.sub_options, 3) msg = f"Unknown option '{k}' in {values.value_path}. " - msg += f"Did you mean: {', '.join(suggested)}\n" + if suggested: + msg += f"Did you mean: {', '.join(suggested)}\n" if msg: raise ConfigError(msg[:-1]) return values.values diff --git a/src/py_build_cmake/config/options/dict.py b/src/py_build_cmake/config/options/dict.py index 46b70ff..d51cb43 100644 --- a/src/py_build_cmake/config/options/dict.py +++ b/src/py_build_cmake/config/options/dict.py @@ -4,7 +4,7 @@ from ...common import ConfigError from .config_option import ConfigOption -from .value_reference import ValueReference +from .value_reference import OverrideAction, OverrideActionEnum, ValueReference class DictOfStrConfigOption(ConfigOption): @@ -12,17 +12,33 @@ def get_typename(self, md: bool = False) -> str: return "dict" def override(self, old_value: ValueReference, new_value: ValueReference): - if old_value.values is None: - old_value.values = {} - if new_value.values is None: - return old_value.values - r = deepcopy(old_value.values) - r.update(deepcopy(new_value.values)) + new, old = new_value.values, old_value.values + if old is None: + old = {} + if new is None: + return old + r = deepcopy(old) + for k, v in new.items(): + if isinstance(v, str): + r[k] = v + else: + assert isinstance(v, OverrideAction) + assert isinstance(v.values, str) + r[k] = v.action.override_string(r.get(k, ""), v.values) return r def verify(self, values: ValueReference): + def validate_type(el): + if isinstance(el, OverrideAction): + return isinstance(el.values, str) + return isinstance(el, str) + if values.values is None: return None + if values.action != OverrideActionEnum.Assign: + msg = f"Option {values.value_path} of type {self.get_typename()} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) valdict = values.values if not isinstance(valdict, dict): msg = f"Type of {values.value_path} should be {dict}, " @@ -31,7 +47,7 @@ def verify(self, values: ValueReference): elif not all(isinstance(el, str) for el in valdict): msg = f"Type of keys in {values.value_path} should be {str}" raise ConfigError(msg) - elif not all(isinstance(el, str) for el in valdict.values()): + elif not all(validate_type(el) for el in valdict.values()): msg = f"Type of values in {values.value_path} should be {str}" raise ConfigError(msg) return valdict diff --git a/src/py_build_cmake/config/options/enum.py b/src/py_build_cmake/config/options/enum.py index a3248e2..76c2e9e 100644 --- a/src/py_build_cmake/config/options/enum.py +++ b/src/py_build_cmake/config/options/enum.py @@ -4,7 +4,7 @@ from .config_option import ConfigOption from .config_path import ConfPath from .default import DefaultValue, NoDefaultValue -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference class EnumConfigOption(ConfigOption): @@ -44,6 +44,10 @@ def override(self, old_value: ValueReference, new_value: ValueReference): return new_value.values def verify(self, values: ValueReference): + if values.action != OverrideActionEnum.Assign: + msg = f"Enumeration option {values.value_path} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) if self.sub_options: msg = f"Type of {values.value_path} should be {str}, " msg += f"not {dict}" diff --git a/src/py_build_cmake/config/options/int.py b/src/py_build_cmake/config/options/int.py index 1c03bef..db00623 100644 --- a/src/py_build_cmake/config/options/int.py +++ b/src/py_build_cmake/config/options/int.py @@ -2,7 +2,7 @@ from ...common import ConfigError from .config_option import ConfigOption -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference class IntConfigOption(ConfigOption): @@ -15,6 +15,10 @@ def override(self, old_value: ValueReference, new_value: ValueReference): return new_value.values def verify(self, values: ValueReference): + if values.action != OverrideActionEnum.Assign: + msg = f"Option {values.value_path} of type {self.get_typename()} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) if self.sub_options: msg = f"Type of {values.value_path} should be {int}, " msg += f"not {dict}" diff --git a/src/py_build_cmake/config/options/list.py b/src/py_build_cmake/config/options/list.py index 81546f6..3c6428d 100644 --- a/src/py_build_cmake/config/options/list.py +++ b/src/py_build_cmake/config/options/list.py @@ -6,7 +6,7 @@ from .config_option import ConfigOption from .config_path import ConfPath from .default import DefaultValue -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference class ListOfStrConfigOption(ConfigOption): @@ -76,23 +76,30 @@ def override(self, old_value: ValueReference, new_value: ValueReference): return self._override_dict(old_value, new_value) return self._override_list(old_value, new_value) + def _verify_dict(self, values): + if values.action != OverrideActionEnum.Assign: + msg = f"Type of {values.value_path} should be {list}, " + msg += f"not {dict}" + raise ConfigError(msg) + invalid_keys = set(values.values.keys()) - self.list_op_keys + if invalid_keys: + inv_str = ", ".join(map(str, invalid_keys)) + val_str = ", ".join(map(str, self.list_op_keys)) + msg = f"Invalid keys in {values.value_path}: {inv_str} " + msg += f"(valid keys are: {val_str})" + raise ConfigError(msg) + for k, v in values.values.items(): + pthname = f"{values.value_path}[{k}]" + if not isinstance(v, list): + msg = f"Type of {pthname} should be {list}, not {type(v)}" + raise ConfigError(msg) + if not all(isinstance(el, str) for el in v): + msg = f"Type of elements in {pthname} should be {str}" + raise ConfigError(msg) + def verify(self, values: ValueReference): if isinstance(values.values, dict): - invalid_keys = set(values.values.keys()) - self.list_op_keys - if invalid_keys: - inv_str = ", ".join(map(str, invalid_keys)) - val_str = ", ".join(map(str, self.list_op_keys)) - msg = f"Invalid keys in {values.value_path}: {inv_str} " - msg += f"(valid keys are: {val_str})" - raise ConfigError(msg) - for k, v in values.values.items(): - pthname = f"{values.value_path}[{k}]" - if not isinstance(v, list): - msg = f"Type of {pthname} should be {list}, not {type(v)}" - raise ConfigError(msg) - if not all(isinstance(el, str) for el in v): - msg = f"Type of elements in {pthname} should be {str}" - raise ConfigError(msg) + self._verify_dict(values) elif not isinstance(values.values, list): if self.convert_str_to_singleton and isinstance(values.values, str): values.values = [values.values] @@ -103,7 +110,18 @@ def verify(self, values: ValueReference): elif not all(isinstance(el, str) for el in values.values): msg = f"Type of elements in {values.value_path} should be {str}" raise ConfigError(msg) - return values.values + if values.action == OverrideActionEnum.Assign: + return values.values + elif values.action == OverrideActionEnum.Append: + return {"append": values.values} + elif values.action == OverrideActionEnum.Prepend: + return {"prepend": values.values} + elif values.action == OverrideActionEnum.Remove: + return {"-": values.values} + else: + msg = f"Option {values.value_path} of type {self.get_typename()} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) def finalize(self, values: ValueReference): if isinstance(values.values, str): diff --git a/src/py_build_cmake/config/options/override.py b/src/py_build_cmake/config/options/override.py index 1407337..773a47c 100644 --- a/src/py_build_cmake/config/options/override.py +++ b/src/py_build_cmake/config/options/override.py @@ -1,7 +1,7 @@ from __future__ import annotations from .config_reference import ConfigReference -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference class ConfigOverrider: @@ -18,23 +18,29 @@ def __init__( self.new_values = new_values def override(self): + # Override our own value overridden_values = ValueReference( self.values.value_path, self.ref.config.override(self.values, self.new_values), ) + # If we have sub-options, override those for name in self.ref.sub_options: + # Skip the sup-option if its value is not set in the override ref = self.ref.sub_ref(name).resolve_inheritance(self.root) try: new_val = self.new_values.sub_ref(name) except KeyError: continue - default = {} if ref.sub_options else None - overridden_values.set_value_default(name, default) - old_val = overridden_values.sub_ref(name) - overridden_values.values[name] = ConfigOverrider( - root=self.root, - ref=ref, - values=old_val, - new_values=new_val, - ).override() + if new_val.action == OverrideActionEnum.Clear: + overridden_values.clear_value(name) + else: + default = {} if ref.sub_options else None + overridden_values.set_value_default(name, default) + old_val = overridden_values.sub_ref(name) + overridden_values.values[name] = ConfigOverrider( + root=self.root, + ref=ref, + values=old_val, + new_values=new_val, + ).override() return overridden_values.values diff --git a/src/py_build_cmake/config/options/path.py b/src/py_build_cmake/config/options/path.py index 132687f..d21ae04 100644 --- a/src/py_build_cmake/config/options/path.py +++ b/src/py_build_cmake/config/options/path.py @@ -7,7 +7,7 @@ from .config_option import ConfigOption from .config_path import ConfPath from .default import DefaultValue -from .value_reference import ValueReference +from .value_reference import OverrideActionEnum, ValueReference @dataclass @@ -64,6 +64,10 @@ def override(self, old_value, new_value): return new_value.values def _verify_string(self, values: ValueReference): + if values.action != OverrideActionEnum.Assign: + msg = f"Option {values.value_path} of type {self.get_typename()} " + msg += f"does not support operation {values.action.value}" + raise ConfigError(msg) if self.sub_options: msg = f"Type of {values.value_path} should be {str}, " msg += f"not {dict}" diff --git a/src/py_build_cmake/config/options/string.py b/src/py_build_cmake/config/options/string.py index 6cf673a..1faeb6e 100644 --- a/src/py_build_cmake/config/options/string.py +++ b/src/py_build_cmake/config/options/string.py @@ -10,9 +10,14 @@ def get_typename(self, md: bool = False) -> str: return "string" def override(self, old_value, new_value): - if new_value.values is None: - return old_value.values - return new_value.values + new, old = new_value.values, old_value.values + if old is None: + old = "" + if new is None: + return old + assert isinstance(new, str) + assert isinstance(old, str) + return new_value.action.override_string(old, new) def verify(self, values: ValueReference): if self.sub_options: diff --git a/src/py_build_cmake/config/options/value_reference.py b/src/py_build_cmake/config/options/value_reference.py index b3f0082..0cd5d01 100644 --- a/src/py_build_cmake/config/options/value_reference.py +++ b/src/py_build_cmake/config/options/value_reference.py @@ -1,14 +1,55 @@ from __future__ import annotations +import os +from dataclasses import dataclass +from enum import Enum from typing import Any from .config_path import ConfPath +class OverrideActionEnum(Enum): + Assign = "=" + Append = "+=" + AppendPath = "+=(path)" + Prepend = "=+" + PrependPath = "=+(path)" + Remove = "-=" + Clear = "=!" + + def override_string(self, old: str, new: str) -> str: + return { + OverrideActionEnum.Assign: lambda: new, + OverrideActionEnum.Append: lambda: old + new, + OverrideActionEnum.AppendPath: lambda: ( + old + os.pathsep + new if old and new else old + new + ), + OverrideActionEnum.Prepend: lambda: new + old, + OverrideActionEnum.PrependPath: lambda: ( + new + os.pathsep + old if old and new else old + new + ), + OverrideActionEnum.Remove: lambda: old.replace(new, ""), + }[self]() + + +@dataclass +class OverrideAction: + action: OverrideActionEnum + values: Any + + class ValueReference: - def __init__(self, value_path: ConfPath, values: dict) -> None: + def __init__( + self, value_path: ConfPath, values: dict | OverrideAction | Any + ) -> None: self.value_path = value_path - self.values: dict | Any = values + self.action = OverrideActionEnum.Assign + self.values: dict | Any + if isinstance(values, OverrideAction): + self.action = values.action + self.values = values.values + else: + self.values = values def is_value_set(self, path: str | ConfPath): if isinstance(path, str): @@ -44,6 +85,20 @@ def set_value(self, path: str | ConfPath, val: Any): return False values = values[name] + def clear_value(self, path: str | ConfPath): + if isinstance(path, str): + self.values.pop(path, None) + return True + values = self.values + while True: + name, path = path.split_front() + if not path: + values.pop(name, None) + return True + if name not in values: + return False + values = values[name] + def set_value_default(self, path: str | ConfPath, val: Any): if isinstance(path, str): self.values.setdefault(path, val) diff --git a/tests/test_cli_override.py b/tests/test_cli_override.py new file mode 100644 index 0000000..cbc2196 --- /dev/null +++ b/tests/test_cli_override.py @@ -0,0 +1,69 @@ +from py_build_cmake.config.cli_override import CLIOption, parse_cli, parse_file + + +def test_parse_cli(): + + assert parse_cli( + r'tools.py-build-cmake.cmake."😄"-=[a, b, c, def, 551, False]' + ) == CLIOption( + action="-=", + key=("tools", "py-build-cmake", "cmake", "😄"), + value=["a", "b", "c", "def", 551, False], + ) + assert parse_cli(r'"a b c\\\""=!') == CLIOption( + action="=!", + key=('a b c\\"',), + value=None, + ) + assert parse_cli(r'"a b c"=+ foo=5 ') == CLIOption( + action="=+", + key=("a b c",), + value=" foo=5 ", + ) + + +def test_parse_file(): + file = r""" + "a b c" =+foo=5 + y= d, e, f# Comment + z= a, b, c # Comment + tools.py-build-cmake.cmake."😄"-=[a, b, c, def, 551, False] + a.b.c=! # Comment + bar ={a = 1, b= 2, c =3, d=4, e=foo=5, f= foo=5, g = foo=5 } + zed -=[] + zed -=[1] + zed -=[1, 2] # Comment + zed -=[1, 2,] +# Foo bar + # Bar foo + baz+=(path)$HOME/opt/python/bin + """ + assert parse_file(file) == [ + CLIOption(action="=+", key=("a b c",), value="foo=5"), + CLIOption(action="=", key=("y",), value=" d, e, f"), + CLIOption(action="=", key=("z",), value=" a, b, c "), + CLIOption( + action="-=", + key=("tools", "py-build-cmake", "cmake", "😄"), + value=["a", "b", "c", "def", 551, False], + ), + CLIOption(action="=!", key=("a", "b", "c"), value=None), + CLIOption( + action="=", + key=("bar",), + value={ + "a": 1, + "b": 2, + "c": 3, + "d": 4, + "e": "foo=5", + "f": "foo=5", + "g": "foo=5", + }, + ), + CLIOption(action="-=", key=("zed",), value=[]), + CLIOption(action="-=", key=("zed",), value=[1]), + CLIOption(action="-=", key=("zed",), value=[1, 2]), + CLIOption(action="-=", key=("zed",), value=[1, 2]), + CLIOption(action="+=(path)", key=("baz",), value="$HOME/opt/python/bin"), + ] diff --git a/tests/test_config_load.py b/tests/test_config_load.py index c33bcb8..2678fe1 100644 --- a/tests/test_config_load.py +++ b/tests/test_config_load.py @@ -1,7 +1,14 @@ +import os from pathlib import PurePosixPath import pytest -from py_build_cmake.config.load import Config, ConfigError, process_config +from py_build_cmake.config.cli_override import parse_file +from py_build_cmake.config.load import ( + Config, + ConfigError, + add_cli_override, + process_config, +) from py_build_cmake.config.options.config_path import ConfPath @@ -442,3 +449,141 @@ def test_real_config_local_override_windows(): } assert conf.cmake is None assert conf.cross is None + + +def test_real_config_cli_override(): + pyproj_path = PurePosixPath("/project/pyproject.toml") + pyproj = { + "project": {"name": "foobar", "version": "1.2.3", "description": "descr"}, + "tool": { + "some-other-tool": {}, + "py-build-cmake": { + "cmake": { + "build_type": "Release", + "generator": "Ninja", + "source_path": "src", + "env": {"foo": "bar"}, + "args": ["arg1", "arg2"], + "build_tool_args": ["-a", "-b"], + "find_python": False, + "find_python3": True, + }, + "linux": { + "cmake": { + "install_components": ["linux_install"], + "env": {"PATH": "/usr/bin"}, + "args": ["arg3", "arg4"], + } + }, + "windows": { + "cmake": { + "install_components": ["win_install"], + } + }, + }, + }, + } + files = {"pyproject.toml": pyproj} + override_config = r""" + linux.cmake.build_type=MinSizeRel + linux.cmake.env.PATH=+(path)$HOME/opt + linux.cmake.options.FOOBAR="abc" + linux.cmake.options.FOOBAR+="def" + linux.cmake.options.FOOBAR+="ghi" + linux.cmake.options.FOOBAR=+"xyz" + linux.cmake.options.FOOBAR-="def" + linux.cmake.args-=["arg3"] + cmake.build_tool_args=["-c"] + """ + overrides = {} + for i, opt in enumerate(parse_file(override_config)): + overrides.update(add_cli_override(files, opt, f"")) + + conf = process_config(pyproj_path, files, overrides, test=True) + assert conf.standard_metadata.name == "foobar" + assert str(conf.standard_metadata.version) == "1.2.3" + assert conf.standard_metadata.description == "descr" + assert conf.module == { + "name": "foobar", + "directory": PurePosixPath("/project"), + "namespace": False, + } + assert conf.editable == { + "linux": {"build_hook": False, "mode": "symlink"}, + "windows": {"build_hook": False, "mode": "symlink"}, + "mac": {"build_hook": False, "mode": "symlink"}, + } + assert conf.sdist == { + "linux": {"include_patterns": [], "exclude_patterns": []}, + "windows": {"include_patterns": [], "exclude_patterns": []}, + "mac": {"include_patterns": [], "exclude_patterns": []}, + } + assert conf.cmake == { + "linux": { + "build_type": "MinSizeRel", + "config": ["MinSizeRel"], + "generator": "Ninja", + "source_path": PurePosixPath("/project/src"), + "build_path": PurePosixPath( + "/project/.py-build-cmake_cache/{build_config}" + ), + "options": {"FOOBAR": "xyz;abc;ghi"}, + "args": ["arg1", "arg2", "arg4"], + "find_python": False, + "find_python3": True, + "build_args": [], + "build_tool_args": ["-a", "-b", "-c"], + "install_args": [], + "install_components": ["linux_install"], + "minimum_version": "3.15", + "env": {"PATH": "$HOME/opt" + os.pathsep + "/usr/bin", "foo": "bar"}, + "pure_python": False, + "python_abi": "auto", + "abi3_minimum_cpython_version": 32, + }, + "windows": { + "build_type": "Release", + "config": ["Release"], + "generator": "Ninja", + "source_path": PurePosixPath("/project/src"), + "build_path": PurePosixPath( + "/project/.py-build-cmake_cache/{build_config}" + ), + "options": {}, + "args": ["arg1", "arg2"], + "find_python": False, + "find_python3": True, + "build_args": [], + "build_tool_args": ["-a", "-b", "-c"], + "install_args": [], + "install_components": ["win_install"], + "minimum_version": "3.15", + "env": {"foo": "bar"}, + "pure_python": False, + "python_abi": "auto", + "abi3_minimum_cpython_version": 32, + }, + "mac": { + "build_type": "Release", + "config": ["Release"], + "generator": "Ninja", + "source_path": PurePosixPath("/project/src"), + "build_path": PurePosixPath( + "/project/.py-build-cmake_cache/{build_config}" + ), + "options": {}, + "args": ["arg1", "arg2"], + "find_python": False, + "find_python3": True, + "build_args": [], + "build_tool_args": ["-a", "-b", "-c"], + "install_args": [], + "install_components": [""], + "minimum_version": "3.15", + "env": {"foo": "bar"}, + "pure_python": False, + "python_abi": "auto", + "abi3_minimum_cpython_version": 32, + }, + } + assert conf.cross is None diff --git a/tests/test_configoptions.py b/tests/test_configoptions.py index ac80f5d..f58c004 100644 --- a/tests/test_configoptions.py +++ b/tests/test_configoptions.py @@ -18,7 +18,11 @@ from py_build_cmake.config.options.list import ListOfStrConfigOption from py_build_cmake.config.options.override import ConfigOverrider from py_build_cmake.config.options.string import StringConfigOption -from py_build_cmake.config.options.value_reference import ValueReference +from py_build_cmake.config.options.value_reference import ( + OverrideAction, + OverrideActionEnum, + ValueReference, +) from py_build_cmake.config.options.verify import ConfigVerifier @@ -195,6 +199,60 @@ def test_override2(): } +def test_override_action(): + opts = gen_test_opts() + values = { + "trunk": { + "mid1": { + "leaf11": "11", + "leaf12": "12", + }, + "mid2": { + "leaf21": "21", + "leaf22": "22", + }, + }, + } + override_values = { + "trunk": { + "mid1": { + "leaf11": OverrideAction(OverrideActionEnum.Clear, None), + "leaf12": OverrideAction(OverrideActionEnum.Append, "34"), + }, + "mid2": { + "leaf21": OverrideAction(OverrideActionEnum.Remove, "2"), + "leaf22": OverrideAction(OverrideActionEnum.Assign, "99"), + }, + }, + } + root_ref = ConfigReference(ConfPath.from_string("/"), opts) + rval = ValueReference(ConfPath.from_string("/"), values) + + ConfigVerifier( + root=root_ref, + ref=root_ref, + values=rval, + ).verify() + overridden_values = ConfigOverrider( + root=root_ref, + ref=root_ref, + values=rval, + new_values=ValueReference(ConfPath.from_string("/override"), override_values), + ).override() + + assert overridden_values == { + "trunk": { + "mid1": { + "leaf12": "1234", + }, + "mid2": { + "leaf21": "1", + "leaf22": "99", + }, + }, + } + + def test_override_trunk(): opts = gen_test_opts() values = {