Allow build file symbols / macros with leading _s (#21246)

In #21028, we started excluding build file prelude symbols and macros
that had a leading `_` in a way that broke potential live code, where a
macro like `_example` wouldn't be available in `BUILD` files.

This moves that skipping to only exclude those symbols from the help
output, meaning such macros are now available in BUILD files again,
although not visible in `pants help ...`.

Fixes #21228

---------

Co-authored-by: Jacob Floyd <[email protected]>

src/python/pants/engine/internals/build_files_test.py

@@ -350,22 +350,33 @@ def test_prelude_docstring_on_constant() -> None:
         """
     )
     result = run_prelude_parsing_rule(prelude_content)
-    assert {"MACRO_CONST", "ANON", "Number", "MULTI_HINTS", "untyped"} == set(result.info)
+    assert {"MACRO_CONST", "ANON", "Number", "MULTI_HINTS", "_PRIVATE", "untyped"} == set(
+        result.info
+    )
 
     info = result.info["MACRO_CONST"]
     assert info.value == "value"
     assert info.help == softwrap(macro_docstring)
     assert info.signature == ": str"
+    assert info.hide_from_help is False
 
     multi = result.info["MULTI_HINTS"]
     assert multi.value == 42
     assert multi.help == "this is it"
     assert multi.signature == ": Number"
+    assert multi.hide_from_help is False
 
     anon = result.info["ANON"]
     assert anon.value == "undocumented"
     assert anon.help is None
     assert anon.signature == ": str"
+    assert anon.hide_from_help is False
+
+    private = result.info["_PRIVATE"]
+    assert private.value == 42
+    assert private.help is None
+    assert private.signature == ": int"
+    assert private.hide_from_help is True
 
 
 def test_prelude_reference_env_vars() -> None:

src/python/pants/engine/internals/parser.py

@@ -67,12 +67,22 @@ class BuildFilePreludeSymbols(BuildFileSymbolsInfo):
     @classmethod
     def create(cls, ns: Mapping[str, Any], env_vars: Iterable[str]) -> BuildFilePreludeSymbols:
         info = {}
-        annotations = ns.get("__annotations__", {})
+        annotations_name = "__annotations__"
+        annotations = ns.get(annotations_name, {})
         for name, symb in ns.items():
-            if name.startswith("_"):
+            if name == annotations_name:
+                # don't include the annotations themselves as a symbol
                 continue
-            # We only need type hints via `annotations` for top-level values which doesn't work with `inspect`.
-            info[name] = BuildFileSymbolInfo(name, symb, type_hints=annotations.get(name))
+
+            info[name] = BuildFileSymbolInfo(
+                name,
+                symb,
+                # We only need type hints via `annotations` for top-level values which doesn't work with `inspect`.
+                type_hints=annotations.get(name),
+                # If the user has defined a _ symbol, we assume they don't want it in `pants help` output.
+                hide_from_help=name.startswith("_"),
+            )
+
         return cls(info=FrozenDict(info), referenced_env_vars=tuple(sorted(env_vars)))
 
 
@@ -84,6 +94,8 @@ class BuildFileSymbolInfo:
     signature: str | None = field(default=None, compare=False, init=False)
     type_hints: InitVar[Any] = None
 
+    hide_from_help: bool = False
+
     def __post_init__(self, type_hints: Any) -> None:
         annotated_type: type = type(self.value)
         help: str | None = self.help

src/python/pants/help/help_info_extracter.py

@@ -989,6 +989,9 @@ def load() -> BuildFileSymbolHelpInfo:
             {
                 symbol.name: get_build_file_symbol_help_info_loader(symbol)
                 for symbol in build_symbols.info.values()
+                # (NB. we don't just check name.startswith("_") because there's symbols like
+                # __default__ that should appear in help & docs)
+                if not symbol.hide_from_help
             }
         )
 

src/python/pants/help/help_info_extracter_test.py

@@ -284,7 +284,12 @@ def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
         UnionMembership({}),
         fake_consumed_scopes_mapper,
         RegisteredTargetTypes({BazLibrary.alias: BazLibrary}),
-        BuildFileSymbolsInfo.from_info((BuildFileSymbolInfo("dummy", rule_info_test),)),
+        BuildFileSymbolsInfo.from_info(
+            (
+                BuildFileSymbolInfo("dummy", rule_info_test),
+                BuildFileSymbolInfo("private", 1, hide_from_help=True),
+            )
+        ),
         bc_builder.create(),
     )