[Call-by-name] JVM linters (#21368)

Auto-migrated the JVM linters to call-by-name

docs/notes/2.23.x.md

@@ -345,7 +345,10 @@ Pants has a new mechanism for `@rule` invocation in backends. In this release th
 - `cc`
 - `cue`
 - `debian`
+- `java.lint`
+- `kotlin.lint`
 - `makeself`
+- `scala.lint`
 - `semgrep`
 - `sql`
 - `swift`

src/python/pants/backend/java/lint/google_java_format/rules.py

@@ -9,14 +9,13 @@
 from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
 from pants.core.goals.resolves import ExportableTool
 from pants.core.util_rules.partitions import PartitionerType
-from pants.engine.internals.selectors import Get
-from pants.engine.process import ProcessResult
-from pants.engine.rules import collect_rules, rule
+from pants.engine.process import fallible_to_exec_result_or_raise
+from pants.engine.rules import collect_rules, implicitly, rule
 from pants.engine.target import FieldSet, Target
 from pants.engine.unions import UnionRule
 from pants.jvm.jdk_rules import InternalJdk, JvmProcess
 from pants.jvm.resolve import jvm_tool
-from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest
+from pants.jvm.resolve.coursier_fetch import ToolClasspathRequest, materialize_classpath_for_tool
 from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool
 from pants.util.logging import LogLevel
 from pants.util.strutil import pluralize
@@ -48,7 +47,9 @@ async def google_java_format_fmt(
     jdk: InternalJdk,
 ) -> FmtResult:
     lockfile_request = GenerateJvmLockfileFromTool.create(tool)
-    tool_classpath = await Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request))
+    tool_classpath = await materialize_classpath_for_tool(
+        ToolClasspathRequest(lockfile=lockfile_request)
+    )
 
     toolcp_relpath = "__toolcp"
     extra_immutable_input_digests = {
@@ -73,20 +74,21 @@ async def google_java_format_fmt(
         *request.files,
     ]
 
-    result = await Get(
-        ProcessResult,
-        JvmProcess(
-            jdk=jdk,
-            argv=args,
-            classpath_entries=tool_classpath.classpath_entries(toolcp_relpath),
-            input_digest=request.snapshot.digest,
-            extra_jvm_options=tool.jvm_options,
-            extra_immutable_input_digests=extra_immutable_input_digests,
-            extra_nailgun_keys=extra_immutable_input_digests,
-            output_files=request.files,
-            description=f"Run Google Java Format on {pluralize(len(request.files), 'file')}.",
-            level=LogLevel.DEBUG,
-        ),
+    result = await fallible_to_exec_result_or_raise(
+        **implicitly(
+            JvmProcess(
+                jdk=jdk,
+                argv=args,
+                classpath_entries=tool_classpath.classpath_entries(toolcp_relpath),
+                input_digest=request.snapshot.digest,
+                extra_jvm_options=tool.jvm_options,
+                extra_immutable_input_digests=extra_immutable_input_digests,
+                extra_nailgun_keys=extra_immutable_input_digests,
+                output_files=request.files,
+                description=f"Run Google Java Format on {pluralize(len(request.files), 'file')}.",
+                level=LogLevel.DEBUG,
+            ),
+        )
     )
     return await FmtResult.create(request, result)
 

src/python/pants/backend/kotlin/lint/ktlint/rules.py

@@ -9,14 +9,13 @@
 from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
 from pants.core.goals.resolves import ExportableTool
 from pants.core.util_rules.partitions import PartitionerType
-from pants.engine.internals.selectors import Get
-from pants.engine.process import ProcessResult
-from pants.engine.rules import collect_rules, rule
+from pants.engine.process import fallible_to_exec_result_or_raise
+from pants.engine.rules import collect_rules, implicitly, rule
 from pants.engine.target import FieldSet, Target
 from pants.engine.unions import UnionRule
 from pants.jvm.jdk_rules import InternalJdk, JvmProcess
 from pants.jvm.resolve import jvm_tool
-from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest
+from pants.jvm.resolve.coursier_fetch import ToolClasspathRequest, materialize_classpath_for_tool
 from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool
 from pants.util.logging import LogLevel
 from pants.util.strutil import pluralize
@@ -46,7 +45,9 @@ async def ktlint_fmt(
     request: KtlintRequest.Batch, tool: KtlintSubsystem, jdk: InternalJdk
 ) -> FmtResult:
     lockfile_request = GenerateJvmLockfileFromTool.create(tool)
-    tool_classpath = await Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request))
+    tool_classpath = await materialize_classpath_for_tool(
+        ToolClasspathRequest(lockfile=lockfile_request)
+    )
 
     toolcp_relpath = "__toolcp"
     extra_immutable_input_digests = {
@@ -59,20 +60,21 @@ async def ktlint_fmt(
         *request.files,
     ]
 
-    result = await Get(
-        ProcessResult,
-        JvmProcess(
-            jdk=jdk,
-            argv=args,
-            classpath_entries=tool_classpath.classpath_entries(toolcp_relpath),
-            input_digest=request.snapshot.digest,
-            extra_jvm_options=tool.jvm_options,
-            extra_immutable_input_digests=extra_immutable_input_digests,
-            extra_nailgun_keys=extra_immutable_input_digests,
-            output_files=request.files,
-            description=f"Run Ktlint on {pluralize(len(request.files), 'file')}.",
-            level=LogLevel.DEBUG,
-        ),
+    result = await fallible_to_exec_result_or_raise(
+        **implicitly(
+            JvmProcess(
+                jdk=jdk,
+                argv=args,
+                classpath_entries=tool_classpath.classpath_entries(toolcp_relpath),
+                input_digest=request.snapshot.digest,
+                extra_jvm_options=tool.jvm_options,
+                extra_immutable_input_digests=extra_immutable_input_digests,
+                extra_nailgun_keys=extra_immutable_input_digests,
+                output_files=request.files,
+                description=f"Run Ktlint on {pluralize(len(request.files), 'file')}.",
+                level=LogLevel.DEBUG,
+            )
+        )
     )
 
     return await FmtResult.create(request, result)

src/python/pants/backend/scala/lint/scalafix/rules.py

@@ -19,27 +19,35 @@
 from pants.core.goals.resolves import ExportableTool
 from pants.core.util_rules.config_files import (
     GatherConfigFilesByDirectoriesRequest,
-    GatheredConfigFilesByDirectories,
+    gather_config_files_by_workspace_dir,
 )
 from pants.core.util_rules.partitions import Partition
 from pants.core.util_rules.source_files import SourceFiles
-from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
+from pants.core.util_rules.stripped_source_files import strip_source_roots
 from pants.engine.addresses import Addresses, UnparsedAddressInputs
 from pants.engine.fs import AddPrefix, Digest, DigestSubset, MergeDigests, PathGlobs, Snapshot
+from pants.engine.intrinsics import (
+    add_prefix_request_to_digest,
+    digest_subset_to_digest,
+    digest_to_snapshot,
+    merge_digests_request_to_digest,
+    process_request_to_process_result,
+)
 from pants.engine.process import FallibleProcessResult
-from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.rules import Get, collect_rules, concurrently, implicitly, rule
 from pants.engine.target import FieldSet, Target
 from pants.engine.unions import UnionRule
 from pants.jvm.classpath import Classpath
+from pants.jvm.classpath import classpath as classpath_get
 from pants.jvm.compile import ClasspathEntry
 from pants.jvm.goals import lockfile
 from pants.jvm.jdk_rules import InternalJdk, JvmProcess
-from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest
+from pants.jvm.resolve.coursier_fetch import ToolClasspathRequest, materialize_classpath_for_tool
 from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool
 from pants.jvm.resolve.key import CoursierResolveKey
 from pants.jvm.subsystems import JvmSubsystem
 from pants.jvm.target_types import JvmResolveField
-from pants.source.source_root import SourceRootsRequest, SourceRootsResult
+from pants.source.source_root import SourceRootsRequest, SourceRootsResult, get_source_roots
 from pants.util.dirutil import group_by_dir
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
@@ -138,24 +146,24 @@ async def _partition_scalafix(
 
     classpaths: Iterable[Classpath] = ()
     if scalafix.semantic_rules:
-        classpaths = await MultiGet(
-            Get(Classpath, Addresses([field_set.address])) for field_set in request.field_sets
+        classpaths = await concurrently(
+            classpath_get(**implicitly(Addresses([field_set.address])))
+            for field_set in request.field_sets
         )
     rule_classpath = await _resolve_scalafix_rule_classpath(scalafix)
 
     filepaths = tuple(field_set.source.file_path for field_set in request.field_sets)
     classpath_by_filepath = dict(zip(filepaths, classpaths))
     lockfile_request = GenerateJvmLockfileFromTool.create(scalafix)
-    tool_runtime_classpath, config_files = await MultiGet(
-        Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
-        Get(
-            GatheredConfigFilesByDirectories,
+    tool_runtime_classpath, config_files = await concurrently(
+        materialize_classpath_for_tool(ToolClasspathRequest(lockfile=lockfile_request)),
+        gather_config_files_by_workspace_dir(
             GatherConfigFilesByDirectoriesRequest(
                 tool_name="scalafix",
                 config_filename=scalafix.config_file_name,
                 filepaths=filepaths,
                 orphan_filepath_behavior=scalafix.orphan_files_behavior,
-            ),
+            )
         ),
     )
 
@@ -179,8 +187,10 @@ async def _partition_scalafix(
             scala_version = sorted(scala_versions_by_filepath[filename])[0]
             partitioned_source_files[(config_file, scala_version)].add(filename)
 
-    config_file_snapshots = await MultiGet(
-        Get(Snapshot, DigestSubset(config_files.snapshot.digest, PathGlobs([config_file])))
+    config_file_snapshots = await concurrently(
+        digest_to_snapshot(
+            **implicitly(DigestSubset(config_files.snapshot.digest, PathGlobs([config_file])))
+        )
         for (config_file, _) in partitioned_source_files
     )
 
@@ -262,16 +272,16 @@ async def _restore_source_roots(source_roots_result: SourceRootsResult, digest:
     for file, root in source_roots_result.path_to_root.items():
         source_roots_to_files[root.path].add(str(file.relative_to(root.path)))
 
-    digest_subsets = await MultiGet(
-        Get(Digest, DigestSubset(digest, PathGlobs(files)))
+    digest_subsets = await concurrently(
+        digest_subset_to_digest(DigestSubset(digest, PathGlobs(files)))
         for files in source_roots_to_files.values()
     )
 
-    restored_digests = await MultiGet(
-        Get(Digest, AddPrefix(digest, source_root))
+    restored_digests = await concurrently(
+        add_prefix_request_to_digest(AddPrefix(digest, source_root))
         for digest, source_root in zip(digest_subsets, source_roots_to_files.keys())
     )
-    return await Get(Snapshot, MergeDigests(restored_digests))
+    return await digest_to_snapshot(**implicitly(MergeDigests(restored_digests)))
 
 
 @dataclass(frozen=True)
@@ -287,60 +297,59 @@ async def _run_scalafix_process(
 ) -> FallibleProcessResult:
     partition_info = request.partition_info
 
-    merged_digest = await Get(
-        Digest, MergeDigests([partition_info.config_snapshot.digest, request.snapshot.digest])
+    merged_digest = await merge_digests_request_to_digest(
+        MergeDigests([partition_info.config_snapshot.digest, request.snapshot.digest])
     )
 
-    return await Get(
-        FallibleProcessResult,
-        JvmProcess(
-            jdk=jdk,
-            argv=[
-                "scalafix.cli.Cli",
-                f"--config={partition_info.config_snapshot.files[0]}",
-                f"--scala-version={partition_info.scala_version}",
-                *(
-                    (f"--classpath={':'.join(partition_info.compile_classpath_entries)}",)
-                    if partition_info.compile_classpath_entries
-                    else ()
-                ),
-                *(
-                    (f"--tool-classpath={':'.join(partition_info.rule_classpath_entries)}",)
-                    if partition_info.rule_classpath_entries
-                    else ()
-                ),
-                *(("--check",) if request.check_only else ()),
-                *((f"--scalac-options={arg}" for arg in scalac.args) if scalac.args else ()),
-                *(f"--files={file}" for file in request.snapshot.files),
-            ],
-            classpath_entries=partition_info.runtime_classpath_entries,
-            input_digest=merged_digest,
-            output_files=request.snapshot.files,
-            extra_jvm_options=scalafix.jvm_options,
-            extra_immutable_input_digests=partition_info.extra_immutable_input_digests,
-            use_nailgun=False,
-            description=f"Run `scalafix` on {pluralize(len(request.snapshot.files), 'file')}.",
-            level=LogLevel.DEBUG,
-        ),
+    return await process_request_to_process_result(
+        **implicitly(
+            JvmProcess(
+                jdk=jdk,
+                argv=[
+                    "scalafix.cli.Cli",
+                    f"--config={partition_info.config_snapshot.files[0]}",
+                    f"--scala-version={partition_info.scala_version}",
+                    *(
+                        (f"--classpath={':'.join(partition_info.compile_classpath_entries)}",)
+                        if partition_info.compile_classpath_entries
+                        else ()
+                    ),
+                    *(
+                        (f"--tool-classpath={':'.join(partition_info.rule_classpath_entries)}",)
+                        if partition_info.rule_classpath_entries
+                        else ()
+                    ),
+                    *(("--check",) if request.check_only else ()),
+                    *((f"--scalac-options={arg}" for arg in scalac.args) if scalac.args else ()),
+                    *(f"--files={file}" for file in request.snapshot.files),
+                ],
+                classpath_entries=partition_info.runtime_classpath_entries,
+                input_digest=merged_digest,
+                output_files=request.snapshot.files,
+                extra_jvm_options=scalafix.jvm_options,
+                extra_immutable_input_digests=partition_info.extra_immutable_input_digests,
+                use_nailgun=False,
+                description=f"Run `scalafix` on {pluralize(len(request.snapshot.files), 'file')}.",
+                level=LogLevel.DEBUG,
+            )
+        )
     )
 
 
 @rule
 async def scalafix_fix(request: ScalafixFixRequest.Batch) -> FixResult:
-    source_roots = await Get(
-        SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(request.snapshot.files)
-    )
+    source_roots = await get_source_roots(SourceRootsRequest.for_files(request.snapshot.files))
 
     # We need to strip the source files to get semantic rules find SemanticDB metadata in the classpath
-    stripped_source_files = await Get(StrippedSourceFiles, SourceFiles(request.snapshot, ()))
+    stripped_source_files = await strip_source_roots(SourceFiles(request.snapshot, ()))
 
-    process_result = await Get(
-        FallibleProcessResult,
+    process_result = await _run_scalafix_process(
         _ScalafixProcess(
             snapshot=stripped_source_files.snapshot,
             partition_info=cast(ScalafixPartitionInfo, request.partition_metadata),
             check_only=False,
         ),
+        **implicitly(),
     )
 
     # We need now to restore the source roots
@@ -359,16 +368,16 @@ async def scalafix_fix(request: ScalafixFixRequest.Batch) -> FixResult:
 @rule
 async def scalafix_lint(request: ScalafixLintRequest.Batch) -> LintResult:
     # We need to strip the source files to get semantic rules find SemanticDB metadata in the classpath
-    source_snapshot = await Get(Snapshot, PathGlobs(request.elements))
-    stripped_source_files = await Get(StrippedSourceFiles, SourceFiles(source_snapshot, ()))
+    source_snapshot = await digest_to_snapshot(**implicitly(PathGlobs(request.elements)))
+    stripped_source_files = await strip_source_roots(SourceFiles(source_snapshot, ()))
 
-    process_result = await Get(
-        FallibleProcessResult,
+    process_result = await _run_scalafix_process(
         _ScalafixProcess(
             snapshot=stripped_source_files.snapshot,
             partition_info=cast(ScalafixPartitionInfo, request.partition_metadata),
             check_only=True,
         ),
+        **implicitly(),
     )
 
     return LintResult.create(request, process_result)