-
-
Notifications
You must be signed in to change notification settings - Fork 628
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add sql backend + sqlfluff linter (#20854)
- Loading branch information
Showing
25 changed files
with
2,320 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
{ | ||
"label": "Ad-Hoc Tools", | ||
"position": 12 | ||
"position": 13 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
{ | ||
"label": "Contributions", | ||
"position": 15 | ||
"position": 16 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
{ | ||
"label": "Releases", | ||
"position": 14 | ||
"position": 15 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
{ | ||
"label": "SQL", | ||
"position": 12 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
--- | ||
title: SQL Overview | ||
sidebar_position: 999 | ||
--- | ||
|
||
--- | ||
|
||
:::caution SQL support is in alpha stage | ||
Pants is currently building support for SQL. Simple use cases might be | ||
supported, but many options are missing. | ||
|
||
Please share feedback for what you need to use Pants with your SQL queries by | ||
either [opening a GitHub | ||
issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our | ||
Slack](/community/getting-help)! | ||
::: | ||
|
||
## Initial setup | ||
|
||
First, activate the relevant backend in `pants.toml`: | ||
|
||
```toml title="pants.toml" | ||
[GLOBAL] | ||
backend_packages = [ | ||
... | ||
"pants.backend.experimental.sql", | ||
... | ||
] | ||
``` | ||
|
||
The SQL backend adds [`sql_source`](../../reference/targets/sql_source.mdx) and | ||
[`sql_sources`](../../reference/targets/sql_sources.mdx) target types for SQL | ||
files. The [`sql_source`](../../reference/targets/sql_source.mdx) behaves like | ||
[`resource`](../../reference/targets/resource.mdx), so you can use it directly | ||
without wrappers. The `tailor` goal will automatically generate the targets for | ||
your .sql files. | ||
|
||
## Enable sqlfluff linter | ||
|
||
To enable the linter activate the relevant backend in `pants.toml`: | ||
|
||
```toml title="pants.toml" | ||
[GLOBAL] | ||
backend_packages = [ | ||
... | ||
"pants.backend.experimental.sql.lint.sqlfluff", | ||
... | ||
] | ||
``` | ||
|
||
You can run the linter via `lint` goal: | ||
``` | ||
pants lint --only=sqlfluff :: | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
{ | ||
"label": "Writing Plugins", | ||
"position": 13 | ||
"position": 14 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
Empty file.
Empty file.
12 changes: 12 additions & 0 deletions
12
src/python/pants/backend/experimental/sql/lint/sqlfluff/register.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
from pants.backend.sql.lint.sqlfluff import rules as sqlfluff_rules | ||
from pants.backend.sql.lint.sqlfluff import skip_field, subsystem | ||
|
||
|
||
def rules(): | ||
return [ | ||
*subsystem.rules(), | ||
*sqlfluff_rules.rules(), | ||
*skip_field.rules(), | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
from pants.backend.sql import tailor | ||
from pants.backend.sql.target_types import SqlSourcesGeneratorTarget, SqlSourceTarget | ||
|
||
|
||
def target_types(): | ||
return [ | ||
SqlSourceTarget, | ||
SqlSourcesGeneratorTarget, | ||
] | ||
|
||
|
||
def rules(): | ||
return [*tailor.rules()] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_sources( | ||
sources=[ | ||
"tailor.py", | ||
"target_types.py", | ||
], | ||
) |
Empty file.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
resource(name="lockfile", source="sqlfluff.lock") | ||
|
||
python_sources( | ||
overrides={"subsystem.py": {"dependencies": [":lockfile"]}}, | ||
) | ||
|
||
python_tests( | ||
name="tests", | ||
) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,144 @@ | ||
# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
from __future__ import annotations | ||
|
||
from dataclasses import dataclass | ||
from typing import Any, Tuple | ||
|
||
from typing_extensions import assert_never | ||
|
||
from pants.backend.python.util_rules import pex | ||
from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess | ||
from pants.backend.sql.lint.sqlfluff.subsystem import Sqlfluff, SqlfluffFieldSet, SqlfluffMode | ||
from pants.core.goals.fix import FixResult, FixTargetsRequest | ||
from pants.core.goals.fmt import FmtResult, FmtTargetsRequest | ||
from pants.core.goals.lint import LintResult, LintTargetsRequest | ||
from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest | ||
from pants.core.util_rules.partitions import PartitionerType | ||
from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest | ||
from pants.engine.fs import Digest, MergeDigests | ||
from pants.engine.internals.native_engine import Snapshot | ||
from pants.engine.process import FallibleProcessResult | ||
from pants.engine.rules import Get, MultiGet, collect_rules, rule | ||
from pants.util.logging import LogLevel | ||
from pants.util.meta import classproperty | ||
from pants.util.strutil import pluralize | ||
|
||
|
||
class SqlfluffFixRequest(FixTargetsRequest): | ||
field_set_type = SqlfluffFieldSet | ||
tool_subsystem = Sqlfluff | ||
partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION | ||
|
||
# We don't need to include automatically added lint rules for this SqlfluffFixRequest, | ||
# because these lint rules are already checked by SqlfluffLintRequest. | ||
enable_lint_rules = False | ||
|
||
|
||
class SqlfluffLintRequest(LintTargetsRequest): | ||
field_set_type = SqlfluffFieldSet | ||
tool_subsystem = Sqlfluff | ||
partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION | ||
|
||
|
||
class SqlfluffFormatRequest(FmtTargetsRequest): | ||
field_set_type = SqlfluffFieldSet | ||
tool_subsystem = Sqlfluff | ||
partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION | ||
|
||
@classproperty | ||
def tool_name(cls) -> str: | ||
return "sqlfluff format" | ||
|
||
@classproperty | ||
def tool_id(cls) -> str: | ||
return "sqlfluff-format" | ||
|
||
|
||
@dataclass(frozen=True) | ||
class _RunSqlfluffRequest: | ||
snapshot: Snapshot | ||
mode: SqlfluffMode | ||
|
||
|
||
@rule(level=LogLevel.DEBUG) | ||
async def run_sqlfluff( | ||
request: _RunSqlfluffRequest, | ||
sqlfluff: Sqlfluff, | ||
) -> FallibleProcessResult: | ||
sqlfluff_pex_get = Get(VenvPex, PexRequest, sqlfluff.to_pex_request()) | ||
|
||
config_files_get = Get( | ||
ConfigFiles, ConfigFilesRequest, sqlfluff.config_request(request.snapshot.dirs) | ||
) | ||
|
||
sqlfluff_pex, config_files = await MultiGet(sqlfluff_pex_get, config_files_get) | ||
|
||
input_digest = await Get( | ||
Digest, | ||
MergeDigests((request.snapshot.digest, config_files.snapshot.digest)), | ||
) | ||
|
||
initial_args: Tuple[str, ...] = () | ||
if request.mode is SqlfluffMode.FMT: | ||
initial_args = ("format",) | ||
elif request.mode is SqlfluffMode.FIX: | ||
initial_args = ("fix", *sqlfluff.fix_args) | ||
elif request.mode is SqlfluffMode.LINT: | ||
initial_args = ("lint",) | ||
else: | ||
assert_never(request.mode) | ||
|
||
conf_args = ["--config", sqlfluff.config] if sqlfluff.config else [] | ||
|
||
result = await Get( | ||
FallibleProcessResult, | ||
VenvPexProcess( | ||
sqlfluff_pex, | ||
argv=(*initial_args, *conf_args, *sqlfluff.args, *request.snapshot.files), | ||
input_digest=input_digest, | ||
output_files=request.snapshot.files, | ||
description=f"Run sqlfluff {' '.join(initial_args)} on {pluralize(len(request.snapshot.files), 'file')}.", | ||
level=LogLevel.DEBUG, | ||
), | ||
) | ||
return result | ||
|
||
|
||
@rule(desc="Fix with sqlfluff fix", level=LogLevel.DEBUG) | ||
async def sqlfluff_fix(request: SqlfluffFixRequest.Batch, sqlfluff: Sqlfluff) -> FixResult: | ||
result = await Get( | ||
FallibleProcessResult, _RunSqlfluffRequest(snapshot=request.snapshot, mode=SqlfluffMode.FIX) | ||
) | ||
return await FixResult.create(request, result) | ||
|
||
|
||
@rule(desc="Lint with sqlfluff lint", level=LogLevel.DEBUG) | ||
async def sqlfluff_lint(request: SqlfluffLintRequest.Batch[SqlfluffFieldSet, Any]) -> LintResult: | ||
source_files = await Get( | ||
SourceFiles, SourceFilesRequest(field_set.source for field_set in request.elements) | ||
) | ||
result = await Get( | ||
FallibleProcessResult, | ||
_RunSqlfluffRequest(snapshot=source_files.snapshot, mode=SqlfluffMode.LINT), | ||
) | ||
return LintResult.create(request, result) | ||
|
||
|
||
@rule(desc="Format with sqlfluff format", level=LogLevel.DEBUG) | ||
async def sqlfluff_fmt(request: SqlfluffFormatRequest.Batch, sqlfluff: Sqlfluff) -> FmtResult: | ||
result = await Get( | ||
FallibleProcessResult, | ||
_RunSqlfluffRequest(snapshot=request.snapshot, mode=SqlfluffMode.FMT), | ||
) | ||
return await FmtResult.create(request, result) | ||
|
||
|
||
def rules(): | ||
return [ | ||
*collect_rules(), | ||
*SqlfluffLintRequest.rules(), | ||
*SqlfluffFixRequest.rules(), | ||
*SqlfluffFormatRequest.rules(), | ||
*pex.rules(), | ||
] |
Oops, something went wrong.