Skip to content

Commit 2dbfec1

Browse files
d4l3kd4l3k
authored
use lintrunner for formatters + pyre (#861)
1 parent dd4c61f commit 2dbfec1

File tree

13 files changed

+347
-98
lines changed

13 files changed

+347
-98
lines changed

‎.github/workflows/lint.yaml

+6-3
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,19 @@ jobs:
1313
- name: Setup Python
1414
uses: actions/setup-python@v2
1515
with:
16-
python-version: 3.8
16+
python-version: "3.10"
1717
architecture: x64
1818
- name: Checkout TorchX
1919
uses: actions/checkout@v2
2020
- name: Install Dependencies
2121
run: |
2222
set -eux
23-
grep -E "(usort|black|flake8)" < dev-requirements.txt > /tmp/lint-requirements.txt
23+
grep -E "(lintrunner)" < dev-requirements.txt > /tmp/lint-requirements.txt
2424
pip install -r /tmp/lint-requirements.txt
25+
26+
lintrunner init
2527
- name: Run Lint
2628
run: |
2729
git config --global url."https://${{ secrets.GITHUB_TOKEN }}:[email protected]/".insteadOf "https://github.com/"
28-
scripts/lint.sh
30+
31+
lintrunner --skip PYRE --force-color --all-files

‎.github/workflows/pyre.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
run: |
2222
set -eux
2323
pip install -e .[dev]
24-
VERSION=$(grep "version" .pyre_configuration | sed -n -e 's/.*\(0\.0\.[0-9]*\).*/\1/p')
25-
pip install pyre-check-nightly==$VERSION
24+
25+
lintrunner init
2626
- name: Run Pyre
2727
run: scripts/pyre.sh

‎.lintrunner.toml

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
[[linter]]
2+
code = 'UFMT'
3+
include_patterns = [
4+
'**/*.py',
5+
'**/*.pyi',
6+
]
7+
command = [
8+
'python3',
9+
'tools/linter/adapters/ufmt_linter.py',
10+
'--',
11+
'@{{PATHSFILE}}'
12+
]
13+
init_command = [
14+
'python3',
15+
'-m',
16+
'lintrunner_adapters',
17+
'run',
18+
'pip_init',
19+
'--dry-run={{DRYRUN}}',
20+
'black==24.2.0',
21+
'ufmt==2.5.1',
22+
'usort==1.0.5',
23+
]
24+
is_formatter = true
25+
26+
[[linter]]
27+
code = 'PYRE'
28+
include_patterns = [
29+
'**/*.py',
30+
'**/*.pyi',
31+
]
32+
command = [
33+
'python3',
34+
'tools/linter/adapters/pyre_linter.py',
35+
'--',
36+
'@{{PATHSFILE}}'
37+
]
38+
init_command = [
39+
'bash',
40+
'scripts/setup_pyre.sh',
41+
'--dry-run={{DRYRUN}}',
42+
]
43+
is_formatter = false

‎CONTRIBUTING.md

+10
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,16 @@ Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
2929
disclosure of security bugs. In those cases, please go through the process
3030
outlined on that page and do not file a public issue.
3131

32+
## Lint + Pyre
33+
34+
Lint and type checking can be run via `lintrunner`
35+
36+
```sh
37+
pip install lintrunner lintrunner-adapters
38+
lintrunner init
39+
lintrunner -a
40+
```
41+
3242
## Integration Tests
3343

3444
See the [KFP integration test](scripts/kfpint.py) file for more details on setup

‎dev-requirements.txt

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
aiobotocore==2.12.1
22
ax-platform[mysql]==0.2.3
3-
black==22.12.0
43
boto3==1.34.51
54
captum>=0.4.0
65
docker
@@ -29,9 +28,12 @@ torchserve>=0.10.0
2928
torchtext==0.17.1
3029
torchvision==0.17.1
3130
ts==0.5.1
32-
usort==1.0.5
3331
ray[default]
3432

33+
# lint (linter versions are managed by lintrunner)
34+
lintrunner
35+
lintrunner-adapters
36+
3537

3638
# reduce backtracking
3739
grpcio==1.62.1

‎pyproject.toml

+3
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
[tool.usort]
22
first_party_detection = false
3+
4+
[tool.black]
5+
target-version = ["py38", "py39", "py310", "py311"]

‎scripts/lint.sh

+3-85
Original file line numberDiff line numberDiff line change
@@ -7,92 +7,10 @@
77

88
set -e
99

10-
if [ ! "$(black --version)" ]
10+
if [ ! "$(lintrunner --version)" ]
1111
then
12-
echo "Please install black."
12+
echo "Please install lintrunner."
1313
exit 1
1414
fi
15-
if [ ! "$(usort --version)" ]
16-
then
17-
echo "Please install usort."
18-
exit 1
19-
fi
20-
if [ ! "$(flake8 --version)" ]
21-
then
22-
echo "Please install flake8."
23-
exit 1
24-
fi
25-
26-
# cd to the project directory
27-
cd "$(dirname "$0")/.." || exit 1
28-
29-
GIT_URL_1="https://github.com/pytorch/torchx.git"
30-
GIT_URL_2="[email protected]:pytorch/torchx.git"
31-
32-
UPSTREAM_URL="$(git config remote.upstream.url)" || true
33-
34-
if [ -z "$UPSTREAM_URL" ]
35-
then
36-
echo "Setting upstream remote to $GIT_URL_1"
37-
git remote add upstream "$GIT_URL_1"
38-
elif [ "$UPSTREAM_URL" != "$GIT_URL_1" ] && \
39-
[ "$UPSTREAM_URL" != "$GIT_URL_2" ]
40-
then
41-
echo "upstream remote set to $UPSTREAM_URL."
42-
echo "Please delete the upstream remote or set it to $GIT_URL_1 to use this script."
43-
exit 1
44-
fi
45-
46-
git fetch upstream
4715

48-
CHANGED_FILES="$(git diff --diff-filter=ACMRT --name-only upstream/main | grep '\.py$' | tr '\n' ' ')"
49-
50-
if [ "$CHANGED_FILES" != "" ]
51-
then
52-
# Processing files one by one since passing directly $CHANGED_FILES will
53-
# treat the whole variable as a single file.
54-
echo "Running linters ..."
55-
for file in $CHANGED_FILES
56-
do
57-
echo "Checking $file"
58-
usort format "$file"
59-
black "$file" -q
60-
flake8 "$file" || LINT_ERRORS=1
61-
done
62-
else
63-
echo "No changes made to any Python files. Nothing to do."
64-
exit 0
65-
fi
66-
67-
if [ "$LINT_ERRORS" != "" ]
68-
then
69-
echo "One of the linters returned an error. See above output."
70-
# need this so that CI fails
71-
exit 1
72-
fi
73-
74-
# Check if any files were modified by running usort + black
75-
# If so, then the files were formatted incorrectly (e.g. did not pass lint)
76-
CHANGED_FILES="$(git diff --name-only | grep '\.py$' | tr '\n' ' ')"
77-
if [ "$CHANGED_FILES" != "" ]
78-
then
79-
RED="\e[31m"
80-
ENDCOLOR="\e[0m"
81-
echo "------------------------------------------"
82-
echo -e "${RED}[format] These files are not well-formatted:${ENDCOLOR}"
83-
git diff --name-only
84-
echo "------------------------------------------"
85-
echo -e "${RED}[format] Suggested format by lint:${ENDCOLOR}"
86-
git diff
87-
echo "------------------------------------------"
88-
echo "To apply the suggested format, run:"
89-
echo "usort format <file_name>"
90-
echo "black <file_name> -q"
91-
echo "flake8 <file_name>"
92-
echo -e "${RED}You must fix them before merging the pull request.${ENDCOLOR}"
93-
# need this so that CI fails
94-
exit 1
95-
else
96-
echo "All files are well-formatted."
97-
exit 0
98-
fi
16+
lintrunner --force-color

‎scripts/setup_pyre.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
VERSION=$(grep "version" .pyre_configuration | sed -n -e 's/.*\(0\.0\.[0-9]*\).*/\1/p')
6+
pip install pyre-check-nightly==$VERSION

‎tools/linter/adapters/pyre_linter.py

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import argparse
2+
import concurrent.futures
3+
import json
4+
import logging
5+
import os
6+
import subprocess
7+
import sys
8+
from enum import Enum
9+
from pathlib import Path
10+
from typing import Any, List, NamedTuple, Optional, Set, TypedDict
11+
12+
logger: logging.Logger = logging.getLogger(__name__)
13+
14+
15+
class LintSeverity(str, Enum):
16+
ERROR = "error"
17+
WARNING = "warning"
18+
ADVICE = "advice"
19+
DISABLED = "disabled"
20+
21+
22+
class LintMessage(NamedTuple):
23+
path: Optional[str]
24+
line: Optional[int]
25+
char: Optional[int]
26+
code: str
27+
severity: LintSeverity
28+
name: str
29+
original: Optional[str]
30+
replacement: Optional[str]
31+
description: Optional[str]
32+
33+
34+
class PyreResult(TypedDict):
35+
line: int
36+
column: int
37+
stop_line: int
38+
stop_column: int
39+
path: str
40+
code: int
41+
name: str
42+
description: str
43+
concise_description: str
44+
45+
46+
def run_pyre() -> List[PyreResult]:
47+
proc = subprocess.run(
48+
["pyre", "--output=json", "incremental"],
49+
capture_output=True,
50+
)
51+
return json.loads(proc.stdout)
52+
53+
54+
def check_pyre(
55+
filenames: Set[str],
56+
) -> List[LintMessage]:
57+
try:
58+
results = run_pyre()
59+
60+
return [
61+
LintMessage(
62+
path=result["path"],
63+
line=result["line"],
64+
char=result["column"],
65+
code="pyre",
66+
severity=LintSeverity.WARNING,
67+
name=result["name"],
68+
description=result["description"],
69+
original=None,
70+
replacement=None,
71+
)
72+
for result in results
73+
]
74+
except Exception as err:
75+
return [
76+
LintMessage(
77+
path=None,
78+
line=None,
79+
char=None,
80+
code="pyre",
81+
severity=LintSeverity.ADVICE,
82+
name="command-failed",
83+
original=None,
84+
replacement=None,
85+
description=(f"Failed due to {err.__class__.__name__}:\n{err}"),
86+
)
87+
]
88+
89+
90+
def main() -> None:
91+
parser = argparse.ArgumentParser(
92+
description="Checks files with pyre",
93+
fromfile_prefix_chars="@",
94+
)
95+
parser.add_argument(
96+
"--verbose",
97+
action="store_true",
98+
help="verbose logging",
99+
)
100+
parser.add_argument(
101+
"filenames",
102+
nargs="+",
103+
help="paths to lint",
104+
)
105+
args = parser.parse_args()
106+
107+
logging.basicConfig(
108+
format="<%(processName)s:%(levelname)s> %(message)s",
109+
level=(
110+
logging.NOTSET
111+
if args.verbose
112+
else logging.DEBUG if len(args.filenames) < 1000 else logging.INFO
113+
),
114+
stream=sys.stderr,
115+
)
116+
117+
lint_messages = check_pyre(set(args.filenames))
118+
119+
for lint_message in lint_messages:
120+
print(json.dumps(lint_message._asdict()), flush=True)
121+
122+
123+
if __name__ == "__main__":
124+
main()

0 commit comments

Comments
 (0)