From 731d321852a44eca07436e35bc727c2fe15d27a1 Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Wed, 20 Jan 2021 21:47:36 -0600 Subject: [PATCH 1/9] format hidden code --- mdformat_rustfmt/__init__.py | 78 +++++++++++++++++++++++++++++------- tests/data/fixtures.md | 42 ++++++++++++++++++- 2 files changed, 104 insertions(+), 16 deletions(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index f4aa609..7f50299 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -4,9 +4,22 @@ import subprocess from typing import Callable +in_commented = False + +from collections.abc import Iterable + +def flatten(l): + for el in l: + if isinstance(el, Iterable) and not isinstance(el, (str, bytes)): + yield from flatten(el) + else: + yield el def format_rust(unformatted: str, _info_str: str) -> str: + global in_commented + unformatted = _for_each_line(unformatted, _hide_sharp) + unformatted_bytes = unformatted.encode("utf-8") result = subprocess.run( ["rustfmt"], @@ -14,32 +27,67 @@ def format_rust(unformatted: str, _info_str: str) -> str: stderr=subprocess.DEVNULL, input=unformatted_bytes, ) - if result.returncode: - raise Exception("Failed to format Rust code") + formatted = result.stdout.decode("utf-8") - formatted = _for_each_line(formatted, _unhide_sharp) - return formatted + + if result.returncode: + raise Exception("Failed to format Rust code\n" + formatted) + + in_commented = False + return _for_each_line(formatted, _unhide_sharp) + "\n" def _for_each_line(string: str, action: Callable[[str], str]) -> str: lines = string.split("\n") - lines = (action(line) for line in lines) + + lines = [action(line) for line in lines] + + lines = list(flatten(lines)) + + lines = list(filter(None, lines)) return "\n".join(lines) -_RUSTFMT_CUSTOM_COMMENT_PREFIX = "//#### " +_RUSTFMT_CUSTOM_COMMENT_PREFIX = "//__MDFORMAT_RUSTFMT__" +_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN = "//__MDFORMAT_RUSTFMT_COMMENT_BEGIN__" +_RUSTFMT_CUSTOM_COMMENT_BLOCK_END = "//__MDFORMAT_RUSTFMT_COMMENT_END__" -def _hide_sharp(line: str) -> str: +def _hide_sharp(line: str): + global in_commented stripped = line.strip() - if stripped.startswith("# ") or stripped == "#": - return _RUSTFMT_CUSTOM_COMMENT_PREFIX + line - return line + if stripped.startswith("#") and not line.startswith("##"): + if not in_commented: + in_commented = True + return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, stripped[1:]] + + return stripped[1:] + + if in_commented: + in_commented = False + return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_END, stripped] + + return stripped + + +def _unhide_sharp(line: str): + global in_commented + + if _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN in line: + in_commented = True + return None + + if _RUSTFMT_CUSTOM_COMMENT_BLOCK_END in line: + in_commented = False + return None + + if in_commented: + if line.startswith("#"): + return "#" + line + if line.startswith(" "): + return "#" + line[1:] + + return "# " + line -def _unhide_sharp(line: str) -> str: - if re.match(r"\s*" + re.escape(_RUSTFMT_CUSTOM_COMMENT_PREFIX), line): - # Remove the first "rustfmt custom comment prefix" and any leading - # whitespace the prefixed line originally had. - return re.sub(re.escape(_RUSTFMT_CUSTOM_COMMENT_PREFIX) + r"\s*", "", line, 1) return line diff --git a/tests/data/fixtures.md b/tests/data/fixtures.md index 5f86195..c310a9e 100644 --- a/tests/data/fixtures.md +++ b/tests/data/fixtures.md @@ -23,10 +23,50 @@ fn main() { . ```rust fn main() { - # let x=a();b();c(); +# let x = a(); +# b(); +# c(); let y = d(); e(); f(); } ``` . + +Format hidden lines +. +~~~rust +fn main() { + # let x=a();b();c(); +} +~~~ +. +```rust +fn main() { +# let x = a(); +# b(); +# c(); +} +``` +. + +Handle empty comment lines +. +~~~rust +fn main() { + # +# + # // comment +let s = "asdf +## literal hash"; +} +~~~ +. +```rust +fn main() { +# // comment + let s = "asdf +## literal hash"; +} +``` +. From da8d5b5f3423cb3cb3ee575854faf5eb443cacd3 Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Wed, 20 Jan 2021 22:25:11 -0600 Subject: [PATCH 2/9] refine tests --- mdformat_rustfmt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index 7f50299..62dd569 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -34,7 +34,7 @@ def format_rust(unformatted: str, _info_str: str) -> str: raise Exception("Failed to format Rust code\n" + formatted) in_commented = False - return _for_each_line(formatted, _unhide_sharp) + "\n" + return _for_each_line(formatted, _unhide_sharp).replace("\r", "") + "\n" def _for_each_line(string: str, action: Callable[[str], str]) -> str: From 5e19ebfb6967fc878f125c52bf65810765860abe Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Wed, 20 Jan 2021 22:38:18 -0600 Subject: [PATCH 3/9] fix linter errors --- mdformat_rustfmt/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index 62dd569..2443ddf 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -1,20 +1,21 @@ __version__ = "0.0.3" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT -import re import subprocess from typing import Callable +from collections.abc import Iterable + in_commented = False -from collections.abc import Iterable -def flatten(l): - for el in l: +def flatten(deep_list): + for el in deep_list: if isinstance(el, Iterable) and not isinstance(el, (str, bytes)): yield from flatten(el) else: yield el + def format_rust(unformatted: str, _info_str: str) -> str: global in_commented @@ -63,7 +64,7 @@ def _hide_sharp(line: str): return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, stripped[1:]] return stripped[1:] - + if in_commented: in_commented = False return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_END, stripped] From 1607d02c0c685f63f48c68fb215271826d4b246b Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Wed, 20 Jan 2021 23:20:03 -0600 Subject: [PATCH 4/9] handle derive statements --- mdformat_rustfmt/__init__.py | 32 +++++++++++++++++++++++++------- tests/data/fixtures.md | 25 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index 2443ddf..6025439 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -1,9 +1,8 @@ __version__ = "0.0.3" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT +from collections.abc import Iterable import subprocess from typing import Callable -from collections.abc import Iterable - in_commented = False @@ -49,21 +48,31 @@ def _for_each_line(string: str, action: Callable[[str], str]) -> str: return "\n".join(lines) -_RUSTFMT_CUSTOM_COMMENT_PREFIX = "//__MDFORMAT_RUSTFMT__" _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN = "//__MDFORMAT_RUSTFMT_COMMENT_BEGIN__" _RUSTFMT_CUSTOM_COMMENT_BLOCK_END = "//__MDFORMAT_RUSTFMT_COMMENT_END__" +_RUSTFMT_CUSTOM_COMMENT_ESCAPE = "//__MDFORMAT_RUSTFMT_COMMENT_ESCAPE__" def _hide_sharp(line: str): global in_commented stripped = line.strip() - if stripped.startswith("#") and not line.startswith("##"): + if stripped.startswith("#"): if not in_commented: in_commented = True - return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, stripped[1:]] - return stripped[1:] + if stripped.startswith("##"): + return [ + _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, + _RUSTFMT_CUSTOM_COMMENT_ESCAPE, + stripped[1:], + ] + else: + return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, stripped[1:]] + if stripped.startswith("##"): + return [_RUSTFMT_CUSTOM_COMMENT_ESCAPE, stripped[1:]] + else: + return stripped[1:] if in_commented: in_commented = False @@ -72,8 +81,12 @@ def _hide_sharp(line: str): return stripped +next_line_escape = False + + def _unhide_sharp(line: str): global in_commented + global next_line_escape if _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN in line: in_commented = True @@ -83,8 +96,13 @@ def _unhide_sharp(line: str): in_commented = False return None + if _RUSTFMT_CUSTOM_COMMENT_ESCAPE in line: + next_line_escape = True + return None + if in_commented: - if line.startswith("#"): + if line.startswith("#") and next_line_escape: + next_line_escape = False return "#" + line if line.startswith(" "): return "#" + line[1:] diff --git a/tests/data/fixtures.md b/tests/data/fixtures.md index c310a9e..88ebd7c 100644 --- a/tests/data/fixtures.md +++ b/tests/data/fixtures.md @@ -70,3 +70,28 @@ fn main() { } ``` . + +Handle hidden derive and attr statements +. +~~~rust +# #[derive(Debug)] +struct MyStruct {} +~~~ +. +```rust +# #[derive(Debug)] +struct MyStruct {} +``` +. +Handle derive and attr statements +. +~~~rust +#[derive(Debug)] +struct MyStruct {} +~~~ +. +```rust +#[derive(Debug)] +struct MyStruct {} +``` +. \ No newline at end of file From 5d708dd9487a02a8a705014d0026e50bf7ab7cc1 Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Thu, 21 Jan 2021 10:40:36 -0600 Subject: [PATCH 5/9] add nightly blank lines test --- mdformat_rustfmt/__init__.py | 26 +++++++++++++++----------- rustfmt.toml | 1 + tests/data/fixtures.md | 19 ++++++++++++++++++- tests/test_mdformat_rustfmt.py | 2 ++ 4 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 rustfmt.toml diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index 6025439..ced3d27 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -51,6 +51,7 @@ def _for_each_line(string: str, action: Callable[[str], str]) -> str: _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN = "//__MDFORMAT_RUSTFMT_COMMENT_BEGIN__" _RUSTFMT_CUSTOM_COMMENT_BLOCK_END = "//__MDFORMAT_RUSTFMT_COMMENT_END__" _RUSTFMT_CUSTOM_COMMENT_ESCAPE = "//__MDFORMAT_RUSTFMT_COMMENT_ESCAPE__" +_RUSTFMT_CUSTOM_COMMENT_BLANK_LINE = "//__MDFORMAT_RUSTFMT_COMMENT_BLANK_LINE__" def _hide_sharp(line: str): @@ -58,21 +59,21 @@ def _hide_sharp(line: str): stripped = line.strip() if stripped.startswith("#"): + tokens = [] + if not in_commented: in_commented = True + tokens.append(_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN) - if stripped.startswith("##"): - return [ - _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, - _RUSTFMT_CUSTOM_COMMENT_ESCAPE, - stripped[1:], - ] - else: - return [_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN, stripped[1:]] if stripped.startswith("##"): - return [_RUSTFMT_CUSTOM_COMMENT_ESCAPE, stripped[1:]] - else: - return stripped[1:] + tokens.append(_RUSTFMT_CUSTOM_COMMENT_ESCAPE) + + # if stripped == "#": + # tokens.append(_RUSTFMT_CUSTOM_COMMENT_BLANK_LINE) + + tokens.append(stripped[1:]) + + return tokens if in_commented: in_commented = False @@ -100,6 +101,9 @@ def _unhide_sharp(line: str): next_line_escape = True return None + if _RUSTFMT_CUSTOM_COMMENT_BLANK_LINE in line: + return "#" + if in_commented: if line.startswith("#") and next_line_escape: next_line_escape = False diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..1b4670e --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +blank_lines_lower_bound = 1 diff --git a/tests/data/fixtures.md b/tests/data/fixtures.md index 88ebd7c..25aef28 100644 --- a/tests/data/fixtures.md +++ b/tests/data/fixtures.md @@ -94,4 +94,21 @@ struct MyStruct {} #[derive(Debug)] struct MyStruct {} ``` -. \ No newline at end of file +. +[NIGHTLY] Preserve blank lines +. +~~~rust +# struct Something {} +# +# +# fn main() {} +# +# +~~~ +. +```rust +# struct Something {} +# +# fn main() {} +``` +. diff --git a/tests/test_mdformat_rustfmt.py b/tests/test_mdformat_rustfmt.py index 425902c..617a4e1 100644 --- a/tests/test_mdformat_rustfmt.py +++ b/tests/test_mdformat_rustfmt.py @@ -12,6 +12,8 @@ ) def test_fixtures(line, title, text, expected): """Test fixtures in tests/data/fixtures.md.""" + if "NIGHTLY" in title: + pytest.skip("nightly test not supported on stable") md_new = mdformat.text(text, codeformatters={"rust"}) if md_new != expected: print("Formatted (unexpected) Markdown below:") From 119cf04a7bebfbb8fadbabcf2844a5d62a36ed14 Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Thu, 21 Jan 2021 11:31:30 -0600 Subject: [PATCH 6/9] slightly better blank lines handling --- mdformat_rustfmt/__init__.py | 17 ++++++++++++++--- tests/data/fixtures.md | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index ced3d27..61dcf2d 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -34,7 +34,9 @@ def format_rust(unformatted: str, _info_str: str) -> str: raise Exception("Failed to format Rust code\n" + formatted) in_commented = False - return _for_each_line(formatted, _unhide_sharp).replace("\r", "") + "\n" + remove_newlines = False + + return _for_each_line(formatted, _unhide_sharp).replace("\r", "") def _for_each_line(string: str, action: Callable[[str], str]) -> str: @@ -44,7 +46,7 @@ def _for_each_line(string: str, action: Callable[[str], str]) -> str: lines = list(flatten(lines)) - lines = list(filter(None, lines)) + lines = [x for x in lines if x != None] return "\n".join(lines) @@ -58,7 +60,7 @@ def _hide_sharp(line: str): global in_commented stripped = line.strip() - if stripped.startswith("#"): + if stripped.startswith("# ") or stripped.startswith("##") or stripped == "#": tokens = [] if not in_commented: @@ -83,13 +85,16 @@ def _hide_sharp(line: str): next_line_escape = False +remove_newlines = False def _unhide_sharp(line: str): global in_commented global next_line_escape + global remove_newlines if _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN in line: + remove_newlines = True in_commented = True return None @@ -105,9 +110,15 @@ def _unhide_sharp(line: str): return "#" if in_commented: + if line == "" and remove_newlines: + return None + + remove_newlines = False + if line.startswith("#") and next_line_escape: next_line_escape = False return "#" + line + if line.startswith(" "): return "#" + line[1:] diff --git a/tests/data/fixtures.md b/tests/data/fixtures.md index 25aef28..9fdcc64 100644 --- a/tests/data/fixtures.md +++ b/tests/data/fixtures.md @@ -59,6 +59,8 @@ fn main() { # // comment let s = "asdf ## literal hash"; +let x = 5; +let y = 6; } ~~~ . @@ -67,6 +69,8 @@ fn main() { # // comment let s = "asdf ## literal hash"; + let x = 5; + let y = 6; } ``` . @@ -104,11 +108,23 @@ struct MyStruct {} # fn main() {} # # +trait SomeTrait { fn nothing() {} } + struct Another; + + impl Another {} ~~~ . ```rust # struct Something {} # # fn main() {} +# +trait SomeTrait { + fn nothing() {} +} + +struct Another; + +impl Another {} ``` . From 862d46fbb0ca6dae642e0c6d0ec068cbb1f32688 Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Thu, 21 Jan 2021 11:37:56 -0600 Subject: [PATCH 7/9] fix lints --- mdformat_rustfmt/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index 61dcf2d..324fe94 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -17,6 +17,7 @@ def flatten(deep_list): def format_rust(unformatted: str, _info_str: str) -> str: global in_commented + global remove_newlines unformatted = _for_each_line(unformatted, _hide_sharp) @@ -46,7 +47,7 @@ def _for_each_line(string: str, action: Callable[[str], str]) -> str: lines = list(flatten(lines)) - lines = [x for x in lines if x != None] + lines = [x for x in lines if x is not None] return "\n".join(lines) From 3f47f24cdcac7d3303bd79737c01d4aa8a9fca2a Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Thu, 21 Jan 2021 18:07:30 -0600 Subject: [PATCH 8/9] fix deleted lines bug --- mdformat_rustfmt/__init__.py | 14 +++++++++++--- tests/data/fixtures.md | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/mdformat_rustfmt/__init__.py b/mdformat_rustfmt/__init__.py index 324fe94..983e15a 100644 --- a/mdformat_rustfmt/__init__.py +++ b/mdformat_rustfmt/__init__.py @@ -1,6 +1,7 @@ __version__ = "0.0.3" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT from collections.abc import Iterable +import re import subprocess from typing import Callable @@ -97,15 +98,22 @@ def _unhide_sharp(line: str): if _RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN in line: remove_newlines = True in_commented = True - return None + line = re.sub( + re.escape(_RUSTFMT_CUSTOM_COMMENT_BLOCK_BEGIN), "", line, 1 + ).rstrip() + return line or None if _RUSTFMT_CUSTOM_COMMENT_BLOCK_END in line: in_commented = False - return None + line = re.sub( + re.escape(_RUSTFMT_CUSTOM_COMMENT_BLOCK_END), "", line, 1 + ).rstrip() + return line or None if _RUSTFMT_CUSTOM_COMMENT_ESCAPE in line: next_line_escape = True - return None + line = re.sub(re.escape(_RUSTFMT_CUSTOM_COMMENT_ESCAPE), "", line, 1).rstrip() + return line or None if _RUSTFMT_CUSTOM_COMMENT_BLANK_LINE in line: return "#" diff --git a/tests/data/fixtures.md b/tests/data/fixtures.md index 9fdcc64..e3af3a4 100644 --- a/tests/data/fixtures.md +++ b/tests/data/fixtures.md @@ -128,3 +128,26 @@ struct Another; impl Another {} ``` . +Not sure +. +~~~rust +# fn main() -> Result<(), amethyst::Error> { +# let game_data = DispatcherBuilder::default().with_bundle( + // inside your rendering bundle setup + RenderingBundle::::new() + .with_plugin(RenderFlat2D::default()) +# )?; +# Ok(()) +# } +~~~ +. +```rust +# fn main() -> Result<(), amethyst::Error> { +# let game_data = DispatcherBuilder::default().with_bundle( + // inside your rendering bundle setup + RenderingBundle::::new().with_plugin(RenderFlat2D::default()), +# )?; +# Ok(()) +# } +``` +. \ No newline at end of file From 6fcfd5e68844884afa62b26df04b1a34ab11d69e Mon Sep 17 00:00:00 2001 From: Emory Petermann Date: Thu, 21 Jan 2021 18:25:59 -0600 Subject: [PATCH 9/9] use better test name --- tests/data/fixtures.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/fixtures.md b/tests/data/fixtures.md index e3af3a4..4b47921 100644 --- a/tests/data/fixtures.md +++ b/tests/data/fixtures.md @@ -128,7 +128,7 @@ struct Another; impl Another {} ``` . -Not sure +Comment collapse does not delete lines . ~~~rust # fn main() -> Result<(), amethyst::Error> {