Improve String Handling (#1132)

This pull request's main intention is to wraps long strings (as requested by #182); however, it also provides better string handling in general and, in doing so, closes the following issues:

Closes #26
Closes #182
Closes #933
Closes #1183
Closes #1243

blib2to3/pgen2/pgen.py

@@ -167,8 +167,8 @@ def calcfirst(self, name: Text) -> None:
             for symbol in itsfirst:
                 if symbol in inverse:
                     raise ValueError(
-                        "rule %s is ambiguous; %s is in the"
-                        " first sets of %s as well as %s"
+                        "rule %s is ambiguous; %s is in the first sets of %s as well"
+                        " as %s"
                         % (name, symbol, label, inverse[symbol])
                     )
                 inverse[symbol] = label

blib2to3/pgen2/tokenize.py

@@ -78,7 +78,9 @@ def _combinations(*l):
 Whitespace = r"[ \f\t]*"
 Comment = r"#[^\r\n]*"
 Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment)
-Name = r"\w+"  # this is invalid but it's fine because Name comes after Number in all groups
+Name = (  # this is invalid but it's fine because Name comes after Number in all groups
+    r"\w+"
+)
 
 Binnumber = r"0[bB]_?[01]+(?:_[01]+)*"
 Hexnumber = r"0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?"

gallery/gallery.py

@@ -268,8 +268,7 @@ def main() -> None:
         "-v",
         "--version",
         help=(
-            "Version for given PyPI package. "
-            "Will be discarded if used with -t option."
+            "Version for given PyPI package. Will be discarded if used with -t option."
         ),
     )
     parser.add_argument(

tests/data/cantfit.py

@@ -35,7 +35,7 @@
 """.split():
     if key in self.connect_kwargs:
         raise ValueError(err.format(key))
-concatenated_strings = "some strings that are" "concatenated implicitly, so if you put them on separate" "lines it will fit"
+concatenated_strings = "some strings that are " "concatenated implicitly, so if you put them on separate " "lines it will fit"
 del concatenated_strings, string_variable_name, normal_function_name, normal_name, need_more_to_make_the_line_long_enough
 
 
@@ -75,8 +75,10 @@
 )
 # long arguments
 normal_name = normal_function_name(
-    "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit",
-    "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs",
+    "but with super long string arguments that on their own exceed the line limit so"
+    " there's no way it can ever fit",
+    "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs"
+    " with spam and eggs and spam with eggs",
     this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0,
 )
 string_variable_name = "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do"  # noqa
@@ -88,8 +90,8 @@
     if key in self.connect_kwargs:
         raise ValueError(err.format(key))
 concatenated_strings = (
-    "some strings that are"
-    "concatenated implicitly, so if you put them on separate"
+    "some strings that are "
+    "concatenated implicitly, so if you put them on separate "
     "lines it will fit"
 )
 del (

tests/data/comments4.py

@@ -13,49 +13,39 @@ class C:
             # metadata_version errors.
             (
                 {},
-                "None is an invalid value for Metadata-Version. "
-                "Error: This field is required. "
-                "see "
-                "https://packaging.python.org/specifications/core-metadata",
+                "None is an invalid value for Metadata-Version. Error: This field is"
+                " required. see"
+                " https://packaging.python.org/specifications/core-metadata",
             ),
             (
                 {"metadata_version": "-1"},
-                "'-1' is an invalid value for Metadata-Version. "
-                "Error: Unknown Metadata Version "
-                "see "
-                "https://packaging.python.org/specifications/core-metadata",
+                "'-1' is an invalid value for Metadata-Version. Error: Unknown Metadata"
+                " Version see"
+                " https://packaging.python.org/specifications/core-metadata",
             ),
             # name errors.
             (
                 {"metadata_version": "1.2"},
-                "'' is an invalid value for Name. "
-                "Error: This field is required. "
-                "see "
-                "https://packaging.python.org/specifications/core-metadata",
+                "'' is an invalid value for Name. Error: This field is required. see"
+                " https://packaging.python.org/specifications/core-metadata",
             ),
             (
                 {"metadata_version": "1.2", "name": "foo-"},
-                "'foo-' is an invalid value for Name. "
-                "Error: Must start and end with a letter or numeral and "
-                "contain only ascii numeric and '.', '_' and '-'. "
-                "see "
-                "https://packaging.python.org/specifications/core-metadata",
+                "'foo-' is an invalid value for Name. Error: Must start and end with a"
+                " letter or numeral and contain only ascii numeric and '.', '_' and"
+                " '-'. see https://packaging.python.org/specifications/core-metadata",
             ),
             # version errors.
             (
                 {"metadata_version": "1.2", "name": "example"},
-                "'' is an invalid value for Version. "
-                "Error: This field is required. "
-                "see "
-                "https://packaging.python.org/specifications/core-metadata",
+                "'' is an invalid value for Version. Error: This field is required. see"
+                " https://packaging.python.org/specifications/core-metadata",
             ),
             (
                 {"metadata_version": "1.2", "name": "example", "version": "dog"},
-                "'dog' is an invalid value for Version. "
-                "Error: Must start and end with a letter or numeral and "
-                "contain only ascii numeric and '.', '_' and '-'. "
-                "see "
-                "https://packaging.python.org/specifications/core-metadata",
+                "'dog' is an invalid value for Version. Error: Must start and end with"
+                " a letter or numeral and contain only ascii numeric and '.', '_' and"
+                " '-'. see https://packaging.python.org/specifications/core-metadata",
             ),
         ],
     )

tests/data/comments6.py

@@ -104,7 +104,9 @@ def func(
     )
 
 
-result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+result = (  # aaa
+    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+)
 
 AAAAAAAAAAAAA = [AAAAAAAAAAAAA] + SHARED_AAAAAAAAAAAAA + USER_AAAAAAAAAAAAA + AAAAAAAAAAAAA  # type: ignore
 

tests/data/comments7.py

@@ -87,9 +87,13 @@ def func():
 
 result = 1  # look ma, no comment migration xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
-result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+result = (  # aaa
+    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+)
 
-result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+result = (  # aaa
+    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+)
 
 
 def func():

tests/data/composition.py

@@ -10,13 +10,13 @@ def test(self) -> None:
             )
             self.assertEqual(
                 unstyle(str(report)),
-                "2 files reformatted, 1 file left unchanged, "
-                "1 file failed to reformat.",
+                "2 files reformatted, 1 file left unchanged, 1 file failed to"
+                " reformat.",
             )
             self.assertEqual(
                 unstyle(str(report)),
-                "2 files reformatted, 2 files left unchanged, "
-                "2 files failed to reformat.",
+                "2 files reformatted, 2 files left unchanged, 2 files failed to"
+                " reformat.",
             )
             for i in (a,):
                 if (
@@ -40,8 +40,7 @@ def test(self) -> None:
             items=items[:num_items]
         )
         return (
-            "Utterly failed doctest test for %s\n"
-            '  File "%s", line %s, in %s\n\n%s'
+            'Utterly failed doctest test for %s\n   File "%s", line %s, in %s\n\n%s'
             % (test.name, test.filename, lineno, lname, err)
         )
 
@@ -150,8 +149,8 @@ def tricky_asserts(self) -> None:
             key8: value8,
             key9: value9,
         }, (
-            "Not what we expected and the message is too long to fit "
-            "in one line because it's too long"
+            "Not what we expected and the message is too long to fit in one line"
+            " because it's too long"
         )
 
         dis_c_instance_method = """\