Merge python#5

5: Support Py3xwarning warnings with a fix argument r=vext01 a=nanjekyejoannah

This PR does the following:

- Adds a `fix` argument to allow for flexibility in adding a possible fix to a warning
- The new warning format is used for the already merged warnings for numbers

Co-authored-by: Joannah Nanjekye <[email protected]>

Doc/c-api/exceptions.rst

@@ -319,6 +319,12 @@ is a separate error indicator for each thread.
    and *registry* arguments may be set to *NULL* to get the default effect
    described there.
 
+.. c:function:: int PyErr_WarnExplicit_WithFix(PyObject *category, const char *message, const char *fix, const char *filename, int lineno, const char *module, PyObject *registry)
+
+   Issue a warning message and a potential fix.  This warning is the 
+   same as `PyErr_WarnExplicit` but adds a *fix* argument to allow 
+   for `Py3xWarning` warnings to suggest potential fixes for Python
+   3.x incompatible code.
 
 .. c:function:: int PyErr_WarnPy3k(char *message, int stacklevel)
 
@@ -715,7 +721,7 @@ the variables:
 +------------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_UserWarning`              | :exc:`UserWarning`              |          |
 +------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_3xWarning`                | :exc:`Py3xWarning`                |          |
+| :c:data:`PyExc_3xWarning`                | :exc:`Py3xWarning`              |          |
 +------------------------------------------+---------------------------------+----------+
 
 Notes:

Doc/library/warnings.rst

@@ -95,7 +95,7 @@ following warnings category classes are currently defined:
 |                                  | bytes and bytearray.                          |
 +----------------------------------+-----------------------------------------------+
 | :exc:`Py3xWarning`               | Base class for warnings about 3.x             |
-                                   | compatibility                                 |                         |
+                                   | compatibility.                                 |                         |
 +----------------------------------+-----------------------------------------------+
 
 While these are technically built-in exceptions, they are documented here,

Include/warnings.h

@@ -9,6 +9,8 @@ PyAPI_FUNC(void) _PyWarnings_Init(void);
 PyAPI_FUNC(int) PyErr_WarnEx(PyObject *, const char *, Py_ssize_t);
 PyAPI_FUNC(int) PyErr_WarnExplicit(PyObject *, const char *, const char *, int,
                                     const char *, PyObject *);
+PyAPI_FUNC(int) PyErr_WarnExplicit_WithFix(PyObject *, const char *, const char *, const char *, int,
+                                    const char *, PyObject *);
 
 #define PyErr_WarnPy3k(msg, stacklevel) \
   (Py_Py3kWarningFlag ? PyErr_WarnEx(PyExc_DeprecationWarning, msg, stacklevel) : 0)

Lib/test/test_grammar.py

@@ -66,14 +66,19 @@ def test_plain_integers(self):
             self.fail('Weird maxint value %r' % maxint)
 
         if sys.py3kwarning:
-            with warnings.catch_warnings():
-                warnings.filterwarnings('error', category=Py3xWarning)
-                with self.assertRaises(Py3xWarning) as oct:
-                    compile('032', '<test string>', 'eval')
-                self.assertIn("octal literals are not supported in 3.x;\n" 
-                              "drop the leading 0",
-                              str(oct.exception))
-
+            with warnings.catch_warnings(record=True) as w:
+                warnings.filterwarnings('always', category=Py3xWarning)
+                self.assertEqual(034, 28)
+                self.assertEqual(01, 1)
+                for warning in w:
+                    self.assertTrue(Py3xWarning is w.category)
+                    self.assertEqual(str(w.message), "using just a '0' prefix for octal literals is not supported in 3.x: " \
+                                        "use the '0o' prefix for octal integers")
+                    self.assertEqual(len(w), 2)
+                    self.assertIn("using just a '0' prefix for octal literals is not supported in 3.x:: \n" 
+                                    "use the '0o' prefix for octal integers",
+                                    str(oct.exception))
+
     def test_long_integers(self):
         x = 0L
         x = 0l

Lib/test/test_optparse.py

@@ -23,7 +23,7 @@
      BadOptionError, OptionValueError, Values
 from optparse import _match_abbrev
 from optparse import _parse_num
-from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import run_unittest, check_py3k_warnings, warnings
 
 retype = type(re.compile(''))
 
@@ -1656,14 +1656,20 @@ def test_numeric_options(self):
                              "option -l: invalid long integer value: '0x12x'")
 
     def test_parse_num_3k_warnings(self):
-        expected = 'the L suffix is not supported in 3.x; simply drop the suffix, \
-                    or accept the auto fixer modifications'
-        with check_py3k_warnings((expected, Py3xWarning)):
+        with warnings.catch_warnings(record=True) as w:
+            warnings.filterwarnings("always", "the L suffix is not supported in 3.x: \
+                                    drop the suffix",
+                                Py3xWarning)
             x = 10L
             y = 8L
             z = x + y
             a = x * y
             b = x - y
+            for warning in w:
+                self.assertTrue(Py3xWarning is w.category)
+                self.assertEqual(str(w.message), "the L suffix is not supported in 3.x: " \
+                                 "drop the suffix")
+                self.assertEqual(len(w), 5)
 
 
 def test_main():

Lib/test/test_warnings.py

@@ -70,7 +70,8 @@ class PublicAPITests(BaseTest):
 
     def test_module_all_attribute(self):
         self.assertTrue(hasattr(self.module, '__all__'))
-        target_api = ["warn", "warn_explicit", "showwarning",
+        target_api = ["warn", "warn_explicit", "warn_explicit_with_fix"
+                      "showwarning", "showwarningwithfix", "formatwarningwithfix",
                       "formatwarning", "filterwarnings", "simplefilter",
                       "resetwarnings", "catch_warnings"]
         self.assertSetEqual(set(self.module.__all__),
@@ -161,6 +162,23 @@ def test_once(self):
                                     42)
             self.assertEqual(len(w), 0)
 
+    def test_once_with_fix(self):
+        with original_warnings.catch_warnings(record=True,
+                module=self.module) as w:
+            self.module.resetwarnings()
+            self.module.filterwarnings("once", category=UserWarning)
+            message = UserWarning("FilterTests.test_once")
+            fix = 'some fix'
+            self.module.warn_explicit_with_fix(message, fix, UserWarning, "test_warnings.py",
+                                    42)
+            del w[:]
+            self.module.warn_explicit_with_fix(message, fix, UserWarning, "test_warnings.py",
+                                    13)
+            self.assertEqual(len(w), 0)
+            self.module.warn_explicit_with_fix(message, fix, UserWarning, "test_warnings2.py",
+                                    42)
+            self.assertEqual(len(w), 0)
+
     def test_inheritance(self):
         with original_warnings.catch_warnings(module=self.module) as w:
             self.module.resetwarnings()
@@ -230,7 +248,8 @@ class PyFilterTests(BaseTest, FilterTests):
 
 class WarnTests(unittest.TestCase):
 
-    """Test warnings.warn() and warnings.warn_explicit()."""
+    """Test warnings.warn(), warnings.warn_explicit()
+    and warnings.warn_explicit_with_fix()."""
 
     def test_message(self):
         with original_warnings.catch_warnings(record=True,
@@ -644,6 +663,25 @@ def test_formatwarning(self):
         self.assertEqual(expect, self.module.formatwarning(message,
                                     category, file_name, line_num, file_line))
 
+    def test_formatwarningwithfix(self):
+        message = "msg"
+        fix = 'fix'
+        category = Warning
+        file_name = os.path.splitext(warning_tests.__file__)[0] + '.py'
+        line_num = 3
+        file_line = linecache.getline(file_name, line_num).strip()
+        format = "%s:%s: %s: %s: %s\n  %s\n"
+        expect = format % (file_name, line_num, category.__name__, message,
+                            fix, file_line)
+        self.assertEqual(expect, self.module.formatwarningwithfix(message, fix,
+                                                category, file_name, line_num))
+        # Test the 'line' argument.
+        file_line += " for the win!"
+        expect = format % (file_name, line_num, category.__name__, message,
+                            fix, file_line)
+        self.assertEqual(expect, self.module.formatwarningwithfix(message, fix,
+                                    category, file_name, line_num, file_line))
+
     @test_support.requires_unicode
     def test_formatwarning_unicode_msg(self):
         message = u"msg"
@@ -722,6 +760,28 @@ def test_showwarning(self):
                                 file_object, expected_file_line)
         self.assertEqual(expect, file_object.getvalue())
 
+    def test_showwarningwithfix(self):
+        file_name = os.path.splitext(warning_tests.__file__)[0] + '.py'
+        line_num = 3
+        expected_file_line = linecache.getline(file_name, line_num).strip()
+        message = 'msg'
+        fix = 'fix'
+        category = Warning
+        file_object = StringIO.StringIO()
+        expect = self.module.formatwarningwithfix(message, fix, category, file_name,
+                                            line_num)
+        self.module.showwarningwithfix(message, fix, category, file_name, line_num,
+                                file_object)
+        self.assertEqual(file_object.getvalue(), expect)
+        # Test 'line' argument.
+        expected_file_line += "for the win!"
+        expect = self.module.formatwarningwithfix(message, fix, category, file_name,
+                                            line_num, expected_file_line)
+        file_object = StringIO.StringIO()
+        self.module.showwarningwithfix(message, fix, category, file_name, line_num,
+                                file_object, expected_file_line)
+        self.assertEqual(expect, file_object.getvalue())
+
 class CWarningsDisplayTests(BaseTest, WarningsDisplayTests):
     module = c_warnings
 

Lib/warnings.py

@@ -7,8 +7,9 @@
 import sys
 import types
 
-__all__ = ["warn", "warn_explicit", "showwarning",
-           "formatwarning", "filterwarnings", "simplefilter",
+__all__ = ["warn", "warn_explicit", "warn_explicit_with_fix",
+           "showwarning", "showwarningwithfix", "formatwarning",
+           "formatwarningwithfix", "filterwarnings", "simplefilter",
            "resetwarnings", "catch_warnings"]
 
 
@@ -37,6 +38,21 @@ def _show_warning(message, category, filename, lineno, file=None, line=None):
 # triggered.
 showwarning = _show_warning
 
+def _show_warning_with_fix(message, fix, category, filename, lineno, file=None, line=None):
+    """Hook to write a warning to a file; replace if you like."""
+    if file is None:
+        file = sys.stderr
+        if file is None:
+            # sys.stderr is None - warnings get lost
+            return
+    try:
+        file.write(formatwarningwithfix(message, fix, category, filename, lineno, line))
+    except (IOError, UnicodeError):
+        pass # the file (probably stderr) is invalid - this warning gets lost.
+# Keep a working version around in case the deprecation of the old API is
+# triggered.
+showwarningwithfix = _show_warning_with_fix
+
 def formatwarning(message, category, filename, lineno, line=None):
     """Function to format a warning the standard way."""
     try:
@@ -64,6 +80,34 @@ def formatwarning(message, category, filename, lineno, line=None):
     s = "%s:%s" % (filename, s)
     return s
 
+def formatwarningwithfix(message, fix, category, filename, lineno, line=None):
+    """Function to format a warning the standard way with a fix."""
+    try:
+        unicodetype = unicode
+    except NameError:
+        unicodetype = ()
+    try:
+        message = str(message)
+        fix = str(fix)
+    except UnicodeEncodeError:
+        pass
+    s =  "%s: %s: %s: %s\n" % (lineno, category.__name__, message, fix)
+    line = linecache.getline(filename, lineno) if line is None else line
+    if line:
+        line = line.strip()
+        if isinstance(s, unicodetype) and isinstance(line, str):
+            line = unicode(line, 'latin1')
+        s += "  %s\n" % line
+    if isinstance(s, unicodetype) and isinstance(filename, str):
+        enc = sys.getfilesystemencoding()
+        if enc:
+            try:
+                filename = unicode(filename, enc)
+            except UnicodeDecodeError:
+                pass
+    s = "%s:%s" % (filename, s)
+    return s
+
 def filterwarnings(action, message="", category=Warning, module="", lineno=0,
                    append=0):
     """Insert an entry into the list of warnings filters (at the front).
@@ -299,6 +343,71 @@ def warn_explicit(message, category, filename, lineno,
     # Print message and context
     showwarning(message, category, filename, lineno)
 
+def warn_explicit_with_fix(message, fix, category, filename, lineno,
+                  module=None, registry=None, module_globals=None):
+    lineno = int(lineno)
+    if module is None:
+        module = filename or "<unknown>"
+        if module[-3:].lower() == ".py":
+            module = module[:-3] # XXX What about leading pathname?
+    if registry is None:
+        registry = {}
+    if isinstance(message, Warning):
+        text = str(message)
+        category = message.__class__
+    else:
+        text = message
+        message = category(message)
+    key = (text, category, lineno)
+    # Quick test for common case
+    if registry.get(key):
+        return
+    # Search the filters
+    for item in filters:
+        action, msg, cat, mod, ln = item
+        if ((msg is None or msg.match(text)) and
+            issubclass(category, cat) and
+            (mod is None or mod.match(module)) and
+            (ln == 0 or lineno == ln)):
+            break
+    else:
+        action = defaultaction
+    # Early exit actions
+    if action == "ignore":
+        registry[key] = 1
+        return
+
+    # Prime the linecache for formatting, in case the
+    # "file" is actually in a zipfile or something.
+    linecache.getlines(filename, module_globals)
+
+    if action == "error":
+        raise message
+    # Other actions
+    if action == "once":
+        registry[key] = 1
+        oncekey = (text, category)
+        if onceregistry.get(oncekey):
+            return
+        onceregistry[oncekey] = 1
+    elif action == "always":
+        pass
+    elif action == "module":
+        registry[key] = 1
+        altkey = (text, category, 0)
+        if registry.get(altkey):
+            return
+        registry[altkey] = 1
+    elif action == "default":
+        registry[key] = 1
+    else:
+        # Unrecognized actions are errors
+        raise RuntimeError(
+              "Unrecognized action (%r) in warnings.filters:\n %s" %
+              (action, item))
+    # Print message and context
+    showwarningwithfix(message, fix, category, filename, lineno)
+
 
 class WarningMessage(object):
 
@@ -395,7 +504,7 @@ def __exit__(self, *exc_info):
 _warnings_defaults = False
 try:
     from _warnings import (filters, default_action, once_registry,
-                            warn, warn_explicit)
+                            warn, warn_explicit, warn_explicit_with_fix)
     defaultaction = default_action
     onceregistry = once_registry
     _warnings_defaults = True

Objects/exceptions.c

@@ -1982,7 +1982,8 @@ SimpleExtendsException(PyExc_Warning, SyntaxWarning,
 
 
 /*
- *    3xWarning extends Warning
+ *    Py3xWarning extends Warning
+ *    XXX(nanjekyejoannah): Suppress this warning for legacy tests for now
  */
 SimpleExtendsException(PyExc_Warning, Py3xWarning,
                        "Base class for warnings about 3.x compatibility.");

PC/os2emx/python27.def

@@ -914,6 +914,7 @@ EXPORTS
   "_PyErr_BadInternalCall"
   "PyErr_Warn"
   "PyErr_WarnExplicit"
+  "PyErr_WarnExplicit_WithFix"
 
 ; From python27_s.lib(frozen)
   "PyImport_FrozenModules"

Parser/tokenizer.c

@@ -1433,14 +1433,6 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
     /* Number */
     if (isdigit(c)) {
         if (c == '0') {
-            if (Py_Py3kWarningFlag) {
-                if (PyErr_WarnExplicit(PyExc_Py3xWarning,
-                                    "octal literals are not supported in 3.x;\n" 
-                                    "drop the leading 0",
-                                    tok->filename, tok->lineno, NULL, NULL)) {
-                    return NULL;
-                }
-            }
             /* Hex, octal or binary -- maybe. */
             c = tok_nextc(tok);
             if (c == '.')
@@ -1449,6 +1441,15 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
             if (c == 'j' || c == 'J')
                 goto imaginary;
 #endif
+            if (c != 'o' || c != 'O') {
+                char buf[100];
+                if (Py_Py3kWarningFlag) {
+                    if (PyErr_WarnExplicit_WithFix(PyExc_Py3xWarning, "using just a '0' prefix for octal literals is not supported in 3.x", 
+                                                   "use the '0o' prefix for octal integers, if you intended the integer to be decimal", tok->filename, tok->lineno, NULL, NULL)) {
+                        return NULL;
+                    }
+                }
+            }
             if (c == 'x' || c == 'X') {
 
                 /* Hex */