bpo-28806: Continue work: improve the netrc library

Doc/library/netrc.rst

@@ -41,6 +41,10 @@ the Unix :program:`ftp` program and other FTP clients.
    .. versionchanged:: 3.10
       :class:`netrc` try UTF-8 encoding before using locale specific
       encoding.
+      The entry in the netrc file no longer needs to contain all tokens.  The missing
+      tokens' value default to an empty string.  All the tokens and their values now
+      can contain arbitrary characters, like whitespace and non-ASCII characters.
+      If the login name is anonymous, it won't trigger the security check.
 
 
 .. exception:: NetrcParseError
@@ -85,10 +89,3 @@ Instances of :class:`~netrc.netrc` have public instance variables:
 .. attribute:: netrc.macros
 
    Dictionary mapping macro names to string lists.
-
-.. note::
-
-   Passwords are limited to a subset of the ASCII character set.  All ASCII
-   punctuation is allowed in passwords, however, note that whitespace and
-   non-printable characters are not allowed in passwords.  This is a limitation
-   of the way the .netrc file is parsed and may be removed in the future.

Lib/netrc.py

@@ -19,6 +19,50 @@ def __str__(self):
         return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno)
 
 
+class _netrclex:
+    def __init__(self, fp):
+        self.lineno = 1
+        self.instream = fp
+        self.whitespace = "\n\t\r "
+        self.pushback = []
+
+    def _read_char(self):
+        ch = self.instream.read(1)
+        if ch == "\n":
+            self.lineno += 1
+        return ch
+
+    def get_token(self):
+        if self.pushback:
+            return self.pushback.pop(0)
+        token = ""
+        fiter = iter(self._read_char, "")
+        for ch in fiter:
+            if ch in self.whitespace:
+                continue
+            if ch == '"':
+                for ch in fiter:
+                    if ch == '"':
+                        return token
+                    elif ch == "\\":
+                        ch = self._read_char()
+                    token += ch
+            else:
+                if ch == "\\":
+                    ch = self._read_char()
+                token += ch
+                for ch in fiter:
+                    if ch in self.whitespace:
+                        return token
+                    elif ch == "\\":
+                        ch = self._read_char()
+                    token += ch
+        return token
+
+    def push_token(self, token):
+        self.pushback.append(token)
+
+
 class netrc:
     def __init__(self, file=None):
         default_netrc = file is None
@@ -34,9 +78,7 @@ def __init__(self, file=None):
                 self._parse(file, fp, default_netrc)
 
     def _parse(self, file, fp, default_netrc):
-        lexer = shlex.shlex(fp)
-        lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
-        lexer.commenters = lexer.commenters.replace('#', '')
+        lexer = _netrclex(fp)
         while 1:
             # Look for a machine, default, or macdef top-level keyword
             saved_lineno = lexer.lineno
@@ -51,68 +93,75 @@ def _parse(self, file, fp, default_netrc):
                 entryname = lexer.get_token()
             elif tt == 'default':
                 entryname = 'default'
-            elif tt == 'macdef':                # Just skip to end of macdefs
+            elif tt == 'macdef':
                 entryname = lexer.get_token()
                 self.macros[entryname] = []
-                lexer.whitespace = ' \t'
                 while 1:
                     line = lexer.instream.readline()
-                    if not line or line == '\012':
-                        lexer.whitespace = ' \t\r\n'
+                    if not line:
+                        raise NetrcParseError(
+                            "Macro definition missing null line terminator.",
+                            file, lexer.lineno)
+                    if line == '\n':
+                        # a macro definition finished with consecutive new-line
+                        # characters. The first \n is encountered by the
+                        # readline() method and this is the second \n.
                         break
                     self.macros[entryname].append(line)
                 continue
             else:
                 raise NetrcParseError(
                     "bad toplevel token %r" % tt, file, lexer.lineno)
 
+            if not entryname:
+                raise NetrcParseError("missing %r name" % tt, file, lexer.lineno)
+
             # We're looking at start of an entry for a named machine or default.
-            login = ''
-            account = password = None
+            login = account = password = ''
             self.hosts[entryname] = {}
             while 1:
+                prev_lineno = lexer.lineno
                 tt = lexer.get_token()
-                if (tt.startswith('#') or
-                    tt in {'', 'machine', 'default', 'macdef'}):
-                    if password:
-                        self.hosts[entryname] = (login, account, password)
-                        lexer.push_token(tt)
-                        break
-                    else:
-                        raise NetrcParseError(
-                            "malformed %s entry %s terminated by %s"
-                            % (toplevel, entryname, repr(tt)),
-                            file, lexer.lineno)
+                if tt.startswith('#'):
+                    if lexer.lineno == prev_lineno:
+                        lexer.instream.readline()
+                    continue
+                if tt in {'', 'machine', 'default', 'macdef'}:
+                    self.hosts[entryname] = (login, account, password)
+                    lexer.push_token(tt)
+                    break
                 elif tt == 'login' or tt == 'user':
                     login = lexer.get_token()
                 elif tt == 'account':
                     account = lexer.get_token()
                 elif tt == 'password':
-                    if os.name == 'posix' and default_netrc:
-                        prop = os.fstat(fp.fileno())
-                        if prop.st_uid != os.getuid():
-                            import pwd
-                            try:
-                                fowner = pwd.getpwuid(prop.st_uid)[0]
-                            except KeyError:
-                                fowner = 'uid %s' % prop.st_uid
-                            try:
-                                user = pwd.getpwuid(os.getuid())[0]
-                            except KeyError:
-                                user = 'uid %s' % os.getuid()
-                            raise NetrcParseError(
-                                ("~/.netrc file owner (%s) does not match"
-                                 " current user (%s)") % (fowner, user),
-                                file, lexer.lineno)
-                        if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
-                            raise NetrcParseError(
-                               "~/.netrc access too permissive: access"
-                               " permissions must restrict access to only"
-                               " the owner", file, lexer.lineno)
                     password = lexer.get_token()
                 else:
                     raise NetrcParseError("bad follower token %r" % tt,
                                           file, lexer.lineno)
+            self._security_check(fp, default_netrc, self.hosts[entryname][0])
+
+    def _security_check(self, fp, default_netrc, login):
+        if os.name == 'posix' and default_netrc and login != "anonymous":
+            prop = os.fstat(fp.fileno())
+            if prop.st_uid != os.getuid():
+                import pwd
+                try:
+                    fowner = pwd.getpwuid(prop.st_uid)[0]
+                except KeyError:
+                    fowner = 'uid %s' % prop.st_uid
+                try:
+                    user = pwd.getpwuid(os.getuid())[0]
+                except KeyError:
+                    user = 'uid %s' % os.getuid()
+                raise NetrcParseError(
+                    (f"~/.netrc file owner ({fowner}, {user}) does not match"
+                     " current user"))
+            if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
+                raise NetrcParseError(
+                    "~/.netrc access too permissive: access"
+                    " permissions must restrict access to only"
+                    " the owner")
 
     def authenticators(self, host):
         """Return a (user, account, password) tuple for given host."""