Merge branch 'release-1.7.46'

* release-1.7.46:
  Bumping version to 1.7.46
  Update changelog
  Fix pep8 issue
  Support nested hashes/lists
  Add more comments to make backtracking more clear
  Add consume_whitespace arg to expect
  Missing values parse to empty string
  Handle newlines with appropriate errors
  fixed typo
  Update changelog with fix from #1447
  Handle case when create-virtual-mfa-device errors out
  Remove 'ur' and use 'u' strings instead
  Add support for hash literals
  Initial commit of imporoved shorthand parser

CHANGELOG.rst

@@ -2,6 +2,14 @@
 CHANGELOG
 =========
 
+1.7.46
+======
+
+* feature:``aws s3``: Add support for event notification filters
+* bugfix:``aws iam create-virtual-mfa-device``: Fix issue when an error
+  response is received from the ``create-virtual-mfa-device`` command
+  (`issue 1447 <https://github.com/aws/aws-cli/pull/1447/>`__)
+
 1.7.45
 ======
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.7.45'
+__version__ = '1.7.46'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/iamvirtmfa.py

@@ -81,6 +81,8 @@ def _add_options(self, argument_table, **kwargs):
         argument_table['bootstrap-method'] = self._method
 
     def _save_file(self, parsed, **kwargs):
+        if 'Error' in parsed:
+            return
         method = self._method.value
         outfile = self._outfile.value
         if method in parsed['VirtualMFADevice']:

awscli/examples/cloudformation/update-stack.rst

@@ -4,7 +4,7 @@ The following ``update-stack`` command updates the template and input parameters
 
   aws cloudformation update-stack --stack-name mystack --template-url https://s3.amazonaws.com/sample/updated.template --parameters ParameterKey=KeyPairName,ParameterValue=SampleKeyPair ParameterKey=SubnetIDs,ParameterValue=SampleSubnetID1\\,SampleSubnetID2
 
-The following ``update-stack`` command updates just the ``SubnetIDs`` paramter value for the ``mystack`` stack. If you
+The following ``update-stack`` command updates just the ``SubnetIDs`` parameter value for the ``mystack`` stack. If you
 don't specify a parameter value, the default value that is specified in the template is used::
 
   aws cloudformation update-stack --stack-name mystack --template-url https://s3.amazonaws.com/sample/updated.template --parameters ParameterKey=KeyPairName,UsePreviousValue=true ParameterKey=SubnetIDs,ParameterValue=SampleSubnetID1\\,UpdatedSampleSubnetID2

awscli/shorthand.py

@@ -0,0 +1,284 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import re
+import string
+
+
+_EOF = object()
+
+
+class _NamedRegex(object):
+    def __init__(self, name, regex_str):
+        self.name = name
+        self.regex = re.compile(regex_str, re.UNICODE)
+
+    def match(self, value):
+        return self.regex.match(value)
+
+
+class ShorthandParseError(Exception):
+    def __init__(self, value, expected, actual, index):
+        self.value = value
+        self.expected = expected
+        self.actual = actual
+        self.index = index
+        msg = self._construct_msg()
+        super(ShorthandParseError, self).__init__(msg)
+
+    def _construct_msg(self):
+        if '\n' in self.value:
+            # If there's newlines in the expression, we want
+            # to make sure we're only counting the spaces
+            # from the last newline:
+            # foo=bar,\n
+            # bar==baz
+            #     ^
+            last_newline = self.value[:self.index].rindex('\n')
+            num_spaces = self.index - last_newline - 1
+        else:
+            num_spaces = self.index
+        msg = (
+            "Expected: '%s', received: '%s' for input:\n"
+            "%s\n"
+            "%s\n"
+        ) % (self.expected, self.actual, self.value,
+             ' ' * num_spaces + '^')
+        return msg
+
+
+class ShorthandParser(object):
+
+    _SINGLE_QUOTED = _NamedRegex('singled quoted', r'\'(?:\\\\|\\\'|[^\'])*\'')
+    _DOUBLE_QUOTED = _NamedRegex('double quoted', r'"(?:\\\\|\\"|[^"])*"')
+    _FIRST_VALUE = _NamedRegex('first',
+                               u'[\!\#-&\(-\+\--\<\>-Z\\\\-z\u007c-\uffff]'
+                               u'[\!\#-&\(-\+\--\\\\\^-\|~-\uffff]*')
+    _SECOND_VALUE = _NamedRegex('second',
+                                u'[\!\#-&\(-\+\--\<\>-Z\\\\-z\u007c-\uffff]'
+                                u'[\!\#-&\(-\+\--\<\>-\uffff]*')
+
+    def __init__(self):
+        self._tokens = []
+
+    def parse(self, value):
+        self._input_value = value
+        self._index = 0
+        return self._parameter()
+
+    def _parameter(self):
+        # parameter = keyval *("," keyval)
+        params = {}
+        params.update(self._keyval())
+        while self._index < len(self._input_value):
+            self._expect(',', consume_whitespace=True)
+            params.update(self._keyval())
+        return params
+
+    def _keyval(self):
+        # keyval = key "=" [values]
+        key = self._key()
+        self._expect('=', consume_whitespace=True)
+        values = self._values()
+        return {key: values}
+
+    def _key(self):
+        # key = 1*(alpha / %x30-39)  ; [a-zA-Z0-9]
+        valid_chars = string.ascii_letters + string.digits
+        start = self._index
+        while not self._at_eof():
+            if self._current() not in valid_chars:
+                break
+            self._index += 1
+        return self._input_value[start:self._index]
+
+    def _values(self):
+        # values = csv-list / explicit-list / hash-literal
+        if self._at_eof():
+            return ''
+        elif self._current() == '[':
+            return self._explicit_list()
+        elif self._current() == '{':
+            return self._hash_literal()
+        else:
+            return self._csv_value()
+
+    def _csv_value(self):
+        # Supports either:
+        # foo=bar     -> 'bar'
+        #     ^
+        # foo=bar,baz -> ['bar', 'baz']
+        #     ^
+        first_value = self._first_value()
+        self._consume_whitespace()
+        if self._at_eof() or self._input_value[self._index] != ',':
+            return first_value
+        self._expect(',', consume_whitespace=True)
+        csv_list = [first_value]
+        # Try to parse remaining list values.
+        # It's possible we don't parse anything:
+        # a=b,c=d
+        #     ^-here
+        # In the case above, we'll hit the ShorthandParser,
+        # backtrack to the comma, and return a single scalar
+        # value 'b'.
+        while True:
+            try:
+                current = self._second_value()
+                if current is None:
+                    break
+                self._consume_whitespace()
+                if self._at_eof():
+                    csv_list.append(current)
+                    break
+                self._expect(',', consume_whitespace=True)
+                csv_list.append(current)
+            except ShorthandParseError:
+                # Backtrack to the previous comma.
+                # This can happen when we reach this case:
+                # foo=a,b,c=d,e=f
+                #     ^-start
+                # foo=a,b,c=d,e=f
+                #          ^-error, "expected ',' received '='
+                # foo=a,b,c=d,e=f
+                #        ^-backtrack to here.
+                self._backtrack_to(',')
+                break
+        if len(csv_list) == 1:
+            # Then this was a foo=bar case, so we expect
+            # this to parse to a scalar value 'bar', i.e
+            # {"foo": "bar"} instead of {"bar": ["bar"]}
+            return first_value
+        return csv_list
+
+    def _value(self):
+        result = self._FIRST_VALUE.match(self._input_value[self._index:])
+        if result is not None:
+            return self._consume_matched_regex(result)
+        return ''
+
+    def _explicit_list(self):
+        # explicit-list = "[" [value *(",' value)] "]"
+        self._expect('[', consume_whitespace=True)
+        values = []
+        while self._current() != ']':
+            val = self._explicit_values()
+            values.append(val)
+            self._consume_whitespace()
+            if self._current() != ']':
+                self._expect(',')
+                self._consume_whitespace()
+        self._expect(']')
+        return values
+
+    def _explicit_values(self):
+        # values = csv-list / explicit-list / hash-literal
+        if self._at_eof():
+            return ''
+        elif self._current() == '[':
+            return self._explicit_list()
+        elif self._current() == '{':
+            return self._hash_literal()
+        else:
+            return self._first_value()
+
+    def _hash_literal(self):
+        self._expect('{', consume_whitespace=True)
+        keyvals = {}
+        while self._current() != '}':
+            key = self._key()
+            self._expect('=', consume_whitespace=True)
+            v = self._explicit_values()
+            self._consume_whitespace()
+            if self._current() != '}':
+                self._expect(',')
+                self._consume_whitespace()
+            keyvals[key] = v
+        self._expect('}')
+        return keyvals
+
+    def _first_value(self):
+        # first-value = value / single-quoted-val / double-quoted-val
+        if self._current() == "'":
+            return self._single_quoted_value()
+        elif self._current() == '"':
+            return self._double_quoted_value()
+        return self._value()
+
+    def _single_quoted_value(self):
+        # single-quoted-value = %x27 *(val-escaped-single) %x27
+        # val-escaped-single  = %x20-26 / %x28-7F / escaped-escape /
+        #                       (escape single-quote)
+        return self._consume_quoted(self._SINGLE_QUOTED, escaped_char="'")
+
+    def _consume_quoted(self, regex, escaped_char=None):
+        value = self._must_consume_regex(regex)[1:-1]
+        if escaped_char is not None:
+            value = value.replace("\\%s" % escaped_char, escaped_char)
+            value = value.replace("\\\\", "\\")
+        return value
+
+    def _double_quoted_value(self):
+        return self._consume_quoted(self._DOUBLE_QUOTED, escaped_char='"')
+
+    def _second_value(self):
+        if self._current() == "'":
+            return self._single_quoted_value()
+        elif self._current() == '"':
+            return self._double_quoted_value()
+        else:
+            return self._must_consume_regex(self._SECOND_VALUE)
+
+    def _expect(self, char, consume_whitespace=False):
+        if consume_whitespace:
+            self._consume_whitespace()
+        if self._index >= len(self._input_value):
+            raise ShorthandParseError(self._input_value, char,
+                                      'EOF', self._index)
+        actual = self._input_value[self._index]
+        if actual != char:
+            raise ShorthandParseError(self._input_value, char,
+                                      actual, self._index)
+        self._index += 1
+        if consume_whitespace:
+            self._consume_whitespace()
+
+    def _must_consume_regex(self, regex):
+        result = regex.match(self._input_value[self._index:])
+        if result is not None:
+            return self._consume_matched_regex(result)
+        raise ShorthandParseError(self._input_value, '<%s>' % regex.name,
+                                  '<none>', self._index)
+
+    def _consume_matched_regex(self, result):
+        start, end = result.span()
+        v = self._input_value[self._index+start:self._index+end]
+        self._index += (end - start)
+        return v
+
+    def _current(self):
+        # If the index is at the end of the input value,
+        # then _EOF will be returned.
+        if self._index < len(self._input_value):
+            return self._input_value[self._index]
+        return _EOF
+
+    def _at_eof(self):
+        return self._index >= len(self._input_value)
+
+    def _backtrack_to(self, char):
+        while self._index >= 0 and self._input_value[self._index] != char:
+            self._index -= 1
+
+    def _consume_whitespace(self):
+        while self._current() != _EOF and self._current() in string.whitespace:
+            self._index += 1

doc/source/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.7.'
 # The full version, including alpha/beta/rc tags.
-release = '1.7.45'
+release = '1.7.46'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.py

@@ -6,7 +6,7 @@
 import awscli
 
 
-requires = ['botocore==1.1.8',
+requires = ['botocore==1.1.9',
             'colorama>=0.2.5,<=0.3.3',
             'docutils>=0.10',
             'rsa>=3.1.2,<=3.1.4']

tests/functional/iam/test_create_virtual_mfa_device.py

@@ -127,3 +127,28 @@ def test_bad_relative_filename(self):
         cmdline += ' --virtual-mfa-device-name fiebaz'
         cmdline += ' --outfile %s --bootstrap-method QRCodePNG' % outfile
         self.assert_params_for_cmd(cmdline, expected_rc=255)
+
+    def test_bad_response(self):
+        # This can happen if you run the create-virtual-mfa-device
+        # command multiple times with the same name.  You'll get
+        # an "already exists" error and we should handle that case
+        # gracefully.
+        self.parsed_response = {
+            'Error': {
+                'Code': 'EntityAlreadyExists',
+                'Message': 'MFADevice entity at the and name already exists.',
+                'Type': 'Sender',
+            },
+            'ResponseMetadata': {
+                'HTTPStatusCode': 409,
+                'RequestId': 'requset-id'}
+        }
+        self.http_response.status_code = 409
+        cmdline = self.prefix
+        cmdline += ' --virtual-mfa-device-name fiebaz'
+        cmdline += ' --outfile foo --bootstrap-method QRCodePNG'
+        # The error message should be in the stderr.
+        self.assert_params_for_cmd(
+            cmdline,
+            stderr_contains=self.parsed_response['Error']['Message'],
+            expected_rc=255)

tests/unit/test_argprocess.py

@@ -238,6 +238,16 @@ def test_list_structure_list_scalar(self):
                 "Name = architecture, Values = i386"])
         self.assertEqual(returned3, expected)
 
+    def test_parse_empty_values(self):
+        # A value can be omitted and will default to an empty string.
+        p = self.get_param_model('ec2.DescribeInstances.Filters')
+        expected = [{"Name": "", "Values": ["i-1", "i-2"]},
+                    {"Name": "architecture", "Values": ['']}]
+        returned = self.simplify(
+            p, ["Name=,Values=i-1,i-2",
+                "Name=architecture,Values="])
+        self.assertEqual(returned, expected)
+
     def test_list_structure_list_scalar_2(self):
         p = self.get_param_model('emr.ModifyInstanceGroups.InstanceGroups')
         expected = [