Fix regression for supported timestamp formats

botocore/serialize.py

@@ -41,13 +41,14 @@
 import time
 import base64
 from xml.etree import ElementTree
+import calendar
 
 import datetime
 from dateutil.tz import tzutc
 import six
 
 from botocore.compat import json, formatdate
-from botocore.utils import parse_timestamp
+from botocore.utils import parse_timestamp, parse_to_aware_datetime
 from botocore.utils import percent_encode
 from botocore import validate
 
@@ -132,34 +133,16 @@ def _timestamp_iso8601(self, value):
             timestamp_format = ISO8601_MICRO
         else:
             timestamp_format = ISO8601
-        if value.tzinfo is None:
-            # I think a case would be made that if no time zone is provided,
-            # we should use the local time.  However, to restore backwards
-            # compat, the previous behavior was to assume UTC, which is
-            # what we're going to do here.
-            datetime_obj = value.replace(tzinfo=tzutc())
-        else:
-            datetime_obj = value.astimezone(tzutc())
-        return datetime_obj.strftime(timestamp_format)
+        return value.strftime(timestamp_format)
 
     def _timestamp_unixtimestamp(self, value):
-        return int(time.mktime(value.timetuple()))
+        return int(calendar.timegm(value.timetuple()))
 
     def _timestamp_rfc822(self, value):
         return formatdate(value)
 
     def _convert_timestamp_to_str(self, value):
-        # This is a general purpose method that handles several cases of
-        # converting the provided value to a string timestamp suitable to be
-        # serialized to an http request. It can handle:
-        # 1) A datetime.datetime object.
-        if isinstance(value, datetime.datetime):
-            datetime_obj = value
-        else:
-            # 2) A string object that's formatted as a timestamp.
-            #    We document this as being an iso8601 timestamp, although
-            #    parse_timestamp is a bit more flexible.
-            datetime_obj = parse_timestamp(value)
+        datetime_obj = parse_to_aware_datetime(value)
         converter = getattr(
             self, '_timestamp_%s' % self.TIMESTAMP_FORMAT.lower())
         final_value = converter(datetime_obj)
@@ -304,6 +287,8 @@ def _serialize_type_list(self, serialized, value, shape, prefix=''):
 
 
 class JSONSerializer(Serializer):
+    TIMESTAMP_FORMAT = 'unixtimestamp'
+
     def serialize_to_request(self, parameters, operation_model):
         target = '%s.%s' % (operation_model.metadata['targetPrefix'],
                             operation_model.name)
@@ -315,9 +300,39 @@ def serialize_to_request(self, parameters, operation_model):
             'X-Amz-Target': target,
             'Content-Type': 'application/x-amz-json-%s' % json_version,
         }
-        serialized['body'] = json.dumps(parameters)
+        body = {}
+        input_shape = operation_model.input_shape
+        if input_shape is not None:
+            self._serialize(body, parameters, input_shape)
+        serialized['body'] = json.dumps(body)
         return serialized
 
+    def _serialize(self, serialized, value, shape, key=None):
+        method = getattr(self, '_serialize_type_%s' % shape.type_name,
+                         self._default_serialize)
+        method(serialized, value, shape, key)
+
+    def _serialize_type_structure(self, serialized, value, shape, key):
+        if key is not None:
+            # If a key is provided, this is a result of a recursive
+            # call so we need to add a new child dict as the value
+            # of the passed in serialized dict.  We'll then add
+            # all the structure members as key/vals in the new serialized
+            # dictionary we just created.
+            new_serialized = {}
+            serialized[key] = new_serialized
+            serialized = new_serialized
+        members = shape.members
+        for member_key, member_value in value.items():
+            member_shape = members[member_key]
+            self._serialize(serialized, member_value, member_shape, member_key)
+
+    def _default_serialize(self, serialized, value, shape, key):
+        serialized[key] = value
+
+    def _serialize_type_timestamp(self, serialized, value, shape, key):
+        serialized[key] = self._convert_timestamp_to_str(value)
+
 
 class BaseRestSerializer(Serializer):
     """Base class for rest protocols.

botocore/utils.py

@@ -15,7 +15,7 @@
 
 from six import string_types, text_type
 import dateutil.parser
-from dateutil.tz import tzlocal
+from dateutil.tz import tzlocal, tzutc
 
 from botocore.exceptions import InvalidExpressionError, ConfigNotFound
 from botocore.compat import json, quote
@@ -288,12 +288,62 @@ def parse_timestamp(value):
     if isinstance(value, (int, float)):
         # Possibly an epoch time.
         return datetime.datetime.fromtimestamp(value, tzlocal())
+    else:
+        try:
+            return datetime.datetime.fromtimestamp(float(value), tzlocal())
+        except (TypeError, ValueError):
+            pass
     try:
         return dateutil.parser.parse(value)
     except (TypeError, ValueError) as e:
         raise ValueError('Invalid timestamp "%s": %s' % (value, e))
 
 
+def parse_to_aware_datetime(value):
+    """Converted the passed in value to a datetime object with tzinfo.
+
+    This function can be used to normalize all timestamp inputs.  This
+    function accepts a number of different types of inputs, but
+    will always return a datetime.datetime object with time zone
+    information.
+
+    The input param ``value`` can be one of several types:
+
+        * A datetime object (both naive and aware)
+        * An integer representing the epoch time (can also be a string
+          of the integer, i.e '0', instead of 0).  The epoch time is
+          considered to be UTC.
+        * An iso8601 formatted timestamp.  This does not need to be
+          a complete timestamp, it can contain just the date portion
+          without the time component.
+
+    The returned value will be a datetime object that will have tzinfo.
+    If no timezone info was provided in the input value, then UTC is
+    assumed, not local time.
+
+    """
+    # This is a general purpose method that handles several cases of
+    # converting the provided value to a string timestamp suitable to be
+    # serialized to an http request. It can handle:
+    # 1) A datetime.datetime object.
+    if isinstance(value, datetime.datetime):
+        datetime_obj = value
+    else:
+        # 2) A string object that's formatted as a timestamp.
+        #    We document this as being an iso8601 timestamp, although
+        #    parse_timestamp is a bit more flexible.
+        datetime_obj = parse_timestamp(value)
+    if datetime_obj.tzinfo is None:
+        # I think a case would be made that if no time zone is provided,
+        # we should use the local time.  However, to restore backwards
+        # compat, the previous behavior was to assume UTC, which is
+        # what we're going to do here.
+        datetime_obj = datetime_obj.replace(tzinfo=tzutc())
+    else:
+        datetime_obj = datetime_obj.astimezone(tzutc())
+    return datetime_obj
+
+
 class CachedProperty(object):
     """A read only property that caches the initially computed value.
 

botocore/validate.py

@@ -17,7 +17,7 @@
 import decimal
 from datetime import datetime
 
-from botocore.utils import parse_timestamp
+from botocore.utils import parse_to_aware_datetime
 from botocore.exceptions import ParamValidationError
 
 
@@ -250,10 +250,8 @@ def _validate_timestamp(self, param, shape, errors, name):
                             valid_types=valid_type_names)
 
     def _type_check_datetime(self, value):
-        if isinstance(value, datetime):
-            return True
         try:
-            parse_timestamp(value)
+            parse_to_aware_datetime(value)
             return True
         except (TypeError, ValueError):
             return False

tests/integration/test_client.py

@@ -13,6 +13,7 @@
 import time
 import random
 import logging
+import datetime
 from tests import unittest
 
 from six import StringIO
@@ -93,3 +94,33 @@ def test_debug_log_contains_headers_and_body(self):
         debug_log_contents = debug_log.getvalue()
         self.assertIn('Response headers', debug_log_contents)
         self.assertIn('Response body', debug_log_contents)
+
+
+class TestAcceptedDateTimeFormats(unittest.TestCase):
+    def setUp(self):
+        self.session = botocore.session.get_session()
+        self.client = self.session.create_client('emr')
+
+    def test_accepts_datetime_object(self):
+        response = self.client.list_clusters(
+            CreatedAfter=datetime.datetime.now())
+        self.assertIn('Clusters', response)
+
+    def test_accepts_epoch_format(self):
+        response = self.client.list_clusters(CreatedAfter=0)
+        self.assertIn('Clusters', response)
+
+    def test_accepts_iso_8601_unaware(self):
+        response = self.client.list_clusters(
+            CreatedAfter='2014-01-01T00:00:00')
+        self.assertIn('Clusters', response)
+
+    def test_accepts_iso_8601_utc(self):
+        response = self.client.list_clusters(
+            CreatedAfter='2014-01-01T00:00:00Z')
+        self.assertIn('Clusters', response)
+
+    def test_accepts_iso_8701_local(self):
+        response = self.client.list_clusters(
+            CreatedAfter='2014-01-01T00:00:00-08:00')
+        self.assertIn('Clusters', response)

tests/unit/test_serialize.py

@@ -12,6 +12,7 @@
 
 """
 import base64
+import json
 import datetime
 import dateutil.tz
 from tests import unittest
@@ -144,3 +145,59 @@ def test_microsecond_timestamp_without_tz_info(self):
             {'Timestamp': '2014-01-01T12:12:12.123456'})
         self.assertEqual(request['body']['Timestamp'],
                          '2014-01-01T12:12:12.123456Z')
+
+
+class TestJSONTimestampSerialization(unittest.TestCase):
+    def setUp(self):
+        self.model = {
+            'metadata': {'protocol': 'json', 'apiVersion': '2014-01-01',
+                         'jsonVersion': '1.1', 'targetPrefix': 'foo'},
+            'documentation': '',
+            'operations': {
+                'TestOperation': {
+                    'name': 'TestOperation',
+                    'http': {
+                        'method': 'POST',
+                        'requestUri': '/',
+                    },
+                    'input': {'shape': 'InputShape'},
+                }
+            },
+            'shapes': {
+                'InputShape': {
+                    'type': 'structure',
+                    'members': {
+                        'Timestamp': {'shape': 'TimestampType'},
+                    }
+                },
+                'TimestampType': {
+                    'type': 'timestamp',
+                }
+            }
+        }
+        self.service_model = ServiceModel(self.model)
+
+    def serialize_to_request(self, input_params):
+        request_serializer = serialize.create_serializer(
+            self.service_model.metadata['protocol'])
+        return request_serializer.serialize_to_request(
+            input_params, self.service_model.operation_model('TestOperation'))
+
+    def test_accepts_iso_8601_format(self):
+        body = json.loads(self.serialize_to_request(
+            {'Timestamp': '1970-01-01T00:00:00'})['body'])
+        self.assertEqual(body['Timestamp'], 0)
+
+    def test_accepts_epoch(self):
+        body = json.loads(self.serialize_to_request(
+            {'Timestamp': '0'})['body'])
+        self.assertEqual(body['Timestamp'], 0)
+        # Can also be an integer 0.
+        body = json.loads(self.serialize_to_request(
+            {'Timestamp': 0})['body'])
+        self.assertEqual(body['Timestamp'], 0)
+
+    def test_accepts_partial_iso_format(self):
+        body = json.loads(self.serialize_to_request(
+            {'Timestamp': '1970-01-01'})['body'])
+        self.assertEqual(body['Timestamp'], 0)

tests/unit/test_utils.py

@@ -12,7 +12,7 @@
 # language governing permissions and limitations under the License.
 
 from tests import unittest
-from dateutil.tz import tzutc
+from dateutil.tz import tzutc, tzoffset
 import datetime
 
 import mock
@@ -26,6 +26,7 @@
 from botocore.utils import parse_key_val_file_contents
 from botocore.utils import parse_key_val_file
 from botocore.utils import parse_timestamp
+from botocore.utils import parse_to_aware_datetime
 from botocore.utils import CachedProperty
 from botocore.utils import ArgumentGenerator
 from botocore.model import DenormalizedStructureBuilder
@@ -196,6 +197,16 @@ def test_parse_epoch(self):
             parse_timestamp(1222172800),
             datetime.datetime(2008, 9, 23, 12, 26, 40, tzinfo=tzutc()))
 
+    def test_parse_epoch_zero_time(self):
+        self.assertEqual(
+            parse_timestamp(0),
+            datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()))
+
+    def test_parse_epoch_as_string(self):
+        self.assertEqual(
+            parse_timestamp('1222172800'),
+            datetime.datetime(2008, 9, 23, 12, 26, 40, tzinfo=tzutc()))
+
     def test_parse_rfc822(self):
         self.assertEqual(
             parse_timestamp('Wed, 02 Oct 2002 13:00:00 GMT'),
@@ -206,6 +217,40 @@ def test_parse_invalid_timestamp(self):
             parse_timestamp('invalid date')
 
 
+class TestParseToUTCDatetime(unittest.TestCase):
+    def test_handles_utc_time(self):
+        original = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+        self.assertEqual(parse_to_aware_datetime(original), original)
+
+    def test_handles_other_timezone(self):
+        tzinfo = tzoffset("BRST", -10800)
+        original = datetime.datetime(2014, 1, 1, 0, 0, 0, tzinfo=tzinfo)
+        self.assertEqual(parse_to_aware_datetime(original), original)
+
+    def test_handles_naive_datetime(self):
+        original = datetime.datetime(1970, 1, 1, 0, 0, 0)
+        expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+        self.assertEqual(parse_to_aware_datetime(original), expected)
+
+    def test_handles_string_epoch(self):
+        expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+        self.assertEqual(parse_to_aware_datetime('0'), expected)
+
+    def test_handles_int_epoch(self):
+        expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+        self.assertEqual(parse_to_aware_datetime(0), expected)
+
+    def test_handles_full_iso_8601(self):
+        expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+        self.assertEqual(
+            parse_to_aware_datetime('1970-01-01T00:00:00Z'),
+            expected)
+
+    def test_year_only_iso_8601(self):
+        expected = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
+        self.assertEqual(parse_to_aware_datetime('1970-01-01'), expected)
+
+
 class TestCachedProperty(unittest.TestCase):
     def test_cached_property_same_value(self):
         class CacheMe(object):