From 0f2b7500816117fb5fd08ebec75547d9cc371c2f Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 16 Jan 2015 16:11:34 -0800 Subject: [PATCH] Set mtime of S3 downloaded object when file is closed This is the original patch from @kirat-singh, but the forked repo is no longer available. I've added a unit test for this that shows that the mtime modification is tied to the execution of an IOCloseRequest instead of the enqueuing of the IOCloseRequest. Fixes #1072 Closes #994 --- awscli/customizations/s3/executor.py | 4 ++++ awscli/customizations/s3/tasks.py | 4 ++-- awscli/customizations/s3/utils.py | 5 ++++- tests/unit/customizations/s3/test_executor.py | 16 ++++++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/awscli/customizations/s3/executor.py b/awscli/customizations/s3/executor.py index 5a54571a92b7..94261821ed08 100644 --- a/awscli/customizations/s3/executor.py +++ b/awscli/customizations/s3/executor.py @@ -10,6 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os import logging import sys import threading @@ -175,6 +176,9 @@ def run(self): if fileobj is not None: fileobj.close() del self.fd_descriptor_cache[task.filename] + if task.desired_mtime is not None: + os.utime(task.filename, (task.desired_mtime, + task.desired_mtime)) def _cleanup(self): for fileobj in self.fd_descriptor_cache.values(): diff --git a/awscli/customizations/s3/tasks.py b/awscli/customizations/s3/tasks.py index 7d1c09eb98d5..ab58078febaa 100644 --- a/awscli/customizations/s3/tasks.py +++ b/awscli/customizations/s3/tasks.py @@ -310,12 +310,12 @@ def __call__(self): self._context.wait_for_completion() last_update_tuple = self._filename.last_update.timetuple() mod_timestamp = time.mktime(last_update_tuple) - os.utime(self._filename.dest, (int(mod_timestamp), int(mod_timestamp))) + desired_mtime = int(mod_timestamp) message = print_operation(self._filename, False, self._parameters['dryrun']) print_task = {'message': message, 'error': False} self._result_queue.put(PrintTask(**print_task)) - self._io_queue.put(IOCloseRequest(self._filename.dest)) + self._io_queue.put(IOCloseRequest(self._filename.dest, desired_mtime)) class DownloadPartTask(OrderableTask): diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 6eed9fcfe6e7..16e22bb935df 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -452,4 +452,7 @@ def __new__(cls, message, error=False, total_parts=None, warning=None): ['filename', 'offset', 'data', 'is_stream']) # Used to signal that IO for the filename is finished, and that # any associated resources may be cleaned up. -IOCloseRequest = namedtuple('IOCloseRequest', ['filename']) +_IOCloseRequest = namedtuple('IOCloseRequest', ['filename', 'desired_mtime']) +class IOCloseRequest(_IOCloseRequest): + def __new__(cls, filename, desired_mtime=None): + return super(IOCloseRequest, cls).__new__(cls, filename, desired_mtime) diff --git a/tests/unit/customizations/s3/test_executor.py b/tests/unit/customizations/s3/test_executor.py index 04f9bc19d962..f5549ecff570 100644 --- a/tests/unit/customizations/s3/test_executor.py +++ b/tests/unit/customizations/s3/test_executor.py @@ -13,12 +13,13 @@ import os import tempfile import shutil -from awscli.compat import six -from six.moves import queue +import time import sys import mock +from awscli.compat import six +from six.moves import queue from awscli.testutils import unittest, temporary_file from awscli.customizations.s3.executor import IOWriterThread from awscli.customizations.s3.executor import ShutdownThreadRequest @@ -74,6 +75,17 @@ def test_multiple_files_in_queue(self): with open(second_file, 'rb') as f: self.assertEqual(f.read(), b'otherstuff') + def test_mtime_set_at_file_close_time(self): + # We're picking something other than the close time so that can verify + # that the IOCloseRequest can specify what the mtime should be. + now_time = int(time.time() - 100) + self.queue.put(IORequest(self.filename, 0, b'foobar', False)) + self.queue.put(IOCloseRequest(self.filename, now_time)) + self.queue.put(ShutdownThreadRequest()) + self.io_thread.run() + actual_mtime = int(os.stat(self.filename).st_mtime) + self.assertEqual(actual_mtime, now_time) + def test_stream_requests(self): # Test that offset has no affect on the order in which requests # are written to stdout. The order of requests for a stream are