Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Storage: add generation arguments (from #7023). #7444

Merged
merged 15 commits into from
Mar 14, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 17 additions & 11 deletions storage/google/cloud/storage/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ class _PropertyMixin(object):
"""Abstract mixin for cloud storage classes with associated properties.

Non-abstract subclasses should implement:
- client
- path
- client
- user_project

:type name: str
:param name: The name of the object. Bucket names must start and end with a
Expand Down Expand Up @@ -98,6 +99,14 @@ def _encryption_headers(self):
"""
return {}

@property
def _query_params(self):
"""Default query parameters."""
params = {}
if self.user_project is not None:
params["userProject"] = self.user_project
return params

def reload(self, client=None):
"""Reload properties from Cloud Storage.

Expand All @@ -109,17 +118,16 @@ def reload(self, client=None):
``client`` stored on the current object.
"""
client = self._require_client(client)
query_params = self._query_params
# Pass only '?projection=noAcl' here because 'acl' and related
# are handled via custom endpoints.
query_params = {"projection": "noAcl"}
if self.user_project is not None:
query_params["userProject"] = self.user_project
query_params["projection"] = "noAcl"
api_response = client._connection.api_request(
method="GET",
path=self.path,
query_params=query_params,
headers=self._encryption_headers(),
_target_object=self
_target_object=self,
)
self._set_properties(api_response)

Expand Down Expand Up @@ -164,11 +172,10 @@ def patch(self, client=None):
``client`` stored on the current object.
"""
client = self._require_client(client)
query_params = self._query_params
# Pass '?projection=full' here because 'PATCH' documented not
# to work properly w/ 'noAcl'.
query_params = {"projection": "full"}
if self.user_project is not None:
query_params["userProject"] = self.user_project
query_params["projection"] = "full"
update_properties = {key: self._properties[key] for key in self._changes}

# Make the API call.
Expand All @@ -194,9 +201,8 @@ def update(self, client=None):
``client`` stored on the current object.
"""
client = self._require_client(client)
query_params = {"projection": "full"}
if self.user_project is not None:
query_params["userProject"] = self.user_project
query_params = self._query_params
query_params["projection"] = "full"
api_response = client._connection.api_request(
method="PUT",
path=self.path,
Expand Down
39 changes: 30 additions & 9 deletions storage/google/cloud/storage/blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,13 @@ class Blob(_PropertyMixin):
"""

def __init__(
self, name, bucket, chunk_size=None, encryption_key=None, kms_key_name=None
self,
name,
bucket,
chunk_size=None,
encryption_key=None,
kms_key_name=None,
generation=None,
):
name = _bytes_to_unicode(name)
super(Blob, self).__init__(name=name)
Expand All @@ -177,6 +183,9 @@ def __init__(
if kms_key_name is not None:
self._properties["kmsKeyName"] = kms_key_name

if generation is not None:
self._properties["generation"] = generation

@property
def chunk_size(self):
"""Get the blob's default chunk size.
Expand Down Expand Up @@ -265,6 +274,16 @@ def _encryption_headers(self):
"""
return _get_encryption_headers(self._encryption_key)

@property
def _query_params(self):
"""Default query parameters."""
params = {}
if self.generation is not None:
params["generation"] = self.generation
if self.user_project is not None:
params["userProject"] = self.user_project
return params

@property
def public_url(self):
"""The public URL for this blob.
Expand Down Expand Up @@ -397,10 +416,8 @@ def exists(self, client=None):
client = self._require_client(client)
# We only need the status code (200 or not) so we seek to
# minimize the returned payload.
query_params = {"fields": "name"}

if self.user_project is not None:
query_params["userProject"] = self.user_project
query_params = self._query_params
query_params["fields"] = "name"

try:
# We intentionally pass `_target_object=None` since fields=name
Expand Down Expand Up @@ -435,7 +452,9 @@ def delete(self, client=None):
(propagated from
:meth:`google.cloud.storage.bucket.Bucket.delete_blob`).
"""
return self.bucket.delete_blob(self.name, client=client)
return self.bucket.delete_blob(
self.name, client=client, generation=self.generation
)

def _get_transport(self, client):
"""Return the client's transport.
Expand Down Expand Up @@ -1492,13 +1511,15 @@ def rewrite(self, source, token=None, client=None):
headers = _get_encryption_headers(self._encryption_key)
headers.update(_get_encryption_headers(source._encryption_key, source=True))

query_params = {}
query_params = self._query_params
if "generation" in query_params:
del query_params["generation"]

if token:
query_params["rewriteToken"] = token

if self.user_project is not None:
query_params["userProject"] = self.user_project
if source.generation:
query_params["sourceGeneration"] = source.generation

if self.kms_key_name is not None:
query_params["destinationKmsKeyName"] = self.kms_key_name
Expand Down
44 changes: 33 additions & 11 deletions storage/google/cloud/storage/bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,14 @@ def user_project(self):
"""
return self._user_project

def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=None):
def blob(
self,
blob_name,
chunk_size=None,
encryption_key=None,
kms_key_name=None,
generation=None,
):
"""Factory constructor for blob object.

.. note::
Expand All @@ -457,6 +464,10 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non
:param kms_key_name:
Optional resource name of KMS key used to encrypt blob's content.

:type generation: long
:param generation: Optional. If present, selects a specific revision of
this object.

:rtype: :class:`google.cloud.storage.blob.Blob`
:returns: The blob object created.
"""
Expand All @@ -466,6 +477,7 @@ def blob(self, blob_name, chunk_size=None, encryption_key=None, kms_key_name=Non
chunk_size=chunk_size,
encryption_key=encryption_key,
kms_key_name=kms_key_name,
generation=generation,
)

def notification(
Expand Down Expand Up @@ -638,7 +650,9 @@ def path(self):

return self.path_helper(self.name)

def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
def get_blob(
self, blob_name, client=None, encryption_key=None, generation=None, **kwargs
):
"""Get a blob object by name.

This will return None if the blob doesn't exist:
Expand All @@ -663,6 +677,10 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
See
https://cloud.google.com/storage/docs/encryption#customer-supplied.

:type generation: long
:param generation: Optional. If present, selects a specific revision of
this object.

:type kwargs: dict
:param kwargs: Keyword arguments to pass to the
:class:`~google.cloud.storage.blob.Blob` constructor.
Expand All @@ -671,7 +689,11 @@ def get_blob(self, blob_name, client=None, encryption_key=None, **kwargs):
:returns: The blob object if it exists, otherwise None.
"""
blob = Blob(
bucket=self, name=blob_name, encryption_key=encryption_key, **kwargs
bucket=self,
name=blob_name,
encryption_key=encryption_key,
generation=generation,
**kwargs
)
try:
# NOTE: This will not fail immediately in a batch. However, when
Expand Down Expand Up @@ -867,7 +889,7 @@ def delete(self, force=False, client=None):
_target_object=None,
)

def delete_blob(self, blob_name, client=None):
def delete_blob(self, blob_name, client=None, generation=None):
"""Deletes a blob from the current bucket.

If the blob isn't found (backend 404), raises a
Expand All @@ -889,6 +911,10 @@ def delete_blob(self, blob_name, client=None):
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.

:type generation: long
:param generation: Optional. If present, permanently deletes a specific
revision of this object.

:raises: :class:`google.cloud.exceptions.NotFound` (to suppress
the exception, call ``delete_blobs``, passing a no-op
``on_error`` callback, e.g.:
Expand All @@ -899,19 +925,15 @@ def delete_blob(self, blob_name, client=None):

"""
client = self._require_client(client)
query_params = {}
blob = Blob(blob_name, bucket=self, generation=generation)

if self.user_project is not None:
query_params["userProject"] = self.user_project

blob_path = Blob.path_helper(self.path, blob_name)
# We intentionally pass `_target_object=None` since a DELETE
# request has no response value (whether in a standard request or
# in a batch request).
client._connection.api_request(
method="DELETE",
path=blob_path,
query_params=query_params,
path=blob.path,
query_params=blob._query_params,
_target_object=None,
)

Expand Down
46 changes: 36 additions & 10 deletions storage/tests/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ def _bad_copy(bad_request):


retry_429 = RetryErrors(exceptions.TooManyRequests, max_tries=6)
retry_429_503 = RetryErrors([exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6)
retry_429_503 = RetryErrors(
[exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=6
)
retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)


Expand Down Expand Up @@ -78,6 +80,7 @@ def setUpModule():
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
Config.TEST_BUCKET.versioning_enabled = True
retry_429(Config.TEST_BUCKET.create)()


Expand Down Expand Up @@ -414,29 +417,52 @@ def test_crud_blob_w_user_project(self):

# Exercise 'objects.insert' w/ userProject.
blob.upload_from_filename(file_data["path"])
gen0 = blob.generation

# Upload a second generation of the blob
blob.upload_from_string(b"gen1")
gen1 = blob.generation

blob0 = with_user_project.blob("SmallFile", generation=gen0)
blob1 = with_user_project.blob("SmallFile", generation=gen1)

# Exercise 'objects.get' w/ generation
self.assertEqual(with_user_project.get_blob(blob.name).generation, gen1)
self.assertEqual(
with_user_project.get_blob(blob.name, generation=gen0).generation, gen0
)

try:
# Exercise 'objects.get' (metadata) w/ userProject.
self.assertTrue(blob.exists())
blob.reload()

# Exercise 'objects.get' (media) w/ userProject.
downloaded = blob.download_as_string()
self.assertEqual(downloaded, file_contents)
self.assertEqual(blob0.download_as_string(), file_contents)
self.assertEqual(blob1.download_as_string(), b"gen1")

# Exercise 'objects.patch' w/ userProject.
blob.content_language = "en"
blob.patch()
self.assertEqual(blob.content_language, "en")
blob0.content_language = "en"
blob0.patch()
self.assertEqual(blob0.content_language, "en")
self.assertIsNone(blob1.content_language)

# Exercise 'objects.update' w/ userProject.
metadata = {"foo": "Foo", "bar": "Bar"}
blob.metadata = metadata
blob.update()
self.assertEqual(blob.metadata, metadata)
blob0.metadata = metadata
blob0.update()
self.assertEqual(blob0.metadata, metadata)
self.assertIsNone(blob1.metadata)
finally:
# Exercise 'objects.delete' (metadata) w/ userProject.
blob.delete()
blobs = with_user_project.list_blobs(prefix=blob.name, versions=True)
self.assertEqual([each.generation for each in blobs], [gen0, gen1])

blob0.delete()
blobs = with_user_project.list_blobs(prefix=blob.name, versions=True)
self.assertEqual([each.generation for each in blobs], [gen1])

blob1.delete()

@unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
def test_blob_acl_w_user_project(self):
Expand Down
9 changes: 9 additions & 0 deletions storage/tests/unit/test__helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,15 @@ def test__encryption_headers(self):
mixin = self._make_one()
self.assertEqual(mixin._encryption_headers(), {})

def test__query_params_wo_user_project(self):
derived = self._derivedClass("/path", None)()
self.assertEqual(derived._query_params, {})

def test__query_params_w_user_project(self):
user_project = "user-project-123"
derived = self._derivedClass("/path", user_project)()
self.assertEqual(derived._query_params, {"userProject": user_project})

def test_reload(self):
connection = _Connection({"foo": "Foo"})
client = _Client(connection)
Expand Down
Loading