Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
File renamed without changes.
File renamed without changes.
File renamed without changes.
36 changes: 34 additions & 2 deletions storage/samples/snippets/snippets_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
import asyncio
import io
import os
import sys
import tempfile
import time
import uuid
import sys

from google.cloud import storage
import google.cloud.exceptions
Expand Down Expand Up @@ -668,7 +668,10 @@ def test_object_get_kms_key(test_bucket):


def test_storage_compose_file(test_bucket):
source_files = ["test_upload_blob_1", "test_upload_blob_2"]
source_files = [
f"test_upload_blob_1_{uuid.uuid4().hex}",
f"test_upload_blob_2_{uuid.uuid4().hex}",
]
for source in source_files:
blob = test_bucket.blob(source)
blob.upload_from_string(source)
Expand All @@ -684,6 +687,35 @@ def test_storage_compose_file(test_bucket):

assert composed.decode("utf-8") == source_files[0] + source_files[1]

# Verify sources are NOT deleted
assert test_bucket.blob(source_files[0]).exists()
assert test_bucket.blob(source_files[1]).exists()


def test_storage_compose_file_delete_source(test_bucket):
source_files = [
f"test_upload_blob_1_{uuid.uuid4().hex}",
f"test_upload_blob_2_{uuid.uuid4().hex}",
]
for source in source_files:
blob = test_bucket.blob(source)
blob.upload_from_string(source)

with tempfile.NamedTemporaryFile() as dest_file:
destination = storage_compose_file.compose_file(
test_bucket.name,
source_files[0],
source_files[1],
dest_file.name,
delete_source_objects=True,
)
composed = destination.download_as_bytes()
assert composed.decode("utf-8") == source_files[0] + source_files[1]
Comment on lines +704 to +713

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Using tempfile.NamedTemporaryFile is unnecessary here because compose_file only interacts with GCS blobs and does not read from or write to the local filesystem. We can simplify this by using a unique string for the destination blob name, avoiding the overhead of creating and deleting a temporary file on disk.

    dest_blob_name = f"test_compose_dest_{uuid.uuid4().hex}"
    destination = storage_compose_file.compose_file(
        test_bucket.name,
        source_files[0],
        source_files[1],
        dest_blob_name,
        delete_source_objects=True,
    )
    composed = destination.download_as_bytes()
    assert composed.decode("utf-8") == source_files[0] + source_files[1]


# Verify sources are deleted
assert not test_bucket.blob(source_files[0]).exists()
assert not test_bucket.blob(source_files[1]).exists()


def test_cors_configuration(test_bucket, capsys):
bucket = storage_cors_configuration.cors_configuration(test_bucket)
Expand Down
25 changes: 21 additions & 4 deletions storage/samples/snippets/storage_compose_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,13 @@
from google.cloud import storage


def compose_file(bucket_name, first_blob_name, second_blob_name, destination_blob_name):
def compose_file(
bucket_name,
first_blob_name,
second_blob_name,
destination_blob_name,
delete_source_objects=False,
):
"""Concatenate source blobs into destination blob."""

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The docstring should be updated to reflect the new capability of optionally deleting the source objects.

Suggested change
"""Concatenate source blobs into destination blob."""
"""Concatenate source blobs into destination blob and optionally delete sources."""

# bucket_name = "your-bucket-name"
# first_blob_name = "first-object-name"
Expand All @@ -44,11 +50,22 @@ def compose_file(bucket_name, first_blob_name, second_blob_name, destination_blo
# There is also an `if_source_generation_match` parameter, which is not used in this example.
destination_generation_match_precondition = 0

destination.compose(sources, if_generation_match=destination_generation_match_precondition)
destination.compose(
sources,
if_generation_match=destination_generation_match_precondition,
delete_source_objects=delete_source_objects,
)

deletion_message = (
" and the source objects were deleted." if delete_source_objects else "."
)
print(
"New composite object {} in the bucket {} was created by combining {} and {}".format(
destination_blob_name, bucket_name, first_blob_name, second_blob_name
"New composite object {} in the bucket {} was created by combining {} and {}{}".format(
destination_blob_name,
bucket_name,
first_blob_name,
second_blob_name,
deletion_message,
)
)
return destination
Expand Down