diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3780e6a7ddf..d43a7b0c072 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -77,6 +77,7 @@ /appengine/standard_python3/spanner/* @GoogleCloudPlatform/api-spanner-python @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /asset/**/* @GoogleCloudPlatform/cloud-asset-analysis-team @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /bigquery/**/* @chalmerlowe @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers +/bigquery/bigframes/**/* @tswast @GoogleCloudPlatform/api-bigquery @GoogleCloudPlatform/cloud-samples-reviewers /bigquery/remote_function/**/* @autoerr @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /cloud-media-livestream/**/* @GoogleCloudPlatform/cloud-media-team @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /bigquery-connection/**/* @GoogleCloudPlatform/api-bigquery @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers diff --git a/alloydb/notebooks/noxfile_config.py b/alloydb/notebooks/noxfile_config.py index f5bc1ea9e2f..7f53f031cd7 100644 --- a/alloydb/notebooks/noxfile_config.py +++ b/alloydb/notebooks/noxfile_config.py @@ -14,7 +14,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.11", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/alloydb/notebooks/requirements-test.txt b/alloydb/notebooks/requirements-test.txt index ba12393197f..0a11cd35d0d 100644 --- a/alloydb/notebooks/requirements-test.txt +++ b/alloydb/notebooks/requirements-test.txt @@ -1,6 +1,6 @@ -google-cloud-alloydb-connector[asyncpg]==1.5.0 +google-cloud-alloydb-connector[asyncpg]==1.12.1 sqlalchemy==2.0.40 -pytest==8.3.3 +pytest==9.0.3; python_version >= "3.10" ipykernel==6.29.5 -pytest-asyncio==0.24.0 +pytest-asyncio==1.3.0 nbconvert==7.16.6 \ No newline at end of file diff --git a/aml-ai/requirements-test.txt b/aml-ai/requirements-test.txt index 060ed652e0b..c9e154ba440 100644 --- a/aml-ai/requirements-test.txt +++ b/aml-ai/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 \ No newline at end of file +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/analytics/noxfile_config.py b/appengine/flexible/analytics/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/analytics/noxfile_config.py +++ b/appengine/flexible/analytics/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/analytics/requirements-test.txt b/appengine/flexible/analytics/requirements-test.txt index 0a501624ec5..a4cc4887361 100644 --- a/appengine/flexible/analytics/requirements-test.txt +++ b/appengine/flexible/analytics/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" responses==0.23.1 diff --git a/appengine/flexible/datastore/noxfile_config.py b/appengine/flexible/datastore/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/datastore/noxfile_config.py +++ b/appengine/flexible/datastore/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/datastore/requirements-test.txt b/appengine/flexible/datastore/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/datastore/requirements-test.txt +++ b/appengine/flexible/datastore/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/django_cloudsql/noxfile_config.py b/appengine/flexible/django_cloudsql/noxfile_config.py index 60e19bd8a96..4f10cf1a8f6 100644 --- a/appengine/flexible/django_cloudsql/noxfile_config.py +++ b/appengine/flexible/django_cloudsql/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/appengine/flexible/django_cloudsql/requirements-test.txt b/appengine/flexible/django_cloudsql/requirements-test.txt index 5e5d2c73a81..75ff3b9675d 100644 --- a/appengine/flexible/django_cloudsql/requirements-test.txt +++ b/appengine/flexible/django_cloudsql/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-django==4.9.0 diff --git a/appengine/flexible/hello_world/noxfile_config.py b/appengine/flexible/hello_world/noxfile_config.py index ae3ed14dc36..0973c8621c7 100644 --- a/appengine/flexible/hello_world/noxfile_config.py +++ b/appengine/flexible/hello_world/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/appengine/flexible/hello_world/requirements-test.txt b/appengine/flexible/hello_world/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/hello_world/requirements-test.txt +++ b/appengine/flexible/hello_world/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/hello_world_django/noxfile_config.py b/appengine/flexible/hello_world_django/noxfile_config.py index 692b834f789..949abebe3ea 100644 --- a/appengine/flexible/hello_world_django/noxfile_config.py +++ b/appengine/flexible/hello_world_django/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/hello_world_django/requirements-test.txt b/appengine/flexible/hello_world_django/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/hello_world_django/requirements-test.txt +++ b/appengine/flexible/hello_world_django/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/metadata/noxfile_config.py b/appengine/flexible/metadata/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/metadata/noxfile_config.py +++ b/appengine/flexible/metadata/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/metadata/requirements-test.txt b/appengine/flexible/metadata/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/metadata/requirements-test.txt +++ b/appengine/flexible/metadata/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/multiple_services/gateway-service/noxfile_config.py b/appengine/flexible/multiple_services/gateway-service/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/multiple_services/gateway-service/noxfile_config.py +++ b/appengine/flexible/multiple_services/gateway-service/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/multiple_services/gateway-service/requirements-test.txt b/appengine/flexible/multiple_services/gateway-service/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/multiple_services/gateway-service/requirements-test.txt +++ b/appengine/flexible/multiple_services/gateway-service/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/multiple_services/static-service/noxfile_config.py b/appengine/flexible/multiple_services/static-service/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/multiple_services/static-service/noxfile_config.py +++ b/appengine/flexible/multiple_services/static-service/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/multiple_services/static-service/requirements-test.txt b/appengine/flexible/multiple_services/static-service/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/multiple_services/static-service/requirements-test.txt +++ b/appengine/flexible/multiple_services/static-service/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/numpy/noxfile_config.py b/appengine/flexible/numpy/noxfile_config.py index 5f744eddc83..949abebe3ea 100644 --- a/appengine/flexible/numpy/noxfile_config.py +++ b/appengine/flexible/numpy/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/numpy/requirements-test.txt b/appengine/flexible/numpy/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/numpy/requirements-test.txt +++ b/appengine/flexible/numpy/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/pubsub/noxfile_config.py b/appengine/flexible/pubsub/noxfile_config.py index 501440bf378..eb510f7a4b8 100644 --- a/appengine/flexible/pubsub/noxfile_config.py +++ b/appengine/flexible/pubsub/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to pyarrow compilation failure. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/pubsub/requirements-test.txt b/appengine/flexible/pubsub/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/pubsub/requirements-test.txt +++ b/appengine/flexible/pubsub/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/scipy/noxfile_config.py b/appengine/flexible/scipy/noxfile_config.py index fa718fc163c..6abbd44a54a 100644 --- a/appengine/flexible/scipy/noxfile_config.py +++ b/appengine/flexible/scipy/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.11", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/scipy/requirements-test.txt b/appengine/flexible/scipy/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/scipy/requirements-test.txt +++ b/appengine/flexible/scipy/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/static_files/noxfile_config.py b/appengine/flexible/static_files/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/static_files/noxfile_config.py +++ b/appengine/flexible/static_files/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/static_files/requirements-test.txt b/appengine/flexible/static_files/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/static_files/requirements-test.txt +++ b/appengine/flexible/static_files/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/storage/noxfile_config.py b/appengine/flexible/storage/noxfile_config.py index 972f04d9a4d..addd1f2d00e 100644 --- a/appengine/flexible/storage/noxfile_config.py +++ b/appengine/flexible/storage/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/appengine/flexible/storage/requirements-test.txt b/appengine/flexible/storage/requirements-test.txt index f27726d7455..3b2c5a8f638 100644 --- a/appengine/flexible/storage/requirements-test.txt +++ b/appengine/flexible/storage/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-storage==2.9.0 diff --git a/appengine/flexible/tasks/Dockerfile b/appengine/flexible/tasks/Dockerfile index 5aaeb51144d..10d577925a9 100644 --- a/appengine/flexible/tasks/Dockerfile +++ b/appengine/flexible/tasks/Dockerfile @@ -14,7 +14,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.11 +FROM python:3.14 # Copy local code to the container image. ENV APP_HOME /app diff --git a/appengine/flexible/tasks/noxfile_config.py b/appengine/flexible/tasks/noxfile_config.py index 196376e7023..93284075acd 100644 --- a/appengine/flexible/tasks/noxfile_config.py +++ b/appengine/flexible/tasks/noxfile_config.py @@ -22,7 +22,8 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/tasks/requirements-test.txt b/appengine/flexible/tasks/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/appengine/flexible/tasks/requirements-test.txt +++ b/appengine/flexible/tasks/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/flexible/twilio/noxfile_config.py b/appengine/flexible/twilio/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/twilio/noxfile_config.py +++ b/appengine/flexible/twilio/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/twilio/requirements-test.txt b/appengine/flexible/twilio/requirements-test.txt index 0a501624ec5..a4cc4887361 100644 --- a/appengine/flexible/twilio/requirements-test.txt +++ b/appengine/flexible/twilio/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" responses==0.23.1 diff --git a/appengine/flexible/websockets/noxfile_config.py b/appengine/flexible/websockets/noxfile_config.py index 196376e7023..949abebe3ea 100644 --- a/appengine/flexible/websockets/noxfile_config.py +++ b/appengine/flexible/websockets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/flexible/websockets/requirements-test.txt b/appengine/flexible/websockets/requirements-test.txt index 92b9194cf63..884df6e946a 100644 --- a/appengine/flexible/websockets/requirements-test.txt +++ b/appengine/flexible/websockets/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" retrying==1.3.4 websocket-client==1.7.0 diff --git a/appengine/standard/analytics/requirements-test.txt b/appengine/standard/analytics/requirements-test.txt index 30b5b4c9f19..1f1919df7e1 100644 --- a/appengine/standard/analytics/requirements-test.txt +++ b/appengine/standard/analytics/requirements-test.txt @@ -1,4 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" responses==0.17.0; python_version < '3.7' responses==0.23.1; python_version > '3.6' diff --git a/appengine/standard/blobstore/blobreader/requirements-test.txt b/appengine/standard/blobstore/blobreader/requirements-test.txt index c607ba3b2ab..201a14af7e3 100644 --- a/appengine/standard/blobstore/blobreader/requirements-test.txt +++ b/appengine/standard/blobstore/blobreader/requirements-test.txt @@ -1,3 +1,2 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" WebTest==2.0.35; python_version < '3.0' diff --git a/appengine/standard/blobstore/gcs/requirements-test.txt b/appengine/standard/blobstore/gcs/requirements-test.txt index c607ba3b2ab..201a14af7e3 100644 --- a/appengine/standard/blobstore/gcs/requirements-test.txt +++ b/appengine/standard/blobstore/gcs/requirements-test.txt @@ -1,3 +1,2 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" WebTest==2.0.35; python_version < '3.0' diff --git a/appengine/standard/endpoints-frameworks-v2/echo/requirements-test.txt b/appengine/standard/endpoints-frameworks-v2/echo/requirements-test.txt index b45b8adfc17..c6c555e9b7d 100644 --- a/appengine/standard/endpoints-frameworks-v2/echo/requirements-test.txt +++ b/appengine/standard/endpoints-frameworks-v2/echo/requirements-test.txt @@ -1,5 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' -pytest==8.3.2; python_version >= '3.0' +pytest==9.0.3; python_version >= "3.10" mock==3.0.5; python_version < '3.0' mock==5.1.0; python_version >= '3.0' diff --git a/appengine/standard/endpoints-frameworks-v2/iata/requirements-test.txt b/appengine/standard/endpoints-frameworks-v2/iata/requirements-test.txt index 7439fc43d48..c9e154ba440 100644 --- a/appengine/standard/endpoints-frameworks-v2/iata/requirements-test.txt +++ b/appengine/standard/endpoints-frameworks-v2/iata/requirements-test.txt @@ -1,2 +1 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard/endpoints-frameworks-v2/quickstart/requirements-test.txt b/appengine/standard/endpoints-frameworks-v2/quickstart/requirements-test.txt index b45b8adfc17..c6c555e9b7d 100644 --- a/appengine/standard/endpoints-frameworks-v2/quickstart/requirements-test.txt +++ b/appengine/standard/endpoints-frameworks-v2/quickstart/requirements-test.txt @@ -1,5 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' -pytest==8.3.2; python_version >= '3.0' +pytest==9.0.3; python_version >= "3.10" mock==3.0.5; python_version < '3.0' mock==5.1.0; python_version >= '3.0' diff --git a/appengine/standard/firebase/firenotes/backend/requirements-test.txt b/appengine/standard/firebase/firenotes/backend/requirements-test.txt index b45b8adfc17..c6c555e9b7d 100644 --- a/appengine/standard/firebase/firenotes/backend/requirements-test.txt +++ b/appengine/standard/firebase/firenotes/backend/requirements-test.txt @@ -1,5 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' -pytest==8.3.2; python_version >= '3.0' +pytest==9.0.3; python_version >= "3.10" mock==3.0.5; python_version < '3.0' mock==5.1.0; python_version >= '3.0' diff --git a/appengine/standard/flask/hello_world/requirements-test.txt b/appengine/standard/flask/hello_world/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/flask/hello_world/requirements-test.txt +++ b/appengine/standard/flask/hello_world/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/iap/requirements-test.txt b/appengine/standard/iap/requirements-test.txt index 2bdf05ee6cd..645c7b65f82 100644 --- a/appengine/standard/iap/requirements-test.txt +++ b/appengine/standard/iap/requirements-test.txt @@ -1,5 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' -pytest==8.3.2; python_version >= '3.0' +pytest==9.0.3; python_version >= "3.10" WebTest==2.0.35; python_version < '3.0' six==1.16.0 diff --git a/appengine/standard/images/api/requirements-test.txt b/appengine/standard/images/api/requirements-test.txt index e32096ac3f2..e44de1e709c 100644 --- a/appengine/standard/images/api/requirements-test.txt +++ b/appengine/standard/images/api/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.3.5 +pytest==9.0.3; python_version >= "3.10" six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/memcache/guestbook/requirements-test.txt b/appengine/standard/memcache/guestbook/requirements-test.txt index fc0672f932b..e44de1e709c 100644 --- a/appengine/standard/memcache/guestbook/requirements-test.txt +++ b/appengine/standard/memcache/guestbook/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.3.4 +pytest==9.0.3; python_version >= "3.10" six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/memcache/snippets/requirements-test.txt b/appengine/standard/memcache/snippets/requirements-test.txt index fc0672f932b..e44de1e709c 100644 --- a/appengine/standard/memcache/snippets/requirements-test.txt +++ b/appengine/standard/memcache/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.3.4 +pytest==9.0.3; python_version >= "3.10" six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/migration/incoming/requirements-test.txt b/appengine/standard/migration/incoming/requirements-test.txt index 9dddb06acfc..81aa28131ef 100644 --- a/appengine/standard/migration/incoming/requirements-test.txt +++ b/appengine/standard/migration/incoming/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.3.5 +pytest==9.0.3; python_version >= "3.10" WebTest==3.0.4 diff --git a/appengine/standard/migration/memorystore/requirements-test.txt b/appengine/standard/migration/memorystore/requirements-test.txt index 97587b45b12..8f9dd472203 100644 --- a/appengine/standard/migration/memorystore/requirements-test.txt +++ b/appengine/standard/migration/memorystore/requirements-test.txt @@ -1,6 +1,4 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' -pytest==8.3.2; python_version >= '3.0' +pytest==9.0.3; python_version >= "3.10" mock==5.0.2; python_version > '3.0' mock==3.0.5; python_version < '3.0' six==1.16.0 diff --git a/appengine/standard/migration/ndb/overview/requirements-test.txt b/appengine/standard/migration/ndb/overview/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/migration/ndb/overview/requirements-test.txt +++ b/appengine/standard/migration/ndb/overview/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/migration/ndb/redis_cache/requirements-test.txt b/appengine/standard/migration/ndb/redis_cache/requirements-test.txt index 729e42b9a2e..325a4c49696 100644 --- a/appengine/standard/migration/ndb/redis_cache/requirements-test.txt +++ b/appengine/standard/migration/ndb/redis_cache/requirements-test.txt @@ -1,7 +1,5 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" # 2025-01-14 Adds support for Python3. -pytest==8.3.2; python_version >= '3.0' WebTest==3.0.1; python_version >= '3.0' six==1.16.0 \ No newline at end of file diff --git a/appengine/standard/migration/storage/requirements-test.txt b/appengine/standard/migration/storage/requirements-test.txt index 7439fc43d48..c9e154ba440 100644 --- a/appengine/standard/migration/storage/requirements-test.txt +++ b/appengine/standard/migration/storage/requirements-test.txt @@ -1,2 +1 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard/migration/taskqueue/pull-counter/requirements-test.txt b/appengine/standard/migration/taskqueue/pull-counter/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/migration/taskqueue/pull-counter/requirements-test.txt +++ b/appengine/standard/migration/taskqueue/pull-counter/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/migration/urlfetch/async/requirements-test.txt b/appengine/standard/migration/urlfetch/async/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/migration/urlfetch/async/requirements-test.txt +++ b/appengine/standard/migration/urlfetch/async/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/migration/urlfetch/requests/requirements-test.txt b/appengine/standard/migration/urlfetch/requests/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/migration/urlfetch/requests/requirements-test.txt +++ b/appengine/standard/migration/urlfetch/requests/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/ndb/overview/requirements-test.txt b/appengine/standard/ndb/overview/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/ndb/overview/requirements-test.txt +++ b/appengine/standard/ndb/overview/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/ndb/transactions/requirements-test.txt b/appengine/standard/ndb/transactions/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/ndb/transactions/requirements-test.txt +++ b/appengine/standard/ndb/transactions/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/noxfile_config.py b/appengine/standard/noxfile_config.py index f39811085fa..c58186d0158 100644 --- a/appengine/standard/noxfile_config.py +++ b/appengine/standard/noxfile_config.py @@ -24,7 +24,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["3.6", "3.7", "3.8", "3.10", "3.11", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard/taskqueue/counter/requirements-test.txt b/appengine/standard/taskqueue/counter/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/taskqueue/counter/requirements-test.txt +++ b/appengine/standard/taskqueue/counter/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/taskqueue/pull-counter/requirements-test.txt b/appengine/standard/taskqueue/pull-counter/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/taskqueue/pull-counter/requirements-test.txt +++ b/appengine/standard/taskqueue/pull-counter/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/urlfetch/async/requirements-test.txt b/appengine/standard/urlfetch/async/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/urlfetch/async/requirements-test.txt +++ b/appengine/standard/urlfetch/async/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/urlfetch/requests/requirements-test.txt b/appengine/standard/urlfetch/requests/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/urlfetch/requests/requirements-test.txt +++ b/appengine/standard/urlfetch/requests/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/urlfetch/snippets/requirements-test.txt b/appengine/standard/urlfetch/snippets/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/urlfetch/snippets/requirements-test.txt +++ b/appengine/standard/urlfetch/snippets/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard/users/requirements-test.txt b/appengine/standard/users/requirements-test.txt index 454c88a573a..e7fb62924c9 100644 --- a/appengine/standard/users/requirements-test.txt +++ b/appengine/standard/users/requirements-test.txt @@ -1,6 +1,3 @@ -# pin pytest to 4.6.11 for Python2. -pytest==4.6.11; python_version < '3.0' +pytest==9.0.3; python_version >= "3.10" -# pytest==8.3.4 and six==1.17.0 for Python3. -pytest==8.3.4; python_version >= '3.0' six==1.17.0 \ No newline at end of file diff --git a/appengine/standard_python3/bigquery/noxfile_config.py b/appengine/standard_python3/bigquery/noxfile_config.py index 3428b4ef27a..5dbb51286c6 100644 --- a/appengine/standard_python3/bigquery/noxfile_config.py +++ b/appengine/standard_python3/bigquery/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # There's no google-cloud-bigquery package for Python 3.9. - "ignored_versions": ["2.7", "3.6", "3.9", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bigquery/requirements-test.txt b/appengine/standard_python3/bigquery/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/bigquery/requirements-test.txt +++ b/appengine/standard_python3/bigquery/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/building-an-app/building-an-app-1/requirements-test.txt b/appengine/standard_python3/building-an-app/building-an-app-1/requirements-test.txt index c987bcfee7e..c9e154ba440 100644 --- a/appengine/standard_python3/building-an-app/building-an-app-1/requirements-test.txt +++ b/appengine/standard_python3/building-an-app/building-an-app-1/requirements-test.txt @@ -1,2 +1 @@ -pytest==7.0.1; python_version == '3.9' -pytest==9.0.2; python_version >= '3.10' +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/building-an-app/building-an-app-2/requirements-test.txt b/appengine/standard_python3/building-an-app/building-an-app-2/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/building-an-app/building-an-app-2/requirements-test.txt +++ b/appengine/standard_python3/building-an-app/building-an-app-2/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/building-an-app/building-an-app-3/requirements-test.txt b/appengine/standard_python3/building-an-app/building-an-app-3/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/building-an-app/building-an-app-3/requirements-test.txt +++ b/appengine/standard_python3/building-an-app/building-an-app-3/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/building-an-app/building-an-app-4/requirements-test.txt b/appengine/standard_python3/building-an-app/building-an-app-4/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/building-an-app/building-an-app-4/requirements-test.txt +++ b/appengine/standard_python3/building-an-app/building-an-app-4/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/bundled-services/blobstore/django/noxfile_config.py b/appengine/standard_python3/bundled-services/blobstore/django/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/blobstore/django/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/blobstore/django/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/blobstore/django/requirements-test.txt b/appengine/standard_python3/bundled-services/blobstore/django/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/blobstore/django/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/blobstore/django/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/blobstore/django/requirements.txt b/appengine/standard_python3/bundled-services/blobstore/django/requirements.txt index c616634cafe..0dca8b45f05 100644 --- a/appengine/standard_python3/bundled-services/blobstore/django/requirements.txt +++ b/appengine/standard_python3/bundled-services/blobstore/django/requirements.txt @@ -1,4 +1,4 @@ -Django==5.1.9; python_version >= "3.10" +Django==5.1.15; python_version >= "3.10" Django==4.2.16; python_version < "3.10" django-environ==0.10.0 google-cloud-logging==3.5.0 diff --git a/appengine/standard_python3/bundled-services/blobstore/flask/noxfile_config.py b/appengine/standard_python3/bundled-services/blobstore/flask/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/blobstore/flask/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/blobstore/flask/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/blobstore/flask/requirements-test.txt b/appengine/standard_python3/bundled-services/blobstore/flask/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/blobstore/flask/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/blobstore/flask/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/blobstore/wsgi/noxfile_config.py b/appengine/standard_python3/bundled-services/blobstore/wsgi/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/blobstore/wsgi/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/blobstore/wsgi/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/blobstore/wsgi/requirements-test.txt b/appengine/standard_python3/bundled-services/blobstore/wsgi/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/blobstore/wsgi/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/blobstore/wsgi/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/deferred/django/noxfile_config.py b/appengine/standard_python3/bundled-services/deferred/django/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/deferred/django/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/deferred/django/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/deferred/django/requirements-test.txt b/appengine/standard_python3/bundled-services/deferred/django/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/deferred/django/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/deferred/django/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/deferred/flask/noxfile_config.py b/appengine/standard_python3/bundled-services/deferred/flask/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/deferred/flask/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/deferred/flask/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/deferred/flask/requirements-test.txt b/appengine/standard_python3/bundled-services/deferred/flask/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/deferred/flask/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/deferred/flask/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/deferred/wsgi/noxfile_config.py b/appengine/standard_python3/bundled-services/deferred/wsgi/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/deferred/wsgi/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/deferred/wsgi/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/deferred/wsgi/requirements-test.txt b/appengine/standard_python3/bundled-services/deferred/wsgi/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/deferred/wsgi/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/deferred/wsgi/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/mail/django/noxfile_config.py b/appengine/standard_python3/bundled-services/mail/django/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/mail/django/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/mail/django/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/mail/django/requirements-test.txt b/appengine/standard_python3/bundled-services/mail/django/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/mail/django/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/mail/django/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/mail/django/requirements.txt b/appengine/standard_python3/bundled-services/mail/django/requirements.txt index bdd07a4620e..d5731eb8861 100644 --- a/appengine/standard_python3/bundled-services/mail/django/requirements.txt +++ b/appengine/standard_python3/bundled-services/mail/django/requirements.txt @@ -1,4 +1,4 @@ -Django==5.1.13; python_version >= "3.10" +Django==5.1.15; python_version >= "3.10" Django==4.2.16; python_version >= "3.8" and python_version < "3.10" Django==3.2.25; python_version < "3.8" django-environ==0.10.0 diff --git a/appengine/standard_python3/bundled-services/mail/flask/noxfile_config.py b/appengine/standard_python3/bundled-services/mail/flask/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/mail/flask/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/mail/flask/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/mail/flask/requirements-test.txt b/appengine/standard_python3/bundled-services/mail/flask/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/mail/flask/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/mail/flask/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/bundled-services/mail/wsgi/noxfile_config.py b/appengine/standard_python3/bundled-services/mail/wsgi/noxfile_config.py index 1bde00988d8..99047da44a9 100644 --- a/appengine/standard_python3/bundled-services/mail/wsgi/noxfile_config.py +++ b/appengine/standard_python3/bundled-services/mail/wsgi/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/appengine/standard_python3/bundled-services/mail/wsgi/requirements-test.txt b/appengine/standard_python3/bundled-services/mail/wsgi/requirements-test.txt index b83aedab5dd..b8e94617358 100644 --- a/appengine/standard_python3/bundled-services/mail/wsgi/requirements-test.txt +++ b/appengine/standard_python3/bundled-services/mail/wsgi/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" requests==2.28.2 \ No newline at end of file diff --git a/appengine/standard_python3/cloudsql/requirements-test.txt b/appengine/standard_python3/cloudsql/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/cloudsql/requirements-test.txt +++ b/appengine/standard_python3/cloudsql/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/custom-server/requirements-test.txt b/appengine/standard_python3/custom-server/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/custom-server/requirements-test.txt +++ b/appengine/standard_python3/custom-server/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/django/noxfile_config.py b/appengine/standard_python3/django/noxfile_config.py index b05bde23ec6..2e8aab8cdeb 100644 --- a/appengine/standard_python3/django/noxfile_config.py +++ b/appengine/standard_python3/django/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/appengine/standard_python3/django/requirements-test.txt b/appengine/standard_python3/django/requirements-test.txt index 99bb7d24ca4..5279f42cfd4 100644 --- a/appengine/standard_python3/django/requirements-test.txt +++ b/appengine/standard_python3/django/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" pytest-django==4.5.0 diff --git a/appengine/standard_python3/hello_world/requirements-test.txt b/appengine/standard_python3/hello_world/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/hello_world/requirements-test.txt +++ b/appengine/standard_python3/hello_world/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/migration/urlfetch/requirements-test.txt b/appengine/standard_python3/migration/urlfetch/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/migration/urlfetch/requirements-test.txt +++ b/appengine/standard_python3/migration/urlfetch/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/pubsub/requirements-test.txt b/appengine/standard_python3/pubsub/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/pubsub/requirements-test.txt +++ b/appengine/standard_python3/pubsub/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/redis/requirements-test.txt b/appengine/standard_python3/redis/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/redis/requirements-test.txt +++ b/appengine/standard_python3/redis/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/spanner/requirements-test.txt b/appengine/standard_python3/spanner/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/spanner/requirements-test.txt +++ b/appengine/standard_python3/spanner/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/appengine/standard_python3/warmup/requirements-test.txt b/appengine/standard_python3/warmup/requirements-test.txt index c2845bffbe8..c9e154ba440 100644 --- a/appengine/standard_python3/warmup/requirements-test.txt +++ b/appengine/standard_python3/warmup/requirements-test.txt @@ -1 +1 @@ -pytest==7.0.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/asset/snippets/noxfile_config.py b/asset/snippets/noxfile_config.py index 9a1680c88df..4eae59976e2 100644 --- a/asset/snippets/noxfile_config.py +++ b/asset/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/asset/snippets/requirements-test.txt b/asset/snippets/requirements-test.txt index d57b0bfd0ab..b85518d5117 100644 --- a/asset/snippets/requirements-test.txt +++ b/asset/snippets/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1 flaky==3.8.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/auth/api-client/requirements-test.txt b/auth/api-client/requirements-test.txt index 6ff70adf77d..975d5ee58c2 100644 --- a/auth/api-client/requirements-test.txt +++ b/auth/api-client/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" backoff==2.2.1 diff --git a/auth/cloud-client-temp/noxfile_config.py b/auth/cloud-client-temp/noxfile_config.py index e892b338fce..658f73ff360 100644 --- a/auth/cloud-client-temp/noxfile_config.py +++ b/auth/cloud-client-temp/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/auth/cloud-client-temp/requirements.txt b/auth/cloud-client-temp/requirements.txt index 8dafe853ea0..c0bda55ed4a 100644 --- a/auth/cloud-client-temp/requirements.txt +++ b/auth/cloud-client-temp/requirements.txt @@ -1,8 +1,7 @@ google-cloud-compute==1.42.0 google-cloud-storage==3.8.0 google-auth==2.47.0 -pytest===8.4.2; python_version == '3.9' -pytest==9.0.2; python_version > '3.9' +pytest==9.0.3; python_version >= "3.10" boto3>=1.26.0 requests==2.32.5 python-dotenv==1.2.1 diff --git a/auth/cloud-client/requirements-test.txt b/auth/cloud-client/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/auth/cloud-client/requirements-test.txt +++ b/auth/cloud-client/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/auth/custom-credentials/aws/Dockerfile b/auth/custom-credentials/aws/Dockerfile index d90d88aa0a8..0cd34429f50 100644 --- a/auth/custom-credentials/aws/Dockerfile +++ b/auth/custom-credentials/aws/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.13-slim +FROM python:3.14-slim RUN useradd -m appuser diff --git a/auth/custom-credentials/aws/noxfile_config.py b/auth/custom-credentials/aws/noxfile_config.py index 0ed973689f7..526763fbcc9 100644 --- a/auth/custom-credentials/aws/noxfile_config.py +++ b/auth/custom-credentials/aws/noxfile_config.py @@ -13,5 +13,6 @@ # limitations under the License. TEST_CONFIG_OVERRIDE = { - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], } diff --git a/auth/custom-credentials/aws/requirements-test.txt b/auth/custom-credentials/aws/requirements-test.txt index 43b24059d3e..8e23e776847 100644 --- a/auth/custom-credentials/aws/requirements-test.txt +++ b/auth/custom-credentials/aws/requirements-test.txt @@ -1,2 +1,2 @@ -r requirements.txt -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/auth/custom-credentials/okta/noxfile_config.py b/auth/custom-credentials/okta/noxfile_config.py index 0ed973689f7..379d24818b5 100644 --- a/auth/custom-credentials/okta/noxfile_config.py +++ b/auth/custom-credentials/okta/noxfile_config.py @@ -13,5 +13,5 @@ # limitations under the License. TEST_CONFIG_OVERRIDE = { - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], } diff --git a/auth/custom-credentials/okta/requirements-test.txt b/auth/custom-credentials/okta/requirements-test.txt index f47609d2651..8e23e776847 100644 --- a/auth/custom-credentials/okta/requirements-test.txt +++ b/auth/custom-credentials/okta/requirements-test.txt @@ -1,2 +1,2 @@ -r requirements.txt -pytest==7.1.2 +pytest==9.0.3; python_version >= "3.10" diff --git a/auth/custom-credentials/okta/requirements.txt b/auth/custom-credentials/okta/requirements.txt index d9669ebee9f..893676b178f 100644 --- a/auth/custom-credentials/okta/requirements.txt +++ b/auth/custom-credentials/okta/requirements.txt @@ -1,4 +1,4 @@ requests==2.32.3 google-cloud-storage==2.19.0 google-auth==2.43.0 -python-dotenv==1.1.1 +python-dotenv==1.2.2 diff --git a/auth/downscoping/requirements-test.txt b/auth/downscoping/requirements-test.txt index 5d399275c93..a967ba62fb1 100644 --- a/auth/downscoping/requirements-test.txt +++ b/auth/downscoping/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-storage==2.9.0; python_version < '3.7' google-cloud-storage==2.9.0; python_version > '3.6' diff --git a/auth/end-user/web/requirements-test.txt b/auth/end-user/web/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/auth/end-user/web/requirements-test.txt +++ b/auth/end-user/web/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/auth/service-to-service/noxfile_config.py b/auth/service-to-service/noxfile_config.py index bf45881273d..f6a3cf7417f 100644 --- a/auth/service-to-service/noxfile_config.py +++ b/auth/service-to-service/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # We only run the cloud run tests in py38 session. - "ignored_versions": ["2.7", "3.6", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/auth/service-to-service/requirements-test.txt b/auth/service-to-service/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/auth/service-to-service/requirements-test.txt +++ b/auth/service-to-service/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/automl/snippets/noxfile_config.py b/automl/snippets/noxfile_config.py index ef111e5e309..b4a45d95757 100644 --- a/automl/snippets/noxfile_config.py +++ b/automl/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them # "enforce_type_hints": False, diff --git a/automl/snippets/requirements-test.txt b/automl/snippets/requirements-test.txt index f3230681cda..79932f83530 100644 --- a/automl/snippets/requirements-test.txt +++ b/automl/snippets/requirements-test.txt @@ -1,2 +1,2 @@ backoff==2.2.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/batch/requirements-test.txt b/batch/requirements-test.txt index 08d1b1b9c1f..4daf4a01072 100644 --- a/batch/requirements-test.txt +++ b/batch/requirements-test.txt @@ -1,4 +1,4 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-compute==1.11.0 google-cloud-resource-manager==1.10.1 google-cloud-storage==2.9.0 diff --git a/bigquery-connection/snippets/noxfile_config.py b/bigquery-connection/snippets/noxfile_config.py index 28c09af52f7..6cd93d1263d 100644 --- a/bigquery-connection/snippets/noxfile_config.py +++ b/bigquery-connection/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery-connection/snippets/requirements-test.txt b/bigquery-connection/snippets/requirements-test.txt index 5b0f38d50e2..334e1bfdbb8 100644 --- a/bigquery-connection/snippets/requirements-test.txt +++ b/bigquery-connection/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-testutils==1.5.0 \ No newline at end of file diff --git a/bigquery-datatransfer/snippets/noxfile_config.py b/bigquery-datatransfer/snippets/noxfile_config.py index 161ffcc14f3..5c1c0aa7b2f 100644 --- a/bigquery-datatransfer/snippets/noxfile_config.py +++ b/bigquery-datatransfer/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/bigquery-datatransfer/snippets/requirements-test.txt b/bigquery-datatransfer/snippets/requirements-test.txt index ae8913096ea..3c6ff1b456d 100644 --- a/bigquery-datatransfer/snippets/requirements-test.txt +++ b/bigquery-datatransfer/snippets/requirements-test.txt @@ -1,4 +1,4 @@ google-cloud-bigquery==3.27.0 google-cloud-pubsub==2.28.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" mock==5.1.0 diff --git a/bigquery-migration/snippets/noxfile_config.py b/bigquery-migration/snippets/noxfile_config.py index 68825a3b2dc..5ed96a67651 100644 --- a/bigquery-migration/snippets/noxfile_config.py +++ b/bigquery-migration/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery-migration/snippets/requirements-test.txt b/bigquery-migration/snippets/requirements-test.txt index d54b3ea50e2..6b60bac71d7 100644 --- a/bigquery-migration/snippets/requirements-test.txt +++ b/bigquery-migration/snippets/requirements-test.txt @@ -1,4 +1,4 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-testutils==1.5.0 google-api-core==2.17.1 google-cloud-storage==2.9.0 \ No newline at end of file diff --git a/bigquery-reservation/snippets/noxfile_config.py b/bigquery-reservation/snippets/noxfile_config.py index e1c631ca86f..4fb2be93998 100644 --- a/bigquery-reservation/snippets/noxfile_config.py +++ b/bigquery-reservation/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery-reservation/snippets/requirements-test.txt b/bigquery-reservation/snippets/requirements-test.txt index 840c3fcffe5..3e1319f761b 100644 --- a/bigquery-reservation/snippets/requirements-test.txt +++ b/bigquery-reservation/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-testutils==1.5.0 diff --git a/bigquery/bigframes/.gitignore b/bigquery/bigframes/.gitignore new file mode 100644 index 00000000000..6e1f113ef05 --- /dev/null +++ b/bigquery/bigframes/.gitignore @@ -0,0 +1 @@ +noxfile.py diff --git a/bigquery/bigframes/README.md b/bigquery/bigframes/README.md new file mode 100644 index 00000000000..a243089ef7f --- /dev/null +++ b/bigquery/bigframes/README.md @@ -0,0 +1,10 @@ +# BigQuery DataFrames code samples + +This directory contains code samples for [BigQuery DataFrames (aka +BigFrames)](https://dataframes.bigquery.dev/). + +To install BigQuery DataFrames, run: + +``` +pip install --upgrade bigframes +``` diff --git a/bigquery/bigframes/call_python_udf.py b/bigquery/bigframes/call_python_udf.py new file mode 100644 index 00000000000..200b1c1fb54 --- /dev/null +++ b/bigquery/bigframes/call_python_udf.py @@ -0,0 +1,105 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START bigquery_dataframes_call_python_udf] +import textwrap +from typing import Tuple + +import bigframes.pandas as bpd +import pandas as pd +import pyarrow as pa + + +# Using partial ordering mode enables more efficient query optimizations. +bpd.options.bigquery.ordering_mode = "partial" + + +def call_python_udf( + project_id: str, location: str, +) -> Tuple[pd.Series, bpd.Series]: + # Set the billing project to use for queries. This step is optional, as the + # project can be inferred from your environment in many cases. + bpd.options.bigquery.project = project_id # "your-project-id" + + # Since this example works with local data, set a processing location. + bpd.options.bigquery.location = location # "US" + + # Create a sample series. + xml_series = pd.Series( + [ + textwrap.dedent( + """ + + The Great Gatsby + F. Scott Fitzgerald + + """ + ), + textwrap.dedent( + """ + + 1984 + George Orwell + + """ + ), + textwrap.dedent( + """ + + Brave New World + Aldous Huxley + + """ + ), + ], + dtype=pd.ArrowDtype(pa.string()), + ) + df = pd.DataFrame({"xml": xml_series}) + + # Use the BigQuery Accessor, which is automatically registered on pandas + # DataFrames when you import bigframes. This example uses a function that + # has been deployed to bigquery-utils for demonstration purposes. To use in + # production, deploy the function at + # https://github.com/GoogleCloudPlatform/bigquery-utils/blob/master/udfs/community/cw_xml_extract.sqlx + # to your own project. + titles_pandas = df.bigquery.sql_scalar( + "`bqutil`.`fn`.cw_xml_extract({xml}, '//title/text()')", + ) + + # Alternatively, call read_gbq_function to get a pointer to the function + # that can be applied on BigQuery DataFrames objects. + cw_xml_extract = bpd.read_gbq_function("bqutil.fn.cw_xml_extract") + xml_bigframes = bpd.read_pandas(xml_series) + + xpath_query = "//title/text()" + titles_bigframes = xml_bigframes.apply(cw_xml_extract, args=(xpath_query,)) + return titles_pandas, titles_bigframes + # [END bigquery_dataframes_call_python_udf] + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + + # Note: GCP project ID can be inferred from the environment if Application + # Default Credentials are set, so None is perfectly valid for --project_id. + parser.add_argument("--project_id", type=str) + parser.add_argument("--location", default="US", type=str) + args = parser.parse_args() + + pddf, bfdf = call_python_udf(project_id=args.project_id, location=args.location) + print(pddf) + print(bfdf.to_pandas()) diff --git a/bigquery/bigframes/call_python_udf_test.py b/bigquery/bigframes/call_python_udf_test.py new file mode 100644 index 00000000000..67475d94718 --- /dev/null +++ b/bigquery/bigframes/call_python_udf_test.py @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bigframes.pandas as bpd + +import call_python_udf + + +def test_call_python_udf(project_id: str, location: str) -> None: + bpd.close_session() + pd_result, bf_result = call_python_udf.call_python_udf(project_id=project_id, location=location) + assert len(pd_result.index) == 3 + assert len(bf_result.index) == 3 diff --git a/bigquery/bigframes/conftest.py b/bigquery/bigframes/conftest.py new file mode 100644 index 00000000000..a3274f08ca3 --- /dev/null +++ b/bigquery/bigframes/conftest.py @@ -0,0 +1,27 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + + +@pytest.fixture(scope="session") +def project_id() -> str: + return os.environ["GOOGLE_CLOUD_PROJECT"] + + +@pytest.fixture(scope="session") +def location() -> str: + return "US" diff --git a/bigquery/bigframes/noxfile_config.py b/bigquery/bigframes/noxfile_config.py new file mode 100644 index 00000000000..322efba33e8 --- /dev/null +++ b/bigquery/bigframes/noxfile_config.py @@ -0,0 +1,38 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} diff --git a/bigquery/bigframes/requirements-test.txt b/bigquery/bigframes/requirements-test.txt new file mode 100644 index 00000000000..8b8e28c484c --- /dev/null +++ b/bigquery/bigframes/requirements-test.txt @@ -0,0 +1,2 @@ +flaky==3.8.1 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/bigframes/requirements.txt b/bigquery/bigframes/requirements.txt new file mode 100644 index 00000000000..a14856a58d8 --- /dev/null +++ b/bigquery/bigframes/requirements.txt @@ -0,0 +1 @@ +bigframes==2.38.0 diff --git a/bigquery/bqml/noxfile_config.py b/bigquery/bqml/noxfile_config.py index 4e2c703470f..a4cb0cac197 100644 --- a/bigquery/bqml/noxfile_config.py +++ b/bigquery/bqml/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to pyarrow compilation failure. - "ignored_versions": ["2.7", "3.6", "3.9", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/bigquery/bqml/requirements-test.txt b/bigquery/bqml/requirements-test.txt index f1684cd8061..8b8e28c484c 100644 --- a/bigquery/bqml/requirements-test.txt +++ b/bigquery/bqml/requirements-test.txt @@ -1,2 +1,2 @@ flaky==3.8.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/cloud-client/requirements-test.txt b/bigquery/cloud-client/requirements-test.txt index 7d32dfc20c7..1fb857b4d88 100644 --- a/bigquery/cloud-client/requirements-test.txt +++ b/bigquery/cloud-client/requirements-test.txt @@ -1,3 +1,3 @@ # samples/snippets should be runnable with no "extras" google-cloud-testutils==1.5.0 -pytest==8.3.4 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/continuous-queries/requirements-test.txt b/bigquery/continuous-queries/requirements-test.txt index ecdd071f48d..a4f09c46e50 100644 --- a/bigquery/continuous-queries/requirements-test.txt +++ b/bigquery/continuous-queries/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.3.5 +pytest==9.0.3; python_version >= "3.10" google-auth==2.38.0 requests==2.32.4 diff --git a/bigquery/pandas-gbq-migration/noxfile_config.py b/bigquery/pandas-gbq-migration/noxfile_config.py index 4e2c703470f..a4cb0cac197 100644 --- a/bigquery/pandas-gbq-migration/noxfile_config.py +++ b/bigquery/pandas-gbq-migration/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to pyarrow compilation failure. - "ignored_versions": ["2.7", "3.6", "3.9", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/bigquery/pandas-gbq-migration/requirements-test.txt b/bigquery/pandas-gbq-migration/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/bigquery/pandas-gbq-migration/requirements-test.txt +++ b/bigquery/pandas-gbq-migration/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/python-db-dtypes-pandas/snippets/requirements-test.txt b/bigquery/python-db-dtypes-pandas/snippets/requirements-test.txt index 9471b3d92fb..c9e154ba440 100644 --- a/bigquery/python-db-dtypes-pandas/snippets/requirements-test.txt +++ b/bigquery/python-db-dtypes-pandas/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==8.4.2 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/remote-function/document/noxfile_config.py b/bigquery/remote-function/document/noxfile_config.py index 129472ab778..05254a9bf8a 100644 --- a/bigquery/remote-function/document/noxfile_config.py +++ b/bigquery/remote-function/document/noxfile_config.py @@ -17,7 +17,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery/remote-function/document/requirements-test.txt b/bigquery/remote-function/document/requirements-test.txt index 254febb7aba..ac936365b2e 100644 --- a/bigquery/remote-function/document/requirements-test.txt +++ b/bigquery/remote-function/document/requirements-test.txt @@ -1,4 +1,4 @@ Flask==2.2.2 functions-framework==3.9.2 google-cloud-documentai==3.0.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/remote-function/translate/noxfile_config.py b/bigquery/remote-function/translate/noxfile_config.py index 881bc58580f..45b90dd8081 100644 --- a/bigquery/remote-function/translate/noxfile_config.py +++ b/bigquery/remote-function/translate/noxfile_config.py @@ -17,7 +17,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery/remote-function/translate/requirements-test.txt b/bigquery/remote-function/translate/requirements-test.txt index 2048a36731f..adb53acce34 100644 --- a/bigquery/remote-function/translate/requirements-test.txt +++ b/bigquery/remote-function/translate/requirements-test.txt @@ -1,4 +1,4 @@ Flask==2.2.2 functions-framework==3.9.2 google-cloud-translate==3.18.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery/remote-function/vision/noxfile_config.py b/bigquery/remote-function/vision/noxfile_config.py index 881bc58580f..45b90dd8081 100644 --- a/bigquery/remote-function/vision/noxfile_config.py +++ b/bigquery/remote-function/vision/noxfile_config.py @@ -17,7 +17,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery/remote-function/vision/requirements-test.txt b/bigquery/remote-function/vision/requirements-test.txt index 62634fcffc0..20f0bc405f4 100644 --- a/bigquery/remote-function/vision/requirements-test.txt +++ b/bigquery/remote-function/vision/requirements-test.txt @@ -1,4 +1,4 @@ Flask==2.2.2 functions-framework==3.9.2 google-cloud-vision==3.8.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery_storage/pyarrow/noxfile_config.py b/bigquery_storage/pyarrow/noxfile_config.py index 29edb31ffe8..fea7f946dff 100644 --- a/bigquery_storage/pyarrow/noxfile_config.py +++ b/bigquery_storage/pyarrow/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery_storage/pyarrow/requirements-test.txt b/bigquery_storage/pyarrow/requirements-test.txt index 7561ed55ce2..c9e154ba440 100644 --- a/bigquery_storage/pyarrow/requirements-test.txt +++ b/bigquery_storage/pyarrow/requirements-test.txt @@ -1,3 +1 @@ -pytest===7.4.3; python_version == '3.7' -pytest===8.3.5; python_version == '3.8' -pytest==8.4.1; python_version >= '3.9' +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery_storage/quickstart/noxfile_config.py b/bigquery_storage/quickstart/noxfile_config.py index f1fa9e5618b..0973c8621c7 100644 --- a/bigquery_storage/quickstart/noxfile_config.py +++ b/bigquery_storage/quickstart/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery_storage/quickstart/requirements-test.txt b/bigquery_storage/quickstart/requirements-test.txt index 7561ed55ce2..c9e154ba440 100644 --- a/bigquery_storage/quickstart/requirements-test.txt +++ b/bigquery_storage/quickstart/requirements-test.txt @@ -1,3 +1 @@ -pytest===7.4.3; python_version == '3.7' -pytest===8.3.5; python_version == '3.8' -pytest==8.4.1; python_version >= '3.9' +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery_storage/snippets/noxfile_config.py b/bigquery_storage/snippets/noxfile_config.py index f1fa9e5618b..0973c8621c7 100644 --- a/bigquery_storage/snippets/noxfile_config.py +++ b/bigquery_storage/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery_storage/snippets/requirements-test.txt b/bigquery_storage/snippets/requirements-test.txt index 230ca56dc3a..06cc720170b 100644 --- a/bigquery_storage/snippets/requirements-test.txt +++ b/bigquery_storage/snippets/requirements-test.txt @@ -1,4 +1,2 @@ google-cloud-testutils==1.6.4 -pytest===7.4.3; python_version == '3.7' -pytest===8.3.5; python_version == '3.8' -pytest==8.4.1; python_version >= '3.9' +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery_storage/snippets/requirements.txt b/bigquery_storage/snippets/requirements.txt index 8a456493526..821c0abb139 100644 --- a/bigquery_storage/snippets/requirements.txt +++ b/bigquery_storage/snippets/requirements.txt @@ -1,6 +1,4 @@ google-cloud-bigquery-storage==2.32.0 google-cloud-bigquery===3.30.0; python_version <= '3.8' google-cloud-bigquery==3.35.1; python_version >= '3.9' -pytest===7.4.3; python_version == '3.7' -pytest===8.3.5; python_version == '3.8' -pytest==8.4.1; python_version >= '3.9' +pytest==9.0.3; python_version >= "3.10" diff --git a/bigquery_storage/to_dataframe/noxfile_config.py b/bigquery_storage/to_dataframe/noxfile_config.py index f1fa9e5618b..0973c8621c7 100644 --- a/bigquery_storage/to_dataframe/noxfile_config.py +++ b/bigquery_storage/to_dataframe/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/bigquery_storage/to_dataframe/requirements-test.txt b/bigquery_storage/to_dataframe/requirements-test.txt index 7561ed55ce2..c9e154ba440 100644 --- a/bigquery_storage/to_dataframe/requirements-test.txt +++ b/bigquery_storage/to_dataframe/requirements-test.txt @@ -1,3 +1 @@ -pytest===7.4.3; python_version == '3.7' -pytest===8.3.5; python_version == '3.8' -pytest==8.4.1; python_version >= '3.9' +pytest==9.0.3; python_version >= "3.10" diff --git a/billing/requirements-test.txt b/billing/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/billing/requirements-test.txt +++ b/billing/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/blog/introduction_to_data_models_in_cloud_datastore/noxfile_config.py b/blog/introduction_to_data_models_in_cloud_datastore/noxfile_config.py index 9a1680c88df..4eae59976e2 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/noxfile_config.py +++ b/blog/introduction_to_data_models_in_cloud_datastore/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/blog/introduction_to_data_models_in_cloud_datastore/requirements-test.txt b/blog/introduction_to_data_models_in_cloud_datastore/requirements-test.txt index 185d62c4204..23df1e03c7e 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/requirements-test.txt +++ b/blog/introduction_to_data_models_in_cloud_datastore/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" flaky==3.8.1 diff --git a/cloud-sql/mysql/client-side-encryption/requirements.txt b/cloud-sql/mysql/client-side-encryption/requirements.txt index 32f632b2ca7..52b5fac5d02 100644 --- a/cloud-sql/mysql/client-side-encryption/requirements.txt +++ b/cloud-sql/mysql/client-side-encryption/requirements.txt @@ -1,3 +1,3 @@ SQLAlchemy==2.0.40 -PyMySQL==1.1.1 +PyMySQL==1.1.2 tink==1.9.0 diff --git a/cloud-sql/mysql/sqlalchemy/Dockerfile b/cloud-sql/mysql/sqlalchemy/Dockerfile index 72a0ef555e1..d0ad90b7fbb 100644 --- a/cloud-sql/mysql/sqlalchemy/Dockerfile +++ b/cloud-sql/mysql/sqlalchemy/Dockerfile @@ -14,7 +14,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.13 +FROM python:3.14 RUN apt-get update diff --git a/cloud-sql/mysql/sqlalchemy/requirements.txt b/cloud-sql/mysql/sqlalchemy/requirements.txt index a5e6f819085..bcb4ea5c4bb 100644 --- a/cloud-sql/mysql/sqlalchemy/requirements.txt +++ b/cloud-sql/mysql/sqlalchemy/requirements.txt @@ -1,7 +1,7 @@ Flask==2.2.2 SQLAlchemy==2.0.40 -PyMySQL==1.1.1 +PyMySQL==1.1.2 gunicorn==23.0.0 -cloud-sql-python-connector==1.20.0 +cloud-sql-python-connector==1.20.1 functions-framework==3.9.2 Werkzeug==2.3.8 diff --git a/cloud-sql/postgres/sqlalchemy/Dockerfile b/cloud-sql/postgres/sqlalchemy/Dockerfile index 72a0ef555e1..d0ad90b7fbb 100644 --- a/cloud-sql/postgres/sqlalchemy/Dockerfile +++ b/cloud-sql/postgres/sqlalchemy/Dockerfile @@ -14,7 +14,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.13 +FROM python:3.14 RUN apt-get update diff --git a/cloud-sql/postgres/sqlalchemy/requirements.txt b/cloud-sql/postgres/sqlalchemy/requirements.txt index ba738cc1669..e44a280e6bf 100644 --- a/cloud-sql/postgres/sqlalchemy/requirements.txt +++ b/cloud-sql/postgres/sqlalchemy/requirements.txt @@ -1,7 +1,7 @@ Flask==2.2.2 pg8000==1.31.5 SQLAlchemy==2.0.40 -cloud-sql-python-connector==1.20.0 +cloud-sql-python-connector==1.20.1 gunicorn==23.0.0 functions-framework==3.9.2 Werkzeug==2.3.8 diff --git a/cloud-sql/sql-server/sqlalchemy/Dockerfile b/cloud-sql/sql-server/sqlalchemy/Dockerfile index 75f4e22a969..1464006d45b 100644 --- a/cloud-sql/sql-server/sqlalchemy/Dockerfile +++ b/cloud-sql/sql-server/sqlalchemy/Dockerfile @@ -14,7 +14,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.13 +FROM python:3.14 RUN apt-get update diff --git a/cloud-sql/sql-server/sqlalchemy/requirements.txt b/cloud-sql/sql-server/sqlalchemy/requirements.txt index a2aae8784d1..32660dce882 100644 --- a/cloud-sql/sql-server/sqlalchemy/requirements.txt +++ b/cloud-sql/sql-server/sqlalchemy/requirements.txt @@ -1,9 +1,9 @@ Flask==2.2.2 gunicorn==23.0.0 python-tds==1.16.0 -pyopenssl==25.0.0 +pyopenssl==26.0.0 SQLAlchemy==2.0.40 -cloud-sql-python-connector==1.20.0 +cloud-sql-python-connector==1.20.1 sqlalchemy-pytds==1.0.2 functions-framework==3.9.2 Werkzeug==2.3.8 diff --git a/composer/airflow_1_samples/gke_operator.py b/composer/airflow_1_samples/gke_operator.py index 082d3333f9a..410a5a2aa7d 100644 --- a/composer/airflow_1_samples/gke_operator.py +++ b/composer/airflow_1_samples/gke_operator.py @@ -97,7 +97,7 @@ # [END composer_gkeoperator_minconfig_airflow_1] # [START composer_gkeoperator_templateconfig_airflow_1] - kubenetes_template_ex = GKEStartPodOperator( + kubernetes_template_ex = GKEStartPodOperator( task_id="ex-kube-templates", name="ex-kube-templates", project_id=PROJECT_ID, @@ -243,6 +243,6 @@ create_cluster >> create_node_pools >> kubernetes_min_pod >> delete_cluster create_cluster >> create_node_pools >> kubernetes_full_pod >> delete_cluster create_cluster >> create_node_pools >> kubernetes_affinity_ex >> delete_cluster - create_cluster >> create_node_pools >> kubenetes_template_ex >> delete_cluster + create_cluster >> create_node_pools >> kubernetes_template_ex >> delete_cluster # [END composer_gkeoperator_airflow_1] diff --git a/composer/airflow_1_samples/kubernetes_pod_operator.py b/composer/airflow_1_samples/kubernetes_pod_operator.py index 2799f467ec9..2062d2cc636 100644 --- a/composer/airflow_1_samples/kubernetes_pod_operator.py +++ b/composer/airflow_1_samples/kubernetes_pod_operator.py @@ -97,7 +97,7 @@ ) # [END composer_kubernetespodoperator_minconfig_airflow_1] # [START composer_kubernetespodoperator_templateconfig_airflow_1] - kubenetes_template_ex = kubernetes_pod_operator.KubernetesPodOperator( + kubernetes_template_ex = kubernetes_pod_operator.KubernetesPodOperator( task_id="ex-kube-templates", name="ex-kube-templates", namespace="default", diff --git a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/Dockerfile b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/Dockerfile index e78e11b3462..a18789f2f09 100644 --- a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/Dockerfile +++ b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.11 +FROM python:3.14 # Allow statements and log messages to immediately appear in the Cloud Run logs ENV PYTHONUNBUFFERED True diff --git a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py index 5751b6d12eb..9577c4f3b42 100644 --- a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py +++ b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py @@ -32,6 +32,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to numpy compilation failure. + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) "ignored_versions": ["2.7", "3.9", "3.10", "3.11"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them diff --git a/composer/cicd_sample/utils/add_dags_to_composer.py b/composer/cicd_sample/utils/add_dags_to_composer.py index 8e5698f0ba8..7df54ff52fb 100644 --- a/composer/cicd_sample/utils/add_dags_to_composer.py +++ b/composer/cicd_sample/utils/add_dags_to_composer.py @@ -57,7 +57,7 @@ def upload_dags_to_composer( if len(dags) > 0: # Note - the GCS client library does not currently support batch requests on uploads # if you have a large number of files, consider using - # the Python subprocess module to run gsutil -m cp -r on your dags + # the Python subprocess module to run gcloud storage cp --recursive on your dags # See https://cloud.google.com/storage/docs/gsutil/commands/cp for more info storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) diff --git a/composer/workflows/gke_operator.py b/composer/workflows/gke_operator.py index 31536ba55e7..acf60c05e5a 100644 --- a/composer/workflows/gke_operator.py +++ b/composer/workflows/gke_operator.py @@ -91,7 +91,7 @@ # [END composer_gkeoperator_minconfig] # [START composer_gkeoperator_templateconfig] - kubenetes_template_ex = GKEStartPodOperator( + kubernetes_template_ex = GKEStartPodOperator( task_id="ex-kube-templates", name="ex-kube-templates", project_id=PROJECT_ID, @@ -238,6 +238,6 @@ create_cluster >> kubernetes_min_pod >> delete_cluster create_cluster >> kubernetes_full_pod >> delete_cluster create_cluster >> kubernetes_affinity_ex >> delete_cluster - create_cluster >> kubenetes_template_ex >> delete_cluster + create_cluster >> kubernetes_template_ex >> delete_cluster # [END composer_gkeoperator] diff --git a/composer/workflows/kubernetes_pod_operator.py b/composer/workflows/kubernetes_pod_operator.py index 26dcb9d5173..835bd108fd4 100644 --- a/composer/workflows/kubernetes_pod_operator.py +++ b/composer/workflows/kubernetes_pod_operator.py @@ -100,7 +100,7 @@ ) # [END composer_kubernetespodoperator_minconfig] # [START composer_kubernetespodoperator_templateconfig] - kubenetes_template_ex = KubernetesPodOperator( + kubernetes_template_ex = KubernetesPodOperator( task_id="ex-kube-templates", name="ex-kube-templates", namespace="default", diff --git a/compute/client_library/requirements.txt b/compute/client_library/requirements.txt index f9faea10a9d..656732bef9a 100644 --- a/compute/client_library/requirements.txt +++ b/compute/client_library/requirements.txt @@ -1,5 +1,4 @@ isort==6.0.0; python_version > "3.9" isort==5.13.2; python_version <= "3.8" -black==24.8.0; python_version < "3.9" -black==24.10.0; python_version >= "3.9" -google-cloud-compute==1.19.1 \ No newline at end of file +black==26.3.1 +google-cloud-compute==1.19.1 diff --git a/dataflow/conftest.py b/dataflow/conftest.py index a1f81eac6f6..2bb6c6b3ecf 100644 --- a/dataflow/conftest.py +++ b/dataflow/conftest.py @@ -85,7 +85,7 @@ def bucket_name(test_name: str, location: str, unique_id: str) -> Iterator[str]: # Try to remove all files before deleting the bucket. # Deleting a bucket with too many files results in an error. try: - run_cmd("gsutil", "-m", "rm", "-rf", f"gs://{bucket_name}/*") + run_cmd("gcloud", "storage", "rm", "--recursive", "--continue-on-error", f"gs://{bucket_name}/*") except RuntimeError: # If no files were found and it fails, ignore the error. pass diff --git a/dataflow/custom-containers/miniconda/Dockerfile b/dataflow/custom-containers/miniconda/Dockerfile index bcc1eadc7f5..31e8c3b03f8 100644 --- a/dataflow/custom-containers/miniconda/Dockerfile +++ b/dataflow/custom-containers/miniconda/Dockerfile @@ -18,7 +18,7 @@ FROM continuumio/miniconda3:22.11.1-alpine AS builder # Create a virtual environment and make it standalone with conda-pack. # https://conda.github.io/conda-pack -RUN conda create -y -n env python=3.9 \ +RUN conda create -y -n env python=3.14 \ && conda install -y conda-pack \ && conda-pack -n env -o /tmp/env.tar \ && mkdir /opt/python \ @@ -31,7 +31,7 @@ FROM ubuntu:latest WORKDIR /pipeline # Set the entrypoint to Apache Beam SDK worker launcher. -COPY --from=apache/beam_python3.9_sdk:2.55.1 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] # Copy the python installation from the builder stage. diff --git a/dataflow/custom-containers/miniconda/noxfile_config.py b/dataflow/custom-containers/miniconda/noxfile_config.py index fb2bcbdea22..60b814e62b1 100644 --- a/dataflow/custom-containers/miniconda/noxfile_config.py +++ b/dataflow/custom-containers/miniconda/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ We're opting out of all Python versions except 3.9. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.10", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/custom-containers/minimal/Dockerfile b/dataflow/custom-containers/minimal/Dockerfile index 2176aa76a81..cb113d33854 100644 --- a/dataflow/custom-containers/minimal/Dockerfile +++ b/dataflow/custom-containers/minimal/Dockerfile @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.9-slim +FROM python:3.14-slim WORKDIR /pipeline # Set the entrypoint to Apache Beam SDK worker launcher. -COPY --from=apache/beam_python3.9_sdk:2.55.1 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] # Install the requirements. diff --git a/dataflow/custom-containers/minimal/noxfile_config.py b/dataflow/custom-containers/minimal/noxfile_config.py index fb2bcbdea22..60b814e62b1 100644 --- a/dataflow/custom-containers/minimal/noxfile_config.py +++ b/dataflow/custom-containers/minimal/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ We're opting out of all Python versions except 3.9. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.10", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/custom-containers/ubuntu/Dockerfile b/dataflow/custom-containers/ubuntu/Dockerfile index c35d23d9957..67e5d80e998 100644 --- a/dataflow/custom-containers/ubuntu/Dockerfile +++ b/dataflow/custom-containers/ubuntu/Dockerfile @@ -17,7 +17,7 @@ FROM ubuntu:focal WORKDIR /pipeline # Set the entrypoint to Apache Beam SDK worker launcher. -COPY --from=apache/beam_python3.8_sdk:2.40.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] # Install Python with pip, dev tools, distutils, and a C++ compiler. diff --git a/dataflow/custom-containers/ubuntu/noxfile_config.py b/dataflow/custom-containers/ubuntu/noxfile_config.py index fb2bcbdea22..60b814e62b1 100644 --- a/dataflow/custom-containers/ubuntu/noxfile_config.py +++ b/dataflow/custom-containers/ubuntu/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ We're opting out of all Python versions except 3.9. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.10", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/encryption-keys/README.md b/dataflow/encryption-keys/README.md index 0545d63d321..036540108d8 100644 --- a/dataflow/encryption-keys/README.md +++ b/dataflow/encryption-keys/README.md @@ -25,7 +25,7 @@ Additionally, for this sample you need the following: ```sh export BUCKET=your-gcs-bucket - gsutil mb gs://$BUCKET + gcloud storage buckets create gs://$BUCKET ``` 1. [Create a symmetric key ring](https://cloud.google.com/kms/docs/creating-keys). @@ -174,10 +174,10 @@ To avoid incurring charges to your GCP account for the resources used: ```sh # Remove only the files created by this sample. -gsutil -m rm -rf "gs://$BUCKET/samples/dataflow/kms" +gcloud storage rm --recursive --continue-on-error "gs://$BUCKET/samples/dataflow/kms" # [optional] Remove the Cloud Storage bucket. -gsutil rb gs://$BUCKET +gcloud storage buckets delete gs://$BUCKET # Remove the BigQuery table. bq rm -f -t $PROJECT:$DATASET.$TABLE diff --git a/dataflow/flex-templates/getting_started/README.md b/dataflow/flex-templates/getting_started/README.md index d7ee5d38bd3..383fdca052e 100644 --- a/dataflow/flex-templates/getting_started/README.md +++ b/dataflow/flex-templates/getting_started/README.md @@ -9,7 +9,7 @@ Make sure you have followed the ```sh export BUCKET="your--bucket" -gsutil mb gs://$BUCKET +gcloud storage buckets create gs://$BUCKET ``` ## create an Artifact Registry repository @@ -51,4 +51,3 @@ gcloud dataflow flex-template run "flex-`date +%Y%m%d-%H%M%S`" \ For more information about building and running flex templates, see 📝 [Use Flex Templates](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates). - diff --git a/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile b/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile index e85016b1411..0939e9e0728 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile +++ b/dataflow/flex-templates/pipeline_with_dependencies/Dockerfile @@ -24,14 +24,14 @@ # This Dockerfile illustrates how to use a custom base image when building # a custom contaier images for Dataflow. A 'slim' base image is smaller in size, # but does not include some preinstalled libraries, like google-cloud-debugger. -# To use a standard image, use apache/beam_python3.11_sdk:2.54.0 instead. +# To use a standard image, use apache/beam_python3.14_sdk:2.73.0 instead. # Use consistent versions of Python interpreter in the project. -FROM python:3.11-slim +FROM python:3.14-slim # Copy SDK entrypoint binary from Apache Beam image, which makes it possible to # use the image as SDK container image. If you explicitly depend on # apache-beam in setup.py, use the same version of Beam in both files. -COPY --from=apache/beam_python3.11_sdk:2.54.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam # Copy Flex Template launcher binary from the launcher image, which makes it # possible to use the image as a Flex Template base image. diff --git a/dataflow/flex-templates/pipeline_with_dependencies/README.md b/dataflow/flex-templates/pipeline_with_dependencies/README.md index 99385639297..79bbabf9761 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/README.md +++ b/dataflow/flex-templates/pipeline_with_dependencies/README.md @@ -73,7 +73,7 @@ rules. It is optional. export PROJECT="project-id" export BUCKET="your-bucket" export REGION="us-central1" -gsutil mb -p $PROJECT gs://$BUCKET +gcloud storage buckets create gs://$BUCKET --project=$PROJECT ``` ## Create an Artifact Registry repository @@ -165,7 +165,7 @@ gcloud dataflow flex-template run "flex-`date +%Y%m%d-%H%M%S`" \ After the pipeline finishes, use the following command to inspect the output: ```bash -gsutil cat gs://$BUCKET/output* +gcloud storage cat gs://$BUCKET/output* ``` ## Optional: Update the dependencies in the requirements file and rebuild the Docker images diff --git a/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py b/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py index 8df70c1108b..c3badedaaba 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py +++ b/dataflow/flex-templates/pipeline_with_dependencies/noxfile_config.py @@ -16,8 +16,8 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ We're opting out of all Python versions except 3.11. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], } diff --git a/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt b/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt index bef166bb943..cc9fda76736 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt +++ b/dataflow/flex-templates/pipeline_with_dependencies/requirements.txt @@ -305,7 +305,7 @@ typing-extensions==4.10.0 # via apache-beam tzlocal==5.2 # via js2py -urllib3==2.6.0 +urllib3==2.6.3 # via requests wrapt==1.16.0 # via deprecated diff --git a/dataflow/flex-templates/streaming_beam/README.md b/dataflow/flex-templates/streaming_beam/README.md index 66d891ce526..2a21e44220b 100644 --- a/dataflow/flex-templates/streaming_beam/README.md +++ b/dataflow/flex-templates/streaming_beam/README.md @@ -25,7 +25,7 @@ Additionally, for this sample you need the following: ```sh export BUCKET="your-gcs-bucket" - gsutil mb gs://$BUCKET + gcloud storage buckets create gs://$BUCKET ``` 1. Create a @@ -231,7 +231,7 @@ The following sections describe how to delete or turn off these resources. 1. Delete the template spec file from Cloud Storage. ```sh - gsutil rm $TEMPLATE_PATH + gcloud storage rm $TEMPLATE_PATH ``` 1. Delete the Flex Template container image from Container Registry. @@ -277,7 +277,7 @@ The following sections describe how to delete or turn off these resources. > These objects cannot be recovered. > > ```sh - > gsutil rm -r gs://$BUCKET + > gcloud storage rm --recursive gs://$BUCKET > ``` ## Limitations diff --git a/dataflow/gemma-flex-template/Dockerfile b/dataflow/gemma-flex-template/Dockerfile index 284474e9759..fb2c52d27d6 100644 --- a/dataflow/gemma-flex-template/Dockerfile +++ b/dataflow/gemma-flex-template/Dockerfile @@ -30,11 +30,11 @@ RUN pip install --no-cache-dir --upgrade pip \ # Copy SDK entrypoint binary from Apache Beam image, which makes it possible to # use the image as SDK container image. # The Beam version should match the version specified in requirements.txt -COPY --from=apache/beam_python3.10_sdk:2.62.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam # Copy Flex Template launcher binary from the launcher image, which makes it # possible to use the image as a Flex Template base image. -COPY --from=gcr.io/dataflow-templates-base/python310-template-launcher-base:latest /opt/google/dataflow/python_template_launcher /opt/google/dataflow/python_template_launcher +COPY --from=gcr.io/dataflow-templates-base/python314-template-launcher-base:latest /opt/google/dataflow/python_template_launcher /opt/google/dataflow/python_template_launcher # Copy the model directory downloaded from Kaggle and the pipeline code. COPY pytorch_model pytorch_model diff --git a/dataflow/gemma-flex-template/README.md b/dataflow/gemma-flex-template/README.md index 8ade42a8c46..0e082cc2bb7 100644 --- a/dataflow/gemma-flex-template/README.md +++ b/dataflow/gemma-flex-template/README.md @@ -43,7 +43,7 @@ Click [here to create a GCS bucket](https://console.cloud.google.com/storage/cre ```sh export GCS_BUCKET="your--bucket" -gsutil mb gs://$GCS_BUCKET +gcloud storage buckets create gs://$GCS_BUCKET ``` Make sure your GCS bucket name does __not__ include the `gs://` prefix diff --git a/dataflow/gemma-flex-template/e2e_test.py b/dataflow/gemma-flex-template/e2e_test.py index f95f78ec089..be281a9c984 100644 --- a/dataflow/gemma-flex-template/e2e_test.py +++ b/dataflow/gemma-flex-template/e2e_test.py @@ -92,7 +92,7 @@ def responses_subscription( @pytest.fixture(scope="session") def flex_template_image(utils: Utils) -> str: - conftest.run_cmd("gsutil", "cp", "-r", GEMMA_GCS, ".") + conftest.run_cmd("gcloud", "storage", "cp", "--recursive", GEMMA_GCS, ".") yield from utils.cloud_build_submit(NAME) diff --git a/dataflow/gemma-flex-template/noxfile_config.py b/dataflow/gemma-flex-template/noxfile_config.py index 7e6ba7ba31b..35321dbbdea 100644 --- a/dataflow/gemma-flex-template/noxfile_config.py +++ b/dataflow/gemma-flex-template/noxfile_config.py @@ -16,10 +16,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # Opting out of all Python versions except 3.10. # The Python version used is defined by the Dockerfile and the job # submission enviornment must match. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], "envs": { "PYTHONPATH": ".." }, diff --git a/dataflow/gemma/Dockerfile b/dataflow/gemma/Dockerfile index d66c298e6eb..b3472a56955 100644 --- a/dataflow/gemma/Dockerfile +++ b/dataflow/gemma/Dockerfile @@ -29,7 +29,7 @@ RUN pip install --upgrade --no-cache-dir pip \ && pip install --no-cache-dir -r requirements.txt # Copy files from official SDK image, including script/dependencies. -COPY --from=apache/beam_python3.11_sdk:2.54.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam # Copy the model directory downloaded from Kaggle and the pipeline code. COPY gemma_2b gemma_2B diff --git a/dataflow/gemma/e2e_test.py b/dataflow/gemma/e2e_test.py index e2510716f4b..6f65fb15959 100644 --- a/dataflow/gemma/e2e_test.py +++ b/dataflow/gemma/e2e_test.py @@ -60,7 +60,7 @@ def test_name() -> str: @pytest.fixture(scope="session") def container_image(utils: Utils) -> str: # Copy Gemma onto the local environment - conftest.run_cmd("gsutil", "cp", "-r", GEMMA_GCS, ".") + conftest.run_cmd("gcloud", "storage", "cp", "--recursive", GEMMA_GCS, ".") yield from utils.cloud_build_submit(NAME) diff --git a/dataflow/gemma/noxfile_config.py b/dataflow/gemma/noxfile_config.py index 7b3b1b9ebf6..35321dbbdea 100644 --- a/dataflow/gemma/noxfile_config.py +++ b/dataflow/gemma/noxfile_config.py @@ -16,10 +16,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # Opting out of all Python versions except 3.11. # The Python version used is defined by the Dockerfile and the job # submission enviornment must match. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], "envs": { "PYTHONPATH": ".." }, diff --git a/dataflow/gpu-examples/pytorch-minimal/Dockerfile b/dataflow/gpu-examples/pytorch-minimal/Dockerfile index f86d8bb388f..df3903a36ed 100644 --- a/dataflow/gpu-examples/pytorch-minimal/Dockerfile +++ b/dataflow/gpu-examples/pytorch-minimal/Dockerfile @@ -27,5 +27,5 @@ RUN pip install --no-cache-dir --upgrade pip \ && pip check # Set the entrypoint to Apache Beam SDK worker launcher. -COPY --from=apache/beam_python3.10_sdk:2.62.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] diff --git a/dataflow/gpu-examples/pytorch-minimal/noxfile_config.py b/dataflow/gpu-examples/pytorch-minimal/noxfile_config.py index 99b1fb47b8e..ac06c5f8740 100644 --- a/dataflow/gpu-examples/pytorch-minimal/noxfile_config.py +++ b/dataflow/gpu-examples/pytorch-minimal/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ We're opting out of all Python versions except 3.10. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/gpu-examples/tensorflow-landsat-prime/Dockerfile b/dataflow/gpu-examples/tensorflow-landsat-prime/Dockerfile index a506a8727a7..9b2fba3c978 100644 --- a/dataflow/gpu-examples/tensorflow-landsat-prime/Dockerfile +++ b/dataflow/gpu-examples/tensorflow-landsat-prime/Dockerfile @@ -25,9 +25,9 @@ COPY *.py ./ RUN apt-get update \ # Install Python and other system dependencies. && apt-get install -y --no-install-recommends \ - curl g++ python3.10-dev python3.10-venv python3-distutils \ + curl g++ python3.14-dev python3.14-venv python3-distutils \ && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.10 10 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.14 10 \ && curl https://bootstrap.pypa.io/get-pip.py | python \ # Install the pipeline requirements. && pip install --no-cache-dir -r requirements.txt \ @@ -35,5 +35,5 @@ RUN apt-get update \ # Set the entrypoint to Apache Beam SDK worker launcher. # Check this matches the apache-beam version in the requirements.txt -COPY --from=apache/beam_python3.10_sdk:2.62.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] diff --git a/dataflow/gpu-examples/tensorflow-landsat-prime/noxfile_config.py b/dataflow/gpu-examples/tensorflow-landsat-prime/noxfile_config.py index 376ea30e3b6..ca12a452f5b 100644 --- a/dataflow/gpu-examples/tensorflow-landsat-prime/noxfile_config.py +++ b/dataflow/gpu-examples/tensorflow-landsat-prime/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ Test only on Python 3.10. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/gpu-examples/tensorflow-landsat/Dockerfile b/dataflow/gpu-examples/tensorflow-landsat/Dockerfile index 39a836fdb0b..f547b13de44 100644 --- a/dataflow/gpu-examples/tensorflow-landsat/Dockerfile +++ b/dataflow/gpu-examples/tensorflow-landsat/Dockerfile @@ -25,9 +25,9 @@ COPY *.py ./ RUN apt-get update \ # Install Python and other system dependencies. && apt-get install -y --no-install-recommends \ - curl g++ python3.10-dev python3.10-venv python3-distutils \ + curl g++ python3.14-dev python3.14-venv python3-distutils \ && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.10 10 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.14 10 \ && curl https://bootstrap.pypa.io/get-pip.py | python \ # Install the pipeline requirements. && pip install --no-cache-dir -r requirements.txt \ @@ -35,5 +35,5 @@ RUN apt-get update \ # Set the entrypoint to Apache Beam SDK worker launcher. # Check this matches the apache-beam version in the requirements.txt -COPY --from=apache/beam_python3.10_sdk:2.62.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] diff --git a/dataflow/gpu-examples/tensorflow-landsat/noxfile_config.py b/dataflow/gpu-examples/tensorflow-landsat/noxfile_config.py index baf97789883..33c259f5bef 100644 --- a/dataflow/gpu-examples/tensorflow-landsat/noxfile_config.py +++ b/dataflow/gpu-examples/tensorflow-landsat/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ Test only on Python 3.10. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/gpu-examples/tensorflow-minimal/Dockerfile b/dataflow/gpu-examples/tensorflow-minimal/Dockerfile index e5f79f6e4ad..16c74bc0d75 100644 --- a/dataflow/gpu-examples/tensorflow-minimal/Dockerfile +++ b/dataflow/gpu-examples/tensorflow-minimal/Dockerfile @@ -25,9 +25,9 @@ COPY *.py ./ RUN apt-get update \ # Install Python and other system dependencies. && apt-get install -y --no-install-recommends \ - curl g++ python3.10-dev python3.10-venv python3-distutils \ + curl g++ python3.14-dev python3.14-venv python3-distutils \ && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.10 10 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.14 10 \ && curl https://bootstrap.pypa.io/get-pip.py | python \ # Install the pipeline requirements. && pip install --no-cache-dir -r requirements.txt \ @@ -35,5 +35,5 @@ RUN apt-get update \ # Set the entrypoint to Apache Beam SDK worker launcher. # Check this matches the apache-beam version in the requirements.txt -COPY --from=apache/beam_python3.10_sdk:2.62.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam ENTRYPOINT [ "/opt/apache/beam/boot" ] diff --git a/dataflow/gpu-examples/tensorflow-minimal/noxfile_config.py b/dataflow/gpu-examples/tensorflow-minimal/noxfile_config.py index baf97789883..33c259f5bef 100644 --- a/dataflow/gpu-examples/tensorflow-minimal/noxfile_config.py +++ b/dataflow/gpu-examples/tensorflow-minimal/noxfile_config.py @@ -22,10 +22,10 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # > ℹ️ Test only on Python 3.10. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.11", "3.12", "3.13"], + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) + "ignored_versions": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/dataflow/run-inference/requirements-test.txt b/dataflow/run-inference/requirements-test.txt index c9095c832fd..1802752120d 100644 --- a/dataflow/run-inference/requirements-test.txt +++ b/dataflow/run-inference/requirements-test.txt @@ -1,4 +1,4 @@ google-cloud-aiplatform==1.57.0 google-cloud-dataflow-client==0.8.14 google-cloud-storage==2.10.0 -pytest==8.2.0 +pytest==9.0.3 diff --git a/dataflow/run-inference/requirements.txt b/dataflow/run-inference/requirements.txt index 585334e1a9b..d0376ac202e 100644 --- a/dataflow/run-inference/requirements.txt +++ b/dataflow/run-inference/requirements.txt @@ -1,3 +1,3 @@ apache-beam[gcp]==2.49.0 torch==2.2.2 -transformers==4.38.0 +transformers==5.0.0rc3 diff --git a/dataflow/run-inference/tests/e2e_test.py b/dataflow/run-inference/tests/e2e_test.py index 0428af3dd28..70be7d6878d 100644 --- a/dataflow/run-inference/tests/e2e_test.py +++ b/dataflow/run-inference/tests/e2e_test.py @@ -95,7 +95,7 @@ def dataflow_job( ) -> Iterator[str]: # Upload the state dict to Cloud Storage. state_dict_gcs = f"gs://{bucket_name}/temp/state_dict.pt" - conftest.run_cmd("gsutil", "cp", "-n", state_dict_path, state_dict_gcs) + conftest.run_cmd("gcloud", "storage", "cp", "--no-clobber", state_dict_path, state_dict_gcs) # Launch the streaming Dataflow pipeline. conftest.run_cmd( diff --git a/dataflow/run_template/README.md b/dataflow/run_template/README.md index c73fb76c504..dda9f4570aa 100644 --- a/dataflow/run_template/README.md +++ b/dataflow/run_template/README.md @@ -29,7 +29,7 @@ Additionally, for this sample you need the following: ```sh export BUCKET=your-gcs-bucket - gsutil mb gs://$BUCKET + gcloud storage buckets create gs://$BUCKET ``` 1. Clone the `python-docs-samples` repository. diff --git a/dataflow/snippets/Dockerfile b/dataflow/snippets/Dockerfile index bb230e64e4d..4234ca52651 100644 --- a/dataflow/snippets/Dockerfile +++ b/dataflow/snippets/Dockerfile @@ -18,7 +18,7 @@ # on the host machine. This Dockerfile is derived from the # dataflow/custom-containers/ubuntu sample. -FROM python:3.12-slim +FROM python:3.14-slim # Install JRE COPY --from=openjdk:8-jre-slim /usr/local/openjdk-8 /usr/local/openjdk-8 @@ -28,7 +28,7 @@ RUN update-alternatives --install /usr/bin/java java /usr/local/openjdk-8/bin/ja WORKDIR /pipeline # Copy files from official SDK image. -COPY --from=apache/beam_python3.11_sdk:2.63.0 /opt/apache/beam /opt/apache/beam +COPY --from=apache/beam_python3.14_sdk:2.73.0 /opt/apache/beam /opt/apache/beam # Set the entrypoint to Apache Beam SDK launcher. ENTRYPOINT [ "/opt/apache/beam/boot" ] @@ -37,7 +37,7 @@ RUN apt-get update RUN apt-get install -y --no-install-recommends docker.io # Install dependencies. -RUN pip3 install --no-cache-dir apache-beam[gcp]==2.63.0 +RUN pip3 install --no-cache-dir apache-beam[gcp]==2.73.0 RUN pip install --no-cache-dir kafka-python==2.0.6 # Verify that the image does not have conflicting dependencies. diff --git a/dataflow/snippets/noxfile_config.py b/dataflow/snippets/noxfile_config.py index 900f58e0ddf..7760eb40877 100644 --- a/dataflow/snippets/noxfile_config.py +++ b/dataflow/snippets/noxfile_config.py @@ -22,6 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. + # Note: Docker-based sample, testing only against version specified in Dockerfile (3.14) "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them diff --git a/dataproc/snippets/README.md b/dataproc/snippets/README.md index 98622be7dc1..442b8fa55bb 100644 --- a/dataproc/snippets/README.md +++ b/dataproc/snippets/README.md @@ -64,7 +64,7 @@ To run list_clusters.py: To run submit_job_to_cluster.py, first create a GCS bucket (used by Cloud Dataproc to stage files) from the Cloud Console or with gsutil: - gsutil mb gs:// + gcloud storage buckets create gs:// Next, set the following environment variables: diff --git a/dataproc/snippets/python-api-walkthrough.md b/dataproc/snippets/python-api-walkthrough.md index c5eb884a8f0..2ca94cca3ca 100644 --- a/dataproc/snippets/python-api-walkthrough.md +++ b/dataproc/snippets/python-api-walkthrough.md @@ -65,7 +65,7 @@ an explanation of how the code works. * To create a new bucket, run the following command. Your bucket name must be unique. - gsutil mb -p {{project-id}} gs://your-bucket-name + gcloud storage buckets create --project={{project-id}} gs://your-bucket-name 2. Set environment variables. @@ -145,12 +145,12 @@ Cluster cluster-name successfully deleted. If you created a Cloud Storage bucket to use for this walkthrough, you can run the following command to delete the bucket (the bucket must be empty). - gsutil rb gs://$BUCKET + gcloud storage buckets delete gs://$BUCKET * You can run the following command to **delete the bucket and all objects within it. Note: the deleted objects cannot be recovered.** - gsutil rm -r gs://$BUCKET + gcloud storage rm --recursive gs://$BUCKET * **For more information.** See the [Dataproc documentation](https://cloud.google.com/dataproc/docs/) diff --git a/dialogflow/detect_intent_texts_with_location.py b/dialogflow/detect_intent_texts_with_location.py index d52ac178dd7..99e2eabb2f3 100644 --- a/dialogflow/detect_intent_texts_with_location.py +++ b/dialogflow/detect_intent_texts_with_location.py @@ -58,7 +58,7 @@ def detect_intent_texts_with_location( print("=" * 20) print(f"Query text: {response.query_result.query_text}") print( - f"Detected intent: {response.query_result.intent.display_name} (confidence: {response.query_result.intent_detection_confidence,})\n" + f"Detected intent: {response.query_result.intent.display_name} (confidence: {response.query_result.intent_detection_confidence})\n" ) print(f"Fulfillment text: {response.query_result.fulfillment_text}\n") diff --git a/dialogflow/participant_management.py b/dialogflow/participant_management.py index e2f9a486c1a..d0bfa9decf3 100644 --- a/dialogflow/participant_management.py +++ b/dialogflow/participant_management.py @@ -196,6 +196,7 @@ def analyze_content_audio_stream( timeout: int, language_code: str, single_utterance=False, + output_multiple_utterances=False, ): import google.auth from google.cloud import dialogflow_v2beta1 as dialogflow @@ -231,7 +232,9 @@ def gen_requests(participant_name, audio_config, stream): """Generates requests for streaming.""" audio_generator = stream.generator() yield dialogflow.types.participant.StreamingAnalyzeContentRequest( - participant=participant_name, audio_config=audio_config + participant=participant_name, + audio_config=audio_config, + output_multiple_utterances=output_multiple_utterances ) for content in audio_generator: yield dialogflow.types.participant.StreamingAnalyzeContentRequest( diff --git a/dialogflow/requirements.txt b/dialogflow/requirements.txt index 4c7d355eb45..ed176a19af0 100644 --- a/dialogflow/requirements.txt +++ b/dialogflow/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-dialogflow==2.36.0 +google-cloud-dialogflow==2.46.0 Flask==3.0.3 pyaudio==0.2.14 termcolor==3.0.0 diff --git a/dialogflow/streaming_transcription.py b/dialogflow/streaming_transcription.py index 6395a30b3a8..fe88afb581f 100644 --- a/dialogflow/streaming_transcription.py +++ b/dialogflow/streaming_transcription.py @@ -34,7 +34,7 @@ import re import sys -from google.api_core.exceptions import DeadlineExceeded +from google.api_core.exceptions import DeadlineExceeded, OutOfRange import pyaudio @@ -51,6 +51,7 @@ CHUNK_SIZE = int(SAMPLE_RATE / 10) # 100ms RESTART_TIMEOUT = 160 # seconds MAX_LOOKBACK = 3 # seconds +HALF_CLOSE_DURATION_MS = 90 * 1000 # milliseconds YELLOW = "\033[0;33m" @@ -198,6 +199,9 @@ def main(): timeout=RESTART_TIMEOUT, language_code="en-US", single_utterance=False, + # Uncomment to process multiple utterances detected in the audio stream + # individually instead of stitching together to form a single utterance. + # output_multiple_utterances=True, ) # Now, print the final transcription responses to user. @@ -213,8 +217,10 @@ def main(): offset.seconds * 1000 + offset.microseconds / 1000 ) transcript = response.recognition_result.transcript - # Half-close the stream with gRPC (in Python just stop yielding requests) - stream.is_final = True + # Half-close upon final results for better streaming experiences + # (in Python just stop yielding requests) + if stream.is_final_offset > HALF_CLOSE_DURATION_MS: + stream.is_final = True # Exit recognition if any of the transcribed phrase could be # one of our keywords. if re.search(r"\b(exit|quit)\b", transcript, re.I): @@ -223,6 +229,8 @@ def main(): terminate = True stream.closed = True break + except OutOfRange: + print("Maximum audio duration exceeded in the stream, restarting.") except DeadlineExceeded: print("Deadline Exceeded, restarting.") diff --git a/discoveryengine/answer_query_sample.py b/discoveryengine/answer_query_sample.py index fcb47bff6b8..92d51c43a71 100644 --- a/discoveryengine/answer_query_sample.py +++ b/discoveryengine/answer_query_sample.py @@ -69,7 +69,8 @@ def answer_query_sample( ignore_non_answer_seeking_query=False, # Optional: Ignore non-answer seeking query ignore_low_relevant_content=False, # Optional: Return fallback answer when content is not relevant model_spec=discoveryengine.AnswerQueryRequest.AnswerGenerationSpec.ModelSpec( - model_version="gemini-2.5-flash/answer_gen/v1", # Optional: Model to use for answer generation + # Use the 2026 stable production model for answer generation + model_version="gemini-2.5-flash/answer_gen/stable", ), prompt_spec=discoveryengine.AnswerQueryRequest.AnswerGenerationSpec.PromptSpec( preamble="Give a detailed answer.", # Optional: Natural language instructions for customizing the answer. diff --git a/endpoints/bookstore-grpc-transcoding/requirements-test.txt b/endpoints/bookstore-grpc-transcoding/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/endpoints/bookstore-grpc-transcoding/requirements-test.txt +++ b/endpoints/bookstore-grpc-transcoding/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/endpoints/bookstore-grpc/requirements-test.txt b/endpoints/bookstore-grpc/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/endpoints/bookstore-grpc/requirements-test.txt +++ b/endpoints/bookstore-grpc/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/endpoints/getting-started-grpc/requirements-test.txt b/endpoints/getting-started-grpc/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/endpoints/getting-started-grpc/requirements-test.txt +++ b/endpoints/getting-started-grpc/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/endpoints/getting-started/clients/service_to_service_non_default/requirements-test.txt b/endpoints/getting-started/clients/service_to_service_non_default/requirements-test.txt index 15d066af319..216d5b8a24a 100644 --- a/endpoints/getting-started/clients/service_to_service_non_default/requirements-test.txt +++ b/endpoints/getting-started/clients/service_to_service_non_default/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +endpoints/getting-started/clients/service_to_service_non_default/requirements-test.txt diff --git a/enterpriseknowledgegraph/entity_reconciliation/requirements-test.txt b/enterpriseknowledgegraph/entity_reconciliation/requirements-test.txt index 55d9a1d34d9..2313637c624 100644 --- a/enterpriseknowledgegraph/entity_reconciliation/requirements-test.txt +++ b/enterpriseknowledgegraph/entity_reconciliation/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-api-core google-cloud-enterpriseknowledgegraph diff --git a/enterpriseknowledgegraph/search/requirements-test.txt b/enterpriseknowledgegraph/search/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/enterpriseknowledgegraph/search/requirements-test.txt +++ b/enterpriseknowledgegraph/search/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/error_reporting/fluent_on_compute/requirements-test.txt b/error_reporting/fluent_on_compute/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/error_reporting/fluent_on_compute/requirements-test.txt +++ b/error_reporting/fluent_on_compute/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/error_reporting/snippets/requirements-test.txt b/error_reporting/snippets/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/error_reporting/snippets/requirements-test.txt +++ b/error_reporting/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/eventarc/audit-storage/Dockerfile b/eventarc/audit-storage/Dockerfile index d5b1fad1c17..e65ba1afa4d 100644 --- a/eventarc/audit-storage/Dockerfile +++ b/eventarc/audit-storage/Dockerfile @@ -16,7 +16,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.11-slim +FROM python:3.14-slim # Allow statements and log messages to immediately appear in the Cloud Run logs ENV PYTHONUNBUFFERED True diff --git a/eventarc/audit-storage/requirements-test.txt b/eventarc/audit-storage/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/eventarc/audit-storage/requirements-test.txt +++ b/eventarc/audit-storage/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/eventarc/audit_iam/requirements-test.txt b/eventarc/audit_iam/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/eventarc/audit_iam/requirements-test.txt +++ b/eventarc/audit_iam/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/eventarc/generic/Dockerfile b/eventarc/generic/Dockerfile index a7a158bcb39..e4f1889dc5b 100644 --- a/eventarc/generic/Dockerfile +++ b/eventarc/generic/Dockerfile @@ -14,7 +14,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.11-slim +FROM python:3.14-slim # Allow statements and log messages to immediately appear in the Cloud Run logs ENV PYTHONUNBUFFERED True diff --git a/eventarc/generic/requirements-test.txt b/eventarc/generic/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/eventarc/generic/requirements-test.txt +++ b/eventarc/generic/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/eventarc/pubsub/Dockerfile b/eventarc/pubsub/Dockerfile index a7a158bcb39..e4f1889dc5b 100644 --- a/eventarc/pubsub/Dockerfile +++ b/eventarc/pubsub/Dockerfile @@ -14,7 +14,7 @@ # Use the official Python image. # https://hub.docker.com/_/python -FROM python:3.11-slim +FROM python:3.14-slim # Allow statements and log messages to immediately appear in the Cloud Run logs ENV PYTHONUNBUFFERED True diff --git a/eventarc/pubsub/requirements-test.txt b/eventarc/pubsub/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/eventarc/pubsub/requirements-test.txt +++ b/eventarc/pubsub/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/eventarc/storage_handler/requirements-test.txt b/eventarc/storage_handler/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/eventarc/storage_handler/requirements-test.txt +++ b/eventarc/storage_handler/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/functions/imagemagick/requirements-dev.txt b/functions/imagemagick/requirements-dev.txt index 158a3587da8..3bbf38755d9 100644 --- a/functions/imagemagick/requirements-dev.txt +++ b/functions/imagemagick/requirements-dev.txt @@ -1,4 +1,4 @@ uuid==1.30 -pytest==8.2.0; python_version > "3.0" +pytest==9.0.3; python_version > "3.0" # pin pytest to 4.6.11 or lower for Python2. -pytest==4.6.11; python_version < "3.0" +pytest==9.0.3; python_version < "3.0" diff --git a/functions/ocr/app/main.py b/functions/ocr/app/main.py index 186c9abfaaa..53bf3467d1d 100644 --- a/functions/ocr/app/main.py +++ b/functions/ocr/app/main.py @@ -69,7 +69,7 @@ def detect_text(bucket: str, filename: str) -> None: filename: name of the file to be read. Returns: - None; the output is written to stdout and Stackdriver Logging. + None; the output is written to stdout and Cloud Logging. """ print("Looking for text in image {}".format(filename)) @@ -123,7 +123,7 @@ def process_image(file_info: dict, context: dict) -> None: context: a dictionary containing metadata about the event. Returns: - None; the output is written to stdout and Stackdriver Logging. + None; the output is written to stdout and Cloud Logging. """ bucket = validate_message(file_info, "bucket") name = validate_message(file_info, "name") @@ -148,7 +148,7 @@ def translate_text(event: dict, context: dict) -> None: context: a dictionary containing metadata about the event. Returns: - None; the output is written to stdout and Stackdriver Logging. + None; the output is written to stdout and Cloud Logging. """ if event.get("data"): message_data = base64.b64decode(event["data"]).decode("utf-8") @@ -189,7 +189,7 @@ def save_result(event: dict, context: dict) -> None: context: a dictionary containing metadata about the event. Returns: - None; the output is written to stdout and Stackdriver Logging. + None; the output is written to stdout and Cloud Logging. """ if event.get("data"): message_data = base64.b64decode(event["data"]).decode("utf-8") diff --git a/functions/tips-retry/main.py b/functions/tips-retry/main.py index 847ae6394dc..dfe11942af2 100644 --- a/functions/tips-retry/main.py +++ b/functions/tips-retry/main.py @@ -27,7 +27,7 @@ def retry_or_not(data, context): data (dict): The event payload. context (google.cloud.functions.Context): The event metadata. Returns: - None; output is written to Stackdriver Logging + None; output is written to Cloud Logging """ # Retry based on a user-defined parameter diff --git a/functions/v2/tips-avoid-infinite-retries/main.py b/functions/v2/tips-avoid-infinite-retries/main.py index e3b735f2f74..9e751d5d0b5 100644 --- a/functions/v2/tips-avoid-infinite-retries/main.py +++ b/functions/v2/tips-avoid-infinite-retries/main.py @@ -29,7 +29,7 @@ def avoid_infinite_retries(cloud_event): Args: cloud_event: The cloud event associated with the current trigger Returns: - None; output is written to Stackdriver Logging + None; output is written to Cloud Logging """ timestamp = cloud_event["time"] diff --git a/functions/v2/tips-retry/main.py b/functions/v2/tips-retry/main.py index 7ff58a954ff..91ce20b8468 100644 --- a/functions/v2/tips-retry/main.py +++ b/functions/v2/tips-retry/main.py @@ -31,7 +31,7 @@ def retry_or_not(cloud_event): Args: cloud_event: The cloud event with a Pub/Sub data payload Returns: - None; output is written to Stackdriver Logging + None; output is written to Cloud Logging """ # The Pub/Sub event payload is passed as the CloudEvent's data payload. diff --git a/gemma2/noxfile_config.py b/gemma2/noxfile_config.py index 494cf15318d..fd69813938c 100644 --- a/gemma2/noxfile_config.py +++ b/gemma2/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/gemma2/requirements-test.txt b/gemma2/requirements-test.txt index 40543aababf..c9e154ba440 100644 --- a/gemma2/requirements-test.txt +++ b/gemma2/requirements-test.txt @@ -1 +1 @@ -pytest==8.3.3 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/batch_prediction/noxfile_config.py b/genai/batch_prediction/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/batch_prediction/noxfile_config.py +++ b/genai/batch_prediction/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/batch_prediction/requirements-test.txt b/genai/batch_prediction/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/batch_prediction/requirements-test.txt +++ b/genai/batch_prediction/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/bounding_box/noxfile_config.py b/genai/bounding_box/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/bounding_box/noxfile_config.py +++ b/genai/bounding_box/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/bounding_box/requirements-test.txt b/genai/bounding_box/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/bounding_box/requirements-test.txt +++ b/genai/bounding_box/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/bounding_box/test_bounding_box_examples.py b/genai/bounding_box/test_bounding_box_examples.py index bb6eca92008..39b0056e7a3 100644 --- a/genai/bounding_box/test_bounding_box_examples.py +++ b/genai/bounding_box/test_bounding_box_examples.py @@ -20,7 +20,7 @@ import boundingbox_with_txt_img -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/code_execution/noxfile_config.py b/genai/code_execution/noxfile_config.py index 29d9e7911eb..650b3c47840 100644 --- a/genai/code_execution/noxfile_config.py +++ b/genai/code_execution/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.13", "3.14"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/code_execution/requirements-test.txt b/genai/code_execution/requirements-test.txt index 8d10ef87035..7289efe2596 100644 --- a/genai/code_execution/requirements-test.txt +++ b/genai/code_execution/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.29.0 -pytest==9.0.2 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==1.3.0 diff --git a/genai/code_execution/test_codeexecution.py b/genai/code_execution/test_codeexecution.py index e3a8bfb7944..ea978643c7c 100644 --- a/genai/code_execution/test_codeexecution.py +++ b/genai/code_execution/test_codeexecution.py @@ -17,7 +17,7 @@ import codeexecution_barplot_with_txt_img import codeexecution_cropimage_with_txt_img -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/content_cache/noxfile_config.py b/genai/content_cache/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/content_cache/noxfile_config.py +++ b/genai/content_cache/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/content_cache/requirements-test.txt b/genai/content_cache/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/content_cache/requirements-test.txt +++ b/genai/content_cache/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/content_cache/test_content_cache_examples.py b/genai/content_cache/test_content_cache_examples.py index d7d9e5abda4..d17c43b1bea 100644 --- a/genai/content_cache/test_content_cache_examples.py +++ b/genai/content_cache/test_content_cache_examples.py @@ -21,7 +21,7 @@ import contentcache_use_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/controlled_generation/noxfile_config.py b/genai/controlled_generation/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/controlled_generation/noxfile_config.py +++ b/genai/controlled_generation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/controlled_generation/requirements-test.txt b/genai/controlled_generation/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/genai/controlled_generation/requirements-test.txt +++ b/genai/controlled_generation/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/genai/controlled_generation/test_controlled_generation_examples.py b/genai/controlled_generation/test_controlled_generation_examples.py index ab27d8e7a46..2bd518be054 100644 --- a/genai/controlled_generation/test_controlled_generation_examples.py +++ b/genai/controlled_generation/test_controlled_generation_examples.py @@ -25,7 +25,7 @@ import ctrlgen_with_nullable_schema import ctrlgen_with_resp_schema -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/count_tokens/noxfile_config.py b/genai/count_tokens/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/count_tokens/noxfile_config.py +++ b/genai/count_tokens/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/count_tokens/requirements-test.txt b/genai/count_tokens/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/genai/count_tokens/requirements-test.txt +++ b/genai/count_tokens/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/genai/count_tokens/test_count_tokens_examples.py b/genai/count_tokens/test_count_tokens_examples.py index e83f20cd14c..94f5b06c036 100644 --- a/genai/count_tokens/test_count_tokens_examples.py +++ b/genai/count_tokens/test_count_tokens_examples.py @@ -25,7 +25,7 @@ import counttoken_with_txt import counttoken_with_txt_vid -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/embeddings/noxfile_config.py b/genai/embeddings/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/embeddings/noxfile_config.py +++ b/genai/embeddings/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/embeddings/requirements-test.txt b/genai/embeddings/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/embeddings/requirements-test.txt +++ b/genai/embeddings/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/embeddings/test_embeddings_examples.py b/genai/embeddings/test_embeddings_examples.py index 5908ccddc6a..5b8d94f07c3 100644 --- a/genai/embeddings/test_embeddings_examples.py +++ b/genai/embeddings/test_embeddings_examples.py @@ -20,7 +20,7 @@ import embeddings_docretrieval_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/express_mode/noxfile_config.py b/genai/express_mode/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/express_mode/noxfile_config.py +++ b/genai/express_mode/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/express_mode/requirements-test.txt b/genai/express_mode/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/express_mode/requirements-test.txt +++ b/genai/express_mode/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/image_generation/imggen_mmflash_edit_img_with_txt_img.py b/genai/image_generation/imggen_mmflash_edit_img_with_txt_img.py index e2d9888a027..405702d4eff 100644 --- a/genai/image_generation/imggen_mmflash_edit_img_with_txt_img.py +++ b/genai/image_generation/imggen_mmflash_edit_img_with_txt_img.py @@ -26,7 +26,7 @@ def generate_content() -> str: image = Image.open("test_resources/example-image-eiffel-tower.png") response = client.models.generate_content( - model="gemini-3-pro-image-preview", + model="gemini-3.1-flash-image", contents=[image, "Edit this image to make it look like a cartoon."], config=GenerateContentConfig(response_modalities=[Modality.TEXT, Modality.IMAGE]), ) diff --git a/genai/image_generation/imggen_mmflash_img_with_vid.py b/genai/image_generation/imggen_mmflash_img_with_vid.py new file mode 100644 index 00000000000..98a0c51f996 --- /dev/null +++ b/genai/image_generation/imggen_mmflash_img_with_vid.py @@ -0,0 +1,51 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def generate_content() -> str: + # [START googlegenaisdk_imggen_mmflash_img_with_vid] + from google import genai + from google.genai.types import GenerateContentConfig, Modality, Part + from PIL import Image + from io import BytesIO + + client = genai.Client() + + # A video on 'The ABCs of agent building' + video = "https://www.youtube.com/watch?v=rjoMZyxncUI" + + response = client.models.generate_content( + model="gemini-3.1-flash-image", + contents=[ + Part.from_uri( + file_uri=video, + mime_type="video/mp4" + ), + "Generate an infographic of the topics covered in this video." + ], + config=GenerateContentConfig(response_modalities=[Modality.TEXT, Modality.IMAGE]), + ) + for part in response.candidates[0].content.parts: + if part.text: + print(part.text) + elif part.inline_data: + image = Image.open(BytesIO((part.inline_data.data))) + image.save("output_folder/video-image.png") + + # [END googlegenaisdk_imggen_mmflash_img_with_vid] + return "output_folder/video-image.png" + + +if __name__ == "__main__": + generate_content() diff --git a/genai/image_generation/imggen_mmflash_locale_aware_with_txt.py b/genai/image_generation/imggen_mmflash_locale_aware_with_txt.py index 305be883d22..2e80866a525 100644 --- a/genai/image_generation/imggen_mmflash_locale_aware_with_txt.py +++ b/genai/image_generation/imggen_mmflash_locale_aware_with_txt.py @@ -23,7 +23,7 @@ def generate_content() -> str: client = genai.Client() response = client.models.generate_content( - model="gemini-2.5-flash-image", + model="gemini-3.1-flash-image", contents=("Generate a photo of a breakfast meal."), config=GenerateContentConfig(response_modalities=[Modality.TEXT, Modality.IMAGE]), ) diff --git a/genai/image_generation/imggen_mmflash_multiple_imgs_with_txt.py b/genai/image_generation/imggen_mmflash_multiple_imgs_with_txt.py index 2b831ca97d9..884dba9d45f 100644 --- a/genai/image_generation/imggen_mmflash_multiple_imgs_with_txt.py +++ b/genai/image_generation/imggen_mmflash_multiple_imgs_with_txt.py @@ -23,7 +23,7 @@ def generate_content() -> str: client = genai.Client() response = client.models.generate_content( - model="gemini-2.5-flash-image", + model="gemini-3.1-flash-image", contents=("Generate 3 images a cat sitting on a chair."), config=GenerateContentConfig(response_modalities=[Modality.TEXT, Modality.IMAGE]), ) diff --git a/genai/image_generation/imggen_mmflash_txt_and_img_with_txt.py b/genai/image_generation/imggen_mmflash_txt_and_img_with_txt.py index 7a9d11103a7..ff2b9fda9f9 100644 --- a/genai/image_generation/imggen_mmflash_txt_and_img_with_txt.py +++ b/genai/image_generation/imggen_mmflash_txt_and_img_with_txt.py @@ -23,7 +23,7 @@ def generate_content() -> int: client = genai.Client() response = client.models.generate_content( - model="gemini-3-pro-image-preview", + model="gemini-3.1-flash-image", contents=( "Generate an illustrated recipe for a paella." "Create images to go alongside the text as you generate the recipe" diff --git a/genai/image_generation/imggen_mmflash_with_txt.py b/genai/image_generation/imggen_mmflash_with_txt.py index cd6c458a757..fd83f89eeda 100644 --- a/genai/image_generation/imggen_mmflash_with_txt.py +++ b/genai/image_generation/imggen_mmflash_with_txt.py @@ -25,7 +25,7 @@ def generate_content() -> str: client = genai.Client() response = client.models.generate_content( - model="gemini-3-pro-image-preview", + model="gemini-3.1-flash-image", contents=("Generate an image of the Eiffel tower with fireworks in the background."), config=GenerateContentConfig( response_modalities=[Modality.TEXT, Modality.IMAGE], diff --git a/genai/image_generation/noxfile_config.py b/genai/image_generation/noxfile_config.py index d63baa25bfa..0973c8621c7 100644 --- a/genai/image_generation/noxfile_config.py +++ b/genai/image_generation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/image_generation/requirements-test.txt b/genai/image_generation/requirements-test.txt index 4ccc4347cbe..14a8493498b 100644 --- a/genai/image_generation/requirements-test.txt +++ b/genai/image_generation/requirements-test.txt @@ -1,3 +1,3 @@ google-api-core==2.24.0 google-cloud-storage==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/image_generation/test_image_generation.py b/genai/image_generation/test_image_generation.py index f30b295f85e..f5cf14bb62c 100644 --- a/genai/image_generation/test_image_generation.py +++ b/genai/image_generation/test_image_generation.py @@ -41,7 +41,7 @@ import imggen_virtual_try_on_with_txt_img import imggen_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/image_generation/test_image_generation_mmflash.py b/genai/image_generation/test_image_generation_mmflash.py index 3ae60ec66ba..1b6b46bae70 100644 --- a/genai/image_generation/test_image_generation_mmflash.py +++ b/genai/image_generation/test_image_generation_mmflash.py @@ -19,13 +19,14 @@ import os import imggen_mmflash_edit_img_with_txt_img +import imggen_mmflash_img_with_vid import imggen_mmflash_locale_aware_with_txt import imggen_mmflash_multiple_imgs_with_txt import imggen_mmflash_txt_and_img_with_txt import imggen_mmflash_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" @@ -49,3 +50,6 @@ def test_imggen_mmflash_locale_aware_with_txt() -> None: def test_imggen_mmflash_multiple_imgs_with_txt() -> None: assert imggen_mmflash_multiple_imgs_with_txt.generate_content() + +def test_imggen_mmflash_img_with_vid() -> None: + assert imggen_mmflash_img_with_vid.generate_content() diff --git a/genai/live/noxfile_config.py b/genai/live/noxfile_config.py index d63baa25bfa..0973c8621c7 100644 --- a/genai/live/noxfile_config.py +++ b/genai/live/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/live/requirements-test.txt b/genai/live/requirements-test.txt index 7d5998c481d..654ef724fd6 100644 --- a/genai/live/requirements-test.txt +++ b/genai/live/requirements-test.txt @@ -1,5 +1,5 @@ backoff==2.2.1 google-api-core==2.25.1 -pytest==8.4.1 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==1.1.0 pytest-mock==3.14.0 \ No newline at end of file diff --git a/genai/live/test_live_examples.py b/genai/live/test_live_examples.py index ffb0f10c689..2d59ee87d10 100644 --- a/genai/live/test_live_examples.py +++ b/genai/live/test_live_examples.py @@ -42,7 +42,7 @@ import live_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/model_optimizer/noxfile_config.py b/genai/model_optimizer/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/model_optimizer/noxfile_config.py +++ b/genai/model_optimizer/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/model_optimizer/requirements-test.txt b/genai/model_optimizer/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/genai/model_optimizer/requirements-test.txt +++ b/genai/model_optimizer/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/genai/model_optimizer/test_modeloptimizer_examples.py b/genai/model_optimizer/test_modeloptimizer_examples.py index c26668b3ad3..71eb67f7575 100644 --- a/genai/model_optimizer/test_modeloptimizer_examples.py +++ b/genai/model_optimizer/test_modeloptimizer_examples.py @@ -15,7 +15,7 @@ import modeloptimizer_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/provisioned_throughput/noxfile_config.py b/genai/provisioned_throughput/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/provisioned_throughput/noxfile_config.py +++ b/genai/provisioned_throughput/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/provisioned_throughput/requirements-test.txt b/genai/provisioned_throughput/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/provisioned_throughput/requirements-test.txt +++ b/genai/provisioned_throughput/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/provisioned_throughput/test_provisioned_throughput_examples.py b/genai/provisioned_throughput/test_provisioned_throughput_examples.py index 693d4fe32da..8bcda182b73 100644 --- a/genai/provisioned_throughput/test_provisioned_throughput_examples.py +++ b/genai/provisioned_throughput/test_provisioned_throughput_examples.py @@ -20,7 +20,7 @@ import provisionedthroughput_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/safety/noxfile_config.py b/genai/safety/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/safety/noxfile_config.py +++ b/genai/safety/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/safety/requirements-test.txt b/genai/safety/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/safety/requirements-test.txt +++ b/genai/safety/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/safety/test_safety_examples.py b/genai/safety/test_safety_examples.py index 593e43fb617..1da4d861cf5 100644 --- a/genai/safety/test_safety_examples.py +++ b/genai/safety/test_safety_examples.py @@ -21,7 +21,7 @@ import safety_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/template_folder/noxfile_config.py b/genai/template_folder/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/template_folder/noxfile_config.py +++ b/genai/template_folder/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/template_folder/requirements-test.txt b/genai/template_folder/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/genai/template_folder/requirements-test.txt +++ b/genai/template_folder/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/genai/template_folder/test_templatefolder_examples.py b/genai/template_folder/test_templatefolder_examples.py index ecae1dce1d2..acc25e25530 100644 --- a/genai/template_folder/test_templatefolder_examples.py +++ b/genai/template_folder/test_templatefolder_examples.py @@ -15,7 +15,7 @@ import templatefolder_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/text_generation/noxfile_config.py b/genai/text_generation/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/text_generation/noxfile_config.py +++ b/genai/text_generation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/text_generation/requirements-test.txt b/genai/text_generation/requirements-test.txt index e43b7792721..22a9617b8e8 100644 --- a/genai/text_generation/requirements-test.txt +++ b/genai/text_generation/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/text_generation/test_text_generation_examples.py b/genai/text_generation/test_text_generation_examples.py index 3477caef9df..d859b4ff73b 100644 --- a/genai/text_generation/test_text_generation_examples.py +++ b/genai/text_generation/test_text_generation_examples.py @@ -39,7 +39,7 @@ import textgen_with_youtube_video import thinking_textgen_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/thinking/noxfile_config.py b/genai/thinking/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/thinking/noxfile_config.py +++ b/genai/thinking/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/thinking/requirements-test.txt b/genai/thinking/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/genai/thinking/requirements-test.txt +++ b/genai/thinking/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/genai/thinking/test_thinking_examples.py b/genai/thinking/test_thinking_examples.py index 71fc75f1f9a..047141ace4e 100644 --- a/genai/thinking/test_thinking_examples.py +++ b/genai/thinking/test_thinking_examples.py @@ -17,7 +17,7 @@ import thinking_includethoughts_with_txt import thinking_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "global" # "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/tools/noxfile_config.py b/genai/tools/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/tools/noxfile_config.py +++ b/genai/tools/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/tools/requirements-test.txt b/genai/tools/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/genai/tools/requirements-test.txt +++ b/genai/tools/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/genai/tools/test_tools_examples.py b/genai/tools/test_tools_examples.py index 60ed069e1a4..95611393b00 100644 --- a/genai/tools/test_tools_examples.py +++ b/genai/tools/test_tools_examples.py @@ -31,7 +31,7 @@ import tools_urlcontext_with_txt import tools_vais_with_txt -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/genai/tuning/noxfile_config.py b/genai/tuning/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/tuning/noxfile_config.py +++ b/genai/tuning/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/tuning/requirements-test.txt b/genai/tuning/requirements-test.txt index 4ccc4347cbe..14a8493498b 100644 --- a/genai/tuning/requirements-test.txt +++ b/genai/tuning/requirements-test.txt @@ -1,3 +1,3 @@ google-api-core==2.24.0 google-cloud-storage==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/video_generation/noxfile_config.py b/genai/video_generation/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/genai/video_generation/noxfile_config.py +++ b/genai/video_generation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/genai/video_generation/requirements-test.txt b/genai/video_generation/requirements-test.txt index 4ccc4347cbe..14a8493498b 100644 --- a/genai/video_generation/requirements-test.txt +++ b/genai/video_generation/requirements-test.txt @@ -1,3 +1,3 @@ google-api-core==2.24.0 google-cloud-storage==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/genai/video_generation/test_video_generation_examples.py b/genai/video_generation/test_video_generation_examples.py index 639793ff9e8..079bdf7e97c 100644 --- a/genai/video_generation/test_video_generation_examples.py +++ b/genai/video_generation/test_video_generation_examples.py @@ -41,7 +41,7 @@ import videogen_with_vid_edit_remove -os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = "True" +os.environ["GOOGLE_GENAI_USE_ENTERPRISE"] = "True" os.environ["GOOGLE_CLOUD_LOCATION"] = "us-central1" # The project name is included in the CICD pipeline # os.environ['GOOGLE_CLOUD_PROJECT'] = "add-your-project-name" diff --git a/generative_ai/README.md b/generative_ai/README.md index 9cd7509813b..caf5c1ea389 100644 --- a/generative_ai/README.md +++ b/generative_ai/README.md @@ -1,15 +1,14 @@ -# Generative AI Samples on Google Cloud +# Generative AI on Google Cloud: Python Samples -Welcome to the Python samples folder for Generative AI on Vertex AI! In this folder, you can find the Python samples -used in [Google Cloud Generative AI documentation](https://cloud.google.com/ai/generative-ai?hl=en). +This directory contains the official Python code samples featured in the [Google Cloud Generative AI documentation](https://cloud.google.com/ai/generative-ai?hl=en). These scripts demonstrate how to integrate and build with Vertex AI. -If you are looking for colab notebook, then this [link](https://github.com/GoogleCloudPlatform/generative-ai/tree/main). +Looking for interactive, step-by-step tutorials? Check out our extensive collection of [Colab notebooks](https://github.com/GoogleCloudPlatform/generative-ai/tree/main). ## Getting Started -To try and run these Code samples, we have following recommend using Google Cloud IDE or Google Colab. +> **Note:** An active Google Cloud Project is required. -Note: A Google Cloud Project is a pre-requisite. +We recommend running these code samples using Google Cloud Shell Editor or Google Colab to minimize environment setup. ### Feature folders @@ -21,47 +20,26 @@ Browse the folders below to find the Generative AI capabilities you're intereste Google Cloud Product - Short Description (With the help of Gemini 1.5) - - - - Context Caching - - https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-overview - - Code samples demonstrating how to use context caching with Vertex AI's generative models. This allows for more consistent and relevant responses across multiple interactions by storing previous conversation history. - - - - Controlled Generation - - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/control-generated-output - - Examples of how to control the output of generative models, such as specifying length, format, or sentiment. - - - - Count Token - - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/list-token - - Code demonstrating how to count tokens in text, which is crucial for managing costs and understanding model limitations. + Short Description (With the help of Gemini 3.1) + Embeddings https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings - Code showing how to generate and use embeddings from text or images. Embeddings can be used for tasks like semantic search, clustering, and classification. + Learn how to use Vertex AI's text and multimodal embedding models. These samples show you how to convert your unstructured data into numerical vectors to power semantic search, clustering, and RAG applications. + + Extensions https://cloud.google.com/vertex-ai/generative-ai/docs/extensions/overview - Demonstrations of how to use extensions with generative models, enabling them to access and process real-time information, use tools, and interact with external systems. + These samples show how to connect Gemini to external APIs and databases so your models can retrieve live data and execute real-world actions. **Note** that as Google Cloud transitions to the Gemini Enterprise Agent Platform, standalone Vertex AI Extensions are evolving into *Tools* managed within the centralized Agent Registry. While these examples teach the core mechanics of model-to-API communication, future production applications should adopt the new Agent Platform architecture.. @@ -69,23 +47,16 @@ Browse the folders below to find the Generative AI capabilities you're intereste https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling - Examples of how to use function calling to enable generative models to execute specific actions or retrieve information from external APIs. - - - - Grounding - - https://cloud.google.com/vertex-ai/generative-ai/docs/grounding/overview - - Code illustrating how to ground generative models with specific knowledge bases or data sources to improve the accuracy and relevance of their responses. + Function calling gives Gemini the ability to interact with your codebase. The model predicts which of your local functions needs to be run and returns the formatted arguments, leaving the actual execution up to your application. + Image Generation https://cloud.google.com/vertex-ai/generative-ai/docs/image/overview - Samples showcasing how to generate images from text prompts using models like Imagen. + Learn how to integrate the Imagen model into your applications. These examples cover text-to-image generation, editing, and using advanced parameters to get the exact visual output you need. @@ -93,7 +64,7 @@ Browse the folders below to find the Generative AI capabilities you're intereste https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/explore-models - Resources related to exploring and utilizing pre-trained models available in Vertex AI's Model Garden. + These examples show you how to provision endpoints and serve predictions from first-party, third-party, and open-source foundation models available in the Vertex AI Model Garden. @@ -101,7 +72,7 @@ Browse the folders below to find the Generative AI capabilities you're intereste https://cloud.google.com/vertex-ai/generative-ai/docs/models/tune-models - Code and guides for fine-tuning pre-trained generative models on specific datasets or for specific tasks. + Tailor Gemini and other foundation models to your specific domain. These examples cover how to format your datasets, kick off tuning jobs on Vertex AI, and deploy your custom-tuned models or adapters to production endpoints. @@ -109,7 +80,7 @@ Browse the folders below to find the Generative AI capabilities you're intereste https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/rag-api - Information and resources about Retrieval Augmented Generation (RAG), which combines information retrieval with generative models. + These examples cover the end-to-end RAG architecture: ingesting data, generating embeddings, querying a vector database, and passing the retrieved context to Gemini to generate informed, accurate answers. @@ -117,23 +88,7 @@ Browse the folders below to find the Generative AI capabilities you're intereste https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/reasoning-engine - Details about the Reasoning Engine, which enables more complex reasoning and logical deduction in generative models. - - - - Safety - - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-attributes - - Examples of how to configure safety attributes and filters to mitigate risks and ensure responsible use of generative models. - - - - System Instructions - - https://cloud.google.com/vertex-ai/generative-ai/docs/learn/prompts/system-instructions?hl=en - - Code demonstrating how to provide system instructions to guide the behavior and responses of generative models. + These examples cover how to use Vertex AI Reasoning Engine to build custom agents. @@ -141,23 +96,7 @@ Browse the folders below to find the Generative AI capabilities you're intereste https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-chat-prompts-gemini - Samples of how to generate text using Gemini models, including chat-based interactions and creative writing. - - - - Understand Audio - - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/audio-understanding - - Examples of how to use generative models for audio understanding tasks, such as transcription and audio classification. - - - - Understand Video - - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/video-understanding - - Samples showcasing how to use generative models for video understanding tasks, such as video summarization and content analysis. + These samples demonstrate how to use Vertex AI's Gemini models to generate, summarize, and extract information from text. diff --git a/generative_ai/chat_completions/noxfile_config.py b/generative_ai/chat_completions/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/chat_completions/noxfile_config.py +++ b/generative_ai/chat_completions/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/chat_completions/requirements-test.txt b/generative_ai/chat_completions/requirements-test.txt index 3b9949d8513..3a0f817634b 100644 --- a/generative_ai/chat_completions/requirements-test.txt +++ b/generative_ai/chat_completions/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/embeddings/noxfile_config.py b/generative_ai/embeddings/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/embeddings/noxfile_config.py +++ b/generative_ai/embeddings/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/embeddings/requirements-test.txt b/generative_ai/embeddings/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/embeddings/requirements-test.txt +++ b/generative_ai/embeddings/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/evaluation/noxfile_config.py b/generative_ai/evaluation/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/evaluation/noxfile_config.py +++ b/generative_ai/evaluation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/evaluation/requirements-test.txt b/generative_ai/evaluation/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/evaluation/requirements-test.txt +++ b/generative_ai/evaluation/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/extensions/noxfile_config.py b/generative_ai/extensions/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/extensions/noxfile_config.py +++ b/generative_ai/extensions/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/extensions/requirements-test.txt b/generative_ai/extensions/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/extensions/requirements-test.txt +++ b/generative_ai/extensions/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/function_calling/noxfile_config.py b/generative_ai/function_calling/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/function_calling/noxfile_config.py +++ b/generative_ai/function_calling/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/function_calling/requirements-test.txt b/generative_ai/function_calling/requirements-test.txt index 3b9949d8513..3a0f817634b 100644 --- a/generative_ai/function_calling/requirements-test.txt +++ b/generative_ai/function_calling/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.24.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/image_generation/noxfile_config.py b/generative_ai/image_generation/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/image_generation/noxfile_config.py +++ b/generative_ai/image_generation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/image_generation/requirements-test.txt b/generative_ai/image_generation/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/image_generation/requirements-test.txt +++ b/generative_ai/image_generation/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/labels/noxfile_config.py b/generative_ai/labels/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/labels/noxfile_config.py +++ b/generative_ai/labels/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/labels/requirements-test.txt b/generative_ai/labels/requirements-test.txt index 2247ce2d832..cefab43dffa 100644 --- a/generative_ai/labels/requirements-test.txt +++ b/generative_ai/labels/requirements-test.txt @@ -1,2 +1,2 @@ google-api-core==2.23.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/generative_ai/model_garden/noxfile_config.py b/generative_ai/model_garden/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/model_garden/noxfile_config.py +++ b/generative_ai/model_garden/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/model_garden/requirements-test.txt b/generative_ai/model_garden/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/model_garden/requirements-test.txt +++ b/generative_ai/model_garden/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/model_tuning/noxfile_config.py b/generative_ai/model_tuning/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/model_tuning/noxfile_config.py +++ b/generative_ai/model_tuning/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/model_tuning/requirements-test.txt b/generative_ai/model_tuning/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/model_tuning/requirements-test.txt +++ b/generative_ai/model_tuning/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/prompts/noxfile_config.py b/generative_ai/prompts/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/prompts/noxfile_config.py +++ b/generative_ai/prompts/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/prompts/requirements-test.txt b/generative_ai/prompts/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/prompts/requirements-test.txt +++ b/generative_ai/prompts/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/provisioned_throughput/noxfile_config.py b/generative_ai/provisioned_throughput/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/provisioned_throughput/noxfile_config.py +++ b/generative_ai/provisioned_throughput/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/provisioned_throughput/requirements-test.txt b/generative_ai/provisioned_throughput/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/provisioned_throughput/requirements-test.txt +++ b/generative_ai/provisioned_throughput/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/rag/noxfile_config.py b/generative_ai/rag/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/rag/noxfile_config.py +++ b/generative_ai/rag/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/rag/requirements-test.txt b/generative_ai/rag/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/rag/requirements-test.txt +++ b/generative_ai/rag/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/generative_ai/reasoning_engine/noxfile_config.py b/generative_ai/reasoning_engine/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/generative_ai/reasoning_engine/noxfile_config.py +++ b/generative_ai/reasoning_engine/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/reasoning_engine/requirements-test.txt b/generative_ai/reasoning_engine/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/generative_ai/reasoning_engine/requirements-test.txt +++ b/generative_ai/reasoning_engine/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/healthcare/api-client/v1/consent/noxfile_config.py b/healthcare/api-client/v1/consent/noxfile_config.py index ecc5247cce9..52036a424ef 100644 --- a/healthcare/api-client/v1/consent/noxfile_config.py +++ b/healthcare/api-client/v1/consent/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/healthcare/api-client/v1/consent/requirements-test.txt b/healthcare/api-client/v1/consent/requirements-test.txt index 0d7187a429e..659ed4bc2c9 100644 --- a/healthcare/api-client/v1/consent/requirements-test.txt +++ b/healthcare/api-client/v1/consent/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" diff --git a/healthcare/api-client/v1/datasets/noxfile_config.py b/healthcare/api-client/v1/datasets/noxfile_config.py index b4d9f9b057c..6cea5520fa8 100644 --- a/healthcare/api-client/v1/datasets/noxfile_config.py +++ b/healthcare/api-client/v1/datasets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/healthcare/api-client/v1/datasets/requirements-test.txt b/healthcare/api-client/v1/datasets/requirements-test.txt index 8b9eaff06c4..cb48e4d986e 100644 --- a/healthcare/api-client/v1/datasets/requirements-test.txt +++ b/healthcare/api-client/v1/datasets/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" retrying==1.3.4 diff --git a/healthcare/api-client/v1/dicom/noxfile_config.py b/healthcare/api-client/v1/dicom/noxfile_config.py index b4d9f9b057c..6cea5520fa8 100644 --- a/healthcare/api-client/v1/dicom/noxfile_config.py +++ b/healthcare/api-client/v1/dicom/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/healthcare/api-client/v1/dicom/requirements-test.txt b/healthcare/api-client/v1/dicom/requirements-test.txt index 0d7187a429e..659ed4bc2c9 100644 --- a/healthcare/api-client/v1/dicom/requirements-test.txt +++ b/healthcare/api-client/v1/dicom/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" diff --git a/healthcare/api-client/v1/fhir/noxfile_config.py b/healthcare/api-client/v1/fhir/noxfile_config.py index 1a9099a80eb..c2250035bd5 100644 --- a/healthcare/api-client/v1/fhir/noxfile_config.py +++ b/healthcare/api-client/v1/fhir/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/healthcare/api-client/v1/fhir/requirements-test.txt b/healthcare/api-client/v1/fhir/requirements-test.txt index 0d7187a429e..659ed4bc2c9 100644 --- a/healthcare/api-client/v1/fhir/requirements-test.txt +++ b/healthcare/api-client/v1/fhir/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" diff --git a/healthcare/api-client/v1/hl7v2/noxfile_config.py b/healthcare/api-client/v1/hl7v2/noxfile_config.py index b4d9f9b057c..6cea5520fa8 100644 --- a/healthcare/api-client/v1/hl7v2/noxfile_config.py +++ b/healthcare/api-client/v1/hl7v2/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/healthcare/api-client/v1/hl7v2/requirements-test.txt b/healthcare/api-client/v1/hl7v2/requirements-test.txt index 0d7187a429e..659ed4bc2c9 100644 --- a/healthcare/api-client/v1/hl7v2/requirements-test.txt +++ b/healthcare/api-client/v1/hl7v2/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" diff --git a/healthcare/api-client/v1beta1/fhir/requirements-test.txt b/healthcare/api-client/v1beta1/fhir/requirements-test.txt index 8ce117fb56e..36921e74453 100644 --- a/healthcare/api-client/v1beta1/fhir/requirements-test.txt +++ b/healthcare/api-client/v1beta1/fhir/requirements-test.txt @@ -1,3 +1,3 @@ backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/iam/api-client/noxfile_config.py b/iam/api-client/noxfile_config.py index 30f1c3971e6..f879875f76a 100644 --- a/iam/api-client/noxfile_config.py +++ b/iam/api-client/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Declare optional test sessions you want to opt-in. Currently we # have the following optional test sessions: # 'cloud_run' # Test session for Cloud Run application. diff --git a/iam/api-client/requirements-test.txt b/iam/api-client/requirements-test.txt index 8b9eaff06c4..cb48e4d986e 100644 --- a/iam/api-client/requirements-test.txt +++ b/iam/api-client/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" retrying==1.3.4 diff --git a/iam/cloud-client/snippets/noxfile_config.py b/iam/cloud-client/snippets/noxfile_config.py index 029a70011cc..658f73ff360 100644 --- a/iam/cloud-client/snippets/noxfile_config.py +++ b/iam/cloud-client/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/iam/cloud-client/snippets/requirements-test.txt b/iam/cloud-client/snippets/requirements-test.txt index 6ff70adf77d..975d5ee58c2 100644 --- a/iam/cloud-client/snippets/requirements-test.txt +++ b/iam/cloud-client/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" backoff==2.2.1 diff --git a/iap/app_engine_app/requirements-test.txt b/iap/app_engine_app/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/iap/app_engine_app/requirements-test.txt +++ b/iap/app_engine_app/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/iap/requirements-test.txt b/iap/requirements-test.txt index 185d62c4204..23df1e03c7e 100644 --- a/iap/requirements-test.txt +++ b/iap/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" flaky==3.8.1 diff --git a/iap/requirements.txt b/iap/requirements.txt index c0d103f39e4..452dc0220c4 100644 --- a/iap/requirements.txt +++ b/iap/requirements.txt @@ -1,4 +1,4 @@ -cryptography==45.0.1 +cryptography==46.0.7 Flask==3.1.3 google-auth==2.38.0 gunicorn==23.0.0 diff --git a/jobs/v3/api_client/noxfile_config.py b/jobs/v3/api_client/noxfile_config.py index b4d9f9b057c..6cea5520fa8 100644 --- a/jobs/v3/api_client/noxfile_config.py +++ b/jobs/v3/api_client/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/jobs/v3/api_client/requirements-test.txt b/jobs/v3/api_client/requirements-test.txt index 2a635ea7b6a..322ae75b68c 100644 --- a/jobs/v3/api_client/requirements-test.txt +++ b/jobs/v3/api_client/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" flaky==3.8.1 diff --git a/kms/attestations/noxfile_config.py b/kms/attestations/noxfile_config.py index a68d8a75ecd..932bef266b6 100644 --- a/kms/attestations/noxfile_config.py +++ b/kms/attestations/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/kms/attestations/requirements-test.txt b/kms/attestations/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/kms/attestations/requirements-test.txt +++ b/kms/attestations/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/kms/attestations/requirements.txt b/kms/attestations/requirements.txt index 21fdd0e1147..12decf81590 100644 --- a/kms/attestations/requirements.txt +++ b/kms/attestations/requirements.txt @@ -1,4 +1,4 @@ cryptography==45.0.1 -pem==21.2.0; python_version < '3.8' +pem==23.1.0; python_version < '3.8' pem==23.1.0; python_version > '3.7' requests==2.31.0 diff --git a/kms/snippets/noxfile_config.py b/kms/snippets/noxfile_config.py index 457e86f5413..0973c8621c7 100644 --- a/kms/snippets/noxfile_config.py +++ b/kms/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/kms/snippets/requirements-test.txt b/kms/snippets/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/kms/snippets/requirements-test.txt +++ b/kms/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/kubernetes_engine/api-client/requirements-test.txt b/kubernetes_engine/api-client/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/kubernetes_engine/api-client/requirements-test.txt +++ b/kubernetes_engine/api-client/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/kubernetes_engine/django_tutorial/requirements-test.txt b/kubernetes_engine/django_tutorial/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/kubernetes_engine/django_tutorial/requirements-test.txt +++ b/kubernetes_engine/django_tutorial/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/language/snippets/api/requirements-test.txt b/language/snippets/api/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/language/snippets/api/requirements-test.txt +++ b/language/snippets/api/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/language/snippets/classify_text/noxfile_config.py b/language/snippets/classify_text/noxfile_config.py index 25d1d4e081c..50cd5669209 100644 --- a/language/snippets/classify_text/noxfile_config.py +++ b/language/snippets/classify_text/noxfile_config.py @@ -25,7 +25,7 @@ # > ℹ️ Test only on Python 3.10. # > The Python version used is defined by the Dockerfile, so it's redundant # > to run multiple tests since they would all be running the same Dockerfile. - "ignored_versions": ["2.7", "3.6", "3.7", "3.9", "3.11", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them # "enforce_type_hints": True, diff --git a/language/snippets/classify_text/requirements-test.txt b/language/snippets/classify_text/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/language/snippets/classify_text/requirements-test.txt +++ b/language/snippets/classify_text/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/language/snippets/cloud-client/v1/requirements-test.txt b/language/snippets/cloud-client/v1/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/language/snippets/cloud-client/v1/requirements-test.txt +++ b/language/snippets/cloud-client/v1/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/language/snippets/sentiment/requirements-test.txt b/language/snippets/sentiment/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/language/snippets/sentiment/requirements-test.txt +++ b/language/snippets/sentiment/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/language/v1/requirements-test.txt b/language/v1/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/language/v1/requirements-test.txt +++ b/language/v1/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/language/v2/noxfile_config.py b/language/v2/noxfile_config.py index 38a32880121..b7d8115d773 100644 --- a/language/v2/noxfile_config.py +++ b/language/v2/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/language/v2/requirements-test.txt b/language/v2/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/language/v2/requirements-test.txt +++ b/language/v2/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/logging/import-logs/noxfile_config.py b/logging/import-logs/noxfile_config.py index e0098f56b2f..8ed5845a7a3 100644 --- a/logging/import-logs/noxfile_config.py +++ b/logging/import-logs/noxfile_config.py @@ -17,5 +17,5 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], } diff --git a/logging/import-logs/requirements-test.txt b/logging/import-logs/requirements-test.txt index 47c9e1b113d..c00563c2dde 100644 --- a/logging/import-logs/requirements-test.txt +++ b/logging/import-logs/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" google-cloud-logging~=3.11.4 google-cloud-storage~=2.10.0 diff --git a/logging/samples/snippets/requirements-test.txt b/logging/samples/snippets/requirements-test.txt index 37eb1f9aa7a..79932f83530 100644 --- a/logging/samples/snippets/requirements-test.txt +++ b/logging/samples/snippets/requirements-test.txt @@ -1,3 +1,2 @@ backoff==2.2.1 -pytest===7.4.4; python_version == '3.7' -pytest==8.2.2; python_version >= '3.8' +pytest==9.0.3; python_version >= "3.10" diff --git a/managedkafka/snippets/connect/clusters/requirements.txt b/managedkafka/snippets/connect/clusters/requirements.txt index 5f372e81c41..2903dc4f44d 100644 --- a/managedkafka/snippets/connect/clusters/requirements.txt +++ b/managedkafka/snippets/connect/clusters/requirements.txt @@ -1,5 +1,5 @@ protobuf==5.29.4 -pytest==8.2.2 +pytest==9.0.3; python_version >= "3.10" google-api-core==2.23.0 google-auth==2.38.0 google-cloud-managedkafka==0.1.12 diff --git a/managedkafka/snippets/requirements.txt b/managedkafka/snippets/requirements.txt index 5f372e81c41..2903dc4f44d 100644 --- a/managedkafka/snippets/requirements.txt +++ b/managedkafka/snippets/requirements.txt @@ -1,5 +1,5 @@ protobuf==5.29.4 -pytest==8.2.2 +pytest==9.0.3; python_version >= "3.10" google-api-core==2.23.0 google-auth==2.38.0 google-cloud-managedkafka==0.1.12 diff --git a/media-translation/snippets/noxfile_config.py b/media-translation/snippets/noxfile_config.py index a68d8a75ecd..932bef266b6 100644 --- a/media-translation/snippets/noxfile_config.py +++ b/media-translation/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/media-translation/snippets/requirements-test.txt b/media-translation/snippets/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/media-translation/snippets/requirements-test.txt +++ b/media-translation/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/media_cdn/requirements-test.txt b/media_cdn/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/media_cdn/requirements-test.txt +++ b/media_cdn/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/memorystore/memcache/noxfile_config.py b/memorystore/memcache/noxfile_config.py index e70e0f7ff70..0d9b541774e 100644 --- a/memorystore/memcache/noxfile_config.py +++ b/memorystore/memcache/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # We only run the cloud run tests in py38 session. - "ignored_versions": ["2.7", "3.6", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/memorystore/redis/cloud_run_deployment/Dockerfile b/memorystore/redis/cloud_run_deployment/Dockerfile index 94548382d70..81e9d8f6404 100644 --- a/memorystore/redis/cloud_run_deployment/Dockerfile +++ b/memorystore/redis/cloud_run_deployment/Dockerfile @@ -14,7 +14,7 @@ # Use the official lightweight Python image. # https://hub.docker.com/_/python -FROM python:3.11-slim +FROM python:3.14-slim # Copy local code to the container image. ENV APP_HOME /app diff --git a/memorystore/redis/gce_deployment/deploy.sh b/memorystore/redis/gce_deployment/deploy.sh index 0fa80846da6..4352fdb8bcf 100755 --- a/memorystore/redis/gce_deployment/deploy.sh +++ b/memorystore/redis/gce_deployment/deploy.sh @@ -37,7 +37,7 @@ fi #Upload the tar to GCS tar -cvf app.tar -C .. requirements.txt main.py # Copy to GCS bucket -gsutil cp app.tar gs://"$GCS_BUCKET_NAME"/gce/ +gcloud storage cp app.tar gs://"$GCS_BUCKET_NAME"/gce/ # Create an instance gcloud compute instances create my-instance \ diff --git a/memorystore/redis/gce_deployment/startup-script.sh b/memorystore/redis/gce_deployment/startup-script.sh index 3e523246114..5a8e0bb0b09 100644 --- a/memorystore/redis/gce_deployment/startup-script.sh +++ b/memorystore/redis/gce_deployment/startup-script.sh @@ -33,7 +33,7 @@ apt-get install -yq \ curl -s "https://storage.googleapis.com/signals-agents/logging/google-fluentd-install.sh" | bash service google-fluentd restart & -gsutil cp gs://"$GCS_BUCKET_NAME"/gce/app.tar /app.tar +gcloud storage cp gs://"$GCS_BUCKET_NAME"/gce/app.tar /app.tar mkdir -p /app tar -x -f /app.tar -C /app cd /app diff --git a/memorystore/redis/noxfile_config.py b/memorystore/redis/noxfile_config.py index bf45881273d..f6a3cf7417f 100644 --- a/memorystore/redis/noxfile_config.py +++ b/memorystore/redis/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # We only run the cloud run tests in py38 session. - "ignored_versions": ["2.7", "3.6", "3.7"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/memorystore/redis/requirements-test.txt b/memorystore/redis/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/memorystore/redis/requirements-test.txt +++ b/memorystore/redis/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/model_armor/snippets/noxfile_config.py b/model_armor/snippets/noxfile_config.py index 29c18b2ba9c..293769315f2 100644 --- a/model_armor/snippets/noxfile_config.py +++ b/model_armor/snippets/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/model_armor/snippets/requirements-test.txt b/model_armor/snippets/requirements-test.txt index 1c987370aa9..c9e154ba440 100644 --- a/model_armor/snippets/requirements-test.txt +++ b/model_armor/snippets/requirements-test.txt @@ -1 +1 @@ -pytest==8.3.4 \ No newline at end of file +pytest==9.0.3; python_version >= "3.10" diff --git a/model_garden/anthropic/noxfile_config.py b/model_garden/anthropic/noxfile_config.py index 2a0f115c38f..0973c8621c7 100644 --- a/model_garden/anthropic/noxfile_config.py +++ b/model_garden/anthropic/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.12"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/model_garden/anthropic/requirements-test.txt b/model_garden/anthropic/requirements-test.txt index 73541a927f4..de695820000 100644 --- a/model_garden/anthropic/requirements-test.txt +++ b/model_garden/anthropic/requirements-test.txt @@ -1,4 +1,4 @@ google-api-core==2.24.0 google-cloud-bigquery==3.29.0 google-cloud-storage==2.19.0 -pytest==8.2.0 \ No newline at end of file +pytest==9.0.3; python_version >= "3.10" diff --git a/model_garden/gemma/noxfile_config.py b/model_garden/gemma/noxfile_config.py index 962ba40a926..0973c8621c7 100644 --- a/model_garden/gemma/noxfile_config.py +++ b/model_garden/gemma/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.8", "3.10", "3.11", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/model_garden/gemma/requirements-test.txt b/model_garden/gemma/requirements-test.txt index 92281986e50..baa23bf9c3e 100644 --- a/model_garden/gemma/requirements-test.txt +++ b/model_garden/gemma/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 google-api-core==2.19.0 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" pytest-asyncio==0.23.6 diff --git a/monitoring/api/v3/api-client/README.rst b/monitoring/api/v3/api-client/README.rst index b905644a9e8..0b83c81e51f 100644 --- a/monitoring/api/v3/api-client/README.rst +++ b/monitoring/api/v3/api-client/README.rst @@ -1,18 +1,18 @@ .. This file is automatically generated. Do not edit this file directly. -Stackdriver Monitoring Python Samples +Cloud Monitoring Python Samples =============================================================================== .. image:: https://gstatic.com/cloudssh/images/open-btn.png :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/api-client/README.rst -This directory contains samples for Stackdriver Monitoring. `Stackdriver Monitoring `_ collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Stackdriver ingests that data and generates insights via dashboards, charts, and alerts. +This directory contains samples for Cloud Monitoring. `Cloud Monitoring `_ collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Cloud Monitoring ingests that data and generates insights via dashboards, charts, and alerts. -.. _Stackdriver Monitoring: https://cloud.google.com/monitoring/docs +.. _Cloud Monitoring: https://cloud.google.com/monitoring/docs Setup ------------------------------------------------------------------------------- @@ -78,7 +78,7 @@ To run this sample: usage: list_resources.py [-h] --project_id PROJECT_ID - Sample command-line program for retrieving Stackdriver Monitoring API V3 + Sample command-line program for retrieving Cloud Monitoring API V3 data. See README.md for instructions on setting up your development environment. @@ -111,7 +111,7 @@ To run this sample: usage: custom_metric.py [-h] --project_id PROJECT_ID - Sample command-line program for writing and reading Stackdriver Monitoring + Sample command-line program for writing and reading Cloud Monitoring API V3 custom metrics. Simple command-line program to demonstrate connecting to the Google diff --git a/monitoring/api/v3/api-client/README.rst.in b/monitoring/api/v3/api-client/README.rst.in index e71bd6de87c..529a45128cb 100644 --- a/monitoring/api/v3/api-client/README.rst.in +++ b/monitoring/api/v3/api-client/README.rst.in @@ -1,15 +1,15 @@ # This file is used to generate README.rst product: - name: Stackdriver Monitoring - short_name: Stackdriver Monitoring + name: Cloud Monitoring + short_name: Cloud Monitoring url: https://cloud.google.com/monitoring/docs description: > - `Stackdriver Monitoring `_ collects metrics, events, and metadata from + `Cloud Monitoring `_ collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many - others. Stackdriver ingests that data and generates insights via + others. Cloud Monitoring ingests that data and generates insights via dashboards, charts, and alerts. setup: diff --git a/monitoring/api/v3/api-client/list_resources.py b/monitoring/api/v3/api-client/list_resources.py index e77e610314f..2a4aac9f904 100644 --- a/monitoring/api/v3/api-client/list_resources.py +++ b/monitoring/api/v3/api-client/list_resources.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Sample command-line program for retrieving Stackdriver Monitoring API V3 +"""Sample command-line program for retrieving Cloud Monitoring API V3 data. See README.md for instructions on setting up your development environment. diff --git a/monitoring/api/v3/api-client/requirements-test.txt b/monitoring/api/v3/api-client/requirements-test.txt index 2a635ea7b6a..322ae75b68c 100644 --- a/monitoring/api/v3/api-client/requirements-test.txt +++ b/monitoring/api/v3/api-client/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1; python_version < "3.7" backoff==2.2.1; python_version >= "3.7" -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" flaky==3.8.1 diff --git a/monitoring/opencensus/requirements-test.txt b/monitoring/opencensus/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/monitoring/opencensus/requirements-test.txt +++ b/monitoring/opencensus/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/monitoring/prometheus/requirements-test.txt b/monitoring/prometheus/requirements-test.txt index 15d066af319..c9e154ba440 100644 --- a/monitoring/prometheus/requirements-test.txt +++ b/monitoring/prometheus/requirements-test.txt @@ -1 +1 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/monitoring/snippets/v3/alerts-client/README.rst b/monitoring/snippets/v3/alerts-client/README.rst index bb59aad5fee..4f941d6fc14 100644 --- a/monitoring/snippets/v3/alerts-client/README.rst +++ b/monitoring/snippets/v3/alerts-client/README.rst @@ -1,18 +1,18 @@ .. This file is automatically generated. Do not edit this file directly. -Google Stackdriver Alerting API Python Samples +Google Cloud Monitoring Alerting API Python Samples =============================================================================== .. image:: https://gstatic.com/cloudssh/images/open-btn.png :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/alerts-client/README.rst -This directory contains samples for Google Stackdriver Alerting API. Stackdriver Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Stackdriver's Alerting API allows you to create, delete, and make back up copies of your alert policies. +This directory contains samples for Google Cloud Monitoring Alerting API. Cloud Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Cloud Monitoring's Alerting API allows you to create, delete, and make back up copies of your alert policies. -.. _Google Stackdriver Alerting API: https://cloud.google.com/monitoring/alerts/ +.. _Google Cloud Monitoring Alerting API: https://cloud.google.com/monitoring/alerts/ To run the sample, you need to enable the API at: https://console.cloud.google.com/apis/library/monitoring.googleapis.com diff --git a/monitoring/snippets/v3/alerts-client/README.rst.in b/monitoring/snippets/v3/alerts-client/README.rst.in index 00b280124ea..c3bd356a750 100644 --- a/monitoring/snippets/v3/alerts-client/README.rst.in +++ b/monitoring/snippets/v3/alerts-client/README.rst.in @@ -1,15 +1,15 @@ # This file is used to generate README.rst product: - name: Google Stackdriver Alerting API - short_name: Stackdriver Alerting API + name: Google Cloud Monitoring Alerting API + short_name: Cloud Monitoring Alerting API url: https://cloud.google.com/monitoring/alerts/ description: > - Stackdriver Monitoring collects metrics, events, and metadata from Google + Cloud Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch - and many others. Stackdriver's Alerting API allows you to create, + and many others. Cloud Monitoring's Alerting API allows you to create, delete, and make back up copies of your alert policies. required_api_url: https://console.cloud.google.com/apis/library/monitoring.googleapis.com diff --git a/monitoring/snippets/v3/alerts-client/noxfile_config.py b/monitoring/snippets/v3/alerts-client/noxfile_config.py index 083b166d18c..f5de96c352a 100644 --- a/monitoring/snippets/v3/alerts-client/noxfile_config.py +++ b/monitoring/snippets/v3/alerts-client/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Declare optional test sessions you want to opt-in. Currently we # have the following optional test sessions: # 'cloud_run' # Test session for Cloud Run application. diff --git a/monitoring/snippets/v3/alerts-client/requirements-test.txt b/monitoring/snippets/v3/alerts-client/requirements-test.txt index e312099c33c..568cffaec48 100644 --- a/monitoring/snippets/v3/alerts-client/requirements-test.txt +++ b/monitoring/snippets/v3/alerts-client/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" retrying==1.3.4 flaky==3.8.1 diff --git a/monitoring/snippets/v3/cloud-client/README.rst b/monitoring/snippets/v3/cloud-client/README.rst index 280f9c4e0a7..64e62d36ad9 100644 --- a/monitoring/snippets/v3/cloud-client/README.rst +++ b/monitoring/snippets/v3/cloud-client/README.rst @@ -1,20 +1,20 @@ .. This file is automatically generated. Do not edit this file directly. -Google Stackdriver Monitoring API Python Samples +Google Cloud Monitoring API Python Samples =============================================================================== .. image:: https://gstatic.com/cloudssh/images/open-btn.png :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/cloud-client/README.rst -This directory contains samples for Google Stackdriver Monitoring API. Stackdriver Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch - and many others. Stackdriver ingests that data and generates insights +This directory contains samples for Google Cloud Monitoring API. Cloud Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch + and many others. Cloud Monitoring ingests that data and generates insights via dashboards, charts, and alerts. -.. _Google Stackdriver Monitoring API: https://cloud.google.com/monitoring/docs/ +.. _Google Cloud Monitoring API: https://cloud.google.com/monitoring/docs/ To run the sample, you need to enable the API at: https://console.cloud.google.com/apis/library/monitoring.googleapis.com diff --git a/monitoring/snippets/v3/cloud-client/README.rst.in b/monitoring/snippets/v3/cloud-client/README.rst.in index 0ab6b2258b7..8ac06335517 100644 --- a/monitoring/snippets/v3/cloud-client/README.rst.in +++ b/monitoring/snippets/v3/cloud-client/README.rst.in @@ -1,15 +1,15 @@ # This file is used to generate README.rst product: - name: Google Stackdriver Monitoring API - short_name: Stackdriver Monitoring API + name: Google Cloud Monitoring API + short_name: Cloud Monitoring API url: https://cloud.google.com/monitoring/docs/ description: > - Stackdriver Monitoring collects metrics, events, and metadata from Google + Cloud Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch - and many others. Stackdriver ingests that data and generates insights + and many others. Cloud Monitoring ingests that data and generates insights via dashboards, charts, and alerts. required_api_url: https://console.cloud.google.com/apis/library/monitoring.googleapis.com diff --git a/monitoring/snippets/v3/cloud-client/noxfile_config.py b/monitoring/snippets/v3/cloud-client/noxfile_config.py index 083b166d18c..f5de96c352a 100644 --- a/monitoring/snippets/v3/cloud-client/noxfile_config.py +++ b/monitoring/snippets/v3/cloud-client/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Declare optional test sessions you want to opt-in. Currently we # have the following optional test sessions: # 'cloud_run' # Test session for Cloud Run application. diff --git a/monitoring/snippets/v3/cloud-client/requirements-test.txt b/monitoring/snippets/v3/cloud-client/requirements-test.txt index f3230681cda..79932f83530 100644 --- a/monitoring/snippets/v3/cloud-client/requirements-test.txt +++ b/monitoring/snippets/v3/cloud-client/requirements-test.txt @@ -1,2 +1,2 @@ backoff==2.2.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/monitoring/snippets/v3/uptime-check-client/README.rst b/monitoring/snippets/v3/uptime-check-client/README.rst index 30046bdef9d..b565f9e4884 100644 --- a/monitoring/snippets/v3/uptime-check-client/README.rst +++ b/monitoring/snippets/v3/uptime-check-client/README.rst @@ -1,18 +1,18 @@ .. This file is automatically generated. Do not edit this file directly. -Google Stackdriver Uptime Checks API Python Samples +Google Cloud Monitoring Uptime Checks API Python Samples =============================================================================== .. image:: https://gstatic.com/cloudssh/images/open-btn.png :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=monitoring/api/v3/uptime-check-client/README.rst -This directory contains samples for Google Stackdriver Uptime Checks API. Stackdriver Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Stackdriver's Uptime Checks API allows you to create, delete, and list your project's Uptime Checks. +This directory contains samples for Google Cloud Monitoring Uptime Checks API. Cloud Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch and many others. Cloud Monitoring's Uptime Checks API allows you to create, delete, and list your project's Uptime Checks. -.. _Google Stackdriver Uptime Checks API: https://cloud.google.com/monitoring/uptime-checks/management +.. _Google Cloud Monitoring Uptime Checks API: https://cloud.google.com/monitoring/uptime-checks/management Setup ------------------------------------------------------------------------------- diff --git a/monitoring/snippets/v3/uptime-check-client/README.rst.in b/monitoring/snippets/v3/uptime-check-client/README.rst.in index 1174962e48d..84998ed3529 100644 --- a/monitoring/snippets/v3/uptime-check-client/README.rst.in +++ b/monitoring/snippets/v3/uptime-check-client/README.rst.in @@ -1,15 +1,15 @@ # This file is used to generate README.rst product: - name: Google Stackdriver Uptime Checks API - short_name: Stackdriver Uptime Checks API + name: Google Cloud Monitoring Uptime Checks API + short_name: Cloud Uptime Checks API url: https://cloud.google.com/monitoring/uptime-checks/management description: > - Stackdriver Monitoring collects metrics, events, and metadata from Google + Cloud Monitoring collects metrics, events, and metadata from Google Cloud Platform, Amazon Web Services (AWS), hosted uptime probes, application instrumentation, and a variety of common application components including Cassandra, Nginx, Apache Web Server, Elasticsearch - and many others. Stackdriver's Uptime Checks API allows you to create, + and many others. Cloud Monitoring's Uptime Checks API allows you to create, delete, and list your project's Uptime Checks. setup: diff --git a/monitoring/snippets/v3/uptime-check-client/noxfile_config.py b/monitoring/snippets/v3/uptime-check-client/noxfile_config.py index 083b166d18c..f5de96c352a 100644 --- a/monitoring/snippets/v3/uptime-check-client/noxfile_config.py +++ b/monitoring/snippets/v3/uptime-check-client/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], + "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], # Declare optional test sessions you want to opt-in. Currently we # have the following optional test sessions: # 'cloud_run' # Test session for Cloud Run application. diff --git a/monitoring/snippets/v3/uptime-check-client/requirements-test.txt b/monitoring/snippets/v3/uptime-check-client/requirements-test.txt index f3230681cda..79932f83530 100644 --- a/monitoring/snippets/v3/uptime-check-client/requirements-test.txt +++ b/monitoring/snippets/v3/uptime-check-client/requirements-test.txt @@ -1,2 +1,2 @@ backoff==2.2.1 -pytest==8.2.0 +pytest==9.0.3; python_version >= "3.10" diff --git a/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb b/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb index 4db540c6ae1..d805d0cb1fe 100644 --- a/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb +++ b/notebooks/tutorials/cloud-ml-engine/Training and prediction with scikit-learn.ipynb @@ -76,7 +76,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil mb gs://$BUCKET_NAME/" + "!gcloud storage buckets create gs://$BUCKET_NAME" ] }, { @@ -377,7 +377,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil ls gs://$BUCKET_NAME/" + "!gcloud storage ls gs://$BUCKET_NAME/" ] }, { @@ -539,7 +539,7 @@ "!gcloud ai-platform models delete $MODEL_NAME --quiet\n", "\n", "# Delete the bucket and contents\n", - "!gsutil rm -r gs://$BUCKET_NAME\n", + "!gcloud storage rm --recursive gs://$BUCKET_NAME\n", "\n", "# Delete the local files created by the tutorial\n", "!rm -rf census_training" diff --git a/notebooks/tutorials/storage/Storage command-line tool.ipynb b/notebooks/tutorials/storage/Storage command-line tool.ipynb index 21e62ae8236..eef0054c790 100644 --- a/notebooks/tutorials/storage/Storage command-line tool.ipynb +++ b/notebooks/tutorials/storage/Storage command-line tool.ipynb @@ -26,6 +26,7 @@ }, "outputs": [], "source": [ + "# ERROR: A migration for this command is not implemented. Please refer to the migration guide.\n", "!gsutil help" ] }, @@ -67,7 +68,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil mb gs://{bucket_name}/" + "!gcloud storage buckets create gs://{bucket_name}/" ] }, { @@ -95,7 +96,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil ls -p $project_id" + "!gcloud storage ls --project $project_id" ] }, { @@ -128,7 +129,7 @@ }, "outputs": [], "source": [ - "!gsutil ls -L -b gs://{bucket_name}/" + "!gcloud storage ls --full --buckets gs://{bucket_name}/" ] }, { @@ -163,7 +164,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil cp resources/us-states.txt gs://{bucket_name}/" + "!gcloud storage cp resources/us-states.txt gs://{bucket_name}/" ] }, { @@ -181,7 +182,7 @@ }, "outputs": [], "source": [ - "!gsutil ls -r gs://{bucket_name}/**" + "!gcloud storage ls --recursive gs://{bucket_name}/**" ] }, { @@ -209,7 +210,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil ls -L gs://{bucket_name}/us-states.txt" + "!gcloud storage ls --full gs://{bucket_name}/us-states.txt" ] }, { @@ -245,7 +246,7 @@ }, "outputs": [], "source": [ - "!gsutil cp gs://{bucket_name}/us-states.txt resources/downloaded-us-states.txt" + "!gcloud storage cp gs://{bucket_name}/us-states.txt resources/downloaded-us-states.txt" ] }, { @@ -270,7 +271,7 @@ }, "outputs": [], "source": [ - "!gsutil rm gs://{bucket_name}/us-states.txt" + "!gcloud storage rm gs://{bucket_name}/us-states.txt" ] }, { @@ -288,7 +289,7 @@ "metadata": {}, "outputs": [], "source": [ - "!gsutil rm -r gs://{bucket_name}/" + "!gcloud storage rm --recursive gs://{bucket_name}/" ] }, { diff --git a/parametermanager/snippets-adk/README.md b/parametermanager/snippets-adk/README.md new file mode 100644 index 00000000000..a2ab8a841eb --- /dev/null +++ b/parametermanager/snippets-adk/README.md @@ -0,0 +1,99 @@ +# ADK Parameter Manager Samples + +This directory contains samples demonstrating how to use the Agent Development Kit (ADK) with Google Cloud Parameter Manager. + +## Folders +* `agent_global`: Sample for accessing parameters from a global Parameter Manager instance. +* `agent_regional`: Sample for accessing parameters from a regional Parameter Manager endpoint. + +## Prerequisites + +1. **Create and activate a virtual environment**: + ```bash + python3 -m venv .venv + source .venv/bin/activate + ``` + +2. **Set up Application Default Credentials**: + ```bash + gcloud auth application-default login + ``` + +3. **Install dependencies**: + You need to install dependencies for the specific sample or test you want to run. + + For global agent samples and tests: + ```bash + pip install -r agent_global/requirements.txt + ``` + + For regional agent samples and tests: + ```bash + pip install -r agent_regional/requirements.txt + ``` + +4. **Set up environment variables**: + * `GOOGLE_CLOUD_PROJECT`: Your Google Cloud Project ID. (Required for both samples and tests). + + The following environment variables are **only required when running the samples manually**. The tests will generate and use their own temporary parameters automatically. + + * `ADK_TEST_PARAMETER_ID`: The ID of the parameter to access. + * `ADK_TEST_PARAMETER_VERSION` (Optional): The version of the parameter (defaults to `latest`). + * `GOOGLE_CLOUD_PROJECT_LOCATION` (Required for regional samples): The region where the parameter is located (e.g., `us-central1`). + +## Running the Samples + +The samples are designed to be run from this directory (`snippets-adk`) using the `adk run` command. + +### Global Parameter Manager Agent + +1. **Create a `.env` file** alongside `agent.py` in the `agent_global` directory: + ```env + GOOGLE_GENAI_USE_VERTEXAI=1 + GOOGLE_CLOUD_PROJECT=your-project-id + GOOGLE_CLOUD_LOCATION=us-central1 + ``` + *Note: Replace `your-project-id` with your GCP project ID.* + +2. **Run the agent**: + ```bash + export GOOGLE_CLOUD_PROJECT="your-project-id" + export ADK_TEST_PARAMETER_ID="your-parameter-id" + adk run agent_global + ``` + +### Regional Parameter Manager Agent + +1. **Create a `.env` file** alongside `agent.py` in the `agent_regional` directory: + ```env + GOOGLE_GENAI_USE_VERTEXAI=1 + GOOGLE_CLOUD_PROJECT=your-project-id + GOOGLE_CLOUD_LOCATION=us-central1 + ``` + *Note: Replace `your-project-id` with your GCP project ID.* + +2. **Run the agent**: + ```bash + export GOOGLE_CLOUD_PROJECT="your-project-id" + export GOOGLE_CLOUD_PROJECT_LOCATION="us-central1" + export ADK_TEST_PARAMETER_ID="your-parameter-id" + adk run agent_regional + ``` + +## Running the Tests + +The tests are located within the specific agent folders and run the samples using `adk run`. + +To run tests for the global agent: +```bash +pip install -r agent_global/requirements.txt +pip install -r agent_global/requirements-test.txt +pytest agent_global/snippets_adk_test.py +``` + +To run tests for the regional agent: +```bash +pip install -r agent_regional/requirements.txt +pip install -r agent_regional/requirements-test.txt +pytest agent_regional/snippets_adk_test.py +``` diff --git a/parametermanager/snippets-adk/agent_global/agent.py b/parametermanager/snippets-adk/agent_global/agent.py new file mode 100644 index 00000000000..3ec915375ae --- /dev/null +++ b/parametermanager/snippets-adk/agent_global/agent.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +ADK agent for accessing parameters from global Parameter Manager. +""" + +import os + +from google.adk import Agent +from google.adk.integrations.parameter_manager.parameter_client import ParameterManagerClient + +# Fetch parameter from global Parameter Manager +project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") +parameter_id = os.environ.get("ADK_TEST_PARAMETER_ID") +parameter_version = os.environ.get("ADK_TEST_PARAMETER_VERSION", "latest") + +if not project_id or not parameter_id: + raise ValueError("GOOGLE_CLOUD_PROJECT and ADK_TEST_PARAMETER_ID environment variables must be set.") + +resource_name = f"projects/{project_id}/locations/global/parameters/{parameter_id}/versions/{parameter_version}" + +print("Fetching parameter from global Parameter Manager...") +# Initialize Parameter Manager Client +client = ParameterManagerClient() + +# Fetch parameter +try: + parameter_payload = client.get_parameter(resource_name) + print("Successfully fetched parameter.") +except Exception as e: + print(f"Error fetching parameter: {e}") + raise e + +# Initialize Agent +root_agent = Agent( + model='gemini-2.5-flash', + name='root_agent', + description='A helpful assistant for user questions.', + instruction='Answer user questions to the best of your knowledge', +) + +print("Agent initialized successfully.") diff --git a/parametermanager/snippets-adk/agent_global/noxfile_config.py b/parametermanager/snippets-adk/agent_global/noxfile_config.py new file mode 100644 index 00000000000..2a42a33e5e8 --- /dev/null +++ b/parametermanager/snippets-adk/agent_global/noxfile_config.py @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # A dictionary you want to inject into your test. + "envs": {}, +} diff --git a/parametermanager/snippets-adk/agent_global/requirements-test.txt b/parametermanager/snippets-adk/agent_global/requirements-test.txt new file mode 100644 index 00000000000..b3b2b3ca4cc --- /dev/null +++ b/parametermanager/snippets-adk/agent_global/requirements-test.txt @@ -0,0 +1 @@ +pytest==9.0.3 diff --git a/parametermanager/snippets-adk/agent_global/requirements.txt b/parametermanager/snippets-adk/agent_global/requirements.txt new file mode 100644 index 00000000000..a7a4fdd4908 --- /dev/null +++ b/parametermanager/snippets-adk/agent_global/requirements.txt @@ -0,0 +1 @@ +google-adk[extensions]>=1.30 diff --git a/parametermanager/snippets-adk/agent_global/snippets_adk_test.py b/parametermanager/snippets-adk/agent_global/snippets_adk_test.py new file mode 100644 index 00000000000..57746824045 --- /dev/null +++ b/parametermanager/snippets-adk/agent_global/snippets_adk_test.py @@ -0,0 +1,112 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import time +from typing import Iterator +import uuid + +from google.api_core import exceptions +from google.cloud import parametermanager_v1 +import pytest + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.dirname(CURRENT_DIR) + + +@pytest.fixture() +def client() -> parametermanager_v1.ParameterManagerClient: + return parametermanager_v1.ParameterManagerClient() + + +@pytest.fixture() +def project_id() -> str: + return os.environ["GOOGLE_CLOUD_PROJECT"] + + +@pytest.fixture() +def parameter_id( + client: parametermanager_v1.ParameterManagerClient, project_id: str +) -> Iterator[str]: + parameter_id = f"python-adk-test-{uuid.uuid4()}" + yield parameter_id + + # Teardown: delete the version then the parameter + version_path = client.parameter_version_path(project_id, "global", parameter_id, "1") + parameter_path = client.parameter_path(project_id, "global", parameter_id) + + print("Deleting parameter version 1...") + try: + time.sleep(1) + client.delete_parameter_version(request={"name": version_path}) + except exceptions.NotFound: + pass + + print(f"Deleting parameter {parameter_id}...") + try: + time.sleep(2) + client.delete_parameter(name=parameter_path) + except exceptions.NotFound: + pass + + +def test_agent_global( + client: parametermanager_v1.ParameterManagerClient, project_id: str, parameter_id: str +) -> None: + # Create the parameter + parent = client.common_location_path(project_id, "global") + print(f"Creating parameter {parameter_id}...") + client.create_parameter( + request={ + "parent": parent, + "parameter_id": parameter_id, + } + ) + + # Add a version + parameter_path = client.parameter_path(project_id, "global", parameter_id) + print(f"Adding version to {parameter_id}...") + client.create_parameter_version( + request={ + "parent": parameter_path, + "parameter_version_id": "1", + "parameter_version": { + "payload": {"data": b"test-payload"} + }, + } + ) + + # Set environment variables required by the agent + os.environ["ADK_TEST_PARAMETER_ID"] = parameter_id + os.environ["ADK_TEST_PARAMETER_VERSION"] = "1" + + print(f"Running adk run agent_global from {PARENT_DIR}...") + # Run the sample using adk run + result = subprocess.run( + ["adk", "run", "agent_global"], + input="exit\n", + capture_output=True, + text=True, + cwd=PARENT_DIR, + ) + + print("STDOUT:") + print(result.stdout) + print("STDERR:") + print(result.stderr) + + assert result.returncode == 0 + assert "Successfully fetched parameter" in result.stdout + assert "Agent initialized successfully" in result.stdout diff --git a/parametermanager/snippets-adk/agent_regional/agent.py b/parametermanager/snippets-adk/agent_regional/agent.py new file mode 100644 index 00000000000..0c633dc8131 --- /dev/null +++ b/parametermanager/snippets-adk/agent_regional/agent.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +ADK agent for accessing parameters from regional Parameter Manager. +""" + +import os + +from google.adk import Agent +from google.adk.integrations.parameter_manager.parameter_client import ParameterManagerClient + +# Fetch parameter from regional Parameter Manager +project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") +location = os.environ.get("GOOGLE_CLOUD_PROJECT_LOCATION") +parameter_id = os.environ.get("ADK_TEST_PARAMETER_ID") +parameter_version = os.environ.get("ADK_TEST_PARAMETER_VERSION", "latest") + +if not project_id or not location or not parameter_id: + raise ValueError("GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_PROJECT_LOCATION, and ADK_TEST_PARAMETER_ID environment variables must be set.") + +resource_name = f"projects/{project_id}/locations/{location}/parameters/{parameter_id}/versions/{parameter_version}" + +print(f"Fetching parameter from regional Parameter Manager ({location})...") +# Initialize Parameter Manager Client (Regional) +client = ParameterManagerClient(location=location) + +# Fetch parameter +try: + parameter_payload = client.get_parameter(resource_name) + print("Successfully fetched parameter.") +except Exception as e: + print(f"Error fetching parameter: {e}") + raise e + +# Initialize Agent +root_agent = Agent( + model='gemini-2.5-flash', + name='root_agent', + description='A helpful assistant for user questions.', + instruction='Answer user questions to the best of your knowledge', +) + +print("Agent initialized successfully.") diff --git a/parametermanager/snippets-adk/agent_regional/noxfile_config.py b/parametermanager/snippets-adk/agent_regional/noxfile_config.py new file mode 100644 index 00000000000..2a42a33e5e8 --- /dev/null +++ b/parametermanager/snippets-adk/agent_regional/noxfile_config.py @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # A dictionary you want to inject into your test. + "envs": {}, +} diff --git a/parametermanager/snippets-adk/agent_regional/requirements-test.txt b/parametermanager/snippets-adk/agent_regional/requirements-test.txt new file mode 100644 index 00000000000..15d066af319 --- /dev/null +++ b/parametermanager/snippets-adk/agent_regional/requirements-test.txt @@ -0,0 +1 @@ +pytest==8.2.0 diff --git a/parametermanager/snippets-adk/agent_regional/requirements.txt b/parametermanager/snippets-adk/agent_regional/requirements.txt new file mode 100644 index 00000000000..a7a4fdd4908 --- /dev/null +++ b/parametermanager/snippets-adk/agent_regional/requirements.txt @@ -0,0 +1 @@ +google-adk[extensions]>=1.30 diff --git a/parametermanager/snippets-adk/agent_regional/snippets_adk_test.py b/parametermanager/snippets-adk/agent_regional/snippets_adk_test.py new file mode 100644 index 00000000000..f523a72b073 --- /dev/null +++ b/parametermanager/snippets-adk/agent_regional/snippets_adk_test.py @@ -0,0 +1,124 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import time +from typing import Iterator +import uuid + +from google.api_core import exceptions +from google.cloud import parametermanager_v1 +import pytest + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.dirname(CURRENT_DIR) + + +@pytest.fixture() +def location() -> str: + return "us-central1" + + +@pytest.fixture() +def client(location: str) -> parametermanager_v1.ParameterManagerClient: + api_endpoint = f"parametermanager.{location}.rep.googleapis.com" + return parametermanager_v1.ParameterManagerClient( + client_options={"api_endpoint": api_endpoint} + ) + + +@pytest.fixture() +def project_id() -> str: + return os.environ["GOOGLE_CLOUD_PROJECT"] + + +@pytest.fixture() +def parameter_id( + client: parametermanager_v1.ParameterManagerClient, project_id: str, location: str +) -> Iterator[str]: + parameter_id = f"python-adk-test-{uuid.uuid4()}" + yield parameter_id + + # Teardown: delete the version then the parameter + version_path = f"projects/{project_id}/locations/{location}/parameters/{parameter_id}/versions/1" + parameter_path = f"projects/{project_id}/locations/{location}/parameters/{parameter_id}" + + print("Deleting parameter version 1...") + try: + time.sleep(1) + client.delete_parameter_version(request={"name": version_path}) + except exceptions.NotFound: + pass + + print(f"Deleting parameter {parameter_id}...") + try: + time.sleep(2) + client.delete_parameter(name=parameter_path) + except exceptions.NotFound: + pass + + +def test_agent_regional( + client: parametermanager_v1.ParameterManagerClient, + project_id: str, + location: str, + parameter_id: str, +) -> None: + # Create the parameter + parent = f"projects/{project_id}/locations/{location}" + print(f"Creating regional parameter {parameter_id} in {location}...") + client.create_parameter( + request={ + "parent": parent, + "parameter_id": parameter_id, + } + ) + + # Add a version + parameter_path = f"projects/{project_id}/locations/{location}/parameters/{parameter_id}" + print(f"Adding version to {parameter_id}...") + client.create_parameter_version( + request={ + "parent": parameter_path, + "parameter_version_id": "1", + "parameter_version": { + "payload": {"data": b"test-payload"} + }, + } + ) + + # Set environment variables required by the agent + os.environ["ADK_TEST_PARAMETER_ID"] = parameter_id + os.environ["ADK_TEST_PARAMETER_VERSION"] = "1" + os.environ["GOOGLE_CLOUD_PROJECT_LOCATION"] = location + + print(f"Running adk run agent_regional from {PARENT_DIR}...") + # Run the sample using adk run + result = subprocess.run( + ["adk", "run", "agent_regional"], + input="exit\n", + capture_output=True, + text=True, + cwd=PARENT_DIR, + ) + + print("STDOUT:") + print(result.stdout) + print("STDERR:") + print(result.stderr) + + assert result.returncode == 0 + assert "Successfully fetched parameter" in result.stdout + assert "Agent initialized successfully" in result.stdout diff --git a/parametermanager/snippets/get_param_version.py b/parametermanager/snippets/get_param_version.py index dace37d53ac..aaf75a4936a 100644 --- a/parametermanager/snippets/get_param_version.py +++ b/parametermanager/snippets/get_param_version.py @@ -20,6 +20,12 @@ # [START parametermanager_get_param_version] + +# The GetParameterVersion operation retrieves parameter metadata and doesn't +# support high-volume access. To access parameter values at scale, +# see Access a parameter version (https://docs.cloud.google.com/secret-manager/parameter-manager/docs/render-parameter-version). + + def get_param_version( project_id: str, parameter_id: str, version_id: str ) -> parametermanager_v1.ParameterVersion: diff --git a/parametermanager/snippets/regional_samples/render_regional_param_version.py b/parametermanager/snippets/regional_samples/render_regional_param_version.py index 106a684bc79..5ff7e122df9 100644 --- a/parametermanager/snippets/regional_samples/render_regional_param_version.py +++ b/parametermanager/snippets/regional_samples/render_regional_param_version.py @@ -34,7 +34,8 @@ def render_regional_param_version( location_id (str): The ID of the region where the parameter is located. parameter_id (str): The ID of the parameter for which version details are to be rendered. - version_id (str): The ID of the version to be rendered. + version_id (str): The ID of the version or alias (e.g. "latest") to + be rendered. Returns: parametermanager_v1.RenderParameterVersionResponse: An object diff --git a/parametermanager/snippets/render_param_version.py b/parametermanager/snippets/render_param_version.py index 7a0cefe3298..1c9a8b64618 100644 --- a/parametermanager/snippets/render_param_version.py +++ b/parametermanager/snippets/render_param_version.py @@ -32,7 +32,7 @@ def render_param_version( project_id (str): The ID of the project where the parameter is located. parameter_id (str): The ID of the parameter for which version details are to be rendered. - version_id (str): The ID of the version to be rendered. + version_id (str): The ID of the version or alias (e.g. "latest") to be rendered. Returns: parametermanager_v1.RenderParameterVersionResponse: An object diff --git a/people-and-planet-ai/land-cover-classification/e2e_test.py b/people-and-planet-ai/land-cover-classification/e2e_test.py index c1c4aeadf9f..04dcc610126 100644 --- a/people-and-planet-ai/land-cover-classification/e2e_test.py +++ b/people-and-planet-ai/land-cover-classification/e2e_test.py @@ -57,7 +57,7 @@ def data_path(bucket_name: str) -> str: def model_path(bucket_name: str) -> str: # This is a different path than where Vertex AI saves its model. gcs_path = f"gs://{bucket_name}/pretrained-model" - conftest.run_cmd("gsutil", "-m", "cp", "-r", "./pretrained-model", gcs_path) + conftest.run_cmd("gcloud", "storage", "cp", "--recursive", "./pretrained-model", gcs_path) return gcs_path diff --git a/people-and-planet-ai/weather-forecasting/notebooks/2-dataset.ipynb b/people-and-planet-ai/weather-forecasting/notebooks/2-dataset.ipynb index d4f505d03bc..969a82e30b2 100644 --- a/people-and-planet-ai/weather-forecasting/notebooks/2-dataset.ipynb +++ b/people-and-planet-ai/weather-forecasting/notebooks/2-dataset.ipynb @@ -700,8 +700,7 @@ }, "outputs": [], "source": [ - "!gsutil ls -lh gs://{bucket}/weather/data-small" - ], + "!gcloud storage ls --long --readable-sizes gs://{bucket}/weather/data-small" ], "id": "F43OAIlrDosG" }, { diff --git a/people-and-planet-ai/weather-forecasting/notebooks/3-training.ipynb b/people-and-planet-ai/weather-forecasting/notebooks/3-training.ipynb index ab637613a91..b8882b1d34d 100644 --- a/people-and-planet-ai/weather-forecasting/notebooks/3-training.ipynb +++ b/people-and-planet-ai/weather-forecasting/notebooks/3-training.ipynb @@ -285,7 +285,7 @@ "data_path_gcs = f\"gs://{bucket}/weather/data\"\n", "\n", "!mkdir -p data-training\n", - "!gsutil -m cp {data_path_gcs}/* data-training" + "!gcloud storage cp {data_path_gcs}/* data-training" ], "metadata": { "id": "h_IUpnqvO-sa" @@ -1336,7 +1336,7 @@ "cell_type": "code", "source": [ "# Stage the `weather-model` package in Cloud Storage.\n", - "!gsutil cp serving/weather-model/dist/weather-model-1.0.0.tar.gz gs://{bucket}/weather/" + "!gcloud storage cp serving/weather-model/dist/weather-model-1.0.0.tar.gz gs://{bucket}/weather/" ], "metadata": { "id": "JA1k9ky02dsx" diff --git a/people-and-planet-ai/weather-forecasting/notebooks/4-predictions.ipynb b/people-and-planet-ai/weather-forecasting/notebooks/4-predictions.ipynb index a2d72385465..405b52a5bd3 100644 --- a/people-and-planet-ai/weather-forecasting/notebooks/4-predictions.ipynb +++ b/people-and-planet-ai/weather-forecasting/notebooks/4-predictions.ipynb @@ -336,8 +336,7 @@ "model_path_gcs = f\"gs://{bucket}/weather/model\"\n", "\n", "!mkdir -p model\n", - "!gsutil cp {model_path_gcs}/* model" - ], + "!gcloud storage cp {model_path_gcs}/* model" ], "metadata": { "id": "5w_uNjluhDMG" }, diff --git a/people-and-planet-ai/weather-forecasting/tests/predictions_tests/test_predictions.py b/people-and-planet-ai/weather-forecasting/tests/predictions_tests/test_predictions.py index 9e3f63d7949..2f4b5b90e3f 100644 --- a/people-and-planet-ai/weather-forecasting/tests/predictions_tests/test_predictions.py +++ b/people-and-planet-ai/weather-forecasting/tests/predictions_tests/test_predictions.py @@ -37,7 +37,7 @@ def test_name() -> str: @pytest.fixture(scope="session") def model_path_gcs(bucket_name: str) -> str: path_gcs = f"gs://{bucket_name}/model" - conftest.run_cmd("gsutil", "cp", "serving/model/*", path_gcs) + conftest.run_cmd("gcloud", "storage", "cp", "serving/model/*", path_gcs) return path_gcs diff --git a/people-and-planet-ai/weather-forecasting/tests/training_tests/test_training.py b/people-and-planet-ai/weather-forecasting/tests/training_tests/test_training.py index 1f921794ec6..140aef9758a 100644 --- a/people-and-planet-ai/weather-forecasting/tests/training_tests/test_training.py +++ b/people-and-planet-ai/weather-forecasting/tests/training_tests/test_training.py @@ -51,7 +51,7 @@ def data_path_gcs(bucket_name: str) -> str: inputs_batch = [inputs] * batch_size labels_batch = [labels] * batch_size np.savez_compressed(f, inputs=inputs_batch, labels=labels_batch) - conftest.run_cmd("gsutil", "cp", f.name, f"{path_gcs}/example.npz") + conftest.run_cmd("gcloud", "storage", "cp", f.name, f"{path_gcs}/example.npz") return path_gcs diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index 35fd9fa5a89..8ce4e0bff2d 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -61,7 +61,7 @@ def test_pubsub_to_gcs(): # Check for output files on GCS. gcs_client = GcsIO() - files = gcs_client.list_prefix(f"gs://{BUCKET}/pubsub/{UUID}") + files = dict(gcs_client.list_files(f"gs://{BUCKET}/pubsub/{UUID}")) assert len(files) > 0 # Clean up. diff --git a/pubsub/streaming-analytics/README.md b/pubsub/streaming-analytics/README.md index 11706cf600b..f22a6ae5a2b 100644 --- a/pubsub/streaming-analytics/README.md +++ b/pubsub/streaming-analytics/README.md @@ -26,7 +26,7 @@ Sample(s) showing how to use [Google Cloud Pub/Sub] with [Google Cloud Dataflow] gcloud init ``` -1. [Enable the APIs](https://console.cloud.google.com/flows/enableapi?apiid=dataflow,compute_component,logging,storage_component,storage_api,pubsub,cloudresourcemanager.googleapis.com,cloudscheduler.googleapis.com,appengine.googleapis.com): Dataflow, Compute Engine, Stackdriver Logging, Cloud Storage, Cloud Storage JSON, Pub/Sub, Cloud Scheduler, Cloud Resource Manager, and App Engine. +1. [Enable the APIs](https://console.cloud.google.com/flows/enableapi?apiid=dataflow,compute_component,logging,storage_component,storage_api,pubsub,cloudresourcemanager.googleapis.com,cloudscheduler.googleapis.com,appengine.googleapis.com): Dataflow, Compute Engine, Cloud Logging, Cloud Storage, Cloud Storage JSON, Pub/Sub, Cloud Scheduler, Cloud Resource Manager, and App Engine. 1. Create a service account JSON key via the [*Create service account key* page], diff --git a/pubsub/streaming-analytics/noxfile_config.py b/pubsub/streaming-analytics/noxfile_config.py index 783945807ee..17b48115e0c 100644 --- a/pubsub/streaming-analytics/noxfile_config.py +++ b/pubsub/streaming-analytics/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.6", "3.9", "3.10", "3.11", "3.12", "3.13"], + "ignored_versions": ["2.7", "3.6", "3.7", "3.8", "3.9", "3.14"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, diff --git a/pubsub/streaming-analytics/requirements.txt b/pubsub/streaming-analytics/requirements.txt index 7dac644ad0f..44955c079c5 100644 --- a/pubsub/streaming-analytics/requirements.txt +++ b/pubsub/streaming-analytics/requirements.txt @@ -1 +1 @@ -apache-beam[gcp,test]==2.42.0 +apache-beam[gcp,test]==2.72.0 diff --git a/recaptcha_enterprise/demosite/app/requirements.txt b/recaptcha_enterprise/demosite/app/requirements.txt index 16c29cd89fa..99a041bbcf9 100644 --- a/recaptcha_enterprise/demosite/app/requirements.txt +++ b/recaptcha_enterprise/demosite/app/requirements.txt @@ -1,4 +1,4 @@ -Flask==3.0.3 -gunicorn==23.0.0 -google-cloud-recaptcha-enterprise==1.25.0 -Werkzeug==3.0.3 +Flask==3.1.3 +gunicorn==25.1.0 +google-cloud-recaptcha-enterprise==1.30.0 +Werkzeug==3.1.6 diff --git a/retail/snippets/README.md b/retail/snippets/README.md new file mode 100644 index 00000000000..7cf3778d6a7 --- /dev/null +++ b/retail/snippets/README.md @@ -0,0 +1,26 @@ +# Vertex AI Search for commerce Samples + +This directory contains Python samples for [Vertex AI Search for commerce](https://cloud.google.com/retail/docs/search-basic#search). + +## Prerequisites + +To run these samples, you must have: + +1. **A Google Cloud Project** with the [Vertex AI Search for commerce API](https://console.cloud.google.com/apis/library/retail.googleapis.com) enabled. +2. **Vertex AI Search for commerce** set up with a valid catalog and serving configuration (placement). +3. **Authentication**: These samples use [Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/provide-credentials-adc). + - If running locally, you can set up ADC by running: + ```bash + gcloud auth application-default login + ``` +4. **IAM Roles**: The service account or user running the samples needs the `roles/retail.viewer` (Retail Viewer) role or higher. + +## Samples + +- **[search_request.py](search_request.py)**: Basic search request showing both text search and browse search (using categories). +- **[search_pagination.py](search_pagination.py)**: Shows how to use `next_page_token` to paginate through search results. +- **[search_offset.py](search_offset.py)**: Shows how to use `offset` to skip a specified number of results. + +## Documentation + +For more information, see the [Vertex AI Search for commerce documentation](https://docs.cloud.google.com/retail/docs/search-basic#search). diff --git a/retail/snippets/conftest.py b/retail/snippets/conftest.py new file mode 100644 index 00000000000..ff8eccf5441 --- /dev/null +++ b/retail/snippets/conftest.py @@ -0,0 +1,26 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + + +@pytest.fixture +def project_id() -> str: + """Get the Google Cloud project ID from the environment.""" + project_id = os.environ.get("BUILD_SPECIFIC_GCLOUD_PROJECT") + if not project_id: + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") + return project_id diff --git a/retail/snippets/requirements-test.txt b/retail/snippets/requirements-test.txt new file mode 100644 index 00000000000..ad59a67d1f7 --- /dev/null +++ b/retail/snippets/requirements-test.txt @@ -0,0 +1,5 @@ +pytest +pytest-xdist +mock +google-cloud-retail>=2.10.0 +google-api-core diff --git a/retail/snippets/requirements.txt b/retail/snippets/requirements.txt new file mode 100644 index 00000000000..7c213ef275a --- /dev/null +++ b/retail/snippets/requirements.txt @@ -0,0 +1 @@ +google-cloud-retail>=2.10.0 diff --git a/retail/snippets/search_offset.py b/retail/snippets/search_offset.py new file mode 100644 index 00000000000..75e19bbfd6d --- /dev/null +++ b/retail/snippets/search_offset.py @@ -0,0 +1,79 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START retail_v2_search_offset] +import sys + +from google.api_core import exceptions +from google.cloud import retail_v2 + +client = retail_v2.SearchServiceClient() + + +def search_offset( + project_id: str, + placement_id: str, + visitor_id: str, + query: str, + offset: int, +) -> None: + """Search for products with an offset using Vertex AI Search for commerce. + + Performs a search request starting from a specified position. + + Args: + project_id: The Google Cloud project ID. + placement_id: The placement name for the search. + visitor_id: A unique identifier for the user. + query: The search term. + offset: The number of results to skip. + """ + placement_path = client.serving_config_path( + project=project_id, + location="global", + catalog="default_catalog", + serving_config=placement_id, + ) + + branch_path = client.branch_path( + project=project_id, + location="global", + catalog="default_catalog", + branch="default_branch", + ) + + request = retail_v2.SearchRequest( + placement=placement_path, + branch=branch_path, + visitor_id=visitor_id, + query=query, + page_size=10, + offset=offset, + ) + + try: + response = client.search(request=request) + + print(f"--- Results for offset: {offset} ---") + for result in response: + product = result.product + print(f"Product ID: {product.id}") + print(f" Title: {product.title}") + print(f" Scores: {result.model_scores}") + + except exceptions.GoogleAPICallError as e: + print(f"error: {e.message}", file=sys.stderr) + + +# [END retail_v2_search_offset] diff --git a/retail/snippets/search_offset_test.py b/retail/snippets/search_offset_test.py new file mode 100644 index 00000000000..d5650891003 --- /dev/null +++ b/retail/snippets/search_offset_test.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from google.cloud import retail_v2 +import pytest + +from search_offset import search_offset + + +@pytest.fixture +def test_config(project_id): + return { + "project_id": project_id, + "placement_id": "default_placement", + "visitor_id": "test_visitor", + } + + +@mock.patch.object(retail_v2.SearchServiceClient, "search") +def test_search_offset(mock_search, test_config, capsys): + # Mock result + mock_product = mock.Mock() + mock_product.id = "product_at_offset" + mock_product.title = "Offset Title" + + mock_result = mock.Mock() + mock_result.product = mock_product + + mock_page = mock.MagicMock() + mock_page.results = [mock_result] + mock_pager = mock.MagicMock() + mock_pager.pages = iter([mock_page]) + mock_pager.__iter__.return_value = [mock_result] + mock_search.return_value = mock_pager + + search_offset( + project_id=test_config["project_id"], + placement_id=test_config["placement_id"], + visitor_id=test_config["visitor_id"], + query="test query", + offset=10, + ) + + out, _ = capsys.readouterr() + assert "--- Results for offset: 10 ---" in out + assert "Product ID: product_at_offset" in out + + # Verify call request + args, kwargs = mock_search.call_args + request = kwargs.get("request") or args[0] + assert request.offset == 10 + assert request.page_size == 10 + assert request.query == "test query" diff --git a/retail/snippets/search_pagination.py b/retail/snippets/search_pagination.py new file mode 100644 index 00000000000..00d3dfa5605 --- /dev/null +++ b/retail/snippets/search_pagination.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START retail_v2_search_pagination] +import sys + +from google.api_core import exceptions +from google.cloud import retail_v2 + +client = retail_v2.SearchServiceClient() + + +def search_pagination( + project_id: str, + placement_id: str, + visitor_id: str, + query: str, +) -> None: + """Search for products with pagination using Vertex AI Search for commerce. + + Performs a search request, then uses the next_page_token to get the next page. + + Args: + project_id: The Google Cloud project ID. + placement_id: The placement name for the search. + visitor_id: A unique identifier for the user. + query: The search term. + """ + placement_path = client.serving_config_path( + project=project_id, + location="global", + catalog="default_catalog", + serving_config=placement_id, + ) + + branch_path = client.branch_path( + project=project_id, + location="global", + catalog="default_catalog", + branch="default_branch", + ) + + # First page request + first_request = retail_v2.SearchRequest( + placement=placement_path, + branch=branch_path, + visitor_id=visitor_id, + query=query, + page_size=5, + ) + + try: + first_response = client.search(request=first_request) + print("--- First Page ---") + first_page = next(first_response.pages) + for result in first_page.results: + print(f"Product ID: {result.product.id}") + + next_page_token = first_response.next_page_token + + if next_page_token: + # Second page request using page_token + second_request = retail_v2.SearchRequest( + placement=placement_path, + branch=branch_path, + visitor_id=visitor_id, + query=query, + page_size=5, + page_token=next_page_token, + ) + second_response = client.search(request=second_request) + print("\n--- Second Page ---") + second_page = next(second_response.pages) + for result in second_page.results: + print(f"Product ID: {result.product.id}") + else: + print("\nNo more pages.") + + except exceptions.GoogleAPICallError as e: + print(f"error: {e.message}", file=sys.stderr) + + +# [END retail_v2_search_pagination] diff --git a/retail/snippets/search_pagination_test.py b/retail/snippets/search_pagination_test.py new file mode 100644 index 00000000000..8afe99ab50a --- /dev/null +++ b/retail/snippets/search_pagination_test.py @@ -0,0 +1,88 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from google.cloud import retail_v2 +import pytest + +from search_pagination import search_pagination + + +@pytest.fixture +def test_config(project_id): + return { + "project_id": project_id, + "placement_id": "default_placement", + "visitor_id": "test_visitor", + } + + +@mock.patch.object(retail_v2.SearchServiceClient, "search") +def test_search_pagination(mock_search, test_config, capsys): + # Mock first response + mock_product_1 = mock.Mock() + mock_product_1.id = "product_1" + + mock_result_1 = mock.Mock() + mock_result_1.product = mock_product_1 + + mock_page_1 = mock.MagicMock() + mock_page_1.results = [mock_result_1] + mock_first_response = mock.MagicMock() + mock_first_response.next_page_token = "token_for_page_2" + mock_first_response.pages = iter([mock_page_1]) + mock_first_response.__iter__.return_value = [mock_result_1] + + # Mock second response + mock_product_2 = mock.Mock() + mock_product_2.id = "product_2" + + mock_result_2 = mock.Mock() + mock_result_2.product = mock_product_2 + + mock_page_2 = mock.MagicMock() + mock_page_2.results = [mock_result_2] + mock_second_response = mock.MagicMock() + mock_second_response.next_page_token = "" + mock_second_response.pages = iter([mock_page_2]) + mock_second_response.__iter__.return_value = [mock_result_2] + + mock_search.side_effect = [mock_first_response, mock_second_response] + + search_pagination( + project_id=test_config["project_id"], + placement_id=test_config["placement_id"], + visitor_id=test_config["visitor_id"], + query="test query", + ) + + out, _ = capsys.readouterr() + assert "--- First Page ---" in out + assert "Product ID: product_1" in out + assert "--- Second Page ---" in out + assert "Product ID: product_2" in out + + # Verify calls + assert mock_search.call_count == 2 + + # Check first call request + first_call_request = mock_search.call_args_list[0].kwargs["request"] + assert first_call_request.page_size == 5 + assert not first_call_request.page_token + + # Check second call request + second_call_request = mock_search.call_args_list[1].kwargs["request"] + assert second_call_request.page_size == 5 + assert second_call_request.page_token == "token_for_page_2" diff --git a/retail/snippets/search_request.py b/retail/snippets/search_request.py new file mode 100644 index 00000000000..80784411fb9 --- /dev/null +++ b/retail/snippets/search_request.py @@ -0,0 +1,85 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START retail_v2_search_request] +import sys +from typing import List + +from google.api_core import exceptions +from google.cloud import retail_v2 + +client = retail_v2.SearchServiceClient() + + +def search_request( + project_id: str, + placement_id: str, + visitor_id: str, + query: str = "", + page_categories: List[str] = None, +) -> None: + """Search for products using Vertex AI Search for commerce. + + Performs a search request for a specific placement. + Handles both text search (using query) and browse search (using page_categories). + + Args: + project_id: The Google Cloud project ID. + placement_id: The placement name for the search. + visitor_id: A unique identifier for the user. + query: The search term for text search. + page_categories: The categories for browse search. + """ + placement_path = client.serving_config_path( + project=project_id, + location="global", + catalog="default_catalog", + serving_config=placement_id, + ) + + branch_path = client.branch_path( + project=project_id, + location="global", + catalog="default_catalog", + branch="default_branch", + ) + + request = retail_v2.SearchRequest( + placement=placement_path, + branch=branch_path, + visitor_id=visitor_id, + query=query, + page_categories=page_categories or [], + page_size=10, + ) + + try: + response = client.search(request=request) + + for result in response: + product = result.product + print(f"Product ID: {product.id}") + print(f" Title: {product.title}") + scores = dict(result.model_scores.items()) + print(f" Scores: {scores}") + + except exceptions.GoogleAPICallError as e: + print(f"error: {e.message}", file=sys.stderr) + print( + f"Troubleshooting Context: Project: {project_id}, Catalog: default_catalog", + file=sys.stderr, + ) + + +# [END retail_v2_search_request] diff --git a/retail/snippets/search_request_test.py b/retail/snippets/search_request_test.py new file mode 100644 index 00000000000..b1472cdf1a9 --- /dev/null +++ b/retail/snippets/search_request_test.py @@ -0,0 +1,121 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from google.cloud import retail_v2 +import pytest + +from search_request import search_request + + +@pytest.fixture +def test_config(project_id): + return { + "project_id": project_id, + "placement_id": "default_placement", + "visitor_id": "test_visitor", + } + + +@mock.patch.object(retail_v2.SearchServiceClient, "search") +def test_search_request_text(mock_search, test_config, capsys): + # Mock return value for search call + mock_product = mock.Mock() + mock_product.id = "test_product_id" + mock_product.title = "Test Product Title" + + mock_result = mock.Mock() + mock_result.product = mock_product + mock_result.model_scores = {"relevance": 0.95} + + mock_page = mock.MagicMock() + mock_page.results = [mock_result] + mock_pager = mock.MagicMock() + mock_pager.pages = iter([mock_page]) + mock_pager.__iter__.return_value = [mock_result] + mock_search.return_value = mock_pager + + search_request( + project_id=test_config["project_id"], + placement_id=test_config["placement_id"], + visitor_id=test_config["visitor_id"], + query="test query", + ) + + out, _ = capsys.readouterr() + assert "Product ID: test_product_id" in out + assert "Title: Test Product Title" in out + assert "Scores: {'relevance': 0.95}" in out + + # Verify that search was called with query + args, kwargs = mock_search.call_args + request = kwargs.get("request") or args[0] + assert request.query == "test query" + assert not request.page_categories + + +@mock.patch.object(retail_v2.SearchServiceClient, "search") +def test_search_request_browse(mock_search, test_config, capsys): + # Mock return value for search call + mock_product = mock.Mock() + mock_product.id = "test_browse_id" + mock_product.title = "Browse Product Title" + + mock_result = mock.Mock() + mock_result.product = mock_product + mock_result.model_scores = {"relevance": 0.8} + + mock_page = mock.MagicMock() + mock_page.results = [mock_result] + mock_pager = mock.MagicMock() + mock_pager.pages = iter([mock_page]) + mock_pager.__iter__.return_value = [mock_result] + mock_search.return_value = mock_pager + + search_request( + project_id=test_config["project_id"], + placement_id=test_config["placement_id"], + visitor_id=test_config["visitor_id"], + page_categories=["Electronics", "Laptops"], + ) + + out, _ = capsys.readouterr() + assert "Product ID: test_browse_id" in out + assert "Title: Browse Product Title" in out + assert "Scores: {'relevance': 0.8}" in out + + # Verify that search was called with page_categories + args, kwargs = mock_search.call_args + request = kwargs.get("request") or args[0] + assert not request.query + assert "Electronics" in request.page_categories + assert "Laptops" in request.page_categories + + +@mock.patch.object(retail_v2.SearchServiceClient, "search") +def test_search_request_error(mock_search, test_config, capsys): + from google.api_core import exceptions + + mock_search.side_effect = exceptions.InvalidArgument("test error") + + search_request( + project_id=test_config["project_id"], + placement_id=test_config["placement_id"], + visitor_id=test_config["visitor_id"], + ) + + _, err = capsys.readouterr() + assert "error: test error" in err + assert f"Project: {test_config['project_id']}" in err diff --git a/run/django/e2e_test_cleanup.yaml b/run/django/e2e_test_cleanup.yaml index 0006140f871..f3af8441112 100644 --- a/run/django/e2e_test_cleanup.yaml +++ b/run/django/e2e_test_cleanup.yaml @@ -22,8 +22,8 @@ steps: ./retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ "gcloud secrets delete ${_SECRET_SETTINGS_NAME} --quiet --project $PROJECT_ID" - ./retry.sh "gsutil ls gs://${_STORAGE_BUCKET}" \ - "gsutil -m rm -r gs://${_STORAGE_BUCKET}" + ./retry.sh "gcloud storage ls gs://${_STORAGE_BUCKET}" \ + "gcloud storage rm --recursive gs://${_STORAGE_BUCKET}" ./retry.sh "gcloud artifacts docker images describe ${_IMAGE_NAME}" \ "gcloud artifacts docker images delete ${_IMAGE_NAME} --quiet" diff --git a/run/django/e2e_test_setup.yaml b/run/django/e2e_test_setup.yaml index dac968f03f6..b4b2a6a634e 100644 --- a/run/django/e2e_test_setup.yaml +++ b/run/django/e2e_test_setup.yaml @@ -42,10 +42,9 @@ steps: args: - "-c" - | - ./retry.sh "gsutil mb \ - -l ${_REGION} \ - -p ${PROJECT_ID} \ - gs://${_STORAGE_BUCKET}" + ./retry.sh "gcloud storage buckets create gs://${_STORAGE_BUCKET} \ + --location ${_REGION} \ + --project ${PROJECT_ID}" - id: "IAM and Secrets" name: "gcr.io/google.com/cloudsdktool/cloud-sdk" @@ -148,4 +147,3 @@ substitutions: _DB_PASS: password1234 _ADMIN_PASSWORD: superpass _ADMIN_EMAIL: example@noop.com - diff --git a/run/logging-manual/README.md b/run/logging-manual/README.md index 378cf44efbe..ed8945290dc 100644 --- a/run/logging-manual/README.md +++ b/run/logging-manual/README.md @@ -1,6 +1,6 @@ # Cloud Run Manual Logging Sample -This sample shows how to send structured logs to Stackdriver Logging. +This sample shows how to send structured logs to Cloud Logging. [![Run in Google Cloud][run_img]][run_link] diff --git a/run/logging-manual/e2e_test.py b/run/logging-manual/e2e_test.py index a2eed020216..80aef7455e2 100644 --- a/run/logging-manual/e2e_test.py +++ b/run/logging-manual/e2e_test.py @@ -169,8 +169,8 @@ def test_end_to_end(service_url_auth_token, deployed_service): assert response.status_code == 200 assert "Hello Logger!" in response.content.decode("UTF-8") - # Test that the logs are writing properly to stackdriver - time.sleep(10) # Slight delay writing to stackdriver + # Test that the logs are writing properly to Cloud Logging + time.sleep(10) # Slight delay writing to Cloud Logging client = LoggingServiceV2Client() resource_names = [f"projects/{PROJECT}"] # We add timestamp for making the query faster. @@ -184,7 +184,7 @@ def test_end_to_end(service_url_auth_token, deployed_service): "AND jsonPayload.component=arbitrary-property" ) - # Retry a maximum number of 10 times to find results in stackdriver + # Retry a maximum number of 10 times to find results in Cloud Logging found = False for x in range(10): iterator = client.list_log_entries( diff --git a/run/logging-manual/requirements-test.txt b/run/logging-manual/requirements-test.txt index 86ed56c0141..cae36808290 100644 --- a/run/logging-manual/requirements-test.txt +++ b/run/logging-manual/requirements-test.txt @@ -1,2 +1,2 @@ pytest==8.2.0 -google-cloud-logging==3.11.4 +google-cloud-logging==3.15.0 diff --git a/run/mcp-server/README.md b/run/mcp-server/README.md index f4c61795eab..53d01ff9d8b 100644 --- a/run/mcp-server/README.md +++ b/run/mcp-server/README.md @@ -6,6 +6,8 @@ This sample uses the `streamable-http` transport, which allows for running MCP servers remotely. You can read more about MCP transports in the [official MCP docs](https://modelcontextprotocol.io/docs/concepts/architecture#transport-layer). +Run on Google Cloud + ## Benefits of running an MCP server remotely Running an MCP server remotely on Cloud Run can provide several benefits: @@ -180,3 +182,32 @@ You should see the following output: You have successfully deployed a remote MCP server to Cloud Run and tested it using the FastMCP client. + +## Observability with OpenTelemetry + +This sample includes integration with OpenTelemetry to send traces, logs, and metrics to Google +Cloud Observability (Cloud Trace, Cloud Logging, and Cloud Monitoring). + +[FastMCP is natively instrumented for +OpenTelemetry](https://gofastmcp.com/servers/telemetry#opentelemetry), so simply setting up the +SDK is enough to get telemetry data. Learn more about OpenTelemetry instrumentation +[here](https://docs.cloud.google.com/stackdriver/docs/instrumentation/overview). + + +### Setup Observability + +1. **Ensure APIs are enabled**: + Make sure you have enabled the Telemetry (OTLP) API, Cloud Logging API, and Cloud Monitoring API in your Google Cloud project. + + ```bash + gcloud services enable logging.googleapis.com monitoring.googleapis.com telemetry.googleapis.com + ``` + +1. **Run the server**: + The sample is pre-configured to use OpenTelemetry. + + * **Locally**: Run `uv run server.py`. You can test it with the client: `uv run test_server.py`. + * **Cloud Run**: Deploy using the instructions in the [Deploy](#deploy) section. The default `Dockerfile` is already set up to run the instrumented server. + +1. **View Traces**: + After interacting with the server to generate traces, you can view them in the Google Cloud Console. For detailed instructions, see the [Google Cloud Trace documentation on finding traces](https://docs.cloud.google.com/trace/docs/finding-traces). diff --git a/run/mcp-server/otel_setup.py b/run/mcp-server/otel_setup.py new file mode 100644 index 00000000000..4d06f043991 --- /dev/null +++ b/run/mcp-server/otel_setup.py @@ -0,0 +1,67 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START cloudrun_mcpserver_setup_otel] +import logging +import google.auth +import google.auth.transport.requests +import grpc +from google.auth.transport.grpc import AuthMetadataPlugin +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +logger = logging.getLogger(__name__) + + +def setup_opentelemetry(service_name: str) -> None: + """Sets up OpenTelemetry to send traces to Google Cloud Observability.""" + credentials, project_id = google.auth.default() + if not project_id: + raise Exception("Could not determine Google Cloud project ID.") + + resource = Resource.create( + attributes={ + SERVICE_NAME: service_name, + "gcp.project_id": project_id, + } + ) + + # Set up OTLP auth + request = google.auth.transport.requests.Request() + auth_metadata_plugin = AuthMetadataPlugin(credentials=credentials, request=request) + channel_creds = grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials(auth_metadata_plugin), + ) + + # Set up OpenTelemetry Python SDK + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter( + credentials=channel_creds, + endpoint="https://telemetry.googleapis.com:443/v1/traces", + ) + ) + ) + trace.set_tracer_provider(tracer_provider) + logger.info("OpenTelemetry successfully initialized.") + + +# [END cloudrun_mcpserver_setup_otel] diff --git a/run/mcp-server/pyproject.toml b/run/mcp-server/pyproject.toml index 32fdf3743bb..d104884f089 100644 --- a/run/mcp-server/pyproject.toml +++ b/run/mcp-server/pyproject.toml @@ -5,5 +5,11 @@ description = "Example of deploying an MCP server on Cloud Run" readme = "README.md" requires-python = ">=3.10" dependencies = [ - "fastmcp==3.0.0", + "fastmcp==3.2.0", + "opentelemetry-api==1.40.0", + "opentelemetry-sdk==1.40.0", + "opentelemetry-exporter-otlp-proto-grpc==1.40.0", + "google-auth==2.49.1", + "grpcio==1.80.0", ] + diff --git a/run/mcp-server/server.py b/run/mcp-server/server.py index 7068b2731ba..37f9ad68fc5 100644 --- a/run/mcp-server/server.py +++ b/run/mcp-server/server.py @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# [START cloudrun_mcpserver_otel] +from otel_setup import setup_opentelemetry +setup_opentelemetry("mcp-server") + # [START cloudrun_mcpserver] import asyncio import logging @@ -64,3 +68,4 @@ def subtract(a: int, b: int) -> int: ) # [END cloudrun_mcpserver] +# [END cloudrun_mcpserver_otel] diff --git a/run/mcp-server/test_server.py b/run/mcp-server/test_server.py index ec7cfe2ab38..e566ffe5eec 100644 --- a/run/mcp-server/test_server.py +++ b/run/mcp-server/test_server.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +# [START cloudrun_mcpserver_test_otel] +from otel_setup import setup_opentelemetry +setup_opentelemetry("test-server") + # [START cloudrun_mcpserver_test] import asyncio from fastmcp import Client + async def test_server(): # Test the MCP server using streamable-http transport. # Use "/sse" endpoint if using sse transport. @@ -28,12 +33,14 @@ async def test_server(): # Call add tool print(">>> 🪛 Calling add tool for 1 + 2") result = await client.call_tool("add", {"a": 1, "b": 2}) - print(f"<<< ✅ Result: {result[0].text}") + print(f"<<< ✅ Result: {result.content[0].text}") # Call subtract tool print(">>> 🪛 Calling subtract tool for 10 - 3") result = await client.call_tool("subtract", {"a": 10, "b": 3}) - print(f"<<< ✅ Result: {result[0].text}") + print(f"<<< ✅ Result: {result.content[0].text}") + if __name__ == "__main__": asyncio.run(test_server()) # [END cloudrun_mcpserver_test] +# [END cloudrun_mcpserver_test_otel] \ No newline at end of file diff --git a/run/mcp-server/uv.lock b/run/mcp-server/uv.lock index 517177da22d..4d787934105 100644 --- a/run/mcp-server/uv.lock +++ b/run/mcp-server/uv.lock @@ -1,6 +1,12 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version >= '3.11' and python_full_version < '3.13'", + "python_full_version < '3.11'", +] [[package]] name = "aiofile" @@ -94,14 +100,24 @@ sdist = { url = "https://files.pythonhosted.org/packages/92/88/b8527e1b00c1811db wheels = [ { url = "https://files.pythonhosted.org/packages/6a/80/ea4ead0c5d52a9828692e7df20f0eafe8d26e671ce4883a0a146bb91049e/caio-0.9.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ca6c8ecda611478b6016cb94d23fd3eb7124852b985bdec7ecaad9f3116b9619", size = 36836, upload-time = "2025-12-26T15:22:04.662Z" }, { url = "https://files.pythonhosted.org/packages/17/b9/36715c97c873649d1029001578f901b50250916295e3dddf20c865438865/caio-0.9.25-cp310-cp310-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db9b5681e4af8176159f0d6598e73b2279bb661e718c7ac23342c550bd78c241", size = 79695, upload-time = "2025-12-26T15:22:18.818Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ab/07080ecb1adb55a02cbd8ec0126aa8e43af343ffabb6a71125b42670e9a1/caio-0.9.25-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:bf61d7d0c4fd10ffdd98ca47f7e8db4d7408e74649ffaf4bef40b029ada3c21b", size = 79457, upload-time = "2026-03-04T22:08:16.024Z" }, + { url = "https://files.pythonhosted.org/packages/88/95/dd55757bb671eb4c376e006c04e83beb413486821f517792ea603ef216e9/caio-0.9.25-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:ab52e5b643f8bbd64a0605d9412796cd3464cb8ca88593b13e95a0f0b10508ae", size = 77705, upload-time = "2026-03-04T22:08:17.202Z" }, { url = "https://files.pythonhosted.org/packages/ec/90/543f556fcfcfa270713eef906b6352ab048e1e557afec12925c991dc93c2/caio-0.9.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d6956d9e4a27021c8bd6c9677f3a59eb1d820cc32d0343cea7961a03b1371965", size = 36839, upload-time = "2025-12-26T15:21:40.267Z" }, { url = "https://files.pythonhosted.org/packages/51/3b/36f3e8ec38dafe8de4831decd2e44c69303d2a3892d16ceda42afed44e1b/caio-0.9.25-cp311-cp311-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf84bfa039f25ad91f4f52944452a5f6f405e8afab4d445450978cd6241d1478", size = 80255, upload-time = "2025-12-26T15:22:20.271Z" }, + { url = "https://files.pythonhosted.org/packages/df/ce/65e64867d928e6aff1b4f0e12dba0ef6d5bf412c240dc1df9d421ac10573/caio-0.9.25-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:ae3d62587332bce600f861a8de6256b1014d6485cfd25d68c15caf1611dd1f7c", size = 80052, upload-time = "2026-03-04T22:08:20.402Z" }, + { url = "https://files.pythonhosted.org/packages/46/90/e278863c47e14ec58309aa2e38a45882fbe67b4cc29ec9bc8f65852d3e45/caio-0.9.25-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:fc220b8533dcf0f238a6b1a4a937f92024c71e7b10b5a2dfc1c73604a25709bc", size = 78273, upload-time = "2026-03-04T22:08:21.368Z" }, { url = "https://files.pythonhosted.org/packages/d3/25/79c98ebe12df31548ba4eaf44db11b7cad6b3e7b4203718335620939083c/caio-0.9.25-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fb7ff95af4c31ad3f03179149aab61097a71fd85e05f89b4786de0359dffd044", size = 36983, upload-time = "2025-12-26T15:21:36.075Z" }, { url = "https://files.pythonhosted.org/packages/a3/2b/21288691f16d479945968a0a4f2856818c1c5be56881d51d4dac9b255d26/caio-0.9.25-cp312-cp312-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:97084e4e30dfa598449d874c4d8e0c8d5ea17d2f752ef5e48e150ff9d240cd64", size = 82012, upload-time = "2025-12-26T15:22:20.983Z" }, + { url = "https://files.pythonhosted.org/packages/03/c4/8a1b580875303500a9c12b9e0af58cb82e47f5bcf888c2457742a138273c/caio-0.9.25-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:4fa69eba47e0f041b9d4f336e2ad40740681c43e686b18b191b6c5f4c5544bfb", size = 81502, upload-time = "2026-03-04T22:08:22.381Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/0fe770b8ffc8362c48134d1592d653a81a3d8748d764bec33864db36319d/caio-0.9.25-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:6bebf6f079f1341d19f7386db9b8b1f07e8cc15ae13bfdaff573371ba0575d69", size = 80200, upload-time = "2026-03-04T22:08:23.382Z" }, { url = "https://files.pythonhosted.org/packages/31/57/5e6ff127e6f62c9f15d989560435c642144aa4210882f9494204bc892305/caio-0.9.25-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d6c2a3411af97762a2b03840c3cec2f7f728921ff8adda53d7ea2315a8563451", size = 36979, upload-time = "2025-12-26T15:21:35.484Z" }, { url = "https://files.pythonhosted.org/packages/a3/9f/f21af50e72117eb528c422d4276cbac11fb941b1b812b182e0a9c70d19c5/caio-0.9.25-cp313-cp313-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0998210a4d5cd5cb565b32ccfe4e53d67303f868a76f212e002a8554692870e6", size = 81900, upload-time = "2025-12-26T15:22:21.919Z" }, + { url = "https://files.pythonhosted.org/packages/9c/12/c39ae2a4037cb10ad5eb3578eb4d5f8c1a2575c62bba675f3406b7ef0824/caio-0.9.25-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:1a177d4777141b96f175fe2c37a3d96dec7911ed9ad5f02bac38aaa1c936611f", size = 81523, upload-time = "2026-03-04T22:08:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/22/59/f8f2e950eb4f1a5a3883e198dca514b9d475415cb6cd7b78b9213a0dd45a/caio-0.9.25-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:9ed3cfb28c0e99fec5e208c934e5c157d0866aa9c32aa4dc5e9b6034af6286b7", size = 80243, upload-time = "2026-03-04T22:08:26.449Z" }, { url = "https://files.pythonhosted.org/packages/69/ca/a08fdc7efdcc24e6a6131a93c85be1f204d41c58f474c42b0670af8c016b/caio-0.9.25-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fab6078b9348e883c80a5e14b382e6ad6aabbc4429ca034e76e730cf464269db", size = 36978, upload-time = "2025-12-26T15:21:41.055Z" }, { url = "https://files.pythonhosted.org/packages/5e/6c/d4d24f65e690213c097174d26eda6831f45f4734d9d036d81790a27e7b78/caio-0.9.25-cp314-cp314-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44a6b58e52d488c75cfaa5ecaa404b2b41cc965e6c417e03251e868ecd5b6d77", size = 81832, upload-time = "2025-12-26T15:22:22.757Z" }, + { url = "https://files.pythonhosted.org/packages/87/a4/e534cf7d2d0e8d880e25dd61e8d921ffcfe15bd696734589826f5a2df727/caio-0.9.25-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:628a630eb7fb22381dd8e3c8ab7f59e854b9c806639811fc3f4310c6bd711d79", size = 81565, upload-time = "2026-03-04T22:08:27.483Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ed/bf81aeac1d290017e5e5ac3e880fd56ee15e50a6d0353986799d1bc5cfd5/caio-0.9.25-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:0ba16aa605ccb174665357fc729cf500679c2d94d5f1458a6f0d5ca48f2060a7", size = 80071, upload-time = "2026-03-04T22:08:28.751Z" }, { url = "https://files.pythonhosted.org/packages/86/93/1f76c8d1bafe3b0614e06b2195784a3765bbf7b0a067661af9e2dd47fc33/caio-0.9.25-py3-none-any.whl", hash = "sha256:06c0bb02d6b929119b1cfbe1ca403c768b2013a369e2db46bfa2a5761cf82e40", size = 19087, upload-time = "2025-12-26T15:22:00.221Z" }, ] @@ -399,7 +415,7 @@ wheels = [ [[package]] name = "fastmcp" -version = "3.0.0" +version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "authlib" }, @@ -419,13 +435,100 @@ dependencies = [ { name = "python-dotenv" }, { name = "pyyaml" }, { name = "rich" }, + { name = "uncalled-for" }, { name = "uvicorn" }, { name = "watchfiles" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/be/beb5d3e485983b9dd122f3f74772bcceeb085ca824e11c52c14ba71cf21a/fastmcp-3.0.0.tar.gz", hash = "sha256:f3b0cfa012f6b2b50b877da181431c6f9a551197f466b0bb7de7f39ceae159a1", size = 16093079, upload-time = "2026-02-18T21:25:34.461Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/32/4f1b2cfd7b50db89114949f90158b1dcc2c92a1917b9f57c0ff24e47a2f4/fastmcp-3.2.0.tar.gz", hash = "sha256:d4830b8ffc3592d3d9c76dc0f398904cf41f04910e41a0de38cc1004e0903bef", size = 26318581, upload-time = "2026-03-30T20:25:37.692Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/67/684fa2d2de1e7504549d4ca457b4f854ccec3cd3be03bd86b33b599fbf58/fastmcp-3.2.0-py3-none-any.whl", hash = "sha256:e71aba3df16f86f546a4a9e513261d3233bcc92bef0dfa647bac3fa33623f681", size = 705550, upload-time = "2026-03-30T20:25:35.499Z" }, +] + +[[package]] +name = "google-auth" +version = "2.49.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/80/6a696a07d3d3b0a92488933532f03dbefa4a24ab80fb231395b9a2a1be77/google_auth-2.49.1.tar.gz", hash = "sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64", size = 333825, upload-time = "2026-03-12T19:30:58.135Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/14/05bebaf3764ea71ce6fa9d3fcf870610bbc8b1e7be2505e870d709375316/fastmcp-3.0.0-py3-none-any.whl", hash = "sha256:561d537cb789f995174c5591f1b54f758ce3f82da3cd951ffe51ce18609569e9", size = 603327, upload-time = "2026-02-18T21:25:36.701Z" }, + { url = "https://files.pythonhosted.org/packages/e9/eb/c6c2478d8a8d633460be40e2a8a6f8f429171997a35a96f81d3b680dec83/google_auth-2.49.1-py3-none-any.whl", hash = "sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7", size = 240737, upload-time = "2026-03-12T19:30:53.159Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, +] + +[[package]] +name = "grpcio" +version = "1.80.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/cd/bb7b7e54084a344c03d68144450da7ddd5564e51a298ae1662de65f48e2d/grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c", size = 6050363, upload-time = "2026-03-30T08:46:20.894Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/1417f5c3460dea65f7a2e3c14e8b31e77f7ffb730e9bfadd89eda7a9f477/grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388", size = 12026037, upload-time = "2026-03-30T08:46:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/43/98/c910254eedf2cae368d78336a2de0678e66a7317d27c02522392f949b5c6/grpcio-1.80.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02", size = 6602306, upload-time = "2026-03-30T08:46:27.593Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f8/88ca4e78c077b2b2113d95da1e1ab43efd43d723c9a0397d26529c2c1a56/grpcio-1.80.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc", size = 7301535, upload-time = "2026-03-30T08:46:29.556Z" }, + { url = "https://files.pythonhosted.org/packages/f9/96/f28660fe2fe0f153288bf4a04e4910b7309d442395135c88ed4f5b3b8b40/grpcio-1.80.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a", size = 6808669, upload-time = "2026-03-30T08:46:31.984Z" }, + { url = "https://files.pythonhosted.org/packages/47/eb/3f68a5e955779c00aeef23850e019c1c1d0e032d90633ba49c01ad5a96e0/grpcio-1.80.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9", size = 7409489, upload-time = "2026-03-30T08:46:34.684Z" }, + { url = "https://files.pythonhosted.org/packages/5b/a7/d2f681a4bfb881be40659a309771f3bdfbfdb1190619442816c3f0ffc079/grpcio-1.80.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199", size = 8423167, upload-time = "2026-03-30T08:46:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/97/8a/29b4589c204959aa35ce5708400a05bba72181807c45c47b3ec000c39333/grpcio-1.80.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81", size = 7846761, upload-time = "2026-03-30T08:46:40.091Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d2/ed143e097230ee121ac5848f6ff14372dba91289b10b536d54fb1b7cbae7/grpcio-1.80.0-cp310-cp310-win32.whl", hash = "sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069", size = 4156534, upload-time = "2026-03-30T08:46:42.026Z" }, + { url = "https://files.pythonhosted.org/packages/d5/c9/df8279bb49b29409995e95efa85b72973d62f8aeff89abee58c91f393710/grpcio-1.80.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58", size = 4889869, upload-time = "2026-03-30T08:46:44.219Z" }, + { url = "https://files.pythonhosted.org/packages/5d/db/1d56e5f5823257b291962d6c0ce106146c6447f405b60b234c4f222a7cde/grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a", size = 6055009, upload-time = "2026-03-30T08:46:46.265Z" }, + { url = "https://files.pythonhosted.org/packages/6e/18/c83f3cad64c5ca63bca7e91e5e46b0d026afc5af9d0a9972472ceba294b3/grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060", size = 12035295, upload-time = "2026-03-30T08:46:49.099Z" }, + { url = "https://files.pythonhosted.org/packages/0f/8e/e14966b435be2dda99fbe89db9525ea436edc79780431a1c2875a3582644/grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2", size = 6610297, upload-time = "2026-03-30T08:46:52.123Z" }, + { url = "https://files.pythonhosted.org/packages/cc/26/d5eb38f42ce0e3fdc8174ea4d52036ef8d58cc4426cb800f2610f625dd75/grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21", size = 7300208, upload-time = "2026-03-30T08:46:54.859Z" }, + { url = "https://files.pythonhosted.org/packages/25/51/bd267c989f85a17a5b3eea65a6feb4ff672af41ca614e5a0279cc0ea381c/grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab", size = 6813442, upload-time = "2026-03-30T08:46:57.056Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d9/d80eef735b19e9169e30164bbf889b46f9df9127598a83d174eb13a48b26/grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1", size = 7414743, upload-time = "2026-03-30T08:46:59.682Z" }, + { url = "https://files.pythonhosted.org/packages/de/f2/567f5bd5054398ed6b0509b9a30900376dcf2786bd936812098808b49d8d/grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106", size = 8426046, upload-time = "2026-03-30T08:47:02.474Z" }, + { url = "https://files.pythonhosted.org/packages/62/29/73ef0141b4732ff5eacd68430ff2512a65c004696997f70476a83e548e7e/grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6", size = 7851641, upload-time = "2026-03-30T08:47:05.462Z" }, + { url = "https://files.pythonhosted.org/packages/46/69/abbfa360eb229a8623bab5f5a4f8105e445bd38ce81a89514ba55d281ad0/grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440", size = 4154368, upload-time = "2026-03-30T08:47:08.027Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d4/ae92206d01183b08613e846076115f5ac5991bae358d2a749fa864da5699/grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9", size = 4894235, upload-time = "2026-03-30T08:47:10.839Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e8/a2b749265eb3415abc94f2e619bbd9e9707bebdda787e61c593004ec927a/grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0", size = 6015616, upload-time = "2026-03-30T08:47:13.428Z" }, + { url = "https://files.pythonhosted.org/packages/3e/97/b1282161a15d699d1e90c360df18d19165a045ce1c343c7f313f5e8a0b77/grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2", size = 12014204, upload-time = "2026-03-30T08:47:15.873Z" }, + { url = "https://files.pythonhosted.org/packages/6e/5e/d319c6e997b50c155ac5a8cb12f5173d5b42677510e886d250d50264949d/grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de", size = 6563866, upload-time = "2026-03-30T08:47:18.588Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f6/fdd975a2cb4d78eb67769a7b3b3830970bfa2e919f1decf724ae4445f42c/grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921", size = 7273060, upload-time = "2026-03-30T08:47:21.113Z" }, + { url = "https://files.pythonhosted.org/packages/db/f0/a3deb5feba60d9538a962913e37bd2e69a195f1c3376a3dd44fe0427e996/grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411", size = 6782121, upload-time = "2026-03-30T08:47:23.827Z" }, + { url = "https://files.pythonhosted.org/packages/ca/84/36c6dcfddc093e108141f757c407902a05085e0c328007cb090d56646cdf/grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd", size = 7383811, upload-time = "2026-03-30T08:47:26.517Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ef/f3a77e3dc5b471a0ec86c564c98d6adfa3510d38f8ee99010410858d591e/grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f", size = 8393860, upload-time = "2026-03-30T08:47:29.439Z" }, + { url = "https://files.pythonhosted.org/packages/9b/8d/9d4d27ed7f33d109c50d6b5ce578a9914aa68edab75d65869a17e630a8d1/grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f", size = 7830132, upload-time = "2026-03-30T08:47:33.254Z" }, + { url = "https://files.pythonhosted.org/packages/14/e4/9990b41c6d7a44e1e9dee8ac11d7a9802ba1378b40d77468a7761d1ad288/grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193", size = 4140904, upload-time = "2026-03-30T08:47:35.319Z" }, + { url = "https://files.pythonhosted.org/packages/2f/2c/296f6138caca1f4b92a31ace4ae1b87dab692fc16a7a3417af3bb3c805bf/grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff", size = 4880944, upload-time = "2026-03-30T08:47:37.831Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/7c3c25789e3f069e581dc342e03613c5b1cb012c4e8c7d9d5cf960a75856/grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad", size = 6017243, upload-time = "2026-03-30T08:47:40.075Z" }, + { url = "https://files.pythonhosted.org/packages/04/19/21a9806eb8240e174fd1ab0cd5b9aa948bb0e05c2f2f55f9d5d7405e6d08/grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0", size = 12010840, upload-time = "2026-03-30T08:47:43.11Z" }, + { url = "https://files.pythonhosted.org/packages/18/3a/23347d35f76f639e807fb7a36fad3068aed100996849a33809591f26eca6/grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f", size = 6567644, upload-time = "2026-03-30T08:47:46.806Z" }, + { url = "https://files.pythonhosted.org/packages/ff/40/96e07ecb604a6a67ae6ab151e3e35b132875d98bc68ec65f3e5ab3e781d7/grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6", size = 7277830, upload-time = "2026-03-30T08:47:49.643Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e2/da1506ecea1f34a5e365964644b35edef53803052b763ca214ba3870c856/grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140", size = 6783216, upload-time = "2026-03-30T08:47:52.817Z" }, + { url = "https://files.pythonhosted.org/packages/44/83/3b20ff58d0c3b7f6caaa3af9a4174d4023701df40a3f39f7f1c8e7c48f9d/grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d", size = 7385866, upload-time = "2026-03-30T08:47:55.687Z" }, + { url = "https://files.pythonhosted.org/packages/47/45/55c507599c5520416de5eefecc927d6a0d7af55e91cfffb2e410607e5744/grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7", size = 8391602, upload-time = "2026-03-30T08:47:58.303Z" }, + { url = "https://files.pythonhosted.org/packages/10/bb/dd06f4c24c01db9cf11341b547d0a016b2c90ed7dbbb086a5710df7dd1d7/grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7", size = 7826752, upload-time = "2026-03-30T08:48:01.311Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1e/9d67992ba23371fd63d4527096eb8c6b76d74d52b500df992a3343fd7251/grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294", size = 4142310, upload-time = "2026-03-30T08:48:04.594Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e6/283326a27da9e2c3038bc93eeea36fb118ce0b2d03922a9cda6688f53c5b/grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50", size = 4882833, upload-time = "2026-03-30T08:48:07.363Z" }, + { url = "https://files.pythonhosted.org/packages/c5/6d/e65307ce20f5a09244ba9e9d8476e99fb039de7154f37fb85f26978b59c3/grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e", size = 6017376, upload-time = "2026-03-30T08:48:10.005Z" }, + { url = "https://files.pythonhosted.org/packages/69/10/9cef5d9650c72625a699c549940f0abb3c4bfdb5ed45a5ce431f92f31806/grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f", size = 12018133, upload-time = "2026-03-30T08:48:12.927Z" }, + { url = "https://files.pythonhosted.org/packages/04/82/983aabaad82ba26113caceeb9091706a0696b25da004fe3defb5b346e15b/grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9", size = 6574748, upload-time = "2026-03-30T08:48:16.386Z" }, + { url = "https://files.pythonhosted.org/packages/07/d7/031666ef155aa0bf399ed7e19439656c38bbd143779ae0861b038ce82abd/grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14", size = 7277711, upload-time = "2026-03-30T08:48:19.627Z" }, + { url = "https://files.pythonhosted.org/packages/e8/43/f437a78f7f4f1d311804189e8f11fb311a01049b2e08557c1068d470cb2e/grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05", size = 6785372, upload-time = "2026-03-30T08:48:22.373Z" }, + { url = "https://files.pythonhosted.org/packages/93/3d/f6558e9c6296cb4227faa5c43c54a34c68d32654b829f53288313d16a86e/grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1", size = 7395268, upload-time = "2026-03-30T08:48:25.638Z" }, + { url = "https://files.pythonhosted.org/packages/06/21/0fdd77e84720b08843c371a2efa6f2e19dbebf56adc72df73d891f5506f0/grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f", size = 8392000, upload-time = "2026-03-30T08:48:28.974Z" }, + { url = "https://files.pythonhosted.org/packages/f5/68/67f4947ed55d2e69f2cc199ab9fd85e0a0034d813bbeef84df6d2ba4d4b7/grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e", size = 7828477, upload-time = "2026-03-30T08:48:32.054Z" }, + { url = "https://files.pythonhosted.org/packages/44/b6/8d4096691b2e385e8271911a0de4f35f0a6c7d05aff7098e296c3de86939/grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae", size = 4218563, upload-time = "2026-03-30T08:48:34.538Z" }, + { url = "https://files.pythonhosted.org/packages/e5/8c/bbe6baf2557262834f2070cf668515fa308b2d38a4bbf771f8f7872a7036/grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f", size = 5019457, upload-time = "2026-03-30T08:48:37.308Z" }, ] [[package]] @@ -652,10 +755,22 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "fastmcp" }, + { name = "google-auth" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-sdk" }, ] [package.metadata] -requires-dist = [{ name = "fastmcp", specifier = "==3.0.0" }] +requires-dist = [ + { name = "fastmcp", specifier = "==3.2.0" }, + { name = "google-auth", specifier = "==2.49.1" }, + { name = "grpcio", specifier = "==1.80.0" }, + { name = "opentelemetry-api", specifier = "==1.40.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = "==1.40.0" }, + { name = "opentelemetry-sdk", specifier = "==1.40.0" }, +] [[package]] name = "mdurl" @@ -689,15 +804,84 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.39.1" +version = "1.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/b9/3161be15bb8e3ad01be8be5a968a9237c3027c5be504362ff800fca3e442/opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c", size = 65767, upload-time = "2025-12-11T13:32:39.182Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/1d/4049a9e8698361cc1a1aa03a6c59e4fa4c71e0c0f94a30f988a6876a2ae6/opentelemetry_api-1.40.0.tar.gz", hash = "sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f", size = 70851, upload-time = "2026-03-04T14:17:21.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/bf/93795954016c522008da367da292adceed71cca6ee1717e1d64c83089099/opentelemetry_api-1.40.0-py3-none-any.whl", hash = "sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9", size = 68676, upload-time = "2026-03-04T14:17:01.24Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/bc/1559d46557fe6eca0b46c88d4c2676285f1f3be2e8d06bb5d15fbffc814a/opentelemetry_exporter_otlp_proto_common-1.40.0.tar.gz", hash = "sha256:1cbee86a4064790b362a86601ee7934f368b81cd4cc2f2e163902a6e7818a0fa", size = 20416, upload-time = "2026-03-04T14:17:23.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ca/8f122055c97a932311a3f640273f084e738008933503d0c2563cd5d591fc/opentelemetry_exporter_otlp_proto_common-1.40.0-py3-none-any.whl", hash = "sha256:7081ff453835a82417bf38dccf122c827c3cbc94f2079b03bba02a3165f25149", size = 18369, upload-time = "2026-03-04T14:17:04.796Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/7f/b9e60435cfcc7590fa87436edad6822240dddbc184643a2a005301cc31f4/opentelemetry_exporter_otlp_proto_grpc-1.40.0.tar.gz", hash = "sha256:bd4015183e40b635b3dab8da528b27161ba83bf4ef545776b196f0fb4ec47740", size = 25759, upload-time = "2026-03-04T14:17:24.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/6f/7ee0980afcbdcd2d40362da16f7f9796bd083bf7f0b8e038abfbc0300f5d/opentelemetry_exporter_otlp_proto_grpc-1.40.0-py3-none-any.whl", hash = "sha256:2aa0ca53483fe0cf6405087a7491472b70335bc5c7944378a0a8e72e86995c52", size = 20304, upload-time = "2026-03-04T14:17:05.942Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/77/dd38991db037fdfce45849491cb61de5ab000f49824a00230afb112a4392/opentelemetry_proto-1.40.0.tar.gz", hash = "sha256:03f639ca129ba513f5819810f5b1f42bcb371391405d99c168fe6937c62febcd", size = 45667, upload-time = "2026-03-04T14:17:31.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/b2/189b2577dde745b15625b3214302605b1353436219d42b7912e77fa8dc24/opentelemetry_proto-1.40.0-py3-none-any.whl", hash = "sha256:266c4385d88923a23d63e353e9761af0f47a6ed0d486979777fe4de59dc9b25f", size = 72073, upload-time = "2026-03-04T14:17:16.673Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/fd/3c3125b20ba18ce2155ba9ea74acb0ae5d25f8cd39cfd37455601b7955cc/opentelemetry_sdk-1.40.0.tar.gz", hash = "sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2", size = 184252, upload-time = "2026-03-04T14:17:31.87Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/df/d3f1ddf4bb4cb50ed9b1139cc7b1c54c34a1e7ce8fd1b9a37c0d1551a6bd/opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950", size = 66356, upload-time = "2025-12-11T13:32:17.304Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c5/6a852903d8bfac758c6dc6e9a68b015d3c33f2f1be5e9591e0f4b69c7e0a/opentelemetry_sdk-1.40.0-py3-none-any.whl", hash = "sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1", size = 141951, upload-time = "2026-03-04T14:17:17.961Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/c0/4ae7973f3c2cfd2b6e321f1675626f0dab0a97027cc7a297474c9c8f3d04/opentelemetry_semantic_conventions-0.61b0.tar.gz", hash = "sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a", size = 145755, upload-time = "2026-03-04T14:17:32.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/37/cc6a55e448deaa9b27377d087da8615a3416d8ad523d5960b78dbeadd02a/opentelemetry_semantic_conventions-0.61b0-py3-none-any.whl", hash = "sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2", size = 231621, upload-time = "2026-03-04T14:17:19.33Z" }, ] [[package]] @@ -727,6 +911,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, ] +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.4" @@ -752,6 +951,27 @@ memory = [ { name = "cachetools" }, ] +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "2.22" @@ -943,11 +1163,11 @@ wheels = [ [[package]] name = "python-dotenv" -version = "1.1.0" +version = "1.2.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/88/2c/7bb1416c5620485aa793f2de31d3df393d3686aa8a8506d11e10e13c5baf/python_dotenv-1.1.0.tar.gz", hash = "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5", size = 39920, upload-time = "2025-03-25T10:14:56.835Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/18/98a99ad95133c6a6e2005fe89faedf294a748bd5dc803008059409ac9b1e/python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d", size = 20256, upload-time = "2025-03-25T10:14:55.034Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] [[package]] @@ -1268,14 +1488,15 @@ wheels = [ [[package]] name = "starlette" -version = "0.47.0" +version = "0.49.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/d0/0332bd8a25779a0e2082b0e179805ad39afad642938b371ae0882e7f880d/starlette-0.47.0.tar.gz", hash = "sha256:1f64887e94a447fed5f23309fb6890ef23349b7e478faa7b24a851cd4eb844af", size = 2582856, upload-time = "2025-05-29T15:45:27.628Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/3f/507c21db33b66fb027a332f2cb3abbbe924cc3a79ced12f01ed8645955c9/starlette-0.49.1.tar.gz", hash = "sha256:481a43b71e24ed8c43b11ea02f5353d77840e01480881b8cb5a26b8cae64a8cb", size = 2654703, upload-time = "2025-10-28T17:34:10.928Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/81/c60b35fe9674f63b38a8feafc414fca0da378a9dbd5fa1e0b8d23fcc7a9b/starlette-0.47.0-py3-none-any.whl", hash = "sha256:9d052d4933683af40ffd47c7465433570b4949dc937e20ad1d73b34e72f10c37", size = 72796, upload-time = "2025-05-29T15:45:26.305Z" }, + { url = "https://files.pythonhosted.org/packages/51/da/545b75d420bb23b5d494b0517757b351963e974e79933f01e05c929f20a6/starlette-0.49.1-py3-none-any.whl", hash = "sha256:d92ce9f07e4a3caa3ac13a79523bd18e3bc0042bb8ff2d759a8e7dd0e1859875", size = 74175, upload-time = "2025-10-28T17:34:09.13Z" }, ] [[package]] @@ -1348,6 +1569,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "uncalled-for" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/7c/b5b7d8136f872e3f13b0584e576886de0489d7213a12de6bebf29ff6ebfc/uncalled_for-0.2.0.tar.gz", hash = "sha256:b4f8fdbcec328c5a113807d653e041c5094473dd4afa7c34599ace69ccb7e69f", size = 49488, upload-time = "2026-02-27T17:40:58.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/7f/4320d9ce3be404e6310b915c3629fe27bf1e2f438a1a7a3cb0396e32e9a9/uncalled_for-0.2.0-py3-none-any.whl", hash = "sha256:2c0bd338faff5f930918f79e7eb9ff48290df2cb05fcc0b40a7f334e55d4d85f", size = 11351, upload-time = "2026-02-27T17:40:56.804Z" }, +] + [[package]] name = "urllib3" version = "2.5.0" diff --git a/run/pubsub/e2e_test.py b/run/pubsub/e2e_test.py index 917657cd5da..3bb78a73566 100644 --- a/run/pubsub/e2e_test.py +++ b/run/pubsub/e2e_test.py @@ -233,7 +233,7 @@ def test_end_to_end(pubsub_topic): future.result() # Check the logs for "Hello Runner" - time.sleep(20) # Slight delay writing to stackdriver + time.sleep(20) # Slight delay writing to Cloud Logging client = LoggingServiceV2Client() resource_names = [f"projects/{PROJECT}"] @@ -246,7 +246,7 @@ def test_end_to_end(pubsub_topic): f"AND resource.labels.service_name={CLOUD_RUN_SERVICE} " ) - # Retry a maximum number of 10 times to find results in stackdriver + # Retry a maximum number of 10 times to find results in Cloud Logging found = False for x in range(10): iterator = client.list_log_entries( diff --git a/run/pubsub/requirements-test.txt b/run/pubsub/requirements-test.txt index 53e3abc66ec..305f4f14156 100644 --- a/run/pubsub/requirements-test.txt +++ b/run/pubsub/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 pytest==8.2.0 -google-cloud-logging==3.11.4 -google-cloud-pubsub==2.28.0 +google-cloud-logging==3.15.0 +google-cloud-pubsub==2.36.0 diff --git a/secretmanager/snippets-adk/README.md b/secretmanager/snippets-adk/README.md new file mode 100644 index 00000000000..73e1c7faf03 --- /dev/null +++ b/secretmanager/snippets-adk/README.md @@ -0,0 +1,99 @@ +# ADK Secret Manager Samples + +This directory contains samples demonstrating how to use the Agent Development Kit (ADK) with Google Cloud Secret Manager. + +## Folders +* `agent_global`: Sample for accessing secrets from a global Secret Manager instance. +* `agent_regional`: Sample for accessing secrets from a regional Secret Manager endpoint. + +## Prerequisites + +1. **Create and activate a virtual environment**: + ```bash + python3 -m venv .venv + source .venv/bin/activate + ``` + +2. **Set up Application Default Credentials**: + ```bash + gcloud auth application-default login + ``` + +3. **Install dependencies**: + You need to install dependencies for the specific sample or test you want to run. + + For global agent samples and tests: + ```bash + pip install -r agent_global/requirements.txt + ``` + + For regional agent samples and tests: + ```bash + pip install -r agent_regional/requirements.txt + ``` + +4. **Set up environment variables**: + * `GOOGLE_CLOUD_PROJECT`: Your Google Cloud Project ID. (Required for both samples and tests). + + The following environment variables are **only required when running the samples manually**. The tests will generate and use their own temporary secrets automatically. + + * `ADK_TEST_SECRET_ID`: The ID of the secret to access. + * `ADK_TEST_SECRET_VERSION` (Optional): The version of the secret (defaults to `latest`). + * `GOOGLE_CLOUD_PROJECT_LOCATION` (Required for regional samples): The region where the secret is located (e.g., `us-central1`). + +## Running the Samples + +The samples are designed to be run from this directory (`snippets-adk`) using the `adk run` command. + +### Global Secret Manager Agent + +1. **Create a `.env` file** alongside `agent_global.py` in the `agent_global` directory: + ```env + GOOGLE_GENAI_USE_VERTEXAI=1 + GOOGLE_CLOUD_PROJECT=your-project-id + GOOGLE_CLOUD_LOCATION=your-region + ``` + *Note: Replace `your-project-id` with your GCP project ID and `your-region` with your desired region.* + +2. **Run the agent**: + ```bash + export GOOGLE_CLOUD_PROJECT="your-project-id" + export ADK_TEST_SECRET_ID="your-secret-id" + adk run agent_global + ``` + +### Regional Secret Manager Agent + +1. **Create a `.env` file** alongside `agent_regional.py` in the `agent_regional` directory: + ```env + GOOGLE_GENAI_USE_VERTEXAI=1 + GOOGLE_CLOUD_PROJECT=your-project-id + GOOGLE_CLOUD_LOCATION=your-region + ``` + *Note: Replace `your-project-id` with your GCP project ID and `your-region` with your desired region.* + +2. **Run the agent**: + ```bash + export GOOGLE_CLOUD_PROJECT="your-project-id" + export GOOGLE_CLOUD_PROJECT_LOCATION="your-region" + export ADK_TEST_SECRET_ID="your-secret-id" + adk run agent_regional + ``` + +## Running the Tests + +The tests are located within the specific agent folders and run the samples using `adk run`. + +To run tests for the global agent: +```bash +pip install -r agent_global/requirements.txt +pip install -r agent_global/requirements-test.txt +pytest agent_global/snippets_adk_test.py +``` + +To run tests for the regional agent: +```bash +pip install -r agent_regional/requirements.txt +pip install -r agent_regional/requirements-test.txt +pytest agent_regional/snippets_adk_test.py +``` diff --git a/secretmanager/snippets-adk/agent_global/agent.py b/secretmanager/snippets-adk/agent_global/agent.py new file mode 100644 index 00000000000..af5dea9d531 --- /dev/null +++ b/secretmanager/snippets-adk/agent_global/agent.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +""" +ADK agent for accessing secrets from global Secret Manager. +""" + +import os + +from google.adk import Agent +from google.adk.integrations.secret_manager.secret_client import SecretManagerClient + +# Fetch secret from global Secret Manager +project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") +secret_id = os.environ.get("ADK_TEST_SECRET_ID") +secret_version = os.environ.get("ADK_TEST_SECRET_VERSION", "latest") + +if not project_id or not secret_id: + raise ValueError("GOOGLE_CLOUD_PROJECT and ADK_TEST_SECRET_ID environment variables must be set.") + +resource_name = f"projects/{project_id}/secrets/{secret_id}/versions/{secret_version}" + +print("Fetching secret from global Secret Manager...") +# Initialize Secret Manager Client (Global) +client = SecretManagerClient() + +# Fetch secret +try: + secret_payload = client.get_secret(resource_name) + print("Successfully fetched secret.") + # The secret_payload can now be used by the agent or its tools as required. +except Exception as e: + print(f"Error fetching secret: {e}") + raise e + +# Initialize Agent +root_agent = Agent( + model='gemini-2.5-flash', + name='root_agent', + description='A helpful assistant for user questions.', + instruction='Answer user questions to the best of your knowledge', +) + +print("Agent initialized successfully.") diff --git a/secretmanager/snippets-adk/agent_global/noxfile_config.py b/secretmanager/snippets-adk/agent_global/noxfile_config.py new file mode 100644 index 00000000000..873ef83d52a --- /dev/null +++ b/secretmanager/snippets-adk/agent_global/noxfile_config.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be inported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} diff --git a/secretmanager/snippets-adk/agent_global/requirements-test.txt b/secretmanager/snippets-adk/agent_global/requirements-test.txt new file mode 100644 index 00000000000..b3b2b3ca4cc --- /dev/null +++ b/secretmanager/snippets-adk/agent_global/requirements-test.txt @@ -0,0 +1 @@ +pytest==9.0.3 diff --git a/secretmanager/snippets-adk/agent_global/requirements.txt b/secretmanager/snippets-adk/agent_global/requirements.txt new file mode 100644 index 00000000000..a4f5a5371c3 --- /dev/null +++ b/secretmanager/snippets-adk/agent_global/requirements.txt @@ -0,0 +1 @@ +google-adk>=1.29 diff --git a/secretmanager/snippets-adk/agent_global/snippets_adk_test.py b/secretmanager/snippets-adk/agent_global/snippets_adk_test.py new file mode 100644 index 00000000000..3d2501cf405 --- /dev/null +++ b/secretmanager/snippets-adk/agent_global/snippets_adk_test.py @@ -0,0 +1,102 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import os +import subprocess +import time +from typing import Iterator +import uuid + +from google.api_core import exceptions +from google.cloud import secretmanager +import pytest + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.dirname(CURRENT_DIR) + + +@pytest.fixture() +def client() -> secretmanager.SecretManagerServiceClient: + return secretmanager.SecretManagerServiceClient() + + +@pytest.fixture() +def project_id() -> str: + return os.environ["GOOGLE_CLOUD_PROJECT"] + + +@pytest.fixture() +def secret_id( + client: secretmanager.SecretManagerServiceClient, project_id: str +) -> Iterator[str]: + secret_id = f"python-adk-test-{uuid.uuid4()}" + yield secret_id + + # Teardown: delete the secret + secret_path = client.secret_path(project_id, secret_id) + print(f"Deleting secret {secret_id}...") + try: + # Wait a bit to avoid conflicts if operations are still pending + time.sleep(2) + client.delete_secret(request={"name": secret_path}) + except exceptions.NotFound: + pass + + +def test_agent_global( + client: secretmanager.SecretManagerServiceClient, project_id: str, secret_id: str +) -> None: + # Create the secret + parent = f"projects/{project_id}" + print(f"Creating secret {secret_id}...") + client.create_secret( + request={ + "parent": parent, + "secret_id": secret_id, + "secret": {"replication": {"automatic": {}}}, + } + ) + + # Add a version + secret_path = client.secret_path(project_id, secret_id) + print(f"Adding version to {secret_id}...") + client.add_secret_version( + request={ + "parent": secret_path, + "payload": {"data": b"test-payload"}, + } + ) + + # Set environment variables required by the agent + # GOOGLE_CLOUD_PROJECT is already set in the environment (required to run this test) + os.environ["ADK_TEST_SECRET_ID"] = secret_id + os.environ["ADK_TEST_SECRET_VERSION"] = "latest" + + print(f"Running adk run agent_global from {PARENT_DIR}...") + # Run the sample using adk run + result = subprocess.run( + ["adk", "run", "agent_global"], + input="exit\n", + capture_output=True, + text=True, + cwd=PARENT_DIR, + ) + + print("STDOUT:") + print(result.stdout) + print("STDERR:") + print(result.stderr) + + assert result.returncode == 0 + assert "Successfully fetched secret" in result.stdout + assert "Agent initialized successfully" in result.stdout diff --git a/secretmanager/snippets-adk/agent_regional/agent.py b/secretmanager/snippets-adk/agent_regional/agent.py new file mode 100644 index 00000000000..b5c6edfc1ee --- /dev/null +++ b/secretmanager/snippets-adk/agent_regional/agent.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +""" +ADK agent for accessing secrets from regional Secret Manager. +""" + +import os + +from google.adk import Agent +from google.adk.integrations.secret_manager.secret_client import SecretManagerClient + +# Fetch secret from regional Secret Manager +project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") +location = os.environ.get("GOOGLE_CLOUD_PROJECT_LOCATION") +secret_id = os.environ.get("ADK_TEST_SECRET_ID") +secret_version = os.environ.get("ADK_TEST_SECRET_VERSION", "latest") + +if not project_id or not location or not secret_id: + raise ValueError("GOOGLE_CLOUD_PROJECT, GOOGLE_CLOUD_PROJECT_LOCATION, and ADK_TEST_SECRET_ID environment variables must be set.") + +resource_name = f"projects/{project_id}/locations/{location}/secrets/{secret_id}/versions/{secret_version}" + +print(f"Fetching secret from regional Secret Manager ({location})...") +# Initialize Secret Manager Client (Regional) +client = SecretManagerClient(location=location) + +# Fetch secret +try: + secret_payload = client.get_secret(resource_name) + print("Successfully fetched secret.") + # The secret_payload can now be used by the agent or its tools as required. +except Exception as e: + print(f"Error fetching secret: {e}") + raise e + +# Initialize the agent +root_agent = Agent( + model='gemini-2.5-flash', + name='root_agent', + description='A helpful assistant for user questions.', + instruction='Answer user questions to the best of your knowledge', +) + +print("Agent initialized successfully.") diff --git a/secretmanager/snippets-adk/agent_regional/noxfile_config.py b/secretmanager/snippets-adk/agent_regional/noxfile_config.py new file mode 100644 index 00000000000..873ef83d52a --- /dev/null +++ b/secretmanager/snippets-adk/agent_regional/noxfile_config.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be inported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} diff --git a/secretmanager/snippets-adk/agent_regional/requirements-test.txt b/secretmanager/snippets-adk/agent_regional/requirements-test.txt new file mode 100644 index 00000000000..b3b2b3ca4cc --- /dev/null +++ b/secretmanager/snippets-adk/agent_regional/requirements-test.txt @@ -0,0 +1 @@ +pytest==9.0.3 diff --git a/secretmanager/snippets-adk/agent_regional/requirements.txt b/secretmanager/snippets-adk/agent_regional/requirements.txt new file mode 100644 index 00000000000..a4f5a5371c3 --- /dev/null +++ b/secretmanager/snippets-adk/agent_regional/requirements.txt @@ -0,0 +1 @@ +google-adk>=1.29 diff --git a/secretmanager/snippets-adk/agent_regional/snippets_adk_test.py b/secretmanager/snippets-adk/agent_regional/snippets_adk_test.py new file mode 100644 index 00000000000..f530c5ae72f --- /dev/null +++ b/secretmanager/snippets-adk/agent_regional/snippets_adk_test.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import os +import subprocess +import time +from typing import Iterator +import uuid + +from google.api_core import exceptions +from google.cloud import secretmanager +import pytest + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.dirname(CURRENT_DIR) + + +@pytest.fixture() +def location() -> str: + return "us-east5" + + +@pytest.fixture() +def client(location: str) -> secretmanager.SecretManagerServiceClient: + client_options = {"api_endpoint": f"secretmanager.{location}.rep.googleapis.com"} + return secretmanager.SecretManagerServiceClient(client_options=client_options) + + +@pytest.fixture() +def project_id() -> str: + return os.environ["GOOGLE_CLOUD_PROJECT"] + + +@pytest.fixture() +def secret_id( + client: secretmanager.SecretManagerServiceClient, project_id: str, location: str +) -> Iterator[str]: + secret_id = f"python-adk-test-{uuid.uuid4()}" + yield secret_id + + # Teardown: delete the secret + secret_path = f"projects/{project_id}/locations/{location}/secrets/{secret_id}" + print(f"Deleting secret {secret_id}...") + try: + # Wait a bit to avoid conflicts if operations are still pending + time.sleep(2) + client.delete_secret(request={"name": secret_path}) + except exceptions.NotFound: + pass + + +def test_agent_regional( + client: secretmanager.SecretManagerServiceClient, + project_id: str, + location: str, + secret_id: str, +) -> None: + # Create the secret + parent = f"projects/{project_id}/locations/{location}" + print(f"Creating regional secret {secret_id} in {location}...") + client.create_secret( + request={ + "parent": parent, + "secret_id": secret_id, + } + ) + + # Add a version + secret_path = f"projects/{project_id}/locations/{location}/secrets/{secret_id}" + print(f"Adding version to {secret_id}...") + client.add_secret_version( + request={ + "parent": secret_path, + "payload": {"data": b"test-payload"}, + } + ) + + # Set environment variables required by the agent + os.environ["ADK_TEST_SECRET_ID"] = secret_id + os.environ["ADK_TEST_SECRET_VERSION"] = "latest" + os.environ["GOOGLE_CLOUD_PROJECT_LOCATION"] = location + + print(f"Running adk run agent_regional from {PARENT_DIR}...") + # Run the sample using adk run + result = subprocess.run( + ["adk", "run", "agent_regional"], + input="exit\n", + capture_output=True, + text=True, + cwd=PARENT_DIR, + ) + + print("STDOUT:") + print(result.stdout) + print("STDERR:") + print(result.stderr) + + assert result.returncode == 0 + assert "Successfully fetched secret" in result.stdout + assert "Agent initialized successfully" in result.stdout diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh new file mode 100644 index 00000000000..25a8e714c7d --- /dev/null +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +set -euxo pipefail +echo '--- Installing git and cloning repository on VM ---' +sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv + +# Clone the repository and checkout the specific commit from the build trigger. +git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/GoogleCloudPlatform/python-docs-samples +cd python-docs-samples +git sparse-checkout set storage +git fetch origin "refs/pull/${_PR_NUMBER}/head" +git checkout ${COMMIT_SHA} +cd storage + + +echo '--- Installing Python and dependencies on VM ---' +python3 -m venv env +source env/bin/activate + +echo 'Install testing libraries explicitly, as they are not in setup.py' +pip install --upgrade pip +pip install pytest pytest-timeout pytest-subtests pytest-asyncio +pip install google-cloud-testutils google-cloud-kms +pip install google-cloud-storage[grpc,testing] + +echo '--- Setting up environment variables on VM ---' +export ZONAL_BUCKET=${_ZONAL_BUCKET} +export RUN_ZONAL_SYSTEM_TESTS=True +export GCE_METADATA_MTLS_MODE=None +CURRENT_ULIMIT=$(ulimit -n) +echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT +pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py diff --git a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml new file mode 100644 index 00000000000..b968a0df18a --- /dev/null +++ b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -0,0 +1,152 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +substitutions: + _REGION: "us-central1" + _ZONE: "us-central1-a" + _SHORT_BUILD_ID: ${BUILD_ID:0:8} + _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}" + _ULIMIT: "10000" # 10k, for gRPC bidi streams + + + +steps: + # Step 0: Generate a persistent SSH key for this build run. + # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + # Generate the SSH key + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + # Save the public key content to a file for the cleanup step + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + waitFor: ["-"] + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-old-keys" + entrypoint: "bash" + args: + - "-c" + - | + #!/bin/bash + set -e + + echo "Fetching OS Login SSH keys..." + echo "Removing all keys." + echo "---------------------------------------------------------------------" + + FINGERPRINTS_TO_DELETE=$$(gcloud compute os-login ssh-keys list \ + --format="value(fingerprint)") + + echo "Keys to delete: $$FINGERPRINTS_TO_DELETE" + + if [ -z "$$FINGERPRINTS_TO_DELETE" ]; then + echo "No keys found to delete. Nothing to do." + exit 0 + fi + + while IFS= read -r FINGERPRINT; do + if [ -n "$$FINGERPRINT" ]; then + echo "Deleting key with fingerprint: $$FINGERPRINT" + gcloud compute os-login ssh-keys remove \ + --key="$$FINGERPRINT" \ + --quiet || true + fi + done <<< "$$FINGERPRINTS_TO_DELETE" + + echo "---------------------------------------------------------------------" + echo "Cleanup complete." + + # Step 1 Create a GCE VM to run the tests. + # The VM is created in the same zone as the buckets to test rapid storage features. + # It's given the 'cloud-platform' scope to allow it to access GCS and other services. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "create" + - "${_VM_NAME}" + - "--project=${PROJECT_ID}" + - "--zone=${_ZONE}" + - "--machine-type=e2-medium" + - "--image-family=debian-13" + - "--image-project=debian-cloud" + - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" + - "--scopes=https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/devstorage.read_write" + - "--metadata=enable-oslogin=TRUE" + waitFor: ["-"] + + # Step 2: Run the integration tests inside the newly created VM and cleanup. + # This step uses 'gcloud compute ssh' to execute a remote script. + # The VM is deleted after tests are run, regardless of success. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-tests-and-delete-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + # Wait for the VM to be fully initialized and SSH to be ready. + for i in {1..10}; do + if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then + break + fi + echo "Waiting for VM to become available... (attempt $i/10)" + sleep 15 + done + # copy the script to the VM + gcloud compute scp storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + # Execute the script on the VM via SSH. + # Capture the exit code to ensure cleanup happens before the build fails. + set +e + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" + EXIT_CODE=$? + set -e + + echo "--- Deleting GCE VM ---" + gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet + + # Exit with the original exit code from the test script. + exit $$EXIT_CODE + waitFor: + - "create-vm" + - "generate-ssh-key" + - "cleanup-old-keys" + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + echo "--- Removing SSH key from OS Login profile to prevent accumulation ---" + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + waitFor: + - "run-tests-and-delete-vm" + +timeout: "3600s" # 60 minutes + +options: + logging: CLOUD_LOGGING_ONLY + pool: + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" diff --git a/storage/hierarchical-namespace/delete_empty_folders.py b/storage/hierarchical-namespace/delete_empty_folders.py deleted file mode 100644 index c1075a742ff..00000000000 --- a/storage/hierarchical-namespace/delete_empty_folders.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright 2025 Google LLC. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import concurrent.futures -import logging -import threading -import time - -from google.api_core import exceptions as google_exceptions -from google.api_core import retry -from google.cloud import storage_control_v2 -import grpc - -ThreadPoolExecutor = concurrent.futures.ThreadPoolExecutor - -# This script may be used to recursively delete a large number of nested empty -# folders in a GCS HNS bucket. Overview of the algorithm: -# 1. Folder Discovery: -# - Lists all folders under the BUCKET_NAME and FOLDER_PREFIX (if set). -# - Partitions all discovered folders into a map, keyed by depth -# (e.g. {1: [foo1/ foo2/], 2: [foo1/bar1/, foo1/bar2/, foo2/bar3/], ...} -# 2. Folder Deletion: -# - Processes depths in reverse order (from deepest to shallowest). -# - For each depth level, submits all folders at that level to a thread pool -# for parallel deletion. -# - Only moves to the next depth level once all folders at the current depth -# have been processed. This ensures that child folders are removed before -# their parents, respecting hierarhical constraints. -# -# Note: This script only deletes folders, not objects; any folders with child -# objects (immediate or nested) will fail to be deleted. -# -# Usage: See README.md for instructions. - -# --- Configuration --- -BUCKET_NAME = "your-gcs-bucket-name" - -# e.g. "archive/old_data/" or "" to delete all folders in the bucket. -# If specified, must end with '/'. -FOLDER_PREFIX = "chain_103/" - -# Max number of concurrent threads to use for deleting folders. -MAX_WORKERS = 100 - -# How often to log statistics during deletion, in seconds. -STATS_REPORT_INTERVAL = 5 - -# --- Data Structures & Globals --- -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(threadName)s - %(message)s" -) - -# Global map to store folders by their depth -# { depth (int) -> set of full_resource_names (str) } -folders_by_depth = {} - -# Stats for monitoring progress -stats = { - "found_total": 0, - "successful_deletes": 0, - "failed_deletes_precondition": 0, - "failed_deletes_internal": 0, -} -stats_lock = threading.Lock() - -# Initialize the Storage Control API client -storage_control_client = storage_control_v2.StorageControlClient() - - -def _get_simple_path_and_depth(full_resource_name: str) -> tuple[str, int]: - """Extracts bucket-relative path and depth from a GCS folder resource name. - - The "simple path" is relative to the bucket (e.g., 'archive/logs/' for - 'projects/_/buckets/my-bucket/folders/archive/logs/'). - - The "depth" is the number of '/' in the simple path (e.g. 'archive/logs/' is - depth 2). - - Args: - full_resource_name: The full resource name of the GCS folder, e.g., - 'projects/_/buckets/your-bucket-name/folders/path/to/folder/'. - - Returns: - A tuple (simple_path: str, depth: int). - - Raises: - ValueError: If the resource name does not match the expected format - (i.e. start with 'projects/_/buckets/BUCKET_NAME/folders/FOLDER_PREFIX' - and ends with a trailing slash). - """ - base_folders_prefix = f"projects/_/buckets/{BUCKET_NAME}/folders/" - # The full prefix to validate against, including the global FOLDER_PREFIX. - # If FOLDER_PREFIX is "", this is equivalent to base_folders_prefix. - expected_validation_prefix = base_folders_prefix + FOLDER_PREFIX - - if not full_resource_name.startswith( - expected_validation_prefix - ) or not full_resource_name.endswith("/"): - raise ValueError( - f"Folder resource name '{full_resource_name}' does not match expected" - f" prefix '{expected_validation_prefix}' or missing trailing slash." - ) - - simple_path = full_resource_name[len(base_folders_prefix) :] - depth = simple_path.count("/") - if depth < 1: - raise ValueError( - f"Folder resource name '{full_resource_name}' has invalid depth" - f" {depth} (expected at least 1)." - ) - return simple_path, depth - - -def discover_and_partition_folders(): - """Discovers all folders in the bucket and partitions them by depth. - - Result is stored in the global folders_by_depth dictionary. - """ - parent_resource = f"projects/_/buckets/{BUCKET_NAME}" - - logging.info( - "Starting folder discovery and partitioning for bucket '%s'." - " Using prefix filter: '%s'.", - BUCKET_NAME, - FOLDER_PREFIX if FOLDER_PREFIX else "NONE (all folders)", - ) - - list_folders_request = storage_control_v2.ListFoldersRequest( - parent=parent_resource, prefix=FOLDER_PREFIX - ) - - num_folders_found = 0 - try: - for folder in storage_control_client.list_folders( - request=list_folders_request - ): - full_resource_name = folder.name - _, depth = _get_simple_path_and_depth(full_resource_name) - - if depth not in folders_by_depth: - folders_by_depth[depth] = set() - folders_by_depth[depth].add(full_resource_name) - - num_folders_found += 1 - with stats_lock: - stats["found_total"] = num_folders_found - - except Exception as e: - logging.error("Failed to list folders: %s", e, exc_info=True) - return - - logging.info( - "Finished discovery. Total folders found: %s.", num_folders_found - ) - if not folders_by_depth: - logging.info("No folders found in the bucket.") - else: - logging.info("Folders partitioned by depth:") - for depth_val in sorted(folders_by_depth.keys()): - logging.info( - " Depth %s: %s folders", depth_val, len(folders_by_depth[depth_val]) - ) - - -# Defines retriable error codes for the DeleteFolder API call. -def should_retry(exception): - if not isinstance( - exception, (google_exceptions.GoogleAPICallError, grpc.RpcError) - ): - return False - - # gRPC status codes to retry on, matching the JSON - retryable_grpc_codes = [ - grpc.StatusCode.RESOURCE_EXHAUSTED, - grpc.StatusCode.UNAVAILABLE, - grpc.StatusCode.INTERNAL, - grpc.StatusCode.UNKNOWN, - ] - - status_code = None - if isinstance(exception, google_exceptions.GoogleAPICallError): - status_code = exception.code - elif isinstance(exception, grpc.RpcError): - # For grpc.RpcError, code() returns the status code enum - status_code = exception.code() - - return status_code in retryable_grpc_codes - - -def delete_folder(folder_full_resource_name: str): - """Attempts to delete a single GCS HNS folder. - - Includes retry logic for transient errors. - - Stores stats in the global stats dictionary. - - Args: - folder_full_resource_name: The full resource name of the GCS folder to - delete, e.g., - 'projects/_/buckets/your-bucket-name/folders/path/to/folder/'. - """ - simple_path, _ = _get_simple_path_and_depth(folder_full_resource_name) - - retry_policy = retry.Retry( - predicate=should_retry, - initial=1.0, # Initial backoff: 1s - maximum=60.0, # Max backoff: 60s - multiplier=2.0, # Backoff multiplier: 2 - deadline=120.0, # Total time allowed for all retries and calls - ) - - try: - request = storage_control_v2.DeleteFolderRequest( - name=folder_full_resource_name - ) - storage_control_client.delete_folder(request=request, retry=retry_policy) - - with stats_lock: - stats["successful_deletes"] += 1 - return # Success - - except google_exceptions.NotFound: - # This can happen if the folder was deleted by another process. - logging.warning( - "Folder not found for deletion (already gone?): %s", simple_path - ) - return # Not a retriable error - except google_exceptions.FailedPrecondition as e: - # This typically means the folder contains objects. - logging.warning("Deletion failed for '%s': %s.", simple_path, e.message) - with stats_lock: - stats["failed_deletes_precondition"] += 1 - return # Not a retriable error - except Exception as e: - logging.error( - "Failed to delete '%s': %s", - simple_path, - e, - exc_info=True, - ) - with stats_lock: - stats["failed_deletes_internal"] += 1 - return # All retries exhausted - - -# --- STATS REPORTER THREAD --- -def stats_reporter_thread_logic(stop_event: threading.Event, start_time: float): - """Logs current statistics periodically.""" - logging.info("Stats Reporter: Started.") - while not stop_event.wait(STATS_REPORT_INTERVAL): - with stats_lock: - elapsed = time.time() - start_time - rate = stats["successful_deletes"] / elapsed if elapsed > 0 else 0 - logging.info( - "[STATS] Total Folders Found: %s | Successful Deletes: %s | Failed" - " Deletes (precondition): %s | Failed Deletes (internal): %s | Rate:" - " %.2f folders/sec", - stats["found_total"], - stats["successful_deletes"], - stats["failed_deletes_precondition"], - stats["failed_deletes_internal"], - rate, - ) - logging.info("Stats Reporter: Shutting down.") - - -# --- MAIN EXECUTION BLOCK --- -if __name__ == "__main__": - if BUCKET_NAME == "your-gcs-bucket-name": - print( - "\nERROR: Please update the BUCKET_NAME variable in the script before" - " running." - ) - exit(1) - - if FOLDER_PREFIX and not FOLDER_PREFIX.endswith("/"): - print("\nERROR: FOLDER_PREFIX must end with a '/' if specified.") - exit(1) - - start_time = time.time() - - logging.info("Starting GCS HNS folder deletion for bucket: %s", BUCKET_NAME) - - # Event to signal threads to stop gracefully. - stop_event = threading.Event() - - # Start the stats reporter thread. - stats_thread = threading.Thread( - target=stats_reporter_thread_logic, - args=(stop_event, start_time), - name="StatsReporter", - daemon=True, - ) - stats_thread.start() - - # Step 1: Discover and Partition Folders. - discover_and_partition_folders() - - if not folders_by_depth: - logging.info("No folders found to delete. Exiting.") - exit(0) - - # Prepare for multi-threaded deletion within each depth level. - deletion_executor = ThreadPoolExecutor( - max_workers=MAX_WORKERS, thread_name_prefix="DeleteFolderWorker" - ) - - try: - # Step 2: Iterate and delete by depth (from max to min). - sorted_depths = sorted(folders_by_depth.keys(), reverse=True) - for current_depth in sorted_depths: - folders_at_current_depth = folders_by_depth.get(current_depth, set()) - - if not folders_at_current_depth: - logging.info( - "Skipping depth %s: No folders found at this depth.", current_depth - ) - continue - - logging.info( - "\nProcessing depth %s: Submitting %s folders for deletion...", - current_depth, - len(folders_at_current_depth), - ) - - # Submit deletion tasks to the executor. - futures = [ - deletion_executor.submit(delete_folder, folder_path) - for folder_path in folders_at_current_depth - ] - - # Wait for all tasks at the current depth to complete. - # This is critical: we must ensure all nested folders are gone before - # tackling their parents. - concurrent.futures.wait(futures) - - logging.info( - "Finished processing all folders at depth %s.", current_depth - ) - - except KeyboardInterrupt: - logging.info( - "Main: Keyboard interrupt received. Attempting graceful shutdown..." - ) - except Exception as e: - logging.error( - "An unexpected error occurred in the main loop: %s", e, exc_info=True - ) - finally: - # Signal all threads to stop. - stop_event.set() - - # Shut down deletion executor and wait for any pending tasks to complete. - logging.info( - "Main: Shutting down deletion workers. Waiting for any final tasks..." - ) - deletion_executor.shutdown(wait=True) - - # Wait for the stats reporter to finish. - if stats_thread.is_alive(): - stats_thread.join( - timeout=STATS_REPORT_INTERVAL + 2 - ) # Give it a bit more time. - - # Log final statistics. - final_elapsed_time = time.time() - start_time - logging.info("\n--- FINAL SUMMARY ---") - with stats_lock: - final_rate = ( - stats["successful_deletes"] / final_elapsed_time - if final_elapsed_time > 0 - else 0 - ) - logging.info( - " - Total Folders Found (Initial Scan): %s\n - Successful Folder" - " Deletes: %s\n - Failed Folder Deletes (Precondition): %s\n -" - " Failed Folder Deletes (Internal): %s\n - Total Runtime: %.2f" - " seconds\n - Average Deletion Rate: %.2f folders/sec", - stats["found_total"], - stats["successful_deletes"], - stats["failed_deletes_precondition"], - stats["failed_deletes_internal"], - final_elapsed_time, - final_rate, - ) - logging.info("Script execution finished.") diff --git a/storage/samples/AUTHORING_GUIDE.md b/storage/samples/AUTHORING_GUIDE.md new file mode 100644 index 00000000000..55c97b32f4c --- /dev/null +++ b/storage/samples/AUTHORING_GUIDE.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/AUTHORING_GUIDE.md \ No newline at end of file diff --git a/storage/samples/CONTRIBUTING.md b/storage/samples/CONTRIBUTING.md new file mode 100644 index 00000000000..34c882b6f1a --- /dev/null +++ b/storage/samples/CONTRIBUTING.md @@ -0,0 +1 @@ +See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/CONTRIBUTING.md \ No newline at end of file diff --git a/storage/samples/snippets/acl_test.py b/storage/samples/snippets/acl_test.py new file mode 100644 index 00000000000..eecee522b57 --- /dev/null +++ b/storage/samples/snippets/acl_test.py @@ -0,0 +1,168 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import backoff +from google.api_core.exceptions import GoogleAPIError +from google.cloud import storage +import pytest + +import storage_add_bucket_default_owner +import storage_add_bucket_owner +import storage_add_file_owner +import storage_print_bucket_acl +import storage_print_bucket_acl_for_user +import storage_print_file_acl +import storage_print_file_acl_for_user +import storage_remove_bucket_default_owner +import storage_remove_bucket_owner +import storage_remove_file_owner + +# Typically we'd use a @example.com address, but GCS requires a real Google +# account. Retrieve a service account email with storage admin permissions. +TEST_EMAIL = "py38-storage-test" "@python-docs-samples-tests.iam.gserviceaccount.com" + + +@pytest.fixture(scope="module") +def test_bucket(): + """Yields a bucket that is deleted after the test completes.""" + + # The new projects have uniform bucket-level access and our tests don't + # pass with those buckets. We need to use the old main project for now. + original_value = os.environ["GOOGLE_CLOUD_PROJECT"] + os.environ["GOOGLE_CLOUD_PROJECT"] = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"] + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"acl-test-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + bucket.create() + yield bucket + bucket.delete(force=True) + # Set the value back. + os.environ["GOOGLE_CLOUD_PROJECT"] = original_value + + +@pytest.fixture +def test_blob(test_bucket): + """Yields a blob that is deleted after the test completes.""" + bucket = test_bucket + blob = bucket.blob(f"storage_acl_test_sigil-{uuid.uuid4()}") + blob.upload_from_string("Hello, is it me you're looking for?") + yield blob + + +def test_print_bucket_acl(test_bucket, capsys): + storage_print_bucket_acl.print_bucket_acl(test_bucket.name) + out, _ = capsys.readouterr() + assert out + + +def test_print_bucket_acl_for_user(test_bucket, capsys): + test_bucket.acl.user(TEST_EMAIL).grant_owner() + test_bucket.acl.save() + + storage_print_bucket_acl_for_user.print_bucket_acl_for_user( + test_bucket.name, TEST_EMAIL + ) + + out, _ = capsys.readouterr() + assert "OWNER" in out + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_add_bucket_owner(test_bucket): + storage_add_bucket_owner.add_bucket_owner(test_bucket.name, TEST_EMAIL) + + test_bucket.acl.reload() + assert "OWNER" in test_bucket.acl.user(TEST_EMAIL).get_roles() + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_remove_bucket_owner(test_bucket): + test_bucket.acl.user(TEST_EMAIL).grant_owner() + test_bucket.acl.save() + + storage_remove_bucket_owner.remove_bucket_owner(test_bucket.name, TEST_EMAIL) + + test_bucket.acl.reload() + assert "OWNER" not in test_bucket.acl.user(TEST_EMAIL).get_roles() + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_add_bucket_default_owner(test_bucket): + storage_add_bucket_default_owner.add_bucket_default_owner( + test_bucket.name, TEST_EMAIL + ) + + test_bucket.default_object_acl.reload() + roles = test_bucket.default_object_acl.user(TEST_EMAIL).get_roles() + assert "OWNER" in roles + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_remove_bucket_default_owner(test_bucket): + test_bucket.acl.user(TEST_EMAIL).grant_owner() + test_bucket.acl.save() + + storage_remove_bucket_default_owner.remove_bucket_default_owner( + test_bucket.name, TEST_EMAIL + ) + + test_bucket.default_object_acl.reload() + roles = test_bucket.default_object_acl.user(TEST_EMAIL).get_roles() + assert "OWNER" not in roles + + +def test_print_blob_acl(test_blob, capsys): + storage_print_file_acl.print_blob_acl(test_blob.bucket.name, test_blob.name) + out, _ = capsys.readouterr() + assert out + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_print_blob_acl_for_user(test_blob, capsys): + test_blob.acl.user(TEST_EMAIL).grant_owner() + test_blob.acl.save() + + storage_print_file_acl_for_user.print_blob_acl_for_user( + test_blob.bucket.name, test_blob.name, TEST_EMAIL + ) + + out, _ = capsys.readouterr() + assert "OWNER" in out + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_add_blob_owner(test_blob): + storage_add_file_owner.add_blob_owner( + test_blob.bucket.name, test_blob.name, TEST_EMAIL + ) + + test_blob.acl.reload() + assert "OWNER" in test_blob.acl.user(TEST_EMAIL).get_roles() + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_remove_blob_owner(test_blob): + test_blob.acl.user(TEST_EMAIL).grant_owner() + test_blob.acl.save() + + storage_remove_file_owner.remove_blob_owner( + test_blob.bucket.name, test_blob.name, TEST_EMAIL + ) + + test_blob.acl.reload() + assert "OWNER" not in test_blob.acl.user(TEST_EMAIL).get_roles() diff --git a/storage/samples/snippets/bucket_lock_test.py b/storage/samples/snippets/bucket_lock_test.py new file mode 100644 index 00000000000..9b7b4fa2a8e --- /dev/null +++ b/storage/samples/snippets/bucket_lock_test.py @@ -0,0 +1,176 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import uuid + +from google.cloud import storage +import pytest + +import storage_disable_default_event_based_hold +import storage_enable_default_event_based_hold +import storage_get_default_event_based_hold +import storage_get_retention_policy +import storage_lock_retention_policy +import storage_release_event_based_hold +import storage_release_temporary_hold +import storage_remove_retention_policy +import storage_set_event_based_hold +import storage_set_retention_policy +import storage_set_temporary_hold + + +BLOB_NAME = "storage_snippets_test_sigil" +BLOB_CONTENT = "Hello, is it me you're looking for?" +# Retention policy for 5 seconds +RETENTION_POLICY = 5 + + +@pytest.fixture +def bucket(): + """Yields a bucket that is deleted after the test completes.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"bucket-lock-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + bucket.create() + yield bucket + bucket.delete(force=True) + + +def test_retention_policy_no_lock(bucket, capsys): + storage_set_retention_policy.set_retention_policy( + bucket.name, RETENTION_POLICY + ) + bucket.reload() + + assert bucket.retention_period is RETENTION_POLICY + assert bucket.retention_policy_effective_time is not None + assert bucket.retention_policy_locked is None + + storage_get_retention_policy.get_retention_policy(bucket.name) + out, _ = capsys.readouterr() + assert f"Retention Policy for {bucket.name}" in out + assert "Retention Period: 5" in out + assert "Effective Time: " in out + assert "Retention Policy is locked" not in out + + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + + assert blob.retention_expiration_time is not None + + storage_remove_retention_policy.remove_retention_policy(bucket.name) + bucket.reload() + assert bucket.retention_period is None + + time.sleep(RETENTION_POLICY) + + +def test_retention_policy_lock(bucket, capsys): + storage_set_retention_policy.set_retention_policy( + bucket.name, RETENTION_POLICY + ) + bucket.reload() + assert bucket.retention_policy_locked is None + + storage_lock_retention_policy.lock_retention_policy(bucket.name) + bucket.reload() + assert bucket.retention_policy_locked is True + + storage_get_retention_policy.get_retention_policy(bucket.name) + out, _ = capsys.readouterr() + assert "Retention Policy is locked" in out + + +def test_enable_disable_bucket_default_event_based_hold(bucket, capsys): + storage_get_default_event_based_hold.get_default_event_based_hold( + bucket.name + ) + out, _ = capsys.readouterr() + assert ( + f"Default event-based hold is not enabled for {bucket.name}" + in out + ) + assert ( + f"Default event-based hold is enabled for {bucket.name}" + not in out + ) + + storage_enable_default_event_based_hold.enable_default_event_based_hold( + bucket.name + ) + bucket.reload() + + assert bucket.default_event_based_hold is True + + storage_get_default_event_based_hold.get_default_event_based_hold( + bucket.name + ) + out, _ = capsys.readouterr() + assert ( + f"Default event-based hold is enabled for {bucket.name}" in out + ) + + # Changes to the bucket will be readable immediately after writing, + # but configuration changes may take time to propagate. + time.sleep(10) + + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + assert blob.event_based_hold is True + + storage_release_event_based_hold.release_event_based_hold( + bucket.name, blob.name + ) + blob.reload() + assert blob.event_based_hold is False + + storage_disable_default_event_based_hold.disable_default_event_based_hold( + bucket.name + ) + bucket.reload() + assert bucket.default_event_based_hold is False + + +def test_enable_disable_temporary_hold(bucket): + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + assert blob.temporary_hold is None + + storage_set_temporary_hold.set_temporary_hold(bucket.name, blob.name) + blob.reload() + assert blob.temporary_hold is True + + storage_release_temporary_hold.release_temporary_hold( + bucket.name, blob.name + ) + blob.reload() + assert blob.temporary_hold is False + + +def test_enable_disable_event_based_hold(bucket): + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string(BLOB_CONTENT) + assert blob.event_based_hold is None + + storage_set_event_based_hold.set_event_based_hold(bucket.name, blob.name) + blob.reload() + assert blob.event_based_hold is True + + storage_release_event_based_hold.release_event_based_hold( + bucket.name, blob.name + ) + blob.reload() + assert blob.event_based_hold is False diff --git a/storage/samples/snippets/conftest.py b/storage/samples/snippets/conftest.py new file mode 100644 index 00000000000..b0db57561d8 --- /dev/null +++ b/storage/samples/snippets/conftest.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import uuid + +from google.cloud import storage +import pytest + + +@pytest.fixture(scope="function") +def bucket(): + """Yields a bucket that is deleted after the test completes.""" + # The new projects enforces uniform bucket level access, so + # we need to use the old main project for now. + original_value = os.environ['GOOGLE_CLOUD_PROJECT'] + os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT'] + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"uniform-bucket-level-access-{uuid.uuid4().hex}" + bucket = storage.Client().bucket(bucket_name) + bucket.create() + yield bucket + time.sleep(3) + bucket.delete(force=True) + # Set the value back. + os.environ['GOOGLE_CLOUD_PROJECT'] = original_value diff --git a/storage/samples/snippets/encryption_test.py b/storage/samples/snippets/encryption_test.py new file mode 100644 index 00000000000..f4d857dd88e --- /dev/null +++ b/storage/samples/snippets/encryption_test.py @@ -0,0 +1,231 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import tempfile +import uuid + +from google.api_core.exceptions import NotFound +from google.cloud import storage +from google.cloud.storage import Blob +import pytest + +import storage_download_encrypted_file +import storage_generate_encryption_key +import storage_object_csek_to_cmek +import storage_rotate_encryption_key +import storage_upload_encrypted_file +import storage_get_bucket_encryption_enforcement_config +import storage_set_bucket_encryption_enforcement_config +import storage_update_bucket_encryption_enforcement_config +from google.cloud.storage.bucket import EncryptionEnforcementConfig + +BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] +KMS_KEY = os.environ["MAIN_CLOUD_KMS_KEY"] + +TEST_ENCRYPTION_KEY = "brtJUWneL92g5q0N2gyDSnlPSYAiIVZ/cWgjyZNeMy0=" +TEST_ENCRYPTION_KEY_DECODED = base64.b64decode(TEST_ENCRYPTION_KEY) + +TEST_ENCRYPTION_KEY_2 = "o4OD7SWCaPjfeEGhAY+YCgMdY9UW+OJ8mvfWD9lNtO4=" +TEST_ENCRYPTION_KEY_2_DECODED = base64.b64decode(TEST_ENCRYPTION_KEY_2) + + +def test_generate_encryption_key(capsys): + storage_generate_encryption_key.generate_encryption_key() + out, _ = capsys.readouterr() + encoded_key = out.split(":", 1).pop().strip() + key = base64.b64decode(encoded_key) + assert len(key) == 32, "Returned key should be 32 bytes" + + +def test_upload_encrypted_blob(): + blob_name = f"test_upload_encrypted_{uuid.uuid4().hex}" + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b"test") + + storage_upload_encrypted_file.upload_encrypted_blob( + BUCKET, + source_file.name, + blob_name, + TEST_ENCRYPTION_KEY, + ) + bucket = storage.Client().bucket(BUCKET) + bucket.delete_blob(blob_name) + + +@pytest.fixture(scope="module") +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob_name = f"test_blob_{uuid.uuid4().hex}" + blob = Blob( + blob_name, + bucket, + encryption_key=TEST_ENCRYPTION_KEY_DECODED, + ) + content = "Hello, is it me you're looking for?" + blob.upload_from_string(content) + + yield blob.name, content + + # To delete an encrypted blob, you have to provide the same key + # used for the blob. When you provide a wrong key, you'll get + # NotFound. + try: + # Clean up for the case that the rotation didn't occur. + blob.delete() + except NotFound as e: + # For the case that the rotation succeeded. + print(f"Ignoring 404, detail: {e}") + blob = Blob(blob_name, bucket, encryption_key=TEST_ENCRYPTION_KEY_2_DECODED) + blob.delete() + + +def test_download_blob(test_blob): + test_blob_name, test_blob_content = test_blob + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_encrypted_file.download_encrypted_blob( + BUCKET, test_blob_name, dest_file.name, TEST_ENCRYPTION_KEY + ) + + downloaded_content = dest_file.read().decode("utf-8") + assert downloaded_content == test_blob_content + + +def test_rotate_encryption_key(test_blob): + test_blob_name, test_blob_content = test_blob + storage_rotate_encryption_key.rotate_encryption_key( + BUCKET, test_blob_name, TEST_ENCRYPTION_KEY, TEST_ENCRYPTION_KEY_2 + ) + + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_encrypted_file.download_encrypted_blob( + BUCKET, test_blob_name, dest_file.name, TEST_ENCRYPTION_KEY_2 + ) + + downloaded_content = dest_file.read().decode("utf-8") + assert downloaded_content == test_blob_content + + +def test_object_csek_to_cmek(test_blob): + test_blob_name, test_blob_content = test_blob + cmek_blob = storage_object_csek_to_cmek.object_csek_to_cmek( + BUCKET, test_blob_name, TEST_ENCRYPTION_KEY_2, KMS_KEY + ) + + assert cmek_blob.download_as_bytes(), test_blob_content + + +@pytest.fixture +def enforcement_bucket(): + bucket_name = f"test_encryption_enforcement_{uuid.uuid4().hex}" + yield bucket_name + + storage_client = storage.Client() + try: + bucket = storage_client.get_bucket(bucket_name) + bucket.delete(force=True) + except Exception: + pass + + +def create_enforcement_bucket(bucket_name): + """Sets up a bucket with GMEK AND CSEK Restricted""" + client = storage.Client() + bucket = client.bucket(bucket_name) + + bucket.encryption.google_managed_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="FullyRestricted") + ) + bucket.encryption.customer_managed_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="NotRestricted") + ) + bucket.encryption.customer_supplied_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="FullyRestricted") + ) + + bucket.create() + return bucket + + +def test_set_bucket_encryption_enforcement_config(enforcement_bucket): + storage_set_bucket_encryption_enforcement_config.set_bucket_encryption_enforcement_config( + enforcement_bucket + ) + + storage_client = storage.Client() + bucket = storage_client.get_bucket(enforcement_bucket) + + assert ( + bucket.encryption.google_managed_encryption_enforcement_config.restriction_mode + == "FullyRestricted" + ) + assert ( + bucket.encryption.customer_managed_encryption_enforcement_config.restriction_mode + == "NotRestricted" + ) + assert ( + bucket.encryption.customer_supplied_encryption_enforcement_config.restriction_mode + == "FullyRestricted" + ) + + +def test_get_bucket_encryption_enforcement_config(enforcement_bucket, capsys): + # Pre-setup: Creating a bucket + create_enforcement_bucket(enforcement_bucket) + + storage_get_bucket_encryption_enforcement_config.get_bucket_encryption_enforcement_config( + enforcement_bucket + ) + + out, _ = capsys.readouterr() + assert f"Encryption Enforcement Config for bucket {enforcement_bucket}" in out + assert ( + "Customer-managed encryption enforcement config restriction mode: NotRestricted" + in out + ) + assert ( + "Customer-supplied encryption enforcement config restriction mode: FullyRestricted" + in out + ) + assert ( + "Google-managed encryption enforcement config restriction mode: FullyRestricted" + in out + ) + + +def test_update_encryption_enforcement_config(enforcement_bucket): + # Pre-setup: Create a bucket in a different state before update + create_enforcement_bucket(enforcement_bucket) + + storage_update_bucket_encryption_enforcement_config.update_bucket_encryption_enforcement_config( + enforcement_bucket + ) + + storage_client = storage.Client() + bucket = storage_client.get_bucket(enforcement_bucket) + + assert ( + bucket.encryption.google_managed_encryption_enforcement_config.restriction_mode + == "NotRestricted" + ) + assert ( + bucket.encryption.customer_managed_encryption_enforcement_config.restriction_mode + == "FullyRestricted" + ) + assert ( + bucket.encryption.customer_supplied_encryption_enforcement_config.restriction_mode + == "FullyRestricted" + ) diff --git a/storage/samples/snippets/fileio_test.py b/storage/samples/snippets/fileio_test.py new file mode 100644 index 00000000000..b8a4b8272f4 --- /dev/null +++ b/storage/samples/snippets/fileio_test.py @@ -0,0 +1,35 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import storage_fileio_pandas +import storage_fileio_write_read + + +def test_fileio_write_read(bucket, capsys): + blob_name = f"test-fileio-{uuid.uuid4()}" + storage_fileio_write_read.write_read(bucket.name, blob_name) + out, _ = capsys.readouterr() + assert "Hello world" in out + + +def test_fileio_pandas(bucket, capsys): + blob_name = f"test-fileio-{uuid.uuid4()}" + storage_fileio_pandas.pandas_write(bucket.name, blob_name) + out, _ = capsys.readouterr() + assert f"Wrote csv with pandas with name {blob_name} from bucket {bucket.name}." in out + storage_fileio_pandas.pandas_read(bucket.name, blob_name) + out, _ = capsys.readouterr() + assert f"Read csv with pandas with name {blob_name} from bucket {bucket.name}." in out diff --git a/storage/samples/snippets/hmac_samples_test.py b/storage/samples/snippets/hmac_samples_test.py new file mode 100644 index 00000000000..fbc2e292df6 --- /dev/null +++ b/storage/samples/snippets/hmac_samples_test.py @@ -0,0 +1,139 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Tests for hmac.py. Requires GOOGLE_CLOUD_PROJECT (valid project) and +HMAC_KEY_TEST_SERVICE_ACCOUNT (valid service account email) env variables to be +set in order to run. +""" + +import os + +import google.api_core.exceptions +from google.cloud import storage +import pytest + +import storage_activate_hmac_key +import storage_create_hmac_key +import storage_deactivate_hmac_key +import storage_delete_hmac_key +import storage_get_hmac_key +import storage_list_hmac_keys + +# We are reaching maximum number of HMAC keys on the service account. +# We change the service account based on the value of +# RUN_TESTS_SESSION in noxfile_config.py. +# The reason we can not use multiple project is that our new projects +# are enforced to have +# 'constraints/iam.disableServiceAccountKeyCreation' policy. + +PROJECT_ID = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"] +SERVICE_ACCOUNT_EMAIL = os.environ["HMAC_KEY_TEST_SERVICE_ACCOUNT"] +STORAGE_CLIENT = storage.Client(project=PROJECT_ID) + + +@pytest.fixture(scope="module") +def new_hmac_key(): + """ + Fixture to create a new HMAC key, and to guarantee all keys are deleted at + the end of the module. + + NOTE: Due to the module scope, test order in this file is significant + """ + try: + hmac_key, secret = STORAGE_CLIENT.create_hmac_key( + service_account_email=SERVICE_ACCOUNT_EMAIL, project_id=PROJECT_ID + ) + except google.api_core.exceptions.PreconditionFailed as e: + # Check if the failure is due to the Organization Policy constraint + if "constraints/iam.disableServiceAccountKeyCreation" in str(e): + pytest.skip( + "Temporary skip: HMAC key creation is disabled by organization policy " + "on project python-docs-samples-tests. See b/493225655." + ) + raise + yield hmac_key + # Re-fetch the key metadata in case state has changed during the test. + hmac_key = STORAGE_CLIENT.get_hmac_key_metadata( + hmac_key.access_id, project_id=PROJECT_ID + ) + if hmac_key.state == "DELETED": + return + if not hmac_key.state == "INACTIVE": + hmac_key.state = "INACTIVE" + hmac_key.update() + try: + hmac_key.delete() + except google.api_core.exceptions.BadRequest: + pass + + +def test_list_keys(capsys, new_hmac_key): + hmac_keys = storage_list_hmac_keys.list_keys(PROJECT_ID) + assert "HMAC Keys:" in capsys.readouterr().out + assert hmac_keys.num_results >= 1 + + +def test_create_key(capsys): + try: + hmac_key = storage_create_hmac_key.create_key(PROJECT_ID, SERVICE_ACCOUNT_EMAIL) + except google.api_core.exceptions.PreconditionFailed as e: + if "constraints/iam.disableServiceAccountKeyCreation" in str(e): + pytest.skip( + "Temporary skip: HMAC key creation is disabled by organization policy " + "on project python-docs-samples-tests. See b/493225655." + ) + raise + + hmac_key.state = "INACTIVE" + hmac_key.update() + hmac_key.delete() + assert "Key ID:" in capsys.readouterr().out + assert hmac_key.access_id + + +def test_get_key(capsys, new_hmac_key): + hmac_key = storage_get_hmac_key.get_key(new_hmac_key.access_id, PROJECT_ID) + assert "HMAC key metadata" in capsys.readouterr().out + assert hmac_key.access_id == new_hmac_key.access_id + + +def test_activate_key(capsys, new_hmac_key): + new_hmac_key.state = "INACTIVE" + new_hmac_key.update() + hmac_key = storage_activate_hmac_key.activate_key( + new_hmac_key.access_id, PROJECT_ID + ) + assert "State: ACTIVE" in capsys.readouterr().out + assert hmac_key.state == "ACTIVE" + + +def test_deactivate_key(capsys, new_hmac_key): + hmac_key = storage_deactivate_hmac_key.deactivate_key( + new_hmac_key.access_id, PROJECT_ID + ) + assert "State: INACTIVE" in capsys.readouterr().out + assert hmac_key.state == "INACTIVE" + + +def test_delete_key(capsys, new_hmac_key): + # Due to reuse of the HMAC key for each test function, the previous + # test has deactivated the key already. + try: + new_hmac_key.state = "INACTIVE" + new_hmac_key.update() + except google.api_core.exceptions.BadRequest: + pass + + storage_delete_hmac_key.delete_key(new_hmac_key.access_id, PROJECT_ID) + assert "The key is deleted" in capsys.readouterr().out diff --git a/storage/samples/snippets/iam_test.py b/storage/samples/snippets/iam_test.py new file mode 100644 index 00000000000..7700b6c6a8a --- /dev/null +++ b/storage/samples/snippets/iam_test.py @@ -0,0 +1,149 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import time +import uuid + +from google.cloud import storage +import pytest + +import storage_add_bucket_conditional_iam_binding +import storage_add_bucket_iam_member +import storage_remove_bucket_conditional_iam_binding +import storage_remove_bucket_iam_member +import storage_set_bucket_public_iam +import storage_view_bucket_iam_members + +MEMBER = "group:dpebot@google.com" +ROLE = "roles/storage.legacyBucketReader" + +CONDITION_TITLE = "match-prefix" +CONDITION_DESCRIPTION = "Applies to objects matching a prefix" +CONDITION_EXPRESSION = ( + 'resource.name.startsWith("projects/_/buckets/bucket-name/objects/prefix-a-")' +) + + +@pytest.fixture(scope="module") +def bucket(): + bucket = None + while bucket is None or bucket.exists(): + storage_client = storage.Client() + bucket_name = f"test-iam-{uuid.uuid4()}" + bucket = storage_client.bucket(bucket_name) + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + storage_client.create_bucket(bucket) + yield bucket + time.sleep(3) + bucket.delete(force=True) + + +@pytest.fixture(scope="function") +def public_bucket(): + # The new projects don't allow to make a bucket available to public, so + # we need to use the old main project for now. + original_value = os.environ['GOOGLE_CLOUD_PROJECT'] + os.environ['GOOGLE_CLOUD_PROJECT'] = os.environ['MAIN_GOOGLE_CLOUD_PROJECT'] + bucket = None + while bucket is None or bucket.exists(): + storage_client = storage.Client() + bucket_name = f"test-iam-{uuid.uuid4()}" + bucket = storage_client.bucket(bucket_name) + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + storage_client.create_bucket(bucket) + yield bucket + time.sleep(3) + bucket.delete(force=True) + # Set the value back. + os.environ['GOOGLE_CLOUD_PROJECT'] = original_value + + +def test_view_bucket_iam_members(capsys, bucket): + storage_view_bucket_iam_members.view_bucket_iam_members(bucket.name) + assert re.match("Role: .*, Members: .*", capsys.readouterr().out) + + +def test_add_bucket_iam_member(bucket): + storage_add_bucket_iam_member.add_bucket_iam_member(bucket.name, ROLE, MEMBER) + policy = bucket.get_iam_policy(requested_policy_version=3) + assert any( + binding["role"] == ROLE and MEMBER in binding["members"] + for binding in policy.bindings + ) + + +def test_add_bucket_conditional_iam_binding(bucket): + storage_add_bucket_conditional_iam_binding.add_bucket_conditional_iam_binding( + bucket.name, + ROLE, + CONDITION_TITLE, + CONDITION_DESCRIPTION, + CONDITION_EXPRESSION, + {MEMBER}, + ) + policy = bucket.get_iam_policy(requested_policy_version=3) + assert any( + binding["role"] == ROLE + and binding["members"] == {MEMBER} + and binding["condition"] + == { + "title": CONDITION_TITLE, + "description": CONDITION_DESCRIPTION, + "expression": CONDITION_EXPRESSION, + } + for binding in policy.bindings + ) + + +def test_remove_bucket_iam_member(public_bucket): + storage_remove_bucket_iam_member.remove_bucket_iam_member( + public_bucket.name, ROLE, MEMBER) + + policy = public_bucket.get_iam_policy(requested_policy_version=3) + assert not any( + binding["role"] == ROLE and MEMBER in binding["members"] + for binding in policy.bindings + ) + + +def test_remove_bucket_conditional_iam_binding(bucket): + storage_remove_bucket_conditional_iam_binding.remove_bucket_conditional_iam_binding( + bucket.name, ROLE, CONDITION_TITLE, CONDITION_DESCRIPTION, CONDITION_EXPRESSION + ) + + policy = bucket.get_iam_policy(requested_policy_version=3) + condition = { + "title": CONDITION_TITLE, + "description": CONDITION_DESCRIPTION, + "expression": CONDITION_EXPRESSION, + } + assert not any( + (binding["role"] == ROLE and binding.get("condition") == condition) + for binding in policy.bindings + ) + + +def test_set_bucket_public_iam(public_bucket): + # The test project has org policy restricting identities by domain. + # Testing "domain:google.com" instead of "allUsers" + storage_set_bucket_public_iam.set_bucket_public_iam(public_bucket.name, ["domain:google.com"]) + policy = public_bucket.get_iam_policy(requested_policy_version=3) + + assert any( + binding["role"] == "roles/storage.objectViewer" + and "domain:google.com" in binding["members"] + for binding in policy.bindings + ) diff --git a/storage/samples/snippets/notification_polling.py b/storage/samples/snippets/notification_polling.py new file mode 100644 index 00000000000..1359c9cfa19 --- /dev/null +++ b/storage/samples/snippets/notification_polling.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to poll for GCS notifications from a +Cloud Pub/Sub subscription, parse the incoming message, and acknowledge the +successful processing of the message. + +This application will work with any subscription configured for pull rather +than push notifications. If you do not already have notifications configured, +you may consult the docs at +https://cloud.google.com/storage/docs/reporting-changes or follow the steps +below: + +1. First, follow the common setup steps for these snippets, specically + configuring auth and installing dependencies. See the README's "Setup" + section. + +2. Activate the Google Cloud Pub/Sub API, if you have not already done so. + https://console.cloud.google.com/flows/enableapi?apiid=pubsub + +3. Create a Google Cloud Storage bucket: + $ gcloud storage buckets create gs://testbucket + +4. Create a Cloud Pub/Sub topic and publish bucket notifications there: + $ gcloud storage buckets notifications create gs://testbucket --topic=testtopic --payload-format=json + +5. Create a subscription for your new topic: + $ gcloud pubsub subscriptions create testsubscription --topic=testtopic + +6. Run this program: + $ python notification_polling.py my-project-id testsubscription + +7. While the program is running, upload and delete some files in the testbucket + bucket (you could use the console or gsutil) and watch as changes scroll by + in the app. +""" + +import argparse +import json +import time + +from google.cloud import pubsub_v1 + + +def summarize(message): + data = message.data.decode("utf-8") + attributes = message.attributes + + event_type = attributes["eventType"] + bucket_id = attributes["bucketId"] + object_id = attributes["objectId"] + generation = attributes["objectGeneration"] + description = ( + "\tEvent type: {event_type}\n" + "\tBucket ID: {bucket_id}\n" + "\tObject ID: {object_id}\n" + "\tGeneration: {generation}\n" + ).format( + event_type=event_type, + bucket_id=bucket_id, + object_id=object_id, + generation=generation, + ) + + if "overwroteGeneration" in attributes: + description += f"\tOverwrote generation: {attributes['overwroteGeneration']}\n" + if "overwrittenByGeneration" in attributes: + description += f"\tOverwritten by generation: {attributes['overwrittenByGeneration']}\n" + + payload_format = attributes["payloadFormat"] + if payload_format == "JSON_API_V1": + object_metadata = json.loads(data) + size = object_metadata["size"] + content_type = object_metadata["contentType"] + metageneration = object_metadata["metageneration"] + description += ( + "\tContent type: {content_type}\n" + "\tSize: {object_size}\n" + "\tMetageneration: {metageneration}\n" + ).format( + content_type=content_type, + object_size=size, + metageneration=metageneration, + ) + return description + + +def poll_notifications(project, subscription_name): + """Polls a Cloud Pub/Sub subscription for new GCS events for display.""" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path( + project, subscription_name + ) + + def callback(message): + print(f"Received message:\n{summarize(message)}") + message.ack() + + subscriber.subscribe(subscription_path, callback=callback) + + # The subscriber is non-blocking, so we must keep the main thread from + # exiting to allow it to process messages in the background. + print(f"Listening for messages on {subscription_path}") + while True: + time.sleep(60) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "project", help="The ID of the project that owns the subscription" + ) + parser.add_argument( + "subscription", help="The ID of the Pub/Sub subscription" + ) + args = parser.parse_args() + poll_notifications(args.project, args.subscription) diff --git a/storage/samples/snippets/notification_polling_test.py b/storage/samples/snippets/notification_polling_test.py new file mode 100644 index 00000000000..dfb241b842d --- /dev/null +++ b/storage/samples/snippets/notification_polling_test.py @@ -0,0 +1,55 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from google.cloud.pubsub_v1.subscriber.message import Message +import mock + +from notification_polling import summarize + + +MESSAGE_ID = 12345 + + +def test_parse_json_message(): + attributes = { + "eventType": "OBJECT_FINALIZE", + "bucketId": "mybucket", + "objectId": "myobject", + "objectGeneration": 1234567, + "resource": "projects/_/buckets/mybucket/objects/myobject#1234567", + "notificationConfig": ( + "projects/_/buckets/mybucket/" "notificationConfigs/5" + ), + "payloadFormat": "JSON_API_V1", + } + data = ( + b"{" + b' "size": 12345,' + b' "contentType": "text/html",' + b' "metageneration": 1' + b"}" + ) + message = Message( + mock.Mock(data=data, attributes=attributes, publish_time=mock.Mock(seconds=0.0, nanos=0.0)), MESSAGE_ID, delivery_attempt=0, request_queue=mock.Mock() + ) + assert summarize(message) == ( + "\tEvent type: OBJECT_FINALIZE\n" + "\tBucket ID: mybucket\n" + "\tObject ID: myobject\n" + "\tGeneration: 1234567\n" + "\tContent type: text/html\n" + "\tSize: 12345\n" + "\tMetageneration: 1\n" + ) diff --git a/storage/samples/snippets/notification_test.py b/storage/samples/snippets/notification_test.py new file mode 100644 index 00000000000..a2fdbe3ef39 --- /dev/null +++ b/storage/samples/snippets/notification_test.py @@ -0,0 +1,120 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import uuid + +from google.api_core.exceptions import NotFound +from google.cloud import storage + +import pytest + +import storage_create_bucket_notifications +import storage_delete_bucket_notification +import storage_list_bucket_notifications +import storage_print_pubsub_bucket_notification + +_topic_name = f"notification-{uuid.uuid4()}" + + +@pytest.fixture(scope="module") +def storage_client(): + return storage.Client() + + +@pytest.fixture(scope="module") +def publisher_client(): + try: + from google.cloud.pubsub_v1 import PublisherClient + except ImportError: + pytest.skip("Cannot import pubsub") + + return PublisherClient() + + +@pytest.fixture(scope="module") +def _notification_topic(storage_client, publisher_client): + topic_path = publisher_client.topic_path(storage_client.project, _topic_name) + try: + topic = publisher_client.get_topic(request={"topic": topic_path}) + except NotFound: + topic = publisher_client.create_topic(request={"name": topic_path}) + + policy = publisher_client.get_iam_policy(request={"resource": topic_path}) + binding = policy.bindings.add() + binding.role = "roles/pubsub.publisher" + binding.members.append( + f"serviceAccount:{storage_client.get_service_account_email()}" + ) + publisher_client.set_iam_policy(request={"resource": topic_path, "policy": policy}) + + yield topic + + try: + publisher_client.delete_topic(request={"topic": topic.name}) + except NotFound: + pass + + +@pytest.fixture(scope="module") +def bucket_w_notification(storage_client, _notification_topic): + """Yields a bucket with notification that is deleted after the tests complete.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"notification-test-{uuid.uuid4()}" + bucket = storage_client.bucket(bucket_name) + bucket.create() + + notification = bucket.notification(topic_name=_topic_name) + notification.create() + + yield bucket + + bucket.delete(force=True) + + +def test_list_bucket_notifications(bucket_w_notification, capsys): + storage_list_bucket_notifications.list_bucket_notifications(bucket_w_notification.name) + out, _ = capsys.readouterr() + assert "Notification ID" in out + + +def test_print_pubsub_bucket_notification(bucket_w_notification, capsys): + notification_id = 1 + storage_print_pubsub_bucket_notification.print_pubsub_bucket_notification(bucket_w_notification.name, notification_id) + out, _ = capsys.readouterr() + assert "Notification ID: 1" in out + + +def test_create_bucket_notifications(bucket_w_notification, capsys): + # test only bucket notification ID 1 was created in the fixture + assert bucket_w_notification.notification(notification_id=1).exists() is True + assert bucket_w_notification.notification(notification_id=2).exists() is False + + storage_create_bucket_notifications.create_bucket_notifications(bucket_w_notification.name, _topic_name) + out, _ = capsys.readouterr() + assert "Successfully created notification" in out + # test succesfully creates new bucket notification with ID 2 + assert bucket_w_notification.notification(notification_id=2).exists() is True + + +def test_delete_bucket_notification(bucket_w_notification, capsys): + # test bucket notification ID 1 was created in the fixture + notification_id = 1 + assert bucket_w_notification.notification(notification_id=notification_id).exists() is True + + storage_delete_bucket_notification.delete_bucket_notification(bucket_w_notification.name, notification_id) + out, _ = capsys.readouterr() + assert "Successfully deleted notification" in out + assert bucket_w_notification.notification(notification_id=notification_id).exists() is False diff --git a/storage/samples/snippets/noxfile.py b/storage/samples/snippets/noxfile.py new file mode 100644 index 00000000000..69bcaf56de6 --- /dev/null +++ b/storage/samples/snippets/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +from pathlib import Path +import sys +from typing import Callable, Dict, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto']) + elif "pytest-xdist" in packages: + concurrent_args.extend(['-n', 'auto']) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/storage/samples/snippets/noxfile_config.py b/storage/samples/snippets/noxfile_config.py new file mode 100644 index 00000000000..7eba203a4b4 --- /dev/null +++ b/storage/samples/snippets/noxfile_config.py @@ -0,0 +1,107 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py + +import os + + +# We are reaching maximum number of HMAC keys on the service account. +# We change the service account based on the value of +# RUN_TESTS_SESSION. The reason we can not use multiple project is +# that our new projects are enforced to have +# 'constraints/iam.disableServiceAccountKeyCreation' policy. +def get_service_account_email(): + session = os.environ.get('RUN_TESTS_SESSION') + if session == 'py-3.6': + return ('py36-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + if session == 'py-3.7': + return ('py37-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + if session == 'py-3.8': + return ('py38-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + if session == 'py-3.9': + return ('py39-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + if session == 'py-3.10': + return ('py310-storage-test@' + 'python-docs-samples-tests.iam.gserviceaccount.com') + return os.environ['HMAC_KEY_TEST_SERVICE_ACCOUNT'] + + +# We change the value of CLOUD_KMS_KEY based on the value of +# RUN_TESTS_SESSION. +def get_cloud_kms_key(): + session = os.environ.get('RUN_TESTS_SESSION') + if session == 'py-3.6': + return ('projects/python-docs-samples-tests-py36/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.7': + return ('projects/python-docs-samples-tests-py37/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.8': + return ('projects/python-docs-samples-tests-py38/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.9': + return ('projects/python-docs-samples-tests-py39/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.10': + return ('projects/python-docs-samples-tests-310/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.11': + return ('projects/python-docs-samples-tests-311/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.12': + return ('projects/python-docs-samples-tests-312/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + if session == 'py-3.13': + return ('projects/python-docs-samples-tests-313/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + return os.environ['CLOUD_KMS_KEY'] + + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7", "3.6", "3.7", "3.11", "3.12", "3.13"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': { + 'HMAC_KEY_TEST_SERVICE_ACCOUNT': get_service_account_email(), + 'CLOUD_KMS_KEY': get_cloud_kms_key(), + # Some tests can not use multiple projects because of several reasons: + # 1. The new projects is enforced to have the + # 'constraints/iam.disableServiceAccountKeyCreation' policy. + # 2. The new projects buckets need to have universal permission model. + # For those tests, we'll use the original project. + 'MAIN_GOOGLE_CLOUD_PROJECT': 'python-docs-samples-tests', + 'MAIN_CLOUD_KMS_KEY': ('projects/python-docs-samples-tests/locations/us/' + 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key') + }, +} diff --git a/storage/samples/snippets/public_access_prevention_test.py b/storage/samples/snippets/public_access_prevention_test.py new file mode 100644 index 00000000000..558a4ef1575 --- /dev/null +++ b/storage/samples/snippets/public_access_prevention_test.py @@ -0,0 +1,39 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import storage_get_public_access_prevention +import storage_set_public_access_prevention_enforced +import storage_set_public_access_prevention_inherited + + +def test_get_public_access_prevention(bucket, capsys): + short_name = storage_get_public_access_prevention + short_name.get_public_access_prevention(bucket.name) + out, _ = capsys.readouterr() + assert f"Public access prevention is inherited for {bucket.name}." in out + + +def test_set_public_access_prevention_enforced(bucket, capsys): + short_name = storage_set_public_access_prevention_enforced + short_name.set_public_access_prevention_enforced(bucket.name) + out, _ = capsys.readouterr() + assert f"Public access prevention is set to enforced for {bucket.name}." in out + + +def test_set_public_access_prevention_inherited(bucket, capsys): + short_name = storage_set_public_access_prevention_inherited + short_name.set_public_access_prevention_inherited(bucket.name) + out, _ = capsys.readouterr() + assert f"Public access prevention is 'inherited' for {bucket.name}." in out diff --git a/storage/samples/snippets/quickstart.py b/storage/samples/snippets/quickstart.py new file mode 100644 index 00000000000..54148b1fb55 --- /dev/null +++ b/storage/samples/snippets/quickstart.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_quickstart(): + # [START storage_quickstart] + # Imports the Google Cloud client library + from google.cloud import storage + + # Instantiates a client + storage_client = storage.Client() + + # The name for the new bucket + bucket_name = "my-new-bucket" + + # Creates the new bucket + bucket = storage_client.create_bucket(bucket_name) + + print(f"Bucket {bucket.name} created.") + # [END storage_quickstart] + + +if __name__ == "__main__": + run_quickstart() diff --git a/storage/samples/snippets/quickstart_test.py b/storage/samples/snippets/quickstart_test.py new file mode 100644 index 00000000000..f6e06ad93e8 --- /dev/null +++ b/storage/samples/snippets/quickstart_test.py @@ -0,0 +1,28 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import quickstart + + +@mock.patch("google.cloud.storage.client.Client.create_bucket") +def test_quickstart(create_bucket_mock, capsys): + # Unlike other quickstart tests, this one mocks out the creation + # because buckets are expensive, globally-namespaced object. + create_bucket_mock.return_value = mock.sentinel.bucket + + quickstart.run_quickstart() + + create_bucket_mock.assert_called_with("my-new-bucket") diff --git a/storage/samples/snippets/requester_pays_test.py b/storage/samples/snippets/requester_pays_test.py new file mode 100644 index 00000000000..4bef0cb8968 --- /dev/null +++ b/storage/samples/snippets/requester_pays_test.py @@ -0,0 +1,73 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import backoff +import os +import tempfile + +from google.api_core.exceptions import GoogleAPIError +from google.cloud import storage +import pytest + +import storage_disable_requester_pays +import storage_download_file_requester_pays +import storage_enable_requester_pays +import storage_get_requester_pays_status + + +# We use a different bucket from other tests. +# The service account for the test needs to have Billing Project Manager role +# in order to make changes on buckets with requester pays enabled. +BUCKET = os.environ["REQUESTER_PAYS_TEST_BUCKET"] +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_enable_requester_pays(capsys): + storage_enable_requester_pays.enable_requester_pays(BUCKET) + out, _ = capsys.readouterr() + assert f"Requester Pays has been enabled for {BUCKET}" in out + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_disable_requester_pays(capsys): + storage_disable_requester_pays.disable_requester_pays(BUCKET) + out, _ = capsys.readouterr() + assert f"Requester Pays has been disabled for {BUCKET}" in out + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_get_requester_pays_status(capsys): + storage_get_requester_pays_status.get_requester_pays_status(BUCKET) + out, _ = capsys.readouterr() + assert f"Requester Pays is disabled for {BUCKET}" in out + + +@pytest.fixture +def test_blob(): + """Provides a pre-existing blob in the test bucket.""" + bucket = storage.Client().bucket(BUCKET) + blob = bucket.blob("storage_snippets_test_sigil") + blob.upload_from_string("Hello, is it me you're looking for?") + return blob + + +@backoff.on_exception(backoff.expo, GoogleAPIError, max_time=60) +def test_download_file_requester_pays(test_blob, capsys): + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_file_requester_pays.download_file_requester_pays( + BUCKET, PROJECT, test_blob.name, dest_file.name + ) + + assert dest_file.read() diff --git a/storage/samples/snippets/requirements-test.txt b/storage/samples/snippets/requirements-test.txt new file mode 100644 index 00000000000..5644295d03e --- /dev/null +++ b/storage/samples/snippets/requirements-test.txt @@ -0,0 +1,4 @@ +pytest===7.4.4; python_version == '3.7' +pytest==8.3.5; python_version >= '3.8' +mock==5.2.0 +backoff==2.2.1 diff --git a/storage/samples/snippets/requirements.txt b/storage/samples/snippets/requirements.txt new file mode 100644 index 00000000000..751f8cfbe53 --- /dev/null +++ b/storage/samples/snippets/requirements.txt @@ -0,0 +1,8 @@ +google-cloud-pubsub==2.29.0 +google-cloud-storage==3.1.0 +pandas===1.3.5; python_version == '3.7' +pandas===2.0.3; python_version == '3.8' +pandas==2.2.3; python_version >= '3.9' +opentelemetry-exporter-gcp-trace +opentelemetry-propagator-gcp +opentelemetry-instrumentation-requests diff --git a/storage/samples/snippets/rpo_test.py b/storage/samples/snippets/rpo_test.py new file mode 100644 index 00000000000..0dcf1574646 --- /dev/null +++ b/storage/samples/snippets/rpo_test.py @@ -0,0 +1,61 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +from google.cloud import storage +import pytest + +import storage_create_bucket_turbo_replication +import storage_get_rpo +import storage_set_rpo_async_turbo +import storage_set_rpo_default + + +@pytest.fixture +def dual_region_bucket(): + """Yields a dual region bucket that is deleted after the test completes.""" + bucket = None + location = "NAM4" + while bucket is None or bucket.exists(): + bucket_name = f"bucket-lock-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + bucket.create(location=location) + yield bucket + bucket.delete(force=True) + + +def test_get_rpo(dual_region_bucket, capsys): + storage_get_rpo.get_rpo(dual_region_bucket.name) + out, _ = capsys.readouterr() + assert f"RPO for {dual_region_bucket.name} is DEFAULT." in out + + +def test_set_rpo_async_turbo(dual_region_bucket, capsys): + storage_set_rpo_async_turbo.set_rpo_async_turbo(dual_region_bucket.name) + out, _ = capsys.readouterr() + assert f"RPO is set to ASYNC_TURBO for {dual_region_bucket.name}." in out + + +def test_set_rpo_default(dual_region_bucket, capsys): + storage_set_rpo_default.set_rpo_default(dual_region_bucket.name) + out, _ = capsys.readouterr() + assert f"RPO is set to DEFAULT for {dual_region_bucket.name}." in out + + +def test_create_bucket_turbo_replication(capsys): + bucket_name = f"test-rpo-{uuid.uuid4()}" + storage_create_bucket_turbo_replication.create_bucket_turbo_replication(bucket_name) + out, _ = capsys.readouterr() + assert f"{bucket_name} created with the recovery point objective (RPO) set to ASYNC_TURBO in NAM4." in out diff --git a/storage/samples/snippets/snippets_test.py b/storage/samples/snippets/snippets_test.py new file mode 100644 index 00000000000..1d3c8c1c442 --- /dev/null +++ b/storage/samples/snippets/snippets_test.py @@ -0,0 +1,1065 @@ +# Copyright 2016 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import io +import os +import tempfile +import time +import uuid +import sys + +from google.cloud import storage +import google.cloud.exceptions +import pytest +import requests + +import storage_add_bucket_label +import storage_async_download +import storage_async_upload +import storage_batch_request +import storage_bucket_delete_default_kms_key +import storage_change_default_storage_class +import storage_change_file_storage_class +import storage_compose_file +import storage_configure_retries +import storage_copy_file +import storage_copy_file_archived_generation +import storage_cors_configuration +import storage_create_bucket_class_location +import storage_create_bucket_dual_region +import storage_create_bucket_hierarchical_namespace +import storage_create_bucket_object_retention +import storage_define_bucket_website_configuration +import storage_delete_file +import storage_delete_file_archived_generation +import storage_disable_bucket_lifecycle_management +import storage_disable_soft_delete +import storage_disable_versioning +import storage_download_byte_range +import storage_download_file +import storage_download_into_memory +import storage_download_public_file +import storage_download_to_stream +import storage_enable_bucket_lifecycle_management +import storage_enable_versioning +import storage_generate_signed_post_policy_v4 +import storage_generate_signed_url_v2 +import storage_generate_signed_url_v4 +import storage_generate_upload_signed_url_v4 +import storage_get_autoclass +import storage_get_bucket_labels +import storage_get_bucket_metadata +import storage_get_metadata +import storage_get_service_account +import storage_get_soft_delete_policy +import storage_get_soft_deleted_bucket +import storage_list_buckets +import storage_list_file_archived_generations +import storage_list_files +import storage_list_files_with_prefix +import storage_list_soft_deleted_buckets +import storage_list_soft_deleted_object_versions +import storage_list_soft_deleted_objects +import storage_make_public +import storage_move_file +import storage_move_file_atomically +import storage_object_get_kms_key +import storage_remove_bucket_label +import storage_remove_cors_configuration +import storage_rename_file +import storage_restore_object +import storage_restore_soft_deleted_bucket +import storage_set_autoclass +import storage_set_bucket_default_kms_key +import storage_set_client_endpoint +import storage_set_metadata +import storage_set_object_retention_policy +import storage_set_soft_delete_policy +import storage_trace_quickstart +import storage_transfer_manager_download_bucket +import storage_transfer_manager_download_chunks_concurrently +import storage_transfer_manager_download_many +import storage_transfer_manager_upload_chunks_concurrently +import storage_transfer_manager_upload_directory +import storage_transfer_manager_upload_many +import storage_upload_file +import storage_upload_from_memory +import storage_upload_from_stream +import storage_upload_with_kms_key + +KMS_KEY = os.environ.get("CLOUD_KMS_KEY") +IS_PYTHON_3_14 = sys.version_info[:2] == (3, 14) + + +@pytest.mark.skipif(IS_PYTHON_3_14, reason="b/470276398") +def test_enable_default_kms_key(test_bucket): + storage_set_bucket_default_kms_key.enable_default_kms_key( + bucket_name=test_bucket.name, kms_key_name=KMS_KEY + ) + time.sleep(2) # Let change propagate as needed + bucket = storage.Client().get_bucket(test_bucket.name) + assert bucket.default_kms_key_name.startswith(KMS_KEY) + bucket.default_kms_key_name = None + bucket.patch() + + +def test_get_bucket_labels(test_bucket): + storage_get_bucket_labels.get_bucket_labels(test_bucket.name) + + +def test_add_bucket_label(test_bucket, capsys): + storage_add_bucket_label.add_bucket_label(test_bucket.name) + out, _ = capsys.readouterr() + assert "example" in out + + +def test_remove_bucket_label(test_bucket, capsys): + storage_add_bucket_label.add_bucket_label(test_bucket.name) + storage_remove_bucket_label.remove_bucket_label(test_bucket.name) + out, _ = capsys.readouterr() + assert "Removed labels" in out + + +@pytest.fixture(scope="module") +def test_bucket(): + """Yields a bucket that is deleted after the test completes.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + bucket.create() + yield bucket + bucket.delete(force=True) + + +@pytest.fixture(scope="module") +def test_soft_deleted_bucket(): + """Yields a soft-deleted bucket.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + bucket.create() + # [Assumption] Bucket is created with default policy , ie soft delete on. + bucket.delete() + yield bucket + + +@pytest.fixture(scope="function") +def test_soft_delete_enabled_bucket(): + """Yields a bucket with soft-delete enabled that is deleted after the test completes.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + # Soft-delete retention for 7 days (minimum allowed by API) + bucket.soft_delete_policy.retention_duration_seconds = 7 * 24 * 60 * 60 + # Soft-delete requires a region + bucket.create(location="US-CENTRAL1") + yield bucket + bucket.delete(force=True) + + +@pytest.fixture(scope="function") +def test_public_bucket(): + # The new projects don't allow to make a bucket available to public, so + # for some tests we need to use the old main project for now. + original_value = os.environ["GOOGLE_CLOUD_PROJECT"] + os.environ["GOOGLE_CLOUD_PROJECT"] = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"] + bucket = None + while bucket is None or bucket.exists(): + storage_client = storage.Client() + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" + bucket = storage_client.bucket(bucket_name) + storage_client.create_bucket(bucket) + yield bucket + bucket.delete(force=True) + # Set the value back. + os.environ["GOOGLE_CLOUD_PROJECT"] = original_value + + +@pytest.fixture(scope="module") +def new_bucket_obj(): + """Yields a new bucket object that is deleted after the test completes.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + yield bucket + bucket.delete(force=True) + + +@pytest.fixture +def test_blob(test_bucket): + """Yields a blob that is deleted after the test completes.""" + bucket = test_bucket + blob = bucket.blob(f"storage_snippets_test_sigil-{uuid.uuid4()}") + blob.upload_from_string("Hello, is it me you're looking for?") + yield blob + + +@pytest.fixture(scope="function") +def test_public_blob(test_public_bucket): + """Yields a blob that is deleted after the test completes.""" + bucket = test_public_bucket + blob = bucket.blob(f"storage_snippets_test_sigil-{uuid.uuid4()}") + blob.upload_from_string("Hello, is it me you're looking for?") + yield blob + + +@pytest.fixture +def test_bucket_create(): + """Yields a bucket object that is deleted after the test completes.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" + bucket = storage.Client().bucket(bucket_name) + yield bucket + bucket.delete(force=True) + + +def test_list_buckets(test_bucket, capsys): + storage_list_buckets.list_buckets() + out, _ = capsys.readouterr() + assert test_bucket.name in out + + +def test_list_soft_deleted_buckets(test_soft_deleted_bucket, capsys): + storage_list_soft_deleted_buckets.list_soft_deleted_buckets() + out, _ = capsys.readouterr() + assert test_soft_deleted_bucket.name in out + + +def test_list_blobs(test_blob, capsys): + storage_list_files.list_blobs(test_blob.bucket.name) + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_bucket_metadata(test_bucket, capsys): + storage_get_bucket_metadata.bucket_metadata(test_bucket.name) + out, _ = capsys.readouterr() + assert test_bucket.name in out + + +def test_get_soft_deleted_bucket(test_soft_deleted_bucket, capsys): + storage_get_soft_deleted_bucket.get_soft_deleted_bucket( + test_soft_deleted_bucket.name, test_soft_deleted_bucket.generation + ) + out, _ = capsys.readouterr() + assert test_soft_deleted_bucket.name in out + + +def test_restore_soft_deleted_bucket(test_soft_deleted_bucket, capsys): + storage_restore_soft_deleted_bucket.restore_bucket( + test_soft_deleted_bucket.name, test_soft_deleted_bucket.generation + ) + out, _ = capsys.readouterr() + assert test_soft_deleted_bucket.name in out + + +def test_list_blobs_with_prefix(test_blob, capsys): + storage_list_files_with_prefix.list_blobs_with_prefix( + test_blob.bucket.name, prefix="storage_snippets" + ) + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_upload_blob(test_bucket): + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b"test") + source_file.flush() + + storage_upload_file.upload_blob( + test_bucket.name, source_file.name, "test_upload_blob" + ) + + +def test_upload_blob_from_memory(test_bucket, capsys): + storage_upload_from_memory.upload_blob_from_memory( + test_bucket.name, "Hello, is it me you're looking for?", "test_upload_blob" + ) + out, _ = capsys.readouterr() + + assert "Hello, is it me you're looking for?" in out + + +def test_upload_blob_from_stream(test_bucket, capsys): + file_obj = io.BytesIO() + file_obj.write(b"This is test data.") + storage_upload_from_stream.upload_blob_from_stream( + test_bucket.name, file_obj, "test_upload_blob" + ) + out, _ = capsys.readouterr() + + assert "Stream data uploaded to test_upload_blob" in out + + +@pytest.mark.skipif(IS_PYTHON_3_14, reason="b/470276398") +def test_upload_blob_with_kms(test_bucket): + blob_name = f"test_upload_with_kms_{uuid.uuid4().hex}" + with tempfile.NamedTemporaryFile() as source_file: + source_file.write(b"test") + source_file.flush() + storage_upload_with_kms_key.upload_blob_with_kms( + test_bucket.name, + source_file.name, + blob_name, + KMS_KEY, + ) + bucket = storage.Client().bucket(test_bucket.name) + kms_blob = bucket.get_blob(blob_name) + assert kms_blob.kms_key_name.startswith(KMS_KEY) + test_bucket.delete_blob(blob_name) + + +def test_async_upload(bucket, capsys): + asyncio.run(storage_async_upload.async_upload_blob(bucket.name)) + out, _ = capsys.readouterr() + assert f"Uploaded 3 files to bucket {bucket.name}" in out + + +def test_async_download(test_bucket, capsys): + object_count = 3 + source_files = [f"async_sample_blob_{x}" for x in range(object_count)] + for source in source_files: + blob = test_bucket.blob(source) + blob.upload_from_string(source) + + asyncio.run( + storage_async_download.async_download_blobs(test_bucket.name, *source_files) + ) + out, _ = capsys.readouterr() + for x in range(object_count): + assert f"Downloaded storage object async_sample_blob_{x}" in out + + +def test_download_byte_range(test_blob): + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_byte_range.download_byte_range( + test_blob.bucket.name, test_blob.name, 0, 4, dest_file.name + ) + assert dest_file.read() == b"Hello" + + +def test_download_blob(test_blob): + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_file.download_blob( + test_blob.bucket.name, test_blob.name, dest_file.name + ) + + assert dest_file.read() + + +def test_download_blob_into_memory(test_blob, capsys): + storage_download_into_memory.download_blob_into_memory( + test_blob.bucket.name, test_blob.name + ) + out, _ = capsys.readouterr() + + assert "Hello, is it me you're looking for?" in out + + +def test_download_blob_to_stream(test_blob, capsys): + file_obj = io.BytesIO() + storage_download_to_stream.download_blob_to_stream( + test_blob.bucket.name, test_blob.name, file_obj + ) + out, _ = capsys.readouterr() + + file_obj.seek(0) + content = file_obj.read() + + assert "Downloaded blob" in out + assert b"Hello, is it me you're looking for?" in content + + +def test_blob_metadata(test_blob, capsys): + storage_get_metadata.blob_metadata(test_blob.bucket.name, test_blob.name) + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_set_blob_metadata(test_blob, capsys): + storage_set_metadata.set_blob_metadata(test_blob.bucket.name, test_blob.name) + out, _ = capsys.readouterr() + assert test_blob.name in out + + +def test_delete_blob(test_blob): + storage_delete_file.delete_blob(test_blob.bucket.name, test_blob.name) + + +@pytest.mark.xfail(reason="wait until b/469643064 is fixed") +def test_make_blob_public(test_public_blob): + storage_make_public.make_blob_public( + test_public_blob.bucket.name, test_public_blob.name + ) + + r = requests.get(test_public_blob.public_url) + assert r.text == "Hello, is it me you're looking for?" + + +def test_generate_signed_url(test_blob, capsys): + url = storage_generate_signed_url_v2.generate_signed_url( + test_blob.bucket.name, test_blob.name + ) + + r = requests.get(url) + assert r.text == "Hello, is it me you're looking for?" + + +def test_generate_download_signed_url_v4(test_blob, capsys): + url = storage_generate_signed_url_v4.generate_download_signed_url_v4( + test_blob.bucket.name, test_blob.name + ) + + r = requests.get(url) + assert r.text == "Hello, is it me you're looking for?" + + +def test_generate_upload_signed_url_v4(test_bucket, capsys): + blob_name = "storage_snippets_test_upload" + content = b"Uploaded via v4 signed url" + url = storage_generate_upload_signed_url_v4.generate_upload_signed_url_v4( + test_bucket.name, blob_name + ) + + requests.put( + url, + data=content, + headers={"content-type": "application/octet-stream"}, + ) + + bucket = storage.Client().bucket(test_bucket.name) + blob = bucket.blob(blob_name) + assert blob.download_as_bytes() == content + + +def test_generate_signed_policy_v4(test_bucket, capsys): + blob_name = "storage_snippets_test_form" + short_name = storage_generate_signed_post_policy_v4 + form = short_name.generate_signed_post_policy_v4(test_bucket.name, blob_name) + assert f"name='key' value='{blob_name}'" in form + assert "name='x-goog-signature'" in form + assert "name='x-goog-date'" in form + assert "name='x-goog-credential'" in form + assert "name='x-goog-algorithm' value='GOOG4-RSA-SHA256'" in form + assert "name='policy'" in form + assert "name='x-goog-meta-test' value='data'" in form + assert "type='file' name='file'/>" in form + + +def test_rename_blob(test_blob): + bucket = storage.Client().bucket(test_blob.bucket.name) + + try: + bucket.delete_blob("test_rename_blob") + except google.cloud.exceptions.exceptions.NotFound: + print(f"test_rename_blob not found in bucket {bucket.name}") + + storage_rename_file.rename_blob(bucket.name, test_blob.name, "test_rename_blob") + + assert bucket.get_blob("test_rename_blob") is not None + assert bucket.get_blob(test_blob.name) is None + + +def test_move_blob(test_bucket_create, test_blob): + bucket = test_blob.bucket + storage.Client().create_bucket(test_bucket_create) + + try: + test_bucket_create.delete_blob("test_move_blob") + except google.cloud.exceptions.NotFound: + print(f"test_move_blob not found in bucket {test_bucket_create.name}") + + storage_move_file.move_blob( + bucket.name, + test_blob.name, + test_bucket_create.name, + "test_move_blob", + ) + + assert test_bucket_create.get_blob("test_move_blob") is not None + assert bucket.get_blob(test_blob.name) is None + + +def test_copy_blob(test_blob): + bucket = storage.Client().bucket(test_blob.bucket.name) + + try: + bucket.delete_blob("test_copy_blob") + except google.cloud.exceptions.NotFound: + pass + + storage_copy_file.copy_blob( + bucket.name, + test_blob.name, + bucket.name, + "test_copy_blob", + ) + + assert bucket.get_blob("test_copy_blob") is not None + assert bucket.get_blob(test_blob.name) is not None + + +def test_versioning(test_bucket, capsys): + bucket = storage_enable_versioning.enable_versioning(test_bucket) + out, _ = capsys.readouterr() + assert "Versioning was enabled for bucket" in out + assert bucket.versioning_enabled is True + + bucket = storage_disable_versioning.disable_versioning(test_bucket) + out, _ = capsys.readouterr() + assert "Versioning was disabled for bucket" in out + assert bucket.versioning_enabled is False + + +def test_get_set_autoclass(new_bucket_obj, test_bucket, capsys): + # Test default values when Autoclass is unset + bucket = storage_get_autoclass.get_autoclass(test_bucket.name) + out, _ = capsys.readouterr() + assert "Autoclass enabled is set to False" in out + assert bucket.autoclass_toggle_time is None + assert bucket.autoclass_terminal_storage_class_update_time is None + + # Test enabling Autoclass at bucket creation + new_bucket_obj.autoclass_enabled = True + bucket = storage.Client().create_bucket(new_bucket_obj) + assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == "NEARLINE" + + # Test set terminal_storage_class to ARCHIVE + bucket = storage_set_autoclass.set_autoclass(bucket.name) + out, _ = capsys.readouterr() + assert "Autoclass enabled is set to True" in out + assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == "ARCHIVE" + + # Test get Autoclass + bucket = storage_get_autoclass.get_autoclass(bucket.name) + out, _ = capsys.readouterr() + assert "Autoclass enabled is set to True" in out + assert bucket.autoclass_toggle_time is not None + assert bucket.autoclass_terminal_storage_class_update_time is not None + + +def test_bucket_lifecycle_management(test_bucket, capsys): + bucket = ( + storage_enable_bucket_lifecycle_management.enable_bucket_lifecycle_management( + test_bucket + ) + ) + out, _ = capsys.readouterr() + assert "[]" in out + assert "Lifecycle management is enable" in out + assert len(list(bucket.lifecycle_rules)) > 0 + + bucket = ( + storage_disable_bucket_lifecycle_management.disable_bucket_lifecycle_management( + test_bucket + ) + ) + out, _ = capsys.readouterr() + assert "[]" in out + assert len(list(bucket.lifecycle_rules)) == 0 + + +def test_create_bucket_class_location(test_bucket_create): + bucket = storage_create_bucket_class_location.create_bucket_class_location( + test_bucket_create.name + ) + + assert bucket.location == "US" + assert bucket.storage_class == "COLDLINE" + + +def test_create_bucket_dual_region(test_bucket_create, capsys): + location = "US" + region_1 = "US-EAST1" + region_2 = "US-WEST1" + storage_create_bucket_dual_region.create_bucket_dual_region( + test_bucket_create.name, location, region_1, region_2 + ) + out, _ = capsys.readouterr() + assert f"Created bucket {test_bucket_create.name}" in out + assert location in out + assert region_1 in out + assert region_2 in out + assert "dual-region" in out + + +@pytest.mark.skipif(IS_PYTHON_3_14, reason="b/470276398") +def test_bucket_delete_default_kms_key(test_bucket, capsys): + test_bucket.default_kms_key_name = KMS_KEY + test_bucket.patch() + + assert test_bucket.default_kms_key_name == KMS_KEY + + bucket = storage_bucket_delete_default_kms_key.bucket_delete_default_kms_key( + test_bucket.name + ) + + out, _ = capsys.readouterr() + assert bucket.default_kms_key_name is None + assert bucket.name in out + + +def test_get_service_account(capsys): + storage_get_service_account.get_service_account() + + out, _ = capsys.readouterr() + + assert "@gs-project-accounts.iam.gserviceaccount.com" in out + + +@pytest.mark.xfail(reason="wait until b/469643064 is fixed") +def test_download_public_file(test_public_blob): + storage_make_public.make_blob_public( + test_public_blob.bucket.name, test_public_blob.name + ) + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_public_file.download_public_file( + test_public_blob.bucket.name, test_public_blob.name, dest_file.name + ) + + assert dest_file.read() == b"Hello, is it me you're looking for?" + + +def test_define_bucket_website_configuration(test_bucket): + bucket = ( + storage_define_bucket_website_configuration.define_bucket_website_configuration( + test_bucket.name, "index.html", "404.html" + ) + ) + + website_val = {"mainPageSuffix": "index.html", "notFoundPage": "404.html"} + + assert bucket._properties["website"] == website_val + + +@pytest.mark.skipif(IS_PYTHON_3_14, reason="b/470276398") +def test_object_get_kms_key(test_bucket): + with tempfile.NamedTemporaryFile() as source_file: + storage_upload_with_kms_key.upload_blob_with_kms( + test_bucket.name, + source_file.name, + "test_upload_blob_encrypted", + KMS_KEY, + ) + kms_key = storage_object_get_kms_key.object_get_kms_key( + test_bucket.name, "test_upload_blob_encrypted" + ) + + assert kms_key.startswith(KMS_KEY) + + +def test_storage_compose_file(test_bucket): + source_files = ["test_upload_blob_1", "test_upload_blob_2"] + for source in source_files: + blob = test_bucket.blob(source) + blob.upload_from_string(source) + + with tempfile.NamedTemporaryFile() as dest_file: + destination = storage_compose_file.compose_file( + test_bucket.name, + source_files[0], + source_files[1], + dest_file.name, + ) + composed = destination.download_as_bytes() + + assert composed.decode("utf-8") == source_files[0] + source_files[1] + + +def test_cors_configuration(test_bucket, capsys): + bucket = storage_cors_configuration.cors_configuration(test_bucket) + out, _ = capsys.readouterr() + assert "Set CORS policies for bucket" in out + assert len(bucket.cors) > 0 + + bucket = storage_remove_cors_configuration.remove_cors_configuration(test_bucket) + out, _ = capsys.readouterr() + assert "Remove CORS policies for bucket" in out + assert len(bucket.cors) == 0 + + +def test_delete_blobs_archived_generation(test_blob, capsys): + storage_delete_file_archived_generation.delete_file_archived_generation( + test_blob.bucket.name, test_blob.name, test_blob.generation + ) + out, _ = capsys.readouterr() + assert "blob " + test_blob.name + " was deleted" in out + blob = test_blob.bucket.get_blob(test_blob.name, generation=test_blob.generation) + assert blob is None + + +def test_change_default_storage_class(test_bucket, capsys): + bucket = storage_change_default_storage_class.change_default_storage_class( + test_bucket + ) + out, _ = capsys.readouterr() + assert "Default storage class for bucket" in out + assert bucket.storage_class == "COLDLINE" + + +def test_change_file_storage_class(test_blob, capsys): + blob = storage_change_file_storage_class.change_file_storage_class( + test_blob.bucket.name, + test_blob.name, + ) + out, _ = capsys.readouterr() + assert f"Blob {blob.name} in bucket {blob.bucket.name}" in out + assert blob.storage_class == "NEARLINE" + + +def test_copy_file_archived_generation(test_blob): + bucket = storage.Client().bucket(test_blob.bucket.name) + + try: + bucket.delete_blob("test_copy_blob") + except google.cloud.exceptions.NotFound: + pass + + storage_copy_file_archived_generation.copy_file_archived_generation( + bucket.name, test_blob.name, bucket.name, "test_copy_blob", test_blob.generation + ) + + assert bucket.get_blob("test_copy_blob") is not None + assert bucket.get_blob(test_blob.name) is not None + + +def test_list_blobs_archived_generation(test_blob, capsys): + storage_list_file_archived_generations.list_file_archived_generations( + test_blob.bucket.name + ) + out, _ = capsys.readouterr() + assert str(test_blob.generation) in out + + +def test_storage_configure_retries(test_blob, capsys): + storage_configure_retries.configure_retries(test_blob.bucket.name, test_blob.name) + + # This simply checks if the retry configurations were set and printed as intended. + out, _ = capsys.readouterr() + assert "The following library method is customized to be retried" in out + assert "_should_retry" in out + assert "initial=1.5, maximum=45.0, multiplier=1.2" in out + assert "500" in out # "deadline" or "timeout" depending on dependency ver. + + +def test_batch_request(test_bucket): + blob1 = test_bucket.blob("b/1.txt") + blob2 = test_bucket.blob("b/2.txt") + blob1.upload_from_string("hello world") + blob2.upload_from_string("hello world") + + storage_batch_request.batch_request(test_bucket.name, "b/") + blob1.reload() + blob2.reload() + + assert blob1.metadata.get("your-metadata-key") == "your-metadata-value" + assert blob2.metadata.get("your-metadata-key") == "your-metadata-value" + + +def test_storage_set_client_endpoint(capsys): + storage_set_client_endpoint.set_client_endpoint("https://storage.googleapis.com") + out, _ = capsys.readouterr() + + assert "client initiated with endpoint: https://storage.googleapis.com" in out + + +def test_transfer_manager_snippets(test_bucket, capsys): + BLOB_NAMES = [ + "test.txt", + "test2.txt", + "blobs/test.txt", + "blobs/nesteddir/test.txt", + ] + + with tempfile.TemporaryDirectory() as uploads: + # Create dirs and nested dirs + for name in BLOB_NAMES: + relpath = os.path.dirname(name) + os.makedirs(os.path.join(uploads, relpath), exist_ok=True) + + # Create files with nested dirs to exercise directory handling. + for name in BLOB_NAMES: + with open(os.path.join(uploads, name), "w") as f: + f.write(name) + + storage_transfer_manager_upload_many.upload_many_blobs_with_transfer_manager( + test_bucket.name, + BLOB_NAMES, + source_directory="{}/".format(uploads), + workers=8, + ) + out, _ = capsys.readouterr() + + for name in BLOB_NAMES: + assert "Uploaded {}".format(name) in out + + with tempfile.TemporaryDirectory() as downloads: + # Download the files. + storage_transfer_manager_download_bucket.download_bucket_with_transfer_manager( + test_bucket.name, + destination_directory=os.path.join(downloads, ""), + workers=8, + max_results=10000, + ) + out, _ = capsys.readouterr() + + for name in BLOB_NAMES: + assert "Downloaded {}".format(name) in out + + with tempfile.TemporaryDirectory() as downloads: + # Download the files. + storage_transfer_manager_download_many.download_many_blobs_with_transfer_manager( + test_bucket.name, + blob_names=BLOB_NAMES, + destination_directory=os.path.join(downloads, ""), + workers=8, + ) + out, _ = capsys.readouterr() + + for name in BLOB_NAMES: + assert "Downloaded {}".format(name) in out + + +def test_transfer_manager_directory_upload(test_bucket, capsys): + BLOB_NAMES = [ + "dirtest/test.txt", + "dirtest/test2.txt", + "dirtest/blobs/test.txt", + "dirtest/blobs/nesteddir/test.txt", + ] + + with tempfile.TemporaryDirectory() as uploads: + # Create dirs and nested dirs + for name in BLOB_NAMES: + relpath = os.path.dirname(name) + os.makedirs(os.path.join(uploads, relpath), exist_ok=True) + + # Create files with nested dirs to exercise directory handling. + for name in BLOB_NAMES: + with open(os.path.join(uploads, name), "w") as f: + f.write(name) + + storage_transfer_manager_upload_directory.upload_directory_with_transfer_manager( + test_bucket.name, source_directory="{}/".format(uploads) + ) + out, _ = capsys.readouterr() + + assert "Found {}".format(len(BLOB_NAMES)) in out + for name in BLOB_NAMES: + assert "Uploaded {}".format(name) in out + + +def test_transfer_manager_download_chunks_concurrently(test_bucket, capsys): + BLOB_NAME = "test_file.txt" + + with tempfile.NamedTemporaryFile() as file: + file.write(b"test") + file.flush() + + storage_upload_file.upload_blob(test_bucket.name, file.name, BLOB_NAME) + + with tempfile.TemporaryDirectory() as downloads: + # Download the file. + storage_transfer_manager_download_chunks_concurrently.download_chunks_concurrently( + test_bucket.name, + BLOB_NAME, + os.path.join(downloads, BLOB_NAME), + workers=8, + ) + out, _ = capsys.readouterr() + + assert ( + "Downloaded {} to {}".format(BLOB_NAME, os.path.join(downloads, BLOB_NAME)) + in out + ) + + +def test_transfer_manager_upload_chunks_concurrently(test_bucket, capsys): + BLOB_NAME = "test_file.txt" + + with tempfile.NamedTemporaryFile() as file: + file.write(b"test") + file.flush() + + storage_transfer_manager_upload_chunks_concurrently.upload_chunks_concurrently( + test_bucket.name, file.name, BLOB_NAME + ) + + out, _ = capsys.readouterr() + assert "File {} uploaded to {}".format(file.name, BLOB_NAME) in out + + +def test_object_retention_policy(test_bucket_create, capsys): + storage_create_bucket_object_retention.create_bucket_object_retention( + test_bucket_create.name + ) + out, _ = capsys.readouterr() + assert ( + f"Created bucket {test_bucket_create.name} with object retention enabled setting" + in out + ) + + blob_name = "test_object_retention" + storage_set_object_retention_policy.set_object_retention_policy( + test_bucket_create.name, "hello world", blob_name + ) + out, _ = capsys.readouterr() + assert f"Retention policy for file {blob_name}" in out + + # Remove retention policy for test cleanup + blob = test_bucket_create.blob(blob_name) + blob.retention.mode = None + blob.retention.retain_until_time = None + blob.patch(override_unlocked_retention=True) + + +def test_create_bucket_hierarchical_namespace(test_bucket_create, capsys): + storage_create_bucket_hierarchical_namespace.create_bucket_hierarchical_namespace( + test_bucket_create.name + ) + out, _ = capsys.readouterr() + assert ( + f"Created bucket {test_bucket_create.name} with hierarchical namespace enabled" + in out + ) + + +def test_storage_trace_quickstart(test_bucket, capsys): + blob_name = f"trace_quickstart_{uuid.uuid4().hex}" + contents = "The quick brown fox jumps over the lazy dog." + storage_trace_quickstart.run_quickstart(test_bucket.name, blob_name, contents) + out, _ = capsys.readouterr() + + assert f"{blob_name} uploaded to {test_bucket.name}" in out + assert ( + f"Downloaded storage object {blob_name} from bucket {test_bucket.name}" in out + ) + + +def test_storage_disable_soft_delete(test_soft_delete_enabled_bucket, capsys): + bucket_name = test_soft_delete_enabled_bucket.name + storage_disable_soft_delete.disable_soft_delete(bucket_name) + out, _ = capsys.readouterr() + assert f"Soft-delete policy is disabled for bucket {bucket_name}" in out + + +def test_storage_get_soft_delete_policy(test_soft_delete_enabled_bucket, capsys): + bucket_name = test_soft_delete_enabled_bucket.name + storage_get_soft_delete_policy.get_soft_delete_policy(bucket_name) + out, _ = capsys.readouterr() + assert f"Soft-delete policy for {bucket_name}" in out + assert "Object soft-delete policy is enabled" in out + assert "Object retention duration: " in out + assert "Policy effective time: " in out + + # Disable the soft-delete policy + test_soft_delete_enabled_bucket.soft_delete_policy.retention_duration_seconds = 0 + test_soft_delete_enabled_bucket.patch() + storage_get_soft_delete_policy.get_soft_delete_policy(bucket_name) + out, _ = capsys.readouterr() + assert f"Soft-delete policy for {bucket_name}" in out + assert "Object soft-delete policy is disabled" in out + + +def test_storage_set_soft_delete_policy(test_soft_delete_enabled_bucket, capsys): + bucket_name = test_soft_delete_enabled_bucket.name + retention_duration_seconds = 10 * 24 * 60 * 60 # 10 days + storage_set_soft_delete_policy.set_soft_delete_policy( + bucket_name, retention_duration_seconds + ) + out, _ = capsys.readouterr() + assert ( + f"Soft delete policy for bucket {bucket_name} was set to {retention_duration_seconds} seconds retention period" + in out + ) + + +def test_storage_list_soft_deleted_objects(test_soft_delete_enabled_bucket, capsys): + bucket_name = test_soft_delete_enabled_bucket.name + blob_name = f"test_object_{uuid.uuid4().hex}.txt" + blob_content = "This object will be soft-deleted for listing." + blob = test_soft_delete_enabled_bucket.blob(blob_name) + blob.upload_from_string(blob_content) + blob_generation = blob.generation + + blob.delete() # Soft-delete the object + storage_list_soft_deleted_objects.list_soft_deleted_objects(bucket_name) + out, _ = capsys.readouterr() + assert f"Name: {blob_name}, Generation: {blob_generation}" in out + + +def test_storage_list_soft_deleted_object_versions( + test_soft_delete_enabled_bucket, capsys +): + bucket_name = test_soft_delete_enabled_bucket.name + blob_name = f"test_object_{uuid.uuid4().hex}.txt" + blob_content = "This object will be soft-deleted for version listing." + blob = test_soft_delete_enabled_bucket.blob(blob_name) + blob.upload_from_string(blob_content) + blob_generation = blob.generation + + blob.delete() # Soft-delete the object + storage_list_soft_deleted_object_versions.list_soft_deleted_object_versions( + bucket_name, blob_name + ) + out, _ = capsys.readouterr() + assert f"Version ID: {blob_generation}" in out + + +def test_storage_restore_soft_deleted_object(test_soft_delete_enabled_bucket, capsys): + bucket_name = test_soft_delete_enabled_bucket.name + blob_name = f"test-restore-sd-obj-{uuid.uuid4().hex}.txt" + blob_content = "This object will be soft-deleted and restored." + blob = test_soft_delete_enabled_bucket.blob(blob_name) + blob.upload_from_string(blob_content) + blob_generation = blob.generation + + blob.delete() # Soft-delete the object + storage_restore_object.restore_soft_deleted_object( + bucket_name, blob_name, blob_generation + ) + out, _ = capsys.readouterr() + assert ( + f"Soft-deleted object {blob_name} is restored in the bucket {bucket_name}" + in out + ) + + # Verify the restoration + blob = test_soft_delete_enabled_bucket.get_blob(blob_name) + assert blob is not None + + +def test_move_object(test_blob): + bucket = test_blob.bucket + try: + bucket.delete_blob("test_move_blob_atomic") + except google.cloud.exceptions.NotFound: + print(f"test_move_blob_atomic not found in bucket {bucket.name}") + + storage_move_file_atomically.move_object( + bucket.name, + test_blob.name, + "test_move_blob_atomic", + ) + + assert bucket.get_blob("test_move_blob_atomic") is not None + assert bucket.get_blob(test_blob.name) is None diff --git a/storage/samples/snippets/storage_activate_hmac_key.py b/storage/samples/snippets/storage_activate_hmac_key.py new file mode 100644 index 00000000000..d3960eb622c --- /dev/null +++ b/storage/samples/snippets/storage_activate_hmac_key.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_activate_hmac_key] +from google.cloud import storage + + +def activate_key(access_id, project_id): + """ + Activate the HMAC key with the given access ID. + """ + # project_id = "Your Google Cloud project ID" + # access_id = "ID of an inactive HMAC key" + + storage_client = storage.Client(project=project_id) + + hmac_key = storage_client.get_hmac_key_metadata( + access_id, project_id=project_id + ) + hmac_key.state = "ACTIVE" + hmac_key.update() + + print("The HMAC key metadata is:") + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") + return hmac_key + + +# [END storage_activate_hmac_key] + +if __name__ == "__main__": + activate_key(access_id=sys.argv[1], project_id=sys.argv[2]) diff --git a/storage/samples/snippets/storage_add_bucket_conditional_iam_binding.py b/storage/samples/snippets/storage_add_bucket_conditional_iam_binding.py new file mode 100644 index 00000000000..d09f528cf72 --- /dev/null +++ b/storage/samples/snippets/storage_add_bucket_conditional_iam_binding.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_add_bucket_conditional_iam_binding] +from google.cloud import storage + + +def add_bucket_conditional_iam_binding( + bucket_name, role, title, description, expression, members +): + """Add a conditional IAM binding to a bucket's IAM policy.""" + # bucket_name = "your-bucket-name" + # role = "IAM role, e.g. roles/storage.objectViewer" + # members = {"IAM identity, e.g. user: name@example.com}" + # title = "Condition title." + # description = "Condition description." + # expression = "Condition expression." + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy(requested_policy_version=3) + + # Set the policy's version to 3 to use condition in bindings. + policy.version = 3 + + policy.bindings.append( + { + "role": role, + "members": members, + "condition": { + "title": title, + "description": description, + "expression": expression, + }, + } + ) + + bucket.set_iam_policy(policy) + + print(f"Added the following member(s) with role {role} to {bucket_name}:") + + for member in members: + print(f" {member}") + + print("with condition:") + print(f" Title: {title}") + print(f" Description: {description}") + print(f" Expression: {expression}") + + +# [END storage_add_bucket_conditional_iam_binding] + + +if __name__ == "__main__": + add_bucket_conditional_iam_binding( + bucket_name=sys.argv[1], + role=sys.argv[2], + title=sys.argv[3], + description=sys.argv[4], + expression=sys.argv[5], + members=set(sys.argv[6::]), + ) diff --git a/storage/samples/snippets/storage_add_bucket_default_owner.py b/storage/samples/snippets/storage_add_bucket_default_owner.py new file mode 100644 index 00000000000..932b1328f3f --- /dev/null +++ b/storage/samples/snippets/storage_add_bucket_default_owner.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_add_bucket_default_owner] +from google.cloud import storage + + +def add_bucket_default_owner(bucket_name, user_email): + """Adds a user as an owner in the given bucket's default object access + control list.""" + # bucket_name = "your-bucket-name" + # user_email = "name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # grant access to different types of entities. You can also use + # `grant_read` or `grant_write` to grant different roles. + bucket.default_object_acl.user(user_email).grant_owner() + bucket.default_object_acl.save() + + print( + "Added user {} as an owner in the default acl on bucket {}.".format( + user_email, bucket_name + ) + ) + + +# [END storage_add_bucket_default_owner] + +if __name__ == "__main__": + add_bucket_default_owner(bucket_name=sys.argv[1], user_email=sys.argv[2]) diff --git a/storage/samples/snippets/storage_add_bucket_iam_member.py b/storage/samples/snippets/storage_add_bucket_iam_member.py new file mode 100644 index 00000000000..0d610eae7ce --- /dev/null +++ b/storage/samples/snippets/storage_add_bucket_iam_member.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_add_bucket_iam_member] +from google.cloud import storage + + +def add_bucket_iam_member(bucket_name, role, member): + """Add a new member to an IAM Policy""" + # bucket_name = "your-bucket-name" + # role = "IAM role, e.g., roles/storage.objectViewer" + # member = "IAM identity, e.g., user: name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy(requested_policy_version=3) + + policy.bindings.append({"role": role, "members": {member}}) + + bucket.set_iam_policy(policy) + + print(f"Added {member} with role {role} to {bucket_name}.") + + +# [END storage_add_bucket_iam_member] + + +if __name__ == "__main__": + add_bucket_iam_member(bucket_name=sys.argv[1], role=sys.argv[2], member=sys.argv[3]) diff --git a/storage/samples/snippets/storage_add_bucket_label.py b/storage/samples/snippets/storage_add_bucket_label.py new file mode 100644 index 00000000000..9c6fcff7af3 --- /dev/null +++ b/storage/samples/snippets/storage_add_bucket_label.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_add_bucket_label] +import pprint +# [END storage_add_bucket_label] +import sys +# [START storage_add_bucket_label] + +from google.cloud import storage + + +def add_bucket_label(bucket_name): + """Add a label to a bucket.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + labels = bucket.labels + labels["example"] = "label" + bucket.labels = labels + bucket.patch() + + print(f"Updated labels on {bucket.name}.") + pprint.pprint(bucket.labels) + + +# [END storage_add_bucket_label] + +if __name__ == "__main__": + add_bucket_label(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_add_bucket_owner.py b/storage/samples/snippets/storage_add_bucket_owner.py new file mode 100644 index 00000000000..bac1f3f6440 --- /dev/null +++ b/storage/samples/snippets/storage_add_bucket_owner.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_add_bucket_owner] +from google.cloud import storage + + +def add_bucket_owner(bucket_name, user_email): + """Adds a user as an owner on the given bucket.""" + # bucket_name = "your-bucket-name" + # user_email = "name@example.com" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group()`, `domain()`, `all_authenticated()` and `all()` + # to grant access to different types of entities. + # You can also use `grant_read()` or `grant_write()` to grant different + # roles. + bucket.acl.user(user_email).grant_owner() + bucket.acl.save() + + print( + f"Added user {user_email} as an owner on bucket {bucket_name}." + ) + + +# [END storage_add_bucket_owner] + +if __name__ == "__main__": + add_bucket_owner(bucket_name=sys.argv[1], user_email=sys.argv[2]) diff --git a/storage/samples/snippets/storage_add_file_owner.py b/storage/samples/snippets/storage_add_file_owner.py new file mode 100644 index 00000000000..9e9342590c4 --- /dev/null +++ b/storage/samples/snippets/storage_add_file_owner.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_add_file_owner] +from google.cloud import storage + + +def add_blob_owner(bucket_name, blob_name, user_email): + """Adds a user as an owner on the given blob.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # user_email = "name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Reload fetches the current ACL from Cloud Storage. + blob.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # grant access to different types of entities. You can also use + # `grant_read` or `grant_write` to grant different roles. + blob.acl.user(user_email).grant_owner() + blob.acl.save() + + print( + "Added user {} as an owner on blob {} in bucket {}.".format( + user_email, blob_name, bucket_name + ) + ) + + +# [END storage_add_file_owner] + +if __name__ == "__main__": + add_blob_owner( + bucket_name=sys.argv[1], blob_name=sys.argv[2], user_email=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_async_download.py b/storage/samples/snippets/storage_async_download.py new file mode 100755 index 00000000000..ed8f3f304f9 --- /dev/null +++ b/storage/samples/snippets/storage_async_download.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import argparse + +"""Sample that asynchronously downloads multiple files from GCS to application's memory. +""" + + +# [START storage_async_download] +# This sample can be run by calling `async.run(async_download_blobs('bucket_name', ['file1', 'file2']))` +async def async_download_blobs(bucket_name, *file_names): + """Downloads a number of files in parallel from the bucket. + """ + # The ID of your GCS bucket. + # bucket_name = "your-bucket-name" + + # The list of files names to download, these files should be present in bucket. + # file_names = ["myfile1", "myfile2"] + + import asyncio + from google.cloud import storage + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + loop = asyncio.get_running_loop() + + tasks = [] + for file_name in file_names: + blob = bucket.blob(file_name) + # The first arg, None, tells it to use the default loops executor + tasks.append(loop.run_in_executor(None, blob.download_as_bytes)) + + # If the method returns a value (such as download_as_bytes), gather will return the values + _ = await asyncio.gather(*tasks) + for file_name in file_names: + print(f"Downloaded storage object {file_name}") + + +# [END storage_async_download] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-b', '--bucket_name', type=str, dest='bucket_name', help='provide the name of the GCS bucket') + parser.add_argument( + '-f', '--file_name', + action='append', + type=str, + dest='file_names', + help='Example: -f file1.txt or --file_name my_fav.mp4 . It can be used multiple times.' + ) + args = parser.parse_args() + + asyncio.run(async_download_blobs(args.bucket_name, *args.file_names)) diff --git a/storage/samples/snippets/storage_async_upload.py b/storage/samples/snippets/storage_async_upload.py new file mode 100644 index 00000000000..25aabb63ee4 --- /dev/null +++ b/storage/samples/snippets/storage_async_upload.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import sys + + +"""Sample that asynchronously uploads a file to GCS +""" + + +# [START storage_async_upload] +# This sample can be run by calling `async.run(async_upload_blob('bucket_name'))` +async def async_upload_blob(bucket_name): + """Uploads a number of files in parallel to the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + import asyncio + from functools import partial + from google.cloud import storage + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + loop = asyncio.get_running_loop() + + tasks = [] + count = 3 + for x in range(count): + blob_name = f"async_sample_blob_{x}" + content = f"Hello world #{x}" + blob = bucket.blob(blob_name) + # The first arg, None, tells it to use the default loops executor + tasks.append(loop.run_in_executor(None, partial(blob.upload_from_string, content))) + + # If the method returns a value (such as download_as_string), gather will return the values + await asyncio.gather(*tasks) + + print(f"Uploaded {count} files to bucket {bucket_name}") + + +# [END storage_async_upload] + + +if __name__ == "__main__": + asyncio.run(async_upload_blob( + bucket_name=sys.argv[1] + )) diff --git a/storage/samples/snippets/storage_batch_request.py b/storage/samples/snippets/storage_batch_request.py new file mode 100644 index 00000000000..7fe11fb1cf7 --- /dev/null +++ b/storage/samples/snippets/storage_batch_request.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that uses a batch request. +This sample is used on this page: + https://cloud.google.com/storage/docs/batch +For more information, see README.md. +""" + +# [START storage_batch_request] + +from google.cloud import storage + + +def batch_request(bucket_name, prefix=None): + """ + Use a batch request to patch a list of objects with the given prefix in a bucket. + + Note that Cloud Storage does not support batch operations for uploading or downloading. + Additionally, the current batch design does not support library methods whose return values + depend on the response payload. + See https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.batch + """ + # The ID of your GCS bucket + # bucket_name = "my-bucket" + # The prefix of the object paths + # prefix = "directory-prefix/" + + client = storage.Client() + bucket = client.bucket(bucket_name) + + # Accumulate in a list the objects with a given prefix. + blobs_to_patch = [blob for blob in bucket.list_blobs(prefix=prefix)] + + # Use a batch context manager to edit metadata in the list of blobs. + # The batch request is sent out when the context manager closes. + # No more than 100 calls should be included in a single batch request. + with client.batch(): + for blob in blobs_to_patch: + metadata = {"your-metadata-key": "your-metadata-value"} + blob.metadata = metadata + blob.patch() + + print( + f"Batch request edited metadata for all objects with the given prefix in {bucket.name}." + ) + + +# [END storage_batch_request] + +if __name__ == "__main__": + batch_request(bucket_name=sys.argv[1], prefix=sys.argv[2]) diff --git a/storage/samples/snippets/storage_bucket_delete_default_kms_key.py b/storage/samples/snippets/storage_bucket_delete_default_kms_key.py new file mode 100644 index 00000000000..0db29375699 --- /dev/null +++ b/storage/samples/snippets/storage_bucket_delete_default_kms_key.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_bucket_delete_default_kms_key] +from google.cloud import storage + + +def bucket_delete_default_kms_key(bucket_name): + """Delete a default KMS key of bucket""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.default_kms_key_name = None + bucket.patch() + + print(f"Default KMS key was removed from {bucket.name}") + return bucket + + +# [END storage_bucket_delete_default_kms_key] + +if __name__ == "__main__": + bucket_delete_default_kms_key(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_change_default_storage_class.py b/storage/samples/snippets/storage_change_default_storage_class.py new file mode 100644 index 00000000000..5d2f924ade7 --- /dev/null +++ b/storage/samples/snippets/storage_change_default_storage_class.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_change_default_storage_class] +from google.cloud import storage +from google.cloud.storage import constants + + +def change_default_storage_class(bucket_name): + """Change the default storage class of the bucket""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.storage_class = constants.COLDLINE_STORAGE_CLASS + bucket.patch() + + print(f"Default storage class for bucket {bucket_name} has been set to {bucket.storage_class}") + return bucket + + +# [END storage_change_default_storage_class] + +if __name__ == "__main__": + change_default_storage_class(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_change_file_storage_class.py b/storage/samples/snippets/storage_change_file_storage_class.py new file mode 100644 index 00000000000..a976ac8a4c8 --- /dev/null +++ b/storage/samples/snippets/storage_change_file_storage_class.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_change_file_storage_class] +from google.cloud import storage + + +def change_file_storage_class(bucket_name, blob_name): + """Change the default storage class of the blob""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + generation_match_precondition = None + + # Optional: set a generation-match precondition to avoid potential race + # conditions and data corruptions. The request is aborted if the + # object's generation number does not match your precondition. + blob.reload() # Fetch blob metadata to use in generation_match_precondition. + generation_match_precondition = blob.generation + + blob.update_storage_class("NEARLINE", if_generation_match=generation_match_precondition) + + print( + "Blob {} in bucket {} had its storage class set to {}".format( + blob_name, + bucket_name, + blob.storage_class + ) + ) + return blob +# [END storage_change_file_storage_class] + + +if __name__ == "__main__": + change_file_storage_class(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_compose_file.py b/storage/samples/snippets/storage_compose_file.py new file mode 100644 index 00000000000..e673912725b --- /dev/null +++ b/storage/samples/snippets/storage_compose_file.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_compose_file] +from google.cloud import storage + + +def compose_file(bucket_name, first_blob_name, second_blob_name, destination_blob_name): + """Concatenate source blobs into destination blob.""" + # bucket_name = "your-bucket-name" + # first_blob_name = "first-object-name" + # second_blob_name = "second-blob-name" + # destination_blob_name = "destination-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + destination = bucket.blob(destination_blob_name) + destination.content_type = "text/plain" + + # Note sources is a list of Blob instances, up to the max of 32 instances per request + sources = [bucket.blob(first_blob_name), bucket.blob(second_blob_name)] + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to compose is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + # There is also an `if_source_generation_match` parameter, which is not used in this example. + destination_generation_match_precondition = 0 + + destination.compose(sources, if_generation_match=destination_generation_match_precondition) + + print( + "New composite object {} in the bucket {} was created by combining {} and {}".format( + destination_blob_name, bucket_name, first_blob_name, second_blob_name + ) + ) + return destination + + +# [END storage_compose_file] + +if __name__ == "__main__": + compose_file( + bucket_name=sys.argv[1], + first_blob_name=sys.argv[2], + second_blob_name=sys.argv[3], + destination_blob_name=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_configure_retries.py b/storage/samples/snippets/storage_configure_retries.py new file mode 100644 index 00000000000..25c2529a42e --- /dev/null +++ b/storage/samples/snippets/storage_configure_retries.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that configures retries on an operation call. +This sample is used on this page: + https://cloud.google.com/storage/docs/retry-strategy +For more information, see README.md. +""" + +# [START storage_configure_retries] +from google.cloud import storage +from google.cloud.storage.retry import DEFAULT_RETRY + + +def configure_retries(bucket_name, blob_name): + """Configures retries with customizations.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of your GCS object + # blob_name = "your-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Customize retry with a timeout of 500 seconds (default=120 seconds). + modified_retry = DEFAULT_RETRY.with_timeout(500.0) + # Customize retry with an initial wait time of 1.5 (default=1.0). + # Customize retry with a wait time multiplier per iteration of 1.2 (default=2.0). + # Customize retry with a maximum wait time of 45.0 (default=60.0). + modified_retry = modified_retry.with_delay(initial=1.5, multiplier=1.2, maximum=45.0) + + # blob.delete() uses DEFAULT_RETRY by default. + # Pass in modified_retry to override the default retry behavior. + print( + f"The following library method is customized to be retried according to the following configurations: {modified_retry}" + ) + + blob.delete(retry=modified_retry) + print(f"Blob {blob_name} deleted with a customized retry strategy.") + + +# [END storage_configure_retries] + + +if __name__ == "__main__": + configure_retries(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_copy_file.py b/storage/samples/snippets/storage_copy_file.py new file mode 100644 index 00000000000..b802de28b1b --- /dev/null +++ b/storage/samples/snippets/storage_copy_file.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_copy_file] +from google.cloud import storage + + +def copy_blob( + bucket_name, blob_name, destination_bucket_name, destination_blob_name, +): + """Copies a blob from one bucket to another with a new name.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # destination_bucket_name = "destination-bucket-name" + # destination_blob_name = "destination-object-name" + + storage_client = storage.Client() + + source_bucket = storage_client.bucket(bucket_name) + source_blob = source_bucket.blob(blob_name) + destination_bucket = storage_client.bucket(destination_bucket_name) + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to copy is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + # There is also an `if_source_generation_match` parameter, which is not used in this example. + destination_generation_match_precondition = 0 + + blob_copy = source_bucket.copy_blob( + source_blob, destination_bucket, destination_blob_name, if_generation_match=destination_generation_match_precondition, + ) + + print( + "Blob {} in bucket {} copied to blob {} in bucket {}.".format( + source_blob.name, + source_bucket.name, + blob_copy.name, + destination_bucket.name, + ) + ) + + +# [END storage_copy_file] + +if __name__ == "__main__": + copy_blob( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + destination_bucket_name=sys.argv[3], + destination_blob_name=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_copy_file_archived_generation.py b/storage/samples/snippets/storage_copy_file_archived_generation.py new file mode 100644 index 00000000000..419d8e5a369 --- /dev/null +++ b/storage/samples/snippets/storage_copy_file_archived_generation.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_copy_file_archived_generation] +from google.cloud import storage + + +def copy_file_archived_generation( + bucket_name, blob_name, destination_bucket_name, destination_blob_name, generation +): + """Copies a blob from one bucket to another with a new name with the same generation.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # destination_bucket_name = "destination-bucket-name" + # destination_blob_name = "destination-object-name" + # generation = 1579287380533984 + + storage_client = storage.Client() + + source_bucket = storage_client.bucket(bucket_name) + source_blob = source_bucket.blob(blob_name) + destination_bucket = storage_client.bucket(destination_bucket_name) + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to copy is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + destination_generation_match_precondition = 0 + + # source_generation selects a specific revision of the source object, as opposed to the latest version. + blob_copy = source_bucket.copy_blob( + source_blob, destination_bucket, destination_blob_name, source_generation=generation, if_generation_match=destination_generation_match_precondition + ) + + print( + "Generation {} of the blob {} in bucket {} copied to blob {} in bucket {}.".format( + generation, + source_blob.name, + source_bucket.name, + blob_copy.name, + destination_bucket.name, + ) + ) + + +# [END storage_copy_file_archived_generation] + +if __name__ == "__main__": + copy_file_archived_generation( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + destination_bucket_name=sys.argv[3], + destination_blob_name=sys.argv[4], + generation=sys.argv[5] + ) diff --git a/storage/samples/snippets/storage_cors_configuration.py b/storage/samples/snippets/storage_cors_configuration.py new file mode 100644 index 00000000000..2c5dd242870 --- /dev/null +++ b/storage/samples/snippets/storage_cors_configuration.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_cors_configuration] +from google.cloud import storage + + +def cors_configuration(bucket_name): + """Set a bucket's CORS policies configuration.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.cors = [ + { + "origin": ["*"], + "responseHeader": [ + "Content-Type", + "x-goog-resumable"], + "method": ['PUT', 'POST'], + "maxAgeSeconds": 3600 + } + ] + bucket.patch() + + print(f"Set CORS policies for bucket {bucket.name} is {bucket.cors}") + return bucket + + +# [END storage_cors_configuration] + +if __name__ == "__main__": + cors_configuration(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_create_bucket.py b/storage/samples/snippets/storage_create_bucket.py new file mode 100644 index 00000000000..c95f32f569b --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_create_bucket] +from google.cloud import storage + + +def create_bucket(bucket_name): + """Creates a new bucket.""" + # bucket_name = "your-new-bucket-name" + + storage_client = storage.Client() + + bucket = storage_client.create_bucket(bucket_name) + + print(f"Bucket {bucket.name} created") + + +# [END storage_create_bucket] + +if __name__ == "__main__": + create_bucket(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_create_bucket_class_location.py b/storage/samples/snippets/storage_create_bucket_class_location.py new file mode 100644 index 00000000000..51fa864405d --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket_class_location.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_create_bucket_class_location] +from google.cloud import storage + + +def create_bucket_class_location(bucket_name): + """ + Create a new bucket in the US region with the coldline storage + class + """ + # bucket_name = "your-new-bucket-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + bucket.storage_class = "COLDLINE" + new_bucket = storage_client.create_bucket(bucket, location="us") + + print( + "Created bucket {} in {} with storage class {}".format( + new_bucket.name, new_bucket.location, new_bucket.storage_class + ) + ) + return new_bucket + + +# [END storage_create_bucket_class_location] + +if __name__ == "__main__": + create_bucket_class_location(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_create_bucket_dual_region.py b/storage/samples/snippets/storage_create_bucket_dual_region.py new file mode 100644 index 00000000000..c5a78fa0f9b --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket_dual_region.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2022 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +""" +Sample that creates a dual region bucket. +""" + +# [START storage_create_bucket_dual_region] +from google.cloud import storage + + +def create_bucket_dual_region(bucket_name, location, region_1, region_2): + """Creates a Dual-Region Bucket with provided location and regions..""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The bucket's pair of regions. Case-insensitive. + # See this documentation for other valid locations: + # https://cloud.google.com/storage/docs/locations + # region_1 = "US-EAST1" + # region_2 = "US-WEST1" + # location = "US" + + storage_client = storage.Client() + bucket = storage_client.create_bucket(bucket_name, location=location, data_locations=[region_1, region_2]) + + print(f"Created bucket {bucket_name}") + print(f" - location: {bucket.location}") + print(f" - location_type: {bucket.location_type}") + print(f" - customPlacementConfig data_locations: {bucket.data_locations}") + + +# [END storage_create_bucket_dual_region] + + +if __name__ == "__main__": + create_bucket_dual_region( + bucket_name=sys.argv[1], location=sys.argv[2], region_1=sys.argv[3], region_2=sys.argv[4] + ) diff --git a/storage/samples/snippets/storage_create_bucket_hierarchical_namespace.py b/storage/samples/snippets/storage_create_bucket_hierarchical_namespace.py new file mode 100644 index 00000000000..d9d31077251 --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket_hierarchical_namespace.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_create_bucket_hierarchical_namespace] +from google.cloud import storage + + +def create_bucket_hierarchical_namespace(bucket_name): + """Creates a bucket with hierarchical namespace enabled.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + bucket.hierarchical_namespace_enabled = True + bucket.create() + + print(f"Created bucket {bucket_name} with hierarchical namespace enabled.") + + +# [END storage_create_bucket_hierarchical_namespace] + + +if __name__ == "__main__": + create_bucket_hierarchical_namespace(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_create_bucket_notifications.py b/storage/samples/snippets/storage_create_bucket_notifications.py new file mode 100644 index 00000000000..a6f218c36fa --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket_notifications.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that creates a notification configuration for a bucket. +This sample is used on this page: + https://cloud.google.com/storage/docs/reporting-changes +For more information, see README.md. +""" + +# [START storage_create_bucket_notifications] +from google.cloud import storage + + +def create_bucket_notifications(bucket_name, topic_name): + """Creates a notification configuration for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The name of a topic + # topic_name = "your-topic-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + notification = bucket.notification(topic_name=topic_name) + notification.create() + + print(f"Successfully created notification with ID {notification.notification_id} for bucket {bucket_name}") + +# [END storage_create_bucket_notifications] + + +if __name__ == "__main__": + create_bucket_notifications(bucket_name=sys.argv[1], topic_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_create_bucket_object_retention.py b/storage/samples/snippets/storage_create_bucket_object_retention.py new file mode 100644 index 00000000000..4ebc32c0a25 --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket_object_retention.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_create_bucket_with_object_retention] +from google.cloud import storage + + +def create_bucket_object_retention(bucket_name): + """Creates a bucket with object retention enabled.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.create_bucket(bucket_name, enable_object_retention=True) + + print(f"Created bucket {bucket_name} with object retention enabled setting: {bucket.object_retention_mode}") + + +# [END storage_create_bucket_with_object_retention] + + +if __name__ == "__main__": + create_bucket_object_retention(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_create_bucket_turbo_replication.py b/storage/samples/snippets/storage_create_bucket_turbo_replication.py new file mode 100644 index 00000000000..bc05597958f --- /dev/null +++ b/storage/samples/snippets/storage_create_bucket_turbo_replication.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that creates a new bucket with dual-region and turbo replication. +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_create_bucket_turbo_replication] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_ASYNC_TURBO + + +def create_bucket_turbo_replication(bucket_name): + """Creates dual-region bucket with turbo replication enabled.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket_location = "NAM4" + bucket.rpo = RPO_ASYNC_TURBO + bucket.create(location=bucket_location) + + print(f"{bucket.name} created with the recovery point objective (RPO) set to {bucket.rpo} in {bucket.location}.") + + +# [END storage_create_bucket_turbo_replication] + +if __name__ == "__main__": + create_bucket_turbo_replication(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_create_hmac_key.py b/storage/samples/snippets/storage_create_hmac_key.py new file mode 100644 index 00000000000..d845738b780 --- /dev/null +++ b/storage/samples/snippets/storage_create_hmac_key.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_create_hmac_key] +from google.cloud import storage + + +def create_key(project_id, service_account_email): + """ + Create a new HMAC key using the given project and service account. + """ + # project_id = 'Your Google Cloud project ID' + # service_account_email = 'Service account used to generate the HMAC key' + + storage_client = storage.Client(project=project_id) + + hmac_key, secret = storage_client.create_hmac_key( + service_account_email=service_account_email, project_id=project_id + ) + + print(f"The base64 encoded secret is {secret}") + print("Do not miss that secret, there is no API to recover it.") + print("The HMAC key metadata is:") + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") + return hmac_key + + +# [END storage_create_hmac_key] + +if __name__ == "__main__": + create_key(project_id=sys.argv[1], service_account_email=sys.argv[2]) diff --git a/storage/samples/snippets/storage_deactivate_hmac_key.py b/storage/samples/snippets/storage_deactivate_hmac_key.py new file mode 100644 index 00000000000..007f7b5a5f2 --- /dev/null +++ b/storage/samples/snippets/storage_deactivate_hmac_key.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_deactivate_hmac_key] +from google.cloud import storage + + +def deactivate_key(access_id, project_id): + """ + Deactivate the HMAC key with the given access ID. + """ + # project_id = "Your Google Cloud project ID" + # access_id = "ID of an active HMAC key" + + storage_client = storage.Client(project=project_id) + + hmac_key = storage_client.get_hmac_key_metadata( + access_id, project_id=project_id + ) + hmac_key.state = "INACTIVE" + hmac_key.update() + + print("The HMAC key is now inactive.") + print("The HMAC key metadata is:") + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") + return hmac_key + + +# [END storage_deactivate_hmac_key] + +if __name__ == "__main__": + deactivate_key(access_id=sys.argv[1], project_id=sys.argv[2]) diff --git a/storage/samples/snippets/storage_define_bucket_website_configuration.py b/storage/samples/snippets/storage_define_bucket_website_configuration.py new file mode 100644 index 00000000000..ce6c7e66cdb --- /dev/null +++ b/storage/samples/snippets/storage_define_bucket_website_configuration.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_define_bucket_website_configuration] +from google.cloud import storage + + +def define_bucket_website_configuration(bucket_name, main_page_suffix, not_found_page): + """Configure website-related properties of bucket""" + # bucket_name = "your-bucket-name" + # main_page_suffix = "index.html" + # not_found_page = "404.html" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.configure_website(main_page_suffix, not_found_page) + bucket.patch() + + print( + "Static website bucket {} is set up to use {} as the index page and {} as the 404 page".format( + bucket.name, main_page_suffix, not_found_page + ) + ) + return bucket + + +# [END storage_define_bucket_website_configuration] + +if __name__ == "__main__": + define_bucket_website_configuration( + bucket_name=sys.argv[1], + main_page_suffix=sys.argv[2], + not_found_page=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_delete_bucket.py b/storage/samples/snippets/storage_delete_bucket.py new file mode 100644 index 00000000000..b12c066361d --- /dev/null +++ b/storage/samples/snippets/storage_delete_bucket.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_delete_bucket] +from google.cloud import storage + + +def delete_bucket(bucket_name): + """Deletes a bucket. The bucket must be empty.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.delete() + + print(f"Bucket {bucket.name} deleted") + + +# [END storage_delete_bucket] + +if __name__ == "__main__": + delete_bucket(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_delete_bucket_notification.py b/storage/samples/snippets/storage_delete_bucket_notification.py new file mode 100644 index 00000000000..efd41771d60 --- /dev/null +++ b/storage/samples/snippets/storage_delete_bucket_notification.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that deletes a notification configuration for a bucket. +This sample is used on this page: + https://cloud.google.com/storage/docs/reporting-changes +For more information, see README.md. +""" + +# [START storage_delete_bucket_notification] +from google.cloud import storage + + +def delete_bucket_notification(bucket_name, notification_id): + """Deletes a notification configuration for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of the notification + # notification_id = "your-notification-id" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + notification = bucket.notification(notification_id=notification_id) + notification.delete() + + print(f"Successfully deleted notification with ID {notification_id} for bucket {bucket_name}") + +# [END storage_delete_bucket_notification] + + +if __name__ == "__main__": + delete_bucket_notification(bucket_name=sys.argv[1], notification_id=sys.argv[2]) diff --git a/storage/samples/snippets/storage_delete_file.py b/storage/samples/snippets/storage_delete_file.py new file mode 100644 index 00000000000..427604145dd --- /dev/null +++ b/storage/samples/snippets/storage_delete_file.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_delete_file] +from google.cloud import storage + + +def delete_blob(bucket_name, blob_name): + """Deletes a blob from the bucket.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + generation_match_precondition = None + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to delete is aborted if the object's + # generation number does not match your precondition. + blob.reload() # Fetch blob metadata to use in generation_match_precondition. + generation_match_precondition = blob.generation + + blob.delete(if_generation_match=generation_match_precondition) + + print(f"Blob {blob_name} deleted.") + + +# [END storage_delete_file] + +if __name__ == "__main__": + delete_blob(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_delete_file_archived_generation.py b/storage/samples/snippets/storage_delete_file_archived_generation.py new file mode 100644 index 00000000000..ff02bca23dc --- /dev/null +++ b/storage/samples/snippets/storage_delete_file_archived_generation.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_delete_file_archived_generation] +from google.cloud import storage + + +def delete_file_archived_generation(bucket_name, blob_name, generation): + """Delete a blob in the bucket with the given generation.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # generation = 1579287380533984 + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.delete_blob(blob_name, generation=generation) + print( + f"Generation {generation} of blob {blob_name} was deleted from {bucket_name}" + ) + + +# [END storage_delete_file_archived_generation] + + +if __name__ == "__main__": + delete_file_archived_generation( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + generation=sys.argv[3] + ) diff --git a/storage/samples/snippets/storage_delete_hmac_key.py b/storage/samples/snippets/storage_delete_hmac_key.py new file mode 100644 index 00000000000..403dc193b22 --- /dev/null +++ b/storage/samples/snippets/storage_delete_hmac_key.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_delete_hmac_key] +from google.cloud import storage + + +def delete_key(access_id, project_id): + """ + Delete the HMAC key with the given access ID. Key must have state INACTIVE + in order to succeed. + """ + # project_id = "Your Google Cloud project ID" + # access_id = "ID of an HMAC key (must be in INACTIVE state)" + + storage_client = storage.Client(project=project_id) + + hmac_key = storage_client.get_hmac_key_metadata( + access_id, project_id=project_id + ) + hmac_key.delete() + + print( + "The key is deleted, though it may still appear in list_hmac_keys()" + " results." + ) + + +# [END storage_delete_hmac_key] + +if __name__ == "__main__": + delete_key(access_id=sys.argv[1], project_id=sys.argv[2]) diff --git a/storage/samples/snippets/storage_disable_bucket_lifecycle_management.py b/storage/samples/snippets/storage_disable_bucket_lifecycle_management.py new file mode 100644 index 00000000000..a5fa56fcf35 --- /dev/null +++ b/storage/samples/snippets/storage_disable_bucket_lifecycle_management.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_disable_bucket_lifecycle_management] +from google.cloud import storage + + +def disable_bucket_lifecycle_management(bucket_name): + """Disable lifecycle management for a bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.clear_lifecyle_rules() + bucket.patch() + rules = bucket.lifecycle_rules + + print(f"Lifecycle management is disable for bucket {bucket_name} and the rules are {list(rules)}") + return bucket + + +# [END storage_disable_bucket_lifecycle_management] + +if __name__ == "__main__": + disable_bucket_lifecycle_management(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_disable_default_event_based_hold.py b/storage/samples/snippets/storage_disable_default_event_based_hold.py new file mode 100644 index 00000000000..48becdac1c0 --- /dev/null +++ b/storage/samples/snippets/storage_disable_default_event_based_hold.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_disable_default_event_based_hold] +from google.cloud import storage + + +def disable_default_event_based_hold(bucket_name): + """Disables the default event based hold on a given bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.default_event_based_hold = False + bucket.patch() + + print(f"Default event based hold was disabled for {bucket_name}") + + +# [END storage_disable_default_event_based_hold] + + +if __name__ == "__main__": + disable_default_event_based_hold(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_disable_requester_pays.py b/storage/samples/snippets/storage_disable_requester_pays.py new file mode 100644 index 00000000000..78e195d8a4a --- /dev/null +++ b/storage/samples/snippets/storage_disable_requester_pays.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_disable_requester_pays] +from google.cloud import storage + + +def disable_requester_pays(bucket_name): + """Disable a bucket's requesterpays metadata""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.requester_pays = False + bucket.patch() + + print(f"Requester Pays has been disabled for {bucket_name}") + + +# [END storage_disable_requester_pays] + + +if __name__ == "__main__": + disable_requester_pays(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_disable_soft_delete.py b/storage/samples/snippets/storage_disable_soft_delete.py new file mode 100644 index 00000000000..dc2447ae873 --- /dev/null +++ b/storage/samples/snippets/storage_disable_soft_delete.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_disable_soft_delete] +from google.cloud import storage + + +def disable_soft_delete(bucket_name): + """Disable soft-delete policy for the bucket.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + # Setting the retention duration to 0 disables soft-delete. + bucket.soft_delete_policy.retention_duration_seconds = 0 + bucket.patch() + + print(f"Soft-delete policy is disabled for bucket {bucket_name}") + + +# [END storage_disable_soft_delete] + +if __name__ == "__main__": + disable_soft_delete(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_disable_uniform_bucket_level_access.py b/storage/samples/snippets/storage_disable_uniform_bucket_level_access.py new file mode 100644 index 00000000000..20a045686c3 --- /dev/null +++ b/storage/samples/snippets/storage_disable_uniform_bucket_level_access.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_disable_uniform_bucket_level_access] +from google.cloud import storage + + +def disable_uniform_bucket_level_access(bucket_name): + """Disable uniform bucket-level access for a bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + bucket.iam_configuration.uniform_bucket_level_access_enabled = False + bucket.patch() + + print( + f"Uniform bucket-level access was disabled for {bucket.name}." + ) + + +# [END storage_disable_uniform_bucket_level_access] + +if __name__ == "__main__": + disable_uniform_bucket_level_access(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_disable_versioning.py b/storage/samples/snippets/storage_disable_versioning.py new file mode 100644 index 00000000000..9dfd0ff909a --- /dev/null +++ b/storage/samples/snippets/storage_disable_versioning.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_disable_versioning] +from google.cloud import storage + + +def disable_versioning(bucket_name): + """Disable versioning for this bucket.""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.versioning_enabled = False + bucket.patch() + + print(f"Versioning was disabled for bucket {bucket}") + return bucket + + +# [END storage_disable_versioning] + +if __name__ == "__main__": + disable_versioning(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_download_byte_range.py b/storage/samples/snippets/storage_download_byte_range.py new file mode 100644 index 00000000000..e6143a04f46 --- /dev/null +++ b/storage/samples/snippets/storage_download_byte_range.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_byte_range] +from google.cloud import storage + + +def download_byte_range( + bucket_name, source_blob_name, start_byte, end_byte, destination_file_name +): + """Downloads a blob from the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # source_blob_name = "storage-object-name" + + # The starting byte at which to begin the download + # start_byte = 0 + + # The ending byte at which to end the download + # end_byte = 20 + + # The path to which the file should be downloaded + # destination_file_name = "local/path/to/file" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name, start=start_byte, end=end_byte) + + print( + "Downloaded bytes {} to {} of object {} from bucket {} to local file {}.".format( + start_byte, end_byte, source_blob_name, bucket_name, destination_file_name + ) + ) + + +# [END storage_download_byte_range] + +if __name__ == "__main__": + download_byte_range( + bucket_name=sys.argv[1], + source_blob_name=sys.argv[2], + start_byte=sys.argv[3], + end_byte=sys.argv[4], + destination_file_name=sys.argv[5], + ) diff --git a/storage/samples/snippets/storage_download_encrypted_file.py b/storage/samples/snippets/storage_download_encrypted_file.py new file mode 100644 index 00000000000..8a81b0de597 --- /dev/null +++ b/storage/samples/snippets/storage_download_encrypted_file.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_download_encrypted_file] +import base64 +# [END storage_download_encrypted_file] +import sys +# [START storage_download_encrypted_file] + +from google.cloud import storage + + +def download_encrypted_blob( + bucket_name, + source_blob_name, + destination_file_name, + base64_encryption_key, +): + """Downloads a previously-encrypted blob from Google Cloud Storage. + + The encryption key provided must be the same key provided when uploading + the blob. + """ + # bucket_name = "your-bucket-name" + # source_blob_name = "storage-object-name" + # destination_file_name = "local/path/to/file" + # base64_encryption_key = "base64-encoded-encryption-key" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + blob = bucket.blob(source_blob_name, encryption_key=encryption_key) + + blob.download_to_filename(destination_file_name) + + print( + f"Blob {source_blob_name} downloaded to {destination_file_name}." + ) + + +# [END storage_download_encrypted_file] + +if __name__ == "__main__": + download_encrypted_blob( + bucket_name=sys.argv[1], + source_blob_name=sys.argv[2], + destination_file_name=sys.argv[3], + base64_encryption_key=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_download_file.py b/storage/samples/snippets/storage_download_file.py new file mode 100644 index 00000000000..f8a1c93c83c --- /dev/null +++ b/storage/samples/snippets/storage_download_file.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_file] +from google.cloud import storage + + +def download_blob(bucket_name, source_blob_name, destination_file_name): + """Downloads a blob from the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # source_blob_name = "storage-object-name" + + # The path to which the file should be downloaded + # destination_file_name = "local/path/to/file" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + + print( + "Downloaded storage object {} from bucket {} to local file {}.".format( + source_blob_name, bucket_name, destination_file_name + ) + ) + + +# [END storage_download_file] + +if __name__ == "__main__": + download_blob( + bucket_name=sys.argv[1], + source_blob_name=sys.argv[2], + destination_file_name=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_download_file_requester_pays.py b/storage/samples/snippets/storage_download_file_requester_pays.py new file mode 100644 index 00000000000..babbafda7c2 --- /dev/null +++ b/storage/samples/snippets/storage_download_file_requester_pays.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_file_requester_pays] +from google.cloud import storage + + +def download_file_requester_pays( + bucket_name, project_id, source_blob_name, destination_file_name +): + """Download file using specified project as the requester""" + # bucket_name = "your-bucket-name" + # project_id = "your-project-id" + # source_blob_name = "source-blob-name" + # destination_file_name = "local-destination-file-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name, user_project=project_id) + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + + print( + "Blob {} downloaded to {} using a requester-pays request.".format( + source_blob_name, destination_file_name + ) + ) + + +# [END storage_download_file_requester_pays] + +if __name__ == "__main__": + download_file_requester_pays( + bucket_name=sys.argv[1], + project_id=sys.argv[2], + source_blob_name=sys.argv[3], + destination_file_name=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_download_into_memory.py b/storage/samples/snippets/storage_download_into_memory.py new file mode 100644 index 00000000000..97f677054d5 --- /dev/null +++ b/storage/samples/snippets/storage_download_into_memory.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_file_download_into_memory] +from google.cloud import storage + + +def download_blob_into_memory(bucket_name, blob_name): + """Downloads a blob into memory.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # blob_name = "storage-object-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(blob_name) + contents = blob.download_as_bytes() + + print( + "Downloaded storage object {} from bucket {} as the following bytes object: {}.".format( + blob_name, bucket_name, contents.decode("utf-8") + ) + ) + + +# [END storage_file_download_into_memory] + +if __name__ == "__main__": + download_blob_into_memory( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + ) diff --git a/storage/samples/snippets/storage_download_public_file.py b/storage/samples/snippets/storage_download_public_file.py new file mode 100644 index 00000000000..8fbb68405af --- /dev/null +++ b/storage/samples/snippets/storage_download_public_file.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_public_file] +from google.cloud import storage + + +def download_public_file(bucket_name, source_blob_name, destination_file_name): + """Downloads a public blob from the bucket.""" + # bucket_name = "your-bucket-name" + # source_blob_name = "storage-object-name" + # destination_file_name = "local/path/to/file" + + storage_client = storage.Client.create_anonymous_client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + + print( + "Downloaded public blob {} from bucket {} to {}.".format( + source_blob_name, bucket.name, destination_file_name + ) + ) + + +# [END storage_download_public_file] + +if __name__ == "__main__": + download_public_file( + bucket_name=sys.argv[1], + source_blob_name=sys.argv[2], + destination_file_name=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_download_to_stream.py b/storage/samples/snippets/storage_download_to_stream.py new file mode 100644 index 00000000000..3834e34c917 --- /dev/null +++ b/storage/samples/snippets/storage_download_to_stream.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_stream_file_download] +from google.cloud import storage + + +def download_blob_to_stream(bucket_name, source_blob_name, file_obj): + """Downloads a blob to a stream or other file-like object.""" + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object (blob) + # source_blob_name = "storage-object-name" + + # The stream or file (file-like object) to which the blob will be written + # import io + # file_obj = io.BytesIO() + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client-side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` in that it doesn't + # retrieve metadata from Google Cloud Storage. As we don't use metadata in + # this example, using `Bucket.blob` is preferred here. + blob = bucket.blob(source_blob_name) + blob.download_to_file(file_obj) + + print(f"Downloaded blob {source_blob_name} to file-like object.") + + return file_obj + # Before reading from file_obj, remember to rewind with file_obj.seek(0). + +# [END storage_stream_file_download] diff --git a/storage/samples/snippets/storage_enable_bucket_lifecycle_management.py b/storage/samples/snippets/storage_enable_bucket_lifecycle_management.py new file mode 100644 index 00000000000..0bbff079c8a --- /dev/null +++ b/storage/samples/snippets/storage_enable_bucket_lifecycle_management.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_enable_bucket_lifecycle_management] +from google.cloud import storage + + +def enable_bucket_lifecycle_management(bucket_name): + """Enable lifecycle management for a bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + rules = bucket.lifecycle_rules + + print(f"Lifecycle management rules for bucket {bucket_name} are {list(rules)}") + bucket.add_lifecycle_delete_rule(age=2) + bucket.patch() + + rules = bucket.lifecycle_rules + print(f"Lifecycle management is enable for bucket {bucket_name} and the rules are {list(rules)}") + + return bucket + + +# [END storage_enable_bucket_lifecycle_management] + +if __name__ == "__main__": + enable_bucket_lifecycle_management(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_enable_default_event_based_hold.py b/storage/samples/snippets/storage_enable_default_event_based_hold.py new file mode 100644 index 00000000000..5dfdf94a983 --- /dev/null +++ b/storage/samples/snippets/storage_enable_default_event_based_hold.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_enable_default_event_based_hold] +from google.cloud import storage + + +def enable_default_event_based_hold(bucket_name): + """Enables the default event based hold on a given bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + bucket.default_event_based_hold = True + bucket.patch() + + print(f"Default event based hold was enabled for {bucket_name}") + + +# [END storage_enable_default_event_based_hold] + + +if __name__ == "__main__": + enable_default_event_based_hold(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_enable_requester_pays.py b/storage/samples/snippets/storage_enable_requester_pays.py new file mode 100644 index 00000000000..fbecb04f47c --- /dev/null +++ b/storage/samples/snippets/storage_enable_requester_pays.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_enable_requester_pays] +from google.cloud import storage + + +def enable_requester_pays(bucket_name): + """Enable a bucket's requesterpays metadata""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.requester_pays = True + bucket.patch() + + print(f"Requester Pays has been enabled for {bucket_name}") + + +# [END storage_enable_requester_pays] + +if __name__ == "__main__": + enable_requester_pays(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_enable_uniform_bucket_level_access.py b/storage/samples/snippets/storage_enable_uniform_bucket_level_access.py new file mode 100644 index 00000000000..9ab71ae3730 --- /dev/null +++ b/storage/samples/snippets/storage_enable_uniform_bucket_level_access.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_enable_uniform_bucket_level_access] +from google.cloud import storage + + +def enable_uniform_bucket_level_access(bucket_name): + """Enable uniform bucket-level access for a bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + bucket.patch() + + print( + f"Uniform bucket-level access was enabled for {bucket.name}." + ) + + +# [END storage_enable_uniform_bucket_level_access] + +if __name__ == "__main__": + enable_uniform_bucket_level_access(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_enable_versioning.py b/storage/samples/snippets/storage_enable_versioning.py new file mode 100644 index 00000000000..9cdc980016e --- /dev/null +++ b/storage/samples/snippets/storage_enable_versioning.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_enable_versioning] +from google.cloud import storage + + +def enable_versioning(bucket_name): + """Enable versioning for this bucket.""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + bucket.versioning_enabled = True + bucket.patch() + + print(f"Versioning was enabled for bucket {bucket.name}") + return bucket + + +# [END storage_enable_versioning] + +if __name__ == "__main__": + enable_versioning(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_fileio_pandas.py b/storage/samples/snippets/storage_fileio_pandas.py new file mode 100644 index 00000000000..d4d01edd784 --- /dev/null +++ b/storage/samples/snippets/storage_fileio_pandas.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that creates and consumes a GCS blob using pandas with file-like IO +""" + +# [START storage_fileio_pandas_write] + + +def pandas_write(bucket_name, blob_name): + """Use pandas to interact with GCS using file-like IO""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your new GCS object + # blob_name = "storage-object-name" + + from google.cloud import storage + import pandas as pd + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + with blob.open("w") as f: + df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}) + f.write(df.to_csv(index=False)) + + print(f"Wrote csv with pandas with name {blob_name} from bucket {bucket.name}.") + + +# [END storage_fileio_pandas_write] + + +# [START storage_fileio_pandas_read] + + +def pandas_read(bucket_name, blob_name): + """Use pandas to interact with GCS using file-like IO""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your new GCS object + # blob_name = "storage-object-name" + + from google.cloud import storage + import pandas as pd + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + with blob.open("r") as f: + pd.read_csv(f) + + print(f"Read csv with pandas with name {blob_name} from bucket {bucket.name}.") + + +# [END storage_fileio_pandas_read] + + +if __name__ == "__main__": + pandas_write( + bucket_name=sys.argv[1], + blob_name=sys.argv[2] + ) + + pandas_read( + bucket_name=sys.argv[1], + blob_name=sys.argv[2] + ) diff --git a/storage/samples/snippets/storage_fileio_write_read.py b/storage/samples/snippets/storage_fileio_write_read.py new file mode 100644 index 00000000000..5d35c84ab51 --- /dev/null +++ b/storage/samples/snippets/storage_fileio_write_read.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that writes and read a blob in GCS using file-like IO +""" + +# [START storage_fileio_write_read] +from google.cloud import storage + + +def write_read(bucket_name, blob_name): + """Write and read a blob from GCS using file-like IO""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your new GCS object + # blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Mode can be specified as wb/rb for bytes mode. + # See: https://docs.python.org/3/library/io.html + with blob.open("w") as f: + f.write("Hello world") + + with blob.open("r") as f: + print(f.read()) + + +# [END storage_fileio_write_read] + +if __name__ == "__main__": + write_read( + bucket_name=sys.argv[1], + blob_name=sys.argv[2] + ) diff --git a/storage/samples/snippets/storage_generate_encryption_key.py b/storage/samples/snippets/storage_generate_encryption_key.py new file mode 100644 index 00000000000..dbeb46b914b --- /dev/null +++ b/storage/samples/snippets/storage_generate_encryption_key.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_generate_encryption_key] +import base64 +import os + + +def generate_encryption_key(): + """Generates a 256 bit (32 byte) AES encryption key and prints the + base64 representation. + + This is included for demonstration purposes. You should generate your own + key. Please remember that encryption keys should be handled with a + comprehensive security policy. + """ + key = os.urandom(32) + encoded_key = base64.b64encode(key).decode("utf-8") + + print(f"Base 64 encoded encryption key: {encoded_key}") + + +# [END storage_generate_encryption_key] + +if __name__ == "__main__": + generate_encryption_key() diff --git a/storage/samples/snippets/storage_generate_signed_post_policy_v4.py b/storage/samples/snippets/storage_generate_signed_post_policy_v4.py new file mode 100644 index 00000000000..0c06ddc2fd4 --- /dev/null +++ b/storage/samples/snippets/storage_generate_signed_post_policy_v4.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_generate_signed_post_policy_v4] +import datetime +# [END storage_generate_signed_post_policy_v4] +import sys +# [START storage_generate_signed_post_policy_v4] + +from google.cloud import storage + + +def generate_signed_post_policy_v4(bucket_name, blob_name): + """Generates a v4 POST Policy and prints an HTML form.""" + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + + policy = storage_client.generate_signed_post_policy_v4( + bucket_name, + blob_name, + expiration=datetime.timedelta(minutes=10), + fields={ + 'x-goog-meta-test': 'data' + } + ) + + # Create an HTML form with the provided policy + header = "
\n" + form = header.format(policy["url"]) + + # Include all fields returned in the HTML form as they're required + for key, value in policy["fields"].items(): + form += f" \n" + + form += "
\n" + form += "
\n" + form += "
" + + print(form) + + return form + + +# [END storage_generate_signed_post_policy_v4] + +if __name__ == "__main__": + generate_signed_post_policy_v4( + bucket_name=sys.argv[1], blob_name=sys.argv[2] + ) diff --git a/storage/samples/snippets/storage_generate_signed_url_v2.py b/storage/samples/snippets/storage_generate_signed_url_v2.py new file mode 100644 index 00000000000..9d34630f115 --- /dev/null +++ b/storage/samples/snippets/storage_generate_signed_url_v2.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_generate_signed_url_v2] +import datetime +# [END storage_generate_signed_url_v2] +import sys +# [START storage_generate_signed_url_v2] + +from google.cloud import storage + + +def generate_signed_url(bucket_name, blob_name): + """Generates a v2 signed URL for downloading a blob. + + Note that this method requires a service account key file. + """ + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + url = blob.generate_signed_url( + # This URL is valid for 1 hour + expiration=datetime.timedelta(hours=1), + # Allow GET requests using this URL. + method="GET", + ) + + print(f"The signed url for {blob.name} is {url}") + return url + + +# [END storage_generate_signed_url_v2] + +if __name__ == "__main__": + generate_signed_url(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_generate_signed_url_v4.py b/storage/samples/snippets/storage_generate_signed_url_v4.py new file mode 100644 index 00000000000..8825a7bb525 --- /dev/null +++ b/storage/samples/snippets/storage_generate_signed_url_v4.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_generate_signed_url_v4] +import datetime +# [END storage_generate_signed_url_v4] +import sys +# [START storage_generate_signed_url_v4] + +from google.cloud import storage + + +def generate_download_signed_url_v4(bucket_name, blob_name): + """Generates a v4 signed URL for downloading a blob. + + Note that this method requires a service account key file. + """ + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + url = blob.generate_signed_url( + version="v4", + # This URL is valid for 15 minutes + expiration=datetime.timedelta(minutes=15), + # Allow GET requests using this URL. + method="GET", + ) + + print("Generated GET signed URL:") + print(url) + print("You can use this URL with any user agent, for example:") + print(f"curl '{url}'") + return url + + +# [END storage_generate_signed_url_v4] + +if __name__ == "__main__": + generate_download_signed_url_v4( + bucket_name=sys.argv[1], blob_name=sys.argv[2] + ) diff --git a/storage/samples/snippets/storage_generate_upload_signed_url_v4.py b/storage/samples/snippets/storage_generate_upload_signed_url_v4.py new file mode 100644 index 00000000000..b096fe59eb7 --- /dev/null +++ b/storage/samples/snippets/storage_generate_upload_signed_url_v4.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_generate_upload_signed_url_v4] +import datetime +# [END storage_generate_upload_signed_url_v4] +import sys +# [START storage_generate_upload_signed_url_v4] + +from google.cloud import storage + + +def generate_upload_signed_url_v4(bucket_name, blob_name): + """Generates a v4 signed URL for uploading a blob using HTTP PUT. + + Note that this method requires a service account key file. + """ + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + url = blob.generate_signed_url( + version="v4", + # This URL is valid for 15 minutes + expiration=datetime.timedelta(minutes=15), + # Allow PUT requests using this URL. + method="PUT", + content_type="application/octet-stream", + ) + + print("Generated PUT signed URL:") + print(url) + print("You can use this URL with any user agent, for example:") + print( + "curl -X PUT -H 'Content-Type: application/octet-stream' " + "--upload-file my-file '{}'".format(url) + ) + return url + + +# [END storage_generate_upload_signed_url_v4] + + +if __name__ == "__main__": + generate_upload_signed_url_v4( + bucket_name=sys.argv[1], blob_name=sys.argv[2] + ) diff --git a/storage/samples/snippets/storage_get_autoclass.py b/storage/samples/snippets/storage_get_autoclass.py new file mode 100644 index 00000000000..30fa0c4f6b3 --- /dev/null +++ b/storage/samples/snippets/storage_get_autoclass.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_autoclass] +from google.cloud import storage + + +def get_autoclass(bucket_name): + """Get the Autoclass setting for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + autoclass_enabled = bucket.autoclass_enabled + autoclass_toggle_time = bucket.autoclass_toggle_time + terminal_storage_class = bucket.autoclass_terminal_storage_class + tsc_update_time = bucket.autoclass_terminal_storage_class_update_time + + print(f"Autoclass enabled is set to {autoclass_enabled} for {bucket.name} at {autoclass_toggle_time}.") + print(f"Autoclass terminal storage class is set to {terminal_storage_class} for {bucket.name} at {tsc_update_time}.") + + return bucket + + +# [END storage_get_autoclass] + +if __name__ == "__main__": + get_autoclass(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_bucket_encryption_enforcement_config.py b/storage/samples/snippets/storage_get_bucket_encryption_enforcement_config.py new file mode 100644 index 00000000000..033dcc8224c --- /dev/null +++ b/storage/samples/snippets/storage_get_bucket_encryption_enforcement_config.py @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_get_bucket_encryption_enforcement_config] +from google.cloud import storage + + +def get_bucket_encryption_enforcement_config(bucket_name): + """Gets the bucket encryption enforcement configuration.""" + # The ID of your GCS bucket + # bucket_name = "your-unique-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + print(f"Encryption Enforcement Config for bucket {bucket.name}:") + + cmek_config = bucket.encryption.customer_managed_encryption_enforcement_config + csek_config = bucket.encryption.customer_supplied_encryption_enforcement_config + gmek_config = bucket.encryption.google_managed_encryption_enforcement_config + + print( + f"Customer-managed encryption enforcement config restriction mode: {cmek_config.restriction_mode if cmek_config else None}" + ) + print( + f"Customer-supplied encryption enforcement config restriction mode: {csek_config.restriction_mode if csek_config else None}" + ) + print( + f"Google-managed encryption enforcement config restriction mode: {gmek_config.restriction_mode if gmek_config else None}" + ) + + +# [END storage_get_bucket_encryption_enforcement_config] + + +if __name__ == "__main__": + get_bucket_encryption_enforcement_config(bucket_name="your-unique-bucket-name") diff --git a/storage/samples/snippets/storage_get_bucket_labels.py b/storage/samples/snippets/storage_get_bucket_labels.py new file mode 100644 index 00000000000..b3bcd6208b8 --- /dev/null +++ b/storage/samples/snippets/storage_get_bucket_labels.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_get_bucket_labels] +import pprint +# [END storage_get_bucket_labels] +import sys +# [START storage_get_bucket_labels] + +from google.cloud import storage + + +def get_bucket_labels(bucket_name): + """Prints out a bucket's labels.""" + # bucket_name = 'your-bucket-name' + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + + labels = bucket.labels + pprint.pprint(labels) + + +# [END storage_get_bucket_labels] + +if __name__ == "__main__": + get_bucket_labels(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_bucket_metadata.py b/storage/samples/snippets/storage_get_bucket_metadata.py new file mode 100644 index 00000000000..c86e154de10 --- /dev/null +++ b/storage/samples/snippets/storage_get_bucket_metadata.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +# [START storage_get_bucket_metadata] + +from google.cloud import storage + + +def bucket_metadata(bucket_name): + """Prints out a bucket's metadata.""" + # bucket_name = 'your-bucket-name' + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + print(f"ID: {bucket.id}") + print(f"Name: {bucket.name}") + print(f"Storage Class: {bucket.storage_class}") + print(f"Location: {bucket.location}") + print(f"Location Type: {bucket.location_type}") + print(f"Cors: {bucket.cors}") + print(f"Default Event Based Hold: {bucket.default_event_based_hold}") + print(f"Default KMS Key Name: {bucket.default_kms_key_name}") + print(f"Metageneration: {bucket.metageneration}") + print( + f"Public Access Prevention: {bucket.iam_configuration.public_access_prevention}" + ) + print(f"Retention Effective Time: {bucket.retention_policy_effective_time}") + print(f"Retention Period: {bucket.retention_period}") + print(f"Retention Policy Locked: {bucket.retention_policy_locked}") + print(f"Object Retention Mode: {bucket.object_retention_mode}") + print(f"Requester Pays: {bucket.requester_pays}") + print(f"Self Link: {bucket.self_link}") + print(f"Time Created: {bucket.time_created}") + print(f"Versioning Enabled: {bucket.versioning_enabled}") + print(f"Labels: {bucket.labels}") + + +# [END storage_get_bucket_metadata] + +if __name__ == "__main__": + bucket_metadata(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_default_event_based_hold.py b/storage/samples/snippets/storage_get_default_event_based_hold.py new file mode 100644 index 00000000000..08a05f8ef55 --- /dev/null +++ b/storage/samples/snippets/storage_get_default_event_based_hold.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_default_event_based_hold] +from google.cloud import storage + + +def get_default_event_based_hold(bucket_name): + """Gets the default event based hold on a given bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + + if bucket.default_event_based_hold: + print(f"Default event-based hold is enabled for {bucket_name}") + else: + print( + f"Default event-based hold is not enabled for {bucket_name}" + ) + + +# [END storage_get_default_event_based_hold] + + +if __name__ == "__main__": + get_default_event_based_hold(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_hmac_key.py b/storage/samples/snippets/storage_get_hmac_key.py new file mode 100644 index 00000000000..82b28ff99e4 --- /dev/null +++ b/storage/samples/snippets/storage_get_hmac_key.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_hmac_key] +from google.cloud import storage + + +def get_key(access_id, project_id): + """ + Retrieve the HMACKeyMetadata with the given access id. + """ + # project_id = "Your Google Cloud project ID" + # access_id = "ID of an HMAC key" + + storage_client = storage.Client(project=project_id) + + hmac_key = storage_client.get_hmac_key_metadata( + access_id, project_id=project_id + ) + + print("The HMAC key metadata is:") + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") + return hmac_key + + +# [END storage_get_hmac_key] + +if __name__ == "__main__": + get_key(access_id=sys.argv[1], project_id=sys.argv[2]) diff --git a/storage/samples/snippets/storage_get_metadata.py b/storage/samples/snippets/storage_get_metadata.py new file mode 100644 index 00000000000..1e332b44565 --- /dev/null +++ b/storage/samples/snippets/storage_get_metadata.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_metadata] +from google.cloud import storage + + +def blob_metadata(bucket_name, blob_name): + """Prints out a blob's metadata.""" + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Retrieve a blob, and its metadata, from Google Cloud Storage. + # Note that `get_blob` differs from `Bucket.blob`, which does not + # make an HTTP request. + blob = bucket.get_blob(blob_name) + + print(f"Blob: {blob.name}") + print(f"Blob finalization: {blob.finalized_time}") + print(f"Bucket: {blob.bucket.name}") + print(f"Storage class: {blob.storage_class}") + print(f"ID: {blob.id}") + print(f"Size: {blob.size} bytes") + print(f"Updated: {blob.updated}") + print(f"Generation: {blob.generation}") + print(f"Metageneration: {blob.metageneration}") + print(f"Etag: {blob.etag}") + print(f"Owner: {blob.owner}") + print(f"Component count: {blob.component_count}") + print(f"Crc32c: {blob.crc32c}") + print(f"md5_hash: {blob.md5_hash}") + print(f"Cache-control: {blob.cache_control}") + print(f"Content-type: {blob.content_type}") + print(f"Content-disposition: {blob.content_disposition}") + print(f"Content-encoding: {blob.content_encoding}") + print(f"Content-language: {blob.content_language}") + print(f"Metadata: {blob.metadata}") + print(f"Medialink: {blob.media_link}") + print(f"Custom Time: {blob.custom_time}") + print("Temporary hold: ", "enabled" if blob.temporary_hold else "disabled") + print( + "Event based hold: ", + "enabled" if blob.event_based_hold else "disabled", + ) + print(f"Retention mode: {blob.retention.mode}") + print(f"Retention retain until time: {blob.retention.retain_until_time}") + if blob.retention_expiration_time: + print( + f"retentionExpirationTime: {blob.retention_expiration_time}" + ) + + +# [END storage_get_metadata] + +if __name__ == "__main__": + blob_metadata(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_get_public_access_prevention.py b/storage/samples/snippets/storage_get_public_access_prevention.py new file mode 100644 index 00000000000..275b84e3553 --- /dev/null +++ b/storage/samples/snippets/storage_get_public_access_prevention.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_public_access_prevention] +from google.cloud import storage + + +def get_public_access_prevention(bucket_name): + """Gets the public access prevention setting (either 'inherited' or 'enforced') for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + iam_configuration = bucket.iam_configuration + + print( + f"Public access prevention is {iam_configuration.public_access_prevention} for {bucket.name}." + ) + + +# [END storage_get_public_access_prevention] + +if __name__ == "__main__": + get_public_access_prevention(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_requester_pays_status.py b/storage/samples/snippets/storage_get_requester_pays_status.py new file mode 100644 index 00000000000..a2eeb34d70f --- /dev/null +++ b/storage/samples/snippets/storage_get_requester_pays_status.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_requester_pays_status] +from google.cloud import storage + + +def get_requester_pays_status(bucket_name): + """Get a bucket's requester pays metadata""" + # bucket_name = "my-bucket" + storage_client = storage.Client() + + bucket = storage_client.get_bucket(bucket_name) + requester_pays_status = bucket.requester_pays + + if requester_pays_status: + print(f"Requester Pays is enabled for {bucket_name}") + else: + print(f"Requester Pays is disabled for {bucket_name}") + + +# [END storage_get_requester_pays_status] + +if __name__ == "__main__": + get_requester_pays_status(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_retention_policy.py b/storage/samples/snippets/storage_get_retention_policy.py new file mode 100644 index 00000000000..215f80d5a59 --- /dev/null +++ b/storage/samples/snippets/storage_get_retention_policy.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_retention_policy] +from google.cloud import storage + + +def get_retention_policy(bucket_name): + """Gets the retention policy on a given bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.reload() + + print(f"Retention Policy for {bucket_name}") + print(f"Retention Period: {bucket.retention_period}") + if bucket.retention_policy_locked: + print("Retention Policy is locked") + + if bucket.retention_policy_effective_time: + print( + f"Effective Time: {bucket.retention_policy_effective_time}" + ) + + +# [END storage_get_retention_policy] + + +if __name__ == "__main__": + get_retention_policy(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_rpo.py b/storage/samples/snippets/storage_get_rpo.py new file mode 100644 index 00000000000..ab40ca3a5f4 --- /dev/null +++ b/storage/samples/snippets/storage_get_rpo.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that gets RPO (Recovery Point Objective) of a bucket +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_get_rpo] + +from google.cloud import storage + + +def get_rpo(bucket_name): + """Gets the RPO of the bucket""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + rpo = bucket.rpo + + print(f"RPO for {bucket.name} is {rpo}.") + + +# [END storage_get_rpo] + +if __name__ == "__main__": + get_rpo(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_service_account.py b/storage/samples/snippets/storage_get_service_account.py new file mode 100644 index 00000000000..5ac0e563835 --- /dev/null +++ b/storage/samples/snippets/storage_get_service_account.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_get_service_account] +from google.cloud import storage + + +def get_service_account(): + """Get the service account email""" + storage_client = storage.Client() + + email = storage_client.get_service_account_email() + print( + f"The GCS service account for project {storage_client.project} is: {email} " + ) + + +# [END storage_get_service_account] + +if __name__ == "__main__": + get_service_account() diff --git a/storage/samples/snippets/storage_get_soft_delete_policy.py b/storage/samples/snippets/storage_get_soft_delete_policy.py new file mode 100644 index 00000000000..99c4e572a24 --- /dev/null +++ b/storage/samples/snippets/storage_get_soft_delete_policy.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_soft_delete_policy] +from google.cloud import storage + + +def get_soft_delete_policy(bucket_name): + """Gets the soft-delete policy of the bucket""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + print(f"Soft-delete policy for {bucket_name}") + if ( + bucket.soft_delete_policy + and bucket.soft_delete_policy.retention_duration_seconds + ): + print("Object soft-delete policy is enabled") + print( + f"Object retention duration: {bucket.soft_delete_policy.retention_duration_seconds} seconds" + ) + print(f"Policy effective time: {bucket.soft_delete_policy.effective_time}") + else: + print("Object soft-delete policy is disabled") + + +# [END storage_get_soft_delete_policy] + +if __name__ == "__main__": + get_soft_delete_policy(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_get_soft_deleted_bucket.py b/storage/samples/snippets/storage_get_soft_deleted_bucket.py new file mode 100644 index 00000000000..2b795504657 --- /dev/null +++ b/storage/samples/snippets/storage_get_soft_deleted_bucket.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +# [START storage_get_soft_deleted_bucket] + +from google.cloud import storage + + +def get_soft_deleted_bucket(bucket_name, generation): + """Prints out a soft-deleted bucket's metadata. + + Args: + bucket_name: str + The name of the bucket to get. + + generation: + The generation of the bucket. + + """ + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name, soft_deleted=True, generation=generation) + + print(f"ID: {bucket.id}") + print(f"Name: {bucket.name}") + print(f"Soft Delete time: {bucket.soft_delete_time}") + print(f"Hard Delete Time : {bucket.hard_delete_time}") + + +# [END storage_get_soft_deleted_bucket] + +if __name__ == "__main__": + get_soft_deleted_bucket(bucket_name=sys.argv[1], generation=sys.argv[2]) diff --git a/storage/samples/snippets/storage_get_uniform_bucket_level_access.py b/storage/samples/snippets/storage_get_uniform_bucket_level_access.py new file mode 100644 index 00000000000..206b9f1ff11 --- /dev/null +++ b/storage/samples/snippets/storage_get_uniform_bucket_level_access.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_get_uniform_bucket_level_access] +from google.cloud import storage + + +def get_uniform_bucket_level_access(bucket_name): + """Get uniform bucket-level access for a bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + iam_configuration = bucket.iam_configuration + + if iam_configuration.uniform_bucket_level_access_enabled: + print( + f"Uniform bucket-level access is enabled for {bucket.name}." + ) + print( + "Bucket will be locked on {}.".format( + iam_configuration.uniform_bucket_level_locked_time + ) + ) + else: + print( + f"Uniform bucket-level access is disabled for {bucket.name}." + ) + + +# [END storage_get_uniform_bucket_level_access] + +if __name__ == "__main__": + get_uniform_bucket_level_access(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_list_bucket_notifications.py b/storage/samples/snippets/storage_list_bucket_notifications.py new file mode 100644 index 00000000000..0d25138bc90 --- /dev/null +++ b/storage/samples/snippets/storage_list_bucket_notifications.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that lists notification configurations for a bucket. +This sample is used on this page: + https://cloud.google.com/storage/docs/reporting-changes +For more information, see README.md. +""" + +# [START storage_list_bucket_notifications] +from google.cloud import storage + + +def list_bucket_notifications(bucket_name): + """Lists notification configurations for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + notifications = bucket.list_notifications() + + for notification in notifications: + print(f"Notification ID: {notification.notification_id}") + +# [END storage_list_bucket_notifications] + + +if __name__ == "__main__": + list_bucket_notifications(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_list_buckets.py b/storage/samples/snippets/storage_list_buckets.py new file mode 100644 index 00000000000..f5897e47a42 --- /dev/null +++ b/storage/samples/snippets/storage_list_buckets.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_list_buckets] +from google.cloud import storage + + +def list_buckets(): + """Lists all buckets.""" + + storage_client = storage.Client() + buckets = storage_client.list_buckets() + + for bucket in buckets: + print(bucket.name) + + +# [END storage_list_buckets] + + +if __name__ == "__main__": + list_buckets() diff --git a/storage/samples/snippets/storage_list_buckets_partial_success.py b/storage/samples/snippets/storage_list_buckets_partial_success.py new file mode 100644 index 00000000000..bea4c9ed35c --- /dev/null +++ b/storage/samples/snippets/storage_list_buckets_partial_success.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright 2025 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_list_buckets_partial_success] +from google.cloud import storage + + +def list_buckets_with_partial_success(): + """Lists buckets and includes unreachable buckets in the response.""" + + storage_client = storage.Client() + + buckets_iterator = storage_client.list_buckets(return_partial_success=True) + + for page in buckets_iterator.pages: + if page.unreachable: + print("Unreachable locations in this page:") + for location in page.unreachable: + print(location) + + print("Reachable buckets in this page:") + for bucket in page: + print(bucket.name) + + +# [END storage_list_buckets_partial_success] + + +if __name__ == "__main__": + list_buckets_with_partial_success() diff --git a/storage/samples/snippets/storage_list_file_archived_generations.py b/storage/samples/snippets/storage_list_file_archived_generations.py new file mode 100644 index 00000000000..419cc3da408 --- /dev/null +++ b/storage/samples/snippets/storage_list_file_archived_generations.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_list_file_archived_generations] +from google.cloud import storage + + +def list_file_archived_generations(bucket_name): + """Lists all the blobs in the bucket with generation.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + + blobs = storage_client.list_blobs(bucket_name, versions=True) + + for blob in blobs: + print(f"{blob.name},{blob.generation}") + + +# [END storage_list_file_archived_generations] + + +if __name__ == "__main__": + list_file_archived_generations(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_list_files.py b/storage/samples/snippets/storage_list_files.py new file mode 100644 index 00000000000..5e80c833afe --- /dev/null +++ b/storage/samples/snippets/storage_list_files.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_list_files] +from google.cloud import storage + + +def list_blobs(bucket_name): + """Lists all the blobs in the bucket.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + + # Note: Client.list_blobs requires at least package version 1.17.0. + blobs = storage_client.list_blobs(bucket_name) + + # Note: The call returns a response only when the iterator is consumed. + for blob in blobs: + print(blob.name) + + +# [END storage_list_files] + + +if __name__ == "__main__": + list_blobs(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_list_files_with_prefix.py b/storage/samples/snippets/storage_list_files_with_prefix.py new file mode 100644 index 00000000000..7f877d1d6bc --- /dev/null +++ b/storage/samples/snippets/storage_list_files_with_prefix.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_list_files_with_prefix] +from google.cloud import storage + + +def list_blobs_with_prefix(bucket_name, prefix, delimiter=None): + """Lists all the blobs in the bucket that begin with the prefix. + + This can be used to list all blobs in a "folder", e.g. "public/". + + The delimiter argument can be used to restrict the results to only the + "files" in the given "folder". Without the delimiter, the entire tree under + the prefix is returned. For example, given these blobs: + + a/1.txt + a/b/2.txt + + If you specify prefix ='a/', without a delimiter, you'll get back: + + a/1.txt + a/b/2.txt + + However, if you specify prefix='a/' and delimiter='/', you'll get back + only the file directly under 'a/': + + a/1.txt + + As part of the response, you'll also get back a blobs.prefixes entity + that lists the "subfolders" under `a/`: + + a/b/ + + + Note: If you only want to list prefixes a/b/ and don't want to iterate over + blobs, you can do + + ``` + for page in blobs.pages: + print(page.prefixes) + ``` + """ + + storage_client = storage.Client() + + # Note: Client.list_blobs requires at least package version 1.17.0. + blobs = storage_client.list_blobs( + bucket_name, prefix=prefix, delimiter=delimiter + ) + + # Note: The call returns a response only when the iterator is consumed. + print("Blobs:") + for blob in blobs: + print(blob.name) + + if delimiter: + print("Prefixes:") + for prefix in blobs.prefixes: + print(prefix) + + +# [END storage_list_files_with_prefix] + +if __name__ == "__main__": + list_blobs_with_prefix( + bucket_name=sys.argv[1], prefix=sys.argv[2], delimiter=sys.argv[3] + ) diff --git a/storage/samples/snippets/storage_list_hmac_keys.py b/storage/samples/snippets/storage_list_hmac_keys.py new file mode 100644 index 00000000000..a09616fa519 --- /dev/null +++ b/storage/samples/snippets/storage_list_hmac_keys.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_list_hmac_keys] +from google.cloud import storage + + +def list_keys(project_id): + """ + List all HMAC keys associated with the project. + """ + # project_id = "Your Google Cloud project ID" + + storage_client = storage.Client(project=project_id) + hmac_keys = storage_client.list_hmac_keys(project_id=project_id) + print("HMAC Keys:") + for hmac_key in hmac_keys: + print( + f"Service Account Email: {hmac_key.service_account_email}" + ) + print(f"Access ID: {hmac_key.access_id}") + return hmac_keys + + +# [END storage_list_hmac_keys] + +if __name__ == "__main__": + list_keys(project_id=sys.argv[1]) diff --git a/storage/samples/snippets/storage_list_soft_deleted_buckets.py b/storage/samples/snippets/storage_list_soft_deleted_buckets.py new file mode 100644 index 00000000000..16abd90f02a --- /dev/null +++ b/storage/samples/snippets/storage_list_soft_deleted_buckets.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_list_soft_deleted_buckets] + +from google.cloud import storage + + +def list_soft_deleted_buckets(): + """Lists all soft-deleted buckets.""" + + storage_client = storage.Client() + buckets = storage_client.list_buckets(soft_deleted=True) + + for bucket in buckets: + print(bucket.name) + + +# [END storage_list_soft_deleted_buckets] + + +if __name__ == "__main__": + list_soft_deleted_buckets() diff --git a/storage/samples/snippets/storage_list_soft_deleted_object_versions.py b/storage/samples/snippets/storage_list_soft_deleted_object_versions.py new file mode 100644 index 00000000000..ecb9851c454 --- /dev/null +++ b/storage/samples/snippets/storage_list_soft_deleted_object_versions.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_list_soft_deleted_object_versions] +from google.cloud import storage + + +def list_soft_deleted_object_versions(bucket_name, blob_name): + """Lists all versions of a soft-deleted object in the bucket.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + + storage_client = storage.Client() + blobs = storage_client.list_blobs(bucket_name, prefix=blob_name, soft_deleted=True) + + # Note: The call returns a response only when the iterator is consumed. + for blob in blobs: + print( + f"Version ID: {blob.generation}, Soft Delete Time: {blob.soft_delete_time}" + ) + + +# [END storage_list_soft_deleted_object_versions] + +if __name__ == "__main__": + list_soft_deleted_object_versions(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_list_soft_deleted_objects.py b/storage/samples/snippets/storage_list_soft_deleted_objects.py new file mode 100644 index 00000000000..764cac56a6d --- /dev/null +++ b/storage/samples/snippets/storage_list_soft_deleted_objects.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_list_soft_deleted_objects] +from google.cloud import storage + + +def list_soft_deleted_objects(bucket_name): + """Lists all soft-deleted objects in the bucket.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + blobs = storage_client.list_blobs(bucket_name, soft_deleted=True) + + # Note: The call returns a response only when the iterator is consumed. + for blob in blobs: + print( + f"Name: {blob.name}, Generation: {blob.generation}, Soft Delete Time: {blob.soft_delete_time}" + ) + + +# [END storage_list_soft_deleted_objects] + +if __name__ == "__main__": + list_soft_deleted_objects(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_lock_retention_policy.py b/storage/samples/snippets/storage_lock_retention_policy.py new file mode 100644 index 00000000000..adff364d749 --- /dev/null +++ b/storage/samples/snippets/storage_lock_retention_policy.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_lock_retention_policy] +from google.cloud import storage + + +def lock_retention_policy(bucket_name): + """Locks the retention policy on a given bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + # get_bucket gets the current metageneration value for the bucket, + # required by lock_retention_policy. + bucket = storage_client.get_bucket(bucket_name) + + # Warning: Once a retention policy is locked it cannot be unlocked + # and retention period can only be increased. + bucket.lock_retention_policy() + + print(f"Retention policy for {bucket_name} is now locked") + print( + f"Retention policy effective as of {bucket.retention_policy_effective_time}" + ) + + +# [END storage_lock_retention_policy] + + +if __name__ == "__main__": + lock_retention_policy(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_make_public.py b/storage/samples/snippets/storage_make_public.py new file mode 100644 index 00000000000..489508cf674 --- /dev/null +++ b/storage/samples/snippets/storage_make_public.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_make_public] +from google.cloud import storage + + +def make_blob_public(bucket_name, blob_name): + """Makes a blob publicly accessible.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.make_public() + + print( + f"Blob {blob.name} is publicly accessible at {blob.public_url}" + ) + + +# [END storage_make_public] + +if __name__ == "__main__": + make_blob_public(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_move_file.py b/storage/samples/snippets/storage_move_file.py new file mode 100644 index 00000000000..b2e5144d0b2 --- /dev/null +++ b/storage/samples/snippets/storage_move_file.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright 2019 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_move_file] +from google.cloud import storage + + +def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name,): + """Moves a blob from one bucket to another with a new name.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of your GCS object + # blob_name = "your-object-name" + # The ID of the bucket to move the object to + # destination_bucket_name = "destination-bucket-name" + # The ID of your new GCS object (optional) + # destination_blob_name = "destination-object-name" + + storage_client = storage.Client() + + source_bucket = storage_client.bucket(bucket_name) + source_blob = source_bucket.blob(blob_name) + destination_bucket = storage_client.bucket(destination_bucket_name) + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + # There is also an `if_source_generation_match` parameter, which is not used in this example. + destination_generation_match_precondition = 0 + + blob_copy = source_bucket.copy_blob( + source_blob, destination_bucket, destination_blob_name, if_generation_match=destination_generation_match_precondition, + ) + source_bucket.delete_blob(blob_name) + + print( + "Blob {} in bucket {} moved to blob {} in bucket {}.".format( + source_blob.name, + source_bucket.name, + blob_copy.name, + destination_bucket.name, + ) + ) + + +# [END storage_move_file] + +if __name__ == "__main__": + move_blob( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + destination_bucket_name=sys.argv[3], + destination_blob_name=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_move_file_atomically.py b/storage/samples/snippets/storage_move_file_atomically.py new file mode 100644 index 00000000000..d659cf3661a --- /dev/null +++ b/storage/samples/snippets/storage_move_file_atomically.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_move_object] +from google.cloud import storage + + +def move_object(bucket_name: str, blob_name: str, new_blob_name: str) -> None: + """Moves a blob to a new name within the same bucket using the move API.""" + # The name of your GCS bucket + # bucket_name = "your-bucket-name" + + # The name of your GCS object to move + # blob_name = "your-file-name" + + # The new name of the GCS object + # new_blob_name = "new-file-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob_to_move = bucket.blob(blob_name) + + # Use move_blob to perform an efficient, server-side move. + moved_blob = bucket.move_blob( + blob=blob_to_move, new_name=new_blob_name + ) + + print(f"Blob {blob_to_move.name} has been moved to {moved_blob.name}.") + + +# [END storage_move_object] + +if __name__ == "__main__": + move_object( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + new_blob_name=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_object_csek_to_cmek.py b/storage/samples/snippets/storage_object_csek_to_cmek.py new file mode 100644 index 00000000000..9a915f08d63 --- /dev/null +++ b/storage/samples/snippets/storage_object_csek_to_cmek.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import sys + +# [START storage_object_csek_to_cmek] +from google.cloud import storage + + +def object_csek_to_cmek(bucket_name, blob_name, encryption_key, kms_key_name): + """Change a blob's customer-supplied encryption key to KMS key""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # encryption_key = "TIbv/fjexq+VmtXzAlc63J4z5kFmWJ6NdAPQulQBT7g=" + # kms_key_name = "projects/PROJ/locations/LOC/keyRings/RING/cryptoKey/KEY" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + current_encryption_key = base64.b64decode(encryption_key) + source_blob = bucket.blob(blob_name, encryption_key=current_encryption_key) + destination_blob = bucket.blob(blob_name, kms_key_name=kms_key_name) + generation_match_precondition = None + token = None + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to rewrite is aborted if the object's + # generation number does not match your precondition. + source_blob.reload() # Fetch blob metadata to use in generation_match_precondition. + generation_match_precondition = source_blob.generation + + while True: + token, bytes_rewritten, total_bytes = destination_blob.rewrite( + source_blob, token=token, if_generation_match=generation_match_precondition + ) + if token is None: + break + + print( + "Blob {} in bucket {} is now managed by the KMS key {} instead of a customer-supplied encryption key".format( + blob_name, bucket_name, kms_key_name + ) + ) + return destination_blob + + +# [END storage_object_csek_to_cmek] + +if __name__ == "__main__": + object_csek_to_cmek( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + encryption_key=sys.argv[3], + kms_key_name=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_object_get_kms_key.py b/storage/samples/snippets/storage_object_get_kms_key.py new file mode 100644 index 00000000000..7604e6eba6e --- /dev/null +++ b/storage/samples/snippets/storage_object_get_kms_key.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_object_get_kms_key] +from google.cloud import storage + + +def object_get_kms_key(bucket_name, blob_name): + """Retrieve the KMS key of a blob""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + blob = bucket.get_blob(blob_name) + + kms_key = blob.kms_key_name + + print(f"The KMS key of a blob is {blob.kms_key_name}") + return kms_key + + +# [END storage_object_get_kms_key] + +if __name__ == "__main__": + object_get_kms_key(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_print_bucket_acl.py b/storage/samples/snippets/storage_print_bucket_acl.py new file mode 100644 index 00000000000..55417f1bc77 --- /dev/null +++ b/storage/samples/snippets/storage_print_bucket_acl.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_print_bucket_acl] +from google.cloud import storage + + +def print_bucket_acl(bucket_name): + """Prints out a bucket's access control list.""" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + for entry in bucket.acl: + print(f"{entry['role']}: {entry['entity']}") + + +# [END storage_print_bucket_acl] + +if __name__ == "__main__": + print_bucket_acl(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_print_bucket_acl_for_user.py b/storage/samples/snippets/storage_print_bucket_acl_for_user.py new file mode 100644 index 00000000000..fa786d03af9 --- /dev/null +++ b/storage/samples/snippets/storage_print_bucket_acl_for_user.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_print_bucket_acl_for_user] +from google.cloud import storage + + +def print_bucket_acl_for_user(bucket_name, user_email): + """Prints out a bucket's access control list for a given user.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # get the roles for different types of entities. + roles = bucket.acl.user(user_email).get_roles() + + print(roles) + + +# [END storage_print_bucket_acl_for_user] + +if __name__ == "__main__": + print_bucket_acl_for_user(bucket_name=sys.argv[1], user_email=sys.argv[2]) diff --git a/storage/samples/snippets/storage_print_file_acl.py b/storage/samples/snippets/storage_print_file_acl.py new file mode 100644 index 00000000000..8dfc4e98464 --- /dev/null +++ b/storage/samples/snippets/storage_print_file_acl.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_print_file_acl] +from google.cloud import storage + + +def print_blob_acl(bucket_name, blob_name): + """Prints out a blob's access control list.""" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + for entry in blob.acl: + print(f"{entry['role']}: {entry['entity']}") + + +# [END storage_print_file_acl] + +if __name__ == "__main__": + print_blob_acl(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_print_file_acl_for_user.py b/storage/samples/snippets/storage_print_file_acl_for_user.py new file mode 100644 index 00000000000..e399b916013 --- /dev/null +++ b/storage/samples/snippets/storage_print_file_acl_for_user.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_print_file_acl_for_user] +from google.cloud import storage + + +def print_blob_acl_for_user(bucket_name, blob_name, user_email): + """Prints out a blob's access control list for a given user.""" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Reload fetches the current ACL from Cloud Storage. + blob.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # get the roles for different types of entities. + roles = blob.acl.user(user_email).get_roles() + + print(roles) + + +# [END storage_print_file_acl_for_user] + +if __name__ == "__main__": + print_blob_acl_for_user( + bucket_name=sys.argv[1], blob_name=sys.argv[2], user_email=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_print_pubsub_bucket_notification.py b/storage/samples/snippets/storage_print_pubsub_bucket_notification.py new file mode 100644 index 00000000000..3df45dc1f57 --- /dev/null +++ b/storage/samples/snippets/storage_print_pubsub_bucket_notification.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that gets a notification configuration for a bucket. +This sample is used on this page: + https://cloud.google.com/storage/docs/reporting-changes +For more information, see README.md. +""" + +# [START storage_print_pubsub_bucket_notification] +from google.cloud import storage + + +def print_pubsub_bucket_notification(bucket_name, notification_id): + """Gets a notification configuration for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of the notification + # notification_id = "your-notification-id" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + notification = bucket.get_notification(notification_id) + + print(f"Notification ID: {notification.notification_id}") + print(f"Topic Name: {notification.topic_name}") + print(f"Event Types: {notification.event_types}") + print(f"Custom Attributes: {notification.custom_attributes}") + print(f"Payload Format: {notification.payload_format}") + print(f"Blob Name Prefix: {notification.blob_name_prefix}") + print(f"Etag: {notification.etag}") + print(f"Self Link: {notification.self_link}") + +# [END storage_print_pubsub_bucket_notification] + + +if __name__ == "__main__": + print_pubsub_bucket_notification(bucket_name=sys.argv[1], notification_id=sys.argv[2]) diff --git a/storage/samples/snippets/storage_release_event_based_hold.py b/storage/samples/snippets/storage_release_event_based_hold.py new file mode 100644 index 00000000000..6b4a2ccb51c --- /dev/null +++ b/storage/samples/snippets/storage_release_event_based_hold.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_release_event_based_hold] +from google.cloud import storage + + +def release_event_based_hold(bucket_name, blob_name): + """Releases the event based hold on a given blob""" + + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + metageneration_match_precondition = None + + # Optional: set a metageneration-match precondition to avoid potential race + # conditions and data corruptions. The request to patch is aborted if the + # object's metageneration does not match your precondition. + blob.reload() # Fetch blob metadata to use in metageneration_match_precondition. + metageneration_match_precondition = blob.metageneration + + blob.event_based_hold = False + blob.patch(if_metageneration_match=metageneration_match_precondition) + + print(f"Event based hold was released for {blob_name}") + + +# [END storage_release_event_based_hold] + + +if __name__ == "__main__": + release_event_based_hold(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_release_temporary_hold.py b/storage/samples/snippets/storage_release_temporary_hold.py new file mode 100644 index 00000000000..64c7607c182 --- /dev/null +++ b/storage/samples/snippets/storage_release_temporary_hold.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_release_temporary_hold] +from google.cloud import storage + + +def release_temporary_hold(bucket_name, blob_name): + """Releases the temporary hold on a given blob""" + + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + metageneration_match_precondition = None + + # Optional: set a metageneration-match precondition to avoid potential race + # conditions and data corruptions. The request to patch is aborted if the + # object's metageneration does not match your precondition. + blob.reload() # Fetch blob metadata to use in metageneration_match_precondition. + metageneration_match_precondition = blob.metageneration + + blob.temporary_hold = False + blob.patch(if_metageneration_match=metageneration_match_precondition) + + print("Temporary hold was release for #{blob_name}") + + +# [END storage_release_temporary_hold] + + +if __name__ == "__main__": + release_temporary_hold(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_remove_bucket_conditional_iam_binding.py b/storage/samples/snippets/storage_remove_bucket_conditional_iam_binding.py new file mode 100644 index 00000000000..242544d8ed2 --- /dev/null +++ b/storage/samples/snippets/storage_remove_bucket_conditional_iam_binding.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_bucket_conditional_iam_binding] +from google.cloud import storage + + +def remove_bucket_conditional_iam_binding( + bucket_name, role, title, description, expression +): + """Remove a conditional IAM binding from a bucket's IAM policy.""" + # bucket_name = "your-bucket-name" + # role = "IAM role, e.g. roles/storage.objectViewer" + # title = "Condition title." + # description = "Condition description." + # expression = "Condition expression." + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy(requested_policy_version=3) + + # Set the policy's version to 3 to use condition in bindings. + policy.version = 3 + + condition = { + "title": title, + "description": description, + "expression": expression, + } + policy.bindings = [ + binding + for binding in policy.bindings + if not (binding["role"] == role and binding.get("condition") == condition) + ] + + bucket.set_iam_policy(policy) + + print("Conditional Binding was removed.") + + +# [END storage_remove_bucket_conditional_iam_binding] + + +if __name__ == "__main__": + remove_bucket_conditional_iam_binding( + bucket_name=sys.argv[1], + role=sys.argv[2], + title=sys.argv[3], + description=sys.argv[4], + expression=sys.argv[5], + ) diff --git a/storage/samples/snippets/storage_remove_bucket_default_owner.py b/storage/samples/snippets/storage_remove_bucket_default_owner.py new file mode 100644 index 00000000000..e6f3c495e5f --- /dev/null +++ b/storage/samples/snippets/storage_remove_bucket_default_owner.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_bucket_default_owner] +from google.cloud import storage + + +def remove_bucket_default_owner(bucket_name, user_email): + """Removes a user from the access control list of the given bucket's + default object access control list.""" + # bucket_name = "your-bucket-name" + # user_email = "name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # remove access for different types of entities. + bucket.default_object_acl.user(user_email).revoke_read() + bucket.default_object_acl.user(user_email).revoke_write() + bucket.default_object_acl.user(user_email).revoke_owner() + bucket.default_object_acl.save() + + print( + f"Removed user {user_email} from the default acl of bucket {bucket_name}." + ) + + +# [END storage_remove_bucket_default_owner] + +if __name__ == "__main__": + remove_bucket_default_owner( + bucket_name=sys.argv[1], user_email=sys.argv[2] + ) diff --git a/storage/samples/snippets/storage_remove_bucket_iam_member.py b/storage/samples/snippets/storage_remove_bucket_iam_member.py new file mode 100644 index 00000000000..2efc29e303c --- /dev/null +++ b/storage/samples/snippets/storage_remove_bucket_iam_member.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_bucket_iam_member] +from google.cloud import storage + + +def remove_bucket_iam_member(bucket_name, role, member): + """Remove member from bucket IAM Policy""" + # bucket_name = "your-bucket-name" + # role = "IAM role, e.g. roles/storage.objectViewer" + # member = "IAM identity, e.g. user: name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy(requested_policy_version=3) + + for binding in policy.bindings: + print(binding) + if binding["role"] == role and binding.get("condition") is None: + binding["members"].discard(member) + + bucket.set_iam_policy(policy) + + print(f"Removed {member} with role {role} from {bucket_name}.") + + +# [END storage_remove_bucket_iam_member] + +if __name__ == "__main__": + remove_bucket_iam_member( + bucket_name=sys.argv[1], role=sys.argv[2], member=sys.argv[3] + ) diff --git a/storage/samples/snippets/storage_remove_bucket_label.py b/storage/samples/snippets/storage_remove_bucket_label.py new file mode 100644 index 00000000000..fc4a5b4e7b2 --- /dev/null +++ b/storage/samples/snippets/storage_remove_bucket_label.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_remove_bucket_label] +import pprint +# [END storage_remove_bucket_label] +import sys +# [START storage_remove_bucket_label] + +from google.cloud import storage + + +def remove_bucket_label(bucket_name): + """Remove a label from a bucket.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + labels = bucket.labels + + if "example" in labels: + del labels["example"] + + bucket.labels = labels + bucket.patch() + + print(f"Removed labels on {bucket.name}.") + pprint.pprint(bucket.labels) + + +# [END storage_remove_bucket_label] + +if __name__ == "__main__": + remove_bucket_label(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_remove_bucket_owner.py b/storage/samples/snippets/storage_remove_bucket_owner.py new file mode 100644 index 00000000000..561ba9175a6 --- /dev/null +++ b/storage/samples/snippets/storage_remove_bucket_owner.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_bucket_owner] +from google.cloud import storage + + +def remove_bucket_owner(bucket_name, user_email): + """Removes a user from the access control list of the given bucket.""" + # bucket_name = "your-bucket-name" + # user_email = "name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Reload fetches the current ACL from Cloud Storage. + bucket.acl.reload() + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # remove access for different types of entities. + bucket.acl.user(user_email).revoke_read() + bucket.acl.user(user_email).revoke_write() + bucket.acl.user(user_email).revoke_owner() + bucket.acl.save() + + print(f"Removed user {user_email} from bucket {bucket_name}.") + + +# [END storage_remove_bucket_owner] + +if __name__ == "__main__": + remove_bucket_owner(bucket_name=sys.argv[1], user_email=sys.argv[2]) diff --git a/storage/samples/snippets/storage_remove_cors_configuration.py b/storage/samples/snippets/storage_remove_cors_configuration.py new file mode 100644 index 00000000000..ad97371f494 --- /dev/null +++ b/storage/samples/snippets/storage_remove_cors_configuration.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_cors_configuration] +from google.cloud import storage + + +def remove_cors_configuration(bucket_name): + """Remove a bucket's CORS policies configuration.""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.cors = [] + bucket.patch() + + print(f"Remove CORS policies for bucket {bucket.name}.") + return bucket + + +# [END storage_remove_cors_configuration] + +if __name__ == "__main__": + remove_cors_configuration(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_remove_file_owner.py b/storage/samples/snippets/storage_remove_file_owner.py new file mode 100644 index 00000000000..315a747adbc --- /dev/null +++ b/storage/samples/snippets/storage_remove_file_owner.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_file_owner] +from google.cloud import storage + + +def remove_blob_owner(bucket_name, blob_name, user_email): + """Removes a user from the access control list of the given blob in the + given bucket.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # user_email = "name@example.com" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # You can also use `group`, `domain`, `all_authenticated` and `all` to + # remove access for different types of entities. + blob.acl.user(user_email).revoke_read() + blob.acl.user(user_email).revoke_write() + blob.acl.user(user_email).revoke_owner() + blob.acl.save() + + print( + f"Removed user {user_email} from blob {blob_name} in bucket {bucket_name}." + ) + + +# [END storage_remove_file_owner] + +if __name__ == "__main__": + remove_blob_owner( + bucket_name=sys.argv[1], blob_name=sys.argv[2], user_email=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_remove_retention_policy.py b/storage/samples/snippets/storage_remove_retention_policy.py new file mode 100644 index 00000000000..9ede8053afd --- /dev/null +++ b/storage/samples/snippets/storage_remove_retention_policy.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_remove_retention_policy] +from google.cloud import storage + + +def remove_retention_policy(bucket_name): + """Removes the retention policy on a given bucket""" + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.reload() + + if bucket.retention_policy_locked: + print( + "Unable to remove retention period as retention policy is locked." + ) + return + + bucket.retention_period = None + bucket.patch() + + print(f"Removed bucket {bucket.name} retention policy") + + +# [END storage_remove_retention_policy] + + +if __name__ == "__main__": + remove_retention_policy(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_rename_file.py b/storage/samples/snippets/storage_rename_file.py new file mode 100644 index 00000000000..1125007c655 --- /dev/null +++ b/storage/samples/snippets/storage_rename_file.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_rename_file] +from google.cloud import storage + + +def rename_blob(bucket_name, blob_name, new_name): + """Renames a blob.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of the GCS object to rename + # blob_name = "your-object-name" + # The new ID of the GCS object + # new_name = "new-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + new_blob = bucket.rename_blob(blob, new_name) + + print(f"Blob {blob.name} has been renamed to {new_blob.name}") + + +# [END storage_rename_file] + +if __name__ == "__main__": + rename_blob(bucket_name=sys.argv[1], blob_name=sys.argv[2], new_name=sys.argv[3]) diff --git a/storage/samples/snippets/storage_restore_object.py b/storage/samples/snippets/storage_restore_object.py new file mode 100644 index 00000000000..d1e3f29372c --- /dev/null +++ b/storage/samples/snippets/storage_restore_object.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +# [START storage_restore_object] +from google.cloud import storage + + +def restore_soft_deleted_object(bucket_name, blob_name, blob_generation): + """Restores a soft-deleted object in the bucket.""" + # bucket_name = "your-bucket-name" + # blob_name = "your-object-name" + # blob_generation = "your-object-version-id" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Restore function will override if a live object already + # exists with the same name. + bucket.restore_blob(blob_name, generation=blob_generation) + + print( + f"Soft-deleted object {blob_name} is restored in the bucket {bucket_name}" + ) + + +# [END storage_restore_object] + +if __name__ == "__main__": + restore_soft_deleted_object( + bucket_name=sys.argv[1], blob_name=sys.argv[2], blob_generation=sys.argv[3] + ) diff --git a/storage/samples/snippets/storage_restore_soft_deleted_bucket.py b/storage/samples/snippets/storage_restore_soft_deleted_bucket.py new file mode 100644 index 00000000000..fb62919978e --- /dev/null +++ b/storage/samples/snippets/storage_restore_soft_deleted_bucket.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +# [START storage_restore_soft_deleted_bucket] + +from google.cloud import storage + + +def restore_bucket(bucket_name, bucket_generation): + storage_client = storage.Client() + bucket = storage_client.restore_bucket(bucket_name=bucket_name, generation=bucket_generation) + print(f"Soft-deleted bucket {bucket.name} with ID: {bucket.id} was restored.") + print(f"Bucket Generation: {bucket.generation}") + + +# [END storage_restore_soft_deleted_bucket] + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Wrong inputs!! Usage of script - \"python storage_restore_soft_deleted_bucket.py \" ") + sys.exit(1) + restore_bucket(bucket_name=sys.argv[1], bucket_generation=sys.argv[2]) diff --git a/storage/samples/snippets/storage_rotate_encryption_key.py b/storage/samples/snippets/storage_rotate_encryption_key.py new file mode 100644 index 00000000000..174947b843e --- /dev/null +++ b/storage/samples/snippets/storage_rotate_encryption_key.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_rotate_encryption_key] +import base64 +# [END storage_rotate_encryption_key] +import sys +# [START storage_rotate_encryption_key] + +from google.cloud import storage + + +def rotate_encryption_key( + bucket_name, blob_name, base64_encryption_key, base64_new_encryption_key +): + """Performs a key rotation by re-writing an encrypted blob with a new + encryption key.""" + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + current_encryption_key = base64.b64decode(base64_encryption_key) + new_encryption_key = base64.b64decode(base64_new_encryption_key) + + # Both source_blob and destination_blob refer to the same storage object, + # but destination_blob has the new encryption key. + source_blob = bucket.blob( + blob_name, encryption_key=current_encryption_key + ) + destination_blob = bucket.blob( + blob_name, encryption_key=new_encryption_key + ) + generation_match_precondition = None + token = None + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to rewrite is aborted if the object's + # generation number does not match your precondition. + source_blob.reload() # Fetch blob metadata to use in generation_match_precondition. + generation_match_precondition = source_blob.generation + + while True: + token, bytes_rewritten, total_bytes = destination_blob.rewrite( + source_blob, token=token, if_generation_match=generation_match_precondition + ) + if token is None: + break + + print(f"Key rotation complete for Blob {blob_name}") + + +# [END storage_rotate_encryption_key] + +if __name__ == "__main__": + rotate_encryption_key( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + base64_encryption_key=sys.argv[3], + base64_new_encryption_key=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_set_autoclass.py b/storage/samples/snippets/storage_set_autoclass.py new file mode 100644 index 00000000000..eec5a550f8c --- /dev/null +++ b/storage/samples/snippets/storage_set_autoclass.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_autoclass] +from google.cloud import storage + + +def set_autoclass(bucket_name): + """Configure the Autoclass setting for a bucket. + + terminal_storage_class field is optional and defaults to NEARLINE if not otherwise specified. + Valid terminal_storage_class values are NEARLINE and ARCHIVE. + """ + # The ID of your GCS bucket + # bucket_name = "my-bucket" + # Enable Autoclass for a bucket. Set enabled to false to disable Autoclass. + # Set Autoclass.TerminalStorageClass, valid values are NEARLINE and ARCHIVE. + enabled = True + terminal_storage_class = "ARCHIVE" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.autoclass_enabled = enabled + bucket.autoclass_terminal_storage_class = terminal_storage_class + bucket.patch() + print(f"Autoclass enabled is set to {bucket.autoclass_enabled} for {bucket.name} at {bucket.autoclass_toggle_time}.") + print(f"Autoclass terminal storage class is {bucket.autoclass_terminal_storage_class}.") + + return bucket + + +# [END storage_set_autoclass] + +if __name__ == "__main__": + set_autoclass(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_set_bucket_default_kms_key.py b/storage/samples/snippets/storage_set_bucket_default_kms_key.py new file mode 100644 index 00000000000..7ba4718b2be --- /dev/null +++ b/storage/samples/snippets/storage_set_bucket_default_kms_key.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_bucket_default_kms_key] +from google.cloud import storage + + +def enable_default_kms_key(bucket_name, kms_key_name): + """Sets a bucket's default KMS key.""" + # bucket_name = "your-bucket-name" + # kms_key_name = "projects/PROJ/locations/LOC/keyRings/RING/cryptoKey/KEY" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + bucket.default_kms_key_name = kms_key_name + bucket.patch() + + print( + "Set default KMS key for bucket {} to {}.".format( + bucket.name, bucket.default_kms_key_name + ) + ) + + +# [END storage_set_bucket_default_kms_key] + +if __name__ == "__main__": + enable_default_kms_key(bucket_name=sys.argv[1], kms_key_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_set_bucket_encryption_enforcement_config.py b/storage/samples/snippets/storage_set_bucket_encryption_enforcement_config.py new file mode 100644 index 00000000000..107564e7f6c --- /dev/null +++ b/storage/samples/snippets/storage_set_bucket_encryption_enforcement_config.py @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_set_bucket_encryption_enforcement_config] +from google.cloud import storage +from google.cloud.storage.bucket import EncryptionEnforcementConfig + + +def set_bucket_encryption_enforcement_config(bucket_name): + """Creates a bucket with encryption enforcement configuration.""" + # The ID of your GCS bucket + # bucket_name = "your-unique-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Setting restriction_mode to "FullyRestricted" for Google-managed encryption (GMEK) + # means objects cannot be created using the default Google-managed keys. + bucket.encryption.google_managed_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="FullyRestricted") + ) + + # Setting restriction_mode to "NotRestricted" for Customer-managed encryption (CMEK) + # ensures that objects ARE permitted to be created using Cloud KMS keys. + bucket.encryption.customer_managed_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="NotRestricted") + ) + + # Setting restriction_mode to "FullyRestricted" for Customer-supplied encryption (CSEK) + # prevents objects from being created using raw, client-side provided keys. + bucket.encryption.customer_supplied_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="FullyRestricted") + ) + + bucket.create() + + print(f"Created bucket {bucket.name} with Encryption Enforcement Config.") + + +# [END storage_set_bucket_encryption_enforcement_config] + + +if __name__ == "__main__": + set_bucket_encryption_enforcement_config(bucket_name="your-unique-bucket-name") diff --git a/storage/samples/snippets/storage_set_bucket_public_iam.py b/storage/samples/snippets/storage_set_bucket_public_iam.py new file mode 100644 index 00000000000..0fb33f59c65 --- /dev/null +++ b/storage/samples/snippets/storage_set_bucket_public_iam.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_bucket_public_iam] +from typing import List + +from google.cloud import storage + + +def set_bucket_public_iam( + bucket_name: str = "your-bucket-name", + members: List[str] = ["allUsers"], +): + """Set a public IAM Policy to bucket""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy(requested_policy_version=3) + policy.bindings.append( + {"role": "roles/storage.objectViewer", "members": members} + ) + + bucket.set_iam_policy(policy) + + print(f"Bucket {bucket.name} is now publicly readable") + + +# [END storage_set_bucket_public_iam] + +if __name__ == "__main__": + set_bucket_public_iam( + bucket_name=sys.argv[1], + ) diff --git a/storage/samples/snippets/storage_set_client_endpoint.py b/storage/samples/snippets/storage_set_client_endpoint.py new file mode 100644 index 00000000000..99ca283a18b --- /dev/null +++ b/storage/samples/snippets/storage_set_client_endpoint.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that creates a new bucket in a specified region +""" + +# [START storage_set_client_endpoint] + +from google.cloud import storage + + +def set_client_endpoint(api_endpoint): + """Initiates client with specified endpoint.""" + # api_endpoint = 'https://storage.googleapis.com' + + storage_client = storage.Client(client_options={'api_endpoint': api_endpoint}) + + print(f"client initiated with endpoint: {storage_client._connection.API_BASE_URL}") + + return storage_client + + +# [END storage_set_client_endpoint] + +if __name__ == "__main__": + set_client_endpoint(api_endpoint=sys.argv[1]) diff --git a/storage/samples/snippets/storage_set_event_based_hold.py b/storage/samples/snippets/storage_set_event_based_hold.py new file mode 100644 index 00000000000..76f7fd7eee4 --- /dev/null +++ b/storage/samples/snippets/storage_set_event_based_hold.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_event_based_hold] +from google.cloud import storage + + +def set_event_based_hold(bucket_name, blob_name): + """Sets a event based hold on a given blob""" + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + metageneration_match_precondition = None + + # Optional: set a metageneration-match precondition to avoid potential race + # conditions and data corruptions. The request to patch is aborted if the + # object's metageneration does not match your precondition. + blob.reload() # Fetch blob metadata to use in metageneration_match_precondition. + metageneration_match_precondition = blob.metageneration + + blob.event_based_hold = True + blob.patch(if_metageneration_match=metageneration_match_precondition) + + print(f"Event based hold was set for {blob_name}") + + +# [END storage_set_event_based_hold] + + +if __name__ == "__main__": + set_event_based_hold(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_set_metadata.py b/storage/samples/snippets/storage_set_metadata.py new file mode 100644 index 00000000000..6a4a9fb9e08 --- /dev/null +++ b/storage/samples/snippets/storage_set_metadata.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_metadata] +from google.cloud import storage + + +def set_blob_metadata(bucket_name, blob_name): + """Set a blob's metadata.""" + # bucket_name = 'your-bucket-name' + # blob_name = 'your-object-name' + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.get_blob(blob_name) + metageneration_match_precondition = None + + # Optional: set a metageneration-match precondition to avoid potential race + # conditions and data corruptions. The request to patch is aborted if the + # object's metageneration does not match your precondition. + metageneration_match_precondition = blob.metageneration + + metadata = {'color': 'Red', 'name': 'Test'} + blob.metadata = metadata + blob.patch(if_metageneration_match=metageneration_match_precondition) + + print(f"The metadata for the blob {blob.name} is {blob.metadata}") + + +# [END storage_set_metadata] + +if __name__ == "__main__": + set_blob_metadata(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_set_object_retention_policy.py b/storage/samples/snippets/storage_set_object_retention_policy.py new file mode 100644 index 00000000000..d0d3a54ec50 --- /dev/null +++ b/storage/samples/snippets/storage_set_object_retention_policy.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import sys + +# [START storage_set_object_retention_policy] +from google.cloud import storage + + +def set_object_retention_policy(bucket_name, contents, destination_blob_name): + """Set the object retention policy of a file.""" + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The contents to upload to the file + # contents = "these are my contents" + + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + blob.upload_from_string(contents) + + # Set the retention policy for the file. + blob.retention.mode = "Unlocked" + retention_date = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=10) + blob.retention.retain_until_time = retention_date + blob.patch() + print( + f"Retention policy for file {destination_blob_name} was set to: {blob.retention.mode}." + ) + + # To modify an existing policy on an unlocked file object, pass in the override parameter. + new_retention_date = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=9) + blob.retention.retain_until_time = new_retention_date + blob.patch(override_unlocked_retention=True) + print( + f"Retention policy for file {destination_blob_name} was updated to: {blob.retention.retain_until_time}." + ) + + +# [END storage_set_object_retention_policy] + + +if __name__ == "__main__": + set_object_retention_policy( + bucket_name=sys.argv[1], + contents=sys.argv[2], + destination_blob_name=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_set_public_access_prevention_enforced.py b/storage/samples/snippets/storage_set_public_access_prevention_enforced.py new file mode 100644 index 00000000000..59ce5ce56ef --- /dev/null +++ b/storage/samples/snippets/storage_set_public_access_prevention_enforced.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_public_access_prevention_enforced] +from google.cloud import storage +from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_ENFORCED + + +def set_public_access_prevention_enforced(bucket_name): + """Enforce public access prevention for a bucket.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + bucket.iam_configuration.public_access_prevention = ( + PUBLIC_ACCESS_PREVENTION_ENFORCED + ) + bucket.patch() + + print(f"Public access prevention is set to enforced for {bucket.name}.") + + +# [END storage_set_public_access_prevention_enforced] + +if __name__ == "__main__": + set_public_access_prevention_enforced(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_set_public_access_prevention_inherited.py b/storage/samples/snippets/storage_set_public_access_prevention_inherited.py new file mode 100644 index 00000000000..97e218f9d0a --- /dev/null +++ b/storage/samples/snippets/storage_set_public_access_prevention_inherited.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that sets public access prevention to inherited. +This sample is used on this page: + https://cloud.google.com/storage/docs/using-public-access-prevention +For more information, see README.md. +""" + +# [START storage_set_public_access_prevention_inherited] + +from google.cloud import storage +from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_INHERITED + + +def set_public_access_prevention_inherited(bucket_name): + """Sets the public access prevention status to inherited, so that the bucket inherits its setting from its parent project.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + bucket.iam_configuration.public_access_prevention = ( + PUBLIC_ACCESS_PREVENTION_INHERITED + ) + bucket.patch() + + print(f"Public access prevention is 'inherited' for {bucket.name}.") + + +# [END storage_set_public_access_prevention_inherited] + +if __name__ == "__main__": + set_public_access_prevention_inherited(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_set_retention_policy.py b/storage/samples/snippets/storage_set_retention_policy.py new file mode 100644 index 00000000000..2b36024919a --- /dev/null +++ b/storage/samples/snippets/storage_set_retention_policy.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_retention_policy] +from google.cloud import storage + + +def set_retention_policy(bucket_name, retention_period): + """Defines a retention policy on a given bucket""" + # bucket_name = "my-bucket" + # retention_period = 10 + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.retention_period = retention_period + bucket.patch() + + print( + "Bucket {} retention period set for {} seconds".format( + bucket.name, bucket.retention_period + ) + ) + + +# [END storage_set_retention_policy] + + +if __name__ == "__main__": + set_retention_policy(bucket_name=sys.argv[1], retention_period=sys.argv[2]) diff --git a/storage/samples/snippets/storage_set_rpo_async_turbo.py b/storage/samples/snippets/storage_set_rpo_async_turbo.py new file mode 100644 index 00000000000..a351cb8f82e --- /dev/null +++ b/storage/samples/snippets/storage_set_rpo_async_turbo.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that sets RPO (Recovery Point Objective) to ASYNC_TURBO +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_set_rpo_async_turbo] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_ASYNC_TURBO + + +def set_rpo_async_turbo(bucket_name): + """Sets the RPO to ASYNC_TURBO, enabling the turbo replication feature""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.rpo = RPO_ASYNC_TURBO + bucket.patch() + + print(f"RPO is set to ASYNC_TURBO for {bucket.name}.") + + +# [END storage_set_rpo_async_turbo] + +if __name__ == "__main__": + set_rpo_async_turbo(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_set_rpo_default.py b/storage/samples/snippets/storage_set_rpo_default.py new file mode 100644 index 00000000000..883fee0c972 --- /dev/null +++ b/storage/samples/snippets/storage_set_rpo_default.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that sets the replication behavior or recovery point objective (RPO) to default. +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_set_rpo_default] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_DEFAULT + + +def set_rpo_default(bucket_name): + """Sets the RPO to DEFAULT, disabling the turbo replication feature""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.rpo = RPO_DEFAULT + bucket.patch() + + print(f"RPO is set to DEFAULT for {bucket.name}.") + + +# [END storage_set_rpo_default] + +if __name__ == "__main__": + set_rpo_default(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/storage_set_soft_delete_policy.py b/storage/samples/snippets/storage_set_soft_delete_policy.py new file mode 100644 index 00000000000..26bc5943664 --- /dev/null +++ b/storage/samples/snippets/storage_set_soft_delete_policy.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_soft_delete_policy] +from google.cloud import storage + + +def set_soft_delete_policy(bucket_name, duration_in_seconds): + """Sets a soft-delete policy on the bucket""" + # bucket_name = "your-bucket-name" + # duration_in_seconds = "your-soft-delete-retention-duration-in-seconds" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.soft_delete_policy.retention_duration_seconds = duration_in_seconds + bucket.patch() + + print( + f"Soft delete policy for bucket {bucket_name} was set to {duration_in_seconds} seconds retention period" + ) + + +# [END storage_set_soft_delete_policy] + +if __name__ == "__main__": + set_soft_delete_policy(bucket_name=sys.argv[1], duration_in_seconds=sys.argv[2]) diff --git a/storage/samples/snippets/storage_set_temporary_hold.py b/storage/samples/snippets/storage_set_temporary_hold.py new file mode 100644 index 00000000000..a91521bcc11 --- /dev/null +++ b/storage/samples/snippets/storage_set_temporary_hold.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_set_temporary_hold] +from google.cloud import storage + + +def set_temporary_hold(bucket_name, blob_name): + """Sets a temporary hold on a given blob""" + # bucket_name = "my-bucket" + # blob_name = "my-blob" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + metageneration_match_precondition = None + + # Optional: set a metageneration-match precondition to avoid potential race + # conditions and data corruptions. The request to patch is aborted if the + # object's metageneration does not match your precondition. + blob.reload() # Fetch blob metadata to use in metageneration_match_precondition. + metageneration_match_precondition = blob.metageneration + + blob.temporary_hold = True + blob.patch(if_metageneration_match=metageneration_match_precondition) + + print("Temporary hold was set for #{blob_name}") + + +# [END storage_set_temporary_hold] + + +if __name__ == "__main__": + set_temporary_hold(bucket_name=sys.argv[1], blob_name=sys.argv[2]) diff --git a/storage/samples/snippets/storage_trace_quickstart.py b/storage/samples/snippets/storage_trace_quickstart.py new file mode 100644 index 00000000000..322edc24051 --- /dev/null +++ b/storage/samples/snippets/storage_trace_quickstart.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +""" +Sample that exports OpenTelemetry Traces collected from the Storage client to Cloud Trace. +""" + + +def run_quickstart(bucket_name, blob_name, data): + # [START storage_enable_otel_tracing] + + from opentelemetry import trace + from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter + from opentelemetry.resourcedetector.gcp_resource_detector import ( + GoogleCloudResourceDetector, + ) + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ALWAYS_ON + # Optional: Enable traces emitted from the requests HTTP library. + from opentelemetry.instrumentation.requests import RequestsInstrumentor + + from google.cloud import storage + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of your GCS object + # blob_name = "your-object-name" + # The contents to upload to the file + # data = "The quick brown fox jumps over the lazy dog." + + # In this sample, we use Google Cloud Trace to export the OpenTelemetry + # traces: https://cloud.google.com/trace/docs/setup/python-ot + # Choose and configure the exporter for your environment. + + tracer_provider = TracerProvider( + # Sampling is set to ALWAYS_ON. + # It is recommended to sample based on a ratio to control trace ingestion volume, + # for instance, sampler=TraceIdRatioBased(0.2) + sampler=ALWAYS_ON, + resource=GoogleCloudResourceDetector().detect(), + ) + + # Export to Google Cloud Trace. + tracer_provider.add_span_processor(BatchSpanProcessor(CloudTraceSpanExporter())) + trace.set_tracer_provider(tracer_provider) + + # Optional: Enable traces emitted from the requests HTTP library. + RequestsInstrumentor().instrument(tracer_provider=tracer_provider) + + # Get the tracer and create a new root span. + tracer = tracer_provider.get_tracer("My App") + with tracer.start_as_current_span("trace-quickstart"): + # Instantiate a storage client and perform a write and read workload. + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + blob.upload_from_string(data) + print(f"{blob_name} uploaded to {bucket_name}.") + + blob.download_as_bytes() + print("Downloaded storage object {} from bucket {}.".format(blob_name, bucket_name)) + + # [END storage_enable_otel_tracing] + + +if __name__ == "__main__": + run_quickstart(bucket_name=sys.argv[1], blob_name=sys.argv[2], data=sys.argv[3]) diff --git a/storage/samples/snippets/storage_transfer_manager_download_bucket.py b/storage/samples/snippets/storage_transfer_manager_download_bucket.py new file mode 100644 index 00000000000..5d94a67aeea --- /dev/null +++ b/storage/samples/snippets/storage_transfer_manager_download_bucket.py @@ -0,0 +1,75 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_transfer_manager_download_bucket] +def download_bucket_with_transfer_manager( + bucket_name, destination_directory="", workers=8, max_results=1000 +): + """Download all of the blobs in a bucket, concurrently in a process pool. + + The filename of each blob once downloaded is derived from the blob name and + the `destination_directory `parameter. For complete control of the filename + of each blob, use transfer_manager.download_many() instead. + + Directories will be created automatically as needed, for instance to + accommodate blob names that include slashes. + """ + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The directory on your computer to which to download all of the files. This + # string is prepended (with os.path.join()) to the name of each blob to form + # the full path. Relative paths and absolute paths are both accepted. An + # empty string means "the current working directory". Note that this + # parameter allows accepts directory traversal ("../" etc.) and is not + # intended for unsanitized end user input. + # destination_directory = "" + + # The maximum number of processes to use for the operation. The performance + # impact of this value depends on the use case, but smaller files usually + # benefit from a higher number of processes. Each additional process occupies + # some CPU and memory resources until finished. Threads can be used instead + # of processes by passing `worker_type=transfer_manager.THREAD`. + # workers=8 + + # The maximum number of results to fetch from bucket.list_blobs(). This + # sample code fetches all of the blobs up to max_results and queues them all + # for download at once. Though they will still be executed in batches up to + # the processes limit, queueing them all at once can be taxing on system + # memory if buckets are very large. Adjust max_results as needed for your + # system environment, or set it to None if you are sure the bucket is not + # too large to hold in memory easily. + # max_results=1000 + + from google.cloud.storage import Client, transfer_manager + + storage_client = Client() + bucket = storage_client.bucket(bucket_name) + + blob_names = [blob.name for blob in bucket.list_blobs(max_results=max_results)] + + results = transfer_manager.download_many_to_path( + bucket, blob_names, destination_directory=destination_directory, max_workers=workers + ) + + for name, result in zip(blob_names, results): + # The results list is either `None` or an exception for each blob in + # the input list, in order. + + if isinstance(result, Exception): + print("Failed to download {} due to exception: {}".format(name, result)) + else: + print("Downloaded {} to {}.".format(name, destination_directory + name)) +# [END storage_transfer_manager_download_bucket] diff --git a/storage/samples/snippets/storage_transfer_manager_download_chunks_concurrently.py b/storage/samples/snippets/storage_transfer_manager_download_chunks_concurrently.py new file mode 100644 index 00000000000..b6ac9982d61 --- /dev/null +++ b/storage/samples/snippets/storage_transfer_manager_download_chunks_concurrently.py @@ -0,0 +1,55 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_transfer_manager_download_chunks_concurrently] +def download_chunks_concurrently( + bucket_name, blob_name, filename, chunk_size=32 * 1024 * 1024, workers=8 +): + """Download a single file in chunks, concurrently in a process pool.""" + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The file to be downloaded + # blob_name = "target-file" + + # The destination filename or path + # filename = "" + + # The size of each chunk. The performance impact of this value depends on + # the use case. The remote service has a minimum of 5 MiB and a maximum of + # 5 GiB. + # chunk_size = 32 * 1024 * 1024 (32 MiB) + + # The maximum number of processes to use for the operation. The performance + # impact of this value depends on the use case, but smaller files usually + # benefit from a higher number of processes. Each additional process occupies + # some CPU and memory resources until finished. Threads can be used instead + # of processes by passing `worker_type=transfer_manager.THREAD`. + # workers=8 + + from google.cloud.storage import Client, transfer_manager + + storage_client = Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + transfer_manager.download_chunks_concurrently( + blob, filename, chunk_size=chunk_size, max_workers=workers + ) + + print("Downloaded {} to {}.".format(blob_name, filename)) + + +# [END storage_transfer_manager_download_chunks_concurrently] diff --git a/storage/samples/snippets/storage_transfer_manager_download_many.py b/storage/samples/snippets/storage_transfer_manager_download_many.py new file mode 100644 index 00000000000..447d0869c5b --- /dev/null +++ b/storage/samples/snippets/storage_transfer_manager_download_many.py @@ -0,0 +1,126 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Example usage: +# python samples/snippets/storage_transfer_manager_download_many.py \ +# --bucket_name \ +# --blobs \ +# --destination_directory \ +# --blob_name_prefix + + +# [START storage_transfer_manager_download_many] +def download_many_blobs_with_transfer_manager( + bucket_name, blob_names, destination_directory="", blob_name_prefix="", workers=8 +): + """Download blobs in a list by name, concurrently in a process pool. + + The filename of each blob once downloaded is derived from the blob name and + the `destination_directory `parameter. For complete control of the filename + of each blob, use transfer_manager.download_many() instead. + + Directories will be created automatically as needed to accommodate blob + names that include slashes. + """ + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The list of blob names to download. The names of each blobs will also + # be the name of each destination file (use transfer_manager.download_many() + # instead to control each destination file name). If there is a "/" in the + # blob name, then corresponding directories will be created on download. + # blob_names = ["myblob", "myblob2"] + + # The directory on your computer to which to download all of the files. This + # string is prepended to the name of each blob to form the full path using + # pathlib. Relative paths and absolute paths are both accepted. An empty + # string means "the current working directory". Note that this parameter + # will NOT allow files to escape the destination_directory and will skip + # downloads that attempt directory traversal outside of it. + # destination_directory = "" + + # The maximum number of processes to use for the operation. The performance + # impact of this value depends on the use case, but smaller files usually + # benefit from a higher number of processes. Each additional process occupies + # some CPU and memory resources until finished. Threads can be used instead + # of processes by passing `worker_type=transfer_manager.THREAD`. + # workers=8 + + from google.cloud.storage import Client, transfer_manager + + storage_client = Client() + bucket = storage_client.bucket(bucket_name) + + results = transfer_manager.download_many_to_path( + bucket, + blob_names, + destination_directory=destination_directory, + blob_name_prefix=blob_name_prefix, + max_workers=workers, + ) + + for name, result in zip(blob_names, results): + # The results list is either `None`, an exception, or a warning for each blob in + # the input list, in order. + if isinstance(result, UserWarning): + print("Skipped download for {} due to warning: {}".format(name, result)) + elif isinstance(result, Exception): + print("Failed to download {} due to exception: {}".format(name, result)) + else: + print( + "Downloaded {} inside {} directory.".format(name, destination_directory) + ) + + +# [END storage_transfer_manager_download_many] + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Download blobs in a list by name, concurrently in a process pool." + ) + parser.add_argument( + "--bucket_name", required=True, help="The name of your GCS bucket" + ) + parser.add_argument( + "--blobs", + nargs="+", + required=True, + help="The list of blob names to download", + ) + parser.add_argument( + "--destination_directory", + default="", + help="The directory on your computer to which to download all of the files", + ) + parser.add_argument( + "--blob_name_prefix", + default="", + help="A string that will be prepended to each blob_name to determine the source blob name", + ) + parser.add_argument( + "--workers", type=int, default=8, help="The maximum number of processes to use" + ) + + args = parser.parse_args() + + download_many_blobs_with_transfer_manager( + bucket_name=args.bucket_name, + blob_names=args.blobs, + destination_directory=args.destination_directory, + blob_name_prefix=args.blob_name_prefix, + workers=args.workers, + ) diff --git a/storage/samples/snippets/storage_transfer_manager_upload_chunks_concurrently.py b/storage/samples/snippets/storage_transfer_manager_upload_chunks_concurrently.py new file mode 100644 index 00000000000..a4abd13b98b --- /dev/null +++ b/storage/samples/snippets/storage_transfer_manager_upload_chunks_concurrently.py @@ -0,0 +1,95 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + + +# [START storage_transfer_manager_upload_chunks_concurrently] +def upload_chunks_concurrently( + bucket_name, + source_filename, + destination_blob_name, + chunk_size=32 * 1024 * 1024, + workers=8, +): + """Upload a single file, in chunks, concurrently in a process pool.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The path to your file to upload + # source_filename = "local/path/to/file" + + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + # The size of each chunk. The performance impact of this value depends on + # the use case. The remote service has a minimum of 5 MiB and a maximum of + # 5 GiB. + # chunk_size = 32 * 1024 * 1024 (32 MiB) + + # The maximum number of processes to use for the operation. The performance + # impact of this value depends on the use case. Each additional process + # occupies some CPU and memory resources until finished. Threads can be used + # instead of processes by passing `worker_type=transfer_manager.THREAD`. + # workers=8 + + from google.cloud.storage import Client, transfer_manager + + storage_client = Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + transfer_manager.upload_chunks_concurrently( + source_filename, blob, chunk_size=chunk_size, max_workers=workers + ) + + print(f"File {source_filename} uploaded to {destination_blob_name}.") + + +if __name__ == "__main__": + argparse = argparse.ArgumentParser( + description="Upload a file to GCS in chunks concurrently." + ) + argparse.add_argument( + "--bucket_name", help="The name of the GCS bucket to upload to." + ) + argparse.add_argument( + "--source_filename", help="The local path to the file to upload." + ) + argparse.add_argument( + "--destination_blob_name", help="The name of the object in GCS." + ) + argparse.add_argument( + "--chunk_size", + type=int, + default=32 * 1024 * 1024, + help="The size of each chunk in bytes (default: 32 MiB). The remote\ + service has a minimum of 5 MiB and a maximum of 5 GiB", + ) + argparse.add_argument( + "--workers", + type=int, + default=8, + help="The number of worker processes to use (default: 8).", + ) + args = argparse.parse_args() + upload_chunks_concurrently( + args.bucket_name, + args.source_filename, + args.destination_blob_name, + args.chunk_size, + args.workers, + ) + + +# [END storage_transfer_manager_upload_chunks_concurrently] diff --git a/storage/samples/snippets/storage_transfer_manager_upload_directory.py b/storage/samples/snippets/storage_transfer_manager_upload_directory.py new file mode 100644 index 00000000000..329ca108133 --- /dev/null +++ b/storage/samples/snippets/storage_transfer_manager_upload_directory.py @@ -0,0 +1,80 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_transfer_manager_upload_directory] +def upload_directory_with_transfer_manager(bucket_name, source_directory, workers=8): + """Upload every file in a directory, including all files in subdirectories. + + Each blob name is derived from the filename, not including the `directory` + parameter itself. For complete control of the blob name for each file (and + other aspects of individual blob metadata), use + transfer_manager.upload_many() instead. + """ + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The directory on your computer to upload. Files in the directory and its + # subdirectories will be uploaded. An empty string means "the current + # working directory". + # source_directory="" + + # The maximum number of processes to use for the operation. The performance + # impact of this value depends on the use case, but smaller files usually + # benefit from a higher number of processes. Each additional process occupies + # some CPU and memory resources until finished. Threads can be used instead + # of processes by passing `worker_type=transfer_manager.THREAD`. + # workers=8 + + from pathlib import Path + + from google.cloud.storage import Client, transfer_manager + + storage_client = Client() + bucket = storage_client.bucket(bucket_name) + + # Generate a list of paths (in string form) relative to the `directory`. + # This can be done in a single list comprehension, but is expanded into + # multiple lines here for clarity. + + # First, recursively get all files in `directory` as Path objects. + directory_as_path_obj = Path(source_directory) + paths = directory_as_path_obj.rglob("*") + + # Filter so the list only includes files, not directories themselves. + file_paths = [path for path in paths if path.is_file()] + + # These paths are relative to the current working directory. Next, make them + # relative to `directory` + relative_paths = [path.relative_to(source_directory) for path in file_paths] + + # Finally, convert them all to strings. + string_paths = [str(path) for path in relative_paths] + + print("Found {} files.".format(len(string_paths))) + + # Start the upload. + results = transfer_manager.upload_many_from_filenames( + bucket, string_paths, source_directory=source_directory, max_workers=workers + ) + + for name, result in zip(string_paths, results): + # The results list is either `None` or an exception for each filename in + # the input list, in order. + + if isinstance(result, Exception): + print("Failed to upload {} due to exception: {}".format(name, result)) + else: + print("Uploaded {} to {}.".format(name, bucket.name)) +# [END storage_transfer_manager_upload_directory] diff --git a/storage/samples/snippets/storage_transfer_manager_upload_many.py b/storage/samples/snippets/storage_transfer_manager_upload_many.py new file mode 100644 index 00000000000..1b9b9fc8983 --- /dev/null +++ b/storage/samples/snippets/storage_transfer_manager_upload_many.py @@ -0,0 +1,67 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_transfer_manager_upload_many] +def upload_many_blobs_with_transfer_manager( + bucket_name, filenames, source_directory="", workers=8 +): + """Upload every file in a list to a bucket, concurrently in a process pool. + + Each blob name is derived from the filename, not including the + `source_directory` parameter. For complete control of the blob name for each + file (and other aspects of individual blob metadata), use + transfer_manager.upload_many() instead. + """ + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # A list (or other iterable) of filenames to upload. + # filenames = ["file_1.txt", "file_2.txt"] + + # The directory on your computer that is the root of all of the files in the + # list of filenames. This string is prepended (with os.path.join()) to each + # filename to get the full path to the file. Relative paths and absolute + # paths are both accepted. This string is not included in the name of the + # uploaded blob; it is only used to find the source files. An empty string + # means "the current working directory". Note that this parameter allows + # directory traversal (e.g. "/", "../") and is not intended for unsanitized + # end user input. + # source_directory="" + + # The maximum number of processes to use for the operation. The performance + # impact of this value depends on the use case, but smaller files usually + # benefit from a higher number of processes. Each additional process occupies + # some CPU and memory resources until finished. Threads can be used instead + # of processes by passing `worker_type=transfer_manager.THREAD`. + # workers=8 + + from google.cloud.storage import Client, transfer_manager + + storage_client = Client() + bucket = storage_client.bucket(bucket_name) + + results = transfer_manager.upload_many_from_filenames( + bucket, filenames, source_directory=source_directory, max_workers=workers + ) + + for name, result in zip(filenames, results): + # The results list is either `None` or an exception for each filename in + # the input list, in order. + + if isinstance(result, Exception): + print("Failed to upload {} due to exception: {}".format(name, result)) + else: + print("Uploaded {} to {}.".format(name, bucket.name)) +# [END storage_transfer_manager_upload_many] diff --git a/storage/samples/snippets/storage_update_bucket_encryption_enforcement_config.py b/storage/samples/snippets/storage_update_bucket_encryption_enforcement_config.py new file mode 100644 index 00000000000..9b704bc0b8d --- /dev/null +++ b/storage/samples/snippets/storage_update_bucket_encryption_enforcement_config.py @@ -0,0 +1,60 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_update_bucket_encryption_enforcement_config] +from google.cloud import storage +from google.cloud.storage.bucket import EncryptionEnforcementConfig + + +def update_bucket_encryption_enforcement_config(bucket_name): + """Updates the encryption enforcement policy for a bucket.""" + # The ID of your GCS bucket with GMEK and CSEK restricted + # bucket_name = "your-unique-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + + # Update a specific type (e.g., change GMEK to NotRestricted) + bucket.encryption.google_managed_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="NotRestricted") + ) + + # Update another type (e.g., change CMEK to FullyRestricted) + bucket.encryption.customer_managed_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="FullyRestricted") + ) + + # Keeping CSEK unchanged + bucket.encryption.customer_supplied_encryption_enforcement_config = ( + EncryptionEnforcementConfig(restriction_mode="FullyRestricted") + ) + + bucket.patch() + + print(f"Encryption enforcement policy updated for bucket {bucket.name}.") + + gmek = bucket.encryption.google_managed_encryption_enforcement_config + cmek = bucket.encryption.customer_managed_encryption_enforcement_config + csek = bucket.encryption.customer_supplied_encryption_enforcement_config + + print(f"GMEK restriction mode: {gmek.restriction_mode if gmek else 'None'}") + print(f"CMEK restriction mode: {cmek.restriction_mode if cmek else 'None'}") + print(f"CSEK restriction mode: {csek.restriction_mode if csek else 'None'}") + + +# [END storage_update_bucket_encryption_enforcement_config] + + +if __name__ == "__main__": + update_bucket_encryption_enforcement_config(bucket_name="your-unique-bucket-name") diff --git a/storage/samples/snippets/storage_upload_encrypted_file.py b/storage/samples/snippets/storage_upload_encrypted_file.py new file mode 100644 index 00000000000..08f58154e07 --- /dev/null +++ b/storage/samples/snippets/storage_upload_encrypted_file.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START storage_upload_encrypted_file] +import base64 +# [END storage_upload_encrypted_file] +import sys +# [START storage_upload_encrypted_file] + +from google.cloud import storage + + +def upload_encrypted_blob( + bucket_name, + source_file_name, + destination_blob_name, + base64_encryption_key, +): + """Uploads a file to a Google Cloud Storage bucket using a custom + encryption key. + + The file will be encrypted by Google Cloud Storage and only + retrievable using the provided encryption key. + """ + # bucket_name = "your-bucket-name" + # source_file_name = "local/path/to/file" + # destination_blob_name = "storage-object-name" + # base64_encryption_key = "TIbv/fjexq+VmtXzAlc63J4z5kFmWJ6NdAPQulQBT7g=" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + # Encryption key must be an AES256 key represented as a bytestring with + # 32 bytes. Since it's passed in as a base64 encoded string, it needs + # to be decoded. + encryption_key = base64.b64decode(base64_encryption_key) + blob = bucket.blob( + destination_blob_name, encryption_key=encryption_key + ) + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to upload is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + generation_match_precondition = 0 + + blob.upload_from_filename(source_file_name, if_generation_match=generation_match_precondition) + + print( + f"File {source_file_name} uploaded to {destination_blob_name}." + ) + + +# [END storage_upload_encrypted_file] + +if __name__ == "__main__": + upload_encrypted_blob( + bucket_name=sys.argv[1], + source_file_name=sys.argv[2], + destination_blob_name=sys.argv[3], + base64_encryption_key=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_upload_file.py b/storage/samples/snippets/storage_upload_file.py new file mode 100644 index 00000000000..1e7ceda5eb4 --- /dev/null +++ b/storage/samples/snippets/storage_upload_file.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_upload_file] +from google.cloud import storage + + +def upload_blob(bucket_name, source_file_name, destination_blob_name): + """Uploads a file to the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The path to your file to upload + # source_file_name = "local/path/to/file" + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to upload is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + generation_match_precondition = 0 + + blob.upload_from_filename(source_file_name, if_generation_match=generation_match_precondition) + + print( + f"File {source_file_name} uploaded to {destination_blob_name}." + ) + + +# [END storage_upload_file] + +if __name__ == "__main__": + upload_blob( + bucket_name=sys.argv[1], + source_file_name=sys.argv[2], + destination_blob_name=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_upload_from_memory.py b/storage/samples/snippets/storage_upload_from_memory.py new file mode 100644 index 00000000000..eff3d222afd --- /dev/null +++ b/storage/samples/snippets/storage_upload_from_memory.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_file_upload_from_memory] +from google.cloud import storage + + +def upload_blob_from_memory(bucket_name, contents, destination_blob_name): + """Uploads a file to the bucket.""" + + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The contents to upload to the file + # contents = "these are my contents" + + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + blob.upload_from_string(contents) + + print( + f"{destination_blob_name} with contents {contents} uploaded to {bucket_name}." + ) + +# [END storage_file_upload_from_memory] + + +if __name__ == "__main__": + upload_blob_from_memory( + bucket_name=sys.argv[1], + contents=sys.argv[2], + destination_blob_name=sys.argv[3], + ) diff --git a/storage/samples/snippets/storage_upload_from_stream.py b/storage/samples/snippets/storage_upload_from_stream.py new file mode 100644 index 00000000000..08eb2588907 --- /dev/null +++ b/storage/samples/snippets/storage_upload_from_stream.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_stream_file_upload] +from google.cloud import storage + + +def upload_blob_from_stream(bucket_name, file_obj, destination_blob_name): + """Uploads bytes from a stream or other file-like object to a blob.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The stream or file (file-like object) from which to read + # import io + # file_obj = io.BytesIO() + # file_obj.write(b"This is test data.") + + # The desired name of the uploaded GCS object (blob) + # destination_blob_name = "storage-object-name" + + # Construct a client-side representation of the blob. + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + # Rewind the stream to the beginning. This step can be omitted if the input + # stream will always be at a correct position. + file_obj.seek(0) + + # Upload data from the stream to your bucket. + blob.upload_from_file(file_obj) + + print( + f"Stream data uploaded to {destination_blob_name} in bucket {bucket_name}." + ) + +# [END storage_stream_file_upload] diff --git a/storage/samples/snippets/storage_upload_with_kms_key.py b/storage/samples/snippets/storage_upload_with_kms_key.py new file mode 100644 index 00000000000..6e8fe039404 --- /dev/null +++ b/storage/samples/snippets/storage_upload_with_kms_key.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_upload_with_kms_key] +from google.cloud import storage + + +def upload_blob_with_kms( + bucket_name, source_file_name, destination_blob_name, kms_key_name, +): + """Uploads a file to the bucket, encrypting it with the given KMS key.""" + # bucket_name = "your-bucket-name" + # source_file_name = "local/path/to/file" + # destination_blob_name = "storage-object-name" + # kms_key_name = "projects/PROJ/locations/LOC/keyRings/RING/cryptoKey/KEY" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name, kms_key_name=kms_key_name) + + # Optional: set a generation-match precondition to avoid potential race conditions + # and data corruptions. The request to upload is aborted if the object's + # generation number does not match your precondition. For a destination + # object that does not yet exist, set the if_generation_match precondition to 0. + # If the destination object already exists in your bucket, set instead a + # generation-match precondition using its generation number. + generation_match_precondition = 0 + + blob.upload_from_filename(source_file_name, if_generation_match=generation_match_precondition) + + print( + "File {} uploaded to {} with encryption key {}.".format( + source_file_name, destination_blob_name, kms_key_name + ) + ) + + +# [END storage_upload_with_kms_key] + +if __name__ == "__main__": + upload_blob_with_kms( + bucket_name=sys.argv[1], + source_file_name=sys.argv[2], + destination_blob_name=sys.argv[3], + kms_key_name=sys.argv[4], + ) diff --git a/storage/samples/snippets/storage_view_bucket_iam_members.py b/storage/samples/snippets/storage_view_bucket_iam_members.py new file mode 100644 index 00000000000..184a1361f0f --- /dev/null +++ b/storage/samples/snippets/storage_view_bucket_iam_members.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_view_bucket_iam_members] +from google.cloud import storage + + +def view_bucket_iam_members(bucket_name): + """View IAM Policy for a bucket""" + # bucket_name = "your-bucket-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + policy = bucket.get_iam_policy(requested_policy_version=3) + + for binding in policy.bindings: + print(f"Role: {binding['role']}, Members: {binding['members']}") + + +# [END storage_view_bucket_iam_members] + + +if __name__ == "__main__": + view_bucket_iam_members(bucket_name=sys.argv[1]) diff --git a/storage/samples/snippets/uniform_bucket_level_access_test.py b/storage/samples/snippets/uniform_bucket_level_access_test.py new file mode 100644 index 00000000000..8b7964038ac --- /dev/null +++ b/storage/samples/snippets/uniform_bucket_level_access_test.py @@ -0,0 +1,52 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import storage_disable_uniform_bucket_level_access +import storage_enable_uniform_bucket_level_access +import storage_get_uniform_bucket_level_access + + +def test_get_uniform_bucket_level_access(bucket, capsys): + storage_get_uniform_bucket_level_access.get_uniform_bucket_level_access( + bucket.name + ) + out, _ = capsys.readouterr() + assert ( + f"Uniform bucket-level access is disabled for {bucket.name}." + in out + ) + + +def test_enable_uniform_bucket_level_access(bucket, capsys): + short_name = storage_enable_uniform_bucket_level_access + short_name.enable_uniform_bucket_level_access( + bucket.name + ) + out, _ = capsys.readouterr() + assert ( + f"Uniform bucket-level access was enabled for {bucket.name}." + in out + ) + + +def test_disable_uniform_bucket_level_access(bucket, capsys): + short_name = storage_disable_uniform_bucket_level_access + short_name.disable_uniform_bucket_level_access( + bucket.name + ) + out, _ = capsys.readouterr() + assert ( + f"Uniform bucket-level access was disabled for {bucket.name}." + in out + ) diff --git a/storage/samples/snippets/zonal_buckets/README.md b/storage/samples/snippets/zonal_buckets/README.md new file mode 100644 index 00000000000..71c17e5c3f1 --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/README.md @@ -0,0 +1,78 @@ +# Google Cloud Storage - Zonal Buckets Snippets + +This directory contains snippets for interacting with Google Cloud Storage zonal buckets. + +## Prerequisites + +- A Google Cloud Platform project with the Cloud Storage API enabled. +- A zonal Google Cloud Storage bucket. + +## Running the snippets + +### Create and write to an appendable object + +This snippet uploads an appendable object to a zonal bucket. + +```bash +python samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py --bucket_name --object_name +``` + +### Finalize an appendable object upload + +This snippet creates, writes to, and finalizes an appendable object. + +```bash +python samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py --bucket_name --object_name +``` + +### Pause and resume an appendable object upload + +This snippet demonstrates pausing and resuming an appendable object upload. + +```bash +python samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py --bucket_name --object_name +``` + +### Tail an appendable object + +This snippet demonstrates tailing an appendable GCS object, similar to `tail -f`. + +```bash +python samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py --bucket_name --object_name --duration +``` + + +### Download a range of bytes from an object + +This snippet downloads a range of bytes from an object. + +```bash +python samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py --bucket_name --object_name --start_byte --size +``` + + +### Download multiple ranges of bytes from a single object + +This snippet downloads multiple ranges of bytes from a single object into different buffers. + +```bash +python samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py --bucket_name --object_name +``` + +### Download the entire content of an object + +This snippet downloads the entire content of an object using a multi-range downloader. + +```bash +python samples/snippets/zonal_buckets/storage_open_object_read_full_object.py --bucket_name --object_name +``` + + + +### Download a range of bytes from multiple objects concurrently + +This snippet downloads a range of bytes from multiple objects concurrently. + +```bash +python samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py --bucket_name --object_names +``` \ No newline at end of file diff --git a/storage/samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py b/storage/samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py new file mode 100644 index 00000000000..725eeb2bd98 --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_create_and_write_appendable_object.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio + +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + AsyncAppendableObjectWriter, +) +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient + + +# [START storage_create_and_write_appendable_object] + + +async def storage_create_and_write_appendable_object( + bucket_name, object_name, grpc_client=None +): + """Uploads an appendable object to zonal bucket. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + + if grpc_client is None: + grpc_client = AsyncGrpcClient() + writer = AsyncAppendableObjectWriter( + client=grpc_client, + bucket_name=bucket_name, + object_name=object_name, + generation=0, # throws `FailedPrecondition` if object already exists. + ) + # This creates a new appendable object of size 0 and opens it for appending. + await writer.open() + + # appends data to the object + # you can perform `.append` multiple times as needed. Data will be appended + # to the end of the object. + await writer.append(b"Some data") + + # Once all appends are done, close the gRPC bidirectional stream. + await writer.close() + + print( + f"Appended object {object_name} created of size {writer.persisted_size} bytes." + ) + + +# [END storage_create_and_write_appendable_object] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument("--object_name", help="Your Cloud Storage object name.") + + args = parser.parse_args() + + asyncio.run( + storage_create_and_write_appendable_object( + bucket_name=args.bucket_name, + object_name=args.object_name, + ) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py b/storage/samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py new file mode 100644 index 00000000000..807fe40a58d --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_finalize_appendable_object_upload.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio + +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + AsyncAppendableObjectWriter, +) +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient + + +# [START storage_finalize_appendable_object_upload] +async def storage_finalize_appendable_object_upload( + bucket_name, object_name, grpc_client=None +): + """Creates, writes to, and finalizes an appendable object. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + + if grpc_client is None: + grpc_client = AsyncGrpcClient() + writer = AsyncAppendableObjectWriter( + client=grpc_client, + bucket_name=bucket_name, + object_name=object_name, + generation=0, # throws `FailedPrecondition` if object already exists. + ) + # This creates a new appendable object of size 0 and opens it for appending. + await writer.open() + + # Appends data to the object. + await writer.append(b"Some data") + + # finalize the appendable object, + # NOTE: + # 1. once finalized no more appends can be done to the object. + # 2. If you don't want to finalize, you can simply call `writer.close` + # 3. calling `.finalize()` also closes the grpc-bidi stream, calling + # `.close` after `.finalize` may lead to undefined behavior. + object_resource = await writer.finalize() + + print(f"Appendable object {object_name} created and finalized.") + print("Object Metadata:") + print(object_resource) + + +# [END storage_finalize_appendable_object_upload] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument("--object_name", help="Your Cloud Storage object name.") + + args = parser.parse_args() + + asyncio.run( + storage_finalize_appendable_object_upload( + bucket_name=args.bucket_name, + object_name=args.object_name, + ) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py b/storage/samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py new file mode 100644 index 00000000000..bed580d3662 --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Downloads a range of bytes from multiple objects concurrently. +Example usage: + ```python samples/snippets/zonal_buckets/storage_open_multiple_objects_ranged_read.py \ + --bucket_name \ + --object_names ``` +""" +import argparse +import asyncio +from io import BytesIO + +from google.cloud.storage.asyncio.async_grpc_client import ( + AsyncGrpcClient, +) +from google.cloud.storage.asyncio.async_multi_range_downloader import ( + AsyncMultiRangeDownloader, +) + + +# [START storage_open_multiple_objects_ranged_read] +async def storage_open_multiple_objects_ranged_read( + bucket_name, object_names, grpc_client=None +): + """Downloads a range of bytes from multiple objects concurrently. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + if grpc_client is None: + grpc_client = AsyncGrpcClient() + + async def _download_range(object_name): + """Helper coroutine to download a range from a single object.""" + mrd = AsyncMultiRangeDownloader(grpc_client, bucket_name, object_name) + try: + # Open the object, mrd always opens in read mode. + await mrd.open() + + # Each object downloads the first 100 bytes. + start_byte = 0 + size = 100 + + # requested range will be downloaded into this buffer, user may provide + # their own buffer or file-like object. + output_buffer = BytesIO() + await mrd.download_ranges([(start_byte, size, output_buffer)]) + finally: + if mrd.is_stream_open: + await mrd.close() + + # Downloaded size can differ from requested size if object is smaller. + # mrd will download at most up to the end of the object. + downloaded_size = output_buffer.getbuffer().nbytes + print(f"Downloaded {downloaded_size} bytes from {object_name}") + + download_tasks = [_download_range(name) for name in object_names] + await asyncio.gather(*download_tasks) + + +# [END storage_open_multiple_objects_ranged_read] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument( + "--object_names", nargs="+", help="Your Cloud Storage object name(s)." + ) + + args = parser.parse_args() + + asyncio.run( + storage_open_multiple_objects_ranged_read(args.bucket_name, args.object_names) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py b/storage/samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py new file mode 100644 index 00000000000..b0f64c48690 --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_open_object_multiple_ranged_read.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio +from io import BytesIO + +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_multi_range_downloader import ( + AsyncMultiRangeDownloader, +) + + +# [START storage_open_object_multiple_ranged_read] +async def storage_open_object_multiple_ranged_read( + bucket_name, object_name, grpc_client=None +): + """Downloads multiple ranges of bytes from a single object into different buffers. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + if grpc_client is None: + grpc_client = AsyncGrpcClient() + + mrd = AsyncMultiRangeDownloader(grpc_client, bucket_name, object_name) + + try: + # Open the object, mrd always opens in read mode. + await mrd.open() + + # Specify four different buffers to download ranges into. + buffers = [BytesIO(), BytesIO(), BytesIO(), BytesIO()] + + # Define the ranges to download. Each range is a tuple of (start_byte, size, buffer). + # All ranges will download 10 bytes from different starting positions. + # We choose arbitrary start bytes for this example. An object should be large enough. + # A user can choose any start byte between 0 and `object_size`. + # If `start_bytes` is greater than `object_size`, mrd will throw an error. + ranges = [ + (0, 10, buffers[0]), + (20, 10, buffers[1]), + (40, 10, buffers[2]), + (60, 10, buffers[3]), + ] + + await mrd.download_ranges(ranges) + + finally: + await mrd.close() + + # Print the downloaded content from each buffer. + for i, output_buffer in enumerate(buffers): + downloaded_size = output_buffer.getbuffer().nbytes + print( + f"Downloaded {downloaded_size} bytes into buffer {i + 1} from start byte {ranges[i][0]}: {output_buffer.getvalue()}" + ) + + +# [END storage_open_object_multiple_ranged_read] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument("--object_name", help="Your Cloud Storage object name.") + + args = parser.parse_args() + + asyncio.run( + storage_open_object_multiple_ranged_read(args.bucket_name, args.object_name) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_open_object_read_full_object.py b/storage/samples/snippets/zonal_buckets/storage_open_object_read_full_object.py new file mode 100644 index 00000000000..2e18caabe23 --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_open_object_read_full_object.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio +from io import BytesIO + +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_multi_range_downloader import ( + AsyncMultiRangeDownloader, +) + + +# [START storage_open_object_read_full_object] +async def storage_open_object_read_full_object( + bucket_name, object_name, grpc_client=None +): + """Downloads the entire content of an object using a multi-range downloader. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + if grpc_client is None: + grpc_client = AsyncGrpcClient() + + # mrd = Multi-Range-Downloader + mrd = AsyncMultiRangeDownloader(grpc_client, bucket_name, object_name) + + try: + # Open the object, mrd always opens in read mode. + await mrd.open() + + # This could be any buffer or file-like object. + output_buffer = BytesIO() + # A download range of (0, 0) means to read from the beginning to the end. + await mrd.download_ranges([(0, 0, output_buffer)]) + finally: + if mrd.is_stream_open: + await mrd.close() + + downloaded_bytes = output_buffer.getvalue() + print( + f"Downloaded all {len(downloaded_bytes)} bytes from object {object_name} in bucket {bucket_name}." + ) + + +# [END storage_open_object_read_full_object] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument("--object_name", help="Your Cloud Storage object name.") + + args = parser.parse_args() + + asyncio.run( + storage_open_object_read_full_object(args.bucket_name, args.object_name) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py b/storage/samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py new file mode 100644 index 00000000000..74bec43f68e --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_open_object_single_ranged_read.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio +from io import BytesIO + +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_multi_range_downloader import ( + AsyncMultiRangeDownloader, +) + + +# [START storage_open_object_single_ranged_read] +async def storage_open_object_single_ranged_read( + bucket_name, object_name, start_byte, size, grpc_client=None +): + """Downloads a range of bytes from an object. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + if grpc_client is None: + grpc_client = AsyncGrpcClient() + + mrd = AsyncMultiRangeDownloader(grpc_client, bucket_name, object_name) + + try: + # Open the object, mrd always opens in read mode. + await mrd.open() + + # requested range will be downloaded into this buffer, user may provide + # their own buffer or file-like object. + output_buffer = BytesIO() + await mrd.download_ranges([(start_byte, size, output_buffer)]) + finally: + if mrd.is_stream_open: + await mrd.close() + + # Downloaded size can differ from requested size if object is smaller. + # mrd will download at most up to the end of the object. + downloaded_size = output_buffer.getbuffer().nbytes + print(f"Downloaded {downloaded_size} bytes from {object_name}") + + +# [END storage_open_object_single_ranged_read] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument("--object_name", help="Your Cloud Storage object name.") + parser.add_argument( + "--start_byte", type=int, help="The starting byte of the range." + ) + parser.add_argument("--size", type=int, help="The number of bytes to download.") + + args = parser.parse_args() + + asyncio.run( + storage_open_object_single_ranged_read( + args.bucket_name, args.object_name, args.start_byte, args.size + ) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py b/storage/samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py new file mode 100644 index 00000000000..c758dc6419d --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_pause_and_resume_appendable_upload.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio + +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + AsyncAppendableObjectWriter, +) +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient + + +# [START storage_pause_and_resume_appendable_upload] +async def storage_pause_and_resume_appendable_upload( + bucket_name, object_name, grpc_client=None +): + """Demonstrates pausing and resuming an appendable object upload. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + if grpc_client is None: + grpc_client = AsyncGrpcClient() + + writer1 = AsyncAppendableObjectWriter( + client=grpc_client, + bucket_name=bucket_name, + object_name=object_name, + ) + await writer1.open() + await writer1.append(b"First part of the data. ") + print(f"Appended {writer1.persisted_size} bytes with the first writer.") + + # 2. After appending some data, close the writer to "pause" the upload. + # NOTE: you can pause indefinitely and still read the conetent uploaded so far using MRD. + await writer1.close() + + print("First writer closed. Upload is 'paused'.") + + # 3. Create a new writer, passing the generation number from the previous + # writer. This is a precondition to ensure that the object hasn't been + # modified since we last accessed it. + generation_to_resume = writer1.generation + print(f"Generation to resume from is: {generation_to_resume}") + + writer2 = AsyncAppendableObjectWriter( + client=grpc_client, + bucket_name=bucket_name, + object_name=object_name, + generation=generation_to_resume, + ) + # 4. Open the new writer. + try: + await writer2.open() + + # 5. Append some more data using the new writer. + await writer2.append(b"Second part of the data.") + print(f"Appended more data. Total size is now {writer2.persisted_size} bytes.") + finally: + # 6. Finally, close the new writer. + if writer2._is_stream_open: + await writer2.close() + print("Second writer closed. Full object uploaded.") + + +# [END storage_pause_and_resume_appendable_upload] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument("--object_name", help="Your Cloud Storage object name.") + + args = parser.parse_args() + + asyncio.run( + storage_pause_and_resume_appendable_upload( + bucket_name=args.bucket_name, + object_name=args.object_name, + ) + ) diff --git a/storage/samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py b/storage/samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py new file mode 100644 index 00000000000..6248980669c --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/storage_read_appendable_object_tail.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import asyncio +import time +from datetime import datetime +from io import BytesIO + +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + AsyncAppendableObjectWriter, +) +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_multi_range_downloader import ( + AsyncMultiRangeDownloader, +) + +BYTES_TO_APPEND = b"fav_bytes." * 100 * 1024 * 1024 +NUM_BYTES_TO_APPEND_EVERY_SECOND = len(BYTES_TO_APPEND) + + +# [START storage_read_appendable_object_tail] +async def appender(writer: AsyncAppendableObjectWriter, duration: int): + """Appends 10 bytes to the object every second for a given duration.""" + print("Appender started.") + bytes_appended = 0 + start_time = time.monotonic() + # Run the appender for the specified duration. + while time.monotonic() - start_time < duration: + await writer.append(BYTES_TO_APPEND) + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + bytes_appended += NUM_BYTES_TO_APPEND_EVERY_SECOND + print( + f"[{now}] Appended {NUM_BYTES_TO_APPEND_EVERY_SECOND} new bytes. Total appended: {bytes_appended} bytes." + ) + await asyncio.sleep(0.1) + print("Appender finished.") + + +async def tailer( + bucket_name: str, object_name: str, duration: int, client: AsyncGrpcClient +): + """Tails the object by reading new data as it is appended.""" + print("Tailer started.") + start_byte = 0 + start_time = time.monotonic() + mrd = AsyncMultiRangeDownloader(client, bucket_name, object_name) + try: + await mrd.open() + # Run the tailer for the specified duration. + while time.monotonic() - start_time < duration: + output_buffer = BytesIO() + # A download range of (start, 0) means to read from 'start' to the end. + await mrd.download_ranges([(start_byte, 0, output_buffer)]) + + bytes_downloaded = output_buffer.getbuffer().nbytes + if bytes_downloaded > 0: + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + print(f"[{now}] Tailer read {bytes_downloaded} new bytes: ") + start_byte += bytes_downloaded + + await asyncio.sleep(0.1) # Poll for new data every 0.1 seconds. + finally: + if mrd.is_stream_open: + await mrd.close() + print("Tailer finished.") + + +# read_appendable_object_tail simulates a "tail -f" command on a GCS object. It +# repeatedly polls an appendable object for new content. In a real +# application, the object would be written to by a separate process. +async def read_appendable_object_tail( + bucket_name: str, object_name: str, duration: int, grpc_client=None +): + """Main function to create an appendable object and run tasks. + + grpc_client: an existing grpc_client to use, this is only for testing. + """ + if grpc_client is None: + grpc_client = AsyncGrpcClient() + writer = AsyncAppendableObjectWriter( + client=grpc_client, + bucket_name=bucket_name, + object_name=object_name, + ) + # 1. Create an empty appendable object. + try: + # 1. Create an empty appendable object. + await writer.open() + print(f"Created empty appendable object: {object_name}") + + # 2. Create the appender and tailer coroutines. + appender_task = asyncio.create_task(appender(writer, duration)) + tailer_task = asyncio.create_task( + tailer(bucket_name, object_name, duration, grpc_client) + ) + + # 3. Execute the coroutines concurrently. + await asyncio.gather(appender_task, tailer_task) + finally: + if writer._is_stream_open: + await writer.close() + print("Writer closed.") + + +# [END storage_read_appendable_object_tail] + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Demonstrates tailing an appendable GCS object.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--bucket_name", help="Your Cloud Storage bucket name.") + parser.add_argument( + "--object_name", help="Your Cloud Storage object name to be created." + ) + parser.add_argument( + "--duration", + type=int, + default=60, + help="Duration in seconds to run the demo.", + ) + + args = parser.parse_args() + + asyncio.run( + read_appendable_object_tail(args.bucket_name, args.object_name, args.duration) + ) diff --git a/storage/samples/snippets/zonal_buckets/zonal_snippets_test.py b/storage/samples/snippets/zonal_buckets/zonal_snippets_test.py new file mode 100644 index 00000000000..6852efe2286 --- /dev/null +++ b/storage/samples/snippets/zonal_buckets/zonal_snippets_test.py @@ -0,0 +1,260 @@ +# Copyright 2025 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import uuid +import os + +import pytest +from google.cloud.storage import Client +import contextlib + +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + AsyncAppendableObjectWriter, +) + +# Import all the snippets +import storage_create_and_write_appendable_object +import storage_finalize_appendable_object_upload +import storage_open_multiple_objects_ranged_read +import storage_open_object_multiple_ranged_read +import storage_open_object_read_full_object +import storage_open_object_single_ranged_read +import storage_pause_and_resume_appendable_upload +import storage_read_appendable_object_tail + +pytestmark = pytest.mark.skipif( + os.getenv("RUN_ZONAL_SYSTEM_TESTS") != "True", + reason="Zonal system tests need to be explicitly enabled. This helps scheduling tests in Kokoro and Cloud Build.", +) + + +# TODO: replace this with a fixture once zonal bucket creation / deletion +# is supported in grpc client or json client client. +_ZONAL_BUCKET = os.getenv("ZONAL_BUCKET") + + +async def create_async_grpc_client(): + """Initializes async client and gets the current event loop.""" + return AsyncGrpcClient() + + +# Forcing a single event loop for the whole test session +@pytest.fixture(scope="session") +def event_loop(): + """Redefine pytest-asyncio's event_loop fixture to be session-scoped.""" + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + +@pytest.fixture(scope="session") +def async_grpc_client(event_loop): + """Yields a StorageAsyncClient that is closed after the test session.""" + grpc_client = event_loop.run_until_complete(create_async_grpc_client()) + yield grpc_client + + +@pytest.fixture(scope="session") +def json_client(): + client = Client() + with contextlib.closing(client): + yield client + + +async def create_appendable_object(grpc_client, object_name, data): + writer = AsyncAppendableObjectWriter( + client=grpc_client, + bucket_name=_ZONAL_BUCKET, + object_name=object_name, + generation=0, # throws `FailedPrecondition` if object already exists. + ) + await writer.open() + await writer.append(data) + await writer.close() + return writer.generation + + +# TODO: replace this with a fixture once zonal bucket creation / deletion +# is supported in grpc client or json client client. +_ZONAL_BUCKET = os.getenv("ZONAL_BUCKET") + + +def test_storage_create_and_write_appendable_object( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"zonal-snippets-test-{uuid.uuid4()}" + + event_loop.run_until_complete( + storage_create_and_write_appendable_object.storage_create_and_write_appendable_object( + _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert f"Appended object {object_name} created of size" in out + + blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob.delete() + + +def test_storage_finalize_appendable_object_upload( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"test-finalize-appendable-{uuid.uuid4()}" + event_loop.run_until_complete( + storage_finalize_appendable_object_upload.storage_finalize_appendable_object_upload( + _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert f"Appendable object {object_name} created and finalized." in out + blob = json_client.bucket(_ZONAL_BUCKET).get_blob(object_name) + blob.delete() + + +def test_storage_pause_and_resume_appendable_upload( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"test-pause-resume-{uuid.uuid4()}" + event_loop.run_until_complete( + storage_pause_and_resume_appendable_upload.storage_pause_and_resume_appendable_upload( + _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert "First writer closed. Upload is 'paused'." in out + assert "Second writer closed. Full object uploaded." in out + + blob = json_client.bucket(_ZONAL_BUCKET).get_blob(object_name) + blob.delete() + + +def test_storage_read_appendable_object_tail( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"test-read-tail-{uuid.uuid4()}" + event_loop.run_until_complete( + storage_read_appendable_object_tail.read_appendable_object_tail( + _ZONAL_BUCKET, object_name, duration=3, grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert f"Created empty appendable object: {object_name}" in out + assert "Appender started." in out + assert "Tailer started." in out + assert "Tailer read" in out + assert "Tailer finished." in out + assert "Writer closed." in out + + bucket = json_client.bucket(_ZONAL_BUCKET) + blob = bucket.blob(object_name) + blob.delete() + + +def test_storage_open_object_read_full_object( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"test-open-read-full-{uuid.uuid4()}" + data = b"Hello, is it me you're looking for?" + event_loop.run_until_complete( + create_appendable_object(async_grpc_client, object_name, data) + ) + event_loop.run_until_complete( + storage_open_object_read_full_object.storage_open_object_read_full_object( + _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert ( + f"Downloaded all {len(data)} bytes from object {object_name} in bucket {_ZONAL_BUCKET}." + in out + ) + blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob.delete() + + +def test_storage_open_object_single_ranged_read( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"test-open-single-range-{uuid.uuid4()}" + event_loop.run_until_complete( + create_appendable_object( + async_grpc_client, object_name, b"Hello, is it me you're looking for?" + ) + ) + download_size = 5 + event_loop.run_until_complete( + storage_open_object_single_ranged_read.storage_open_object_single_ranged_read( + _ZONAL_BUCKET, + object_name, + start_byte=0, + size=download_size, + grpc_client=async_grpc_client, + ) + ) + out, _ = capsys.readouterr() + assert f"Downloaded {download_size} bytes from {object_name}" in out + blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob.delete() + + +def test_storage_open_object_multiple_ranged_read( + async_grpc_client, json_client, event_loop, capsys +): + object_name = f"test-open-multi-range-{uuid.uuid4()}" + data = b"a" * 100 + event_loop.run_until_complete( + create_appendable_object(async_grpc_client, object_name, data) + ) + event_loop.run_until_complete( + storage_open_object_multiple_ranged_read.storage_open_object_multiple_ranged_read( + _ZONAL_BUCKET, object_name, grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert "Downloaded 10 bytes into buffer 1 from start byte 0: b'aaaaaaaaaa'" in out + assert "Downloaded 10 bytes into buffer 2 from start byte 20: b'aaaaaaaaaa'" in out + assert "Downloaded 10 bytes into buffer 3 from start byte 40: b'aaaaaaaaaa'" in out + assert "Downloaded 10 bytes into buffer 4 from start byte 60: b'aaaaaaaaaa'" in out + blob = json_client.bucket(_ZONAL_BUCKET).blob(object_name) + blob.delete() + + +def test_storage_open_multiple_objects_ranged_read( + async_grpc_client, json_client, event_loop, capsys +): + blob1_name = f"multi-obj-1-{uuid.uuid4()}" + blob2_name = f"multi-obj-2-{uuid.uuid4()}" + data1 = b"Content of object 1" + data2 = b"Content of object 2" + event_loop.run_until_complete( + create_appendable_object(async_grpc_client, blob1_name, data1) + ) + event_loop.run_until_complete( + create_appendable_object(async_grpc_client, blob2_name, data2) + ) + + event_loop.run_until_complete( + storage_open_multiple_objects_ranged_read.storage_open_multiple_objects_ranged_read( + _ZONAL_BUCKET, [blob1_name, blob2_name], grpc_client=async_grpc_client + ) + ) + out, _ = capsys.readouterr() + assert f"Downloaded {len(data1)} bytes from {blob1_name}" in out + assert f"Downloaded {len(data2)} bytes from {blob2_name}" in out + blob1 = json_client.bucket(_ZONAL_BUCKET).blob(blob1_name) + blob2 = json_client.bucket(_ZONAL_BUCKET).blob(blob2_name) + blob1.delete() + blob2.delete() diff --git a/storage/hierarchical-namespace/README.md b/storagecontrol/hierarchical-namespace/README.md similarity index 90% rename from storage/hierarchical-namespace/README.md rename to storagecontrol/hierarchical-namespace/README.md index abe0066526b..70fe6525e7a 100644 --- a/storage/hierarchical-namespace/README.md +++ b/storagecontrol/hierarchical-namespace/README.md @@ -34,4 +34,6 @@ This script recursively deletes empty folders within a specified GCS bucket and 3. **Run:** `bash python3 delete_empty_folders.py` **Note:** This script *only* deletes folders. Folders containing any objects -will not be deleted, and a "Failed Precondition" warning will be logged. +will not be deleted, and a "Failed Precondition" warning will be logged. Creating empty folders +through the web interface will result in folders that are not truly empty, with a hidden 0 byte +file. Use CLI instead. diff --git a/storagecontrol/hierarchical-namespace/delete_empty_folders.py b/storagecontrol/hierarchical-namespace/delete_empty_folders.py new file mode 100644 index 00000000000..2aa9d8bf054 --- /dev/null +++ b/storagecontrol/hierarchical-namespace/delete_empty_folders.py @@ -0,0 +1,387 @@ +# Copyright 2025 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import concurrent.futures +import logging +import threading +import time + +from google.api_core import exceptions as google_exceptions +from google.api_core import retry +from google.cloud import storage_control_v2 +import grpc + +ThreadPoolExecutor = concurrent.futures.ThreadPoolExecutor + +# This script may be used to recursively delete a large number of nested empty +# folders in a GCS HNS bucket. Overview of the algorithm: +# 1. Folder Discovery: +# - Lists all folders under the BUCKET_NAME and FOLDER_PREFIX (if set). +# - Partitions all discovered folders into a map, keyed by depth +# (e.g. {1: [foo1/ foo2/], 2: [foo1/bar1/, foo1/bar2/, foo2/bar3/], ...} +# 2. Folder Deletion: +# - Processes depths in reverse order (from deepest to shallowest). +# - For each depth level, submits all folders at that level to a thread pool +# for parallel deletion. +# - Only moves to the next depth level once all folders at the current depth +# have been processed. This ensures that child folders are removed before +# their parents, respecting hierarhical constraints. +# +# Note: This script only deletes folders, not objects; any folders with child +# objects (immediate or nested) will fail to be deleted. +# +# Usage: See README.md for instructions. + +# --- Configuration --- +BUCKET_NAME = "YOUR_BUCKET_NAME" + +# e.g. "archive/old_data/" or "" to delete all folders in the bucket. +# If specified, must end with '/'. +FOLDER_PREFIX = "YOUR_FOLDER_PREFIX/" + +# Max number of concurrent threads to use for deleting folders. +MAX_WORKERS = 32 + +# How often to log statistics during deletion, in seconds. +STATS_REPORT_INTERVAL = 5 + +# --- Data Structures & Globals --- +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(threadName)s - %(message)s" +) + +# Global map to store folders by their depth +# { depth (int) -> set of full_resource_names (str) } +folders_by_depth = {} + +# Stats for monitoring progress +stats = { + "found_total": 0, + "successful_deletes": 0, + "failed_deletes_precondition": 0, + "failed_deletes_internal": 0, +} +stats_lock = threading.Lock() + +# Initialize the Storage Control API client +storage_control_client = storage_control_v2.StorageControlClient() + + +def _get_simple_path_and_depth(full_resource_name: str) -> tuple[str, int]: + """Extracts bucket-relative path and depth from a GCS folder resource name. + + The "simple path" is relative to the bucket (e.g., 'archive/logs/' for + 'projects/_/buckets/my-bucket/folders/archive/logs/'). + + The "depth" is the number of '/' in the simple path (e.g. 'archive/logs/' is + depth 2). + + Args: + full_resource_name: The full resource name of the GCS folder, e.g., + 'projects/_/buckets/your-bucket-name/folders/path/to/folder/'. + + Returns: + A tuple (simple_path: str, depth: int). + + Raises: + ValueError: If the resource name does not match the expected format + (i.e. start with 'projects/_/buckets/BUCKET_NAME/folders/FOLDER_PREFIX' + and ends with a trailing slash). + """ + base_folders_prefix = f"projects/_/buckets/{BUCKET_NAME}/folders/" + # The full prefix to validate against, including the global FOLDER_PREFIX. + # If FOLDER_PREFIX is "", this is equivalent to base_folders_prefix. + expected_validation_prefix = base_folders_prefix + FOLDER_PREFIX + + if not full_resource_name.startswith( + expected_validation_prefix + ) or not full_resource_name.endswith("/"): + raise ValueError( + f"Folder resource name '{full_resource_name}' does not match expected" + f" prefix '{expected_validation_prefix}' or missing trailing slash." + ) + + simple_path = full_resource_name[len(base_folders_prefix) :] + depth = simple_path.count("/") + if depth < 1: + raise ValueError( + f"Folder resource name '{full_resource_name}' has invalid depth" + f" {depth} (expected at least 1)." + ) + return simple_path, depth + + +def discover_and_partition_folders() -> None: + """Discovers all folders in the bucket and partitions them by depth. + + Result is stored in the global folders_by_depth dictionary. + """ + parent_resource = f"projects/_/buckets/{BUCKET_NAME}" + + logging.info( + "Starting folder discovery and partitioning for bucket '%s'." + " Using prefix filter: '%s'.", + BUCKET_NAME, + FOLDER_PREFIX if FOLDER_PREFIX else "NONE (all folders)", + ) + + list_folders_request = storage_control_v2.ListFoldersRequest( + parent=parent_resource, prefix=FOLDER_PREFIX + ) + + num_folders_found = 0 + try: + for folder in storage_control_client.list_folders(request=list_folders_request): + full_resource_name = folder.name + _, depth = _get_simple_path_and_depth(full_resource_name) + + if depth not in folders_by_depth: + folders_by_depth[depth] = set() + folders_by_depth[depth].add(full_resource_name) + + num_folders_found += 1 + with stats_lock: + stats["found_total"] = num_folders_found + + except Exception as e: + logging.error("Failed to list folders: %s", e, exc_info=True) + return + + logging.info("Finished discovery. Total folders found: %s.", num_folders_found) + if not folders_by_depth: + logging.info("No folders found in the bucket.") + else: + logging.info("Folders partitioned by depth:") + for depth_val in sorted(folders_by_depth.keys()): + logging.info( + " Depth %s: %s folders", depth_val, len(folders_by_depth[depth_val]) + ) + + +# Defines retriable error codes for the DeleteFolder API call. +def should_retry(exception: Exception) -> int | None: + if not isinstance(exception, (google_exceptions.GoogleAPICallError, grpc.RpcError)): + return False + + # gRPC status codes to retry on, matching the JSON + retryable_grpc_codes = [ + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.UNKNOWN, + ] + + status_code = None + if isinstance(exception, google_exceptions.GoogleAPICallError): + status_code = exception.code + elif isinstance(exception, grpc.RpcError): + # For grpc.RpcError, code() returns the status code enum + status_code = exception.code() + + return status_code in retryable_grpc_codes + + +def delete_folder(folder_full_resource_name: str) -> None: + """Attempts to delete a single GCS HNS folder. + + Includes retry logic for transient errors. + + Stores stats in the global stats dictionary. + + Args: + folder_full_resource_name: The full resource name of the GCS folder to + delete, e.g., + 'projects/_/buckets/your-bucket-name/folders/path/to/folder/'. + """ + simple_path, _ = _get_simple_path_and_depth(folder_full_resource_name) + + retry_policy = retry.Retry( + predicate=should_retry, + initial=1.0, # Initial backoff: 1s + maximum=60.0, # Max backoff: 60s + multiplier=2.0, # Backoff multiplier: 2 + deadline=120.0, # Total time allowed for all retries and calls + ) + + try: + request = storage_control_v2.DeleteFolderRequest(name=folder_full_resource_name) + storage_control_client.delete_folder(request=request, retry=retry_policy) + + with stats_lock: + stats["successful_deletes"] += 1 + return # Success + + except google_exceptions.NotFound: + # This can happen if the folder was deleted by another process. + logging.warning( + "Folder not found for deletion (already gone?): %s", simple_path + ) + return # Not a retriable error + except google_exceptions.FailedPrecondition as e: + # This typically means the folder contains objects. + logging.warning("Deletion failed for '%s': %s.", simple_path, e.message) + with stats_lock: + stats["failed_deletes_precondition"] += 1 + return # Not a retriable error + except Exception as e: + logging.error( + "Failed to delete '%s': %s", + simple_path, + e, + exc_info=True, + ) + with stats_lock: + stats["failed_deletes_internal"] += 1 + return # All retries exhausted + + +# --- STATS REPORTER THREAD --- +def stats_reporter_thread_logic(stop_event: threading.Event, start_time: float) -> None: + """Logs current statistics periodically.""" + logging.info("Stats Reporter: Started.") + while not stop_event.wait(STATS_REPORT_INTERVAL): + with stats_lock: + elapsed = time.time() - start_time + rate = stats["successful_deletes"] / elapsed if elapsed > 0 else 0 + logging.info( + "[STATS] Total Folders Found: %s | Successful Deletes: %s | Failed" + " Deletes (precondition): %s | Failed Deletes (internal): %s | Rate:" + " %.2f folders/sec", + stats["found_total"], + stats["successful_deletes"], + stats["failed_deletes_precondition"], + stats["failed_deletes_internal"], + rate, + ) + logging.info("Stats Reporter: Shutting down.") + + +# --- MAIN EXECUTION BLOCK --- +if __name__ == "__main__": + if BUCKET_NAME == "your-gcs-bucket-name": + print( + "\nERROR: Please update the BUCKET_NAME variable in the script before" + " running." + ) + exit(1) + + if FOLDER_PREFIX and not FOLDER_PREFIX.endswith("/"): + print("\nERROR: FOLDER_PREFIX must end with a '/' if specified.") + exit(1) + + start_time = time.time() + + logging.info("Starting GCS HNS folder deletion for bucket: %s", BUCKET_NAME) + + # Event to signal threads to stop gracefully. + stop_event = threading.Event() + + # Start the stats reporter thread. + stats_thread = threading.Thread( + target=stats_reporter_thread_logic, + args=(stop_event, start_time), + name="StatsReporter", + daemon=True, + ) + stats_thread.start() + + # Step 1: Discover and Partition Folders. + discover_and_partition_folders() + + if not folders_by_depth: + logging.info("No folders found to delete. Exiting.") + exit(0) + + # Prepare for multi-threaded deletion within each depth level. + deletion_executor = ThreadPoolExecutor( + max_workers=MAX_WORKERS, thread_name_prefix="DeleteFolderWorker" + ) + + try: + # Step 2: Iterate and delete by depth (from max to min). + sorted_depths = sorted(folders_by_depth.keys(), reverse=True) + for current_depth in sorted_depths: + folders_at_current_depth = folders_by_depth.get(current_depth, set()) + + if not folders_at_current_depth: + logging.info( + "Skipping depth %s: No folders found at this depth.", current_depth + ) + continue + + logging.info( + "\nProcessing depth %s: Submitting %s folders for deletion...", + current_depth, + len(folders_at_current_depth), + ) + + # Submit deletion tasks to the executor. + futures = [ + deletion_executor.submit(delete_folder, folder_path) + for folder_path in folders_at_current_depth + ] + + # Wait for all tasks at the current depth to complete. + # This is critical: we must ensure all nested folders are gone before + # tackling their parents. + concurrent.futures.wait(futures) + + logging.info("Finished processing all folders at depth %s.", current_depth) + + except KeyboardInterrupt: + logging.info( + "Main: Keyboard interrupt received. Attempting graceful shutdown..." + ) + except Exception as e: + logging.error( + "An unexpected error occurred in the main loop: %s", e, exc_info=True + ) + finally: + # Signal all threads to stop. + stop_event.set() + + # Shut down deletion executor and wait for any pending tasks to complete. + logging.info( + "Main: Shutting down deletion workers. Waiting for any final tasks..." + ) + deletion_executor.shutdown(wait=True) + + # Wait for the stats reporter to finish. + if stats_thread.is_alive(): + stats_thread.join( + timeout=STATS_REPORT_INTERVAL + 2 + ) # Give it a bit more time. + + # Log final statistics. + final_elapsed_time = time.time() - start_time + logging.info("\n--- FINAL SUMMARY ---") + with stats_lock: + final_rate = ( + stats["successful_deletes"] / final_elapsed_time + if final_elapsed_time > 0 + else 0 + ) + logging.info( + " - Total Folders Found (Initial Scan): %s\n - Successful Folder" + " Deletes: %s\n - Failed Folder Deletes (Precondition): %s\n -" + " Failed Folder Deletes (Internal): %s\n - Total Runtime: %.2f" + " seconds\n - Average Deletion Rate: %.2f folders/sec", + stats["found_total"], + stats["successful_deletes"], + stats["failed_deletes_precondition"], + stats["failed_deletes_internal"], + final_elapsed_time, + final_rate, + ) + logging.info("Script execution finished.") diff --git a/storagecontrol/requirements.txt b/storagecontrol/requirements.txt index b360c1102db..e7b93b6c245 100644 --- a/storagecontrol/requirements.txt +++ b/storagecontrol/requirements.txt @@ -1 +1 @@ -google-cloud-storage-control==1.1.1 \ No newline at end of file +google-cloud-storage-control==1.11.0 \ No newline at end of file diff --git a/trace/trace-python-sample-opentelemetry/README.rst.in b/trace/trace-python-sample-opentelemetry/README.rst.in index 471ffa008eb..c97c06d71f3 100644 --- a/trace/trace-python-sample-opentelemetry/README.rst.in +++ b/trace/trace-python-sample-opentelemetry/README.rst.in @@ -1,11 +1,11 @@ # This file is used to generate README.rst product: - name: Stackdriver Trace - short_name: Stackdriver Trace + name: Cloud Trace + short_name: Cloud Trace url: https://cloud.google.com/trace/docs description: > - `Stackdriver Trace`_ collects latency data from applications and displays + `Cloud Trace`_ collects latency data from applications and displays it in near real time in the Google Cloud Platform Console. setup: diff --git a/trace/trace-python-sample/README.md b/trace/trace-python-sample/README.md deleted file mode 100644 index 91151999ab3..00000000000 --- a/trace/trace-python-sample/README.md +++ /dev/null @@ -1,7 +0,0 @@ -## Google Cloud Trace Python Samples - -The Cloud Trace samples have been moved. - -[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=trace/trace-python-sample-opentelemetry/README.rst) - -This directory is no longer contains samples for Google Cloud Trace. To see how to use Cloud Trace with OpenTelemetry please see [trace-python-sample-opentelemetry](../trace-python-sample-opentelemetry/README.rst). diff --git a/vision/snippets/crop_hints/requirements.txt b/vision/snippets/crop_hints/requirements.txt index 0267eb1541f..e813a2c08ea 100644 --- a/vision/snippets/crop_hints/requirements.txt +++ b/vision/snippets/crop_hints/requirements.txt @@ -1,3 +1,3 @@ google-cloud-vision==3.8.1 -pillow==9.5.0; python_version < '3.8' +pillow==10.3.0; python_version < '3.8' pillow==10.4.0; python_version >= '3.8' diff --git a/vision/snippets/document_text/requirements.txt b/vision/snippets/document_text/requirements.txt index 0267eb1541f..e813a2c08ea 100644 --- a/vision/snippets/document_text/requirements.txt +++ b/vision/snippets/document_text/requirements.txt @@ -1,3 +1,3 @@ google-cloud-vision==3.8.1 -pillow==9.5.0; python_version < '3.8' +pillow==10.3.0; python_version < '3.8' pillow==10.4.0; python_version >= '3.8' diff --git a/vision/snippets/face_detection/requirements.txt b/vision/snippets/face_detection/requirements.txt index 0267eb1541f..e813a2c08ea 100644 --- a/vision/snippets/face_detection/requirements.txt +++ b/vision/snippets/face_detection/requirements.txt @@ -1,3 +1,3 @@ google-cloud-vision==3.8.1 -pillow==9.5.0; python_version < '3.8' +pillow==10.3.0; python_version < '3.8' pillow==10.4.0; python_version >= '3.8'