From 632c320f9cf6be439b6fb595364597a23963f560 Mon Sep 17 00:00:00 2001 From: Stefan Amberger <1277330+snamber@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:55:44 +0000 Subject: [PATCH] Support Azure automation reads and shared obstore storage Use StorageLocationService metadata and obstore for cloud automation reads and S3, GCS, and Azure caches. Preserve provider credential discovery and lazy imports. Require patched gRPC and document macOS fork diagnostics. Include generated protobufs, regression tests, offline test isolation, and documentation. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd95-f057-731f-8eb9-cf195588c2cd --- CHANGELOG.md | 22 +- tilebox-grpc/_tilebox/grpc/replay.py | 3 +- tilebox-grpc/pyproject.toml | 2 +- tilebox-grpc/tests/test_fork_logging.py | 73 ++ tilebox-workflows/README.md | 13 + tilebox-workflows/pyproject.toml | 5 +- .../tests/automations/test_storage_read.py | 526 +++++++++++++ tilebox-workflows/tests/runner/test_runner.py | 13 +- tilebox-workflows/tests/tasks_data.py | 9 + tilebox-workflows/tests/test_cache.py | 216 +++++- .../tilebox/workflows/automations/service.py | 16 +- .../workflows/automations/storage_event.py | 2 +- tilebox-workflows/tilebox/workflows/cache.py | 141 +--- tilebox-workflows/tilebox/workflows/client.py | 9 +- tilebox-workflows/tilebox/workflows/data.py | 224 ++++-- .../workflows/v1/automation_connect.py | 260 ------- .../workflows/workflows/v1/automation_pb2.py | 47 +- .../workflows/workflows/v1/automation_pb2.pyi | 50 +- .../workflows/v1/automation_pb2_grpc.py | 176 ----- .../workflows/v1/storage_location_connect.py | 716 ++++++++++++++++++ .../workflows/v1/storage_location_pb2.py | 114 +++ .../workflows/v1/storage_location_pb2.pyi | 260 +++++++ .../workflows/v1/storage_location_pb2_grpc.py | 479 ++++++++++++ uv.lock | 177 ++--- 24 files changed, 2750 insertions(+), 803 deletions(-) create mode 100644 tilebox-grpc/tests/test_fork_logging.py create mode 100644 tilebox-workflows/tests/automations/test_storage_read.py create mode 100644 tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py create mode 100644 tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py create mode 100644 tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi create mode 100644 tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b06fb..677a51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support Python 3.14 across all packages. - Test the lowest compatible direct dependencies on Python 3.11 in CI. +- `tilebox-workflows`: Read Azure blobs in storage-event tasks. Use the account and container returned by the API. + Authenticate with Azure Identity, or use an account key or SAS token configured in the environment. +- `tilebox-workflows`: Add `AzureBlobCache(account_name, container, prefix="jobs")` with the same authentication + as Azure automation reads. Cache groups share the store and credentials. ### Changed +- `tilebox-workflows`: List storage locations through `StorageLocationService`. Read S3, GCS, and Azure objects + through obstore, with boto3, Google Auth, and Azure Identity handling credentials. Cloud reads now raise + obstore/Python exceptions instead of AWS/GCP SDK exceptions. Missing objects raise `FileNotFoundError`. +- `tilebox-workflows`: Replace the Google storage SDK dependency with `google-auth[requests]` and add `azure-identity`. + Import cloud authentication libraries only when needed and reuse cloud storage clients between automation reads. +- `tilebox-workflows`: Use obstore and Google Auth for `GoogleStorageCache`. Pass a bucket name instead of a + Google SDK bucket object. Cache groups share the same store and credentials. +- `tilebox-workflows`: Use obstore for `AmazonS3Cache`, with the same boto3 session credentials as automation reads. + Cache groups share the store and credentials instead of creating new S3 clients. - Require Python 3.11 or newer across all packages, removing Python 3.10 compatibility code and typing backfills. - `tilebox-storage`: Replace legacy synchronous client patching with explicit wrappers using `asyncio.run()`. When called inside a running event loop (including notebooks), run the operation in a worker thread instead. Remove the internal `syncify` helper and the `nest-asyncio2` dependency from `tilebox-grpc`. - Raise dependency minimums to remove obsolete compatibility workarounds: boto3 1.40.2, OpenTelemetry 1.43.0 - (logging instrumentation 0.64b0), grpcio 1.84.0, and pyqwest 0.7.0. + (logging instrumentation 0.64b0), and pyqwest 0.7.0. - `tilebox-datasets`: Require NumPy 1.25, pandas 2.2.2, xarray 2024.6, and Shapely 2.0.6 or newer. - Require protobuf 6.31.0 for repeated-field descriptors and protobuf-py 0.2.0 for integer-to-float serialization. - `tilebox-workflows`: Make interactive notebook progress an optional `notebook` extra; without it, jobs display @@ -29,6 +42,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `tilebox-workflows`: Use S3 bucket regions from API metadata or discover them with `HeadBucket`. + `AmazonS3Cache` also accepts `region=` to skip discovery. Let botocore refresh temporary credentials. +- `tilebox-workflows`: Preserve cloud cache permission, network, and backend errors instead of reporting cache misses. +- Require `grpcio>=1.84.0` for the memory-exhaustion fixes in + [GHSA-hf3w-6hpw-qp67](https://github.com/grpc/grpc/security/advisories/GHSA-hf3w-6hpw-qp67). + macOS CLI authentication may still emit native fork diagnostics during successful reads. +- `tilebox-workflows`: Fix GCS automation reads when the API returns a bucket name without a project prefix. - Explicitly trust system certificates for Connect HTTP/1 transports with pyqwest 0.7 and newer. ## [0.62.0] - 2026-09-16 diff --git a/tilebox-grpc/_tilebox/grpc/replay.py b/tilebox-grpc/_tilebox/grpc/replay.py index 1417162..149306e 100644 --- a/tilebox-grpc/_tilebox/grpc/replay.py +++ b/tilebox-grpc/_tilebox/grpc/replay.py @@ -24,6 +24,7 @@ from grpc.aio import ( AioRpcError, ClientCallDetails, # import from aio, since grpc.ClientCallDetails is an empty base class + Metadata, ) RequestType = TypeVar("RequestType") @@ -163,7 +164,7 @@ def unary_unary_call( if recorded_status != StatusCode.OK.value[0]: # the recorded call was an error, so raise it again code = _STATUS_CODES[recorded_status] - error = AioRpcError(code, None, None, recorded_response.decode()) + error = AioRpcError(code, Metadata(), Metadata(), recorded_response.decode()) raise error return response_deserializer(base64.b64decode(recorded_response)) diff --git a/tilebox-grpc/pyproject.toml b/tilebox-grpc/pyproject.toml index 674703c..c26142b 100644 --- a/tilebox-grpc/pyproject.toml +++ b/tilebox-grpc/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - # Rate-limits the logging noise from https://github.com/grpc/grpc/issues/42293. + # Includes fixes for https://github.com/grpc/grpc/security/advisories/GHSA-hf3w-6hpw-qp67. "grpcio>=1.84.0", # connectrpc 0.11 switches from Google's protobuf runtime to protobuf-py "connectrpc>=0.10.1,<0.11.0", diff --git a/tilebox-grpc/tests/test_fork_logging.py b/tilebox-grpc/tests/test_fork_logging.py new file mode 100644 index 0000000..ebd9475 --- /dev/null +++ b/tilebox-grpc/tests/test_fork_logging.py @@ -0,0 +1,73 @@ +"""Check native gRPC diagnostics when a CLI starts during active RPCs. + +Run on macOS to check RPCs and report the known fork-log issue: + +uv run --isolated --no-project --with grpcio==1.84.0 --with pytest pytest -c /dev/null -p no:cacheprovider -rx tilebox-grpc/tests/test_fork_logging.py + +RPC failures fail the test. Known fork diagnostics produce XFAIL after the RPC checks pass. +Linux may use vfork and not reproduce the diagnostics. This check uses no cloud credentials. +""" + +import os +import subprocess +import sys +from textwrap import dedent + +import pytest + + +# RPCs must survive CLI subprocess launches; known fork-log noise is reported separately. +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not use POSIX fork handlers") +def test_cli_spawn_during_rpcs() -> None: + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-c", + dedent(""" + import subprocess + import sys + from concurrent.futures import ThreadPoolExecutor + from threading import Event + + import grpc + + with ThreadPoolExecutor(max_workers=4) as executor: + server = grpc.server(executor) + server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler( + "probe", {"echo": grpc.unary_unary_rpc_method_handler(lambda request, context: request)} + ),)) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + stop = Event() + try: + with grpc.insecure_channel(f"127.0.0.1:{port}") as channel: + echo = channel.unary_unary("/probe/echo") + assert echo(b"before", timeout=5) == b"before" + + def poll(): + while not stop.is_set(): + assert echo(b"during", timeout=5) == b"during" + + poller = executor.submit(poll) + try: + for _ in range(25): + # Azure CLI authentication supplies cwd, which prevents posix_spawn. + subprocess.run([sys.executable, "-c", "pass"], cwd=".", check=True, timeout=5) + finally: + stop.set() + poller.result(timeout=10) + assert echo(b"after", timeout=5) == b"after" + finally: + stop.set() + server.stop(0).wait(10) + """), + ], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "GRPC_VERBOSITY": "INFO"}, + check=False, + ) + assert result.returncode == 0, result.stderr + if "FD from fork parent still in poll list" in result.stderr: + pytest.xfail("Known gRPC fork diagnostics: https://github.com/grpc/grpc/issues/42293") diff --git a/tilebox-workflows/README.md b/tilebox-workflows/README.md index 6d079d7..1db0107 100644 --- a/tilebox-workflows/README.md +++ b/tilebox-workflows/README.md @@ -90,6 +90,19 @@ unless the cache implementation explicitly provides that guarantee. Concurrency in one runtime avoids repeated process initialization and allows overlapping I/O or native code that releases Python's GIL. CPU-bound Python code still needs multiple runtime processes for parallel execution. +## Reading automation objects + +Storage-event tasks can read objects from Amazon S3, Google Cloud Storage, Azure Blob Storage, or the local filesystem: + +```python +content = self.trigger.storage.read(self.trigger.location) +``` + +The method returns bytes. Cloud reads use the credentials configured for the runner. + +On macOS, GCS or Azure CLI authentication can emit [gRPC fork diagnostics](https://github.com/grpc/grpc/issues/42293) +even when a read succeeds. These messages alone do not indicate a failed read; do not downgrade gRPC to hide them. + ## Documentation Check out the [Tilebox Workflows documentation](https://docs.tilebox.com/workflows/introduction) for more information. diff --git a/tilebox-workflows/pyproject.toml b/tilebox-workflows/pyproject.toml index 98258dd..19d436a 100644 --- a/tilebox-workflows/pyproject.toml +++ b/tilebox-workflows/pyproject.toml @@ -23,7 +23,10 @@ requires-python = ">=3.11" dependencies = [ "tilebox-datasets>=0.56.0", "tilebox-grpc>=0.28.0", - "google-cloud-storage>=2.10", + # Require the security fixes even with an older tilebox-grpc release. + "grpcio>=1.84.0", + "google-auth[requests]>=2.29", + "azure-identity>=1.23", "opentelemetry-api>=1.43.0", "opentelemetry-exporter-otlp-proto-http>=1.43.0", "opentelemetry-sdk>=1.43.0", diff --git a/tilebox-workflows/tests/automations/test_storage_read.py b/tilebox-workflows/tests/automations/test_storage_read.py new file mode 100644 index 0000000..76fff6d --- /dev/null +++ b/tilebox-workflows/tests/automations/test_storage_read.py @@ -0,0 +1,526 @@ +import json +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from unittest.mock import MagicMock, patch +from uuid import UUID + +import boto3 +import obstore +import pytest +from azure.core.credentials import AccessToken +from botocore.credentials import RefreshableCredentials +from botocore.exceptions import ClientError +from botocore.stub import Stubber +from google.auth.exceptions import DefaultCredentialsError +from obstore.store import AzureStore, MemoryStore, S3Store + +from tilebox.datasets.uuid import uuid_to_uuid_message +from tilebox.workflows.automations import StorageEventTask +from tilebox.workflows.cache import AmazonS3Cache +from tilebox.workflows.client import Client +from tilebox.workflows.data import ( + AzureStorageLocation, + RunnerContext, + S3StorageLocation, + StorageLocation, + StorageType, + _default_azure_storage_client, + _default_gcs_storage_client, + _default_s3_storage_client, +) +from tilebox.workflows.observability.tracing import NoopWorkflowTracer +from tilebox.workflows.workflows.v1 import storage_location_pb2 as storage_pb + +_ID = UUID("01a0fd71-8262-2fde-7283-43dab469c23f") +_ACCOUNT = "/subscriptions/test/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/sourceaccount" + + +@pytest.fixture(autouse=True) +def isolated_storage_clients() -> Iterator[None]: + clients = (_default_gcs_storage_client, _default_s3_storage_client, _default_azure_storage_client) + for client in clients: + client.cache_clear() + yield + for client in clients: + client.cache_clear() + + +@pytest.mark.parametrize(("method", "store_class"), [("gcs_client", "GCSStore"), ("s3_client", "S3Store")]) +def test_cloud_client_cache(method: str, store_class: str, monkeypatch: pytest.MonkeyPatch) -> None: + # GCS and S3 reuse a store for the same bucket and keep different buckets separate. + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + with ( + patch("google.auth.default", return_value=(MagicMock(), "project")), + patch(f"obstore.store.{store_class}", side_effect=[MemoryStore(), MemoryStore()]) as constructor, + ): + client = getattr(RunnerContext(), method) + if method == "s3_client": + client = partial(client, region="us-east-1") + first = client("first-bucket") + assert client("first-bucket") is first + assert client("other-bucket") is not first + assert constructor.call_count == 2 + + +def test_gcs_adc_project_lookup_is_not_repeated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Repeated reads reuse ADC credentials instead of launching gcloud to rediscover the project. + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + monkeypatch.setenv("CLOUDSDK_CONFIG", str(tmp_path)) + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "explicit-project") + (tmp_path / "application_default_credentials.json").write_text( + json.dumps( + {"type": "authorized_user", "client_id": "test-client", "client_secret": "test", "refresh_token": "test"} + ) + ) + store = MemoryStore() + obstore.put(store, "first.txt", b"first") + obstore.put(store, "second.txt", b"second") + location = StorageLocation(_ID, "adc-bucket", StorageType.GCS) + with ( + patch("subprocess.check_output", return_value=b"discovered-project\n") as command, + patch("obstore.store.GCSStore", return_value=store) as constructor, + ): + assert location.read("first.txt") == b"first" + assert location.read("second.txt") == b"second" + command.assert_called_once() + assert command.call_args.args[0] == ("gcloud", "config", "get", "project") + constructor.assert_called_once() + + +def test_gcs_discovery_error_is_not_hidden_or_cached() -> None: + # Credential discovery failures reach the caller and a later read can retry discovery. + store = MemoryStore() + obstore.put(store, "file.txt", b"retried") + location = StorageLocation(_ID, "retry-bucket", StorageType.GCS) + with ( + patch("google.auth.default", side_effect=[DefaultCredentialsError("missing credentials"), (MagicMock(), None)]), + patch("obstore.store.GCSStore", return_value=store), + ): + with pytest.raises(DefaultCredentialsError, match="missing credentials"): + location.read("file.txt") + assert location.read("file.txt") == b"retried" + + +def _azure_message() -> storage_pb.StorageLocation: + return storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), + location="container-only", + type=storage_pb.STORAGE_TYPE_AZURE_BLOB, + reference=storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AZURE_BLOB, + azure_blob=storage_pb.AzureBlobReference(storage_account_resource_id=_ACCOUNT, container="sourcecontainer"), + ), + ) + + +@pytest.mark.parametrize("storage_type", [StorageType.GCS, StorageType.S3, StorageType.FS]) +def test_mixed_storage_location_ordering(storage_type: StorageType) -> None: + # Azure and other storage locations sort together by ID before their bucket or container names. + plain = StorageLocation(UUID(int=1), "z", storage_type) + azure = AzureStorageLocation(UUID(int=2), "a", StorageType.AZURE, storage_account_resource_id=_ACCOUNT) + assert plain < azure + assert azure > plain + assert sorted([azure, plain]) == [plain, azure] + + +def test_azure_trigger_read() -> None: + # Deserialized Azure tasks read from the account and container supplied by the runner context. + location = StorageLocation.from_message(_azure_message()) + assert isinstance(location, AzureStorageLocation) + assert location.type == StorageType.AZURE + assert location.location == "sourcecontainer" + assert location.storage_account_resource_id == _ACCOUNT + assert StorageLocation.from_message(location.to_message()) == location + + store = MemoryStore() + key = "nested/hello + %20 ü.txt" + expected = "Hello Azure! ü\n".encode() + obstore.put(store, key, expected) + context = RunnerContext(storage_locations=[location]) + task = StorageEventTask().once(location, key) + restored = StorageEventTask._deserialize(task._serialize(), context) + assert isinstance(restored.trigger.storage, AzureStorageLocation) + assert restored.trigger.storage.storage_account_resource_id == _ACCOUNT + assert restored.trigger.storage.runner_context is context + with patch.object(context, "azure_client", return_value=store) as client: + assert restored.trigger.storage.read(restored.trigger.location) == expected + client.assert_called_once_with(_ACCOUNT, "sourcecontainer") + + +def test_azure_client_coordinates_override_environment(monkeypatch: pytest.MonkeyPatch) -> None: + # The account from API metadata takes precedence over the account configured in the environment. + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "wrongaccount") + store = RunnerContext().azure_client(_ACCOUNT, "explicit-container") + assert store.config["account_name"] == "sourceaccount" + assert store.config["container_name"] == "explicit-container" + + +def test_azure_client_cache_separates_accounts_and_containers() -> None: + # Azure clients are reused only when both the account and container match. + context = RunnerContext() + first = context.azure_client(_ACCOUNT, "one") + assert context.azure_client(_ACCOUNT, "one") is first + assert context.azure_client(_ACCOUNT, "two") is not first + other = context.azure_client(_ACCOUNT.replace("sourceaccount", "otheraccount"), "one") + assert other is not first + assert other.config["account_name"] == "otheraccount" + + +def test_azure_identity_credentials() -> None: + # Azure Identity supplies a storage-scoped token and its expiry to obstore. + with ( + patch("azure.identity.DefaultAzureCredential") as credential, + patch("obstore.store.AzureStore", wraps=AzureStore) as constructor, + ): + credential.return_value.get_token.return_value = AccessToken("test-token", 2000000000) + RunnerContext().azure_client(_ACCOUNT, "identity-test") + provider = constructor.call_args.kwargs["credential_provider"] + assert provider() == {"token": "test-token", "expires_at": datetime.fromtimestamp(2000000000, UTC)} + credential.return_value.get_token.assert_called_once_with("https://storage.azure.com/.default", tenant_id=None) + credential.assert_called_once_with() + + +@pytest.mark.parametrize( + ("variable", "value"), + [ + ("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdA=="), + ("AZURE_STORAGE_ACCESS_KEY", "dGVzdA=="), + ("AZURE_STORAGE_MASTER_KEY", "dGVzdA=="), + ("AZURE_STORAGE_SAS_KEY", "sv=1&sig=test"), + ("AZURE_STORAGE_SAS_TOKEN", "sv=1&sig=test"), + ], +) +def test_azure_explicit_credentials(variable: str, value: str, monkeypatch: pytest.MonkeyPatch) -> None: + # Explicit keys and SAS tokens use native obstore authentication without constructing Azure Identity. + monkeypatch.setenv(variable, value) + with ( + patch("azure.identity.DefaultAzureCredential") as credential, + patch("obstore.store.AzureStore", wraps=AzureStore) as constructor, + ): + container = variable.lower().replace("_", "-") + RunnerContext().azure_client(_ACCOUNT, container) + constructor.assert_called_once_with(container, account_name="sourceaccount") + credential.assert_not_called() + + +def test_azure_read_requires_account_metadata() -> None: + # Missing or malformed Azure account resource IDs raise a clear error before a cloud request. + location = StorageLocation(_ID, "container-only", StorageType.AZURE) + with pytest.raises(ValueError, match="missing its storage account resource ID"): + location.read("file.txt") + with pytest.raises(ValueError, match="Invalid Azure storage account resource ID"): + RunnerContext().azure_client("https://sourceaccount.blob.core.windows.net", "container") + + +@pytest.mark.parametrize("metadata", ["bare", "reference"]) +def test_gcs_trigger_read(metadata: str) -> None: + # GCS tasks read bare bucket names using scoped google-auth credentials from either API metadata form. + bucket = "storage-automation-test-gcp-2aa460e" + message = storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), location=bucket, type=storage_pb.STORAGE_TYPE_GCS_BUCKET + ) + if metadata == "reference": + message.reference.type = storage_pb.STORAGE_TYPE_GCS_BUCKET + message.reference.gcs_bucket.project_id = "test-project" + message.reference.gcs_bucket.bucket = bucket + location = StorageLocation.from_message(message) + context = RunnerContext(storage_locations=[location]) + key = "nested/hello + %20 ü.txt" + expected = b"GCS blob contents\n" + store = MemoryStore() + obstore.put(store, key, expected) + task = StorageEventTask().once(location, key) + restored = StorageEventTask._deserialize(task._serialize(), context) + credentials = MagicMock() + with ( + patch("google.auth.default", return_value=(credentials, "test-project")) as default_credentials, + patch("obstore.store.GCSStore", return_value=store) as constructor, + ): + assert restored.trigger.storage.read(restored.trigger.location) == expected + default_credentials.assert_called_once_with(scopes=["https://www.googleapis.com/auth/devstorage.read_only"]) + assert restored.trigger.storage.read(restored.trigger.location) == expected + constructor.assert_called_once() + assert constructor.call_args.args == (bucket,) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider.credentials is credentials + credentials.token = "test-token" # noqa: S105 + credentials.expiry = None + assert provider()["token"] == "test-token" # noqa: S105 + credentials.refresh.assert_called_once_with(provider.request) + + +def test_s3_read_preserves_aws_profile(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # S3 reads use credentials from the selected AWS profile rather than the default profile. + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + credentials = tmp_path / "credentials" + credentials.write_text( + "[default]\naws_access_key_id = default-key\naws_secret_access_key = default-secret\n" + "[automation]\naws_access_key_id = profile-key\naws_secret_access_key = profile-secret\n" + ) + config = tmp_path / "config" + config.write_text("[profile automation]\nregion = eu-west-2\n") + for name in ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_SECURITY_TOKEN", + "AWS_DEFAULT_PROFILE", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(credentials)) + monkeypatch.setenv("AWS_CONFIG_FILE", str(config)) + monkeypatch.setenv("AWS_PROFILE", "automation") + store = MemoryStore() + key = "nested/aws.txt" + expected = b"S3 blob contents\n" + obstore.put(store, key, expected) + with patch("obstore.store.S3Store", return_value=store) as constructor: + assert S3StorageLocation(_ID, "aws-bucket", StorageType.S3, region="eu-west-2").read(key) == expected + assert constructor.call_args.args == ("aws-bucket",) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider()["access_key_id"] == "profile-key" + + +def test_s3_read_preserves_default_session(monkeypatch: pytest.MonkeyPatch) -> None: + # S3 reads retain credentials and region configured with boto3.setup_default_session(). + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "environment-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "environment-secret") + boto3.setup_default_session( + aws_access_key_id="configured-key", + aws_secret_access_key="configured-secret", # noqa: S106 + region_name="eu-west-2", + ) + store = MemoryStore() + obstore.put(store, "file.txt", b"configured session") + with patch("obstore.store.S3Store", return_value=store) as constructor: + assert ( + S3StorageLocation(_ID, "aws-bucket", StorageType.S3, region="eu-west-2").read("file.txt") + == b"configured session" + ) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider()["access_key_id"] == "configured-key" + assert provider.config["region"] == "eu-west-2" + + +def test_s3_reference_region_and_store_cache(monkeypatch: pytest.MonkeyPatch) -> None: + # The API bucket region survives task serialization and overrides the session region in cached stores. + session = MagicMock(region_name="us-east-1") + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + message = storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), + reference=storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AWS_S3_BUCKET, + aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket="source-bucket", region="eu-west-2"), + ), + ) + location = StorageLocation.from_message(message) + assert isinstance(location, S3StorageLocation) + assert StorageLocation.from_message(location.to_message()) == location + context = RunnerContext(storage_locations=[location]) + task = StorageEventTask().once(location, "file.txt") + restored = StorageEventTask._deserialize(task._serialize(), context) + store = context.s3_client("source-bucket", region="eu-west-2") + assert store.config["region"] == "eu-west-2" + assert context.s3_client("source-bucket", region="eu-west-2") is store + assert context.s3_client("source-bucket", region="ap-southeast-2") is not store + session.client.assert_not_called() + memory = MemoryStore() + obstore.put(memory, "file.txt", b"regional object") + with patch.object(context, "s3_client", return_value=memory) as client: + assert restored.trigger.storage.read(restored.trigger.location) == b"regional object" + client.assert_called_once_with("source-bucket", region="eu-west-2") + + +@pytest.mark.parametrize("forbidden", [False, True]) +@pytest.mark.parametrize("cache", [False, True]) +def test_s3_region_discovery(forbidden: bool, cache: bool, monkeypatch: pytest.MonkeyPatch) -> None: + # Bucket-only stores discover the region once, including when HeadBucket denies listing permission. + client = boto3.Session( + aws_access_key_id="test-key", + aws_secret_access_key="test-secret", # noqa: S106 + region_name="us-east-1", + ).client("s3") + session = MagicMock(region_name=None) + session.client.return_value = client + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + metadata = {"HTTPHeaders": {"x-amz-bucket-region": "eu-west-2"}} + with Stubber(client) as stub: + if forbidden: + stub.add_client_error( + "head_bucket", + "403", + http_status_code=403, + response_meta=metadata, + expected_params={"Bucket": "source-bucket"}, + ) + else: + stub.add_response("head_bucket", {"ResponseMetadata": metadata}, {"Bucket": "source-bucket"}) + if cache: + root = AmazonS3Cache("source-bucket") + store = root.store + assert root.group("nested").store is store + else: + store = RunnerContext().s3_client("source-bucket") + assert RunnerContext().s3_client("source-bucket") is store + stub.assert_no_pending_responses() + assert store.config["region"] == "eu-west-2" + session.client.assert_called_once_with("s3") + + +def test_s3_region_discovery_error(monkeypatch: pytest.MonkeyPatch) -> None: + # Discovery errors without a bucket-region header propagate instead of using the session region. + session = MagicMock(region_name="us-east-1") + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + error = ClientError({"Error": {"Code": "AccessDenied"}}, "HeadBucket") + session.client.return_value.head_bucket.side_effect = error + with pytest.raises(ClientError) as raised: + RunnerContext().s3_client("source-bucket") + assert raised.value is error + + +@pytest.mark.parametrize("cache", [False, True]) +def test_s3_temporary_credentials_refresh(cache: bool, monkeypatch: pytest.MonkeyPatch) -> None: + # Real signed requests use refreshed credentials after a short session expires, without recreating the store. + now = datetime.now(UTC) + refresh = MagicMock( + return_value={ + "access_key": "refreshed-key", + "secret_key": "secret", + "token": "new-token", + "expiry_time": (now + timedelta(hours=1)).isoformat(), + } + ) + credentials = RefreshableCredentials( + "initial-key", + "secret", + "initial-token", + now + timedelta(minutes=16), + refresh_using=refresh, + method="test", + time_fetcher=lambda: now, + ) + session = MagicMock(region_name="us-east-1") + session.get_credentials.return_value = credentials + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + authorizations: list[str] = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + authorizations.append(self.headers["Authorization"]) + self.send_response(200) + self.send_header("Content-Length", "4") + self.end_headers() + self.wfile.write(b"data") + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = Thread(target=server.serve_forever) + thread.start() + try: + constructor = partial(S3Store, endpoint=f"http://127.0.0.1:{server.server_port}", allow_http=True) + with patch("obstore.store.S3Store", side_effect=constructor): + store = ( + AmazonS3Cache("bucket", region="eu-west-2").store + if cache + else RunnerContext().s3_client("bucket", "eu-west-2") + ) + assert obstore.get(store, "key").bytes() == b"data" + refresh.assert_not_called() + now += timedelta(minutes=17) + assert obstore.get(store, "key").bytes() == b"data" + refresh.assert_called_once() + finally: + server.shutdown() + thread.join() + assert len(authorizations) == 2 + assert "Credential=initial-key/" in authorizations[0] + assert "Credential=refreshed-key/" in authorizations[1] + assert all("/eu-west-2/s3/" in authorization for authorization in authorizations) + + +def test_filesystem_read(tmp_path: Path) -> None: + # Filesystem reads return the contents of the file under the configured root directory. + key = "file.txt" + expected = b"existing provider content" + storage = StorageLocation(_ID, str(tmp_path), StorageType.FS) + (tmp_path / key).write_bytes(expected) + assert storage.read(key) == expected + + +@pytest.mark.parametrize( + ("reference", "expected_type", "expected_location"), + [ + ( + storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_GCS_BUCKET, + gcs_bucket=storage_pb.GCSBucketReference(project_id="project", bucket="bucket"), + ), + StorageType.GCS, + "bucket", + ), + ( + storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AWS_S3_BUCKET, + aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket="bucket"), + ), + StorageType.S3, + "bucket", + ), + ( + storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_FILESYSTEM, filesystem=storage_pb.FilesystemReference(path="/data") + ), + StorageType.FS, + "/data", + ), + ], +) +def test_reference_coordinates( + reference: storage_pb.StorageLocationReference, expected_type: StorageType, expected_location: str +) -> None: + # Structured GCS, S3, and filesystem references map to the expected storage type and location. + storage = StorageLocation.from_message( + storage_pb.StorageLocation(id=uuid_to_uuid_message(_ID), reference=reference) + ) + assert storage.type == expected_type + assert storage.location == expected_location + + +def test_grpc_storage_location_service_route() -> None: + # gRPC listing uses StorageLocationService and retains Azure account metadata. + channel = MagicMock() + channel.unary_unary.return_value.return_value = storage_pb.StorageLocations(locations=[_azure_message()]) + with ( + patch("tilebox.workflows.client.open_channel", return_value=channel), + patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), + ): + locations = Client(token="test-key").automations().storage_locations() # noqa: S106 + assert isinstance(locations[0], AzureStorageLocation) + assert locations[0].storage_account_resource_id == _ACCOUNT + assert "/workflows.v1.StorageLocationService/ListStorageLocations" in [ + call.args[0] for call in channel.unary_unary.call_args_list + ] + + +def test_http1_storage_location_service_route() -> None: + # HTTP/1.1 listing uses the Connect storage-location client and retains Azure account metadata. + with ( + patch( + "tilebox.workflows.workflows.v1.storage_location_connect.StorageLocationServiceClientSync.list_storage_locations", + return_value=storage_pb.StorageLocations(locations=[_azure_message()]), + ) as list_locations, + patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), + ): + locations = Client(token="test-key", transport="http1").automations().storage_locations() # noqa: S106 + assert isinstance(locations[0], AzureStorageLocation) + assert locations[0].storage_account_resource_id == _ACCOUNT + list_locations.assert_called_once() diff --git a/tilebox-workflows/tests/runner/test_runner.py b/tilebox-workflows/tests/runner/test_runner.py index f697aea..12460b2 100644 --- a/tilebox-workflows/tests/runner/test_runner.py +++ b/tilebox-workflows/tests/runner/test_runner.py @@ -20,6 +20,12 @@ from tilebox.workflows.runner.task_runner import TaskRunner +@pytest.fixture(autouse=True) +def offline_logging(monkeypatch: pytest.MonkeyPatch) -> None: + # Offline runner tests must not start background network log exporters. + monkeypatch.setattr("tilebox.workflows.client.initialize_logging", MagicMock()) + + def test_public_execution_context_attributes() -> None: class RenameTask(Task): def execute(self, context: ExecutionContext) -> None: @@ -40,7 +46,7 @@ def test_task_authoring_imports_are_lazy() -> None: code = ( "import sys\n" "import tilebox.workflows as workflows\n" - "heavy = {'pandas', 'xarray', 'boto3', 'google.cloud.storage', 'ipywidgets', 'opentelemetry.sdk'}\n" + "heavy = {'pandas', 'xarray', 'boto3', 'google.cloud.storage', 'google.auth', 'azure.identity', 'obstore', 'ipywidgets', 'opentelemetry.sdk'}\n" "assert not heavy & sys.modules.keys()\n" "assert set(workflows.__all__) <= set(dir(workflows))\n" "task = workflows.Task\n" @@ -195,7 +201,10 @@ def replay_client(replay_file: str, assert_request_matches: bool = True) -> Clie replay = Path(__file__).parent / "testdata" / "recordings" / replay_file replay_channel = open_replay_channel(replay, assert_request_matches) - with patch("tilebox.workflows.client.open_channel") as open_channel_mock: + with ( + patch("tilebox.workflows.client.open_channel") as open_channel_mock, + patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), + ): open_channel_mock.return_value = replay_channel # url/token doesn't matter since its a mocked channel client = Client( diff --git a/tilebox-workflows/tests/tasks_data.py b/tilebox-workflows/tests/tasks_data.py index 381569e..75f1092 100644 --- a/tilebox-workflows/tests/tasks_data.py +++ b/tilebox-workflows/tests/tasks_data.py @@ -30,6 +30,7 @@ from tilebox.workflows.data import ( Artifact, AutomationPrototype, + AzureStorageLocation, Cluster, ComputedTask, CronTrigger, @@ -331,6 +332,14 @@ def storage_locations(draw: DrawFn) -> StorageLocation: location = draw(alphanumerical_text(min_size=4, max_size=30)) storage_type = draw(sampled_from(StorageType)) + if storage_type == StorageType.AZURE: + account = draw(alphanumerical_text(min_size=3, max_size=24)) + account_resource_id = ( + f"/subscriptions/example/resourceGroups/storage/providers/Microsoft.Storage/storageAccounts/{account}" + ) + return AzureStorageLocation( + storage_location_id, location, storage_type, storage_account_resource_id=account_resource_id + ) return StorageLocation(storage_location_id, location, storage_type) diff --git a/tilebox-workflows/tests/test_cache.py b/tilebox-workflows/tests/test_cache.py index 28026ac..80ee621 100644 --- a/tilebox-workflows/tests/test_cache.py +++ b/tilebox-workflows/tests/test_cache.py @@ -1,59 +1,215 @@ -import os -from collections.abc import Iterator +from datetime import timedelta from pathlib import Path +from unittest.mock import MagicMock, patch import boto3 import pytest from _pytest.fixtures import SubRequest -from moto import mock_aws -from mypy_boto3_s3 import S3Client +from obstore.exceptions import GenericError from obstore.store import LocalStore, MemoryStore -from tilebox.workflows.cache import AmazonS3Cache, InMemoryCache, JobCache, LocalFileSystemCache, ObstoreCache +from tilebox.workflows.cache import ( + AmazonS3Cache, + AzureBlobCache, + GoogleStorageCache, + InMemoryCache, + JobCache, + LocalFileSystemCache, + ObstoreCache, +) @pytest.fixture -def _aws_credentials() -> None: - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" # noqa: S105 - os.environ["AWS_SECURITY_TOKEN"] = "testing" # noqa: S105 - os.environ["AWS_SESSION_TOKEN"] = "testing" # noqa: S105 - os.environ["AWS_DEFAULT_REGION"] = "us-east-1" +def _aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + """Use test credentials without contacting AWS metadata services.""" + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +@pytest.fixture(autouse=True) +def _azure_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ACCESS_KEY", + "AZURE_STORAGE_MASTER_KEY", + "AZURE_STORAGE_SAS_KEY", + "AZURE_STORAGE_SAS_TOKEN", + ): + monkeypatch.delenv(name, raising=False) + + +caches = [ + "LocalFileSystem", + "InMemory", + "AmazonS3", + "AmazonS3_no_prefix", + "AzureBlob", + "AzureBlob_no_prefix", + "ObstoreMemory", + "ObstoreLocal", + "GoogleStorage", +] @pytest.fixture -def aws(_aws_credentials: None) -> Iterator[S3Client]: - with mock_aws(): - yield boto3.client("s3", region_name="us-east-1") - - -caches = ["LocalFileSystem", "InMemory", "AmazonS3", "AmazonS3_no_prefix", "ObstoreMemory", "ObstoreLocal"] - - -@pytest.fixture -def cache(request: SubRequest, tmp_path: Path, aws: S3Client) -> JobCache: +def cache(request: SubRequest, tmp_path: Path, _aws_credentials: None) -> JobCache: # noqa: PLR0911 match request.param: case "LocalFileSystem": return LocalFileSystemCache(tmp_path) case "InMemory": return InMemoryCache() - case "AmazonS3": - bucket = "bucket1" - aws.create_bucket(Bucket=bucket) - return AmazonS3Cache(bucket, prefix="test") - case "AmazonS3_no_prefix": - bucket = "bucket1" - aws.create_bucket(Bucket=bucket) - return AmazonS3Cache(bucket, prefix="") + case "AmazonS3" | "AmazonS3_no_prefix": + with patch("obstore.store.S3Store", return_value=MemoryStore()): + return AmazonS3Cache( + "bucket1", prefix="test" if request.param == "AmazonS3" else "", region="us-east-1" + ) + case "AzureBlob" | "AzureBlob_no_prefix": + with ( + patch("azure.identity.DefaultAzureCredential"), + patch("obstore.store.AzureStore", return_value=MemoryStore()), + ): + return AzureBlobCache("account", "container", prefix="test" if request.param == "AzureBlob" else "") case "ObstoreMemory": return ObstoreCache(MemoryStore()) case "ObstoreLocal": return ObstoreCache(LocalStore(tmp_path)) + case "GoogleStorage": + with ( + patch("google.auth.default", return_value=(MagicMock(), "project")), + patch("obstore.store.GCSStore", return_value=MemoryStore()), + ): + return GoogleStorageCache("test-bucket") case _: raise ValueError("Invalid cache type") +def test_azure_cache_credentials_and_groups(monkeypatch: pytest.MonkeyPatch) -> None: + # Azure cache groups share one store and credential provider without listing sibling prefixes. + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "wrongaccount") + store = MemoryStore() + with ( + patch("obstore.auth.azure.AzureCredentialProvider") as provider, + patch("obstore.store.AzureStore", return_value=store) as constructor, + ): + cache = AzureBlobCache("cacheaccount", "cache-container") + first = cache.group("folder") + first["one"] = b"first" + cache.group("folder2")["two"] = b"second" + assert list(first) == ["one"] + assert first["one"] == b"first" + assert isinstance(first, ObstoreCache) + assert first.store is store + assert store.get("jobs/folder/one").bytes() == b"first" + constructor.assert_called_once_with( + "cache-container", account_name="cacheaccount", credential_provider=provider.return_value + ) + provider.assert_called_once_with() + + +@pytest.mark.parametrize( + ("variable", "value"), + [ + ("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdA=="), + ("AZURE_STORAGE_ACCESS_KEY", "dGVzdA=="), + ("AZURE_STORAGE_MASTER_KEY", "dGVzdA=="), + ("AZURE_STORAGE_SAS_KEY", "sv=1&sig=test"), + ("AZURE_STORAGE_SAS_TOKEN", "sv=1&sig=test"), + ], +) +def test_azure_cache_explicit_credentials(variable: str, value: str, monkeypatch: pytest.MonkeyPatch) -> None: + # Account keys and SAS tokens bypass Azure Identity when constructing the cache store. + monkeypatch.setenv(variable, value) + with ( + patch("azure.identity.DefaultAzureCredential") as credential, + patch("obstore.store.AzureStore", return_value=MemoryStore()) as constructor, + ): + AzureBlobCache("cacheaccount", "cache-container") + constructor.assert_called_once_with("cache-container", account_name="cacheaccount") + credential.assert_not_called() + + +def test_s3_cache_credentials_and_groups(monkeypatch: pytest.MonkeyPatch) -> None: + # S3 caches use session credentials and the bucket region, sharing one store across groups. + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "environment-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "environment-secret") + boto3.setup_default_session( + aws_access_key_id="configured-key", + aws_secret_access_key="configured-secret", # noqa: S106 + region_name="eu-west-2", + ) + store = MemoryStore() + with patch("obstore.store.S3Store", return_value=store) as constructor: + cache = AmazonS3Cache("cache-bucket", region="ap-southeast-2") + first = cache.group("folder") + first["one"] = b"first" + cache.group("folder2")["two"] = b"second" + assert list(first) == ["one"] + assert first["one"] == b"first" + assert isinstance(first, ObstoreCache) + assert first.store is store + assert store.get("jobs/folder/one").bytes() == b"first" + assert constructor.call_count == 1 + assert constructor.call_args.args == ("cache-bucket",) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider()["access_key_id"] == "configured-key" + assert provider.config["region"] == "eu-west-2" + assert constructor.call_args.kwargs["region"] == "ap-southeast-2" + assert provider.ttl == timedelta(0) + + +def test_google_storage_credentials_and_groups() -> None: + # GCS caches request write access and share credentials across groups without listing sibling prefixes. + credentials = MagicMock() + store = MemoryStore() + with ( + patch("google.auth.default", return_value=(credentials, "project")) as default, + patch("obstore.store.GCSStore", return_value=store) as constructor, + ): + cache = GoogleStorageCache("cache-bucket") + first = cache.group("folder") + first["one"] = b"first" + cache.group("folder2")["two"] = b"second" + assert list(first) == ["one"] + assert first["one"] == b"first" + assert isinstance(first, ObstoreCache) + assert first.store is store + assert store.get("jobs/folder/one").bytes() == b"first" + default.assert_called_once_with(scopes=["https://www.googleapis.com/auth/devstorage.read_write"]) + assert constructor.call_count == 1 + assert constructor.call_args.args == ("cache-bucket",) + assert constructor.call_args.kwargs["credential_provider"].credentials is credentials + + +@pytest.mark.parametrize("cache", ["AmazonS3", "GoogleStorage", "AzureBlob"], indirect=True) +@pytest.mark.parametrize("error_type", [GenericError, PermissionError, TimeoutError, OSError]) +def test_cloud_cache_backend_errors(cache: ObstoreCache, error_type: type[Exception]) -> None: + # Cloud caches and their groups preserve backend errors during reads, membership checks, and deletion. + error = error_type("backend unavailable") + store = MagicMock() + store.get.side_effect = error + store.delete.side_effect = error + cache.store = store + for target in (cache, cache.group("nested")): + for operation in (target.__getitem__, target.__contains__, target.__delitem__): + with pytest.raises(error_type) as raised: + operation("key") + assert raised.value is error + + +def test_local_cache_generic_error_is_not_a_missing_path(tmp_path: Path) -> None: + # Local backend errors propagate unless the requested path has a file where a directory is needed. + error = GenericError("disk unavailable") + store = MagicMock(spec=LocalStore) + store.prefix = tmp_path + store.get.side_effect = error + with pytest.raises(GenericError) as raised: + ObstoreCache(store)["missing/key"] + assert raised.value is error + + @pytest.mark.parametrize("cache", caches, indirect=True) def test_contains(cache: JobCache) -> None: assert "test" not in cache diff --git a/tilebox-workflows/tilebox/workflows/automations/service.py b/tilebox-workflows/tilebox/workflows/automations/service.py index d49acfc..06cdc50 100644 --- a/tilebox-workflows/tilebox/workflows/automations/service.py +++ b/tilebox-workflows/tilebox/workflows/automations/service.py @@ -12,25 +12,35 @@ StorageLocation, ) from tilebox.workflows.workflows.v1.automation_pb2 import AutomationPrototype as AutomationPrototypeMessage -from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest, StorageLocations +from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest from tilebox.workflows.workflows.v1.automation_pb2_grpc import AutomationServiceStub +from tilebox.workflows.workflows.v1.storage_location_pb2 import StorageLocations +from tilebox.workflows.workflows.v1.storage_location_pb2_grpc import StorageLocationServiceStub class AutomationService: - def __init__(self, channel: Channel | Any) -> None: + def __init__(self, channel: Channel | Any, storage_channel: Channel | Any = None) -> None: """ A wrapper around the AutomationServiceStub that provides a more pythonic interface and converts the protobuf messages to and from the data classes used in the rest of the tilebox-workflows codebase. Args: channel: The gRPC channel to use for the service. + storage_channel: Optional separate storage-location client for the Connect transport. """ self.service = ( with_pythonic_errors(AutomationServiceStub(channel)) if hasattr(channel, "unary_unary") else channel ) + if storage_channel is None: + storage_channel = channel + self.storage_service = ( + with_pythonic_errors(StorageLocationServiceStub(storage_channel)) + if hasattr(storage_channel, "unary_unary") + else storage_channel + ) def list_storage_locations(self) -> builtins.list[StorageLocation]: - response: StorageLocations = self.service.ListStorageLocations(Empty()) + response: StorageLocations = self.storage_service.ListStorageLocations(Empty()) return [StorageLocation.from_message(sl) for sl in response.locations] def list(self) -> builtins.list[AutomationPrototype]: diff --git a/tilebox-workflows/tilebox/workflows/automations/storage_event.py b/tilebox-workflows/tilebox/workflows/automations/storage_event.py index 3bb42cf..9e27f81 100644 --- a/tilebox-workflows/tilebox/workflows/automations/storage_event.py +++ b/tilebox-workflows/tilebox/workflows/automations/storage_event.py @@ -10,7 +10,7 @@ ) from tilebox.workflows.task import RunnerContext, Task, deserialize_task, serialize_task from tilebox.workflows.workflows.v1.automation_pb2 import Automation as AutomationMessage -from tilebox.workflows.workflows.v1.automation_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage +from tilebox.workflows.workflows.v1.storage_location_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage _NOT_TRIGGERED = TriggeredStorageEvent( StorageLocation(UUID(int=0), "", StorageType.FS), diff --git a/tilebox-workflows/tilebox/workflows/cache.py b/tilebox-workflows/tilebox/workflows/cache.py index 2810dac..ffc32b8 100644 --- a/tilebox-workflows/tilebox/workflows/cache.py +++ b/tilebox-workflows/tilebox/workflows/cache.py @@ -1,18 +1,14 @@ import contextlib from abc import ABC, abstractmethod from collections.abc import Iterator -from io import BytesIO from pathlib import Path from pathlib import PurePosixPath as ObjectPath from threading import RLock from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from google.cloud.storage import Blob, Bucket from obstore.store import ObjectStore else: - Blob = Any - Bucket = Any ObjectStore = Any @@ -91,7 +87,7 @@ def __init__(self, store: ObjectStore, prefix: str | ObjectPath = ObjectPath("." self.prefix = ObjectPath(prefix) def __contains__(self, key: str) -> bool: - with contextlib.suppress(OSError): + with contextlib.suppress(FileNotFoundError, NotADirectoryError): self.store.get(str(self.prefix / key)) return True # if get is successful, we know the key is in the cache @@ -103,19 +99,28 @@ def __setitem__(self, key: str, value: bytes) -> None: def __delitem__(self, key: str) -> None: try: self.store.delete(str(self.prefix / key)) - except OSError: + except (FileNotFoundError, NotADirectoryError): raise KeyError(f"{key} is not cached!") from None def __getitem__(self, key: str) -> bytes: from obstore.exceptions import GenericError # noqa: PLC0415 + from obstore.store import LocalStore # noqa: PLC0415 try: entry = self.store.get(str(self.prefix / key)) return bytes(entry.bytes()) - except (OSError, GenericError): - # GenericError is raised if the key contains separator characters, but one of the parents is a file - # instead of a directory + except (FileNotFoundError, NotADirectoryError): raise KeyError(f"{key} is not cached!") from None + except GenericError: + # LocalStore wraps a file used as a parent directory in GenericError. + if isinstance(self.store, LocalStore) and self.store.prefix is not None: + try: + (self.store.prefix / self.prefix / key).stat() + except NotADirectoryError: + raise KeyError(f"{key} is not cached!") from None + except OSError: + pass + raise def __iter__(self) -> Iterator[str]: prefix = "" if self.prefix == ObjectPath(".") else str(self.prefix) @@ -252,113 +257,47 @@ def group(self, key: str) -> "LocalFileSystemCache": return LocalFileSystemCache(self.root / key) -class GoogleStorageCache(JobCache): - def __init__(self, bucket: Bucket, prefix: str | ObjectPath = "jobs") -> None: +class GoogleStorageCache(ObstoreCache): + def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs") -> None: """A cache implementation that stores data in Google Cloud Storage. Args: - bucket: The Google Cloud Storage bucket to use for the cache. + bucket: The Google Cloud Storage bucket name. Credentials come from Google Auth. prefix: A path prefix to append to all objects stored in the cache. Defaults to "jobs". """ - self.bucket = bucket - self.prefix = ObjectPath( - prefix - ) # we still use pathlib here, because it's easier to work with when joining paths - - def _blob(self, key: str) -> Blob: - return self.bucket.blob(str(self.prefix / key)) - - def __contains__(self, key: str) -> bool: - # GCS library has some weird typing issues, so let's ignore them for now - return self._blob(key).exists() - - def __setitem__(self, key: str, value: bytes) -> None: - # GCS library has some weird typing issues, so let's ignore them for now - self._blob(key).upload_from_file(BytesIO(value)) - - def __getitem__(self, key: str) -> bytes: - from google.cloud.exceptions import NotFound # noqa: PLC0415 - - try: - # GCS library has some weird typing issues, so let's ignore them for now - return self._blob(key).download_as_bytes() - except NotFound: - raise KeyError(f"{key} is not cached!") from None - - def __iter__(self) -> Iterator[str]: - # we need to add the trailing slash, to avoid listing other blobs that start with the same prefix, e.g. - # consider the following blobs: - # jobs/folder/some_file.txt - # jobs/folder2/other_file.txt - # if we just list all blobs with prefix "jobs/folder", we would get both of them, but we only want the - # ones in the folder, so we add the trailing slash to only get the blobs in the folder - prefix = str(self.prefix) + "/" - # by specifying the delimiter as "/", we can emulate a directory structure, and only get the blobs directly - # in the "folder", and not the ones in subfolders - - # GCS library has some weird typing issues, so let's ignore them for now - blobs = self.bucket.list_blobs(prefix=prefix, delimiter="/") + import google.auth # noqa: PLC0415 + from obstore.auth.google import GoogleCredentialProvider # noqa: PLC0415 + from obstore.store import GCSStore # noqa: PLC0415 - # make the names relative to the cache prefix (but including the key in the name) - for blob in blobs: - yield str(ObjectPath(blob.name).relative_to(self.prefix)) + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/devstorage.read_write"]) + store = GCSStore(bucket, credential_provider=GoogleCredentialProvider(credentials=credentials)) + super().__init__(store, prefix) - def group(self, key: str) -> "GoogleStorageCache": - return GoogleStorageCache(self.bucket, prefix=str(self.prefix / key)) - -class AmazonS3Cache(JobCache): - def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs") -> None: +class AmazonS3Cache(ObstoreCache): + def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs", *, region: str | None = None) -> None: """A cache implementation that stores data in Amazon S3. Args: - bucket: The Amazon S3 bucket to use for the cache. + bucket: The Amazon S3 bucket name. Credentials come from the boto3 session. prefix: A path prefix to append to all objects stored in the cache. Defaults to "jobs". + region: The bucket region. If omitted, discover it with HeadBucket. """ - import boto3 # noqa: PLC0415 + from tilebox.workflows.data import _s3_storage_client # noqa: PLC0415 - self.bucket = bucket - self.prefix = ObjectPath(prefix) - self._s3 = boto3.client("s3") + super().__init__(_s3_storage_client(bucket, region), prefix) - def __contains__(self, key: str) -> bool: - from botocore.exceptions import ClientError # noqa: PLC0415 - try: - self._s3.head_object(Bucket=self.bucket, Key=str(self.prefix / key)) - except ClientError as e: - err = e.response.get("Error", {}) - if err.get("Code", "") == "404": - return False - raise # not a 404, re-raise the exception - else: - return True - - def __setitem__(self, key: str, value: bytes) -> None: - self._s3.upload_fileobj(BytesIO(value), self.bucket, str(self.prefix / key)) +class AzureBlobCache(ObstoreCache): + def __init__(self, account_name: str, container: str, prefix: str | ObjectPath = "jobs") -> None: + """Store cached data in Azure Blob Storage. - def __getitem__(self, key: str) -> bytes: - from botocore.exceptions import ClientError # noqa: PLC0415 - - item = BytesIO() - try: - self._s3.download_fileobj(self.bucket, str(self.prefix / key), item) - except ClientError as e: - err = e.response.get("Error", {}) - if err.get("Code", "") == "404": - raise KeyError(f"{key} is not cached!") from None - raise # not a 404, re-raise the exception - else: - return item.getvalue() + Args: + account_name: The storage account name. Credentials come from Azure Identity, + or an account key or SAS token configured in the environment. + container: The Azure Blob Storage container name. + prefix: A path prefix for cached objects. Defaults to "jobs". + """ + from tilebox.workflows.data import _azure_storage_client # noqa: PLC0415 - def __iter__(self) -> Iterator[str]: - paginator = self._s3.get_paginator("list_objects_v2") - prefix = str(self.prefix) + "/" if self.prefix != Path() else "" - for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix, Delimiter="/"): - for item in page.get("Contents", []): - bucket = str(Path(item.get("Key", "")).relative_to(self.prefix)) - if "/" not in bucket: # only yield files directly in the prefix - yield bucket - - def group(self, key: str) -> "AmazonS3Cache": - return AmazonS3Cache(self.bucket, prefix=str(self.prefix / key)) + super().__init__(_azure_storage_client(account_name, container), prefix) diff --git a/tilebox-workflows/tilebox/workflows/client.py b/tilebox-workflows/tilebox/workflows/client.py index 21b45ba..c19a167 100644 --- a/tilebox-workflows/tilebox/workflows/client.py +++ b/tilebox-workflows/tilebox/workflows/client.py @@ -68,6 +68,7 @@ def __init__( self._cluster_service = self._job_service self._workflow_service = self._job_service self._automation_service = self._job_service + self._storage_location_service = self._job_service self._task_service = self._job_service case "http1": from pyqwest import HTTPVersion, SyncClient, SyncHTTPTransport # noqa: PLC0415 @@ -76,6 +77,9 @@ def __init__( AutomationServiceClientSync, ) from tilebox.workflows.workflows.v1.job_connect import JobServiceClientSync # noqa: PLC0415 + from tilebox.workflows.workflows.v1.storage_location_connect import ( # noqa: PLC0415 + StorageLocationServiceClientSync, + ) from tilebox.workflows.workflows.v1.task_connect import TaskServiceClientSync # noqa: PLC0415 from tilebox.workflows.workflows.v1.telemetry_connect import ( # noqa: PLC0415 TelemetryQueryServiceClientSync, @@ -98,6 +102,9 @@ def __init__( self._automation_service = ConnectStubAdapter( AutomationServiceClientSync(address, http_client=http_client), headers ) + self._storage_location_service = ConnectStubAdapter( + StorageLocationServiceClientSync(address, http_client=http_client), headers + ) self._task_service = ConnectStubAdapter( TaskServiceClientSync(address, http_client=http_client), headers ) @@ -200,7 +207,7 @@ def automations(self) -> AutomationClient: Returns: A client for the automations service. """ - return AutomationClient(AutomationService(self._automation_service)) + return AutomationClient(AutomationService(self._automation_service, self._storage_location_service)) def _token_from_env(url: str, token: str | None) -> str | None: diff --git a/tilebox-workflows/tilebox/workflows/data.py b/tilebox-workflows/tilebox/workflows/data.py index 6bcbb56..f72dbb9 100644 --- a/tilebox-workflows/tilebox/workflows/data.py +++ b/tilebox-workflows/tilebox/workflows/data.py @@ -1,7 +1,9 @@ +import os import re import warnings from collections.abc import Callable, Mapping, MutableMapping -from dataclasses import asdict, dataclass, field +from contextlib import closing +from dataclasses import asdict, dataclass, field, replace from datetime import UTC, datetime, timedelta from enum import Enum from functools import lru_cache, total_ordering @@ -27,17 +29,15 @@ from tilebox.datasets.uuid import must_uuid_to_uuid_message, uuid_message_to_uuid, uuid_to_uuid_message if TYPE_CHECKING: - from google.cloud.storage.bucket import Bucket - from mypy_boto3_s3.client import S3Client + from obstore.store import AzureStore, GCSStore, S3Store from tilebox.workflows.observability.tracing import WorkflowTracer else: - Bucket = Any - S3Client = Any WorkflowTracer = Any from tilebox.workflows.workflows.v1 import automation_pb2 as automation_pb from tilebox.workflows.workflows.v1 import core_pb2, job_pb2, task_pb2, telemetry_pb2, workflows_pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as storage_pb _VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)$") # matches a version string in the format "v3.2" @@ -969,65 +969,124 @@ def from_message(cls, page: Any) -> "QueryJobSpansResponse": class StorageType(Enum): - GCS = automation_pb.STORAGE_TYPE_GCS # Google Cloud Storage - S3 = automation_pb.STORAGE_TYPE_S3 # Amazon Web Services S3 - FS = automation_pb.STORAGE_TYPE_FS # Local Filesystem + GCS = storage_pb.STORAGE_TYPE_GCS_BUCKET # Google Cloud Storage + S3 = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET # Amazon Web Services S3 + FS = storage_pb.STORAGE_TYPE_FILESYSTEM # Local Filesystem + AZURE = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage _STORAGE_TYPE_TO_ENUM = {storage_type.value: storage_type for storage_type in StorageType} -@dataclass(frozen=True, order=True) +@total_ordering +@dataclass(frozen=True) class StorageLocation: id: UUID location: str type: StorageType runner_context: "RunnerContext | None" = None + def __lt__(self, other: object) -> bool: + if not isinstance(other, StorageLocation): + return NotImplemented + return (self.id, self.location, self.type.value) < (other.id, other.location, other.type.value) + @classmethod - def from_message(cls, storage_location: automation_pb.StorageLocation) -> Self: + def from_message(cls, storage_location: storage_pb.StorageLocation) -> "StorageLocation": """Convert a StorageLocation protobuf message to a StorageLocation object.""" + location = storage_location.location + storage_type = storage_location.type + if storage_location.HasField("reference"): + reference = storage_location.reference + storage_type = reference.type + match storage_type: + case storage_pb.STORAGE_TYPE_AZURE_BLOB: + return AzureStorageLocation( + id=uuid_message_to_uuid(storage_location.id), + location=reference.azure_blob.container, + type=StorageType.AZURE, + storage_account_resource_id=reference.azure_blob.storage_account_resource_id, + ) + case storage_pb.STORAGE_TYPE_GCS_BUCKET: + location = reference.gcs_bucket.bucket + case storage_pb.STORAGE_TYPE_AWS_S3_BUCKET: + return S3StorageLocation( + id=uuid_message_to_uuid(storage_location.id), + location=reference.aws_s3_bucket.bucket, + type=StorageType.S3, + region=reference.aws_s3_bucket.region or None, + ) + case storage_pb.STORAGE_TYPE_FILESYSTEM: + location = reference.filesystem.path return cls( id=uuid_message_to_uuid(storage_location.id), - location=storage_location.location, - type=_STORAGE_TYPE_TO_ENUM[storage_location.type], + location=location, + type=_STORAGE_TYPE_TO_ENUM[storage_type], ) def _with_runner_context(self, runner_context: "RunnerContext") -> "StorageLocation": - return StorageLocation( - id=self.id, - location=self.location, - type=self.type, - runner_context=runner_context, - ) + return replace(self, runner_context=runner_context) - def to_message(self) -> automation_pb.StorageLocation: + def to_message(self) -> storage_pb.StorageLocation: """Convert a StorageLocation object to a StorageLocation protobuf message.""" - return automation_pb.StorageLocation( + return storage_pb.StorageLocation( id=uuid_to_uuid_message(self.id), location=self.location, type=self.type.value ) def read(self, path: str) -> bytes: runner_context = self.runner_context or RunnerContext() - match self.type: - case StorageType.GCS: - with runner_context.tracer.span("gcs.read") as span: + with runner_context.tracer.span(f"{self.type.name.lower()}.read") as span: + span.set_attribute("path", path) + match self.type: + case StorageType.GCS: span.set_attribute("bucket", self.location) - span.set_attribute("path", path) - # GCS library has some weird typing issues, so let's ignore them for now - blob = runner_context.gcs_client(self.location).blob(path) - return blob.download_as_bytes() - case StorageType.S3: - with runner_context.tracer.span("s3.read") as span: + store = runner_context.gcs_client(self.location) + case StorageType.S3: span.set_attribute("bucket", self.location) - span.set_attribute("path", path) - return runner_context.s3_client(self.location).get_object(Bucket=self.location, Key=path)["Body"].read() - case StorageType.FS: - with runner_context.tracer.span("fs.read") as span: + if isinstance(self, S3StorageLocation) and self.region: + store = runner_context.s3_client(self.location, region=self.region) + else: + store = runner_context.s3_client(self.location) + case StorageType.AZURE: + if not isinstance(self, AzureStorageLocation) or not self.storage_account_resource_id: + raise ValueError("Azure storage location is missing its storage account resource ID") + span.set_attribute("storage_account_resource_id", self.storage_account_resource_id) + span.set_attribute("container", self.location) + store = runner_context.azure_client(self.storage_account_resource_id, self.location) + case StorageType.FS: span.set_attribute("root_directory", self.location) - span.set_attribute("path", path) - return Path(self.location).joinpath(path).read_bytes() + return Path(self.location).joinpath(path).read_bytes() + + import obstore # noqa: PLC0415 + + return bytes(obstore.get(store, path).bytes()) + + +@dataclass(frozen=True) +class S3StorageLocation(StorageLocation): + region: str | None = field(default=None, kw_only=True) + + def to_message(self) -> storage_pb.StorageLocation: + """Include the S3 bucket and its region in the storage reference.""" + message = super().to_message() + message.reference.type = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET + message.reference.aws_s3_bucket.bucket = self.location + message.reference.aws_s3_bucket.region = self.region or "" + return message + + +@dataclass(frozen=True) +class AzureStorageLocation(StorageLocation): + storage_account_resource_id: str = field(kw_only=True) + + def to_message(self) -> storage_pb.StorageLocation: + """Include the Azure account and container in the storage reference.""" + message = super().to_message() + message.reference.type = storage_pb.STORAGE_TYPE_AZURE_BLOB + message.reference.azure_blob.storage_account_resource_id = self.storage_account_resource_id + message.reference.azure_blob.container = self.location + return message @dataclass(order=True) @@ -1055,7 +1114,7 @@ def to_message(self) -> automation_pb.StorageEventTrigger: class StorageEventType(Enum): - CREATED = automation_pb.STORAGE_EVENT_TYPE_CREATED + CREATED = storage_pb.STORAGE_EVENT_TYPE_CREATED _STORAGE_EVENT_TYPE_TO_ENUM = {storage_event_type.value: storage_event_type for storage_event_type in StorageEventType} @@ -1069,7 +1128,7 @@ class TriggeredStorageEvent: @classmethod def from_message( - cls, event: automation_pb.TriggeredStorageEvent, locations: Mapping[UUID, StorageLocation] + cls, event: storage_pb.TriggeredStorageEvent, locations: Mapping[UUID, StorageLocation] ) -> "TriggeredStorageEvent": """Convert a TriggeredStorageEvent protobuf message to a TriggeredStorageEvent object.""" storage_location_id = uuid_message_to_uuid(event.storage_location_id) @@ -1085,9 +1144,9 @@ def from_message( location=event.location, ) - def to_message(self) -> automation_pb.TriggeredStorageEvent: + def to_message(self) -> storage_pb.TriggeredStorageEvent: """Convert a TriggeredStorageEvent object to a TriggeredStorageEvent protobuf message.""" - return automation_pb.TriggeredStorageEvent( + return storage_pb.TriggeredStorageEvent( storage_location_id=uuid_to_uuid_message(self.storage.id), type=self.type.value, location=self.location, @@ -1177,25 +1236,90 @@ def __init__( for sl in storage_locations or [] } - def gcs_client(self, location: str) -> Bucket: - return _default_google_storage_client(location) + def gcs_client(self, location: str) -> "GCSStore": + return _default_gcs_storage_client(location) - def s3_client(self, location: str) -> S3Client: - import boto3 # noqa: PLC0415 - - _ = location # we always use the default s3 client, regardless of the location - return boto3.client("s3") + def s3_client(self, location: str, region: str | None = None) -> "S3Store": + return _default_s3_storage_client(location, region) def local_path(self, location: str) -> Path: return Path(location) + def azure_client(self, storage_account_resource_id: str, container: str) -> "AzureStore": + return _default_azure_storage_client(storage_account_resource_id, container) + + +@lru_cache +def _default_gcs_storage_client(bucket: str) -> "GCSStore": + import google.auth # noqa: PLC0415 + from obstore.auth.google import GoogleCredentialProvider # noqa: PLC0415 + from obstore.store import GCSStore # noqa: PLC0415 + + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/devstorage.read_only"]) + return GCSStore(bucket, credential_provider=GoogleCredentialProvider(credentials=credentials)) + @lru_cache -def _default_google_storage_client(location: str) -> Bucket: - from google.cloud.storage import Client as GoogleStorageClient # noqa: PLC0415 +def _default_s3_storage_client(bucket: str, region: str | None = None) -> "S3Store": + return _s3_storage_client(bucket, region) + + +def _s3_storage_client(bucket: str, region: str | None = None) -> "S3Store": + import boto3 # noqa: PLC0415 + from botocore.exceptions import ClientError # noqa: PLC0415 + from obstore.auth.boto3 import Boto3CredentialProvider # noqa: PLC0415 + from obstore.store import S3Store # noqa: PLC0415 + + session = boto3.DEFAULT_SESSION or boto3.Session() + if not region: + with closing(session.client("s3")) as client: + try: + response = client.head_bucket(Bucket=bucket) + except ClientError as error: + region = error.response.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region") + if not region: + raise + else: + region = response.get("BucketRegion") or response.get("ResponseMetadata", {}).get( + "HTTPHeaders", {} + ).get("x-amz-bucket-region") + if not region: + raise ValueError(f"S3 did not return a region for bucket {bucket!r}") + return S3Store( + bucket, region=region, credential_provider=Boto3CredentialProvider(session=session, ttl=timedelta(0)) + ) + + +@lru_cache +def _default_azure_storage_client(storage_account_resource_id: str, container: str) -> "AzureStore": + match = re.fullmatch( + r"/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft\.Storage/storageAccounts/([^/]+)", + storage_account_resource_id, + flags=re.IGNORECASE, + ) + if match is None: + raise ValueError(f"Invalid Azure storage account resource ID: {storage_account_resource_id!r}") + return _azure_storage_client(match[1], container) + + +def _azure_storage_client(account_name: str, container: str) -> "AzureStore": + from obstore.store import AzureStore # noqa: PLC0415 + + if any( + name in os.environ + for name in ( + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ACCESS_KEY", + "AZURE_STORAGE_MASTER_KEY", + "AZURE_STORAGE_SAS_KEY", + "AZURE_STORAGE_SAS_TOKEN", + ) + ): + return AzureStore(container, account_name=account_name) + + from obstore.auth.azure import AzureCredentialProvider # noqa: PLC0415 - project, bucket = location.split(":") - return GoogleStorageClient(project=project).bucket(bucket) + return AzureStore(container, account_name=account_name, credential_provider=AzureCredentialProvider()) @dataclass(frozen=True) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py index 61e2a79..cedd0ed 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py @@ -20,18 +20,6 @@ class AutomationService(Protocol): - async def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - - async def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - - async def create_storage_location(self, request: workflows_dot_v1_dot_automation__pb2.StorageLocation, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - - async def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - async def list_automations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.Automations: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") @@ -53,46 +41,6 @@ def __init__(self, service: AutomationService | AsyncGenerator[AutomationService super().__init__( service=service, endpoints=lambda svc: { - "/workflows.v1.AutomationService/ListStorageLocations": Endpoint.unary( - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.list_storage_locations, - ), - "/workflows.v1.AutomationService/GetStorageLocation": Endpoint.unary( - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.get_storage_location, - ), - "/workflows.v1.AutomationService/CreateStorageLocation": Endpoint.unary( - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.create_storage_location, - ), - "/workflows.v1.AutomationService/DeleteStorageLocation": Endpoint.unary( - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.delete_storage_location, - ), "/workflows.v1.AutomationService/ListAutomations": Endpoint.unary( method=MethodInfo( name="ListAutomations", @@ -157,86 +105,6 @@ def path(self) -> str: class AutomationServiceClient(ConnectClient): - async def list_storage_locations( - self, - request: google_dot_protobuf_dot_empty__pb2.Empty, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - async def get_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - async def create_storage_location( - self, - request: workflows_dot_v1_dot_automation__pb2.StorageLocation, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - async def delete_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> google_dot_protobuf_dot_empty__pb2.Empty: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - async def list_automations( self, request: google_dot_protobuf_dot_empty__pb2.Empty, @@ -342,14 +210,6 @@ async def delete_automation( class AutomationServiceSync(Protocol): - def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - def create_storage_location(self, request: workflows_dot_v1_dot_automation__pb2.StorageLocation, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def list_automations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.Automations: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def get_automation(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.AutomationPrototype: @@ -366,46 +226,6 @@ class AutomationServiceWSGIApplication(ConnectWSGIApplication): def __init__(self, service: AutomationServiceSync, interceptors: Iterable[InterceptorSync]=(), read_max_bytes: int | None = None, compressions: Iterable[Compression] | None = None, codecs: Iterable[Codec] | None = None) -> None: super().__init__( endpoints={ - "/workflows.v1.AutomationService/ListStorageLocations": EndpointSync.unary( - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.list_storage_locations, - ), - "/workflows.v1.AutomationService/GetStorageLocation": EndpointSync.unary( - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.get_storage_location, - ), - "/workflows.v1.AutomationService/CreateStorageLocation": EndpointSync.unary( - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.create_storage_location, - ), - "/workflows.v1.AutomationService/DeleteStorageLocation": EndpointSync.unary( - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.delete_storage_location, - ), "/workflows.v1.AutomationService/ListAutomations": EndpointSync.unary( method=MethodInfo( name="ListAutomations", @@ -470,86 +290,6 @@ def path(self) -> str: class AutomationServiceClientSync(ConnectClientSync): - def list_storage_locations( - self, - request: google_dot_protobuf_dot_empty__pb2.Empty, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - return self.execute_unary( - request=request, - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - def get_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return self.execute_unary( - request=request, - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - def create_storage_location( - self, - request: workflows_dot_v1_dot_automation__pb2.StorageLocation, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return self.execute_unary( - request=request, - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - def delete_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> google_dot_protobuf_dot_empty__pb2.Empty: - return self.execute_unary( - request=request, - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - def list_automations( self, request: google_dot_protobuf_dot_empty__pb2.Empty, diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py index 552115d..6d76160 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py @@ -27,42 +27,33 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflows/v1/automation.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\"|\n\x0fStorageLocation\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08location\x18\x02 \x01(\tR\x08location\x12-\n\x04type\x18\x03 \x01(\x0e\x32\x19.workflows.v1.StorageTypeR\x04type\"O\n\x10StorageLocations\x12;\n\tlocations\x18\x01 \x03(\x0b\x32\x1d.workflows.v1.StorageLocationR\tlocations\"\xc0\x02\n\x13\x41utomationPrototype\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12@\n\tprototype\x18\x03 \x01(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\tprototype\x12W\n\x16storage_event_triggers\x18\x04 \x03(\x0b\x32!.workflows.v1.StorageEventTriggerR\x14storageEventTriggers\x12>\n\rcron_triggers\x18\x05 \x03(\x0b\x32\x19.workflows.v1.CronTriggerR\x0c\x63ronTriggers\x12\x1a\n\x08\x64isabled\x18\x06 \x01(\x08R\x08\x64isabled\"R\n\x0b\x41utomations\x12\x43\n\x0b\x61utomations\x18\x01 \x03(\x0b\x32!.workflows.v1.AutomationPrototypeR\x0b\x61utomations\"\xa2\x01\n\x13StorageEventTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12H\n\x10storage_location\x18\x02 \x01(\x0b\x32\x1d.workflows.v1.StorageLocationR\x0fstorageLocation\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"\x96\x01\n\x0b\x43ronTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08schedule\x18\x02 \x01(\tR\x08schedule\x12K\n\x11next_scheduled_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\x0fnextScheduledAt\"E\n\nAutomation\x12#\n\rtrigger_event\x18\x01 \x01(\x0cR\x0ctriggerEvent\x12\x12\n\x04\x61rgs\x18\x02 \x01(\x0cR\x04\x61rgs\"\xa7\x01\n\x15TriggeredStorageEvent\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location\"S\n\x12TriggeredCronEvent\x12=\n\x0ctrigger_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0btriggerTime\"o\n\x17\x44\x65leteAutomationRequest\x12\x33\n\rautomation_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x1f\n\x0b\x63\x61ncel_jobs\x18\x02 \x01(\x08R\ncancelJobs*k\n\x0bStorageType\x12\x1c\n\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10STORAGE_TYPE_GCS\x10\x01\x12\x13\n\x0fSTORAGE_TYPE_S3\x10\x02\x12\x13\n\x0fSTORAGE_TYPE_FS\x10\x03*V\n\x10StorageEventType\x12\"\n\x1eSTORAGE_EVENT_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTORAGE_EVENT_TYPE_CREATED\x10\x01\x32\xd1\x05\n\x11\x41utomationService\x12N\n\x14ListStorageLocations\x12\x16.google.protobuf.Empty\x1a\x1e.workflows.v1.StorageLocations\x12\x43\n\x12GetStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x1d.workflows.v1.StorageLocation\x12U\n\x15\x43reateStorageLocation\x12\x1d.workflows.v1.StorageLocation\x1a\x1d.workflows.v1.StorageLocation\x12?\n\x15\x44\x65leteStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12\x44\n\x0fListAutomations\x12\x16.google.protobuf.Empty\x1a\x19.workflows.v1.Automations\x12\x42\n\rGetAutomation\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10\x43reateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10UpdateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12Q\n\x10\x44\x65leteAutomation\x12%.workflows.v1.DeleteAutomationRequest\x1a\x16.google.protobuf.EmptyBy\n\x10\x63om.workflows.v1B\x0f\x41utomationProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflows/v1/automation.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\x1a#workflows/v1/storage_location.proto\"\xc0\x02\n\x13\x41utomationPrototype\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12@\n\tprototype\x18\x03 \x01(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\tprototype\x12W\n\x16storage_event_triggers\x18\x04 \x03(\x0b\x32!.workflows.v1.StorageEventTriggerR\x14storageEventTriggers\x12>\n\rcron_triggers\x18\x05 \x03(\x0b\x32\x19.workflows.v1.CronTriggerR\x0c\x63ronTriggers\x12\x1a\n\x08\x64isabled\x18\x06 \x01(\x08R\x08\x64isabled\"R\n\x0b\x41utomations\x12\x43\n\x0b\x61utomations\x18\x01 \x03(\x0b\x32!.workflows.v1.AutomationPrototypeR\x0b\x61utomations\"\xa2\x01\n\x13StorageEventTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12H\n\x10storage_location\x18\x02 \x01(\x0b\x32\x1d.workflows.v1.StorageLocationR\x0fstorageLocation\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"\x96\x01\n\x0b\x43ronTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08schedule\x18\x02 \x01(\tR\x08schedule\x12K\n\x11next_scheduled_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\x0fnextScheduledAt\"E\n\nAutomation\x12#\n\rtrigger_event\x18\x01 \x01(\x0cR\x0ctriggerEvent\x12\x12\n\x04\x61rgs\x18\x02 \x01(\x0cR\x04\x61rgs\"S\n\x12TriggeredCronEvent\x12=\n\x0ctrigger_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0btriggerTime\"o\n\x17\x44\x65leteAutomationRequest\x12\x33\n\rautomation_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x1f\n\x0b\x63\x61ncel_jobs\x18\x02 \x01(\x08R\ncancelJobs2\xa4\x03\n\x11\x41utomationService\x12\x44\n\x0fListAutomations\x12\x16.google.protobuf.Empty\x1a\x19.workflows.v1.Automations\x12\x42\n\rGetAutomation\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10\x43reateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10UpdateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12Q\n\x10\x44\x65leteAutomation\x12%.workflows.v1.DeleteAutomationRequest\x1a\x16.google.protobuf.EmptyB\x05\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.automation_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\020com.workflows.v1B\017AutomationProtoP\001\242\002\003WXX\252\002\014Workflows.V1\312\002\014Workflows\\V1\342\002\030Workflows\\V1\\GPBMetadata\352\002\rWorkflows::V1\222\003\002\010\002' + _globals['DESCRIPTOR']._serialized_options = b'\222\003\002\010\002' _globals['_CRONTRIGGER'].fields_by_name['next_scheduled_at']._loaded_options = None _globals['_CRONTRIGGER'].fields_by_name['next_scheduled_at']._serialized_options = b'\340A\003' - _globals['_STORAGETYPE']._serialized_start=1559 - _globals['_STORAGETYPE']._serialized_end=1666 - _globals['_STORAGEEVENTTYPE']._serialized_start=1668 - _globals['_STORAGEEVENTTYPE']._serialized_end=1754 - _globals['_STORAGELOCATION']._serialized_start=188 - _globals['_STORAGELOCATION']._serialized_end=312 - _globals['_STORAGELOCATIONS']._serialized_start=314 - _globals['_STORAGELOCATIONS']._serialized_end=393 - _globals['_AUTOMATIONPROTOTYPE']._serialized_start=396 - _globals['_AUTOMATIONPROTOTYPE']._serialized_end=716 - _globals['_AUTOMATIONS']._serialized_start=718 - _globals['_AUTOMATIONS']._serialized_end=800 - _globals['_STORAGEEVENTTRIGGER']._serialized_start=803 - _globals['_STORAGEEVENTTRIGGER']._serialized_end=965 - _globals['_CRONTRIGGER']._serialized_start=968 - _globals['_CRONTRIGGER']._serialized_end=1118 - _globals['_AUTOMATION']._serialized_start=1120 - _globals['_AUTOMATION']._serialized_end=1189 - _globals['_TRIGGEREDSTORAGEEVENT']._serialized_start=1192 - _globals['_TRIGGEREDSTORAGEEVENT']._serialized_end=1359 - _globals['_TRIGGEREDCRONEVENT']._serialized_start=1361 - _globals['_TRIGGEREDCRONEVENT']._serialized_end=1444 - _globals['_DELETEAUTOMATIONREQUEST']._serialized_start=1446 - _globals['_DELETEAUTOMATIONREQUEST']._serialized_end=1557 - _globals['_AUTOMATIONSERVICE']._serialized_start=1757 - _globals['_AUTOMATIONSERVICE']._serialized_end=2478 + _globals['_AUTOMATIONPROTOTYPE']._serialized_start=226 + _globals['_AUTOMATIONPROTOTYPE']._serialized_end=546 + _globals['_AUTOMATIONS']._serialized_start=548 + _globals['_AUTOMATIONS']._serialized_end=630 + _globals['_STORAGEEVENTTRIGGER']._serialized_start=633 + _globals['_STORAGEEVENTTRIGGER']._serialized_end=795 + _globals['_CRONTRIGGER']._serialized_start=798 + _globals['_CRONTRIGGER']._serialized_end=948 + _globals['_AUTOMATION']._serialized_start=950 + _globals['_AUTOMATION']._serialized_end=1019 + _globals['_TRIGGEREDCRONEVENT']._serialized_start=1021 + _globals['_TRIGGEREDCRONEVENT']._serialized_end=1104 + _globals['_DELETEAUTOMATIONREQUEST']._serialized_start=1106 + _globals['_DELETEAUTOMATIONREQUEST']._serialized_end=1217 + _globals['_AUTOMATIONSERVICE']._serialized_start=1220 + _globals['_AUTOMATIONSERVICE']._serialized_end=1640 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi index 38da1bd..74be9b9 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi @@ -3,8 +3,8 @@ from google.protobuf import empty_pb2 as _empty_pb2 from google.protobuf import timestamp_pb2 as _timestamp_pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 from tilebox.workflows.workflows.v1 import core_pb2 as _core_pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as _storage_location_pb2 from google.protobuf.internal import containers as _containers -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from collections.abc import Iterable as _Iterable, Mapping as _Mapping @@ -12,40 +12,6 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor -class StorageType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - STORAGE_TYPE_UNSPECIFIED: _ClassVar[StorageType] - STORAGE_TYPE_GCS: _ClassVar[StorageType] - STORAGE_TYPE_S3: _ClassVar[StorageType] - STORAGE_TYPE_FS: _ClassVar[StorageType] - -class StorageEventType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - STORAGE_EVENT_TYPE_UNSPECIFIED: _ClassVar[StorageEventType] - STORAGE_EVENT_TYPE_CREATED: _ClassVar[StorageEventType] -STORAGE_TYPE_UNSPECIFIED: StorageType -STORAGE_TYPE_GCS: StorageType -STORAGE_TYPE_S3: StorageType -STORAGE_TYPE_FS: StorageType -STORAGE_EVENT_TYPE_UNSPECIFIED: StorageEventType -STORAGE_EVENT_TYPE_CREATED: StorageEventType - -class StorageLocation(_message.Message): - __slots__ = ("id", "location", "type") - ID_FIELD_NUMBER: _ClassVar[int] - LOCATION_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - id: _id_pb2.ID - location: str - type: StorageType - def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., location: _Optional[str] = ..., type: _Optional[_Union[StorageType, str]] = ...) -> None: ... - -class StorageLocations(_message.Message): - __slots__ = ("locations",) - LOCATIONS_FIELD_NUMBER: _ClassVar[int] - locations: _containers.RepeatedCompositeFieldContainer[StorageLocation] - def __init__(self, locations: _Optional[_Iterable[_Union[StorageLocation, _Mapping]]] = ...) -> None: ... - class AutomationPrototype(_message.Message): __slots__ = ("id", "name", "prototype", "storage_event_triggers", "cron_triggers", "disabled") ID_FIELD_NUMBER: _ClassVar[int] @@ -74,9 +40,9 @@ class StorageEventTrigger(_message.Message): STORAGE_LOCATION_FIELD_NUMBER: _ClassVar[int] GLOB_PATTERN_FIELD_NUMBER: _ClassVar[int] id: _id_pb2.ID - storage_location: StorageLocation + storage_location: _storage_location_pb2.StorageLocation glob_pattern: str - def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., storage_location: _Optional[_Union[StorageLocation, _Mapping]] = ..., glob_pattern: _Optional[str] = ...) -> None: ... + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., storage_location: _Optional[_Union[_storage_location_pb2.StorageLocation, _Mapping]] = ..., glob_pattern: _Optional[str] = ...) -> None: ... class CronTrigger(_message.Message): __slots__ = ("id", "schedule", "next_scheduled_at") @@ -96,16 +62,6 @@ class Automation(_message.Message): args: bytes def __init__(self, trigger_event: _Optional[bytes] = ..., args: _Optional[bytes] = ...) -> None: ... -class TriggeredStorageEvent(_message.Message): - __slots__ = ("storage_location_id", "type", "location") - STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - LOCATION_FIELD_NUMBER: _ClassVar[int] - storage_location_id: _id_pb2.ID - type: StorageEventType - location: str - def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageEventType, str]] = ..., location: _Optional[str] = ...) -> None: ... - class TriggeredCronEvent(_message.Message): __slots__ = ("trigger_time",) TRIGGER_TIME_FIELD_NUMBER: _ClassVar[int] diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py index 0ab210d..288c874 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py @@ -19,26 +19,6 @@ def __init__(self, channel): Args: channel: A grpc.Channel. """ - self.ListStorageLocations = channel.unary_unary( - '/workflows.v1.AutomationService/ListStorageLocations', - request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - response_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocations.FromString, - _registered_method=True) - self.GetStorageLocation = channel.unary_unary( - '/workflows.v1.AutomationService/GetStorageLocation', - request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - response_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - _registered_method=True) - self.CreateStorageLocation = channel.unary_unary( - '/workflows.v1.AutomationService/CreateStorageLocation', - request_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - response_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - _registered_method=True) - self.DeleteStorageLocation = channel.unary_unary( - '/workflows.v1.AutomationService/DeleteStorageLocation', - request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - _registered_method=True) self.ListAutomations = channel.unary_unary( '/workflows.v1.AutomationService/ListAutomations', request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, @@ -72,34 +52,6 @@ class AutomationServiceServicer: - Cron triggers, which triggers tasks on a schedule """ - def ListStorageLocations(self, request, context): - """ListStorageLocations lists all the storage buckets that are available for use as bucket triggers. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetStorageLocation(self, request, context): - """GetStorageLocation gets a storage location by its ID. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def CreateStorageLocation(self, request, context): - """CreateStorageLocation creates a new storage bucket. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def DeleteStorageLocation(self, request, context): - """DeleteStorageLocation deletes a storage location. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - def ListAutomations(self, request, context): """ListAutomations lists all the automations that are currently registered in a namespace. """ @@ -138,26 +90,6 @@ def DeleteAutomation(self, request, context): def add_AutomationServiceServicer_to_server(servicer, server): rpc_method_handlers = { - 'ListStorageLocations': grpc.unary_unary_rpc_method_handler( - servicer.ListStorageLocations, - request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - response_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocations.SerializeToString, - ), - 'GetStorageLocation': grpc.unary_unary_rpc_method_handler( - servicer.GetStorageLocation, - request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, - response_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - ), - 'CreateStorageLocation': grpc.unary_unary_rpc_method_handler( - servicer.CreateStorageLocation, - request_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - response_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - ), - 'DeleteStorageLocation': grpc.unary_unary_rpc_method_handler( - servicer.DeleteStorageLocation, - request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, - response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - ), 'ListAutomations': grpc.unary_unary_rpc_method_handler( servicer.ListAutomations, request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, @@ -197,114 +129,6 @@ class AutomationService: - Cron triggers, which triggers tasks on a schedule """ - @staticmethod - def ListStorageLocations(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/ListStorageLocations', - google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - workflows_dot_v1_dot_automation__pb2.StorageLocations.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def GetStorageLocation(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/GetStorageLocation', - tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def CreateStorageLocation(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/CreateStorageLocation', - workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def DeleteStorageLocation(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/DeleteStorageLocation', - tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - google_dot_protobuf_dot_empty__pb2.Empty.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - @staticmethod def ListAutomations(request, target, diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py new file mode 100644 index 0000000..63b2cb2 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py @@ -0,0 +1,716 @@ +# -*- coding: utf-8 -*- +# Generated by https://github.com/connectrpc/connect-python. DO NOT EDIT! +# source: workflows/v1/storage_location.proto + +from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Iterator, Mapping +from typing import Protocol + +from connectrpc.client import ConnectClient, ConnectClientSync +from connectrpc.code import Code +from connectrpc.codec import Codec +from connectrpc.compression import Compression +from connectrpc.errors import ConnectError +from connectrpc.interceptor import Interceptor, InterceptorSync +from connectrpc.method import IdempotencyLevel, MethodInfo +from connectrpc.request import Headers, RequestContext +from connectrpc.server import ConnectASGIApplication, ConnectWSGIApplication, Endpoint, EndpointSync +import google.protobuf.empty_pb2 as google_dot_protobuf_dot_empty__pb2 +import tilebox.datasets.tilebox.v1.id_pb2 as tilebox_dot_v1_dot_id__pb2 +import tilebox.workflows.workflows.v1.storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 + + +class StorageLocationService(Protocol): + async def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def create_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def update_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def create_storage_subscription(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def list_storage_subscriptions(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def get_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def delete_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def list_storage_subscription_events(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + +class StorageLocationServiceASGIApplication(ConnectASGIApplication[StorageLocationService]): + def __init__(self, service: StorageLocationService | AsyncGenerator[StorageLocationService], *, interceptors: Iterable[Interceptor]=(), read_max_bytes: int | None = None, compressions: Iterable[Compression] | None = None, codecs: Iterable[Codec] | None = None) -> None: + super().__init__( + service=service, + endpoints=lambda svc: { + "/workflows.v1.StorageLocationService/ListStorageLocations": Endpoint.unary( + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_storage_locations, + ), + "/workflows.v1.StorageLocationService/GetStorageLocation": Endpoint.unary( + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.get_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageLocation": Endpoint.unary( + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.create_storage_location, + ), + "/workflows.v1.StorageLocationService/UpdateStorageLocation": Endpoint.unary( + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.update_storage_location, + ), + "/workflows.v1.StorageLocationService/DeleteStorageLocation": Endpoint.unary( + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.delete_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageSubscription": Endpoint.unary( + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.create_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptions": Endpoint.unary( + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_storage_subscriptions, + ), + "/workflows.v1.StorageLocationService/GetStorageSubscription": Endpoint.unary( + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.get_storage_subscription, + ), + "/workflows.v1.StorageLocationService/DeleteStorageSubscription": Endpoint.unary( + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.delete_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents": Endpoint.unary( + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_storage_subscription_events, + ), + }, + interceptors=interceptors, + read_max_bytes=read_max_bytes, + compressions=compressions, + codecs=codecs, + ) + + @property + def path(self) -> str: + """Returns the URL path to mount the application to when serving multiple applications.""" + return "/workflows.v1.StorageLocationService" + + +class StorageLocationServiceClient(ConnectClient): + async def list_storage_locations( + self, + request: google_dot_protobuf_dot_empty__pb2.Empty, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def get_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def create_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def update_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def delete_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def create_storage_subscription( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def list_storage_subscriptions( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def get_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def delete_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def list_storage_subscription_events( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + + + + +class StorageLocationServiceSync(Protocol): + def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def create_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def update_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def create_storage_subscription(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def list_storage_subscriptions(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def get_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def delete_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def list_storage_subscription_events(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + +class StorageLocationServiceWSGIApplication(ConnectWSGIApplication): + def __init__(self, service: StorageLocationServiceSync, interceptors: Iterable[InterceptorSync]=(), read_max_bytes: int | None = None, compressions: Iterable[Compression] | None = None, codecs: Iterable[Codec] | None = None) -> None: + super().__init__( + endpoints={ + "/workflows.v1.StorageLocationService/ListStorageLocations": EndpointSync.unary( + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_storage_locations, + ), + "/workflows.v1.StorageLocationService/GetStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.get_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.create_storage_location, + ), + "/workflows.v1.StorageLocationService/UpdateStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.update_storage_location, + ), + "/workflows.v1.StorageLocationService/DeleteStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.delete_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageSubscription": EndpointSync.unary( + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.create_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptions": EndpointSync.unary( + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_storage_subscriptions, + ), + "/workflows.v1.StorageLocationService/GetStorageSubscription": EndpointSync.unary( + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.get_storage_subscription, + ), + "/workflows.v1.StorageLocationService/DeleteStorageSubscription": EndpointSync.unary( + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.delete_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents": EndpointSync.unary( + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_storage_subscription_events, + ), + }, + interceptors=interceptors, + read_max_bytes=read_max_bytes, + compressions=compressions, + codecs=codecs, + ) + + @property + def path(self) -> str: + """Returns the URL path to mount the application to when serving multiple applications.""" + return "/workflows.v1.StorageLocationService" + + +class StorageLocationServiceClientSync(ConnectClientSync): + def list_storage_locations( + self, + request: google_dot_protobuf_dot_empty__pb2.Empty, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def get_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def create_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def update_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return self.execute_unary( + request=request, + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def delete_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def create_storage_subscription( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def list_storage_subscriptions( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def get_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def delete_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def list_storage_subscription_events( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py new file mode 100644 index 0000000..fcfb70e --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: workflows/v1/storage_location.proto +# Protobuf Python Version: 6.30.0 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 30, + 0, + '', + 'workflows/v1/storage_location.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 +from tilebox.datasets.tilebox.v1 import query_pb2 as tilebox_dot_v1_dot_query__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#workflows/v1/storage_location.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\"\xde\x01\n\x0fStorageLocation\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1e\n\x08location\x18\x02 \x01(\tB\x02\x18\x01R\x08location\x12\x31\n\x04type\x18\x03 \x01(\x0e\x32\x19.workflows.v1.StorageTypeB\x02\x18\x01R\x04type\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x44\n\treference\x18\x05 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"\xd6\x02\n\x18StorageLocationReference\x12\x46\n\raws_s3_bucket\x18\x01 \x01(\x0b\x32\".workflows.v1.AWSS3BucketReferenceR\x0b\x61wsS3Bucket\x12?\n\ngcs_bucket\x18\x02 \x01(\x0b\x32 .workflows.v1.GCSBucketReferenceR\tgcsBucket\x12?\n\nazure_blob\x18\x03 \x01(\x0b\x32 .workflows.v1.AzureBlobReferenceR\tazureBlob\x12\x41\n\nfilesystem\x18\x04 \x01(\x0b\x32!.workflows.v1.FilesystemReferenceR\nfilesystem\x12-\n\x04type\x18\x05 \x01(\x0e\x32\x19.workflows.v1.StorageTypeR\x04type\"F\n\x14\x41WSS3BucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x16\n\x06region\x18\x02 \x01(\tR\x06region\"g\n\x12GCSBucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x1d\n\nproject_id\x18\x02 \x01(\tR\tprojectId\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location\"\x89\x01\n\x12\x41zureBlobReference\x12=\n\x1bstorage_account_resource_id\x18\x01 \x01(\tR\x18storageAccountResourceId\x12\x1c\n\tcontainer\x18\x02 \x01(\tR\tcontainer\x12\x16\n\x06region\x18\x03 \x01(\tR\x06region\")\n\x13\x46ilesystemReference\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\"\x8d\x01\n\x0cTriggeredJob\x12%\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x05jobId\x12\x33\n\rautomation_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"R\n\rTriggeredJobs\x12\x41\n\x0etriggered_jobs\x18\x01 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\"x\n\x1c\x43reateStorageLocationRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\treference\x18\x02 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"r\n\x1cUpdateStorageLocationRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"8\n\x19\x41WSSNSStorageSubscription\x12\x1b\n\ttopic_arn\x18\x01 \x01(\tR\x08topicArn\"\x9a\x01\n\x1fGooglePubSubStorageSubscription\x12\"\n\x0csubscription\x18\x01 \x01(\tR\x0csubscription\x12\x32\n\x15service_account_email\x18\x02 \x01(\tR\x13serviceAccountEmail\x12\x1f\n\x08\x61udience\x18\x03 \x01(\tB\x03\xe0\x41\x03R\x08\x61udience\"\xbd\x01\n!AzureEventGridStorageSubscription\x12\x37\n\x15webhook_secret_header\x18\x01 \x01(\tB\x03\xe0\x41\x03R\x13webhookSecretHeader\x12*\n\x0ewebhook_secret\x18\x02 \x01(\tB\x03\xe0\x41\x03R\rwebhookSecret\x12\x33\n\x13webhook_secret_hash\x18\x03 \x01(\x0c\x42\x03\xe0\x41\x03R\x11webhookSecretHash\"\x87\x04\n\x13StorageSubscription\x12#\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x03\xe0\x41\x03R\x02id\x12>\n\x13storage_location_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x03 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12\x1f\n\x08\x65ndpoint\x18\x04 \x01(\tB\x03\xe0\x41\x03R\x08\x65ndpoint\x12>\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\tcreatedAt\x12@\n\x07\x61ws_sns\x18\x07 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x08 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\t \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"\x8e\x03\n CreateStorageSubscriptionRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x02 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12@\n\x07\x61ws_sns\x18\x03 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x04 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\x05 \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"a\n\x1fListStorageSubscriptionsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\"_\n\x14StorageSubscriptions\x12G\n\rsubscriptions\x18\x01 \x03(\x0b\x32!.workflows.v1.StorageSubscriptionR\rsubscriptions\"\x90\x03\n\x18StorageSubscriptionEvent\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1d\n\nobject_key\x18\x02 \x01(\tR\tobjectKey\x12\x32\n\x04type\x18\x03 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x39\n\nevent_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12;\n\x0breceived_at\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\nreceivedAt\x12\x41\n\x0etriggered_jobs\x18\x06 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\x12\x46\n\x17storage_subscription_id\x18\x07 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x15storageSubscriptionId\"\x99\x01\n$ListStorageSubscriptionEventsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"\x97\x01\n\x19StorageSubscriptionEvents\x12>\n\x06\x65vents\x18\x01 \x03(\x0b\x32&.workflows.v1.StorageSubscriptionEventR\x06\x65vents\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"O\n\x10StorageLocations\x12;\n\tlocations\x18\x01 \x03(\x0b\x32\x1d.workflows.v1.StorageLocationR\tlocations\"\xa7\x01\n\x15TriggeredStorageEvent\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location*\xa2\x01\n\x0bStorageType\x12\x1c\n\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17STORAGE_TYPE_GCS_BUCKET\x10\x01\x12\x1e\n\x1aSTORAGE_TYPE_AWS_S3_BUCKET\x10\x02\x12\x1b\n\x17STORAGE_TYPE_FILESYSTEM\x10\x03\x12\x1b\n\x17STORAGE_TYPE_AZURE_BLOB\x10\x04*\xc8\x01\n\x17StorageSubscriptionType\x12)\n%STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED\x10\x00\x12%\n!STORAGE_SUBSCRIPTION_TYPE_AWS_SNS\x10\x01\x12+\n\'STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB\x10\x02\x12.\n*STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID\x10\x03*V\n\x10StorageEventType\x12\"\n\x1eSTORAGE_EVENT_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTORAGE_EVENT_TYPE_CREATED\x10\x01\x32\xa5\x07\n\x16StorageLocationService\x12N\n\x14ListStorageLocations\x12\x16.google.protobuf.Empty\x1a\x1e.workflows.v1.StorageLocations\x12\x43\n\x12GetStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15\x43reateStorageLocation\x12*.workflows.v1.CreateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15UpdateStorageLocation\x12*.workflows.v1.UpdateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12?\n\x15\x44\x65leteStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12n\n\x19\x43reateStorageSubscription\x12..workflows.v1.CreateStorageSubscriptionRequest\x1a!.workflows.v1.StorageSubscription\x12m\n\x18ListStorageSubscriptions\x12-.workflows.v1.ListStorageSubscriptionsRequest\x1a\".workflows.v1.StorageSubscriptions\x12K\n\x16GetStorageSubscription\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.StorageSubscription\x12\x43\n\x19\x44\x65leteStorageSubscription\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12|\n\x1dListStorageSubscriptionEvents\x12\x32.workflows.v1.ListStorageSubscriptionEventsRequest\x1a\'.workflows.v1.StorageSubscriptionEventsB\x05\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.storage_location_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\222\003\002\010\002' + _globals['_STORAGELOCATION'].fields_by_name['location']._loaded_options = None + _globals['_STORAGELOCATION'].fields_by_name['location']._serialized_options = b'\030\001' + _globals['_STORAGELOCATION'].fields_by_name['type']._loaded_options = None + _globals['_STORAGELOCATION'].fields_by_name['type']._serialized_options = b'\030\001' + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION'].fields_by_name['audience']._loaded_options = None + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION'].fields_by_name['audience']._serialized_options = b'\340A\003' + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_header']._loaded_options = None + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_header']._serialized_options = b'\340A\003' + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret']._loaded_options = None + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret']._serialized_options = b'\340A\003' + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_hash']._loaded_options = None + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_hash']._serialized_options = b'\340A\003' + _globals['_STORAGESUBSCRIPTION'].fields_by_name['id']._loaded_options = None + _globals['_STORAGESUBSCRIPTION'].fields_by_name['id']._serialized_options = b'\340A\003' + _globals['_STORAGESUBSCRIPTION'].fields_by_name['endpoint']._loaded_options = None + _globals['_STORAGESUBSCRIPTION'].fields_by_name['endpoint']._serialized_options = b'\340A\003' + _globals['_STORAGESUBSCRIPTION'].fields_by_name['created_at']._loaded_options = None + _globals['_STORAGESUBSCRIPTION'].fields_by_name['created_at']._serialized_options = b'\340A\003' + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST'].fields_by_name['page']._loaded_options = None + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST'].fields_by_name['page']._serialized_options = b'\252\001\002\010\001' + _globals['_STORAGESUBSCRIPTIONEVENTS'].fields_by_name['next_page']._loaded_options = None + _globals['_STORAGESUBSCRIPTIONEVENTS'].fields_by_name['next_page']._serialized_options = b'\252\001\002\010\001' + _globals['_STORAGETYPE']._serialized_start=4080 + _globals['_STORAGETYPE']._serialized_end=4242 + _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_start=4245 + _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_end=4445 + _globals['_STORAGEEVENTTYPE']._serialized_start=4447 + _globals['_STORAGEEVENTTYPE']._serialized_end=4533 + _globals['_STORAGELOCATION']._serialized_start=194 + _globals['_STORAGELOCATION']._serialized_end=416 + _globals['_STORAGELOCATIONREFERENCE']._serialized_start=419 + _globals['_STORAGELOCATIONREFERENCE']._serialized_end=761 + _globals['_AWSS3BUCKETREFERENCE']._serialized_start=763 + _globals['_AWSS3BUCKETREFERENCE']._serialized_end=833 + _globals['_GCSBUCKETREFERENCE']._serialized_start=835 + _globals['_GCSBUCKETREFERENCE']._serialized_end=938 + _globals['_AZUREBLOBREFERENCE']._serialized_start=941 + _globals['_AZUREBLOBREFERENCE']._serialized_end=1078 + _globals['_FILESYSTEMREFERENCE']._serialized_start=1080 + _globals['_FILESYSTEMREFERENCE']._serialized_end=1121 + _globals['_TRIGGEREDJOB']._serialized_start=1124 + _globals['_TRIGGEREDJOB']._serialized_end=1265 + _globals['_TRIGGEREDJOBS']._serialized_start=1267 + _globals['_TRIGGEREDJOBS']._serialized_end=1349 + _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_start=1351 + _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_end=1471 + _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_start=1473 + _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_end=1587 + _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_start=1589 + _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_end=1645 + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_start=1648 + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_end=1802 + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_start=1805 + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_end=1994 + _globals['_STORAGESUBSCRIPTION']._serialized_start=1997 + _globals['_STORAGESUBSCRIPTION']._serialized_end=2516 + _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_start=2519 + _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_end=2917 + _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_start=2919 + _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_end=3016 + _globals['_STORAGESUBSCRIPTIONS']._serialized_start=3018 + _globals['_STORAGESUBSCRIPTIONS']._serialized_end=3113 + _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_start=3116 + _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_end=3516 + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_start=3519 + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_end=3672 + _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_start=3675 + _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_end=3826 + _globals['_STORAGELOCATIONS']._serialized_start=3828 + _globals['_STORAGELOCATIONS']._serialized_end=3907 + _globals['_TRIGGEREDSTORAGEEVENT']._serialized_start=3910 + _globals['_TRIGGEREDSTORAGEEVENT']._serialized_end=4077 + _globals['_STORAGELOCATIONSERVICE']._serialized_start=4536 + _globals['_STORAGELOCATIONSERVICE']._serialized_end=5469 +# @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi new file mode 100644 index 0000000..3e46926 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi @@ -0,0 +1,260 @@ +from google.api import field_behavior_pb2 as _field_behavior_pb2 +from google.protobuf import empty_pb2 as _empty_pb2 +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 +from tilebox.datasets.tilebox.v1 import query_pb2 as _query_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class StorageType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + STORAGE_TYPE_UNSPECIFIED: _ClassVar[StorageType] + STORAGE_TYPE_GCS_BUCKET: _ClassVar[StorageType] + STORAGE_TYPE_AWS_S3_BUCKET: _ClassVar[StorageType] + STORAGE_TYPE_FILESYSTEM: _ClassVar[StorageType] + STORAGE_TYPE_AZURE_BLOB: _ClassVar[StorageType] + +class StorageSubscriptionType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_AWS_SNS: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID: _ClassVar[StorageSubscriptionType] + +class StorageEventType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + STORAGE_EVENT_TYPE_UNSPECIFIED: _ClassVar[StorageEventType] + STORAGE_EVENT_TYPE_CREATED: _ClassVar[StorageEventType] +STORAGE_TYPE_UNSPECIFIED: StorageType +STORAGE_TYPE_GCS_BUCKET: StorageType +STORAGE_TYPE_AWS_S3_BUCKET: StorageType +STORAGE_TYPE_FILESYSTEM: StorageType +STORAGE_TYPE_AZURE_BLOB: StorageType +STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_AWS_SNS: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID: StorageSubscriptionType +STORAGE_EVENT_TYPE_UNSPECIFIED: StorageEventType +STORAGE_EVENT_TYPE_CREATED: StorageEventType + +class StorageLocation(_message.Message): + __slots__ = ("id", "location", "type", "name", "reference") + ID_FIELD_NUMBER: _ClassVar[int] + LOCATION_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + REFERENCE_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + location: str + type: StorageType + name: str + reference: StorageLocationReference + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., location: _Optional[str] = ..., type: _Optional[_Union[StorageType, str]] = ..., name: _Optional[str] = ..., reference: _Optional[_Union[StorageLocationReference, _Mapping]] = ...) -> None: ... + +class StorageLocationReference(_message.Message): + __slots__ = ("aws_s3_bucket", "gcs_bucket", "azure_blob", "filesystem", "type") + AWS_S3_BUCKET_FIELD_NUMBER: _ClassVar[int] + GCS_BUCKET_FIELD_NUMBER: _ClassVar[int] + AZURE_BLOB_FIELD_NUMBER: _ClassVar[int] + FILESYSTEM_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + aws_s3_bucket: AWSS3BucketReference + gcs_bucket: GCSBucketReference + azure_blob: AzureBlobReference + filesystem: FilesystemReference + type: StorageType + def __init__(self, aws_s3_bucket: _Optional[_Union[AWSS3BucketReference, _Mapping]] = ..., gcs_bucket: _Optional[_Union[GCSBucketReference, _Mapping]] = ..., azure_blob: _Optional[_Union[AzureBlobReference, _Mapping]] = ..., filesystem: _Optional[_Union[FilesystemReference, _Mapping]] = ..., type: _Optional[_Union[StorageType, str]] = ...) -> None: ... + +class AWSS3BucketReference(_message.Message): + __slots__ = ("bucket", "region") + BUCKET_FIELD_NUMBER: _ClassVar[int] + REGION_FIELD_NUMBER: _ClassVar[int] + bucket: str + region: str + def __init__(self, bucket: _Optional[str] = ..., region: _Optional[str] = ...) -> None: ... + +class GCSBucketReference(_message.Message): + __slots__ = ("bucket", "project_id", "location") + BUCKET_FIELD_NUMBER: _ClassVar[int] + PROJECT_ID_FIELD_NUMBER: _ClassVar[int] + LOCATION_FIELD_NUMBER: _ClassVar[int] + bucket: str + project_id: str + location: str + def __init__(self, bucket: _Optional[str] = ..., project_id: _Optional[str] = ..., location: _Optional[str] = ...) -> None: ... + +class AzureBlobReference(_message.Message): + __slots__ = ("storage_account_resource_id", "container", "region") + STORAGE_ACCOUNT_RESOURCE_ID_FIELD_NUMBER: _ClassVar[int] + CONTAINER_FIELD_NUMBER: _ClassVar[int] + REGION_FIELD_NUMBER: _ClassVar[int] + storage_account_resource_id: str + container: str + region: str + def __init__(self, storage_account_resource_id: _Optional[str] = ..., container: _Optional[str] = ..., region: _Optional[str] = ...) -> None: ... + +class FilesystemReference(_message.Message): + __slots__ = ("path",) + PATH_FIELD_NUMBER: _ClassVar[int] + path: str + def __init__(self, path: _Optional[str] = ...) -> None: ... + +class TriggeredJob(_message.Message): + __slots__ = ("job_id", "automation_id", "glob_pattern") + JOB_ID_FIELD_NUMBER: _ClassVar[int] + AUTOMATION_ID_FIELD_NUMBER: _ClassVar[int] + GLOB_PATTERN_FIELD_NUMBER: _ClassVar[int] + job_id: _id_pb2.ID + automation_id: _id_pb2.ID + glob_pattern: str + def __init__(self, job_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., automation_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., glob_pattern: _Optional[str] = ...) -> None: ... + +class TriggeredJobs(_message.Message): + __slots__ = ("triggered_jobs",) + TRIGGERED_JOBS_FIELD_NUMBER: _ClassVar[int] + triggered_jobs: _containers.RepeatedCompositeFieldContainer[TriggeredJob] + def __init__(self, triggered_jobs: _Optional[_Iterable[_Union[TriggeredJob, _Mapping]]] = ...) -> None: ... + +class CreateStorageLocationRequest(_message.Message): + __slots__ = ("name", "reference") + NAME_FIELD_NUMBER: _ClassVar[int] + REFERENCE_FIELD_NUMBER: _ClassVar[int] + name: str + reference: StorageLocationReference + def __init__(self, name: _Optional[str] = ..., reference: _Optional[_Union[StorageLocationReference, _Mapping]] = ...) -> None: ... + +class UpdateStorageLocationRequest(_message.Message): + __slots__ = ("storage_location_id", "name") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + name: str + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., name: _Optional[str] = ...) -> None: ... + +class AWSSNSStorageSubscription(_message.Message): + __slots__ = ("topic_arn",) + TOPIC_ARN_FIELD_NUMBER: _ClassVar[int] + topic_arn: str + def __init__(self, topic_arn: _Optional[str] = ...) -> None: ... + +class GooglePubSubStorageSubscription(_message.Message): + __slots__ = ("subscription", "service_account_email", "audience") + SUBSCRIPTION_FIELD_NUMBER: _ClassVar[int] + SERVICE_ACCOUNT_EMAIL_FIELD_NUMBER: _ClassVar[int] + AUDIENCE_FIELD_NUMBER: _ClassVar[int] + subscription: str + service_account_email: str + audience: str + def __init__(self, subscription: _Optional[str] = ..., service_account_email: _Optional[str] = ..., audience: _Optional[str] = ...) -> None: ... + +class AzureEventGridStorageSubscription(_message.Message): + __slots__ = ("webhook_secret_header", "webhook_secret", "webhook_secret_hash") + WEBHOOK_SECRET_HEADER_FIELD_NUMBER: _ClassVar[int] + WEBHOOK_SECRET_FIELD_NUMBER: _ClassVar[int] + WEBHOOK_SECRET_HASH_FIELD_NUMBER: _ClassVar[int] + webhook_secret_header: str + webhook_secret: str + webhook_secret_hash: bytes + def __init__(self, webhook_secret_header: _Optional[str] = ..., webhook_secret: _Optional[str] = ..., webhook_secret_hash: _Optional[bytes] = ...) -> None: ... + +class StorageSubscription(_message.Message): + __slots__ = ("id", "storage_location_id", "type", "endpoint", "created_at", "aws_sns", "google_pubsub", "azure_event_grid") + ID_FIELD_NUMBER: _ClassVar[int] + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + ENDPOINT_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + AWS_SNS_FIELD_NUMBER: _ClassVar[int] + GOOGLE_PUBSUB_FIELD_NUMBER: _ClassVar[int] + AZURE_EVENT_GRID_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + storage_location_id: _id_pb2.ID + type: StorageSubscriptionType + endpoint: str + created_at: _timestamp_pb2.Timestamp + aws_sns: AWSSNSStorageSubscription + google_pubsub: GooglePubSubStorageSubscription + azure_event_grid: AzureEventGridStorageSubscription + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageSubscriptionType, str]] = ..., endpoint: _Optional[str] = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., aws_sns: _Optional[_Union[AWSSNSStorageSubscription, _Mapping]] = ..., google_pubsub: _Optional[_Union[GooglePubSubStorageSubscription, _Mapping]] = ..., azure_event_grid: _Optional[_Union[AzureEventGridStorageSubscription, _Mapping]] = ...) -> None: ... + +class CreateStorageSubscriptionRequest(_message.Message): + __slots__ = ("storage_location_id", "type", "aws_sns", "google_pubsub", "azure_event_grid") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + AWS_SNS_FIELD_NUMBER: _ClassVar[int] + GOOGLE_PUBSUB_FIELD_NUMBER: _ClassVar[int] + AZURE_EVENT_GRID_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + type: StorageSubscriptionType + aws_sns: AWSSNSStorageSubscription + google_pubsub: GooglePubSubStorageSubscription + azure_event_grid: AzureEventGridStorageSubscription + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageSubscriptionType, str]] = ..., aws_sns: _Optional[_Union[AWSSNSStorageSubscription, _Mapping]] = ..., google_pubsub: _Optional[_Union[GooglePubSubStorageSubscription, _Mapping]] = ..., azure_event_grid: _Optional[_Union[AzureEventGridStorageSubscription, _Mapping]] = ...) -> None: ... + +class ListStorageSubscriptionsRequest(_message.Message): + __slots__ = ("storage_location_id",) + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ...) -> None: ... + +class StorageSubscriptions(_message.Message): + __slots__ = ("subscriptions",) + SUBSCRIPTIONS_FIELD_NUMBER: _ClassVar[int] + subscriptions: _containers.RepeatedCompositeFieldContainer[StorageSubscription] + def __init__(self, subscriptions: _Optional[_Iterable[_Union[StorageSubscription, _Mapping]]] = ...) -> None: ... + +class StorageSubscriptionEvent(_message.Message): + __slots__ = ("id", "object_key", "type", "event_time", "received_at", "triggered_jobs", "storage_subscription_id") + ID_FIELD_NUMBER: _ClassVar[int] + OBJECT_KEY_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + EVENT_TIME_FIELD_NUMBER: _ClassVar[int] + RECEIVED_AT_FIELD_NUMBER: _ClassVar[int] + TRIGGERED_JOBS_FIELD_NUMBER: _ClassVar[int] + STORAGE_SUBSCRIPTION_ID_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + object_key: str + type: StorageEventType + event_time: _timestamp_pb2.Timestamp + received_at: _timestamp_pb2.Timestamp + triggered_jobs: _containers.RepeatedCompositeFieldContainer[TriggeredJob] + storage_subscription_id: _id_pb2.ID + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., object_key: _Optional[str] = ..., type: _Optional[_Union[StorageEventType, str]] = ..., event_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., received_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., triggered_jobs: _Optional[_Iterable[_Union[TriggeredJob, _Mapping]]] = ..., storage_subscription_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ...) -> None: ... + +class ListStorageSubscriptionEventsRequest(_message.Message): + __slots__ = ("storage_location_id", "page") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + page: _query_pb2.Pagination + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + +class StorageSubscriptionEvents(_message.Message): + __slots__ = ("events", "next_page") + EVENTS_FIELD_NUMBER: _ClassVar[int] + NEXT_PAGE_FIELD_NUMBER: _ClassVar[int] + events: _containers.RepeatedCompositeFieldContainer[StorageSubscriptionEvent] + next_page: _query_pb2.Pagination + def __init__(self, events: _Optional[_Iterable[_Union[StorageSubscriptionEvent, _Mapping]]] = ..., next_page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + +class StorageLocations(_message.Message): + __slots__ = ("locations",) + LOCATIONS_FIELD_NUMBER: _ClassVar[int] + locations: _containers.RepeatedCompositeFieldContainer[StorageLocation] + def __init__(self, locations: _Optional[_Iterable[_Union[StorageLocation, _Mapping]]] = ...) -> None: ... + +class TriggeredStorageEvent(_message.Message): + __slots__ = ("storage_location_id", "type", "location") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + LOCATION_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + type: StorageEventType + location: str + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageEventType, str]] = ..., location: _Optional[str] = ...) -> None: ... diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py new file mode 100644 index 0000000..8ff71ea --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py @@ -0,0 +1,479 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 + + +class StorageLocationServiceStub: + """StorageLocationService manages storage locations, notification subscriptions, and received events. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListStorageLocations = channel.unary_unary( + '/workflows.v1.StorageLocationService/ListStorageLocations', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocations.FromString, + _registered_method=True) + self.GetStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/GetStorageLocation', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + _registered_method=True) + self.CreateStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/CreateStorageLocation', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + _registered_method=True) + self.UpdateStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/UpdateStorageLocation', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + _registered_method=True) + self.DeleteStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/DeleteStorageLocation', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + _registered_method=True) + self.CreateStorageSubscription = channel.unary_unary( + '/workflows.v1.StorageLocationService/CreateStorageSubscription', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + _registered_method=True) + self.ListStorageSubscriptions = channel.unary_unary( + '/workflows.v1.StorageLocationService/ListStorageSubscriptions', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions.FromString, + _registered_method=True) + self.GetStorageSubscription = channel.unary_unary( + '/workflows.v1.StorageLocationService/GetStorageSubscription', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + _registered_method=True) + self.DeleteStorageSubscription = channel.unary_unary( + '/workflows.v1.StorageLocationService/DeleteStorageSubscription', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + _registered_method=True) + self.ListStorageSubscriptionEvents = channel.unary_unary( + '/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents.FromString, + _registered_method=True) + + +class StorageLocationServiceServicer: + """StorageLocationService manages storage locations, notification subscriptions, and received events. + """ + + def ListStorageLocations(self, request, context): + """ListStorageLocations lists storage locations available for automation triggers. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetStorageLocation(self, request, context): + """GetStorageLocation gets a storage location by its ID. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateStorageLocation(self, request, context): + """CreateStorageLocation registers an existing bucket, container, or directory for automation triggers. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UpdateStorageLocation(self, request, context): + """UpdateStorageLocation changes the display name of a storage location. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteStorageLocation(self, request, context): + """DeleteStorageLocation removes the location and its subscriptions from Tilebox without deleting cloud resources. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateStorageSubscription(self, request, context): + """CreateStorageSubscription registers notification delivery and returns the provider setup values. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListStorageSubscriptions(self, request, context): + """ListStorageSubscriptions lists notification subscriptions for a storage location. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetStorageSubscription(self, request, context): + """GetStorageSubscription gets a subscription by its ID. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteStorageSubscription(self, request, context): + """DeleteStorageSubscription stops accepting notifications at its endpoint without deleting cloud resources. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListStorageSubscriptionEvents(self, request, context): + """ListStorageSubscriptionEvents lists received object notifications and the jobs they triggered. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_StorageLocationServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListStorageLocations': grpc.unary_unary_rpc_method_handler( + servicer.ListStorageLocations, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocations.SerializeToString, + ), + 'GetStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.GetStorageLocation, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.SerializeToString, + ), + 'CreateStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.CreateStorageLocation, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.SerializeToString, + ), + 'UpdateStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.UpdateStorageLocation, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.SerializeToString, + ), + 'DeleteStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.DeleteStorageLocation, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'CreateStorageSubscription': grpc.unary_unary_rpc_method_handler( + servicer.CreateStorageSubscription, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.SerializeToString, + ), + 'ListStorageSubscriptions': grpc.unary_unary_rpc_method_handler( + servicer.ListStorageSubscriptions, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions.SerializeToString, + ), + 'GetStorageSubscription': grpc.unary_unary_rpc_method_handler( + servicer.GetStorageSubscription, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.SerializeToString, + ), + 'DeleteStorageSubscription': grpc.unary_unary_rpc_method_handler( + servicer.DeleteStorageSubscription, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'ListStorageSubscriptionEvents': grpc.unary_unary_rpc_method_handler( + servicer.ListStorageSubscriptionEvents, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'workflows.v1.StorageLocationService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('workflows.v1.StorageLocationService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class StorageLocationService: + """StorageLocationService manages storage locations, notification subscriptions, and received events. + """ + + @staticmethod + def ListStorageLocations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/ListStorageLocations', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocations.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/GetStorageLocation', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def CreateStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/CreateStorageLocation', + workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def UpdateStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/UpdateStorageLocation', + workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def DeleteStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/DeleteStorageLocation', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + google_dot_protobuf_dot_empty__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def CreateStorageSubscription(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/CreateStorageSubscription', + workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListStorageSubscriptions(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/ListStorageSubscriptions', + workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetStorageSubscription(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/GetStorageSubscription', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def DeleteStorageSubscription(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/DeleteStorageSubscription', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + google_dot_protobuf_dot_empty__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListStorageSubscriptionEvents(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents', + workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/uv.lock b/uv.lock index 9f1d507..ade3ce2 100644 --- a/uv.lock +++ b/uv.lock @@ -121,6 +121,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "azure-core" +version = "1.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/87/e88098a024d74b7434de5b110c261da024fe5d6c139f1847f6f8a2b76678/azure_identity-1.26.0.tar.gz", hash = "sha256:61b4538b87df10d2d6d44b77cd5ea28a8929fd4929d58b45bf3f230f7daba02e", size = 310151, upload-time = "2026-10-01T14:17:47.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/d5/ae45ce8009d0d015304b206c7c4a6cbab02c03f8f07eedefb32b099ccee3/azure_identity-1.26.0-py3-none-any.whl", hash = "sha256:3c60d9682b3ac01507388d474cb5a5ffbe54152a96b6cdcaa7013da9315cf186", size = 195958, upload-time = "2026-10-01T14:17:49.705Z" }, +] + [[package]] name = "boto3" version = "1.43.101" @@ -751,23 +780,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] -[[package]] -name = "google-api-core" -version = "2.38.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/59/e5/18aeff14213db86267a0f79d869352954394ab4ab3a3866d9b9b0e82dc6a/google_api_core-2.38.0.tar.gz", hash = "sha256:31e326eafa31b34f1db7a715f50f61dbca7e6e37277762f252ed90e6ce94c246", size = 206784, upload-time = "2026-09-17T20:30:08.766Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/55/e545e4b0040cb812f65d2abc545c6420f57b687d2af130a447c86d14d00b/google_api_core-2.38.0-py3-none-any.whl", hash = "sha256:8db2730375eba434bd75041fbe06b5374287407689ce132ac5650c8c8f042fcc", size = 187996, upload-time = "2026-09-17T20:29:32.813Z" }, -] - [[package]] name = "google-auth" version = "2.58.0" @@ -781,77 +793,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/13/477d90d09591b3938b45c4e11f4d8a51291682112cb5efcac961e815d562/google_auth-2.58.0-py3-none-any.whl", hash = "sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d", size = 262404, upload-time = "2026-09-09T20:49:33.951Z" }, ] -[[package]] -name = "google-cloud-core" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f9/54/790548b190cff9cf07225c811d2668eacd8f2cbf04114475cbe3e5738752/google_cloud_core-2.7.0.tar.gz", hash = "sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d", size = 36603, upload-time = "2026-08-25T19:18:43.076Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/64/904dbc9bee128e7b87eeb60da67c8fa4d1a8acdc9a45dd2fedca67ef184d/google_cloud_core-2.7.0-py3-none-any.whl", hash = "sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1", size = 31046, upload-time = "2026-08-24T21:55:28.36Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "3.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, +[package.optional-dependencies] +requests = [ { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/06/33e124df40437c292c7d666616d890c12cfea5b83842acf5be7dc3fde016/google_cloud_storage-3.14.1.tar.gz", hash = "sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7", size = 17348933, upload-time = "2026-09-08T17:10:11.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/65/7201c1816ae46e0b4e70a3ce61e5121963315dc9df2e00fcb8897a3d0926/google_cloud_storage-3.14.1-py3-none-any.whl", hash = "sha256:8f0fe2b74bd80ddcbd67247a3b47b81035a0a4f448569ed358fa40a46c940118", size = 343228, upload-time = "2026-09-08T17:10:09.833Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" }, - { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" }, - { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" }, - { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, - { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, - { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, - { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, - { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, - { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, - { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" }, - { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/17/4b/44e128cd7b0e72cd26da4a3052cd8a9ef1b5a890be7c611558067ea0b354/google_resumable_media-2.10.2.tar.gz", hash = "sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c", size = 2164918, upload-time = "2026-08-25T19:19:11.814Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/55/30/3e976681b5319715ab022363f58f6f8e10f546a1777756fe3ebd918857e4/google_resumable_media-2.10.2-py3-none-any.whl", hash = "sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7", size = 81531, upload-time = "2026-08-25T19:18:07.193Z" }, -] [[package]] name = "googleapis-common-protos" @@ -1363,6 +1308,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/80/ee7ace7a49bd525455497fa2ff198111877d6f93b908b657adeb2cf057a4/moto-5.2.3-py3-none-any.whl", hash = "sha256:5ec31b3cdbb383a19d46030157bd82aaaa7a9cc5f564200f995663d347bd4728", size = 6780972, upload-time = "2026-08-22T18:45:55.719Z" }, ] +[[package]] +name = "msal" +version = "1.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/85/747d28986a44b715cc51cf57aa021c8fd94e9614064261ceec77320e8b0b/msal-1.39.0.tar.gz", hash = "sha256:6ab7de335e6d7f5717e2c7e1dbf86e4dda2f6acf3c56773b78dc53ebc6395b5f", size = 205360, upload-time = "2026-09-17T16:07:45.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/32/178457385925ce301c7b3232bd85665d4df57da39568b62dcfc74031873f/msal-1.39.0-py3-none-any.whl", hash = "sha256:2d2577886906cd7293850dffa2da29119966c213bfc6ec0cecf8bf7621e1ca77", size = 131629, upload-time = "2026-09-17T16:07:46.722Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, +] + [[package]] name = "msgspec" version = "0.21.1" @@ -1948,18 +1919,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, ] -[[package]] -name = "proto-plus" -version = "1.28.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/40/a6/4fbadcc2044034449b3f8f0ce82dcf3005d53f37c136642103fd4836a31c/proto_plus-1.28.4.tar.gz", hash = "sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63", size = 58679, upload-time = "2026-08-25T19:19:15.102Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/5d/0f04b85dafdc3250ced7f2592efc17dce7f40712e941e9632202481e600d/proto_plus-1.28.4-py3-none-any.whl", hash = "sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa", size = 50797, upload-time = "2026-08-25T19:18:12.338Z" }, -] - [[package]] name = "protobuf" version = "7.36.2" @@ -2151,6 +2110,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyproj" version = "3.7.2" @@ -2875,8 +2848,10 @@ dev = [ name = "tilebox-workflows" source = { editable = "tilebox-workflows" } dependencies = [ + { name = "azure-identity" }, { name = "boto3" }, - { name = "google-cloud-storage" }, + { name = "google-auth", extra = ["requests"] }, + { name = "grpcio" }, { name = "msgspec" }, { name = "obstore" }, { name = "opentelemetry-api" }, @@ -2916,8 +2891,10 @@ dev = [ [package.metadata] requires-dist = [ { name = "affine", marker = "extra == 'geospatial'", specifier = ">=2" }, + { name = "azure-identity", specifier = ">=1.23" }, { name = "boto3", specifier = ">=1.40.2" }, - { name = "google-cloud-storage", specifier = ">=2.10" }, + { name = "google-auth", extras = ["requests"], specifier = ">=2.29" }, + { name = "grpcio", specifier = ">=1.84.0" }, { name = "ipywidgets", marker = "extra == 'notebook'", specifier = ">=8.1.7" }, { name = "msgspec", specifier = ">=0.19" }, { name = "obstore", specifier = ">=0.8.2" },