diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b06fb..677a51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support Python 3.14 across all packages. - Test the lowest compatible direct dependencies on Python 3.11 in CI. +- `tilebox-workflows`: Read Azure blobs in storage-event tasks. Use the account and container returned by the API. + Authenticate with Azure Identity, or use an account key or SAS token configured in the environment. +- `tilebox-workflows`: Add `AzureBlobCache(account_name, container, prefix="jobs")` with the same authentication + as Azure automation reads. Cache groups share the store and credentials. ### Changed +- `tilebox-workflows`: List storage locations through `StorageLocationService`. Read S3, GCS, and Azure objects + through obstore, with boto3, Google Auth, and Azure Identity handling credentials. Cloud reads now raise + obstore/Python exceptions instead of AWS/GCP SDK exceptions. Missing objects raise `FileNotFoundError`. +- `tilebox-workflows`: Replace the Google storage SDK dependency with `google-auth[requests]` and add `azure-identity`. + Import cloud authentication libraries only when needed and reuse cloud storage clients between automation reads. +- `tilebox-workflows`: Use obstore and Google Auth for `GoogleStorageCache`. Pass a bucket name instead of a + Google SDK bucket object. Cache groups share the same store and credentials. +- `tilebox-workflows`: Use obstore for `AmazonS3Cache`, with the same boto3 session credentials as automation reads. + Cache groups share the store and credentials instead of creating new S3 clients. - Require Python 3.11 or newer across all packages, removing Python 3.10 compatibility code and typing backfills. - `tilebox-storage`: Replace legacy synchronous client patching with explicit wrappers using `asyncio.run()`. When called inside a running event loop (including notebooks), run the operation in a worker thread instead. Remove the internal `syncify` helper and the `nest-asyncio2` dependency from `tilebox-grpc`. - Raise dependency minimums to remove obsolete compatibility workarounds: boto3 1.40.2, OpenTelemetry 1.43.0 - (logging instrumentation 0.64b0), grpcio 1.84.0, and pyqwest 0.7.0. + (logging instrumentation 0.64b0), and pyqwest 0.7.0. - `tilebox-datasets`: Require NumPy 1.25, pandas 2.2.2, xarray 2024.6, and Shapely 2.0.6 or newer. - Require protobuf 6.31.0 for repeated-field descriptors and protobuf-py 0.2.0 for integer-to-float serialization. - `tilebox-workflows`: Make interactive notebook progress an optional `notebook` extra; without it, jobs display @@ -29,6 +42,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `tilebox-workflows`: Use S3 bucket regions from API metadata or discover them with `HeadBucket`. + `AmazonS3Cache` also accepts `region=` to skip discovery. Let botocore refresh temporary credentials. +- `tilebox-workflows`: Preserve cloud cache permission, network, and backend errors instead of reporting cache misses. +- Require `grpcio>=1.84.0` for the memory-exhaustion fixes in + [GHSA-hf3w-6hpw-qp67](https://github.com/grpc/grpc/security/advisories/GHSA-hf3w-6hpw-qp67). + macOS CLI authentication may still emit native fork diagnostics during successful reads. +- `tilebox-workflows`: Fix GCS automation reads when the API returns a bucket name without a project prefix. - Explicitly trust system certificates for Connect HTTP/1 transports with pyqwest 0.7 and newer. ## [0.62.0] - 2026-09-16 diff --git a/tilebox-grpc/_tilebox/grpc/replay.py b/tilebox-grpc/_tilebox/grpc/replay.py index 1417162..149306e 100644 --- a/tilebox-grpc/_tilebox/grpc/replay.py +++ b/tilebox-grpc/_tilebox/grpc/replay.py @@ -24,6 +24,7 @@ from grpc.aio import ( AioRpcError, ClientCallDetails, # import from aio, since grpc.ClientCallDetails is an empty base class + Metadata, ) RequestType = TypeVar("RequestType") @@ -163,7 +164,7 @@ def unary_unary_call( if recorded_status != StatusCode.OK.value[0]: # the recorded call was an error, so raise it again code = _STATUS_CODES[recorded_status] - error = AioRpcError(code, None, None, recorded_response.decode()) + error = AioRpcError(code, Metadata(), Metadata(), recorded_response.decode()) raise error return response_deserializer(base64.b64decode(recorded_response)) diff --git a/tilebox-grpc/pyproject.toml b/tilebox-grpc/pyproject.toml index 674703c..c26142b 100644 --- a/tilebox-grpc/pyproject.toml +++ b/tilebox-grpc/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - # Rate-limits the logging noise from https://github.com/grpc/grpc/issues/42293. + # Includes fixes for https://github.com/grpc/grpc/security/advisories/GHSA-hf3w-6hpw-qp67. "grpcio>=1.84.0", # connectrpc 0.11 switches from Google's protobuf runtime to protobuf-py "connectrpc>=0.10.1,<0.11.0", diff --git a/tilebox-grpc/tests/test_fork_logging.py b/tilebox-grpc/tests/test_fork_logging.py new file mode 100644 index 0000000..ebd9475 --- /dev/null +++ b/tilebox-grpc/tests/test_fork_logging.py @@ -0,0 +1,73 @@ +"""Check native gRPC diagnostics when a CLI starts during active RPCs. + +Run on macOS to check RPCs and report the known fork-log issue: + +uv run --isolated --no-project --with grpcio==1.84.0 --with pytest pytest -c /dev/null -p no:cacheprovider -rx tilebox-grpc/tests/test_fork_logging.py + +RPC failures fail the test. Known fork diagnostics produce XFAIL after the RPC checks pass. +Linux may use vfork and not reproduce the diagnostics. This check uses no cloud credentials. +""" + +import os +import subprocess +import sys +from textwrap import dedent + +import pytest + + +# RPCs must survive CLI subprocess launches; known fork-log noise is reported separately. +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not use POSIX fork handlers") +def test_cli_spawn_during_rpcs() -> None: + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-c", + dedent(""" + import subprocess + import sys + from concurrent.futures import ThreadPoolExecutor + from threading import Event + + import grpc + + with ThreadPoolExecutor(max_workers=4) as executor: + server = grpc.server(executor) + server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler( + "probe", {"echo": grpc.unary_unary_rpc_method_handler(lambda request, context: request)} + ),)) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + stop = Event() + try: + with grpc.insecure_channel(f"127.0.0.1:{port}") as channel: + echo = channel.unary_unary("/probe/echo") + assert echo(b"before", timeout=5) == b"before" + + def poll(): + while not stop.is_set(): + assert echo(b"during", timeout=5) == b"during" + + poller = executor.submit(poll) + try: + for _ in range(25): + # Azure CLI authentication supplies cwd, which prevents posix_spawn. + subprocess.run([sys.executable, "-c", "pass"], cwd=".", check=True, timeout=5) + finally: + stop.set() + poller.result(timeout=10) + assert echo(b"after", timeout=5) == b"after" + finally: + stop.set() + server.stop(0).wait(10) + """), + ], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "GRPC_VERBOSITY": "INFO"}, + check=False, + ) + assert result.returncode == 0, result.stderr + if "FD from fork parent still in poll list" in result.stderr: + pytest.xfail("Known gRPC fork diagnostics: https://github.com/grpc/grpc/issues/42293") diff --git a/tilebox-workflows/README.md b/tilebox-workflows/README.md index 6d079d7..1db0107 100644 --- a/tilebox-workflows/README.md +++ b/tilebox-workflows/README.md @@ -90,6 +90,19 @@ unless the cache implementation explicitly provides that guarantee. Concurrency in one runtime avoids repeated process initialization and allows overlapping I/O or native code that releases Python's GIL. CPU-bound Python code still needs multiple runtime processes for parallel execution. +## Reading automation objects + +Storage-event tasks can read objects from Amazon S3, Google Cloud Storage, Azure Blob Storage, or the local filesystem: + +```python +content = self.trigger.storage.read(self.trigger.location) +``` + +The method returns bytes. Cloud reads use the credentials configured for the runner. + +On macOS, GCS or Azure CLI authentication can emit [gRPC fork diagnostics](https://github.com/grpc/grpc/issues/42293) +even when a read succeeds. These messages alone do not indicate a failed read; do not downgrade gRPC to hide them. + ## Documentation Check out the [Tilebox Workflows documentation](https://docs.tilebox.com/workflows/introduction) for more information. diff --git a/tilebox-workflows/pyproject.toml b/tilebox-workflows/pyproject.toml index 98258dd..19d436a 100644 --- a/tilebox-workflows/pyproject.toml +++ b/tilebox-workflows/pyproject.toml @@ -23,7 +23,10 @@ requires-python = ">=3.11" dependencies = [ "tilebox-datasets>=0.56.0", "tilebox-grpc>=0.28.0", - "google-cloud-storage>=2.10", + # Require the security fixes even with an older tilebox-grpc release. + "grpcio>=1.84.0", + "google-auth[requests]>=2.29", + "azure-identity>=1.23", "opentelemetry-api>=1.43.0", "opentelemetry-exporter-otlp-proto-http>=1.43.0", "opentelemetry-sdk>=1.43.0", diff --git a/tilebox-workflows/tests/automations/test_storage_read.py b/tilebox-workflows/tests/automations/test_storage_read.py new file mode 100644 index 0000000..76fff6d --- /dev/null +++ b/tilebox-workflows/tests/automations/test_storage_read.py @@ -0,0 +1,526 @@ +import json +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from unittest.mock import MagicMock, patch +from uuid import UUID + +import boto3 +import obstore +import pytest +from azure.core.credentials import AccessToken +from botocore.credentials import RefreshableCredentials +from botocore.exceptions import ClientError +from botocore.stub import Stubber +from google.auth.exceptions import DefaultCredentialsError +from obstore.store import AzureStore, MemoryStore, S3Store + +from tilebox.datasets.uuid import uuid_to_uuid_message +from tilebox.workflows.automations import StorageEventTask +from tilebox.workflows.cache import AmazonS3Cache +from tilebox.workflows.client import Client +from tilebox.workflows.data import ( + AzureStorageLocation, + RunnerContext, + S3StorageLocation, + StorageLocation, + StorageType, + _default_azure_storage_client, + _default_gcs_storage_client, + _default_s3_storage_client, +) +from tilebox.workflows.observability.tracing import NoopWorkflowTracer +from tilebox.workflows.workflows.v1 import storage_location_pb2 as storage_pb + +_ID = UUID("01a0fd71-8262-2fde-7283-43dab469c23f") +_ACCOUNT = "/subscriptions/test/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/sourceaccount" + + +@pytest.fixture(autouse=True) +def isolated_storage_clients() -> Iterator[None]: + clients = (_default_gcs_storage_client, _default_s3_storage_client, _default_azure_storage_client) + for client in clients: + client.cache_clear() + yield + for client in clients: + client.cache_clear() + + +@pytest.mark.parametrize(("method", "store_class"), [("gcs_client", "GCSStore"), ("s3_client", "S3Store")]) +def test_cloud_client_cache(method: str, store_class: str, monkeypatch: pytest.MonkeyPatch) -> None: + # GCS and S3 reuse a store for the same bucket and keep different buckets separate. + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + with ( + patch("google.auth.default", return_value=(MagicMock(), "project")), + patch(f"obstore.store.{store_class}", side_effect=[MemoryStore(), MemoryStore()]) as constructor, + ): + client = getattr(RunnerContext(), method) + if method == "s3_client": + client = partial(client, region="us-east-1") + first = client("first-bucket") + assert client("first-bucket") is first + assert client("other-bucket") is not first + assert constructor.call_count == 2 + + +def test_gcs_adc_project_lookup_is_not_repeated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Repeated reads reuse ADC credentials instead of launching gcloud to rediscover the project. + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + monkeypatch.setenv("CLOUDSDK_CONFIG", str(tmp_path)) + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "explicit-project") + (tmp_path / "application_default_credentials.json").write_text( + json.dumps( + {"type": "authorized_user", "client_id": "test-client", "client_secret": "test", "refresh_token": "test"} + ) + ) + store = MemoryStore() + obstore.put(store, "first.txt", b"first") + obstore.put(store, "second.txt", b"second") + location = StorageLocation(_ID, "adc-bucket", StorageType.GCS) + with ( + patch("subprocess.check_output", return_value=b"discovered-project\n") as command, + patch("obstore.store.GCSStore", return_value=store) as constructor, + ): + assert location.read("first.txt") == b"first" + assert location.read("second.txt") == b"second" + command.assert_called_once() + assert command.call_args.args[0] == ("gcloud", "config", "get", "project") + constructor.assert_called_once() + + +def test_gcs_discovery_error_is_not_hidden_or_cached() -> None: + # Credential discovery failures reach the caller and a later read can retry discovery. + store = MemoryStore() + obstore.put(store, "file.txt", b"retried") + location = StorageLocation(_ID, "retry-bucket", StorageType.GCS) + with ( + patch("google.auth.default", side_effect=[DefaultCredentialsError("missing credentials"), (MagicMock(), None)]), + patch("obstore.store.GCSStore", return_value=store), + ): + with pytest.raises(DefaultCredentialsError, match="missing credentials"): + location.read("file.txt") + assert location.read("file.txt") == b"retried" + + +def _azure_message() -> storage_pb.StorageLocation: + return storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), + location="container-only", + type=storage_pb.STORAGE_TYPE_AZURE_BLOB, + reference=storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AZURE_BLOB, + azure_blob=storage_pb.AzureBlobReference(storage_account_resource_id=_ACCOUNT, container="sourcecontainer"), + ), + ) + + +@pytest.mark.parametrize("storage_type", [StorageType.GCS, StorageType.S3, StorageType.FS]) +def test_mixed_storage_location_ordering(storage_type: StorageType) -> None: + # Azure and other storage locations sort together by ID before their bucket or container names. + plain = StorageLocation(UUID(int=1), "z", storage_type) + azure = AzureStorageLocation(UUID(int=2), "a", StorageType.AZURE, storage_account_resource_id=_ACCOUNT) + assert plain < azure + assert azure > plain + assert sorted([azure, plain]) == [plain, azure] + + +def test_azure_trigger_read() -> None: + # Deserialized Azure tasks read from the account and container supplied by the runner context. + location = StorageLocation.from_message(_azure_message()) + assert isinstance(location, AzureStorageLocation) + assert location.type == StorageType.AZURE + assert location.location == "sourcecontainer" + assert location.storage_account_resource_id == _ACCOUNT + assert StorageLocation.from_message(location.to_message()) == location + + store = MemoryStore() + key = "nested/hello + %20 ü.txt" + expected = "Hello Azure! ü\n".encode() + obstore.put(store, key, expected) + context = RunnerContext(storage_locations=[location]) + task = StorageEventTask().once(location, key) + restored = StorageEventTask._deserialize(task._serialize(), context) + assert isinstance(restored.trigger.storage, AzureStorageLocation) + assert restored.trigger.storage.storage_account_resource_id == _ACCOUNT + assert restored.trigger.storage.runner_context is context + with patch.object(context, "azure_client", return_value=store) as client: + assert restored.trigger.storage.read(restored.trigger.location) == expected + client.assert_called_once_with(_ACCOUNT, "sourcecontainer") + + +def test_azure_client_coordinates_override_environment(monkeypatch: pytest.MonkeyPatch) -> None: + # The account from API metadata takes precedence over the account configured in the environment. + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "wrongaccount") + store = RunnerContext().azure_client(_ACCOUNT, "explicit-container") + assert store.config["account_name"] == "sourceaccount" + assert store.config["container_name"] == "explicit-container" + + +def test_azure_client_cache_separates_accounts_and_containers() -> None: + # Azure clients are reused only when both the account and container match. + context = RunnerContext() + first = context.azure_client(_ACCOUNT, "one") + assert context.azure_client(_ACCOUNT, "one") is first + assert context.azure_client(_ACCOUNT, "two") is not first + other = context.azure_client(_ACCOUNT.replace("sourceaccount", "otheraccount"), "one") + assert other is not first + assert other.config["account_name"] == "otheraccount" + + +def test_azure_identity_credentials() -> None: + # Azure Identity supplies a storage-scoped token and its expiry to obstore. + with ( + patch("azure.identity.DefaultAzureCredential") as credential, + patch("obstore.store.AzureStore", wraps=AzureStore) as constructor, + ): + credential.return_value.get_token.return_value = AccessToken("test-token", 2000000000) + RunnerContext().azure_client(_ACCOUNT, "identity-test") + provider = constructor.call_args.kwargs["credential_provider"] + assert provider() == {"token": "test-token", "expires_at": datetime.fromtimestamp(2000000000, UTC)} + credential.return_value.get_token.assert_called_once_with("https://storage.azure.com/.default", tenant_id=None) + credential.assert_called_once_with() + + +@pytest.mark.parametrize( + ("variable", "value"), + [ + ("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdA=="), + ("AZURE_STORAGE_ACCESS_KEY", "dGVzdA=="), + ("AZURE_STORAGE_MASTER_KEY", "dGVzdA=="), + ("AZURE_STORAGE_SAS_KEY", "sv=1&sig=test"), + ("AZURE_STORAGE_SAS_TOKEN", "sv=1&sig=test"), + ], +) +def test_azure_explicit_credentials(variable: str, value: str, monkeypatch: pytest.MonkeyPatch) -> None: + # Explicit keys and SAS tokens use native obstore authentication without constructing Azure Identity. + monkeypatch.setenv(variable, value) + with ( + patch("azure.identity.DefaultAzureCredential") as credential, + patch("obstore.store.AzureStore", wraps=AzureStore) as constructor, + ): + container = variable.lower().replace("_", "-") + RunnerContext().azure_client(_ACCOUNT, container) + constructor.assert_called_once_with(container, account_name="sourceaccount") + credential.assert_not_called() + + +def test_azure_read_requires_account_metadata() -> None: + # Missing or malformed Azure account resource IDs raise a clear error before a cloud request. + location = StorageLocation(_ID, "container-only", StorageType.AZURE) + with pytest.raises(ValueError, match="missing its storage account resource ID"): + location.read("file.txt") + with pytest.raises(ValueError, match="Invalid Azure storage account resource ID"): + RunnerContext().azure_client("https://sourceaccount.blob.core.windows.net", "container") + + +@pytest.mark.parametrize("metadata", ["bare", "reference"]) +def test_gcs_trigger_read(metadata: str) -> None: + # GCS tasks read bare bucket names using scoped google-auth credentials from either API metadata form. + bucket = "storage-automation-test-gcp-2aa460e" + message = storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), location=bucket, type=storage_pb.STORAGE_TYPE_GCS_BUCKET + ) + if metadata == "reference": + message.reference.type = storage_pb.STORAGE_TYPE_GCS_BUCKET + message.reference.gcs_bucket.project_id = "test-project" + message.reference.gcs_bucket.bucket = bucket + location = StorageLocation.from_message(message) + context = RunnerContext(storage_locations=[location]) + key = "nested/hello + %20 ü.txt" + expected = b"GCS blob contents\n" + store = MemoryStore() + obstore.put(store, key, expected) + task = StorageEventTask().once(location, key) + restored = StorageEventTask._deserialize(task._serialize(), context) + credentials = MagicMock() + with ( + patch("google.auth.default", return_value=(credentials, "test-project")) as default_credentials, + patch("obstore.store.GCSStore", return_value=store) as constructor, + ): + assert restored.trigger.storage.read(restored.trigger.location) == expected + default_credentials.assert_called_once_with(scopes=["https://www.googleapis.com/auth/devstorage.read_only"]) + assert restored.trigger.storage.read(restored.trigger.location) == expected + constructor.assert_called_once() + assert constructor.call_args.args == (bucket,) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider.credentials is credentials + credentials.token = "test-token" # noqa: S105 + credentials.expiry = None + assert provider()["token"] == "test-token" # noqa: S105 + credentials.refresh.assert_called_once_with(provider.request) + + +def test_s3_read_preserves_aws_profile(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # S3 reads use credentials from the selected AWS profile rather than the default profile. + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + credentials = tmp_path / "credentials" + credentials.write_text( + "[default]\naws_access_key_id = default-key\naws_secret_access_key = default-secret\n" + "[automation]\naws_access_key_id = profile-key\naws_secret_access_key = profile-secret\n" + ) + config = tmp_path / "config" + config.write_text("[profile automation]\nregion = eu-west-2\n") + for name in ( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_SECURITY_TOKEN", + "AWS_DEFAULT_PROFILE", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", str(credentials)) + monkeypatch.setenv("AWS_CONFIG_FILE", str(config)) + monkeypatch.setenv("AWS_PROFILE", "automation") + store = MemoryStore() + key = "nested/aws.txt" + expected = b"S3 blob contents\n" + obstore.put(store, key, expected) + with patch("obstore.store.S3Store", return_value=store) as constructor: + assert S3StorageLocation(_ID, "aws-bucket", StorageType.S3, region="eu-west-2").read(key) == expected + assert constructor.call_args.args == ("aws-bucket",) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider()["access_key_id"] == "profile-key" + + +def test_s3_read_preserves_default_session(monkeypatch: pytest.MonkeyPatch) -> None: + # S3 reads retain credentials and region configured with boto3.setup_default_session(). + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "environment-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "environment-secret") + boto3.setup_default_session( + aws_access_key_id="configured-key", + aws_secret_access_key="configured-secret", # noqa: S106 + region_name="eu-west-2", + ) + store = MemoryStore() + obstore.put(store, "file.txt", b"configured session") + with patch("obstore.store.S3Store", return_value=store) as constructor: + assert ( + S3StorageLocation(_ID, "aws-bucket", StorageType.S3, region="eu-west-2").read("file.txt") + == b"configured session" + ) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider()["access_key_id"] == "configured-key" + assert provider.config["region"] == "eu-west-2" + + +def test_s3_reference_region_and_store_cache(monkeypatch: pytest.MonkeyPatch) -> None: + # The API bucket region survives task serialization and overrides the session region in cached stores. + session = MagicMock(region_name="us-east-1") + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + message = storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), + reference=storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AWS_S3_BUCKET, + aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket="source-bucket", region="eu-west-2"), + ), + ) + location = StorageLocation.from_message(message) + assert isinstance(location, S3StorageLocation) + assert StorageLocation.from_message(location.to_message()) == location + context = RunnerContext(storage_locations=[location]) + task = StorageEventTask().once(location, "file.txt") + restored = StorageEventTask._deserialize(task._serialize(), context) + store = context.s3_client("source-bucket", region="eu-west-2") + assert store.config["region"] == "eu-west-2" + assert context.s3_client("source-bucket", region="eu-west-2") is store + assert context.s3_client("source-bucket", region="ap-southeast-2") is not store + session.client.assert_not_called() + memory = MemoryStore() + obstore.put(memory, "file.txt", b"regional object") + with patch.object(context, "s3_client", return_value=memory) as client: + assert restored.trigger.storage.read(restored.trigger.location) == b"regional object" + client.assert_called_once_with("source-bucket", region="eu-west-2") + + +@pytest.mark.parametrize("forbidden", [False, True]) +@pytest.mark.parametrize("cache", [False, True]) +def test_s3_region_discovery(forbidden: bool, cache: bool, monkeypatch: pytest.MonkeyPatch) -> None: + # Bucket-only stores discover the region once, including when HeadBucket denies listing permission. + client = boto3.Session( + aws_access_key_id="test-key", + aws_secret_access_key="test-secret", # noqa: S106 + region_name="us-east-1", + ).client("s3") + session = MagicMock(region_name=None) + session.client.return_value = client + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + metadata = {"HTTPHeaders": {"x-amz-bucket-region": "eu-west-2"}} + with Stubber(client) as stub: + if forbidden: + stub.add_client_error( + "head_bucket", + "403", + http_status_code=403, + response_meta=metadata, + expected_params={"Bucket": "source-bucket"}, + ) + else: + stub.add_response("head_bucket", {"ResponseMetadata": metadata}, {"Bucket": "source-bucket"}) + if cache: + root = AmazonS3Cache("source-bucket") + store = root.store + assert root.group("nested").store is store + else: + store = RunnerContext().s3_client("source-bucket") + assert RunnerContext().s3_client("source-bucket") is store + stub.assert_no_pending_responses() + assert store.config["region"] == "eu-west-2" + session.client.assert_called_once_with("s3") + + +def test_s3_region_discovery_error(monkeypatch: pytest.MonkeyPatch) -> None: + # Discovery errors without a bucket-region header propagate instead of using the session region. + session = MagicMock(region_name="us-east-1") + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + error = ClientError({"Error": {"Code": "AccessDenied"}}, "HeadBucket") + session.client.return_value.head_bucket.side_effect = error + with pytest.raises(ClientError) as raised: + RunnerContext().s3_client("source-bucket") + assert raised.value is error + + +@pytest.mark.parametrize("cache", [False, True]) +def test_s3_temporary_credentials_refresh(cache: bool, monkeypatch: pytest.MonkeyPatch) -> None: + # Real signed requests use refreshed credentials after a short session expires, without recreating the store. + now = datetime.now(UTC) + refresh = MagicMock( + return_value={ + "access_key": "refreshed-key", + "secret_key": "secret", + "token": "new-token", + "expiry_time": (now + timedelta(hours=1)).isoformat(), + } + ) + credentials = RefreshableCredentials( + "initial-key", + "secret", + "initial-token", + now + timedelta(minutes=16), + refresh_using=refresh, + method="test", + time_fetcher=lambda: now, + ) + session = MagicMock(region_name="us-east-1") + session.get_credentials.return_value = credentials + monkeypatch.setattr(boto3, "DEFAULT_SESSION", session) + authorizations: list[str] = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + authorizations.append(self.headers["Authorization"]) + self.send_response(200) + self.send_header("Content-Length", "4") + self.end_headers() + self.wfile.write(b"data") + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = Thread(target=server.serve_forever) + thread.start() + try: + constructor = partial(S3Store, endpoint=f"http://127.0.0.1:{server.server_port}", allow_http=True) + with patch("obstore.store.S3Store", side_effect=constructor): + store = ( + AmazonS3Cache("bucket", region="eu-west-2").store + if cache + else RunnerContext().s3_client("bucket", "eu-west-2") + ) + assert obstore.get(store, "key").bytes() == b"data" + refresh.assert_not_called() + now += timedelta(minutes=17) + assert obstore.get(store, "key").bytes() == b"data" + refresh.assert_called_once() + finally: + server.shutdown() + thread.join() + assert len(authorizations) == 2 + assert "Credential=initial-key/" in authorizations[0] + assert "Credential=refreshed-key/" in authorizations[1] + assert all("/eu-west-2/s3/" in authorization for authorization in authorizations) + + +def test_filesystem_read(tmp_path: Path) -> None: + # Filesystem reads return the contents of the file under the configured root directory. + key = "file.txt" + expected = b"existing provider content" + storage = StorageLocation(_ID, str(tmp_path), StorageType.FS) + (tmp_path / key).write_bytes(expected) + assert storage.read(key) == expected + + +@pytest.mark.parametrize( + ("reference", "expected_type", "expected_location"), + [ + ( + storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_GCS_BUCKET, + gcs_bucket=storage_pb.GCSBucketReference(project_id="project", bucket="bucket"), + ), + StorageType.GCS, + "bucket", + ), + ( + storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AWS_S3_BUCKET, + aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket="bucket"), + ), + StorageType.S3, + "bucket", + ), + ( + storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_FILESYSTEM, filesystem=storage_pb.FilesystemReference(path="/data") + ), + StorageType.FS, + "/data", + ), + ], +) +def test_reference_coordinates( + reference: storage_pb.StorageLocationReference, expected_type: StorageType, expected_location: str +) -> None: + # Structured GCS, S3, and filesystem references map to the expected storage type and location. + storage = StorageLocation.from_message( + storage_pb.StorageLocation(id=uuid_to_uuid_message(_ID), reference=reference) + ) + assert storage.type == expected_type + assert storage.location == expected_location + + +def test_grpc_storage_location_service_route() -> None: + # gRPC listing uses StorageLocationService and retains Azure account metadata. + channel = MagicMock() + channel.unary_unary.return_value.return_value = storage_pb.StorageLocations(locations=[_azure_message()]) + with ( + patch("tilebox.workflows.client.open_channel", return_value=channel), + patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), + ): + locations = Client(token="test-key").automations().storage_locations() # noqa: S106 + assert isinstance(locations[0], AzureStorageLocation) + assert locations[0].storage_account_resource_id == _ACCOUNT + assert "/workflows.v1.StorageLocationService/ListStorageLocations" in [ + call.args[0] for call in channel.unary_unary.call_args_list + ] + + +def test_http1_storage_location_service_route() -> None: + # HTTP/1.1 listing uses the Connect storage-location client and retains Azure account metadata. + with ( + patch( + "tilebox.workflows.workflows.v1.storage_location_connect.StorageLocationServiceClientSync.list_storage_locations", + return_value=storage_pb.StorageLocations(locations=[_azure_message()]), + ) as list_locations, + patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), + ): + locations = Client(token="test-key", transport="http1").automations().storage_locations() # noqa: S106 + assert isinstance(locations[0], AzureStorageLocation) + assert locations[0].storage_account_resource_id == _ACCOUNT + list_locations.assert_called_once() diff --git a/tilebox-workflows/tests/runner/test_runner.py b/tilebox-workflows/tests/runner/test_runner.py index f697aea..12460b2 100644 --- a/tilebox-workflows/tests/runner/test_runner.py +++ b/tilebox-workflows/tests/runner/test_runner.py @@ -20,6 +20,12 @@ from tilebox.workflows.runner.task_runner import TaskRunner +@pytest.fixture(autouse=True) +def offline_logging(monkeypatch: pytest.MonkeyPatch) -> None: + # Offline runner tests must not start background network log exporters. + monkeypatch.setattr("tilebox.workflows.client.initialize_logging", MagicMock()) + + def test_public_execution_context_attributes() -> None: class RenameTask(Task): def execute(self, context: ExecutionContext) -> None: @@ -40,7 +46,7 @@ def test_task_authoring_imports_are_lazy() -> None: code = ( "import sys\n" "import tilebox.workflows as workflows\n" - "heavy = {'pandas', 'xarray', 'boto3', 'google.cloud.storage', 'ipywidgets', 'opentelemetry.sdk'}\n" + "heavy = {'pandas', 'xarray', 'boto3', 'google.cloud.storage', 'google.auth', 'azure.identity', 'obstore', 'ipywidgets', 'opentelemetry.sdk'}\n" "assert not heavy & sys.modules.keys()\n" "assert set(workflows.__all__) <= set(dir(workflows))\n" "task = workflows.Task\n" @@ -195,7 +201,10 @@ def replay_client(replay_file: str, assert_request_matches: bool = True) -> Clie replay = Path(__file__).parent / "testdata" / "recordings" / replay_file replay_channel = open_replay_channel(replay, assert_request_matches) - with patch("tilebox.workflows.client.open_channel") as open_channel_mock: + with ( + patch("tilebox.workflows.client.open_channel") as open_channel_mock, + patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), + ): open_channel_mock.return_value = replay_channel # url/token doesn't matter since its a mocked channel client = Client( diff --git a/tilebox-workflows/tests/tasks_data.py b/tilebox-workflows/tests/tasks_data.py index 381569e..75f1092 100644 --- a/tilebox-workflows/tests/tasks_data.py +++ b/tilebox-workflows/tests/tasks_data.py @@ -30,6 +30,7 @@ from tilebox.workflows.data import ( Artifact, AutomationPrototype, + AzureStorageLocation, Cluster, ComputedTask, CronTrigger, @@ -331,6 +332,14 @@ def storage_locations(draw: DrawFn) -> StorageLocation: location = draw(alphanumerical_text(min_size=4, max_size=30)) storage_type = draw(sampled_from(StorageType)) + if storage_type == StorageType.AZURE: + account = draw(alphanumerical_text(min_size=3, max_size=24)) + account_resource_id = ( + f"/subscriptions/example/resourceGroups/storage/providers/Microsoft.Storage/storageAccounts/{account}" + ) + return AzureStorageLocation( + storage_location_id, location, storage_type, storage_account_resource_id=account_resource_id + ) return StorageLocation(storage_location_id, location, storage_type) diff --git a/tilebox-workflows/tests/test_cache.py b/tilebox-workflows/tests/test_cache.py index 28026ac..80ee621 100644 --- a/tilebox-workflows/tests/test_cache.py +++ b/tilebox-workflows/tests/test_cache.py @@ -1,59 +1,215 @@ -import os -from collections.abc import Iterator +from datetime import timedelta from pathlib import Path +from unittest.mock import MagicMock, patch import boto3 import pytest from _pytest.fixtures import SubRequest -from moto import mock_aws -from mypy_boto3_s3 import S3Client +from obstore.exceptions import GenericError from obstore.store import LocalStore, MemoryStore -from tilebox.workflows.cache import AmazonS3Cache, InMemoryCache, JobCache, LocalFileSystemCache, ObstoreCache +from tilebox.workflows.cache import ( + AmazonS3Cache, + AzureBlobCache, + GoogleStorageCache, + InMemoryCache, + JobCache, + LocalFileSystemCache, + ObstoreCache, +) @pytest.fixture -def _aws_credentials() -> None: - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" # noqa: S105 - os.environ["AWS_SECURITY_TOKEN"] = "testing" # noqa: S105 - os.environ["AWS_SESSION_TOKEN"] = "testing" # noqa: S105 - os.environ["AWS_DEFAULT_REGION"] = "us-east-1" +def _aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + """Use test credentials without contacting AWS metadata services.""" + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +@pytest.fixture(autouse=True) +def _azure_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ACCESS_KEY", + "AZURE_STORAGE_MASTER_KEY", + "AZURE_STORAGE_SAS_KEY", + "AZURE_STORAGE_SAS_TOKEN", + ): + monkeypatch.delenv(name, raising=False) + + +caches = [ + "LocalFileSystem", + "InMemory", + "AmazonS3", + "AmazonS3_no_prefix", + "AzureBlob", + "AzureBlob_no_prefix", + "ObstoreMemory", + "ObstoreLocal", + "GoogleStorage", +] @pytest.fixture -def aws(_aws_credentials: None) -> Iterator[S3Client]: - with mock_aws(): - yield boto3.client("s3", region_name="us-east-1") - - -caches = ["LocalFileSystem", "InMemory", "AmazonS3", "AmazonS3_no_prefix", "ObstoreMemory", "ObstoreLocal"] - - -@pytest.fixture -def cache(request: SubRequest, tmp_path: Path, aws: S3Client) -> JobCache: +def cache(request: SubRequest, tmp_path: Path, _aws_credentials: None) -> JobCache: # noqa: PLR0911 match request.param: case "LocalFileSystem": return LocalFileSystemCache(tmp_path) case "InMemory": return InMemoryCache() - case "AmazonS3": - bucket = "bucket1" - aws.create_bucket(Bucket=bucket) - return AmazonS3Cache(bucket, prefix="test") - case "AmazonS3_no_prefix": - bucket = "bucket1" - aws.create_bucket(Bucket=bucket) - return AmazonS3Cache(bucket, prefix="") + case "AmazonS3" | "AmazonS3_no_prefix": + with patch("obstore.store.S3Store", return_value=MemoryStore()): + return AmazonS3Cache( + "bucket1", prefix="test" if request.param == "AmazonS3" else "", region="us-east-1" + ) + case "AzureBlob" | "AzureBlob_no_prefix": + with ( + patch("azure.identity.DefaultAzureCredential"), + patch("obstore.store.AzureStore", return_value=MemoryStore()), + ): + return AzureBlobCache("account", "container", prefix="test" if request.param == "AzureBlob" else "") case "ObstoreMemory": return ObstoreCache(MemoryStore()) case "ObstoreLocal": return ObstoreCache(LocalStore(tmp_path)) + case "GoogleStorage": + with ( + patch("google.auth.default", return_value=(MagicMock(), "project")), + patch("obstore.store.GCSStore", return_value=MemoryStore()), + ): + return GoogleStorageCache("test-bucket") case _: raise ValueError("Invalid cache type") +def test_azure_cache_credentials_and_groups(monkeypatch: pytest.MonkeyPatch) -> None: + # Azure cache groups share one store and credential provider without listing sibling prefixes. + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "wrongaccount") + store = MemoryStore() + with ( + patch("obstore.auth.azure.AzureCredentialProvider") as provider, + patch("obstore.store.AzureStore", return_value=store) as constructor, + ): + cache = AzureBlobCache("cacheaccount", "cache-container") + first = cache.group("folder") + first["one"] = b"first" + cache.group("folder2")["two"] = b"second" + assert list(first) == ["one"] + assert first["one"] == b"first" + assert isinstance(first, ObstoreCache) + assert first.store is store + assert store.get("jobs/folder/one").bytes() == b"first" + constructor.assert_called_once_with( + "cache-container", account_name="cacheaccount", credential_provider=provider.return_value + ) + provider.assert_called_once_with() + + +@pytest.mark.parametrize( + ("variable", "value"), + [ + ("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdA=="), + ("AZURE_STORAGE_ACCESS_KEY", "dGVzdA=="), + ("AZURE_STORAGE_MASTER_KEY", "dGVzdA=="), + ("AZURE_STORAGE_SAS_KEY", "sv=1&sig=test"), + ("AZURE_STORAGE_SAS_TOKEN", "sv=1&sig=test"), + ], +) +def test_azure_cache_explicit_credentials(variable: str, value: str, monkeypatch: pytest.MonkeyPatch) -> None: + # Account keys and SAS tokens bypass Azure Identity when constructing the cache store. + monkeypatch.setenv(variable, value) + with ( + patch("azure.identity.DefaultAzureCredential") as credential, + patch("obstore.store.AzureStore", return_value=MemoryStore()) as constructor, + ): + AzureBlobCache("cacheaccount", "cache-container") + constructor.assert_called_once_with("cache-container", account_name="cacheaccount") + credential.assert_not_called() + + +def test_s3_cache_credentials_and_groups(monkeypatch: pytest.MonkeyPatch) -> None: + # S3 caches use session credentials and the bucket region, sharing one store across groups. + monkeypatch.setattr(boto3, "DEFAULT_SESSION", None) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "environment-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "environment-secret") + boto3.setup_default_session( + aws_access_key_id="configured-key", + aws_secret_access_key="configured-secret", # noqa: S106 + region_name="eu-west-2", + ) + store = MemoryStore() + with patch("obstore.store.S3Store", return_value=store) as constructor: + cache = AmazonS3Cache("cache-bucket", region="ap-southeast-2") + first = cache.group("folder") + first["one"] = b"first" + cache.group("folder2")["two"] = b"second" + assert list(first) == ["one"] + assert first["one"] == b"first" + assert isinstance(first, ObstoreCache) + assert first.store is store + assert store.get("jobs/folder/one").bytes() == b"first" + assert constructor.call_count == 1 + assert constructor.call_args.args == ("cache-bucket",) + provider = constructor.call_args.kwargs["credential_provider"] + assert provider()["access_key_id"] == "configured-key" + assert provider.config["region"] == "eu-west-2" + assert constructor.call_args.kwargs["region"] == "ap-southeast-2" + assert provider.ttl == timedelta(0) + + +def test_google_storage_credentials_and_groups() -> None: + # GCS caches request write access and share credentials across groups without listing sibling prefixes. + credentials = MagicMock() + store = MemoryStore() + with ( + patch("google.auth.default", return_value=(credentials, "project")) as default, + patch("obstore.store.GCSStore", return_value=store) as constructor, + ): + cache = GoogleStorageCache("cache-bucket") + first = cache.group("folder") + first["one"] = b"first" + cache.group("folder2")["two"] = b"second" + assert list(first) == ["one"] + assert first["one"] == b"first" + assert isinstance(first, ObstoreCache) + assert first.store is store + assert store.get("jobs/folder/one").bytes() == b"first" + default.assert_called_once_with(scopes=["https://www.googleapis.com/auth/devstorage.read_write"]) + assert constructor.call_count == 1 + assert constructor.call_args.args == ("cache-bucket",) + assert constructor.call_args.kwargs["credential_provider"].credentials is credentials + + +@pytest.mark.parametrize("cache", ["AmazonS3", "GoogleStorage", "AzureBlob"], indirect=True) +@pytest.mark.parametrize("error_type", [GenericError, PermissionError, TimeoutError, OSError]) +def test_cloud_cache_backend_errors(cache: ObstoreCache, error_type: type[Exception]) -> None: + # Cloud caches and their groups preserve backend errors during reads, membership checks, and deletion. + error = error_type("backend unavailable") + store = MagicMock() + store.get.side_effect = error + store.delete.side_effect = error + cache.store = store + for target in (cache, cache.group("nested")): + for operation in (target.__getitem__, target.__contains__, target.__delitem__): + with pytest.raises(error_type) as raised: + operation("key") + assert raised.value is error + + +def test_local_cache_generic_error_is_not_a_missing_path(tmp_path: Path) -> None: + # Local backend errors propagate unless the requested path has a file where a directory is needed. + error = GenericError("disk unavailable") + store = MagicMock(spec=LocalStore) + store.prefix = tmp_path + store.get.side_effect = error + with pytest.raises(GenericError) as raised: + ObstoreCache(store)["missing/key"] + assert raised.value is error + + @pytest.mark.parametrize("cache", caches, indirect=True) def test_contains(cache: JobCache) -> None: assert "test" not in cache diff --git a/tilebox-workflows/tilebox/workflows/automations/service.py b/tilebox-workflows/tilebox/workflows/automations/service.py index d49acfc..06cdc50 100644 --- a/tilebox-workflows/tilebox/workflows/automations/service.py +++ b/tilebox-workflows/tilebox/workflows/automations/service.py @@ -12,25 +12,35 @@ StorageLocation, ) from tilebox.workflows.workflows.v1.automation_pb2 import AutomationPrototype as AutomationPrototypeMessage -from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest, StorageLocations +from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest from tilebox.workflows.workflows.v1.automation_pb2_grpc import AutomationServiceStub +from tilebox.workflows.workflows.v1.storage_location_pb2 import StorageLocations +from tilebox.workflows.workflows.v1.storage_location_pb2_grpc import StorageLocationServiceStub class AutomationService: - def __init__(self, channel: Channel | Any) -> None: + def __init__(self, channel: Channel | Any, storage_channel: Channel | Any = None) -> None: """ A wrapper around the AutomationServiceStub that provides a more pythonic interface and converts the protobuf messages to and from the data classes used in the rest of the tilebox-workflows codebase. Args: channel: The gRPC channel to use for the service. + storage_channel: Optional separate storage-location client for the Connect transport. """ self.service = ( with_pythonic_errors(AutomationServiceStub(channel)) if hasattr(channel, "unary_unary") else channel ) + if storage_channel is None: + storage_channel = channel + self.storage_service = ( + with_pythonic_errors(StorageLocationServiceStub(storage_channel)) + if hasattr(storage_channel, "unary_unary") + else storage_channel + ) def list_storage_locations(self) -> builtins.list[StorageLocation]: - response: StorageLocations = self.service.ListStorageLocations(Empty()) + response: StorageLocations = self.storage_service.ListStorageLocations(Empty()) return [StorageLocation.from_message(sl) for sl in response.locations] def list(self) -> builtins.list[AutomationPrototype]: diff --git a/tilebox-workflows/tilebox/workflows/automations/storage_event.py b/tilebox-workflows/tilebox/workflows/automations/storage_event.py index 3bb42cf..9e27f81 100644 --- a/tilebox-workflows/tilebox/workflows/automations/storage_event.py +++ b/tilebox-workflows/tilebox/workflows/automations/storage_event.py @@ -10,7 +10,7 @@ ) from tilebox.workflows.task import RunnerContext, Task, deserialize_task, serialize_task from tilebox.workflows.workflows.v1.automation_pb2 import Automation as AutomationMessage -from tilebox.workflows.workflows.v1.automation_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage +from tilebox.workflows.workflows.v1.storage_location_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage _NOT_TRIGGERED = TriggeredStorageEvent( StorageLocation(UUID(int=0), "", StorageType.FS), diff --git a/tilebox-workflows/tilebox/workflows/cache.py b/tilebox-workflows/tilebox/workflows/cache.py index 2810dac..ffc32b8 100644 --- a/tilebox-workflows/tilebox/workflows/cache.py +++ b/tilebox-workflows/tilebox/workflows/cache.py @@ -1,18 +1,14 @@ import contextlib from abc import ABC, abstractmethod from collections.abc import Iterator -from io import BytesIO from pathlib import Path from pathlib import PurePosixPath as ObjectPath from threading import RLock from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from google.cloud.storage import Blob, Bucket from obstore.store import ObjectStore else: - Blob = Any - Bucket = Any ObjectStore = Any @@ -91,7 +87,7 @@ def __init__(self, store: ObjectStore, prefix: str | ObjectPath = ObjectPath("." self.prefix = ObjectPath(prefix) def __contains__(self, key: str) -> bool: - with contextlib.suppress(OSError): + with contextlib.suppress(FileNotFoundError, NotADirectoryError): self.store.get(str(self.prefix / key)) return True # if get is successful, we know the key is in the cache @@ -103,19 +99,28 @@ def __setitem__(self, key: str, value: bytes) -> None: def __delitem__(self, key: str) -> None: try: self.store.delete(str(self.prefix / key)) - except OSError: + except (FileNotFoundError, NotADirectoryError): raise KeyError(f"{key} is not cached!") from None def __getitem__(self, key: str) -> bytes: from obstore.exceptions import GenericError # noqa: PLC0415 + from obstore.store import LocalStore # noqa: PLC0415 try: entry = self.store.get(str(self.prefix / key)) return bytes(entry.bytes()) - except (OSError, GenericError): - # GenericError is raised if the key contains separator characters, but one of the parents is a file - # instead of a directory + except (FileNotFoundError, NotADirectoryError): raise KeyError(f"{key} is not cached!") from None + except GenericError: + # LocalStore wraps a file used as a parent directory in GenericError. + if isinstance(self.store, LocalStore) and self.store.prefix is not None: + try: + (self.store.prefix / self.prefix / key).stat() + except NotADirectoryError: + raise KeyError(f"{key} is not cached!") from None + except OSError: + pass + raise def __iter__(self) -> Iterator[str]: prefix = "" if self.prefix == ObjectPath(".") else str(self.prefix) @@ -252,113 +257,47 @@ def group(self, key: str) -> "LocalFileSystemCache": return LocalFileSystemCache(self.root / key) -class GoogleStorageCache(JobCache): - def __init__(self, bucket: Bucket, prefix: str | ObjectPath = "jobs") -> None: +class GoogleStorageCache(ObstoreCache): + def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs") -> None: """A cache implementation that stores data in Google Cloud Storage. Args: - bucket: The Google Cloud Storage bucket to use for the cache. + bucket: The Google Cloud Storage bucket name. Credentials come from Google Auth. prefix: A path prefix to append to all objects stored in the cache. Defaults to "jobs". """ - self.bucket = bucket - self.prefix = ObjectPath( - prefix - ) # we still use pathlib here, because it's easier to work with when joining paths - - def _blob(self, key: str) -> Blob: - return self.bucket.blob(str(self.prefix / key)) - - def __contains__(self, key: str) -> bool: - # GCS library has some weird typing issues, so let's ignore them for now - return self._blob(key).exists() - - def __setitem__(self, key: str, value: bytes) -> None: - # GCS library has some weird typing issues, so let's ignore them for now - self._blob(key).upload_from_file(BytesIO(value)) - - def __getitem__(self, key: str) -> bytes: - from google.cloud.exceptions import NotFound # noqa: PLC0415 - - try: - # GCS library has some weird typing issues, so let's ignore them for now - return self._blob(key).download_as_bytes() - except NotFound: - raise KeyError(f"{key} is not cached!") from None - - def __iter__(self) -> Iterator[str]: - # we need to add the trailing slash, to avoid listing other blobs that start with the same prefix, e.g. - # consider the following blobs: - # jobs/folder/some_file.txt - # jobs/folder2/other_file.txt - # if we just list all blobs with prefix "jobs/folder", we would get both of them, but we only want the - # ones in the folder, so we add the trailing slash to only get the blobs in the folder - prefix = str(self.prefix) + "/" - # by specifying the delimiter as "/", we can emulate a directory structure, and only get the blobs directly - # in the "folder", and not the ones in subfolders - - # GCS library has some weird typing issues, so let's ignore them for now - blobs = self.bucket.list_blobs(prefix=prefix, delimiter="/") + import google.auth # noqa: PLC0415 + from obstore.auth.google import GoogleCredentialProvider # noqa: PLC0415 + from obstore.store import GCSStore # noqa: PLC0415 - # make the names relative to the cache prefix (but including the key in the name) - for blob in blobs: - yield str(ObjectPath(blob.name).relative_to(self.prefix)) + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/devstorage.read_write"]) + store = GCSStore(bucket, credential_provider=GoogleCredentialProvider(credentials=credentials)) + super().__init__(store, prefix) - def group(self, key: str) -> "GoogleStorageCache": - return GoogleStorageCache(self.bucket, prefix=str(self.prefix / key)) - -class AmazonS3Cache(JobCache): - def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs") -> None: +class AmazonS3Cache(ObstoreCache): + def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs", *, region: str | None = None) -> None: """A cache implementation that stores data in Amazon S3. Args: - bucket: The Amazon S3 bucket to use for the cache. + bucket: The Amazon S3 bucket name. Credentials come from the boto3 session. prefix: A path prefix to append to all objects stored in the cache. Defaults to "jobs". + region: The bucket region. If omitted, discover it with HeadBucket. """ - import boto3 # noqa: PLC0415 + from tilebox.workflows.data import _s3_storage_client # noqa: PLC0415 - self.bucket = bucket - self.prefix = ObjectPath(prefix) - self._s3 = boto3.client("s3") + super().__init__(_s3_storage_client(bucket, region), prefix) - def __contains__(self, key: str) -> bool: - from botocore.exceptions import ClientError # noqa: PLC0415 - try: - self._s3.head_object(Bucket=self.bucket, Key=str(self.prefix / key)) - except ClientError as e: - err = e.response.get("Error", {}) - if err.get("Code", "") == "404": - return False - raise # not a 404, re-raise the exception - else: - return True - - def __setitem__(self, key: str, value: bytes) -> None: - self._s3.upload_fileobj(BytesIO(value), self.bucket, str(self.prefix / key)) +class AzureBlobCache(ObstoreCache): + def __init__(self, account_name: str, container: str, prefix: str | ObjectPath = "jobs") -> None: + """Store cached data in Azure Blob Storage. - def __getitem__(self, key: str) -> bytes: - from botocore.exceptions import ClientError # noqa: PLC0415 - - item = BytesIO() - try: - self._s3.download_fileobj(self.bucket, str(self.prefix / key), item) - except ClientError as e: - err = e.response.get("Error", {}) - if err.get("Code", "") == "404": - raise KeyError(f"{key} is not cached!") from None - raise # not a 404, re-raise the exception - else: - return item.getvalue() + Args: + account_name: The storage account name. Credentials come from Azure Identity, + or an account key or SAS token configured in the environment. + container: The Azure Blob Storage container name. + prefix: A path prefix for cached objects. Defaults to "jobs". + """ + from tilebox.workflows.data import _azure_storage_client # noqa: PLC0415 - def __iter__(self) -> Iterator[str]: - paginator = self._s3.get_paginator("list_objects_v2") - prefix = str(self.prefix) + "/" if self.prefix != Path() else "" - for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix, Delimiter="/"): - for item in page.get("Contents", []): - bucket = str(Path(item.get("Key", "")).relative_to(self.prefix)) - if "/" not in bucket: # only yield files directly in the prefix - yield bucket - - def group(self, key: str) -> "AmazonS3Cache": - return AmazonS3Cache(self.bucket, prefix=str(self.prefix / key)) + super().__init__(_azure_storage_client(account_name, container), prefix) diff --git a/tilebox-workflows/tilebox/workflows/client.py b/tilebox-workflows/tilebox/workflows/client.py index 21b45ba..c19a167 100644 --- a/tilebox-workflows/tilebox/workflows/client.py +++ b/tilebox-workflows/tilebox/workflows/client.py @@ -68,6 +68,7 @@ def __init__( self._cluster_service = self._job_service self._workflow_service = self._job_service self._automation_service = self._job_service + self._storage_location_service = self._job_service self._task_service = self._job_service case "http1": from pyqwest import HTTPVersion, SyncClient, SyncHTTPTransport # noqa: PLC0415 @@ -76,6 +77,9 @@ def __init__( AutomationServiceClientSync, ) from tilebox.workflows.workflows.v1.job_connect import JobServiceClientSync # noqa: PLC0415 + from tilebox.workflows.workflows.v1.storage_location_connect import ( # noqa: PLC0415 + StorageLocationServiceClientSync, + ) from tilebox.workflows.workflows.v1.task_connect import TaskServiceClientSync # noqa: PLC0415 from tilebox.workflows.workflows.v1.telemetry_connect import ( # noqa: PLC0415 TelemetryQueryServiceClientSync, @@ -98,6 +102,9 @@ def __init__( self._automation_service = ConnectStubAdapter( AutomationServiceClientSync(address, http_client=http_client), headers ) + self._storage_location_service = ConnectStubAdapter( + StorageLocationServiceClientSync(address, http_client=http_client), headers + ) self._task_service = ConnectStubAdapter( TaskServiceClientSync(address, http_client=http_client), headers ) @@ -200,7 +207,7 @@ def automations(self) -> AutomationClient: Returns: A client for the automations service. """ - return AutomationClient(AutomationService(self._automation_service)) + return AutomationClient(AutomationService(self._automation_service, self._storage_location_service)) def _token_from_env(url: str, token: str | None) -> str | None: diff --git a/tilebox-workflows/tilebox/workflows/data.py b/tilebox-workflows/tilebox/workflows/data.py index 6bcbb56..f72dbb9 100644 --- a/tilebox-workflows/tilebox/workflows/data.py +++ b/tilebox-workflows/tilebox/workflows/data.py @@ -1,7 +1,9 @@ +import os import re import warnings from collections.abc import Callable, Mapping, MutableMapping -from dataclasses import asdict, dataclass, field +from contextlib import closing +from dataclasses import asdict, dataclass, field, replace from datetime import UTC, datetime, timedelta from enum import Enum from functools import lru_cache, total_ordering @@ -27,17 +29,15 @@ from tilebox.datasets.uuid import must_uuid_to_uuid_message, uuid_message_to_uuid, uuid_to_uuid_message if TYPE_CHECKING: - from google.cloud.storage.bucket import Bucket - from mypy_boto3_s3.client import S3Client + from obstore.store import AzureStore, GCSStore, S3Store from tilebox.workflows.observability.tracing import WorkflowTracer else: - Bucket = Any - S3Client = Any WorkflowTracer = Any from tilebox.workflows.workflows.v1 import automation_pb2 as automation_pb from tilebox.workflows.workflows.v1 import core_pb2, job_pb2, task_pb2, telemetry_pb2, workflows_pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as storage_pb _VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)$") # matches a version string in the format "v3.2" @@ -969,65 +969,124 @@ def from_message(cls, page: Any) -> "QueryJobSpansResponse": class StorageType(Enum): - GCS = automation_pb.STORAGE_TYPE_GCS # Google Cloud Storage - S3 = automation_pb.STORAGE_TYPE_S3 # Amazon Web Services S3 - FS = automation_pb.STORAGE_TYPE_FS # Local Filesystem + GCS = storage_pb.STORAGE_TYPE_GCS_BUCKET # Google Cloud Storage + S3 = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET # Amazon Web Services S3 + FS = storage_pb.STORAGE_TYPE_FILESYSTEM # Local Filesystem + AZURE = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage _STORAGE_TYPE_TO_ENUM = {storage_type.value: storage_type for storage_type in StorageType} -@dataclass(frozen=True, order=True) +@total_ordering +@dataclass(frozen=True) class StorageLocation: id: UUID location: str type: StorageType runner_context: "RunnerContext | None" = None + def __lt__(self, other: object) -> bool: + if not isinstance(other, StorageLocation): + return NotImplemented + return (self.id, self.location, self.type.value) < (other.id, other.location, other.type.value) + @classmethod - def from_message(cls, storage_location: automation_pb.StorageLocation) -> Self: + def from_message(cls, storage_location: storage_pb.StorageLocation) -> "StorageLocation": """Convert a StorageLocation protobuf message to a StorageLocation object.""" + location = storage_location.location + storage_type = storage_location.type + if storage_location.HasField("reference"): + reference = storage_location.reference + storage_type = reference.type + match storage_type: + case storage_pb.STORAGE_TYPE_AZURE_BLOB: + return AzureStorageLocation( + id=uuid_message_to_uuid(storage_location.id), + location=reference.azure_blob.container, + type=StorageType.AZURE, + storage_account_resource_id=reference.azure_blob.storage_account_resource_id, + ) + case storage_pb.STORAGE_TYPE_GCS_BUCKET: + location = reference.gcs_bucket.bucket + case storage_pb.STORAGE_TYPE_AWS_S3_BUCKET: + return S3StorageLocation( + id=uuid_message_to_uuid(storage_location.id), + location=reference.aws_s3_bucket.bucket, + type=StorageType.S3, + region=reference.aws_s3_bucket.region or None, + ) + case storage_pb.STORAGE_TYPE_FILESYSTEM: + location = reference.filesystem.path return cls( id=uuid_message_to_uuid(storage_location.id), - location=storage_location.location, - type=_STORAGE_TYPE_TO_ENUM[storage_location.type], + location=location, + type=_STORAGE_TYPE_TO_ENUM[storage_type], ) def _with_runner_context(self, runner_context: "RunnerContext") -> "StorageLocation": - return StorageLocation( - id=self.id, - location=self.location, - type=self.type, - runner_context=runner_context, - ) + return replace(self, runner_context=runner_context) - def to_message(self) -> automation_pb.StorageLocation: + def to_message(self) -> storage_pb.StorageLocation: """Convert a StorageLocation object to a StorageLocation protobuf message.""" - return automation_pb.StorageLocation( + return storage_pb.StorageLocation( id=uuid_to_uuid_message(self.id), location=self.location, type=self.type.value ) def read(self, path: str) -> bytes: runner_context = self.runner_context or RunnerContext() - match self.type: - case StorageType.GCS: - with runner_context.tracer.span("gcs.read") as span: + with runner_context.tracer.span(f"{self.type.name.lower()}.read") as span: + span.set_attribute("path", path) + match self.type: + case StorageType.GCS: span.set_attribute("bucket", self.location) - span.set_attribute("path", path) - # GCS library has some weird typing issues, so let's ignore them for now - blob = runner_context.gcs_client(self.location).blob(path) - return blob.download_as_bytes() - case StorageType.S3: - with runner_context.tracer.span("s3.read") as span: + store = runner_context.gcs_client(self.location) + case StorageType.S3: span.set_attribute("bucket", self.location) - span.set_attribute("path", path) - return runner_context.s3_client(self.location).get_object(Bucket=self.location, Key=path)["Body"].read() - case StorageType.FS: - with runner_context.tracer.span("fs.read") as span: + if isinstance(self, S3StorageLocation) and self.region: + store = runner_context.s3_client(self.location, region=self.region) + else: + store = runner_context.s3_client(self.location) + case StorageType.AZURE: + if not isinstance(self, AzureStorageLocation) or not self.storage_account_resource_id: + raise ValueError("Azure storage location is missing its storage account resource ID") + span.set_attribute("storage_account_resource_id", self.storage_account_resource_id) + span.set_attribute("container", self.location) + store = runner_context.azure_client(self.storage_account_resource_id, self.location) + case StorageType.FS: span.set_attribute("root_directory", self.location) - span.set_attribute("path", path) - return Path(self.location).joinpath(path).read_bytes() + return Path(self.location).joinpath(path).read_bytes() + + import obstore # noqa: PLC0415 + + return bytes(obstore.get(store, path).bytes()) + + +@dataclass(frozen=True) +class S3StorageLocation(StorageLocation): + region: str | None = field(default=None, kw_only=True) + + def to_message(self) -> storage_pb.StorageLocation: + """Include the S3 bucket and its region in the storage reference.""" + message = super().to_message() + message.reference.type = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET + message.reference.aws_s3_bucket.bucket = self.location + message.reference.aws_s3_bucket.region = self.region or "" + return message + + +@dataclass(frozen=True) +class AzureStorageLocation(StorageLocation): + storage_account_resource_id: str = field(kw_only=True) + + def to_message(self) -> storage_pb.StorageLocation: + """Include the Azure account and container in the storage reference.""" + message = super().to_message() + message.reference.type = storage_pb.STORAGE_TYPE_AZURE_BLOB + message.reference.azure_blob.storage_account_resource_id = self.storage_account_resource_id + message.reference.azure_blob.container = self.location + return message @dataclass(order=True) @@ -1055,7 +1114,7 @@ def to_message(self) -> automation_pb.StorageEventTrigger: class StorageEventType(Enum): - CREATED = automation_pb.STORAGE_EVENT_TYPE_CREATED + CREATED = storage_pb.STORAGE_EVENT_TYPE_CREATED _STORAGE_EVENT_TYPE_TO_ENUM = {storage_event_type.value: storage_event_type for storage_event_type in StorageEventType} @@ -1069,7 +1128,7 @@ class TriggeredStorageEvent: @classmethod def from_message( - cls, event: automation_pb.TriggeredStorageEvent, locations: Mapping[UUID, StorageLocation] + cls, event: storage_pb.TriggeredStorageEvent, locations: Mapping[UUID, StorageLocation] ) -> "TriggeredStorageEvent": """Convert a TriggeredStorageEvent protobuf message to a TriggeredStorageEvent object.""" storage_location_id = uuid_message_to_uuid(event.storage_location_id) @@ -1085,9 +1144,9 @@ def from_message( location=event.location, ) - def to_message(self) -> automation_pb.TriggeredStorageEvent: + def to_message(self) -> storage_pb.TriggeredStorageEvent: """Convert a TriggeredStorageEvent object to a TriggeredStorageEvent protobuf message.""" - return automation_pb.TriggeredStorageEvent( + return storage_pb.TriggeredStorageEvent( storage_location_id=uuid_to_uuid_message(self.storage.id), type=self.type.value, location=self.location, @@ -1177,25 +1236,90 @@ def __init__( for sl in storage_locations or [] } - def gcs_client(self, location: str) -> Bucket: - return _default_google_storage_client(location) + def gcs_client(self, location: str) -> "GCSStore": + return _default_gcs_storage_client(location) - def s3_client(self, location: str) -> S3Client: - import boto3 # noqa: PLC0415 - - _ = location # we always use the default s3 client, regardless of the location - return boto3.client("s3") + def s3_client(self, location: str, region: str | None = None) -> "S3Store": + return _default_s3_storage_client(location, region) def local_path(self, location: str) -> Path: return Path(location) + def azure_client(self, storage_account_resource_id: str, container: str) -> "AzureStore": + return _default_azure_storage_client(storage_account_resource_id, container) + + +@lru_cache +def _default_gcs_storage_client(bucket: str) -> "GCSStore": + import google.auth # noqa: PLC0415 + from obstore.auth.google import GoogleCredentialProvider # noqa: PLC0415 + from obstore.store import GCSStore # noqa: PLC0415 + + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/devstorage.read_only"]) + return GCSStore(bucket, credential_provider=GoogleCredentialProvider(credentials=credentials)) + @lru_cache -def _default_google_storage_client(location: str) -> Bucket: - from google.cloud.storage import Client as GoogleStorageClient # noqa: PLC0415 +def _default_s3_storage_client(bucket: str, region: str | None = None) -> "S3Store": + return _s3_storage_client(bucket, region) + + +def _s3_storage_client(bucket: str, region: str | None = None) -> "S3Store": + import boto3 # noqa: PLC0415 + from botocore.exceptions import ClientError # noqa: PLC0415 + from obstore.auth.boto3 import Boto3CredentialProvider # noqa: PLC0415 + from obstore.store import S3Store # noqa: PLC0415 + + session = boto3.DEFAULT_SESSION or boto3.Session() + if not region: + with closing(session.client("s3")) as client: + try: + response = client.head_bucket(Bucket=bucket) + except ClientError as error: + region = error.response.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region") + if not region: + raise + else: + region = response.get("BucketRegion") or response.get("ResponseMetadata", {}).get( + "HTTPHeaders", {} + ).get("x-amz-bucket-region") + if not region: + raise ValueError(f"S3 did not return a region for bucket {bucket!r}") + return S3Store( + bucket, region=region, credential_provider=Boto3CredentialProvider(session=session, ttl=timedelta(0)) + ) + + +@lru_cache +def _default_azure_storage_client(storage_account_resource_id: str, container: str) -> "AzureStore": + match = re.fullmatch( + r"/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft\.Storage/storageAccounts/([^/]+)", + storage_account_resource_id, + flags=re.IGNORECASE, + ) + if match is None: + raise ValueError(f"Invalid Azure storage account resource ID: {storage_account_resource_id!r}") + return _azure_storage_client(match[1], container) + + +def _azure_storage_client(account_name: str, container: str) -> "AzureStore": + from obstore.store import AzureStore # noqa: PLC0415 + + if any( + name in os.environ + for name in ( + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ACCESS_KEY", + "AZURE_STORAGE_MASTER_KEY", + "AZURE_STORAGE_SAS_KEY", + "AZURE_STORAGE_SAS_TOKEN", + ) + ): + return AzureStore(container, account_name=account_name) + + from obstore.auth.azure import AzureCredentialProvider # noqa: PLC0415 - project, bucket = location.split(":") - return GoogleStorageClient(project=project).bucket(bucket) + return AzureStore(container, account_name=account_name, credential_provider=AzureCredentialProvider()) @dataclass(frozen=True) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py index 61e2a79..cedd0ed 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_connect.py @@ -20,18 +20,6 @@ class AutomationService(Protocol): - async def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - - async def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - - async def create_storage_location(self, request: workflows_dot_v1_dot_automation__pb2.StorageLocation, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - - async def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - async def list_automations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.Automations: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") @@ -53,46 +41,6 @@ def __init__(self, service: AutomationService | AsyncGenerator[AutomationService super().__init__( service=service, endpoints=lambda svc: { - "/workflows.v1.AutomationService/ListStorageLocations": Endpoint.unary( - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.list_storage_locations, - ), - "/workflows.v1.AutomationService/GetStorageLocation": Endpoint.unary( - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.get_storage_location, - ), - "/workflows.v1.AutomationService/CreateStorageLocation": Endpoint.unary( - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.create_storage_location, - ), - "/workflows.v1.AutomationService/DeleteStorageLocation": Endpoint.unary( - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=svc.delete_storage_location, - ), "/workflows.v1.AutomationService/ListAutomations": Endpoint.unary( method=MethodInfo( name="ListAutomations", @@ -157,86 +105,6 @@ def path(self) -> str: class AutomationServiceClient(ConnectClient): - async def list_storage_locations( - self, - request: google_dot_protobuf_dot_empty__pb2.Empty, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - async def get_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - async def create_storage_location( - self, - request: workflows_dot_v1_dot_automation__pb2.StorageLocation, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - async def delete_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> google_dot_protobuf_dot_empty__pb2.Empty: - return await self.execute_unary( - request=request, - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - async def list_automations( self, request: google_dot_protobuf_dot_empty__pb2.Empty, @@ -342,14 +210,6 @@ async def delete_automation( class AutomationServiceSync(Protocol): - def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - def create_storage_location(self, request: workflows_dot_v1_dot_automation__pb2.StorageLocation, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") - def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: - raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def list_automations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.Automations: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def get_automation(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_automation__pb2.AutomationPrototype: @@ -366,46 +226,6 @@ class AutomationServiceWSGIApplication(ConnectWSGIApplication): def __init__(self, service: AutomationServiceSync, interceptors: Iterable[InterceptorSync]=(), read_max_bytes: int | None = None, compressions: Iterable[Compression] | None = None, codecs: Iterable[Codec] | None = None) -> None: super().__init__( endpoints={ - "/workflows.v1.AutomationService/ListStorageLocations": EndpointSync.unary( - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.list_storage_locations, - ), - "/workflows.v1.AutomationService/GetStorageLocation": EndpointSync.unary( - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.get_storage_location, - ), - "/workflows.v1.AutomationService/CreateStorageLocation": EndpointSync.unary( - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.create_storage_location, - ), - "/workflows.v1.AutomationService/DeleteStorageLocation": EndpointSync.unary( - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - function=service.delete_storage_location, - ), "/workflows.v1.AutomationService/ListAutomations": EndpointSync.unary( method=MethodInfo( name="ListAutomations", @@ -470,86 +290,6 @@ def path(self) -> str: class AutomationServiceClientSync(ConnectClientSync): - def list_storage_locations( - self, - request: google_dot_protobuf_dot_empty__pb2.Empty, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocations: - return self.execute_unary( - request=request, - method=MethodInfo( - name="ListStorageLocations", - service_name="workflows.v1.AutomationService", - input=google_dot_protobuf_dot_empty__pb2.Empty, - output=workflows_dot_v1_dot_automation__pb2.StorageLocations, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - def get_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return self.execute_unary( - request=request, - method=MethodInfo( - name="GetStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - def create_storage_location( - self, - request: workflows_dot_v1_dot_automation__pb2.StorageLocation, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> workflows_dot_v1_dot_automation__pb2.StorageLocation: - return self.execute_unary( - request=request, - method=MethodInfo( - name="CreateStorageLocation", - service_name="workflows.v1.AutomationService", - input=workflows_dot_v1_dot_automation__pb2.StorageLocation, - output=workflows_dot_v1_dot_automation__pb2.StorageLocation, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - - def delete_storage_location( - self, - request: tilebox_dot_v1_dot_id__pb2.ID, - *, - headers: Headers | Mapping[str, str] | None = None, - timeout_ms: int | None = None, - ) -> google_dot_protobuf_dot_empty__pb2.Empty: - return self.execute_unary( - request=request, - method=MethodInfo( - name="DeleteStorageLocation", - service_name="workflows.v1.AutomationService", - input=tilebox_dot_v1_dot_id__pb2.ID, - output=google_dot_protobuf_dot_empty__pb2.Empty, - idempotency_level=IdempotencyLevel.UNKNOWN, - ), - headers=headers, - timeout_ms=timeout_ms, - ) - def list_automations( self, request: google_dot_protobuf_dot_empty__pb2.Empty, diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py index 552115d..6d76160 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py @@ -27,42 +27,33 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflows/v1/automation.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\"|\n\x0fStorageLocation\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08location\x18\x02 \x01(\tR\x08location\x12-\n\x04type\x18\x03 \x01(\x0e\x32\x19.workflows.v1.StorageTypeR\x04type\"O\n\x10StorageLocations\x12;\n\tlocations\x18\x01 \x03(\x0b\x32\x1d.workflows.v1.StorageLocationR\tlocations\"\xc0\x02\n\x13\x41utomationPrototype\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12@\n\tprototype\x18\x03 \x01(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\tprototype\x12W\n\x16storage_event_triggers\x18\x04 \x03(\x0b\x32!.workflows.v1.StorageEventTriggerR\x14storageEventTriggers\x12>\n\rcron_triggers\x18\x05 \x03(\x0b\x32\x19.workflows.v1.CronTriggerR\x0c\x63ronTriggers\x12\x1a\n\x08\x64isabled\x18\x06 \x01(\x08R\x08\x64isabled\"R\n\x0b\x41utomations\x12\x43\n\x0b\x61utomations\x18\x01 \x03(\x0b\x32!.workflows.v1.AutomationPrototypeR\x0b\x61utomations\"\xa2\x01\n\x13StorageEventTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12H\n\x10storage_location\x18\x02 \x01(\x0b\x32\x1d.workflows.v1.StorageLocationR\x0fstorageLocation\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"\x96\x01\n\x0b\x43ronTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08schedule\x18\x02 \x01(\tR\x08schedule\x12K\n\x11next_scheduled_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\x0fnextScheduledAt\"E\n\nAutomation\x12#\n\rtrigger_event\x18\x01 \x01(\x0cR\x0ctriggerEvent\x12\x12\n\x04\x61rgs\x18\x02 \x01(\x0cR\x04\x61rgs\"\xa7\x01\n\x15TriggeredStorageEvent\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location\"S\n\x12TriggeredCronEvent\x12=\n\x0ctrigger_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0btriggerTime\"o\n\x17\x44\x65leteAutomationRequest\x12\x33\n\rautomation_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x1f\n\x0b\x63\x61ncel_jobs\x18\x02 \x01(\x08R\ncancelJobs*k\n\x0bStorageType\x12\x1c\n\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10STORAGE_TYPE_GCS\x10\x01\x12\x13\n\x0fSTORAGE_TYPE_S3\x10\x02\x12\x13\n\x0fSTORAGE_TYPE_FS\x10\x03*V\n\x10StorageEventType\x12\"\n\x1eSTORAGE_EVENT_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTORAGE_EVENT_TYPE_CREATED\x10\x01\x32\xd1\x05\n\x11\x41utomationService\x12N\n\x14ListStorageLocations\x12\x16.google.protobuf.Empty\x1a\x1e.workflows.v1.StorageLocations\x12\x43\n\x12GetStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x1d.workflows.v1.StorageLocation\x12U\n\x15\x43reateStorageLocation\x12\x1d.workflows.v1.StorageLocation\x1a\x1d.workflows.v1.StorageLocation\x12?\n\x15\x44\x65leteStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12\x44\n\x0fListAutomations\x12\x16.google.protobuf.Empty\x1a\x19.workflows.v1.Automations\x12\x42\n\rGetAutomation\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10\x43reateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10UpdateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12Q\n\x10\x44\x65leteAutomation\x12%.workflows.v1.DeleteAutomationRequest\x1a\x16.google.protobuf.EmptyBy\n\x10\x63om.workflows.v1B\x0f\x41utomationProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflows/v1/automation.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\x1a#workflows/v1/storage_location.proto\"\xc0\x02\n\x13\x41utomationPrototype\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12@\n\tprototype\x18\x03 \x01(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\tprototype\x12W\n\x16storage_event_triggers\x18\x04 \x03(\x0b\x32!.workflows.v1.StorageEventTriggerR\x14storageEventTriggers\x12>\n\rcron_triggers\x18\x05 \x03(\x0b\x32\x19.workflows.v1.CronTriggerR\x0c\x63ronTriggers\x12\x1a\n\x08\x64isabled\x18\x06 \x01(\x08R\x08\x64isabled\"R\n\x0b\x41utomations\x12\x43\n\x0b\x61utomations\x18\x01 \x03(\x0b\x32!.workflows.v1.AutomationPrototypeR\x0b\x61utomations\"\xa2\x01\n\x13StorageEventTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12H\n\x10storage_location\x18\x02 \x01(\x0b\x32\x1d.workflows.v1.StorageLocationR\x0fstorageLocation\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"\x96\x01\n\x0b\x43ronTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08schedule\x18\x02 \x01(\tR\x08schedule\x12K\n\x11next_scheduled_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\x0fnextScheduledAt\"E\n\nAutomation\x12#\n\rtrigger_event\x18\x01 \x01(\x0cR\x0ctriggerEvent\x12\x12\n\x04\x61rgs\x18\x02 \x01(\x0cR\x04\x61rgs\"S\n\x12TriggeredCronEvent\x12=\n\x0ctrigger_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0btriggerTime\"o\n\x17\x44\x65leteAutomationRequest\x12\x33\n\rautomation_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x1f\n\x0b\x63\x61ncel_jobs\x18\x02 \x01(\x08R\ncancelJobs2\xa4\x03\n\x11\x41utomationService\x12\x44\n\x0fListAutomations\x12\x16.google.protobuf.Empty\x1a\x19.workflows.v1.Automations\x12\x42\n\rGetAutomation\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10\x43reateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10UpdateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12Q\n\x10\x44\x65leteAutomation\x12%.workflows.v1.DeleteAutomationRequest\x1a\x16.google.protobuf.EmptyB\x05\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.automation_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\n\020com.workflows.v1B\017AutomationProtoP\001\242\002\003WXX\252\002\014Workflows.V1\312\002\014Workflows\\V1\342\002\030Workflows\\V1\\GPBMetadata\352\002\rWorkflows::V1\222\003\002\010\002' + _globals['DESCRIPTOR']._serialized_options = b'\222\003\002\010\002' _globals['_CRONTRIGGER'].fields_by_name['next_scheduled_at']._loaded_options = None _globals['_CRONTRIGGER'].fields_by_name['next_scheduled_at']._serialized_options = b'\340A\003' - _globals['_STORAGETYPE']._serialized_start=1559 - _globals['_STORAGETYPE']._serialized_end=1666 - _globals['_STORAGEEVENTTYPE']._serialized_start=1668 - _globals['_STORAGEEVENTTYPE']._serialized_end=1754 - _globals['_STORAGELOCATION']._serialized_start=188 - _globals['_STORAGELOCATION']._serialized_end=312 - _globals['_STORAGELOCATIONS']._serialized_start=314 - _globals['_STORAGELOCATIONS']._serialized_end=393 - _globals['_AUTOMATIONPROTOTYPE']._serialized_start=396 - _globals['_AUTOMATIONPROTOTYPE']._serialized_end=716 - _globals['_AUTOMATIONS']._serialized_start=718 - _globals['_AUTOMATIONS']._serialized_end=800 - _globals['_STORAGEEVENTTRIGGER']._serialized_start=803 - _globals['_STORAGEEVENTTRIGGER']._serialized_end=965 - _globals['_CRONTRIGGER']._serialized_start=968 - _globals['_CRONTRIGGER']._serialized_end=1118 - _globals['_AUTOMATION']._serialized_start=1120 - _globals['_AUTOMATION']._serialized_end=1189 - _globals['_TRIGGEREDSTORAGEEVENT']._serialized_start=1192 - _globals['_TRIGGEREDSTORAGEEVENT']._serialized_end=1359 - _globals['_TRIGGEREDCRONEVENT']._serialized_start=1361 - _globals['_TRIGGEREDCRONEVENT']._serialized_end=1444 - _globals['_DELETEAUTOMATIONREQUEST']._serialized_start=1446 - _globals['_DELETEAUTOMATIONREQUEST']._serialized_end=1557 - _globals['_AUTOMATIONSERVICE']._serialized_start=1757 - _globals['_AUTOMATIONSERVICE']._serialized_end=2478 + _globals['_AUTOMATIONPROTOTYPE']._serialized_start=226 + _globals['_AUTOMATIONPROTOTYPE']._serialized_end=546 + _globals['_AUTOMATIONS']._serialized_start=548 + _globals['_AUTOMATIONS']._serialized_end=630 + _globals['_STORAGEEVENTTRIGGER']._serialized_start=633 + _globals['_STORAGEEVENTTRIGGER']._serialized_end=795 + _globals['_CRONTRIGGER']._serialized_start=798 + _globals['_CRONTRIGGER']._serialized_end=948 + _globals['_AUTOMATION']._serialized_start=950 + _globals['_AUTOMATION']._serialized_end=1019 + _globals['_TRIGGEREDCRONEVENT']._serialized_start=1021 + _globals['_TRIGGEREDCRONEVENT']._serialized_end=1104 + _globals['_DELETEAUTOMATIONREQUEST']._serialized_start=1106 + _globals['_DELETEAUTOMATIONREQUEST']._serialized_end=1217 + _globals['_AUTOMATIONSERVICE']._serialized_start=1220 + _globals['_AUTOMATIONSERVICE']._serialized_end=1640 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi index 38da1bd..74be9b9 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.pyi @@ -3,8 +3,8 @@ from google.protobuf import empty_pb2 as _empty_pb2 from google.protobuf import timestamp_pb2 as _timestamp_pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 from tilebox.workflows.workflows.v1 import core_pb2 as _core_pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as _storage_location_pb2 from google.protobuf.internal import containers as _containers -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from collections.abc import Iterable as _Iterable, Mapping as _Mapping @@ -12,40 +12,6 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor -class StorageType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - STORAGE_TYPE_UNSPECIFIED: _ClassVar[StorageType] - STORAGE_TYPE_GCS: _ClassVar[StorageType] - STORAGE_TYPE_S3: _ClassVar[StorageType] - STORAGE_TYPE_FS: _ClassVar[StorageType] - -class StorageEventType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - STORAGE_EVENT_TYPE_UNSPECIFIED: _ClassVar[StorageEventType] - STORAGE_EVENT_TYPE_CREATED: _ClassVar[StorageEventType] -STORAGE_TYPE_UNSPECIFIED: StorageType -STORAGE_TYPE_GCS: StorageType -STORAGE_TYPE_S3: StorageType -STORAGE_TYPE_FS: StorageType -STORAGE_EVENT_TYPE_UNSPECIFIED: StorageEventType -STORAGE_EVENT_TYPE_CREATED: StorageEventType - -class StorageLocation(_message.Message): - __slots__ = ("id", "location", "type") - ID_FIELD_NUMBER: _ClassVar[int] - LOCATION_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - id: _id_pb2.ID - location: str - type: StorageType - def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., location: _Optional[str] = ..., type: _Optional[_Union[StorageType, str]] = ...) -> None: ... - -class StorageLocations(_message.Message): - __slots__ = ("locations",) - LOCATIONS_FIELD_NUMBER: _ClassVar[int] - locations: _containers.RepeatedCompositeFieldContainer[StorageLocation] - def __init__(self, locations: _Optional[_Iterable[_Union[StorageLocation, _Mapping]]] = ...) -> None: ... - class AutomationPrototype(_message.Message): __slots__ = ("id", "name", "prototype", "storage_event_triggers", "cron_triggers", "disabled") ID_FIELD_NUMBER: _ClassVar[int] @@ -74,9 +40,9 @@ class StorageEventTrigger(_message.Message): STORAGE_LOCATION_FIELD_NUMBER: _ClassVar[int] GLOB_PATTERN_FIELD_NUMBER: _ClassVar[int] id: _id_pb2.ID - storage_location: StorageLocation + storage_location: _storage_location_pb2.StorageLocation glob_pattern: str - def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., storage_location: _Optional[_Union[StorageLocation, _Mapping]] = ..., glob_pattern: _Optional[str] = ...) -> None: ... + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., storage_location: _Optional[_Union[_storage_location_pb2.StorageLocation, _Mapping]] = ..., glob_pattern: _Optional[str] = ...) -> None: ... class CronTrigger(_message.Message): __slots__ = ("id", "schedule", "next_scheduled_at") @@ -96,16 +62,6 @@ class Automation(_message.Message): args: bytes def __init__(self, trigger_event: _Optional[bytes] = ..., args: _Optional[bytes] = ...) -> None: ... -class TriggeredStorageEvent(_message.Message): - __slots__ = ("storage_location_id", "type", "location") - STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - LOCATION_FIELD_NUMBER: _ClassVar[int] - storage_location_id: _id_pb2.ID - type: StorageEventType - location: str - def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageEventType, str]] = ..., location: _Optional[str] = ...) -> None: ... - class TriggeredCronEvent(_message.Message): __slots__ = ("trigger_time",) TRIGGER_TIME_FIELD_NUMBER: _ClassVar[int] diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py index 0ab210d..288c874 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py @@ -19,26 +19,6 @@ def __init__(self, channel): Args: channel: A grpc.Channel. """ - self.ListStorageLocations = channel.unary_unary( - '/workflows.v1.AutomationService/ListStorageLocations', - request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - response_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocations.FromString, - _registered_method=True) - self.GetStorageLocation = channel.unary_unary( - '/workflows.v1.AutomationService/GetStorageLocation', - request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - response_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - _registered_method=True) - self.CreateStorageLocation = channel.unary_unary( - '/workflows.v1.AutomationService/CreateStorageLocation', - request_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - response_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - _registered_method=True) - self.DeleteStorageLocation = channel.unary_unary( - '/workflows.v1.AutomationService/DeleteStorageLocation', - request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - _registered_method=True) self.ListAutomations = channel.unary_unary( '/workflows.v1.AutomationService/ListAutomations', request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, @@ -72,34 +52,6 @@ class AutomationServiceServicer: - Cron triggers, which triggers tasks on a schedule """ - def ListStorageLocations(self, request, context): - """ListStorageLocations lists all the storage buckets that are available for use as bucket triggers. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetStorageLocation(self, request, context): - """GetStorageLocation gets a storage location by its ID. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def CreateStorageLocation(self, request, context): - """CreateStorageLocation creates a new storage bucket. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def DeleteStorageLocation(self, request, context): - """DeleteStorageLocation deletes a storage location. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - def ListAutomations(self, request, context): """ListAutomations lists all the automations that are currently registered in a namespace. """ @@ -138,26 +90,6 @@ def DeleteAutomation(self, request, context): def add_AutomationServiceServicer_to_server(servicer, server): rpc_method_handlers = { - 'ListStorageLocations': grpc.unary_unary_rpc_method_handler( - servicer.ListStorageLocations, - request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - response_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocations.SerializeToString, - ), - 'GetStorageLocation': grpc.unary_unary_rpc_method_handler( - servicer.GetStorageLocation, - request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, - response_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - ), - 'CreateStorageLocation': grpc.unary_unary_rpc_method_handler( - servicer.CreateStorageLocation, - request_deserializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - response_serializer=workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - ), - 'DeleteStorageLocation': grpc.unary_unary_rpc_method_handler( - servicer.DeleteStorageLocation, - request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, - response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - ), 'ListAutomations': grpc.unary_unary_rpc_method_handler( servicer.ListAutomations, request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, @@ -197,114 +129,6 @@ class AutomationService: - Cron triggers, which triggers tasks on a schedule """ - @staticmethod - def ListStorageLocations(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/ListStorageLocations', - google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - workflows_dot_v1_dot_automation__pb2.StorageLocations.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def GetStorageLocation(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/GetStorageLocation', - tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def CreateStorageLocation(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/CreateStorageLocation', - workflows_dot_v1_dot_automation__pb2.StorageLocation.SerializeToString, - workflows_dot_v1_dot_automation__pb2.StorageLocation.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def DeleteStorageLocation(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/workflows.v1.AutomationService/DeleteStorageLocation', - tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, - google_dot_protobuf_dot_empty__pb2.Empty.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - @staticmethod def ListAutomations(request, target, diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py new file mode 100644 index 0000000..63b2cb2 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_connect.py @@ -0,0 +1,716 @@ +# -*- coding: utf-8 -*- +# Generated by https://github.com/connectrpc/connect-python. DO NOT EDIT! +# source: workflows/v1/storage_location.proto + +from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Iterator, Mapping +from typing import Protocol + +from connectrpc.client import ConnectClient, ConnectClientSync +from connectrpc.code import Code +from connectrpc.codec import Codec +from connectrpc.compression import Compression +from connectrpc.errors import ConnectError +from connectrpc.interceptor import Interceptor, InterceptorSync +from connectrpc.method import IdempotencyLevel, MethodInfo +from connectrpc.request import Headers, RequestContext +from connectrpc.server import ConnectASGIApplication, ConnectWSGIApplication, Endpoint, EndpointSync +import google.protobuf.empty_pb2 as google_dot_protobuf_dot_empty__pb2 +import tilebox.datasets.tilebox.v1.id_pb2 as tilebox_dot_v1_dot_id__pb2 +import tilebox.workflows.workflows.v1.storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 + + +class StorageLocationService(Protocol): + async def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def create_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def update_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def create_storage_subscription(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def list_storage_subscriptions(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def get_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def delete_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + async def list_storage_subscription_events(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + +class StorageLocationServiceASGIApplication(ConnectASGIApplication[StorageLocationService]): + def __init__(self, service: StorageLocationService | AsyncGenerator[StorageLocationService], *, interceptors: Iterable[Interceptor]=(), read_max_bytes: int | None = None, compressions: Iterable[Compression] | None = None, codecs: Iterable[Codec] | None = None) -> None: + super().__init__( + service=service, + endpoints=lambda svc: { + "/workflows.v1.StorageLocationService/ListStorageLocations": Endpoint.unary( + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_storage_locations, + ), + "/workflows.v1.StorageLocationService/GetStorageLocation": Endpoint.unary( + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.get_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageLocation": Endpoint.unary( + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.create_storage_location, + ), + "/workflows.v1.StorageLocationService/UpdateStorageLocation": Endpoint.unary( + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.update_storage_location, + ), + "/workflows.v1.StorageLocationService/DeleteStorageLocation": Endpoint.unary( + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.delete_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageSubscription": Endpoint.unary( + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.create_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptions": Endpoint.unary( + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_storage_subscriptions, + ), + "/workflows.v1.StorageLocationService/GetStorageSubscription": Endpoint.unary( + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.get_storage_subscription, + ), + "/workflows.v1.StorageLocationService/DeleteStorageSubscription": Endpoint.unary( + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.delete_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents": Endpoint.unary( + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_storage_subscription_events, + ), + }, + interceptors=interceptors, + read_max_bytes=read_max_bytes, + compressions=compressions, + codecs=codecs, + ) + + @property + def path(self) -> str: + """Returns the URL path to mount the application to when serving multiple applications.""" + return "/workflows.v1.StorageLocationService" + + +class StorageLocationServiceClient(ConnectClient): + async def list_storage_locations( + self, + request: google_dot_protobuf_dot_empty__pb2.Empty, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def get_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def create_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def update_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def delete_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def create_storage_subscription( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def list_storage_subscriptions( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def get_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def delete_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + async def list_storage_subscription_events( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + + + + +class StorageLocationServiceSync(Protocol): + def list_storage_locations(self, request: google_dot_protobuf_dot_empty__pb2.Empty, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def get_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def create_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def update_storage_location(self, request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def delete_storage_location(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def create_storage_subscription(self, request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def list_storage_subscriptions(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def get_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def delete_storage_subscription(self, request: tilebox_dot_v1_dot_id__pb2.ID, ctx: RequestContext) -> google_dot_protobuf_dot_empty__pb2.Empty: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def list_storage_subscription_events(self, request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + + +class StorageLocationServiceWSGIApplication(ConnectWSGIApplication): + def __init__(self, service: StorageLocationServiceSync, interceptors: Iterable[InterceptorSync]=(), read_max_bytes: int | None = None, compressions: Iterable[Compression] | None = None, codecs: Iterable[Codec] | None = None) -> None: + super().__init__( + endpoints={ + "/workflows.v1.StorageLocationService/ListStorageLocations": EndpointSync.unary( + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_storage_locations, + ), + "/workflows.v1.StorageLocationService/GetStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.get_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.create_storage_location, + ), + "/workflows.v1.StorageLocationService/UpdateStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.update_storage_location, + ), + "/workflows.v1.StorageLocationService/DeleteStorageLocation": EndpointSync.unary( + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.delete_storage_location, + ), + "/workflows.v1.StorageLocationService/CreateStorageSubscription": EndpointSync.unary( + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.create_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptions": EndpointSync.unary( + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_storage_subscriptions, + ), + "/workflows.v1.StorageLocationService/GetStorageSubscription": EndpointSync.unary( + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.get_storage_subscription, + ), + "/workflows.v1.StorageLocationService/DeleteStorageSubscription": EndpointSync.unary( + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.delete_storage_subscription, + ), + "/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents": EndpointSync.unary( + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_storage_subscription_events, + ), + }, + interceptors=interceptors, + read_max_bytes=read_max_bytes, + compressions=compressions, + codecs=codecs, + ) + + @property + def path(self) -> str: + """Returns the URL path to mount the application to when serving multiple applications.""" + return "/workflows.v1.StorageLocationService" + + +class StorageLocationServiceClientSync(ConnectClientSync): + def list_storage_locations( + self, + request: google_dot_protobuf_dot_empty__pb2.Empty, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocations: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageLocations", + service_name="workflows.v1.StorageLocationService", + input=google_dot_protobuf_dot_empty__pb2.Empty, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocations, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def get_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def create_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def update_storage_location( + self, + request: workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageLocation: + return self.execute_unary( + request=request, + method=MethodInfo( + name="UpdateStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageLocation, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def delete_storage_location( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageLocation", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def create_storage_subscription( + self, + request: workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return self.execute_unary( + request=request, + method=MethodInfo( + name="CreateStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def list_storage_subscriptions( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptions", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def get_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscription: + return self.execute_unary( + request=request, + method=MethodInfo( + name="GetStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def delete_storage_subscription( + self, + request: tilebox_dot_v1_dot_id__pb2.ID, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> google_dot_protobuf_dot_empty__pb2.Empty: + return self.execute_unary( + request=request, + method=MethodInfo( + name="DeleteStorageSubscription", + service_name="workflows.v1.StorageLocationService", + input=tilebox_dot_v1_dot_id__pb2.ID, + output=google_dot_protobuf_dot_empty__pb2.Empty, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + + def list_storage_subscription_events( + self, + request: workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListStorageSubscriptionEvents", + service_name="workflows.v1.StorageLocationService", + input=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest, + output=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py new file mode 100644 index 0000000..fcfb70e --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: workflows/v1/storage_location.proto +# Protobuf Python Version: 6.30.0 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 30, + 0, + '', + 'workflows/v1/storage_location.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 +from tilebox.datasets.tilebox.v1 import query_pb2 as tilebox_dot_v1_dot_query__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#workflows/v1/storage_location.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\"\xde\x01\n\x0fStorageLocation\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1e\n\x08location\x18\x02 \x01(\tB\x02\x18\x01R\x08location\x12\x31\n\x04type\x18\x03 \x01(\x0e\x32\x19.workflows.v1.StorageTypeB\x02\x18\x01R\x04type\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x44\n\treference\x18\x05 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"\xd6\x02\n\x18StorageLocationReference\x12\x46\n\raws_s3_bucket\x18\x01 \x01(\x0b\x32\".workflows.v1.AWSS3BucketReferenceR\x0b\x61wsS3Bucket\x12?\n\ngcs_bucket\x18\x02 \x01(\x0b\x32 .workflows.v1.GCSBucketReferenceR\tgcsBucket\x12?\n\nazure_blob\x18\x03 \x01(\x0b\x32 .workflows.v1.AzureBlobReferenceR\tazureBlob\x12\x41\n\nfilesystem\x18\x04 \x01(\x0b\x32!.workflows.v1.FilesystemReferenceR\nfilesystem\x12-\n\x04type\x18\x05 \x01(\x0e\x32\x19.workflows.v1.StorageTypeR\x04type\"F\n\x14\x41WSS3BucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x16\n\x06region\x18\x02 \x01(\tR\x06region\"g\n\x12GCSBucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x1d\n\nproject_id\x18\x02 \x01(\tR\tprojectId\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location\"\x89\x01\n\x12\x41zureBlobReference\x12=\n\x1bstorage_account_resource_id\x18\x01 \x01(\tR\x18storageAccountResourceId\x12\x1c\n\tcontainer\x18\x02 \x01(\tR\tcontainer\x12\x16\n\x06region\x18\x03 \x01(\tR\x06region\")\n\x13\x46ilesystemReference\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\"\x8d\x01\n\x0cTriggeredJob\x12%\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x05jobId\x12\x33\n\rautomation_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"R\n\rTriggeredJobs\x12\x41\n\x0etriggered_jobs\x18\x01 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\"x\n\x1c\x43reateStorageLocationRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\treference\x18\x02 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"r\n\x1cUpdateStorageLocationRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"8\n\x19\x41WSSNSStorageSubscription\x12\x1b\n\ttopic_arn\x18\x01 \x01(\tR\x08topicArn\"\x9a\x01\n\x1fGooglePubSubStorageSubscription\x12\"\n\x0csubscription\x18\x01 \x01(\tR\x0csubscription\x12\x32\n\x15service_account_email\x18\x02 \x01(\tR\x13serviceAccountEmail\x12\x1f\n\x08\x61udience\x18\x03 \x01(\tB\x03\xe0\x41\x03R\x08\x61udience\"\xbd\x01\n!AzureEventGridStorageSubscription\x12\x37\n\x15webhook_secret_header\x18\x01 \x01(\tB\x03\xe0\x41\x03R\x13webhookSecretHeader\x12*\n\x0ewebhook_secret\x18\x02 \x01(\tB\x03\xe0\x41\x03R\rwebhookSecret\x12\x33\n\x13webhook_secret_hash\x18\x03 \x01(\x0c\x42\x03\xe0\x41\x03R\x11webhookSecretHash\"\x87\x04\n\x13StorageSubscription\x12#\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x03\xe0\x41\x03R\x02id\x12>\n\x13storage_location_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x03 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12\x1f\n\x08\x65ndpoint\x18\x04 \x01(\tB\x03\xe0\x41\x03R\x08\x65ndpoint\x12>\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\tcreatedAt\x12@\n\x07\x61ws_sns\x18\x07 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x08 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\t \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"\x8e\x03\n CreateStorageSubscriptionRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x02 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12@\n\x07\x61ws_sns\x18\x03 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x04 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\x05 \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"a\n\x1fListStorageSubscriptionsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\"_\n\x14StorageSubscriptions\x12G\n\rsubscriptions\x18\x01 \x03(\x0b\x32!.workflows.v1.StorageSubscriptionR\rsubscriptions\"\x90\x03\n\x18StorageSubscriptionEvent\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1d\n\nobject_key\x18\x02 \x01(\tR\tobjectKey\x12\x32\n\x04type\x18\x03 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x39\n\nevent_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12;\n\x0breceived_at\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\nreceivedAt\x12\x41\n\x0etriggered_jobs\x18\x06 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\x12\x46\n\x17storage_subscription_id\x18\x07 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x15storageSubscriptionId\"\x99\x01\n$ListStorageSubscriptionEventsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"\x97\x01\n\x19StorageSubscriptionEvents\x12>\n\x06\x65vents\x18\x01 \x03(\x0b\x32&.workflows.v1.StorageSubscriptionEventR\x06\x65vents\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"O\n\x10StorageLocations\x12;\n\tlocations\x18\x01 \x03(\x0b\x32\x1d.workflows.v1.StorageLocationR\tlocations\"\xa7\x01\n\x15TriggeredStorageEvent\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location*\xa2\x01\n\x0bStorageType\x12\x1c\n\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17STORAGE_TYPE_GCS_BUCKET\x10\x01\x12\x1e\n\x1aSTORAGE_TYPE_AWS_S3_BUCKET\x10\x02\x12\x1b\n\x17STORAGE_TYPE_FILESYSTEM\x10\x03\x12\x1b\n\x17STORAGE_TYPE_AZURE_BLOB\x10\x04*\xc8\x01\n\x17StorageSubscriptionType\x12)\n%STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED\x10\x00\x12%\n!STORAGE_SUBSCRIPTION_TYPE_AWS_SNS\x10\x01\x12+\n\'STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB\x10\x02\x12.\n*STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID\x10\x03*V\n\x10StorageEventType\x12\"\n\x1eSTORAGE_EVENT_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTORAGE_EVENT_TYPE_CREATED\x10\x01\x32\xa5\x07\n\x16StorageLocationService\x12N\n\x14ListStorageLocations\x12\x16.google.protobuf.Empty\x1a\x1e.workflows.v1.StorageLocations\x12\x43\n\x12GetStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15\x43reateStorageLocation\x12*.workflows.v1.CreateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15UpdateStorageLocation\x12*.workflows.v1.UpdateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12?\n\x15\x44\x65leteStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12n\n\x19\x43reateStorageSubscription\x12..workflows.v1.CreateStorageSubscriptionRequest\x1a!.workflows.v1.StorageSubscription\x12m\n\x18ListStorageSubscriptions\x12-.workflows.v1.ListStorageSubscriptionsRequest\x1a\".workflows.v1.StorageSubscriptions\x12K\n\x16GetStorageSubscription\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.StorageSubscription\x12\x43\n\x19\x44\x65leteStorageSubscription\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12|\n\x1dListStorageSubscriptionEvents\x12\x32.workflows.v1.ListStorageSubscriptionEventsRequest\x1a\'.workflows.v1.StorageSubscriptionEventsB\x05\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.storage_location_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\222\003\002\010\002' + _globals['_STORAGELOCATION'].fields_by_name['location']._loaded_options = None + _globals['_STORAGELOCATION'].fields_by_name['location']._serialized_options = b'\030\001' + _globals['_STORAGELOCATION'].fields_by_name['type']._loaded_options = None + _globals['_STORAGELOCATION'].fields_by_name['type']._serialized_options = b'\030\001' + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION'].fields_by_name['audience']._loaded_options = None + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION'].fields_by_name['audience']._serialized_options = b'\340A\003' + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_header']._loaded_options = None + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_header']._serialized_options = b'\340A\003' + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret']._loaded_options = None + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret']._serialized_options = b'\340A\003' + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_hash']._loaded_options = None + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION'].fields_by_name['webhook_secret_hash']._serialized_options = b'\340A\003' + _globals['_STORAGESUBSCRIPTION'].fields_by_name['id']._loaded_options = None + _globals['_STORAGESUBSCRIPTION'].fields_by_name['id']._serialized_options = b'\340A\003' + _globals['_STORAGESUBSCRIPTION'].fields_by_name['endpoint']._loaded_options = None + _globals['_STORAGESUBSCRIPTION'].fields_by_name['endpoint']._serialized_options = b'\340A\003' + _globals['_STORAGESUBSCRIPTION'].fields_by_name['created_at']._loaded_options = None + _globals['_STORAGESUBSCRIPTION'].fields_by_name['created_at']._serialized_options = b'\340A\003' + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST'].fields_by_name['page']._loaded_options = None + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST'].fields_by_name['page']._serialized_options = b'\252\001\002\010\001' + _globals['_STORAGESUBSCRIPTIONEVENTS'].fields_by_name['next_page']._loaded_options = None + _globals['_STORAGESUBSCRIPTIONEVENTS'].fields_by_name['next_page']._serialized_options = b'\252\001\002\010\001' + _globals['_STORAGETYPE']._serialized_start=4080 + _globals['_STORAGETYPE']._serialized_end=4242 + _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_start=4245 + _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_end=4445 + _globals['_STORAGEEVENTTYPE']._serialized_start=4447 + _globals['_STORAGEEVENTTYPE']._serialized_end=4533 + _globals['_STORAGELOCATION']._serialized_start=194 + _globals['_STORAGELOCATION']._serialized_end=416 + _globals['_STORAGELOCATIONREFERENCE']._serialized_start=419 + _globals['_STORAGELOCATIONREFERENCE']._serialized_end=761 + _globals['_AWSS3BUCKETREFERENCE']._serialized_start=763 + _globals['_AWSS3BUCKETREFERENCE']._serialized_end=833 + _globals['_GCSBUCKETREFERENCE']._serialized_start=835 + _globals['_GCSBUCKETREFERENCE']._serialized_end=938 + _globals['_AZUREBLOBREFERENCE']._serialized_start=941 + _globals['_AZUREBLOBREFERENCE']._serialized_end=1078 + _globals['_FILESYSTEMREFERENCE']._serialized_start=1080 + _globals['_FILESYSTEMREFERENCE']._serialized_end=1121 + _globals['_TRIGGEREDJOB']._serialized_start=1124 + _globals['_TRIGGEREDJOB']._serialized_end=1265 + _globals['_TRIGGEREDJOBS']._serialized_start=1267 + _globals['_TRIGGEREDJOBS']._serialized_end=1349 + _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_start=1351 + _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_end=1471 + _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_start=1473 + _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_end=1587 + _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_start=1589 + _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_end=1645 + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_start=1648 + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_end=1802 + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_start=1805 + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_end=1994 + _globals['_STORAGESUBSCRIPTION']._serialized_start=1997 + _globals['_STORAGESUBSCRIPTION']._serialized_end=2516 + _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_start=2519 + _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_end=2917 + _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_start=2919 + _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_end=3016 + _globals['_STORAGESUBSCRIPTIONS']._serialized_start=3018 + _globals['_STORAGESUBSCRIPTIONS']._serialized_end=3113 + _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_start=3116 + _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_end=3516 + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_start=3519 + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_end=3672 + _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_start=3675 + _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_end=3826 + _globals['_STORAGELOCATIONS']._serialized_start=3828 + _globals['_STORAGELOCATIONS']._serialized_end=3907 + _globals['_TRIGGEREDSTORAGEEVENT']._serialized_start=3910 + _globals['_TRIGGEREDSTORAGEEVENT']._serialized_end=4077 + _globals['_STORAGELOCATIONSERVICE']._serialized_start=4536 + _globals['_STORAGELOCATIONSERVICE']._serialized_end=5469 +# @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi new file mode 100644 index 0000000..3e46926 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi @@ -0,0 +1,260 @@ +from google.api import field_behavior_pb2 as _field_behavior_pb2 +from google.protobuf import empty_pb2 as _empty_pb2 +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 +from tilebox.datasets.tilebox.v1 import query_pb2 as _query_pb2 +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Iterable as _Iterable, Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class StorageType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + STORAGE_TYPE_UNSPECIFIED: _ClassVar[StorageType] + STORAGE_TYPE_GCS_BUCKET: _ClassVar[StorageType] + STORAGE_TYPE_AWS_S3_BUCKET: _ClassVar[StorageType] + STORAGE_TYPE_FILESYSTEM: _ClassVar[StorageType] + STORAGE_TYPE_AZURE_BLOB: _ClassVar[StorageType] + +class StorageSubscriptionType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_AWS_SNS: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID: _ClassVar[StorageSubscriptionType] + +class StorageEventType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + STORAGE_EVENT_TYPE_UNSPECIFIED: _ClassVar[StorageEventType] + STORAGE_EVENT_TYPE_CREATED: _ClassVar[StorageEventType] +STORAGE_TYPE_UNSPECIFIED: StorageType +STORAGE_TYPE_GCS_BUCKET: StorageType +STORAGE_TYPE_AWS_S3_BUCKET: StorageType +STORAGE_TYPE_FILESYSTEM: StorageType +STORAGE_TYPE_AZURE_BLOB: StorageType +STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_AWS_SNS: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID: StorageSubscriptionType +STORAGE_EVENT_TYPE_UNSPECIFIED: StorageEventType +STORAGE_EVENT_TYPE_CREATED: StorageEventType + +class StorageLocation(_message.Message): + __slots__ = ("id", "location", "type", "name", "reference") + ID_FIELD_NUMBER: _ClassVar[int] + LOCATION_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + REFERENCE_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + location: str + type: StorageType + name: str + reference: StorageLocationReference + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., location: _Optional[str] = ..., type: _Optional[_Union[StorageType, str]] = ..., name: _Optional[str] = ..., reference: _Optional[_Union[StorageLocationReference, _Mapping]] = ...) -> None: ... + +class StorageLocationReference(_message.Message): + __slots__ = ("aws_s3_bucket", "gcs_bucket", "azure_blob", "filesystem", "type") + AWS_S3_BUCKET_FIELD_NUMBER: _ClassVar[int] + GCS_BUCKET_FIELD_NUMBER: _ClassVar[int] + AZURE_BLOB_FIELD_NUMBER: _ClassVar[int] + FILESYSTEM_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + aws_s3_bucket: AWSS3BucketReference + gcs_bucket: GCSBucketReference + azure_blob: AzureBlobReference + filesystem: FilesystemReference + type: StorageType + def __init__(self, aws_s3_bucket: _Optional[_Union[AWSS3BucketReference, _Mapping]] = ..., gcs_bucket: _Optional[_Union[GCSBucketReference, _Mapping]] = ..., azure_blob: _Optional[_Union[AzureBlobReference, _Mapping]] = ..., filesystem: _Optional[_Union[FilesystemReference, _Mapping]] = ..., type: _Optional[_Union[StorageType, str]] = ...) -> None: ... + +class AWSS3BucketReference(_message.Message): + __slots__ = ("bucket", "region") + BUCKET_FIELD_NUMBER: _ClassVar[int] + REGION_FIELD_NUMBER: _ClassVar[int] + bucket: str + region: str + def __init__(self, bucket: _Optional[str] = ..., region: _Optional[str] = ...) -> None: ... + +class GCSBucketReference(_message.Message): + __slots__ = ("bucket", "project_id", "location") + BUCKET_FIELD_NUMBER: _ClassVar[int] + PROJECT_ID_FIELD_NUMBER: _ClassVar[int] + LOCATION_FIELD_NUMBER: _ClassVar[int] + bucket: str + project_id: str + location: str + def __init__(self, bucket: _Optional[str] = ..., project_id: _Optional[str] = ..., location: _Optional[str] = ...) -> None: ... + +class AzureBlobReference(_message.Message): + __slots__ = ("storage_account_resource_id", "container", "region") + STORAGE_ACCOUNT_RESOURCE_ID_FIELD_NUMBER: _ClassVar[int] + CONTAINER_FIELD_NUMBER: _ClassVar[int] + REGION_FIELD_NUMBER: _ClassVar[int] + storage_account_resource_id: str + container: str + region: str + def __init__(self, storage_account_resource_id: _Optional[str] = ..., container: _Optional[str] = ..., region: _Optional[str] = ...) -> None: ... + +class FilesystemReference(_message.Message): + __slots__ = ("path",) + PATH_FIELD_NUMBER: _ClassVar[int] + path: str + def __init__(self, path: _Optional[str] = ...) -> None: ... + +class TriggeredJob(_message.Message): + __slots__ = ("job_id", "automation_id", "glob_pattern") + JOB_ID_FIELD_NUMBER: _ClassVar[int] + AUTOMATION_ID_FIELD_NUMBER: _ClassVar[int] + GLOB_PATTERN_FIELD_NUMBER: _ClassVar[int] + job_id: _id_pb2.ID + automation_id: _id_pb2.ID + glob_pattern: str + def __init__(self, job_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., automation_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., glob_pattern: _Optional[str] = ...) -> None: ... + +class TriggeredJobs(_message.Message): + __slots__ = ("triggered_jobs",) + TRIGGERED_JOBS_FIELD_NUMBER: _ClassVar[int] + triggered_jobs: _containers.RepeatedCompositeFieldContainer[TriggeredJob] + def __init__(self, triggered_jobs: _Optional[_Iterable[_Union[TriggeredJob, _Mapping]]] = ...) -> None: ... + +class CreateStorageLocationRequest(_message.Message): + __slots__ = ("name", "reference") + NAME_FIELD_NUMBER: _ClassVar[int] + REFERENCE_FIELD_NUMBER: _ClassVar[int] + name: str + reference: StorageLocationReference + def __init__(self, name: _Optional[str] = ..., reference: _Optional[_Union[StorageLocationReference, _Mapping]] = ...) -> None: ... + +class UpdateStorageLocationRequest(_message.Message): + __slots__ = ("storage_location_id", "name") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + name: str + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., name: _Optional[str] = ...) -> None: ... + +class AWSSNSStorageSubscription(_message.Message): + __slots__ = ("topic_arn",) + TOPIC_ARN_FIELD_NUMBER: _ClassVar[int] + topic_arn: str + def __init__(self, topic_arn: _Optional[str] = ...) -> None: ... + +class GooglePubSubStorageSubscription(_message.Message): + __slots__ = ("subscription", "service_account_email", "audience") + SUBSCRIPTION_FIELD_NUMBER: _ClassVar[int] + SERVICE_ACCOUNT_EMAIL_FIELD_NUMBER: _ClassVar[int] + AUDIENCE_FIELD_NUMBER: _ClassVar[int] + subscription: str + service_account_email: str + audience: str + def __init__(self, subscription: _Optional[str] = ..., service_account_email: _Optional[str] = ..., audience: _Optional[str] = ...) -> None: ... + +class AzureEventGridStorageSubscription(_message.Message): + __slots__ = ("webhook_secret_header", "webhook_secret", "webhook_secret_hash") + WEBHOOK_SECRET_HEADER_FIELD_NUMBER: _ClassVar[int] + WEBHOOK_SECRET_FIELD_NUMBER: _ClassVar[int] + WEBHOOK_SECRET_HASH_FIELD_NUMBER: _ClassVar[int] + webhook_secret_header: str + webhook_secret: str + webhook_secret_hash: bytes + def __init__(self, webhook_secret_header: _Optional[str] = ..., webhook_secret: _Optional[str] = ..., webhook_secret_hash: _Optional[bytes] = ...) -> None: ... + +class StorageSubscription(_message.Message): + __slots__ = ("id", "storage_location_id", "type", "endpoint", "created_at", "aws_sns", "google_pubsub", "azure_event_grid") + ID_FIELD_NUMBER: _ClassVar[int] + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + ENDPOINT_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + AWS_SNS_FIELD_NUMBER: _ClassVar[int] + GOOGLE_PUBSUB_FIELD_NUMBER: _ClassVar[int] + AZURE_EVENT_GRID_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + storage_location_id: _id_pb2.ID + type: StorageSubscriptionType + endpoint: str + created_at: _timestamp_pb2.Timestamp + aws_sns: AWSSNSStorageSubscription + google_pubsub: GooglePubSubStorageSubscription + azure_event_grid: AzureEventGridStorageSubscription + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageSubscriptionType, str]] = ..., endpoint: _Optional[str] = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., aws_sns: _Optional[_Union[AWSSNSStorageSubscription, _Mapping]] = ..., google_pubsub: _Optional[_Union[GooglePubSubStorageSubscription, _Mapping]] = ..., azure_event_grid: _Optional[_Union[AzureEventGridStorageSubscription, _Mapping]] = ...) -> None: ... + +class CreateStorageSubscriptionRequest(_message.Message): + __slots__ = ("storage_location_id", "type", "aws_sns", "google_pubsub", "azure_event_grid") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + AWS_SNS_FIELD_NUMBER: _ClassVar[int] + GOOGLE_PUBSUB_FIELD_NUMBER: _ClassVar[int] + AZURE_EVENT_GRID_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + type: StorageSubscriptionType + aws_sns: AWSSNSStorageSubscription + google_pubsub: GooglePubSubStorageSubscription + azure_event_grid: AzureEventGridStorageSubscription + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageSubscriptionType, str]] = ..., aws_sns: _Optional[_Union[AWSSNSStorageSubscription, _Mapping]] = ..., google_pubsub: _Optional[_Union[GooglePubSubStorageSubscription, _Mapping]] = ..., azure_event_grid: _Optional[_Union[AzureEventGridStorageSubscription, _Mapping]] = ...) -> None: ... + +class ListStorageSubscriptionsRequest(_message.Message): + __slots__ = ("storage_location_id",) + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ...) -> None: ... + +class StorageSubscriptions(_message.Message): + __slots__ = ("subscriptions",) + SUBSCRIPTIONS_FIELD_NUMBER: _ClassVar[int] + subscriptions: _containers.RepeatedCompositeFieldContainer[StorageSubscription] + def __init__(self, subscriptions: _Optional[_Iterable[_Union[StorageSubscription, _Mapping]]] = ...) -> None: ... + +class StorageSubscriptionEvent(_message.Message): + __slots__ = ("id", "object_key", "type", "event_time", "received_at", "triggered_jobs", "storage_subscription_id") + ID_FIELD_NUMBER: _ClassVar[int] + OBJECT_KEY_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + EVENT_TIME_FIELD_NUMBER: _ClassVar[int] + RECEIVED_AT_FIELD_NUMBER: _ClassVar[int] + TRIGGERED_JOBS_FIELD_NUMBER: _ClassVar[int] + STORAGE_SUBSCRIPTION_ID_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + object_key: str + type: StorageEventType + event_time: _timestamp_pb2.Timestamp + received_at: _timestamp_pb2.Timestamp + triggered_jobs: _containers.RepeatedCompositeFieldContainer[TriggeredJob] + storage_subscription_id: _id_pb2.ID + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., object_key: _Optional[str] = ..., type: _Optional[_Union[StorageEventType, str]] = ..., event_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., received_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., triggered_jobs: _Optional[_Iterable[_Union[TriggeredJob, _Mapping]]] = ..., storage_subscription_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ...) -> None: ... + +class ListStorageSubscriptionEventsRequest(_message.Message): + __slots__ = ("storage_location_id", "page") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + PAGE_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + page: _query_pb2.Pagination + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + +class StorageSubscriptionEvents(_message.Message): + __slots__ = ("events", "next_page") + EVENTS_FIELD_NUMBER: _ClassVar[int] + NEXT_PAGE_FIELD_NUMBER: _ClassVar[int] + events: _containers.RepeatedCompositeFieldContainer[StorageSubscriptionEvent] + next_page: _query_pb2.Pagination + def __init__(self, events: _Optional[_Iterable[_Union[StorageSubscriptionEvent, _Mapping]]] = ..., next_page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + +class StorageLocations(_message.Message): + __slots__ = ("locations",) + LOCATIONS_FIELD_NUMBER: _ClassVar[int] + locations: _containers.RepeatedCompositeFieldContainer[StorageLocation] + def __init__(self, locations: _Optional[_Iterable[_Union[StorageLocation, _Mapping]]] = ...) -> None: ... + +class TriggeredStorageEvent(_message.Message): + __slots__ = ("storage_location_id", "type", "location") + STORAGE_LOCATION_ID_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + LOCATION_FIELD_NUMBER: _ClassVar[int] + storage_location_id: _id_pb2.ID + type: StorageEventType + location: str + def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., type: _Optional[_Union[StorageEventType, str]] = ..., location: _Optional[str] = ...) -> None: ... diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py new file mode 100644 index 0000000..8ff71ea --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2_grpc.py @@ -0,0 +1,479 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 +from tilebox.workflows.workflows.v1 import storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 + + +class StorageLocationServiceStub: + """StorageLocationService manages storage locations, notification subscriptions, and received events. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListStorageLocations = channel.unary_unary( + '/workflows.v1.StorageLocationService/ListStorageLocations', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocations.FromString, + _registered_method=True) + self.GetStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/GetStorageLocation', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + _registered_method=True) + self.CreateStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/CreateStorageLocation', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + _registered_method=True) + self.UpdateStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/UpdateStorageLocation', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + _registered_method=True) + self.DeleteStorageLocation = channel.unary_unary( + '/workflows.v1.StorageLocationService/DeleteStorageLocation', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + _registered_method=True) + self.CreateStorageSubscription = channel.unary_unary( + '/workflows.v1.StorageLocationService/CreateStorageSubscription', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + _registered_method=True) + self.ListStorageSubscriptions = channel.unary_unary( + '/workflows.v1.StorageLocationService/ListStorageSubscriptions', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions.FromString, + _registered_method=True) + self.GetStorageSubscription = channel.unary_unary( + '/workflows.v1.StorageLocationService/GetStorageSubscription', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + _registered_method=True) + self.DeleteStorageSubscription = channel.unary_unary( + '/workflows.v1.StorageLocationService/DeleteStorageSubscription', + request_serializer=tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + _registered_method=True) + self.ListStorageSubscriptionEvents = channel.unary_unary( + '/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents', + request_serializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents.FromString, + _registered_method=True) + + +class StorageLocationServiceServicer: + """StorageLocationService manages storage locations, notification subscriptions, and received events. + """ + + def ListStorageLocations(self, request, context): + """ListStorageLocations lists storage locations available for automation triggers. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetStorageLocation(self, request, context): + """GetStorageLocation gets a storage location by its ID. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateStorageLocation(self, request, context): + """CreateStorageLocation registers an existing bucket, container, or directory for automation triggers. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UpdateStorageLocation(self, request, context): + """UpdateStorageLocation changes the display name of a storage location. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteStorageLocation(self, request, context): + """DeleteStorageLocation removes the location and its subscriptions from Tilebox without deleting cloud resources. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CreateStorageSubscription(self, request, context): + """CreateStorageSubscription registers notification delivery and returns the provider setup values. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListStorageSubscriptions(self, request, context): + """ListStorageSubscriptions lists notification subscriptions for a storage location. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetStorageSubscription(self, request, context): + """GetStorageSubscription gets a subscription by its ID. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteStorageSubscription(self, request, context): + """DeleteStorageSubscription stops accepting notifications at its endpoint without deleting cloud resources. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListStorageSubscriptionEvents(self, request, context): + """ListStorageSubscriptionEvents lists received object notifications and the jobs they triggered. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_StorageLocationServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListStorageLocations': grpc.unary_unary_rpc_method_handler( + servicer.ListStorageLocations, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocations.SerializeToString, + ), + 'GetStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.GetStorageLocation, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.SerializeToString, + ), + 'CreateStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.CreateStorageLocation, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.SerializeToString, + ), + 'UpdateStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.UpdateStorageLocation, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageLocation.SerializeToString, + ), + 'DeleteStorageLocation': grpc.unary_unary_rpc_method_handler( + servicer.DeleteStorageLocation, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'CreateStorageSubscription': grpc.unary_unary_rpc_method_handler( + servicer.CreateStorageSubscription, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.SerializeToString, + ), + 'ListStorageSubscriptions': grpc.unary_unary_rpc_method_handler( + servicer.ListStorageSubscriptions, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions.SerializeToString, + ), + 'GetStorageSubscription': grpc.unary_unary_rpc_method_handler( + servicer.GetStorageSubscription, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.SerializeToString, + ), + 'DeleteStorageSubscription': grpc.unary_unary_rpc_method_handler( + servicer.DeleteStorageSubscription, + request_deserializer=tilebox_dot_v1_dot_id__pb2.ID.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + 'ListStorageSubscriptionEvents': grpc.unary_unary_rpc_method_handler( + servicer.ListStorageSubscriptionEvents, + request_deserializer=workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest.FromString, + response_serializer=workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'workflows.v1.StorageLocationService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('workflows.v1.StorageLocationService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class StorageLocationService: + """StorageLocationService manages storage locations, notification subscriptions, and received events. + """ + + @staticmethod + def ListStorageLocations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/ListStorageLocations', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocations.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/GetStorageLocation', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def CreateStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/CreateStorageLocation', + workflows_dot_v1_dot_storage__location__pb2.CreateStorageLocationRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def UpdateStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/UpdateStorageLocation', + workflows_dot_v1_dot_storage__location__pb2.UpdateStorageLocationRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageLocation.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def DeleteStorageLocation(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/DeleteStorageLocation', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + google_dot_protobuf_dot_empty__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def CreateStorageSubscription(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/CreateStorageSubscription', + workflows_dot_v1_dot_storage__location__pb2.CreateStorageSubscriptionRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListStorageSubscriptions(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/ListStorageSubscriptions', + workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionsRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptions.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetStorageSubscription(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/GetStorageSubscription', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscription.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def DeleteStorageSubscription(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/DeleteStorageSubscription', + tilebox_dot_v1_dot_id__pb2.ID.SerializeToString, + google_dot_protobuf_dot_empty__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListStorageSubscriptionEvents(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.StorageLocationService/ListStorageSubscriptionEvents', + workflows_dot_v1_dot_storage__location__pb2.ListStorageSubscriptionEventsRequest.SerializeToString, + workflows_dot_v1_dot_storage__location__pb2.StorageSubscriptionEvents.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/uv.lock b/uv.lock index 9f1d507..ade3ce2 100644 --- a/uv.lock +++ b/uv.lock @@ -121,6 +121,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "azure-core" +version = "1.41.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, +] + +[[package]] +name = "azure-identity" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "msal" }, + { name = "msal-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/87/e88098a024d74b7434de5b110c261da024fe5d6c139f1847f6f8a2b76678/azure_identity-1.26.0.tar.gz", hash = "sha256:61b4538b87df10d2d6d44b77cd5ea28a8929fd4929d58b45bf3f230f7daba02e", size = 310151, upload-time = "2026-10-01T14:17:47.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/d5/ae45ce8009d0d015304b206c7c4a6cbab02c03f8f07eedefb32b099ccee3/azure_identity-1.26.0-py3-none-any.whl", hash = "sha256:3c60d9682b3ac01507388d474cb5a5ffbe54152a96b6cdcaa7013da9315cf186", size = 195958, upload-time = "2026-10-01T14:17:49.705Z" }, +] + [[package]] name = "boto3" version = "1.43.101" @@ -751,23 +780,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] -[[package]] -name = "google-api-core" -version = "2.38.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/59/e5/18aeff14213db86267a0f79d869352954394ab4ab3a3866d9b9b0e82dc6a/google_api_core-2.38.0.tar.gz", hash = "sha256:31e326eafa31b34f1db7a715f50f61dbca7e6e37277762f252ed90e6ce94c246", size = 206784, upload-time = "2026-09-17T20:30:08.766Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/55/e545e4b0040cb812f65d2abc545c6420f57b687d2af130a447c86d14d00b/google_api_core-2.38.0-py3-none-any.whl", hash = "sha256:8db2730375eba434bd75041fbe06b5374287407689ce132ac5650c8c8f042fcc", size = 187996, upload-time = "2026-09-17T20:29:32.813Z" }, -] - [[package]] name = "google-auth" version = "2.58.0" @@ -781,77 +793,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/13/477d90d09591b3938b45c4e11f4d8a51291682112cb5efcac961e815d562/google_auth-2.58.0-py3-none-any.whl", hash = "sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d", size = 262404, upload-time = "2026-09-09T20:49:33.951Z" }, ] -[[package]] -name = "google-cloud-core" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f9/54/790548b190cff9cf07225c811d2668eacd8f2cbf04114475cbe3e5738752/google_cloud_core-2.7.0.tar.gz", hash = "sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d", size = 36603, upload-time = "2026-08-25T19:18:43.076Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/64/904dbc9bee128e7b87eeb60da67c8fa4d1a8acdc9a45dd2fedca67ef184d/google_cloud_core-2.7.0-py3-none-any.whl", hash = "sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1", size = 31046, upload-time = "2026-08-24T21:55:28.36Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "3.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, +[package.optional-dependencies] +requests = [ { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/06/33e124df40437c292c7d666616d890c12cfea5b83842acf5be7dc3fde016/google_cloud_storage-3.14.1.tar.gz", hash = "sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7", size = 17348933, upload-time = "2026-09-08T17:10:11.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/65/7201c1816ae46e0b4e70a3ce61e5121963315dc9df2e00fcb8897a3d0926/google_cloud_storage-3.14.1-py3-none-any.whl", hash = "sha256:8f0fe2b74bd80ddcbd67247a3b47b81035a0a4f448569ed358fa40a46c940118", size = 343228, upload-time = "2026-09-08T17:10:09.833Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" }, - { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" }, - { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" }, - { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, - { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, - { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, - { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, - { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, - { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, - { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" }, - { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/17/4b/44e128cd7b0e72cd26da4a3052cd8a9ef1b5a890be7c611558067ea0b354/google_resumable_media-2.10.2.tar.gz", hash = "sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c", size = 2164918, upload-time = "2026-08-25T19:19:11.814Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/55/30/3e976681b5319715ab022363f58f6f8e10f546a1777756fe3ebd918857e4/google_resumable_media-2.10.2-py3-none-any.whl", hash = "sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7", size = 81531, upload-time = "2026-08-25T19:18:07.193Z" }, -] [[package]] name = "googleapis-common-protos" @@ -1363,6 +1308,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/80/ee7ace7a49bd525455497fa2ff198111877d6f93b908b657adeb2cf057a4/moto-5.2.3-py3-none-any.whl", hash = "sha256:5ec31b3cdbb383a19d46030157bd82aaaa7a9cc5f564200f995663d347bd4728", size = 6780972, upload-time = "2026-08-22T18:45:55.719Z" }, ] +[[package]] +name = "msal" +version = "1.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/85/747d28986a44b715cc51cf57aa021c8fd94e9614064261ceec77320e8b0b/msal-1.39.0.tar.gz", hash = "sha256:6ab7de335e6d7f5717e2c7e1dbf86e4dda2f6acf3c56773b78dc53ebc6395b5f", size = 205360, upload-time = "2026-09-17T16:07:45.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/32/178457385925ce301c7b3232bd85665d4df57da39568b62dcfc74031873f/msal-1.39.0-py3-none-any.whl", hash = "sha256:2d2577886906cd7293850dffa2da29119966c213bfc6ec0cecf8bf7621e1ca77", size = 131629, upload-time = "2026-09-17T16:07:46.722Z" }, +] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, +] + [[package]] name = "msgspec" version = "0.21.1" @@ -1948,18 +1919,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, ] -[[package]] -name = "proto-plus" -version = "1.28.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/40/a6/4fbadcc2044034449b3f8f0ce82dcf3005d53f37c136642103fd4836a31c/proto_plus-1.28.4.tar.gz", hash = "sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63", size = 58679, upload-time = "2026-08-25T19:19:15.102Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/5d/0f04b85dafdc3250ced7f2592efc17dce7f40712e941e9632202481e600d/proto_plus-1.28.4-py3-none-any.whl", hash = "sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa", size = 50797, upload-time = "2026-08-25T19:18:12.338Z" }, -] - [[package]] name = "protobuf" version = "7.36.2" @@ -2151,6 +2110,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyproj" version = "3.7.2" @@ -2875,8 +2848,10 @@ dev = [ name = "tilebox-workflows" source = { editable = "tilebox-workflows" } dependencies = [ + { name = "azure-identity" }, { name = "boto3" }, - { name = "google-cloud-storage" }, + { name = "google-auth", extra = ["requests"] }, + { name = "grpcio" }, { name = "msgspec" }, { name = "obstore" }, { name = "opentelemetry-api" }, @@ -2916,8 +2891,10 @@ dev = [ [package.metadata] requires-dist = [ { name = "affine", marker = "extra == 'geospatial'", specifier = ">=2" }, + { name = "azure-identity", specifier = ">=1.23" }, { name = "boto3", specifier = ">=1.40.2" }, - { name = "google-cloud-storage", specifier = ">=2.10" }, + { name = "google-auth", extras = ["requests"], specifier = ">=2.29" }, + { name = "grpcio", specifier = ">=1.84.0" }, { name = "ipywidgets", marker = "extra == 'notebook'", specifier = ">=8.1.7" }, { name = "msgspec", specifier = ">=0.19" }, { name = "obstore", specifier = ">=0.8.2" },