From 7a2254bc8011b86b6bf4a8d85e36015333492c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Han=20Zhenyu=20=E9=9F=A9=E6=8C=AF=E5=AE=87?= Date: Fri, 13 Nov 2026 14:47:19 +0800 Subject: [PATCH 1/2] [fix] Stop controller metadata fetches from blocking Check ready samples once so the request loop can continue processing producer notifications. Keep sampler cache hits available and return an empty BatchMeta when no batch is ready. Warn on legacy polling_mode settings and remove them from examples and fixtures. Signed-off-by: 0oshowero0 --- recipe/simple_use_case/relax_demo.py | 2 +- scripts/performance_test/perftest_config.yaml | 2 - .../perftest_config.yaml | 2 - tests/e2e/test_checkpoint_e2e.py | 1 - .../e2e/test_data_dump_cross_topology_e2e.py | 1 - tests/e2e/test_data_dump_e2e.py | 1 - tests/e2e/test_e2e_lifecycle_consistency.py | 12 --- tests/e2e/test_kv_global_lock_e2e.py | 1 - tests/e2e/test_kv_interface_e2e.py | 9 -- tests/e2e/test_kv_link_e2e.py | 1 - tests/e2e/test_ssd_offload_e2e.py | 1 - tests/test_controller.py | 35 ++++++++ transfer_queue/config.yaml | 5 -- transfer_queue/controller.py | 84 +++++++------------ transfer_queue/interface.py | 19 ++++- transfer_queue/sampler/base.py | 5 +- .../sampler/seqlen_balanced_sampler.py | 6 +- tutorial/06_streaming_dataloader.py | 7 +- tutorial/basic.ipynb | 1 - 19 files changed, 89 insertions(+), 106 deletions(-) diff --git a/recipe/simple_use_case/relax_demo.py b/recipe/simple_use_case/relax_demo.py index 6cb1766e..b6e8ce4e 100644 --- a/recipe/simple_use_case/relax_demo.py +++ b/recipe/simple_use_case/relax_demo.py @@ -324,7 +324,7 @@ def build_tq_config(config: DemoConfig): base = OmegaConf.load(resources.files("transfer_queue") / "config.yaml") override = OmegaConf.create( { - "controller": {"sampler": RankAwareSampler, "polling_mode": True}, + "controller": {"sampler": RankAwareSampler}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": {"num_data_storage_units": config.num_data_storage_units}, diff --git a/scripts/performance_test/perftest_config.yaml b/scripts/performance_test/perftest_config.yaml index bf2692fa..ca6cb792 100644 --- a/scripts/performance_test/perftest_config.yaml +++ b/scripts/performance_test/perftest_config.yaml @@ -10,8 +10,6 @@ metrics: controller: # User-defined sampler. User can pass sampler instance to overwrite this string config. sampler: SequentialSampler - # Whether return an empty BatchMeta to prevent request blocking when no enough data is available - polling_mode: False # ZMQ Server IP & Ports (automatically generated during init) zmq_info: null diff --git a/scripts/performance_test_ssd_offload/perftest_config.yaml b/scripts/performance_test_ssd_offload/perftest_config.yaml index 381c6b0d..a7c6a6c4 100644 --- a/scripts/performance_test_ssd_offload/perftest_config.yaml +++ b/scripts/performance_test_ssd_offload/perftest_config.yaml @@ -10,8 +10,6 @@ metrics: controller: # User-defined sampler. User can pass sampler instance to overwrite this string config. sampler: SequentialSampler - # Whether return an empty BatchMeta to prevent request blocking when no enough data is available - polling_mode: False # ZMQ Server IP & Ports (automatically generated during init) zmq_info: null diff --git a/tests/e2e/test_checkpoint_e2e.py b/tests/e2e/test_checkpoint_e2e.py index f2a8da92..de0baa88 100644 --- a/tests/e2e/test_checkpoint_e2e.py +++ b/tests/e2e/test_checkpoint_e2e.py @@ -34,7 +34,6 @@ _TQ_CONFIG = OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/e2e/test_data_dump_cross_topology_e2e.py b/tests/e2e/test_data_dump_cross_topology_e2e.py index 445bdbd3..1bf4d9f9 100644 --- a/tests/e2e/test_data_dump_cross_topology_e2e.py +++ b/tests/e2e/test_data_dump_cross_topology_e2e.py @@ -43,7 +43,6 @@ def _tq_config(num_storage_units: int) -> OmegaConf: return OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/e2e/test_data_dump_e2e.py b/tests/e2e/test_data_dump_e2e.py index 95816533..b849ac52 100644 --- a/tests/e2e/test_data_dump_e2e.py +++ b/tests/e2e/test_data_dump_e2e.py @@ -49,7 +49,6 @@ def _tq_config(num_storage_units: int) -> OmegaConf: return OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/e2e/test_e2e_lifecycle_consistency.py b/tests/e2e/test_e2e_lifecycle_consistency.py index 97028878..a246311c 100644 --- a/tests/e2e/test_e2e_lifecycle_consistency.py +++ b/tests/e2e/test_e2e_lifecycle_consistency.py @@ -46,9 +46,6 @@ # Backend configurations for E2E tests BACKEND_CONFIGS = { "SimpleStorage": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { @@ -58,9 +55,6 @@ }, }, "MooncakeStore": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "MooncakeStore", "MooncakeStore": { @@ -74,9 +68,6 @@ }, }, "Yuanrong": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "Yuanrong", "Yuanrong": { @@ -90,9 +81,6 @@ # GPUDirect RDMA transfers directly from the registered CUDA staging buffer. # auto_init=true: TQ starts mooncake_master automatically on transfer_queue.init(). "MooncakeStore_GDR": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "MooncakeStore", "MooncakeStore": { diff --git a/tests/e2e/test_kv_global_lock_e2e.py b/tests/e2e/test_kv_global_lock_e2e.py index 661bf7fa..8623fb34 100644 --- a/tests/e2e/test_kv_global_lock_e2e.py +++ b/tests/e2e/test_kv_global_lock_e2e.py @@ -34,7 +34,6 @@ # Bound every wait so that a lock regression fails the test instead of hanging it. TIMEOUT_S = 60 CONF = { - "controller": {"polling_mode": True}, "backend": {"storage_backend": "SimpleStorage", "SimpleStorage": {"total_storage_size": 100}}, } diff --git a/tests/e2e/test_kv_interface_e2e.py b/tests/e2e/test_kv_interface_e2e.py index c3ec0c13..52b0d7dd 100644 --- a/tests/e2e/test_kv_interface_e2e.py +++ b/tests/e2e/test_kv_interface_e2e.py @@ -67,9 +67,6 @@ def tq_api(request): # Adjust values for GitHub CI environment (smaller memory footprint) BACKEND_CONFIGS = { "SimpleStorage": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { @@ -79,9 +76,6 @@ def tq_api(request): }, }, "MooncakeStore": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "MooncakeStore", "MooncakeStore": { @@ -96,9 +90,6 @@ def tq_api(request): }, }, "Yuanrong": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "Yuanrong", "Yuanrong": { diff --git a/tests/e2e/test_kv_link_e2e.py b/tests/e2e/test_kv_link_e2e.py index d5ecc61f..74ec0e23 100644 --- a/tests/e2e/test_kv_link_e2e.py +++ b/tests/e2e/test_kv_link_e2e.py @@ -39,7 +39,6 @@ def tq_system(): tq.init( OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": {"total_storage_size": 200, "num_data_storage_units": 2}, diff --git a/tests/e2e/test_ssd_offload_e2e.py b/tests/e2e/test_ssd_offload_e2e.py index eb9060b6..86737798 100644 --- a/tests/e2e/test_ssd_offload_e2e.py +++ b/tests/e2e/test_ssd_offload_e2e.py @@ -52,7 +52,6 @@ def tq_system(ssd_root): config = OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/test_controller.py b/tests/test_controller.py index edbfdff2..caeda11e 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -20,7 +20,9 @@ import ray import torch import zmq +from omegaconf import OmegaConf +import transfer_queue as tq from transfer_queue.controller import TQ_CONTROLLER_ZMQ_MAX_SOCKETS, TransferQueueController from transfer_queue.utils.zmq_utils import ZMQMessage, ZMQRequestType, create_zmq_socket @@ -45,6 +47,39 @@ def ray_setup(): class TestTransferQueueController: + @pytest.mark.parametrize("polling_mode", [False, True]) + def test_init_ignores_deprecated_polling_mode(self, ray_setup, monkeypatch, polling_mode): + monkeypatch.setenv("TQ_RAY_NAMESPACE", f"polling-mode-{uuid4().hex}") + conf = OmegaConf.create( + { + "controller": {"polling_mode": polling_mode}, + "backend": {"SimpleStorage": {"num_data_storage_units": 1}}, + } + ) + + try: + with pytest.warns(FutureWarning, match=r"controller\.polling_mode"): + final_conf = tq.init(conf) + assert "polling_mode" not in final_conf.controller + assert conf.controller.polling_mode is polling_mode + + with pytest.warns(FutureWarning, match=r"controller\.polling_mode"): + tq.init(conf) + finally: + tq.close() + + @pytest.mark.parametrize("polling_mode", [False, True]) + def test_controller_ignores_deprecated_polling_mode(self, monkeypatch, polling_mode): + controller_class = TransferQueueController.__ray_metadata__.modified_class + monkeypatch.setattr(controller_class, "_init_zmq_socket", lambda self: None) + monkeypatch.setattr(controller_class, "_start_process_handshake", lambda self: None) + monkeypatch.setattr(controller_class, "_start_process_request", lambda self: None) + + with pytest.warns(FutureWarning, match=r"`polling_mode` argument"): + controller = controller_class(polling_mode=polling_mode) + metadata = controller.get_metadata(["tokens"], "train", batch_size=1, task_name="trainer") + assert metadata.size == 0 + def test_controller_with_single_partition(self, ray_setup): gbs = 8 num_n_samples = 4 diff --git a/transfer_queue/config.yaml b/transfer_queue/config.yaml index e0ac6599..af35f2fd 100644 --- a/transfer_queue/config.yaml +++ b/transfer_queue/config.yaml @@ -10,8 +10,6 @@ metrics: controller: # User-defined sampler. User can pass sampler instance to overwrite this string config. sampler: SequentialSampler - # Whether return an empty BatchMeta to prevent request blocking when no enough data is available - polling_mode: False # ZMQ Server IP & Ports (automatically generated during init) zmq_info: null @@ -86,9 +84,6 @@ backend: # from ANY node's memory to this offload client via TCP/RDMA. # Ensure that `local_hostname` (above) is set to a cluster-reachable address so that # all nodes can transfer data to the offload client. - # - # TODO: Support multi-node distributed offload (one client per node) for better - # scalability in large clusters. offload: # Master switch for SSD offload. enabled: false diff --git a/transfer_queue/controller.py b/transfer_queue/controller.py index d500c027..db01b24e 100644 --- a/transfer_queue/controller.py +++ b/transfer_queue/controller.py @@ -17,6 +17,7 @@ import os import pickle import time +import warnings from collections import defaultdict, deque from collections.abc import Sequence from dataclasses import dataclass, field @@ -55,10 +56,6 @@ logger = get_logger(__name__) -TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1)) -TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5)) -# Above libzmq's default of 1023: the metrics exporter borrows this context to query -# storage units, so the budget covers a fleet rather than the two ROUTERs bound here. TQ_CONTROLLER_ZMQ_MAX_SOCKETS = int(os.environ.get("TQ_CONTROLLER_ZMQ_MAX_SOCKETS", 4096)) # Holds payloads that keys of other partitions read through kv_link, e.g. a prompt video @@ -1124,7 +1121,7 @@ class TransferQueueController: def __init__( self, sampler: BaseSampler | type[BaseSampler] = SequentialSampler, - polling_mode: bool = False, + polling_mode: bool | None = None, ) -> None: """Initialize the TransferQueue Controller. @@ -1135,11 +1132,17 @@ def __init__( - Defaults to SequentialSampler for simple sequential sampling - Example: sampler=GRPOGroupNSampler() (instance) - Example: sampler=SequentialSampler (class) - polling_mode: Whether to use polling mode for TransferQueue controller. - - If False, the controller will raise an error when no enough data is available. - - If True, the controller will return an empty BatchMeta when no enough data is available. - The user side is responsible for handling this empty case (retrying later). - """ + polling_mode: Deprecated. A metadata fetch returns an empty batch when no batch is available. + """ + if polling_mode is not None: + warnings.warn( + "The `polling_mode` argument is deprecated and ignored. It will be removed in the next release. " + "Remove it when creating the controller. " + "A metadata fetch returns an empty BatchMeta when no batch is available.", + FutureWarning, + stacklevel=2, + ) + if isinstance(sampler, BaseSampler): self.sampler = sampler elif isinstance(sampler, type) and issubclass(sampler, BaseSampler): @@ -1150,7 +1153,6 @@ def __init__( ) self.controller_id = f"TQ_CONTROLLER_{uuid4().hex[:8]}" - self.polling_mode = polling_mode self.tq_config = None # global config for TransferQueue system # Initialize ZMQ sockets for communication @@ -1417,10 +1419,8 @@ def get_metadata( **kwargs: Additional keyword arguments Returns: - BatchMeta object containing the requested metadata - - Raises: - TimeoutError: If waiting for sufficient data times out in fetch mode + BatchMeta containing the requested metadata. A fetch returns an empty batch + when no batch is available. """ if mode == "insert": @@ -1458,40 +1458,19 @@ def get_metadata( if batch_size is None: raise ValueError("must provide batch_size in fetch mode") - start_time = time.time() - while True: - # ready_for_consume_indexes: samples where all required fields are produced - # (production status is ready) and not yet consumed - ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name) - - if len(ready_for_consume_indexes) < batch_size: - if self.polling_mode: - # Return cached result if available - if self.sampler.has_cached_result(partition_id, task_name, sampling_config): - break - else: - logger.debug( - f"[{self.controller_id}]: Not enough data for task {task_name} in " - f"partition {partition_id}. Required: {batch_size}, " - f"Available: {len(ready_for_consume_indexes)}." - f" Returning None due to polling mode." - ) - return BatchMeta.empty() - else: - logger.warning( - f"[{self.controller_id}]: Insufficient data for task {task_name}. Required: {batch_size} " - f"samples with fields {data_fields} in partition {partition_id}, but only have " - f"{len(ready_for_consume_indexes)} samples meeting the criteria. " - f"Retrying in {TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL}s..." - ) - time.sleep(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL) - if time.time() - start_time > TQ_CONTROLLER_GET_METADATA_TIMEOUT: - raise TimeoutError( - f"Timeout while waiting for sufficient data for task {task_name}. " - f"Required: {batch_size}, Available: {len(ready_for_consume_indexes)}" - ) - else: - break + # ready_for_consume_indexes: samples where all required fields are produced + # (production status is ready) and not yet consumed + ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name) + + if len(ready_for_consume_indexes) < batch_size: + # Return cached result if available + if not self.sampler.has_cached_result(partition_id, task_name, sampling_config): + logger.debug( + f"[{self.controller_id}]: Insufficient data for task {task_name}. Required: {batch_size} " + f"samples with fields {data_fields} in partition {partition_id}, but only have " + f"{len(ready_for_consume_indexes)} samples meeting the criteria. " + ) + return BatchMeta.empty() batch_global_indexes, consumed_indexes = self.sampler( ready_for_consume_indexes, @@ -1505,14 +1484,14 @@ def get_metadata( # Some samplers (e.g. SeqlenBalancedSampler) may return variable-size # batches per DP rank, so we only check for empty results. if len(batch_global_indexes) == 0: - if self.polling_mode: - return BatchMeta.empty() - raise RuntimeError( + logger.debug( f"Sampler returned no samples. Please check the sampler logic. " f"Expected: {batch_size}, before sampling: {len(ready_for_consume_indexes)}, " f"after sampling: {len(batch_global_indexes)}" ) + return BatchMeta.empty() + # Mark samples as consumed if in fetch mode if consumed_indexes: partition = self.partitions[partition_id] @@ -1520,7 +1499,6 @@ def get_metadata( elif mode == "force_fetch": batch_global_indexes = self.index_manager.get_indexes_for_partition(partition_id) - consumed_indexes = [] # Package into metadata metadata = self.generate_batch_meta(partition_id, batch_global_indexes, data_fields, mode) diff --git a/transfer_queue/interface.py b/transfer_queue/interface.py index a9615c9b..7e05949d 100644 --- a/transfer_queue/interface.py +++ b/transfer_queue/interface.py @@ -33,6 +33,7 @@ import shutil import subprocess import time +import warnings from collections import Counter, defaultdict from importlib import resources from pathlib import Path @@ -41,7 +42,7 @@ import ray import torch -from omegaconf import DictConfig, OmegaConf +from omegaconf import DictConfig, OmegaConf, open_dict from tensordict import TensorDict from tensordict.tensorclass import NonTensorStack @@ -177,6 +178,17 @@ def init(conf: DictConfig | None = None) -> DictConfig | None: >>> metadata = tq.get_meta(...) >>> data = tq.get_data(metadata) """ + controller_conf = conf.get("controller") if conf is not None else None + has_legacy_polling_mode = controller_conf is not None and "polling_mode" in controller_conf.keys() + if has_legacy_polling_mode: + warnings.warn( + "`controller.polling_mode` is deprecated and ignored. It will be removed in the next release. " + "Remove this setting. " + "A metadata fetch returns an empty BatchMeta when no batch is available.", + FutureWarning, + stacklevel=2, + ) + if _init_from_existing(): return conf @@ -187,6 +199,9 @@ def init(conf: DictConfig | None = None) -> DictConfig | None: final_conf = OmegaConf.merge(final_conf, default_conf) if conf: final_conf = OmegaConf.merge(final_conf, conf) + if has_legacy_polling_mode: + with open_dict(final_conf.controller): + del final_conf.controller["polling_mode"] # TODO(hz): support load custom sampler class from external module. try: @@ -209,7 +224,7 @@ def init(conf: DictConfig | None = None) -> DictConfig | None: global _TQ_CONTROLLER, _TQ_IS_OWNER _TQ_CONTROLLER = TransferQueueController.options( # type: ignore[attr-defined] name="TransferQueueController", namespace=ray_namespace() - ).remote(sampler=sampler, polling_mode=final_conf.controller.polling_mode) + ).remote(sampler=sampler) _TQ_IS_OWNER = True logger.info("TransferQueueController has been created.") _start_lock_managers(lock_settings) diff --git a/transfer_queue/sampler/base.py b/transfer_queue/sampler/base.py index 173b19da..da19559b 100644 --- a/transfer_queue/sampler/base.py +++ b/transfer_queue/sampler/base.py @@ -83,9 +83,8 @@ def has_cached_result( ) -> bool: """Check whether the sampler has a cached sampling result for the given context. - This is used by the controller in polling mode to determine if a previously - computed sampling result is already available, so that it can skip waiting - for more data and directly proceed to return the cached result. + The controller checks this when there are too few unconsumed samples for a + new batch. A cache hit lets the sampler return its previous result. The check is based on the ``_states`` cache structure: ``_states[partition_id][task_name][dp_rank][batch_index]``. diff --git a/transfer_queue/sampler/seqlen_balanced_sampler.py b/transfer_queue/sampler/seqlen_balanced_sampler.py index 20c904d7..29d47fdd 100644 --- a/transfer_queue/sampler/seqlen_balanced_sampler.py +++ b/transfer_queue/sampler/seqlen_balanced_sampler.py @@ -169,10 +169,8 @@ def sample( # Cache the result self._balanced_cache[cache_key] = partitions - # Populate the inherited _states cache for ALL dp_ranks so that - # the controller's polling check (which looks at self.sampler._states) - # works correctly even when ready_indexes < batch_size for later ranks - # (because earlier ranks already consumed their portion). + # Cache each DP rank's assignment so later fetches can reuse it after + # earlier fetches reduce the ready sample count. if partition_id not in self._states: self._states[partition_id] = {} if task_name not in self._states[partition_id]: diff --git a/tutorial/06_streaming_dataloader.py b/tutorial/06_streaming_dataloader.py index c12186aa..b8906c2b 100644 --- a/tutorial/06_streaming_dataloader.py +++ b/tutorial/06_streaming_dataloader.py @@ -83,17 +83,12 @@ def setup_transfer_queue(): "Note: Using RankAwareSampler when each rank retrieves data independently. It guarantees that " "The same DP rank receives the same sample indices." ) - print( - "Note: When using streaming data retrieval, please set polling_mode=True when initializing " - "TransferQueueController. In polling_mode, the controller will return empty BatchMeta when " - "available data cannot meet the consumption requirements. User side need to retry later." - ) + print("StreamingDataset retries when a metadata fetch returns an empty batch.") config = OmegaConf.create( { "controller": { "sampler": RankAwareSampler, # RankAwareSampler enables consistent sampling for each DP rank - "polling_mode": True, # Enable polling mode for streaming data retrieval }, "backend": {"SimpleStorage": {"num_data_storage_units": 2}}, }, diff --git a/tutorial/basic.ipynb b/tutorial/basic.ipynb index 09f946d8..94c38ae2 100644 --- a/tutorial/basic.ipynb +++ b/tutorial/basic.ipynb @@ -77,7 +77,6 @@ "\n", "config = OmegaConf.create(\n", " {\n", - " \"controller\": {\"polling_mode\": True},\n", " \"backend\": {\n", " \"storage_backend\": \"SimpleStorage\",\n", " \"SimpleStorage\": {\n", From d40954493318b22175f3917484acae4d0816863a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Han=20Zhenyu=20=E9=9F=A9=E6=8C=AF=E5=AE=87?= Date: Fri, 13 Nov 2026 16:30:04 +0800 Subject: [PATCH 2/2] [test] Drop polling mode deprecation tests Signed-off-by: 0oshowero0 --- tests/test_controller.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/tests/test_controller.py b/tests/test_controller.py index caeda11e..edbfdff2 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -20,9 +20,7 @@ import ray import torch import zmq -from omegaconf import OmegaConf -import transfer_queue as tq from transfer_queue.controller import TQ_CONTROLLER_ZMQ_MAX_SOCKETS, TransferQueueController from transfer_queue.utils.zmq_utils import ZMQMessage, ZMQRequestType, create_zmq_socket @@ -47,39 +45,6 @@ def ray_setup(): class TestTransferQueueController: - @pytest.mark.parametrize("polling_mode", [False, True]) - def test_init_ignores_deprecated_polling_mode(self, ray_setup, monkeypatch, polling_mode): - monkeypatch.setenv("TQ_RAY_NAMESPACE", f"polling-mode-{uuid4().hex}") - conf = OmegaConf.create( - { - "controller": {"polling_mode": polling_mode}, - "backend": {"SimpleStorage": {"num_data_storage_units": 1}}, - } - ) - - try: - with pytest.warns(FutureWarning, match=r"controller\.polling_mode"): - final_conf = tq.init(conf) - assert "polling_mode" not in final_conf.controller - assert conf.controller.polling_mode is polling_mode - - with pytest.warns(FutureWarning, match=r"controller\.polling_mode"): - tq.init(conf) - finally: - tq.close() - - @pytest.mark.parametrize("polling_mode", [False, True]) - def test_controller_ignores_deprecated_polling_mode(self, monkeypatch, polling_mode): - controller_class = TransferQueueController.__ray_metadata__.modified_class - monkeypatch.setattr(controller_class, "_init_zmq_socket", lambda self: None) - monkeypatch.setattr(controller_class, "_start_process_handshake", lambda self: None) - monkeypatch.setattr(controller_class, "_start_process_request", lambda self: None) - - with pytest.warns(FutureWarning, match=r"`polling_mode` argument"): - controller = controller_class(polling_mode=polling_mode) - metadata = controller.get_metadata(["tokens"], "train", batch_size=1, task_name="trainer") - assert metadata.size == 0 - def test_controller_with_single_partition(self, ray_setup): gbs = 8 num_n_samples = 4