Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion recipe/simple_use_case/relax_demo.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ def build_tq_config(config: DemoConfig):
base = OmegaConf.load(resources.files("transfer_queue") / "config.yaml")
override = OmegaConf.create(
{
"controller": {"sampler": RankAwareSampler, "polling_mode": True},
"controller": {"sampler": RankAwareSampler},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {"num_data_storage_units": config.num_data_storage_units},
Expand Down
2 changes: 0 additions & 2 deletions scripts/performance_test/perftest_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ metrics:
controller:
# User-defined sampler. User can pass sampler instance to overwrite this string config.
sampler: SequentialSampler
# Whether return an empty BatchMeta to prevent request blocking when no enough data is available
polling_mode: False
# ZMQ Server IP & Ports (automatically generated during init)
zmq_info: null

Expand Down
2 changes: 0 additions & 2 deletions scripts/performance_test_ssd_offload/perftest_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ metrics:
controller:
# User-defined sampler. User can pass sampler instance to overwrite this string config.
sampler: SequentialSampler
# Whether return an empty BatchMeta to prevent request blocking when no enough data is available
polling_mode: False
# ZMQ Server IP & Ports (automatically generated during init)
zmq_info: null

Expand Down
1 change: 0 additions & 1 deletion tests/e2e/test_checkpoint_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@

_TQ_CONFIG = OmegaConf.create(
{
"controller": {"polling_mode": True},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {
Expand Down
1 change: 0 additions & 1 deletion tests/e2e/test_data_dump_cross_topology_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
def _tq_config(num_storage_units: int) -> OmegaConf:
return OmegaConf.create(
{
"controller": {"polling_mode": True},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {
Expand Down
1 change: 0 additions & 1 deletion tests/e2e/test_data_dump_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
def _tq_config(num_storage_units: int) -> OmegaConf:
return OmegaConf.create(
{
"controller": {"polling_mode": True},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {
Expand Down
12 changes: 0 additions & 12 deletions tests/e2e/test_e2e_lifecycle_consistency.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,6 @@
# Backend configurations for E2E tests
BACKEND_CONFIGS = {
"SimpleStorage": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {
Expand All @@ -58,9 +55,6 @@
},
},
"MooncakeStore": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "MooncakeStore",
"MooncakeStore": {
Expand All @@ -74,9 +68,6 @@
},
},
"Yuanrong": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "Yuanrong",
"Yuanrong": {
Expand All @@ -90,9 +81,6 @@
# GPUDirect RDMA transfers directly from the registered CUDA staging buffer.
# auto_init=true: TQ starts mooncake_master automatically on transfer_queue.init().
"MooncakeStore_GDR": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "MooncakeStore",
"MooncakeStore": {
Expand Down
1 change: 0 additions & 1 deletion tests/e2e/test_kv_global_lock_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
# Bound every wait so that a lock regression fails the test instead of hanging it.
TIMEOUT_S = 60
CONF = {
"controller": {"polling_mode": True},
"backend": {"storage_backend": "SimpleStorage", "SimpleStorage": {"total_storage_size": 100}},
}

Expand Down
9 changes: 0 additions & 9 deletions tests/e2e/test_kv_interface_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ def tq_api(request):
# Adjust values for GitHub CI environment (smaller memory footprint)
BACKEND_CONFIGS = {
"SimpleStorage": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {
Expand All @@ -79,9 +76,6 @@ def tq_api(request):
},
},
"MooncakeStore": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "MooncakeStore",
"MooncakeStore": {
Expand All @@ -96,9 +90,6 @@ def tq_api(request):
},
},
"Yuanrong": {
"controller": {
"polling_mode": True,
},
"backend": {
"storage_backend": "Yuanrong",
"Yuanrong": {
Expand Down
1 change: 0 additions & 1 deletion tests/e2e/test_kv_link_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ def tq_system():
tq.init(
OmegaConf.create(
{
"controller": {"polling_mode": True},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {"total_storage_size": 200, "num_data_storage_units": 2},
Expand Down
1 change: 0 additions & 1 deletion tests/e2e/test_ssd_offload_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ def tq_system(ssd_root):

config = OmegaConf.create(
{
"controller": {"polling_mode": True},
"backend": {
"storage_backend": "SimpleStorage",
"SimpleStorage": {
Expand Down
5 changes: 0 additions & 5 deletions transfer_queue/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ metrics:
controller:
# User-defined sampler. User can pass sampler instance to overwrite this string config.
sampler: SequentialSampler
# Whether return an empty BatchMeta to prevent request blocking when no enough data is available
polling_mode: False
# ZMQ Server IP & Ports (automatically generated during init)
zmq_info: null

Expand Down Expand Up @@ -86,9 +84,6 @@ backend:
# from ANY node's memory to this offload client via TCP/RDMA.
# Ensure that `local_hostname` (above) is set to a cluster-reachable address so that
# all nodes can transfer data to the offload client.
#
# TODO: Support multi-node distributed offload (one client per node) for better
# scalability in large clusters.
offload:
# Master switch for SSD offload.
enabled: false
Expand Down
84 changes: 31 additions & 53 deletions transfer_queue/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import os
import pickle
import time
import warnings
from collections import defaultdict, deque
from collections.abc import Sequence
from dataclasses import dataclass, field
Expand Down Expand Up @@ -55,10 +56,6 @@

logger = get_logger(__name__)

TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1))
TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5))
# Above libzmq's default of 1023: the metrics exporter borrows this context to query
# storage units, so the budget covers a fleet rather than the two ROUTERs bound here.
TQ_CONTROLLER_ZMQ_MAX_SOCKETS = int(os.environ.get("TQ_CONTROLLER_ZMQ_MAX_SOCKETS", 4096))

# Holds payloads that keys of other partitions read through kv_link, e.g. a prompt video
Expand Down Expand Up @@ -1124,7 +1121,7 @@ class TransferQueueController:
def __init__(
self,
sampler: BaseSampler | type[BaseSampler] = SequentialSampler,
polling_mode: bool = False,
polling_mode: bool | None = None,
) -> None:
"""Initialize the TransferQueue Controller.

Expand All @@ -1135,11 +1132,17 @@ def __init__(
- Defaults to SequentialSampler for simple sequential sampling
- Example: sampler=GRPOGroupNSampler() (instance)
- Example: sampler=SequentialSampler (class)
polling_mode: Whether to use polling mode for TransferQueue controller.
- If False, the controller will raise an error when no enough data is available.
- If True, the controller will return an empty BatchMeta when no enough data is available.
The user side is responsible for handling this empty case (retrying later).
"""
polling_mode: Deprecated. A metadata fetch returns an empty batch when no batch is available.
"""
if polling_mode is not None:
warnings.warn(
"The `polling_mode` argument is deprecated and ignored. It will be removed in the next release. "
"Remove it when creating the controller. "
"A metadata fetch returns an empty BatchMeta when no batch is available.",
FutureWarning,
stacklevel=2,
)

if isinstance(sampler, BaseSampler):
self.sampler = sampler
elif isinstance(sampler, type) and issubclass(sampler, BaseSampler):
Expand All @@ -1150,7 +1153,6 @@ def __init__(
)

self.controller_id = f"TQ_CONTROLLER_{uuid4().hex[:8]}"
self.polling_mode = polling_mode
self.tq_config = None # global config for TransferQueue system

# Initialize ZMQ sockets for communication
Expand Down Expand Up @@ -1417,10 +1419,8 @@ def get_metadata(
**kwargs: Additional keyword arguments

Returns:
BatchMeta object containing the requested metadata

Raises:
TimeoutError: If waiting for sufficient data times out in fetch mode
BatchMeta containing the requested metadata. A fetch returns an empty batch
when no batch is available.
"""

if mode == "insert":
Expand Down Expand Up @@ -1458,40 +1458,19 @@ def get_metadata(
if batch_size is None:
raise ValueError("must provide batch_size in fetch mode")

start_time = time.time()
while True:
# ready_for_consume_indexes: samples where all required fields are produced
# (production status is ready) and not yet consumed
ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name)

if len(ready_for_consume_indexes) < batch_size:
if self.polling_mode:
# Return cached result if available
if self.sampler.has_cached_result(partition_id, task_name, sampling_config):
break
else:
logger.debug(
f"[{self.controller_id}]: Not enough data for task {task_name} in "
f"partition {partition_id}. Required: {batch_size}, "
f"Available: {len(ready_for_consume_indexes)}."
f" Returning None due to polling mode."
)
return BatchMeta.empty()
else:
logger.warning(
f"[{self.controller_id}]: Insufficient data for task {task_name}. Required: {batch_size} "
f"samples with fields {data_fields} in partition {partition_id}, but only have "
f"{len(ready_for_consume_indexes)} samples meeting the criteria. "
f"Retrying in {TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL}s..."
)
time.sleep(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL)
if time.time() - start_time > TQ_CONTROLLER_GET_METADATA_TIMEOUT:
raise TimeoutError(
f"Timeout while waiting for sufficient data for task {task_name}. "
f"Required: {batch_size}, Available: {len(ready_for_consume_indexes)}"
)
else:
break
# ready_for_consume_indexes: samples where all required fields are produced
# (production status is ready) and not yet consumed
ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name)

if len(ready_for_consume_indexes) < batch_size:
# Return cached result if available
if not self.sampler.has_cached_result(partition_id, task_name, sampling_config):
logger.debug(
f"[{self.controller_id}]: Insufficient data for task {task_name}. Required: {batch_size} "
f"samples with fields {data_fields} in partition {partition_id}, but only have "
f"{len(ready_for_consume_indexes)} samples meeting the criteria. "
)
return BatchMeta.empty()

batch_global_indexes, consumed_indexes = self.sampler(
ready_for_consume_indexes,
Expand All @@ -1505,22 +1484,21 @@ def get_metadata(
# Some samplers (e.g. SeqlenBalancedSampler) may return variable-size
# batches per DP rank, so we only check for empty results.
if len(batch_global_indexes) == 0:
if self.polling_mode:
return BatchMeta.empty()
raise RuntimeError(
logger.debug(
f"Sampler returned no samples. Please check the sampler logic. "
f"Expected: {batch_size}, before sampling: {len(ready_for_consume_indexes)}, "
f"after sampling: {len(batch_global_indexes)}"
)

return BatchMeta.empty()

# Mark samples as consumed if in fetch mode
if consumed_indexes:
partition = self.partitions[partition_id]
partition.mark_consumed(task_name, consumed_indexes)

elif mode == "force_fetch":
batch_global_indexes = self.index_manager.get_indexes_for_partition(partition_id)
consumed_indexes = []

# Package into metadata
metadata = self.generate_batch_meta(partition_id, batch_global_indexes, data_fields, mode)
Expand Down
19 changes: 17 additions & 2 deletions transfer_queue/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import shutil
import subprocess
import time
import warnings
from collections import Counter, defaultdict
from importlib import resources
from pathlib import Path
Expand All @@ -41,7 +42,7 @@

import ray
import torch
from omegaconf import DictConfig, OmegaConf
from omegaconf import DictConfig, OmegaConf, open_dict
from tensordict import TensorDict
from tensordict.tensorclass import NonTensorStack

Expand Down Expand Up @@ -177,6 +178,17 @@ def init(conf: DictConfig | None = None) -> DictConfig | None:
>>> metadata = tq.get_meta(...)
>>> data = tq.get_data(metadata)
"""
controller_conf = conf.get("controller") if conf is not None else None
has_legacy_polling_mode = controller_conf is not None and "polling_mode" in controller_conf.keys()
if has_legacy_polling_mode:
warnings.warn(
"`controller.polling_mode` is deprecated and ignored. It will be removed in the next release. "
"Remove this setting. "
"A metadata fetch returns an empty BatchMeta when no batch is available.",
FutureWarning,
stacklevel=2,
)

if _init_from_existing():
return conf

Expand All @@ -187,6 +199,9 @@ def init(conf: DictConfig | None = None) -> DictConfig | None:
final_conf = OmegaConf.merge(final_conf, default_conf)
if conf:
final_conf = OmegaConf.merge(final_conf, conf)
if has_legacy_polling_mode:
with open_dict(final_conf.controller):
del final_conf.controller["polling_mode"]

# TODO(hz): support load custom sampler class from external module.
try:
Expand All @@ -209,7 +224,7 @@ def init(conf: DictConfig | None = None) -> DictConfig | None:
global _TQ_CONTROLLER, _TQ_IS_OWNER
_TQ_CONTROLLER = TransferQueueController.options( # type: ignore[attr-defined]
name="TransferQueueController", namespace=ray_namespace()
).remote(sampler=sampler, polling_mode=final_conf.controller.polling_mode)
).remote(sampler=sampler)
_TQ_IS_OWNER = True
logger.info("TransferQueueController has been created.")
_start_lock_managers(lock_settings)
Expand Down
5 changes: 2 additions & 3 deletions transfer_queue/sampler/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,8 @@ def has_cached_result(
) -> bool:
"""Check whether the sampler has a cached sampling result for the given context.

This is used by the controller in polling mode to determine if a previously
computed sampling result is already available, so that it can skip waiting
for more data and directly proceed to return the cached result.
The controller checks this when there are too few unconsumed samples for a
new batch. A cache hit lets the sampler return its previous result.

The check is based on the ``_states`` cache structure:
``_states[partition_id][task_name][dp_rank][batch_index]``.
Expand Down
6 changes: 2 additions & 4 deletions transfer_queue/sampler/seqlen_balanced_sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,10 +169,8 @@ def sample(
# Cache the result
self._balanced_cache[cache_key] = partitions

# Populate the inherited _states cache for ALL dp_ranks so that
# the controller's polling check (which looks at self.sampler._states)
# works correctly even when ready_indexes < batch_size for later ranks
# (because earlier ranks already consumed their portion).
# Cache each DP rank's assignment so later fetches can reuse it after
# earlier fetches reduce the ready sample count.
if partition_id not in self._states:
self._states[partition_id] = {}
if task_name not in self._states[partition_id]:
Expand Down
Loading
Loading