From 48b2e89202d72b40cb1de8459b1eca3d1a0d0f24 Mon Sep 17 00:00:00 2001 From: Keigh Rim Date: Sun, 8 Nov 2026 18:31:21 -0500 Subject: [PATCH] more clarification and fixes in PromptableApp documentations --- clams/app/__init__.py | 84 ++++++++++++++++++--------- clams/backends/llamaserver.py | 20 +++---- documentation/app-baseclasses.rst | 36 +++++++----- documentation/promptable-backends.rst | 16 ++--- tests/test_llamaserver.py | 47 ++++++++++++--- 5 files changed, 134 insertions(+), 69 deletions(-) diff --git a/clams/app/__init__.py b/clams/app/__init__.py index b79ab46..ad2711a 100644 --- a/clams/app/__init__.py +++ b/clams/app/__init__.py @@ -911,8 +911,8 @@ class ClamsPromptableApp(ClamsApp): 'GPU memory for long dynamic scenes. The value is ' 'operator-set, so the number of output TextDocuments is ' 'deterministic and does not vary by GPU. Composes with ' - '``parallelPrompts``, which batches independent prompts ' - 'into one forward pass.', + 'the number of prompts the backend runs together ' + '(``parallelPrompts`` in apps that offer it).', }, { 'name': 'imagesPerPromptMode', 'type': 'string', @@ -1989,17 +1989,18 @@ def _annotate(self, mmif, **parameters): #: the repository file listing. MODEL_FILES: dict[str, dict[str, Any]] | None = None #: Default values of llama.cpp's ``LLAMA_ARG_*`` environment - #: variables for what the app's models need, e.g. - #: ``{'LLAMA_ARG_UBATCH': '2048'}`` for the token budget per image - #: of a vision model. A variable set in the environment of the app - #: process takes precedence, and settings not given here or there - #: (GPU offload, context size, threads) are left to llama-server's + #: variables, for the server settings the app's models need (see + #: the llama.cpp documentation for the variables and their + #: defaults, which change between llama.cpp releases). A variable + #: set in the environment of the app process takes precedence, and + #: settings not given here or there are left to llama-server's #: defaults; see :mod:`clams.backends.llamaserver`. The settings in #: effect are logged when the server starts, and the context size #: and slot count are kept in ``self.server.slot_context`` and #: ``self.server.slots``. A prompt (system and user text, images, #: and the generated tokens) that does not fit in a slot fails the - #: request with an error naming the size. + #: request with an error naming the size. For the recommended slot + #: count, see :ref:`gguf-promptable`. LLAMA_ARGS: dict[str, str] = {} # ``RESTIFIER_*``: the HTTP wrapper constraints any app may declare @@ -2169,26 +2170,25 @@ def generate_batched( **generation_params, ) -> List[str]: """ - Run N prompts through :py:meth:`generate` in batches of the - server's slot count (``LLAMA_ARG_N_PARALLEL``, 1 by default), so - every batch fills the slots, and return N outputs in input - order. The other arguments are as in - :py:meth:`ClamsPromptableApp.generate_batched`. + Run N prompts as :py:meth:`generate` does, keeping every server + slot busy (a prompt starts as soon as a slot is free), and + report progress through :py:meth:`report_progress` as each + prompt finishes. Returns N outputs in input order. The other + arguments are as in :py:meth:`ClamsPromptableApp.generate_batched`. :param parallel_prompts: not accepted; the server's slot count - sets the batch size. - :raises ValueError: if ``parallel_prompts`` is given. + sets how many prompts run together. + :raises ValueError: if ``parallel_prompts`` is given, or as in + :py:meth:`generate`. """ if parallel_prompts is not None: raise ValueError( f"{type(self).__name__} takes no parallel_prompts: " f"llama-server decodes as many prompts together as it has " - f"slots; set LLAMA_ARG_N_PARALLEL to change that.") - return super().generate_batched( - prompt, system_prompt=system_prompt, images=images, - audios=audios, prompt_mode=prompt_mode, - parallel_prompts=getattr(self.server, 'slots', 1), - **generation_params) + f"slots, a server setting.") + return self._run_prompts( + prompt, system_prompt, images, audios, prompt_mode, + generation_params, report=True) def generate( self, @@ -2217,8 +2217,25 @@ def generate( :raises RuntimeError: if no model is loaded. :raises ValueError: for audio input, which this backend does not support yet, or a prompt that does not fit in the - context of a server slot. + context of a server slot; the prompts not yet sent are + then dropped. """ + return self._run_prompts( + prompt, system_prompt, images, audios, prompt_mode, + generation_params, report=False) + + def _run_prompts(self, prompt: List[str], system_prompt: str, + images: Optional[List[List[Any]]], + audios: Optional[List[List[Any]]], prompt_mode: str, + generation_params: Dict[str, Any], + report: bool) -> List[str]: + """ + Send the prompts of :py:meth:`generate` to the server, as many + at a time as it has slots, and return the outputs in input + order; with ``report``, call :py:meth:`report_progress` before + the first and after each finished prompt. + """ + from concurrent.futures import ThreadPoolExecutor, as_completed from clams.backends.llamaserver import ContextSizeError if audios is not None: raise ValueError( @@ -2248,20 +2265,31 @@ def run_one(i: int) -> str: f"A prompt with {n_images} images does not fit in " f"the context of a server slot " f"({slot or 'unknown'} tokens). Lower " - f"maxImagesPerPrompt, free GPU memory, or set the " - f"context size with LLAMA_ARG_CTX_SIZE. Server " + f"maxImagesPerPrompt, or give the server a larger " + f"context (see the llama.cpp documentation). Server " f"message: {e}") from e except Exception as e: self.logger.error( f"Error processing prompt {i}: {e}", exc_info=True) return '' - if n == 1: - return [run_one(0)] - from concurrent.futures import ThreadPoolExecutor + if report: + self.report_progress(0, n) + outputs = [''] * n slots = getattr(self.server, 'slots', 1) with ThreadPoolExecutor(max_workers=max(1, min(n, slots))) as pool: - return list(pool.map(run_one, range(n))) + futures = {pool.submit(run_one, i): i for i in range(n)} + try: + for done, future in enumerate(as_completed(futures), 1): + outputs[futures[future]] = future.result() + if report: + self.report_progress(done, n) + except BaseException: + # leave the prompts not yet sent unsent + for future in futures: + future.cancel() + raise + return outputs def _chat(self, conversation: Union[List[dict], List[List[dict]]], params: dict) -> str: diff --git a/clams/backends/llamaserver.py b/clams/backends/llamaserver.py index 1569580..1fa2d79 100644 --- a/clams/backends/llamaserver.py +++ b/clams/backends/llamaserver.py @@ -32,14 +32,14 @@ ``llama-server`` reads for every argument not given on its command line. Their values come from, in increasing precedence: -1. ``llama-server``'s defaults, which fit the run to the hardware: - they place what fits on the GPUs, keep the rest in system memory, - and size the context to the free memory; +1. ``llama-server``'s defaults; 1. the ``llama_args`` of :class:`LlamaServer`, for what a model needs (e.g. ``{'LLAMA_ARG_UBATCH': '2048'}``); -1. the environment of the process that starts the server (e.g. - ``LLAMA_ARG_CTX_SIZE``, ``LLAMA_ARG_THREADS``, - ``LLAMA_ARG_N_GPU_LAYERS`` set by whoever runs the app). +1. the environment of the process that starts the server, set by + whoever runs the app. + +The variables and their defaults belong to llama.cpp and change +between its releases; see the llama.cpp documentation. """ import atexit @@ -284,9 +284,9 @@ def _log_level(line: str) -> int: if re.search(r'\b(error|failed|warning|warn)\b', lowered) \ or _LEVEL_LINE.match(lowered): return logging.WARNING - # device placement, load, readiness and timing lines + # load, readiness and timing lines if 'print_timing' in lowered or 'model loaded' in lowered \ - or 'server is listening' in lowered or 'offloaded' in lowered: + or 'server is listening' in lowered: return logging.INFO return logging.DEBUG @@ -341,8 +341,8 @@ class LlamaServer: server reads the whole file at startup). :param request_timeout: seconds to wait for one chat completion. :param logger: logger receiving the server's output: errors and - warnings at WARNING, placement, load and timing lines at INFO, - the rest at DEBUG. + warnings at WARNING, load and timing lines at INFO, the rest at + DEBUG. :raises ValueError: if a ``llama_args`` key is not an ``LLAMA_ARG_*`` variable, or names the server's address. """ diff --git a/documentation/app-baseclasses.rst b/documentation/app-baseclasses.rst index d08d2d3..bd16a69 100644 --- a/documentation/app-baseclasses.rst +++ b/documentation/app-baseclasses.rst @@ -703,11 +703,17 @@ Audio input is not supported yet. GGUF apps have no ``parallelPrompts`` parameter. ``llama-server`` decodes at most as many requests together as it has slots, a number -fixed when the server starts, so -:meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched` always -sends that many prompts at a time. The slot count is a server setting -(``LLAMA_ARG_N_PARALLEL``, see below); more slots split the same -context into smaller parts. +fixed when the server starts, and +:meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched` keeps every +slot busy. The slot count is a server setting (see below), and its +default, like the way slots share the context, belongs to llama.cpp +and changes between its releases; see the llama.cpp documentation. The +startup log reports the slot count and the context of a slot. + +We recommend one slot, set in the app's ``LLAMA_ARGS``. The server +decodes the prompts in its slots together, so with more slots an +output can change with the other prompts in flight. In our tests on +Maxwell and Pascal GPUs, more slots were also no faster. Class-attribute hooks ^^^^^^^^^^^^^^^^^^^^^ @@ -726,10 +732,10 @@ Class-attribute hooks - no * - ``LLAMA_ARGS`` - Default values of llama.cpp's ``LLAMA_ARG_*`` environment - variables for what the app's models need, e.g. - ``{'LLAMA_ARG_UBATCH': '2048'}`` to raise the token budget per - image of a vision model. The environment of the app process - takes precedence (see below). + variables for what the app's models need, e.g. a larger batch + size for the image tokens of a vision model, or the slot count. + The environment of the app process takes precedence (see + below). - no * - ``RESTIFIER_PRODUCTION_WORKERS``, ``RESTIFIER_PRODUCTION_MAX_REQUESTS`` @@ -755,17 +761,17 @@ comes from, in increasing precedence: several GPUs. The server places what fits on the GPUs, keeps the rest in system memory, and sizes the context to the free memory. 1. The app's ``LLAMA_ARGS``, for what its models need. -1. The environment of the app process, set by whoever runs the app - (e.g. ``LLAMA_ARG_CTX_SIZE``, ``LLAMA_ARG_THREADS``, - ``LLAMA_ARG_N_GPU_LAYERS``). +1. The environment of the app process, set by whoever runs the app. -The settings in effect are logged with their source when the server -starts, and the context size and slot count when it is ready. +The variables and their defaults belong to llama.cpp and change +between its releases; see the llama.cpp documentation. The settings in +effect are logged with their source when the server starts, and the +context size and slot count when it is ready. One prompt (system and user text, all its images, and the generated tokens) must fit in the context of a server slot. A prompt that does not fails the request with an error naming the size, so a caller can -lower ``maxImagesPerPrompt`` or raise ``LLAMA_ARG_CTX_SIZE``. +lower ``maxImagesPerPrompt`` or give the server a larger context. Installation and containers ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/documentation/promptable-backends.rst b/documentation/promptable-backends.rst index dcb2366..41d117f 100644 --- a/documentation/promptable-backends.rst +++ b/documentation/promptable-backends.rst @@ -62,13 +62,13 @@ the latest release of each backend, built with CUDA support. * - Prompts run together - ``parallelPrompts`` runtime parameter, per request; the prompts are padded into one batch - - the server's slot count (``LLAMA_ARG_N_PARALLEL``), fixed when - the server starts; no ``parallelPrompts`` parameter + - the server's slot count, fixed when the server starts (one + recommended); no ``parallelPrompts`` parameter * - Memory for one request - grows with ``parallelPrompts`` and the prompt size; too much runs the GPU out of memory - - fixed when the server starts; the context is split over the - slots + - fixed when the server starts; how the slots share the context + depends on the llama.cpp release * - Prompt too large - out-of-memory error - context-size error (an error view naming the context size) @@ -97,10 +97,10 @@ Who sets the batch size An HF app takes ``parallelPrompts`` with each request and stacks that many prompts into one forward pass. A GGUF app sends as many prompts at a time as ``llama-server`` has slots, and the server decodes them -together. The slot count is set once, by whoever runs the app, and -more slots split the same context into smaller parts. An app moved to -the GGUF backend must stop reading ``parameters['parallelPrompts']`` -and stop passing ``parallel_prompts`` to +together. The slot count is a server setting, fixed when the server +starts; see :ref:`gguf-promptable` for the recommended value. An app +moved to the GGUF backend must stop reading +``parameters['parallelPrompts']`` and stop passing ``parallel_prompts`` to :meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched`. Outputs differ between backends diff --git a/tests/test_llamaserver.py b/tests/test_llamaserver.py index f03611e..c1359f6 100644 --- a/tests/test_llamaserver.py +++ b/tests/test_llamaserver.py @@ -14,6 +14,7 @@ import textwrap import types import unittest +import urllib.request from unittest import mock import numpy as np @@ -31,8 +32,14 @@ import json import os import sys + import threading + import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + # chat requests received, for /requests + COUNTS = {{'requests': 0}} + LOCK = threading.Lock() + if os.environ.get('FAKE_LLAMA_EXIT'): print('error: failed to load model', flush=True) sys.exit(int(os.environ['FAKE_LLAMA_EXIT'])) @@ -69,10 +76,15 @@ def setting(flag, env, default): slots = setting('--parallel', 'LLAMA_ARG_N_PARALLEL', 1) self._send({{'default_generation_settings': {{ 'n_ctx': total // slots}}, 'total_slots': slots}}) + elif self.path == '/requests': + self._send(COUNTS) else: self.send_error(404) def do_POST(self): + started = time.time() + with LOCK: + COUNTS['requests'] += 1 length = int(self.headers['Content-Length']) request = json.loads(self.rfile.read(length)) messages = request.pop('messages') @@ -90,7 +102,11 @@ def do_POST(self): 'message': 'request exceeds the available context'}}}}, status=400) return + # optional decode time, longer for prompts with more images + time.sleep(images * float(os.environ.get( + 'FAKE_LLAMA_SECONDS_PER_IMAGE', 0))) echo ={{'images': images, 'last_user': content, + 'started': started, 'ended': time.time(), 'roles': [m['role'] for m in messages], 'assistant': [m['content'] for m in messages if m['role'] == 'assistant'], @@ -360,8 +376,6 @@ def test_log_levels_of_server_output(self): logging.WARNING) self.assertEqual(level('slot print_timing: id 0 | prompt eval'), logging.INFO) - self.assertEqual(level('load_tensors: offloaded 49/49 layers to GPU'), - logging.INFO) self.assertEqual(level('load_tensors: CPU_Mapped model buffer size'), logging.DEBUG) @@ -536,19 +550,36 @@ def test_no_parallel_prompts_parameter(self): names = {p.name for p in app.metadata.parameters} self.assertNotIn('parallelPrompts', names) self.assertIn('maxImagesPerPrompt', names) - with self.assertRaisesRegex(ValueError, 'LLAMA_ARG_N_PARALLEL'): + with self.assertRaisesRegex(ValueError, 'parallel_prompts'): app.generate_batched(['hi'], images=[[]], parallel_prompts=2) - def test_batches_follow_the_server_slots(self): - app = self.make_app(LLAMA_ARGS={'LLAMA_ARG_N_PARALLEL': '2'}) + def test_free_slots_take_the_next_prompt(self): + # 2 slots; the first prompt takes three times as long as the + # others, so the third must start before the first ends + with _environ({'FAKE_LLAMA_SECONDS_PER_IMAGE': '0.3'}): + app = self.make_app(LLAMA_ARGS={'LLAMA_ARG_N_PARALLEL': '2'}) self.assertEqual(app.server.slots, 2) progress = [] app.report_progress = lambda done, total: progress.append(done) - images = [[Image.new('RGB', (4, 4))] * k for k in (1, 2, 3)] + images = [[Image.new('RGB', (4, 4))] * k for k in (3, 1, 1)] outputs = [json.loads(o) for o in app.generate_batched( ['describe'], images=images)] - self.assertEqual([o['images'] for o in outputs], [1, 2, 3]) - self.assertEqual(progress, [0, 2, 3]) + self.assertEqual([o['images'] for o in outputs], [3, 1, 1]) + self.assertLess(outputs[2]['started'], outputs[0]['ended']) + # progress before the first and after each finished prompt + self.assertEqual(progress, [0, 1, 2, 3]) + + def test_context_error_leaves_queued_prompts_unsent(self): + app = self.make_app() # one slot + images = [[Image.new('RGB', (4, 4))] for _ in range(5)] + with self.assertRaisesRegex(ValueError, 'does not fit'): + app.generate_batched(['TOO_LONG'], images=images) + with urllib.request.urlopen( + f"{app.server.base_url}/requests") as response: + sent = json.load(response)['requests'] + # the failing prompt, and at most the one the free worker had + # already taken before the others were cancelled + self.assertLessEqual(sent, 2) def test_text_only_prompt_runs_once(self): app = self.make_app()