more clarification and fixes in PromptableApp documentations

clams/app/__init__.py

@@ -911,8 +911,8 @@ class ClamsPromptableApp(ClamsApp):
                 'GPU memory for long dynamic scenes. The value is '
                 'operator-set, so the number of output TextDocuments is '
                 'deterministic and does not vary by GPU. Composes with '
-                '``parallelPrompts``, which batches independent prompts '
-                'into one forward pass.',
+                'the number of prompts the backend runs together '
+                '(``parallelPrompts`` in apps that offer it).',
         },
         {
             'name': 'imagesPerPromptMode', 'type': 'string',
@@ -1989,17 +1989,18 @@ def _annotate(self, mmif, **parameters):
     #: the repository file listing.
     MODEL_FILES: dict[str, dict[str, Any]] | None = None
     #: Default values of llama.cpp's ``LLAMA_ARG_*`` environment
-    #: variables for what the app's models need, e.g.
-    #: ``{'LLAMA_ARG_UBATCH': '2048'}`` for the token budget per image
-    #: of a vision model. A variable set in the environment of the app
-    #: process takes precedence, and settings not given here or there
-    #: (GPU offload, context size, threads) are left to llama-server's
+    #: variables, for the server settings the app's models need (see
+    #: the llama.cpp documentation for the variables and their
+    #: defaults, which change between llama.cpp releases). A variable
+    #: set in the environment of the app process takes precedence, and
+    #: settings not given here or there are left to llama-server's
     #: defaults; see :mod:`clams.backends.llamaserver`. The settings in
     #: effect are logged when the server starts, and the context size
     #: and slot count are kept in ``self.server.slot_context`` and
     #: ``self.server.slots``. A prompt (system and user text, images,
     #: and the generated tokens) that does not fit in a slot fails the
-    #: request with an error naming the size.
+    #: request with an error naming the size. For the recommended slot
+    #: count, see :ref:`gguf-promptable`.
     LLAMA_ARGS: dict[str, str] = {}
 
     # ``RESTIFIER_*``: the HTTP wrapper constraints any app may declare
@@ -2169,26 +2170,25 @@ def generate_batched(
             **generation_params,
     ) -> List[str]:
         """
-        Run N prompts through :py:meth:`generate` in batches of the
-        server's slot count (``LLAMA_ARG_N_PARALLEL``, 1 by default), so
-        every batch fills the slots, and return N outputs in input
-        order. The other arguments are as in
-        :py:meth:`ClamsPromptableApp.generate_batched`.
+        Run N prompts as :py:meth:`generate` does, keeping every server
+        slot busy (a prompt starts as soon as a slot is free), and
+        report progress through :py:meth:`report_progress` as each
+        prompt finishes. Returns N outputs in input order. The other
+        arguments are as in :py:meth:`ClamsPromptableApp.generate_batched`.
 
         :param parallel_prompts: not accepted; the server's slot count
-            sets the batch size.
-        :raises ValueError: if ``parallel_prompts`` is given.
+            sets how many prompts run together.
+        :raises ValueError: if ``parallel_prompts`` is given, or as in
+            :py:meth:`generate`.
         """
         if parallel_prompts is not None:
             raise ValueError(
                 f"{type(self).__name__} takes no parallel_prompts: "
                 f"llama-server decodes as many prompts together as it has "
-                f"slots; set LLAMA_ARG_N_PARALLEL to change that.")
-        return super().generate_batched(
-            prompt, system_prompt=system_prompt, images=images,
-            audios=audios, prompt_mode=prompt_mode,
-            parallel_prompts=getattr(self.server, 'slots', 1),
-            **generation_params)
+                f"slots, a server setting.")
+        return self._run_prompts(
+            prompt, system_prompt, images, audios, prompt_mode,
+            generation_params, report=True)
 
     def generate(
             self,
@@ -2217,8 +2217,25 @@ def generate(
         :raises RuntimeError: if no model is loaded.
         :raises ValueError: for audio input, which this backend does
             not support yet, or a prompt that does not fit in the
-            context of a server slot.
+            context of a server slot; the prompts not yet sent are
+            then dropped.
         """
+        return self._run_prompts(
+            prompt, system_prompt, images, audios, prompt_mode,
+            generation_params, report=False)
+
+    def _run_prompts(self, prompt: List[str], system_prompt: str,
+                     images: Optional[List[List[Any]]],
+                     audios: Optional[List[List[Any]]], prompt_mode: str,
+                     generation_params: Dict[str, Any],
+                     report: bool) -> List[str]:
+        """
+        Send the prompts of :py:meth:`generate` to the server, as many
+        at a time as it has slots, and return the outputs in input
+        order; with ``report``, call :py:meth:`report_progress` before
+        the first and after each finished prompt.
+        """
+        from concurrent.futures import ThreadPoolExecutor, as_completed
         from clams.backends.llamaserver import ContextSizeError
         if audios is not None:
             raise ValueError(
@@ -2248,20 +2265,31 @@ def run_one(i: int) -> str:
                     f"A prompt with {n_images} images does not fit in "
                     f"the context of a server slot "
                     f"({slot or 'unknown'} tokens). Lower "
-                    f"maxImagesPerPrompt, free GPU memory, or set the "
-                    f"context size with LLAMA_ARG_CTX_SIZE. Server "
+                    f"maxImagesPerPrompt, or give the server a larger "
+                    f"context (see the llama.cpp documentation). Server "
                     f"message: {e}") from e
             except Exception as e:
                 self.logger.error(
                     f"Error processing prompt {i}: {e}", exc_info=True)
                 return ''
 
-        if n == 1:
-            return [run_one(0)]
-        from concurrent.futures import ThreadPoolExecutor
+        if report:
+            self.report_progress(0, n)
+        outputs = [''] * n
         slots = getattr(self.server, 'slots', 1)
         with ThreadPoolExecutor(max_workers=max(1, min(n, slots))) as pool:
-            return list(pool.map(run_one, range(n)))
+            futures = {pool.submit(run_one, i): i for i in range(n)}
+            try:
+                for done, future in enumerate(as_completed(futures), 1):
+                    outputs[futures[future]] = future.result()
+                    if report:
+                        self.report_progress(done, n)
+            except BaseException:
+                # leave the prompts not yet sent unsent
+                for future in futures:
+                    future.cancel()
+                raise
+        return outputs
 
     def _chat(self, conversation: Union[List[dict], List[List[dict]]],
               params: dict) -> str:

clams/backends/llamaserver.py

@@ -32,14 +32,14 @@
 ``llama-server`` reads for every argument not given on its command
 line. Their values come from, in increasing precedence:
 
-1. ``llama-server``'s defaults, which fit the run to the hardware:
-   they place what fits on the GPUs, keep the rest in system memory,
-   and size the context to the free memory;
+1. ``llama-server``'s defaults;
 1. the ``llama_args`` of :class:`LlamaServer`, for what a model needs
    (e.g. ``{'LLAMA_ARG_UBATCH': '2048'}``);
-1. the environment of the process that starts the server (e.g.
-   ``LLAMA_ARG_CTX_SIZE``, ``LLAMA_ARG_THREADS``,
-   ``LLAMA_ARG_N_GPU_LAYERS`` set by whoever runs the app).
+1. the environment of the process that starts the server, set by
+   whoever runs the app.
+
+The variables and their defaults belong to llama.cpp and change
+between its releases; see the llama.cpp documentation.
 """
 
 import atexit
@@ -284,9 +284,9 @@ def _log_level(line: str) -> int:
     if re.search(r'\b(error|failed|warning|warn)\b', lowered) \
             or _LEVEL_LINE.match(lowered):
         return logging.WARNING
-    # device placement, load, readiness and timing lines
+    # load, readiness and timing lines
     if 'print_timing' in lowered or 'model loaded' in lowered \
-            or 'server is listening' in lowered or 'offloaded' in lowered:
+            or 'server is listening' in lowered:
         return logging.INFO
     return logging.DEBUG
 
@@ -341,8 +341,8 @@ class LlamaServer:
         server reads the whole file at startup).
     :param request_timeout: seconds to wait for one chat completion.
     :param logger: logger receiving the server's output: errors and
-        warnings at WARNING, placement, load and timing lines at INFO,
-        the rest at DEBUG.
+        warnings at WARNING, load and timing lines at INFO, the rest at
+        DEBUG.
     :raises ValueError: if a ``llama_args`` key is not an
         ``LLAMA_ARG_*`` variable, or names the server's address.
     """

documentation/app-baseclasses.rst

@@ -703,11 +703,17 @@ Audio input is not supported yet.
 
 GGUF apps have no ``parallelPrompts`` parameter. ``llama-server``
 decodes at most as many requests together as it has slots, a number
-fixed when the server starts, so
-:meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched` always
-sends that many prompts at a time. The slot count is a server setting
-(``LLAMA_ARG_N_PARALLEL``, see below); more slots split the same
-context into smaller parts.
+fixed when the server starts, and
+:meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched` keeps every
+slot busy. The slot count is a server setting (see below), and its
+default, like the way slots share the context, belongs to llama.cpp
+and changes between its releases; see the llama.cpp documentation. The
+startup log reports the slot count and the context of a slot.
+
+We recommend one slot, set in the app's ``LLAMA_ARGS``. The server
+decodes the prompts in its slots together, so with more slots an
+output can change with the other prompts in flight. In our tests on
+Maxwell and Pascal GPUs, more slots were also no faster.
 
 Class-attribute hooks
 ^^^^^^^^^^^^^^^^^^^^^
@@ -726,10 +732,10 @@ Class-attribute hooks
      - no
    * - ``LLAMA_ARGS``
      - Default values of llama.cpp's ``LLAMA_ARG_*`` environment
-       variables for what the app's models need, e.g.
-       ``{'LLAMA_ARG_UBATCH': '2048'}`` to raise the token budget per
-       image of a vision model. The environment of the app process
-       takes precedence (see below).
+       variables for what the app's models need, e.g. a larger batch
+       size for the image tokens of a vision model, or the slot count.
+       The environment of the app process takes precedence (see
+       below).
      - no
    * - ``RESTIFIER_PRODUCTION_WORKERS``,
        ``RESTIFIER_PRODUCTION_MAX_REQUESTS``
@@ -755,17 +761,17 @@ comes from, in increasing precedence:
    several GPUs. The server places what fits on the GPUs, keeps the
    rest in system memory, and sizes the context to the free memory.
 1. The app's ``LLAMA_ARGS``, for what its models need.
-1. The environment of the app process, set by whoever runs the app
-   (e.g. ``LLAMA_ARG_CTX_SIZE``, ``LLAMA_ARG_THREADS``,
-   ``LLAMA_ARG_N_GPU_LAYERS``).
+1. The environment of the app process, set by whoever runs the app.
 
-The settings in effect are logged with their source when the server
-starts, and the context size and slot count when it is ready.
+The variables and their defaults belong to llama.cpp and change
+between its releases; see the llama.cpp documentation. The settings in
+effect are logged with their source when the server starts, and the
+context size and slot count when it is ready.
 
 One prompt (system and user text, all its images, and the generated
 tokens) must fit in the context of a server slot. A prompt that does
 not fails the request with an error naming the size, so a caller can
-lower ``maxImagesPerPrompt`` or raise ``LLAMA_ARG_CTX_SIZE``.
+lower ``maxImagesPerPrompt`` or give the server a larger context.
 
 Installation and containers
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^

documentation/promptable-backends.rst

@@ -62,13 +62,13 @@ the latest release of each backend, built with CUDA support.
    * - Prompts run together
      - ``parallelPrompts`` runtime parameter, per request; the prompts
        are padded into one batch
-     - the server's slot count (``LLAMA_ARG_N_PARALLEL``), fixed when
-       the server starts; no ``parallelPrompts`` parameter
+     - the server's slot count, fixed when the server starts (one
+       recommended); no ``parallelPrompts`` parameter
    * - Memory for one request
      - grows with ``parallelPrompts`` and the prompt size; too much
        runs the GPU out of memory
-     - fixed when the server starts; the context is split over the
-       slots
+     - fixed when the server starts; how the slots share the context
+       depends on the llama.cpp release
    * - Prompt too large
      - out-of-memory error
      - context-size error (an error view naming the context size)
@@ -97,10 +97,10 @@ Who sets the batch size
 An HF app takes ``parallelPrompts`` with each request and stacks that
 many prompts into one forward pass. A GGUF app sends as many prompts
 at a time as ``llama-server`` has slots, and the server decodes them
-together. The slot count is set once, by whoever runs the app, and
-more slots split the same context into smaller parts. An app moved to
-the GGUF backend must stop reading ``parameters['parallelPrompts']``
-and stop passing ``parallel_prompts`` to
+together. The slot count is a server setting, fixed when the server
+starts; see :ref:`gguf-promptable` for the recommended value. An app
+moved to the GGUF backend must stop reading
+``parameters['parallelPrompts']`` and stop passing ``parallel_prompts`` to
 :meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched`.
 
 Outputs differ between backends

tests/test_llamaserver.py

@@ -14,6 +14,7 @@
 import textwrap
 import types
 import unittest
+import urllib.request
 from unittest import mock
 
 import numpy as np
@@ -31,8 +32,14 @@
     import json
     import os
     import sys
+    import threading
+    import time
     from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
+    # chat requests received, for /requests
+    COUNTS = {{'requests': 0}}
+    LOCK = threading.Lock()
+
     if os.environ.get('FAKE_LLAMA_EXIT'):
         print('error: failed to load model', flush=True)
         sys.exit(int(os.environ['FAKE_LLAMA_EXIT']))
@@ -69,10 +76,15 @@ def setting(flag, env, default):
                 slots = setting('--parallel', 'LLAMA_ARG_N_PARALLEL', 1)
                 self._send({{'default_generation_settings': {{
                     'n_ctx': total // slots}}, 'total_slots': slots}})
+            elif self.path == '/requests':
+                self._send(COUNTS)
             else:
                 self.send_error(404)
 
         def do_POST(self):
+            started = time.time()
+            with LOCK:
+                COUNTS['requests'] += 1
             length = int(self.headers['Content-Length'])
             request = json.loads(self.rfile.read(length))
             messages = request.pop('messages')
@@ -90,7 +102,11 @@ def do_POST(self):
                     'message': 'request exceeds the available context'}}}},
                     status=400)
                 return
+            # optional decode time, longer for prompts with more images
+            time.sleep(images * float(os.environ.get(
+                'FAKE_LLAMA_SECONDS_PER_IMAGE', 0)))
             echo ={{'images': images, 'last_user': content,
+                    'started': started, 'ended': time.time(),
                     'roles': [m['role'] for m in messages],
                     'assistant': [m['content'] for m in messages
                                   if m['role'] == 'assistant'],
@@ -360,8 +376,6 @@ def test_log_levels_of_server_output(self):
                          logging.WARNING)
         self.assertEqual(level('slot print_timing: id 0 | prompt eval'),
                          logging.INFO)
-        self.assertEqual(level('load_tensors: offloaded 49/49 layers to GPU'),
-                         logging.INFO)
         self.assertEqual(level('load_tensors: CPU_Mapped model buffer size'),
                          logging.DEBUG)
 
@@ -536,19 +550,36 @@ def test_no_parallel_prompts_parameter(self):
         names = {p.name for p in app.metadata.parameters}
         self.assertNotIn('parallelPrompts', names)
         self.assertIn('maxImagesPerPrompt', names)
-        with self.assertRaisesRegex(ValueError, 'LLAMA_ARG_N_PARALLEL'):
+        with self.assertRaisesRegex(ValueError, 'parallel_prompts'):
             app.generate_batched(['hi'], images=[[]], parallel_prompts=2)
 
-    def test_batches_follow_the_server_slots(self):
-        app = self.make_app(LLAMA_ARGS={'LLAMA_ARG_N_PARALLEL': '2'})
+    def test_free_slots_take_the_next_prompt(self):
+        # 2 slots; the first prompt takes three times as long as the
+        # others, so the third must start before the first ends
+        with _environ({'FAKE_LLAMA_SECONDS_PER_IMAGE': '0.3'}):
+            app = self.make_app(LLAMA_ARGS={'LLAMA_ARG_N_PARALLEL': '2'})
         self.assertEqual(app.server.slots, 2)
         progress = []
         app.report_progress = lambda done, total: progress.append(done)
-        images = [[Image.new('RGB', (4, 4))] * k for k in (1, 2, 3)]
+        images = [[Image.new('RGB', (4, 4))] * k for k in (3, 1, 1)]
         outputs = [json.loads(o) for o in app.generate_batched(
             ['describe'], images=images)]
-        self.assertEqual([o['images'] for o in outputs], [1, 2, 3])
-        self.assertEqual(progress, [0, 2, 3])
+        self.assertEqual([o['images'] for o in outputs], [3, 1, 1])
+        self.assertLess(outputs[2]['started'], outputs[0]['ended'])
+        # progress before the first and after each finished prompt
+        self.assertEqual(progress, [0, 1, 2, 3])
+
+    def test_context_error_leaves_queued_prompts_unsent(self):
+        app = self.make_app()  # one slot
+        images = [[Image.new('RGB', (4, 4))] for _ in range(5)]
+        with self.assertRaisesRegex(ValueError, 'does not fit'):
+            app.generate_batched(['TOO_LONG'], images=images)
+        with urllib.request.urlopen(
+                f"{app.server.base_url}/requests") as response:
+            sent = json.load(response)['requests']
+        # the failing prompt, and at most the one the free worker had
+        # already taken before the others were cancelled
+        self.assertLessEqual(sent, 2)
 
     def test_text_only_prompt_runs_once(self):
         app = self.make_app()