diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 4cdbca9..4bb427d 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -53,6 +53,15 @@ jobs: buildfilename: './container/Containerfile' version: ${{ needs.set-version.outputs.version }} + call-build-llamaserver: + name: "🤙 Call container workflow with `llamaserver`" + needs: ['set-version', 'check-deployment'] + uses: ./.github/workflows/container.yml + secrets: inherit + with: + buildfilename: './container/llamaserver.containerfile' + version: ${{ needs.set-version.outputs.version }} + call-build-ffmpeg: name: "🤙 Call container workflow with `ffmpeg`" needs: ['set-version', 'call-build-base'] diff --git a/build-tools/test.py b/build-tools/test.py index ea64a8c..20dd06e 100644 --- a/build-tools/test.py +++ b/build-tools/test.py @@ -38,6 +38,10 @@ def main(): "(useful if already installed)." ) args = parser.parse_args() + # print each line at once: in CI stdout is a pipe, where python + # would hold these lines until exit, after the output of the + # commands they introduce + sys.stdout.reconfigure(line_buffering=True) project_root = SCRIPT_DIR.parent diff --git a/clams/app/__init__.py b/clams/app/__init__.py index d004ce4..ad2711a 100644 --- a/clams/app/__init__.py +++ b/clams/app/__init__.py @@ -4,6 +4,7 @@ import os import pathlib import sys +import threading import warnings from abc import ABC, abstractmethod from contextlib import contextmanager @@ -13,7 +14,7 @@ from urllib import parse as urlparser __all__ = ['ClamsApp', 'ClamsPromptableApp', 'ClamsHFPromptableApp', - 'PromptTask', 'ImagesPerPromptMode'] + 'ClamsGGUFPromptableApp', 'PromptTask', 'ImagesPerPromptMode'] from typing import Union, Any, Optional, Dict, List, Mapping, Tuple, cast @@ -101,6 +102,19 @@ class ClamsApp(ABC): An abstract class to define API's for ClamsApps. A CLAMS app should inherit this class and then can be used with classes in :mod:`.restify` to work as web applications. + + Optional class attributes read by + :meth:`clams.restify.Restifier.serve_production` (see :mod:`clams.restify` + for their precedence): + + * ``RESTIFIER_PRODUCTION_WORKERS``: number of gunicorn workers. + * ``RESTIFIER_PRODUCTION_MAX_REQUESTS``: requests a worker serves before + it is replaced; ``0`` keeps workers. + + Declare them when the app holds something that must not be forked or + restarted with the worker, e.g. a model loaded in a child process in + GGUF subclass of :class:`~clams.app.ClamsHFPromptableApp`. Also note that + gunicorn configuration can be overridden with environment variables. """ # A set of "universal runtime parameters that can be used for both GET and POST anytime". # The behavioral changes based on these parameters must be implemented on the SDK level. @@ -139,6 +153,7 @@ class ClamsApp(ABC): def __init__(self): self.metadata: AppMetadata = self._load_appmetadata() + self._check_env_requirements() super().__init__() # data type spec to be used in type caster @@ -189,6 +204,30 @@ def _load_appmetadata(self) -> AppMetadata: metadata = self._appmetadata() return metadata + def _check_env_requirements(self) -> None: + """ + A private method to stop the app at startup when an environment + variable declared in ``env_requirements`` of the app metadata is unset + or empty. Only presence is checked, and the error message names the + missing variables. Values are never stored, logged, or included in the + message. + + :raises RuntimeError: if any required environment variable is unset + or empty + """ + # `os.environ.get` is used only for its truthiness, so no value is + # kept in a local variable (some traceback tools print locals) + missing = [env for env in self.metadata.env_requirements + if not os.environ.get(env.name)] + if missing: + listing = '\n'.join(f' {env.name}: {env.description}' + for env in missing) + raise RuntimeError( + f'{self.metadata.name} cannot start because these required ' + f'environment variables are unset or empty:\n{listing}\n' + 'See https://clams.ai/clams-python/clamsapp.html' + '#clamsapp-envvars for how to pass them.') + @abstractmethod def _appmetadata(self) -> AppMetadata: """ @@ -759,6 +798,10 @@ class ClamsPromptableApp(ClamsApp): * :py:meth:`response_to_grounded_textdocument` : persists a generated response into a view as ``TextDocument`` + ``Alignment`` (+ optional ``origins`` / ``origination``) + * an opt-in ``model`` parameter for apps whose models are pinned in + ``analyzer_versions`` (used by the HF and GGUF backends); once it + is added, parameter refinement records the pinned revision as + ``@`` in the view's ``appConfiguration`` """ #: SDK-managed runtime parameters injected into every promptable app. @@ -868,8 +911,8 @@ class ClamsPromptableApp(ClamsApp): 'GPU memory for long dynamic scenes. The value is ' 'operator-set, so the number of output TextDocuments is ' 'deterministic and does not vary by GPU. Composes with ' - '``parallelPrompts``, which batches independent prompts ' - 'into one forward pass.', + 'the number of prompts the backend runs together ' + '(``parallelPrompts`` in apps that offer it).', }, { 'name': 'imagesPerPromptMode', 'type': 'string', @@ -918,7 +961,7 @@ def __init__(self): # here. We only validate that the helper was actually called. super().__init__() declared = {p.name for p in self.metadata.parameters} - expected = {p['name'] for p in ClamsPromptableApp.promptable_parameters} + expected = {p['name'] for p in self.promptable_parameters} missing = expected - declared if missing: raise ValueError( @@ -1415,7 +1458,10 @@ def split_tagged_reasoning_trace( Only the inline-tag style is handled; models that isolate reasoning in a separate token-delimited channel (e.g. gpt-oss, Gemma) need their own - parsing in the app. + parsing in the app when their channel markers are stripped from the + decoded text. With :class:`ClamsGGUFPromptableApp`, llama-server keeps + the markers in the text, so Gemma 4's trace splits with + ``open_tag='<|channel>thought'`` and ``close_tag=''``. :param text: raw decoded model output. :param open_tag: opening marker of the inline reasoning block. @@ -1440,6 +1486,120 @@ def split_tagged_reasoning_trace( else: trace = None return answer, trace + + # ``model`` parameter for apps whose models are pinned in + # ``metadata.analyzer_versions`` (model id --> immutable reference, + # e.g. a Hugging Face commit hash). Opt-in: only apps that call + # ``_add_model_parameter`` get the parameter, and ``_refine_params`` + # touches a ``model`` value only when it is a key of + # ``analyzer_versions``. + + @staticmethod + def _add_model_parameter(metadata: AppMetadata, helper_name: str, + id_kind: str, description: str) -> None: + """ + Add a ``model`` runtime parameter whose choices are the keys of + ``metadata.analyzer_versions``. A singleton family defaults to + its only member; a multi-member family has no default. Called + by backend helpers such as + :py:meth:`ClamsHFPromptableApp.inject_promptable_parameters`. + + :param metadata: the :class:`AppMetadata` being built. + :param helper_name: name of the calling helper, for the error + message. + :param id_kind: what a key is (e.g. ``'HF model id'``), for the + error message. + :param description: description of the ``model`` parameter. + :raises ValueError: if ``metadata.analyzer_versions`` is + missing or empty. + """ + analyzer_versions = metadata.analyzer_versions or {} + if not analyzer_versions: + raise ValueError( + f"{helper_name} " + f"requires ``metadata.analyzer_versions`` to be a " + f"non-empty dict ({id_kind} -> commit hash). Set " + f"it on the ``AppMetadata`` constructor call before " + f"invoking this helper.") + choices: List[real_valued_primitives] = list( + analyzer_versions.keys()) + default = choices[0] if len(choices) == 1 else None + metadata.add_parameter( + name='model', + type='string', + choices=choices, + default=default, + multivalued=False, + description=description, + ) + + def _check_model_family(self, helper_name: str, + id_kind: str) -> Dict[str, str]: + """ + Validate at app init that ``analyzer_versions`` is declared and + that the ``model`` parameter was injected. Backends that require + pinned models call this from ``__init__``. + + :return: ``self.metadata.analyzer_versions``. + :raises ValueError: if either is missing. + """ + cls_name = type(self).__name__ + metadata = self.metadata + analyzer_versions = metadata.analyzer_versions + if not analyzer_versions: + raise ValueError( + f"{cls_name} must declare ``analyzer_versions`` in " + f"``metadata.py`` as a non-empty Dict[str, str] " + f"mapping {id_kind} to pinned commit " + f"hashes (7-char abbreviation is sufficient). This is " + f"required for reproducibility: an unpinned download " + f"silently floats on whatever ``main`` points at and " + f"cannot be reproduced. Singleton families (one " + f"entry) are fine; multi-model families list every " + f"member.") + if 'model' not in {p.name for p in metadata.parameters}: + raise ValueError( + f"{cls_name} must call " + f"``{helper_name}" + f"(metadata)`` (the override that also adds the " + f"``model`` parameter) inside ``appmetadata()`` in " + f"``metadata.py``; calling " + f"``ClamsPromptableApp.inject_promptable_parameters`` " + f"directly skips the ``model`` parameter and trips " + f"this check.") + return analyzer_versions + + def _refine_params(self, **runtime_params): + """ + Refine as :py:meth:`ClamsApp._refine_params` does, then expand a + ``model`` value that is a key of ``analyzer_versions`` to + ``@``, so the resolved revision lands in + ``view.metadata.appConfiguration['model']``. Other ``model`` + values (e.g. a remote API's model name) are left as they are. + """ + refined = super()._refine_params(**runtime_params) + model_id = refined.get('model') + if isinstance(model_id, str) and '@' not in model_id: + revision = (self.metadata.analyzer_versions or {}).get(model_id) + if revision is not None: + refined['model'] = f"{model_id}@{revision}" + return refined + + def _split_model_ref(self, model_ref: str) -> Tuple[str, str]: + """ + Parse ``@`` or a raw ```` (whose + revision is looked up in ``analyzer_versions``) into + ``(model_id, revision)``. + + :raises KeyError: if a raw model id is not in + ``analyzer_versions``. + """ + if '@' in model_ref: + model_id, _, revision = model_ref.rpartition('@') + return model_id, revision + return model_ref, (self.metadata.analyzer_versions or {})[model_ref] + + class ClamsHFPromptableApp(ClamsPromptableApp): """ Base class for promptable CLAMS apps backed by a local @@ -1523,23 +1683,10 @@ def inject_promptable_parameters(metadata: AppMetadata) -> None: missing or empty. """ ClamsPromptableApp.inject_promptable_parameters(metadata) - analyzer_versions = metadata.analyzer_versions or {} - if not analyzer_versions: - raise ValueError( - "ClamsHFPromptableApp.inject_promptable_parameters " - "requires ``metadata.analyzer_versions`` to be a " - "non-empty dict (HF model id -> commit hash). Set " - "it on the ``AppMetadata`` constructor call before " - "invoking this helper.") - choices: List[real_valued_primitives] = list( - analyzer_versions.keys()) - default = choices[0] if len(choices) == 1 else None - metadata.add_parameter( - name='model', - type='string', - choices=choices, - default=default, - multivalued=False, + ClamsPromptableApp._add_model_parameter( + metadata, + helper_name='ClamsHFPromptableApp.inject_promptable_parameters', + id_kind='HF model id', description=( "HuggingFace model identifier to use for this " "request. Must be one of the model ids declared in " @@ -1560,28 +1707,9 @@ def __init__(self): raise ValueError( f"{cls_name} must set the ``MODEL_CLS`` class attribute " f"(a ``transformers`` model class).") - analyzer_versions = self.metadata.analyzer_versions - if not analyzer_versions: - raise ValueError( - f"{cls_name} must declare ``analyzer_versions`` in " - f"``metadata.py`` as a non-empty Dict[str, str] " - f"mapping HuggingFace model ids to pinned commit " - f"hashes (7-char abbreviation is sufficient). This is " - f"required for reproducibility: an unpinned download " - f"silently floats on whatever ``main`` points at and " - f"cannot be reproduced. Singleton families (one " - f"entry) are fine; multi-model families list every " - f"member.") - if 'model' not in {p.name for p in self.metadata.parameters}: - raise ValueError( - f"{cls_name} must call " - f"``ClamsHFPromptableApp.inject_promptable_parameters" - f"(metadata)`` (the HF override that also adds the " - f"``model`` parameter) inside ``appmetadata()`` in " - f"``metadata.py``; calling " - f"``ClamsPromptableApp.inject_promptable_parameters`` " - f"directly skips the ``model`` parameter and trips " - f"this check.") + analyzer_versions = self._check_model_family( + helper_name='ClamsHFPromptableApp.inject_promptable_parameters', + id_kind='HuggingFace model ids') #: Per-(model_id, revision) cache of loaded #: ``(processor, model, device)`` triples. Populated by #: :py:meth:`load_model`; survives for the lifetime of this @@ -1602,20 +1730,6 @@ def __init__(self): only_model_id = next(iter(analyzer_versions.keys())) self.load_model(only_model_id) - def _refine_params(self, **runtime_params): - """ - Expand ``model`` from the raw HF id (``org/name``) to - ``org/name@`` so the resolved revision lands in - ``view.metadata.appConfiguration['model']``. - """ - refined = super()._refine_params(**runtime_params) - model_id = refined.get('model') - if isinstance(model_id, str) and '@' not in model_id: - revision = (self.metadata.analyzer_versions or {}).get(model_id) - if revision is not None: - refined['model'] = f"{model_id}@{revision}" - return refined - def load_model( self, model_id_or_with_rev: str, ) -> Tuple[Any, Any, str]: @@ -1636,11 +1750,7 @@ def load_model( :raises KeyError: if a raw model id is passed and is not in ``analyzer_versions``. """ - if '@' in model_id_or_with_rev: - model_id, _, revision = model_id_or_with_rev.rpartition('@') - else: - model_id = model_id_or_with_rev - revision = (self.metadata.analyzer_versions or {})[model_id] + model_id, revision = self._split_model_ref(model_id_or_with_rev) cache_key = (model_id, revision) cached = self._model_cache.get(cache_key) if cached is not None: @@ -1828,6 +1938,421 @@ def build_template_kwargs(self, **generation_params) -> dict: return {} +class ClamsGGUFPromptableApp(ClamsPromptableApp): + """ + Base class for promptable CLAMS apps that run a GGUF model with + llama.cpp (experimental). The model runs in a private inference + server that the app owns: the app picks the pinned model, starts + the server on localhost, talks to it through the OpenAI-compatible + chat API, and stops it on exit. It never connects to a server it + did not start. + + The default launcher (:py:meth:`launch_server`) runs llama.cpp's + ``llama-server`` on a GGUF model from the Hugging Face Hub, via + :mod:`clams.backends.llamaserver`. A subclass can override + :py:meth:`launch_server` to start another OpenAI-compatible server; + :py:meth:`generate` only needs the returned object's ``chat()``. + + Models are declared like :class:`ClamsHFPromptableApp` families: + ``analyzer_versions`` maps each model id to a pinned commit hash, + and the SDK derives the ``model`` runtime parameter from it. For + the llama-server launcher, a model id is ```` or + ``:``; the variant picks the GGUF file + (and the repo's ``mmproj`` file, for vision input) by name. + :py:attr:`MODEL_FILES` names the files explicitly when that is + ambiguous. Example:: + + class MyGGUFCaptioner(ClamsGGUFPromptableApp): + # In metadata.py: + # analyzer_versions={ + # "ggml-org/gemma-3-4b-it-GGUF:Q8_0": "", + # } + # plus a call to + # ClamsGGUFPromptableApp.inject_promptable_parameters( + # metadata). + + def _annotate(self, mmif, **parameters): + self.load_model(parameters['model']) + # ... self.generate_batched(prompt, images=..., ...) + # ... self.response_to_grounded_textdocument(...) + ... + + Requires the ``[llamaserver]`` extra and a ``llama-server`` + executable (see :func:`clams.backends.llamaserver.find_llama_server`). + """ + + #: Explicit file names per model id, for repositories where the + #: quantization tag does not pick exactly one file, e.g. + #: ``{'org/repo-GGUF:Q8_0': {'model': 'm-Q8_0.gguf', + #: 'mmproj': 'mmproj-m-f16.gguf'}}``. ``'model'`` may also be a + #: list of shard files. Models not listed here are resolved from + #: the repository file listing. + MODEL_FILES: dict[str, dict[str, Any]] | None = None + #: Default values of llama.cpp's ``LLAMA_ARG_*`` environment + #: variables, for the server settings the app's models need (see + #: the llama.cpp documentation for the variables and their + #: defaults, which change between llama.cpp releases). A variable + #: set in the environment of the app process takes precedence, and + #: settings not given here or there are left to llama-server's + #: defaults; see :mod:`clams.backends.llamaserver`. The settings in + #: effect are logged when the server starts, and the context size + #: and slot count are kept in ``self.server.slot_context`` and + #: ``self.server.slots``. A prompt (system and user text, images, + #: and the generated tokens) that does not fit in a slot fails the + #: request with an error naming the size. For the recommended slot + #: count, see :ref:`gguf-promptable`. + LLAMA_ARGS: dict[str, str] = {} + + # ``RESTIFIER_*``: the HTTP wrapper constraints any app may declare + # (see :class:`ClamsApp`); the model lives in a process this app + # owns, so the serving process must not be forked or restarted + # around it. + # + #: One worker. A singleton family starts its server when the app is + #: created, before gunicorn forks, so all workers would share that + #: one server; a multi-member family starts the server on first use + #: inside a worker, so each worker would hold its own copy of the + #: model. Raise this only if the GPUs can hold one model per worker. + RESTIFIER_PRODUCTION_WORKERS: int = 1 + #: Never replace a worker: a replacement would stop and restart the + #: server, and reload the model, on every request. + RESTIFIER_PRODUCTION_MAX_REQUESTS: int = 0 + + #: The promptable parameters of + #: :py:attr:`ClamsPromptableApp.promptable_parameters` except + #: ``parallelPrompts``: llama-server decodes at most as many requests + #: together as it has slots, so :py:meth:`generate_batched` always + #: sends that many at a time. + promptable_parameters = [ + p for p in ClamsPromptableApp.promptable_parameters + if p['name'] != 'parallelPrompts'] + + @staticmethod + def inject_promptable_parameters(metadata: AppMetadata) -> None: + """ + Add the SDK-managed promptable parameters (see + :py:attr:`promptable_parameters`) AND a ``model`` parameter + derived from ``metadata.analyzer_versions``. Call this at the end + of ``appmetadata()`` in ``metadata.py`` if your app subclasses + :py:class:`ClamsGGUFPromptableApp`. + + :raises ValueError: if ``metadata.analyzer_versions`` is + missing or empty. + """ + for param in ClamsGGUFPromptableApp.promptable_parameters: + metadata.add_parameter(**param) + ClamsPromptableApp._add_model_parameter( + metadata, + helper_name=( + 'ClamsGGUFPromptableApp.inject_promptable_parameters'), + id_kind='model id', + description=( + "Model to use for this request. Must be one of the " + "model ids declared in this app's " + "``analyzer_versions``; the SDK pins the corresponding " + "commit hash at load time. A model id is a Hugging " + "Face repository, optionally followed by " + "``:`` (e.g. ``org/repo-GGUF:Q8_0``). " + "When the app ships a single model, this parameter " + "defaults to it and can be omitted." + ), + ) + + def __init__(self): + super().__init__() + analyzer_versions = self._check_model_family( + helper_name=( + 'ClamsGGUFPromptableApp.inject_promptable_parameters'), + id_kind='model ids') + #: The running server (``None`` until :py:meth:`load_model`). + self.server: Any = None + #: ``(model_id, revision)`` the running server serves. + self.loaded_model: Optional[Tuple[str, str]] = None + self._load_lock = threading.Lock() + # singleton families start their server at init, like the HF + # backend pre-loads; multi-member families start on first use + if len(analyzer_versions) == 1: + self.load_model(next(iter(analyzer_versions.keys()))) + + def load_model(self, model_id_or_with_rev: str) -> Any: + """ + Make sure the server for the given model is running, stopping + the server of a previously loaded model first, so one model + holds GPU memory at a time. Accepts both refined + (``@``) and raw (````) forms. + + A server started by another process (the gunicorn master, for a + singleton family that starts its server before the fork) is + used as it is: this process cannot stop or replace it. + + :return: the server object (also stored on ``self.server``). + :raises RuntimeError: if a server owned by another process + serves a different model, or has stopped. + """ + model_id, revision = self._split_model_ref(model_id_or_with_rev) + with self._load_lock: + if (self.loaded_model == (model_id, revision) + and self.server is not None + and self.server.is_running()): + return self.server + owned = getattr(self.server, 'owned_here', lambda: True)() + if self.server is not None and not owned: + state = ('is not running' + if self.loaded_model == (model_id, revision) + else 'serves a different model') + raise RuntimeError( + f"The model server was started by another process " + f"for {self.loaded_model} and {state}; this process " + f"cannot replace it. Restart the app.") + if self.server is not None: + self.logger.info( + f"Stopping server for {self.loaded_model}") + self.server.stop() + self.server = None + self.loaded_model = None + self.logger.info( + f"Starting server for {model_id} @ {revision}") + self.server = self.launch_server(model_id, revision) + self.loaded_model = (model_id, revision) + return self.server + + def _model_files(self, model_id: str, + revision: str) -> Tuple[List[str], Optional[str]]: + """ + File names of the given model in its repository: + ``(model_files, mmproj_file)``. Uses :py:attr:`MODEL_FILES` + when it lists the model, the repository file listing otherwise. + """ + explicit = (self.MODEL_FILES or {}).get(model_id) + if explicit is not None: + files = explicit['model'] + files = [files] if isinstance(files, str) else list(files) + return files, explicit.get('mmproj') + from clams.backends.llamaserver import resolve_gguf_files + repo_id, _, variant = model_id.partition(':') + return resolve_gguf_files(repo_id, revision, variant) + + def launch_server(self, model_id: str, revision: str) -> Any: + """ + Download the pinned GGUF files and start ``llama-server`` on + localhost, with :py:attr:`LLAMA_ARGS` as default settings. + + Override to launch a different OpenAI-compatible server, or to + pass other :class:`~clams.backends.llamaserver.LlamaServer` + arguments (e.g. longer timeouts); the returned object needs + ``chat(messages, **params) -> str``, ``is_running() -> bool`` + and ``stop()``. + + :raises RuntimeError: if the server fails to start. + """ + from clams.backends import llamaserver + repo_id = model_id.partition(':')[0] + model_names, mmproj_name = self._model_files(model_id, revision) + model_paths, mmproj_path = llamaserver.download_gguf( + repo_id, revision, model_names, mmproj_name) + # TODO: record the server's slot count, context per slot and + # llama.cpp build (``/props``) in the view's ``appProfiling``; + # the slot count can change outputs. ``hwFetch`` reports no GPU + # memory for this backend: ``_profile_cuda_memory`` needs torch + # and cannot see the server process. + return llamaserver.LlamaServer( + model_paths[0], mmproj_path, llama_args=self.LLAMA_ARGS, + logger=self.logger).start() + + def generate_batched( + self, + prompt: List[str], + system_prompt: str = '', + images: Optional[List[List[Any]]] = None, + audios: Optional[List[List[Any]]] = None, + prompt_mode: str = 'turn-taking', + parallel_prompts: Optional[int] = None, + **generation_params, + ) -> List[str]: + """ + Run N prompts as :py:meth:`generate` does, keeping every server + slot busy (a prompt starts as soon as a slot is free), and + report progress through :py:meth:`report_progress` as each + prompt finishes. Returns N outputs in input order. The other + arguments are as in :py:meth:`ClamsPromptableApp.generate_batched`. + + :param parallel_prompts: not accepted; the server's slot count + sets how many prompts run together. + :raises ValueError: if ``parallel_prompts`` is given, or as in + :py:meth:`generate`. + """ + if parallel_prompts is not None: + raise ValueError( + f"{type(self).__name__} takes no parallel_prompts: " + f"llama-server decodes as many prompts together as it has " + f"slots, a server setting.") + return self._run_prompts( + prompt, system_prompt, images, audios, prompt_mode, + generation_params, report=True) + + def generate( + self, + prompt: List[str], + system_prompt: str = '', + images: Optional[List[List[Any]]] = None, + audios: Optional[List[List[Any]]] = None, + prompt_mode: str = 'turn-taking', + **generation_params, + ) -> List[str]: + """ + Run N independent prompts against the server, as many at a time + as the server has slots, and return N outputs in order. Images + go to the server as base64 data URLs. A prompt that fails is + logged and yields ``''``, as in the HF backend, except a prompt + longer than the context of a server slot (see + :py:attr:`LLAMA_ARGS`): that is a configuration problem, so it + fails the request. + + Generation parameters are mapped by :py:meth:`build_request_params`; + :py:meth:`build_template_kwargs` adds ``chat_template_kwargs`` + (e.g. a reasoning switch). In ``user-only`` multi-turn mode, + each user turn is sent in sequence with the earlier replies + filled in, and the last reply is returned. + + :raises RuntimeError: if no model is loaded. + :raises ValueError: for audio input, which this backend does + not support yet, or a prompt that does not fit in the + context of a server slot; the prompts not yet sent are + then dropped. + """ + return self._run_prompts( + prompt, system_prompt, images, audios, prompt_mode, + generation_params, report=False) + + def _run_prompts(self, prompt: List[str], system_prompt: str, + images: Optional[List[List[Any]]], + audios: Optional[List[List[Any]]], prompt_mode: str, + generation_params: Dict[str, Any], + report: bool) -> List[str]: + """ + Send the prompts of :py:meth:`generate` to the server, as many + at a time as it has slots, and return the outputs in input + order; with ``report``, call :py:meth:`report_progress` before + the first and after each finished prompt. + """ + from concurrent.futures import ThreadPoolExecutor, as_completed + from clams.backends.llamaserver import ContextSizeError + if audios is not None: + raise ValueError( + f"{type(self).__name__} does not support audio input.") + if self.server is None: + raise RuntimeError( + "No model is loaded; call ``load_model`` first.") + n = len(images) if images is not None else 1 + if n == 0: + return [] + params = self.build_request_params(**generation_params) + template_kwargs = self.build_template_kwargs(**generation_params) + if template_kwargs: + params['chat_template_kwargs'] = template_kwargs + + def run_one(i: int) -> str: + conversation = self.build_conversation( + prompt, system_prompt=system_prompt, + images=images[i] if images is not None else None, + prompt_mode=prompt_mode) + try: + return self._chat(conversation, params) + except ContextSizeError as e: + n_images = len(images[i]) if images is not None else 0 + slot = getattr(self.server, 'slot_context', None) + raise ValueError( + f"A prompt with {n_images} images does not fit in " + f"the context of a server slot " + f"({slot or 'unknown'} tokens). Lower " + f"maxImagesPerPrompt, or give the server a larger " + f"context (see the llama.cpp documentation). Server " + f"message: {e}") from e + except Exception as e: + self.logger.error( + f"Error processing prompt {i}: {e}", exc_info=True) + return '' + + if report: + self.report_progress(0, n) + outputs = [''] * n + slots = getattr(self.server, 'slots', 1) + with ThreadPoolExecutor(max_workers=max(1, min(n, slots))) as pool: + futures = {pool.submit(run_one, i): i for i in range(n)} + try: + for done, future in enumerate(as_completed(futures), 1): + outputs[futures[future]] = future.result() + if report: + self.report_progress(done, n) + except BaseException: + # leave the prompts not yet sent unsent + for future in futures: + future.cancel() + raise + return outputs + + def _chat(self, conversation: Union[List[dict], List[List[dict]]], + params: dict) -> str: + from clams.backends.llamaserver import to_openai_messages + if conversation and isinstance(conversation[0], list): + # user-only multi-turn: progressively extending prefixes + # with ``content=None`` assistant placeholders + replies: List[str] = [] + for prefix in cast(List[List[dict]], conversation): + filled, used = [], 0 + for message in prefix: + if (message['role'] == 'assistant' + and message['content'] is None): + message = {**message, 'content': replies[used]} + used += 1 + filled.append(message) + replies.append(self.server.chat( + to_openai_messages(filled), **params)) + return replies[-1] + return self.server.chat( + to_openai_messages(cast(List[dict], conversation)), **params) + + @staticmethod + def build_request_params( + max_new_tokens: int = 512, + temperature: float = 0.0, + top_p: float = 1.0, + top_k: int = 50, + **_unused, + ) -> dict: + """ + Translate the SDK's promptable-parameter values into + chat-completion request fields. ``temperature == 0.0`` gives + greedy decoding. ``top_k`` is not part of the OpenAI API, but + llama-server (and vLLM) accept it. + + ``cache_prompt`` is off: llama-server's prompt cache reuses the + computed prefix of an earlier request, which changes outputs + even with greedy decoding, so the same input would not always + give the same output. Turn it on in an override when speed on + repeated prefixes matters more than reproducibility. + + Subclasses MAY override to add server-specific fields + (``repeat_penalty``, ``seed``, ``response_format``, ...). Extra + keyword args are ignored, so ``_annotate`` can pass the full + generation parameter dict. + """ + return {'max_tokens': max_new_tokens, 'temperature': temperature, + 'top_p': top_p, 'top_k': top_k, 'cache_prompt': False} + + def build_template_kwargs(self, **generation_params) -> dict: + """ + Extra variables for the server-side chat template, sent as the + request's ``chat_template_kwargs``. The base implementation maps + the SDK ``useReasoning`` parameter onto ``enable_thinking``, the + switch of Gemma 4, Qwen3 and similar templates, so reasoning is + off unless requested; templates that do not use the variable + ignore it. Override for other template variables. + """ + return {'enable_thinking': + bool(generation_params.get('use_reasoning', False))} + + class ParameterCaster(object): def __init__(self, param_spec: Mapping[str, Tuple[str, bool]]): diff --git a/clams/appmetadata/__init__.py b/clams/appmetadata/__init__.py index 588a867..3bce45f 100644 --- a/clams/appmetadata/__init__.py +++ b/clams/appmetadata/__init__.py @@ -268,6 +268,29 @@ def __init__(self, **kwargs): } +class EnvRequirement(_BaseModel): + """ + Defines a data model that describes an environment variable that a CLAMS + app cannot run without, such as an access token for a gated model. Only + the name of the variable is part of the app metadata, never its value. + """ + name: str = pydantic.Field( + ..., + pattern=r'^[A-Za-z_][A-Za-z0-9_]*$', + description="Name of the environment variable." + ) + description: str = pydantic.Field( + ..., + description="Description of what the variable is for and how to " + "obtain its value." + ) + + model_config = { + 'title': 'CLAMS App Environment Requirement', + 'extra': 'forbid', + } + + class AppMetadata(pydantic.BaseModel): """ Data model that describes a CLAMS app. @@ -379,6 +402,14 @@ class AppMetadata(pydantic.BaseModel): "The values declared here are propagated by the SDK into the ``appTags`` field of every view " "the app signs." ) + env_requirements: List[EnvRequirement] = pydantic.Field( + [], + description="(optional) List of environment variables that the app " + "cannot run without. Being listed here means the variable " + "is required, so optional settings are left out. Values " + "of these variables are never part of app metadata or " + "output MMIF." + ) est_gpu_mem_min: int = pydantic.Field( 0, description="(optional) Minimum GPU memory required to run the app, in megabytes (MB). " @@ -567,6 +598,24 @@ def add_more(self, key: str, value: str): raise ValueError(f"'{key}' is already being used in the appmetadata!") else: raise ValueError("Key and value should not be empty!") + + def add_env_requirement(self, name: str, + description: str) -> EnvRequirement: + """ + Helper method to declare an environment variable that the app cannot + run without. + + :param name: name of the environment variable + :param description: what the variable is for and how to get its value + :returns: the newly added + :class:`~clams.appmetadata.EnvRequirement` object + :raises ValueError: if the name is invalid or is already declared + """ + if any(env.name == name for env in self.env_requirements): + raise ValueError(f"'{name}' is already declared as required!") + new_env = EnvRequirement(name=name, description=description) + self.env_requirements.append(new_env) + return new_env def jsonify(self, pretty=False): """ diff --git a/clams/backends/llamaserver.py b/clams/backends/llamaserver.py new file mode 100644 index 0000000..1fa2d79 --- /dev/null +++ b/clams/backends/llamaserver.py @@ -0,0 +1,576 @@ +""" +llama.cpp ``llama-server`` backend helpers. + +A CLAMS app that uses this backend runs its model in a private +``llama-server`` process: the app downloads a pinned GGUF file (and, +for vision models, its ``mmproj`` projector file), starts the server +on a free localhost port, talks to it through the OpenAI-compatible +chat API, and stops it when the app exits. The server is never shared +with, or borrowed from, anything outside the app; model selection stays +with the app. + +* :func:`find_llama_server` -- locate the ``llama-server`` executable. +* :func:`resolve_gguf_files` -- find the GGUF file(s) and the ``mmproj`` + file of a quantization variant in a Hugging Face Hub repository. +* :func:`download_gguf` -- download those files at a pinned revision. +* :class:`LlamaServer` -- start, health-check, query and stop one + ``llama-server`` process. +* :func:`to_openai_messages` -- turn a conversation built by + :py:meth:`clams.app.ClamsPromptableApp.build_conversation` into + OpenAI chat messages, with images as base64 data URLs. + +The ``llama-server`` binary itself is not a python dependency. It is +found, in this order, from the ``binary`` argument, the +``LLAMA_SERVER`` environment variable, or ``llama-server`` on the +``PATH``. ``huggingface_hub`` and ``pillow`` come with the +``[llamaserver]`` extra:: + + pip install clams-python[llamaserver] + +Server settings other than the model files and the address are +llama.cpp's own ``LLAMA_ARG_*`` environment variables, which +``llama-server`` reads for every argument not given on its command +line. Their values come from, in increasing precedence: + +1. ``llama-server``'s defaults; +1. the ``llama_args`` of :class:`LlamaServer`, for what a model needs + (e.g. ``{'LLAMA_ARG_UBATCH': '2048'}``); +1. the environment of the process that starts the server, set by + whoever runs the app. + +The variables and their defaults belong to llama.cpp and change +between its releases; see the llama.cpp documentation. +""" + +import atexit +import base64 +import collections +import io +import json +import logging +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from typing import Any, Deque, List, Mapping, Optional, Sequence, Tuple + +__all__ = ['LLAMA_SERVER_ENV', 'ContextSizeError', 'LlamaServer', + 'download_gguf', 'find_llama_server', 'resolve_gguf_files', + 'to_openai_messages'] + +#: environment variable naming the ``llama-server`` binary to run +LLAMA_SERVER_ENV = 'LLAMA_SERVER' + +_SHARD_PATTERN = re.compile(r'-(\d{5})-of-(\d{5})\.gguf$') +_DRAFT_PREFIXES = ('mtp-', 'dflash-', 'draft-') +_LOG_TAIL_LINES = 40 +_LLAMA_ARG_PREFIX = 'LLAMA_ARG_' +# the server's address is always given on the command line, so these two +# are dropped from its environment (the upstream image sets them) +_ADDRESS_ENV = ('LLAMA_ARG_HOST', 'LLAMA_ARG_PORT') +# server log lines: optional timestamp, then a level letter +_LEVEL_LINE = re.compile(r'^(?:[\d.]+\s+)?[ew]\s') + + +def find_llama_server(binary: Optional[str] = None) -> str: + """ + Locate the ``llama-server`` executable. + + :param binary: explicit path or command name. When ``None``, the + ``LLAMA_SERVER`` environment variable is used, then + ``llama-server`` on the ``PATH``. + :return: absolute path of the executable. + :raises FileNotFoundError: if no executable is found. + """ + candidate = binary or os.environ.get(LLAMA_SERVER_ENV) or 'llama-server' + resolved = shutil.which(candidate) + if resolved is None: + raise FileNotFoundError( + f"llama-server executable not found (looked for " + f"{candidate!r}). Install llama.cpp, put llama-server on the " + f"PATH, or set the {LLAMA_SERVER_ENV} environment variable " + f"to its path.") + return resolved + + +def _free_port(host: str = '127.0.0.1') -> int: + """Return a TCP port on ``host`` that is free at the time of the call.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((host, 0)) + return sock.getsockname()[1] + + +def _pick_gguf_files( + filenames: Sequence[str], + variant: str = '', +) -> Tuple[List[str], Optional[str]]: + """ + Choose the model file(s) and the ``mmproj`` file of one + quantization variant from a repository file listing. + + Files whose base name starts with ``mmproj`` are projector files. + Files with a speculative-decoding prefix (``mtp-``, ``dflash-``, + ``draft-``) are draft models and are skipped. Every other + ``.gguf`` file is a model file. A model file matches ``variant`` + when the variant string occurs in its name, ignoring case. When a + repository has several projectors, the one matching ``variant`` + is chosen if exactly one does. A model split into shards + (``*-00001-of-00003.gguf``) is returned as all of its shards, in + order; llama.cpp loads the rest from the first. + + :param filenames: repository file names (paths relative to the + repository root). + :param variant: quantization tag such as ``Q8_0`` or ``Q4_K_M``. + Empty means the repository holds exactly one model. + :return: ``(model_files, mmproj_file)``; ``mmproj_file`` is + ``None`` when the repository has no projector. + :raises ValueError: if no model file, several model files, or + several projector files match. + """ + ggufs = sorted(f for f in filenames if f.lower().endswith('.gguf')) + projectors = [f for f in ggufs + if os.path.basename(f).lower().startswith('mmproj')] + models = [f for f in ggufs if f not in projectors + and not os.path.basename(f).lower().startswith(_DRAFT_PREFIXES)] + if variant: + models = [f for f in models if variant.lower() in f.lower()] + matching = [f for f in projectors if variant.lower() in f.lower()] + if len(matching) == 1: + projectors = matching + if not models: + raise ValueError( + f"no GGUF model file matches variant {variant!r}; " + f"available: {[f for f in ggufs if f not in projectors]}") + if len(models) > 1: + shards = [_SHARD_PATTERN.search(f) for f in models] + totals = {m.group(2) for m in shards if m} + if not (all(shards) and len(totals) == 1 + and len(models) == int(totals.pop())): + raise ValueError( + f"variant {variant!r} matches several GGUF files: " + f"{models}; name the file explicitly in MODEL_FILES") + if len(projectors) > 1: + raise ValueError( + f"repository has several mmproj files: {projectors}; name " + f"the one to use explicitly in MODEL_FILES") + return models, (projectors[0] if projectors else None) + + +def resolve_gguf_files( + repo_id: str, revision: str, variant: str = '', +) -> Tuple[List[str], Optional[str]]: + """ + List a Hugging Face Hub repository at ``revision`` and pick the + files of ``variant`` with :func:`_pick_gguf_files`. When the Hub + cannot be reached (offline mode, or no network), the files of the + revision already in the local cache are listed instead, so a + cached model starts without network. + """ + try: + from huggingface_hub import HfApi, snapshot_download + except ImportError as e: + raise ImportError( + "The llama-server backend requires huggingface_hub. Install " + "it via `pip install clams-python[llamaserver]`.") from e + try: + files = HfApi().list_repo_files(repo_id, revision=revision) + except Exception as online_error: + try: + snapshot = snapshot_download( + repo_id, revision=revision, local_files_only=True) + except Exception: + raise online_error + files = [os.path.relpath(os.path.join(root, name), snapshot) + for root, _, names in os.walk(snapshot) for name in names] + return _pick_gguf_files(files, variant) + + +def download_gguf( + repo_id: str, + revision: str, + model_files: Sequence[str], + mmproj_file: Optional[str] = None, +) -> Tuple[List[str], Optional[str]]: + """ + Download pinned GGUF files from the Hugging Face Hub (or reuse the + local cache) and return their local paths. + + :param repo_id: Hub repository, e.g. ``ggml-org/gemma-3-4b-it-GGUF``. + :param revision: commit hash the files are pinned to. + :param model_files: model file name(s); several for a sharded model. + :param mmproj_file: projector file name, or ``None``. + :return: ``(model_paths, mmproj_path)``. + """ + try: + from huggingface_hub import hf_hub_download + except ImportError as e: + raise ImportError( + "The llama-server backend requires huggingface_hub. Install " + "it via `pip install clams-python[llamaserver]`.") from e + model_paths = [hf_hub_download(repo_id, f, revision=revision) + for f in model_files] + mmproj_path = (hf_hub_download(repo_id, mmproj_file, revision=revision) + if mmproj_file else None) + return model_paths, mmproj_path + + +def _image_to_data_url(image: Any) -> str: + """ + Encode an image as a ``data:image/png;base64,...`` URL. + + :param image: a PIL image, a numpy array (as returned by + ``mmif.utils.video_document_helper``), a path to an image + file, or a string that already is an ``http(s)://`` or + ``data:`` URL (returned unchanged). + """ + if isinstance(image, str): + if image.startswith(('http://', 'https://', 'data:')): + return image + with open(image, 'rb') as f: + raw = f.read() + return 'data:image/png;base64,' + base64.b64encode(raw).decode() + from PIL import Image + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + buffer = io.BytesIO() + image.save(buffer, format='PNG') + return 'data:image/png;base64,' + base64.b64encode( + buffer.getvalue()).decode() + + +def to_openai_messages(conversation: List[dict]) -> List[dict]: + """ + Convert a conversation in the SDK's chat format (see + :py:meth:`clams.app.ClamsPromptableApp.build_conversation`) into + OpenAI chat messages. Text parts pass through; image parts become + ``image_url`` parts with base64 data URLs. + + :raises ValueError: for audio parts, which this backend does not + support yet. + """ + messages = [] + for message in conversation: + content = message.get('content') + if isinstance(content, list): + parts = [] + for part in content: + kind = part.get('type') + if kind == 'text': + parts.append({'type': 'text', 'text': part['text']}) + elif kind == 'image': + parts.append({ + 'type': 'image_url', + 'image_url': {'url': _image_to_data_url( + part['image'])}}) + else: + raise ValueError( + f"content part type {kind!r} is not supported by " + f"the llama-server backend") + content = parts + messages.append({'role': message['role'], 'content': content}) + return messages + + +def _log_level(line: str) -> int: + """Logging level for one line of ``llama-server`` output.""" + lowered = line.lower() + if re.search(r'\b(error|failed|warning|warn)\b', lowered) \ + or _LEVEL_LINE.match(lowered): + return logging.WARNING + # load, readiness and timing lines + if 'print_timing' in lowered or 'model loaded' in lowered \ + or 'server is listening' in lowered: + return logging.INFO + return logging.DEBUG + + +class ContextSizeError(RuntimeError): + """A request was longer than the context of the server's slot.""" + + +def _exit_on_sigterm() -> None: + """ + Turn a SIGTERM with the default action into ``SystemExit``, so that + ``atexit`` handlers (and with them :py:meth:`LlamaServer.stop`) run + when the app is stopped with ``kill`` or ``docker stop``. Leaves a + SIGTERM handler that someone else installed (e.g. gunicorn's) alone, + and does nothing outside the main thread, where python cannot set + signal handlers. + """ + if threading.current_thread() is not threading.main_thread(): + return + if signal.getsignal(signal.SIGTERM) is signal.SIG_DFL: + signal.signal(signal.SIGTERM, lambda *_: sys.exit(128 + 15)) + + +class LlamaServer: + """ + One ``llama-server`` process serving one model on localhost. + + The process belongs to the python process that started it: only + that process stops it, so a forked child (e.g. a gunicorn worker) + exiting does not take the server down with it. The server is also + stopped at interpreter exit, including a SIGTERM to the main + process. A SIGKILL (or an out-of-memory kill) of the python process + leaves the server running. + + :param model_path: local path of the GGUF model file (the first + shard for a sharded model). + :param mmproj_path: local path of the projector file for vision + input, or ``None``. + :param binary: ``llama-server`` executable; see + :func:`find_llama_server`. + :param host: interface to bind; keep the default so the server is + reachable only from this machine. + :param port: port to bind; a free port is chosen when ``None``. + :param llama_args: default values of ``LLAMA_ARG_*`` environment + variables for the server, e.g. ``{'LLAMA_ARG_UBATCH': '2048'}``. + A variable set in the environment of this process takes + precedence (see the module docstring). The context size and slot + count the server ends up with are read after startup into + :py:attr:`slot_context` and :py:attr:`slots`. + :param startup_timeout: seconds to wait for the model to load. A + large model on a slow disk needs more than the default (the + server reads the whole file at startup). + :param request_timeout: seconds to wait for one chat completion. + :param logger: logger receiving the server's output: errors and + warnings at WARNING, load and timing lines at INFO, the rest at + DEBUG. + :raises ValueError: if a ``llama_args`` key is not an + ``LLAMA_ARG_*`` variable, or names the server's address. + """ + + def __init__( + self, + model_path: str, + mmproj_path: Optional[str] = None, + *, + binary: Optional[str] = None, + host: str = '127.0.0.1', + port: Optional[int] = None, + llama_args: Optional[Mapping[str, Any]] = None, + startup_timeout: float = 600.0, + request_timeout: float = 1800.0, + logger: Optional[logging.Logger] = None, + ): + llama_args = dict(llama_args or {}) + invalid = [k for k in llama_args + if not k.startswith(_LLAMA_ARG_PREFIX) + or k in _ADDRESS_ENV] + if invalid: + raise ValueError( + f"llama_args takes {_LLAMA_ARG_PREFIX}* environment " + f"variables other than {', '.join(_ADDRESS_ENV)}: " + f"{invalid}") + self.model_path = model_path + self.mmproj_path = mmproj_path + self.binary = binary + self.host = host + self.port = port + self.llama_args = {k: str(v) for k, v in llama_args.items()} + self.startup_timeout = startup_timeout + self.request_timeout = request_timeout + self.logger = logger or logging.getLogger(__name__) + #: context size of one slot in tokens, as the running server + #: reports it (``None`` until started, or if it cannot tell) + self.slot_context: Optional[int] = None + #: number of requests the running server decodes together + #: (``1`` until started, or if it cannot tell) + self.slots: int = 1 + self._process: Optional[subprocess.Popen] = None + self._owner_pid: Optional[int] = None + self._log_tail: Deque[str] = collections.deque( + maxlen=_LOG_TAIL_LINES) + + @property + def base_url(self) -> str: + """``http://host:port`` of the running server.""" + return f"http://{self.host}:{self.port}" + + def _command(self) -> List[str]: + """The command line used to start the server.""" + cmd = [find_llama_server(self.binary), + '--model', self.model_path, + '--host', self.host, '--port', str(self.port), + # keep reasoning ("thinking") tags in the text, as the HF + # backend does, so apps split them with + # ``split_tagged_reasoning_trace`` + '--reasoning-format', 'none'] + if self.mmproj_path: + cmd += ['--mmproj', self.mmproj_path] + return cmd + + def _environment(self) -> Tuple[dict, List[str]]: + """ + The server's environment: this process's, over the defaults in + :py:attr:`llama_args`, without the address variables. Also + returns the ``LLAMA_ARG_*`` settings in effect with their + source, for the log. + """ + env = {**self.llama_args, **os.environ} + for key in _ADDRESS_ENV: + env.pop(key, None) + settings = [ + f"{k}={env[k]} " + f"({'environment' if k in os.environ else 'app'})" + for k in sorted(env) if k.startswith(_LLAMA_ARG_PREFIX)] + return env, settings + + def owned_here(self) -> bool: + """Whether this python process started (and may stop) the server.""" + return self._process is not None and os.getpid() == self._owner_pid + + def is_running(self) -> bool: + """ + Whether the server process is alive. A forked child (e.g. a + gunicorn worker of an app whose server started before the + fork) cannot wait on its parent's child process, so it asks the + server's ``/health`` endpoint instead. + """ + if self._process is None: + return False + if not self.owned_here(): + return self._healthy() + return self._process.poll() is None + + def start(self) -> 'LlamaServer': + """ + Start the server and block until it reports healthy. + + :raises RuntimeError: if the process exits or does not become + healthy within ``startup_timeout``; the message includes the + last lines of the server's output. + """ + if self.is_running(): + return self + if self.port is None: + self.port = _free_port(self.host) + cmd = self._command() + env, settings = self._environment() + self.logger.info(f"Starting llama-server: {' '.join(cmd)}" + + (f"; {', '.join(settings)}" if settings + else '')) + self._log_tail.clear() + self._process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, text=True, encoding='utf-8', + errors='replace', bufsize=1, env=env) + self._owner_pid = os.getpid() + threading.Thread(target=self._forward_output, daemon=True).start() + atexit.register(self.stop) + _exit_on_sigterm() + deadline = time.monotonic() + self.startup_timeout + while time.monotonic() < deadline: + if self._process.poll() is not None: + time.sleep(0.2) # let the reader thread drain the pipe + raise RuntimeError( + f"llama-server exited with code " + f"{self._process.returncode} during startup:\n" + + '\n'.join(self._log_tail)) + if self._healthy(): + self.slot_context, self.slots = self._read_props() + self.logger.info( + f"llama-server ready at {self.base_url}; " + f"{self.slots} slot(s) of " + f"{self.slot_context or 'unknown'} tokens") + return self + time.sleep(0.5) + self.stop() + raise RuntimeError( + f"llama-server did not become healthy within " + f"{self.startup_timeout} s:\n" + '\n'.join(self._log_tail)) + + def stop(self) -> None: + """Stop the server if this process started it.""" + process = self._process + if process is None or os.getpid() != self._owner_pid: + return + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + self._process = None + atexit.unregister(self.stop) + + def __enter__(self) -> 'LlamaServer': + return self.start() + + def __exit__(self, *exc) -> None: + self.stop() + + def chat(self, messages: List[dict], **params) -> str: + """ + Send one chat completion request and return the reply text. + + :param messages: OpenAI chat messages (see + :func:`to_openai_messages`). + :param params: further request fields, e.g. ``max_tokens``, + ``temperature``, ``top_p``, ``top_k``, + ``chat_template_kwargs``. + :raises ContextSizeError: if the request does not fit in the + context of a server slot. + :raises RuntimeError: for any other error response. + """ + body = json.dumps({'messages': messages, **params}).encode() + request = urllib.request.Request( + f"{self.base_url}/v1/chat/completions", data=body, + headers={'Content-Type': 'application/json'}) + try: + with urllib.request.urlopen( + request, timeout=self.request_timeout) as response: + reply = json.load(response) + except urllib.error.HTTPError as e: + detail = e.read().decode(errors='replace') + if 'exceed_context_size' in detail: + raise ContextSizeError( + f"the prompt does not fit in the context of a " + f"llama-server slot ({self.slot_context or 'unknown'}" + f" tokens): {detail}") from e + raise RuntimeError( + f"llama-server returned HTTP {e.code}: {detail}") from e + return reply['choices'][0]['message'].get('content') or '' + + def _read_props(self) -> Tuple[Optional[int], int]: + """Context size of one slot and the slot count, from ``/props``.""" + try: + with urllib.request.urlopen( + f"{self.base_url}/props", timeout=5) as response: + props = json.load(response) + except (urllib.error.URLError, OSError, ValueError): + return None, 1 + try: + ctx = int(props['default_generation_settings']['n_ctx']) + except (KeyError, TypeError, ValueError): + ctx = None + try: + slots = max(1, int(props['total_slots'])) + except (KeyError, TypeError, ValueError): + slots = 1 + return ctx, slots + + def _healthy(self) -> bool: + try: + with urllib.request.urlopen( + f"{self.base_url}/health", timeout=2) as response: + return response.status == 200 + except (urllib.error.URLError, OSError): + return False + + def _forward_output(self) -> None: + process = self._process + if process is None or process.stdout is None: + return + for line in process.stdout: + line = line.rstrip() + self._log_tail.append(line) + self.logger.log(_log_level(line), f"[llama-server] {line}") diff --git a/clams/develop/templates/app/.dockerignore.template b/clams/develop/templates/app/.dockerignore.template index 4bf612e..e883442 100644 --- a/clams/develop/templates/app/.dockerignore.template +++ b/clams/develop/templates/app/.dockerignore.template @@ -8,4 +8,4 @@ bin/ build/ *~ **/*.pyc -**/__pyche__ +**/__pycache__ diff --git a/clams/restify/__init__.py b/clams/restify/__init__.py index 5f99045..c0e082c 100644 --- a/clams/restify/__init__.py +++ b/clams/restify/__init__.py @@ -7,6 +7,16 @@ from clams.app import ClamsApp from clams.envelop import EnvelopeError +#: Names of the optional :class:`.ClamsApp` class attributes that +#: :meth:`Restifier.serve_production` reads: the gunicorn worker count, and +#: the number of requests a worker serves before it is replaced. An app +#: declares them when it holds something that must not be forked or +#: restarted with the worker, e.g. a model loaded in a child process +#: (:class:`~clams.app.ClamsGGUFPromptableApp`). See +#: :meth:`Restifier.serve_production` for their precedence. +PRODUCTION_WORKERS_ATTR = 'RESTIFIER_PRODUCTION_WORKERS' +PRODUCTION_MAX_REQUESTS_ATTR = 'RESTIFIER_PRODUCTION_MAX_REQUESTS' + class Restifier(object): """ @@ -45,6 +55,20 @@ def serve_production(self, **options): """ Runs the CLAMS app as a flask webapp, using a production-ready web server (gunicorn, https://docs.gunicorn.org/en/stable/#). + Two gunicorn settings are derived rather than fixed, in this order of + precedence (first match wins): + + ``workers`` + the ``workers`` keyword of this call; the + ``CLAMS_GUNICORN_WORKERS`` environment variable; the app's + :data:`PRODUCTION_WORKERS_ATTR` class attribute; else the smaller + of ``(CPU cores x 2) + 1`` and ``total VRAM / est_gpu_mem_typ`` + of the app metadata. + ``max_requests`` (requests a worker serves before it is replaced) + the ``max_requests`` keyword of this call; the app's + :data:`PRODUCTION_MAX_REQUESTS_ATTR` class attribute; else 1, so + that a worker's GPU memory is freed after each request. + :param options: any additional options to pass to the web server. """ import gunicorn.app.base @@ -56,6 +80,10 @@ def number_of_workers(): if 'CLAMS_GUNICORN_WORKERS' in os.environ: return int(os.environ['CLAMS_GUNICORN_WORKERS']) + declared_workers = getattr(self.cla, PRODUCTION_WORKERS_ATTR, None) + if declared_workers: + return int(declared_workers) + cpu_workers = (multiprocessing.cpu_count() * 2) + 1 # Get GPU memory requirement from app metadata @@ -93,7 +121,10 @@ def number_of_workers(): pass return cpu_workers - + + default_max_requests = getattr( + self.cla, PRODUCTION_MAX_REQUESTS_ATTR, 1) + class ProductionApplication(gunicorn.app.base.BaseApplication): def __init__(self, app, host, port, **options): @@ -108,9 +139,7 @@ def __init__(self, app, host, port, **options): # errorlog, however, is redirected to stderr by default since 19.2, so no need to set # log level is warning by default 'loglevel': os.environ.get('CLAMS_LOGLEVEL', 'warning').lower(), - # default to 1 to free GPU memory after each request - # developers can override via serve_production(max_requests=N) for single-model apps - 'max_requests': 1, + 'max_requests': default_max_requests, } self.options.update(options) self.application = app @@ -126,7 +155,7 @@ def load(self): return self.application # Log max_requests setting - max_req = options.get('max_requests', 1) # default is 1, meaning workers are killed after each request + max_req = options.get('max_requests', default_max_requests) if max_req == 0: self.cla.logger.info("Worker recycling: disabled (workers persist)") else: diff --git a/container/generate_containers_yml.py b/container/generate_containers_yml.py index 607f900..cf24861 100644 --- a/container/generate_containers_yml.py +++ b/container/generate_containers_yml.py @@ -5,28 +5,44 @@ def scan_container_files(directory): container_files = [f.name for f in Path(directory).iterdir() if f.is_file() and (f.name.endswith('.containerfile') or f.name == 'Containerfile')] return container_files +IMAGE_PREFIX = "ghcr.io/clamsproject/clams-python" + + +def job_name_of(file): + return 'base' if file == 'Containerfile' else file.replace('.containerfile', '') + + +def image_job(image): + """Job name that builds a clams-python image reference, or None for other images.""" + name = image.split(':')[0] + if not name.startswith(IMAGE_PREFIX): + return None + suffix = name[len(IMAGE_PREFIX):] + return 'base' if suffix == '' else suffix.lstrip('-') + + def determine_dependencies(container_files): - """Determine dependencies based on the `FROM` line in the file content.""" + """ + Determine dependencies from every `FROM` line that names a clams-python image. + A multi-stage file can name several and then depends on all of them. `FROM` + lines of other images (e.g. `llamaserver` builds on llama.cpp's image) are ignored. + """ dependencies = {} for file in container_files: file_path = Path(__file__).parent / file + deps = [] with open(file_path, 'r') as f: for line in f: if line.startswith('FROM'): - base_image = line.split()[1].strip() - # common prefix: ghcr.io/clamsproject/clams-python - # common suffix: :#clams_version - # just get the middle - base_image = base_image[len("ghcr.io/clamsproject/clams-python"):-len(":#clams_version")] - if len(base_image) == 0: - dependencies[file] = [] - else: - dependencies[file] = base_image.split('-')[1:] # assuming suffix starts with a dash - break + job = image_job(line.split()[1].strip()) + if job is not None and job not in deps: + deps.append(job) + dependencies[file] = deps for k in sorted(dependencies.keys()): print(f"{k}: {dependencies[k]}") # Debugging output to check dependencies return dependencies + def generate_workflow(container_files, dependencies, output_file): """Generate the containers.yml workflow file.""" with open(output_file, 'w') as f: @@ -68,23 +84,23 @@ def generate_workflow(container_files, dependencies, output_file): f.write(" timeout: 1200000\n") f.write(" interval: 5000\n") - # Sort jobs: base first, then by dependency length and filename + # Sort jobs: base first, then by number of dependencies, dependencies and filename sorted_files = sorted(container_files, key=lambda x: (x != 'Containerfile', len(dependencies.get(x, [])), '-'.join(dependencies.get(x, [])), x)) for file in sorted_files: - job_name = 'base' if file == 'Containerfile' else file.replace('.containerfile', '') + job_name = job_name_of(file) f.write(f"\n call-build-{job_name}:\n") human_friendly_job_name = "the base" if job_name == 'base' else f"`{job_name}`" f.write(f" name: \"🤙 Call container workflow with {human_friendly_job_name}\"\n") - dependency_job = dependencies.get(file, None) - print(job_name, dependency_job) - if len(dependency_job) > 0: - f.write(f" needs: ['set-version', 'call-build-{'-'.join(dependency_job)}']\n") + dependency_jobs = dependencies.get(file, []) + print(job_name, dependency_jobs) + if len(dependency_jobs) > 0: + needs = ', '.join(f"'call-build-{d}'" for d in dependency_jobs) + f.write(f" needs: ['set-version', {needs}]\n") else: - if job_name == 'base': - f.write(f" needs: ['set-version', 'check-deployment']\n") - else: - f.write(f" needs: ['set-version', 'call-build-base']\n") + # the base image, or an image built on a non-clams image: + # both pip-install clams-python, so both wait for PyPI + f.write(f" needs: ['set-version', 'check-deployment']\n") f.write(" uses: ./.github/workflows/container.yml\n") f.write(" secrets: inherit\n") f.write(" with:\n") diff --git a/container/llamaserver.containerfile b/container/llamaserver.containerfile new file mode 100644 index 0000000..5ac3076 --- /dev/null +++ b/container/llamaserver.containerfile @@ -0,0 +1,41 @@ +ARG clams_version +# llama.cpp's own CUDA server image (Ubuntu 24.04, CUDA 12.8 runtime, +# ffmpeg, llama-server and its backend libraries in /app), pinned by tag +# and digest. CLAMS builds no llama.cpp: to upgrade, bump both. Upstream +# compiles machine code for Ampere/Ada (8.6, 8.9) and Blackwell, and PTX +# for older GPUs (Maxwell to Turing, A100, Hopper), which the driver +# compiles at the first start; keep CUDA_CACHE_PATH (under /cache) on a +# volume so that happens once. +FROM ghcr.io/ggml-org/llama.cpp:server-cuda-b12169@sha256:5bb779a449a18d8f4333e9814e887746342fe555c528f5eabc92b2093e32673a +ARG clams_version +LABEL org.opencontainers.image.description="clams-python-llamaserver image is shipped with clams-python and llama.cpp's llama-server (experimental GGUF backend), on llama.cpp's CUDA image" + +# Ubuntu blocks system-wide pip installs, so python packages live in a venv +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* +RUN python3 -m venv /opt/clams +ENV PATH=/opt/clams/bin:$PATH + +# upstream's binary finds its libraries in /app only when that is the +# working directory; apps also live in /app, but keep the path explicit +ENV LLAMA_SERVER=/app/llama-server +ENV LD_LIBRARY_PATH=/app + +# same cache layout as the clams-python base image, plus the CUDA compile +# cache, so one volume at /cache keeps models and compiled kernels +ENV XDG_CACHE_HOME=/cache +ENV HF_HOME=/cache/huggingface +ENV TORCH_HOME=/cache/torch +ENV CUDA_CACHE_PATH=/cache/cuda-compute +ENV CUDA_CACHE_MAXSIZE=4294967296 +RUN mkdir -p /cache && rm -rf /root/.cache && ln -s /cache /root/.cache + +# fail the build if a library is missing (libcuda.so.1 comes from the host +# driver at run time) +RUN ! ldd /app/llama-server /app/*.so | grep 'not found' | grep -v 'libcuda.so.1' + +RUN pip install --no-cache-dir "clams-python[llamaserver]==$clams_version" + +# upstream starts llama-server; CLAMS apps set their own command +ENTRYPOINT [] diff --git a/documentation/app-baseclasses.rst b/documentation/app-baseclasses.rst index 895d1a9..bd16a69 100644 --- a/documentation/app-baseclasses.rst +++ b/documentation/app-baseclasses.rst @@ -494,9 +494,10 @@ Choose :class:`~clams.app.ClamsHFPromptableApp` over plain - doesn't need bespoke pixel-value preprocessing or vision-token stitching at inference time. -If your app uses a remote API instead (OpenAI, Anthropic, etc.), or a -non-HF local backend, inherit from -:class:`~clams.app.ClamsPromptableApp` directly and implement +For a model in GGUF format run with llama.cpp, see +:ref:`gguf-promptable` (experimental). If your app uses a remote API +instead (OpenAI, Anthropic, etc.), or another local backend, inherit +from :class:`~clams.app.ClamsPromptableApp` directly and implement :meth:`~clams.app.ClamsPromptableApp.generate` yourself. .. _hf-promptable-declaring: @@ -600,7 +601,7 @@ Reproducibility: ``model`` refinement and view metadata The user-facing ``model`` parameter accepts raw HF model ids (``org/repo-name``). The SDK's -:py:meth:`~clams.app.ClamsHFPromptableApp._refine_params` expands the +:py:meth:`~clams.app.ClamsPromptableApp._refine_params` expands the raw value to ``org/repo-name@`` form (using the dict lookup) during parameter refinement. The standard ``sign_view`` flow then stamps: @@ -649,3 +650,141 @@ See each method's docstring for full details. Apps using the HF backend (with or without the promptable wrapper) must install the ``[hf]`` extra: ``pip install clams-python[hf]``. +.. _gguf-promptable: + +GGUF Promptable Apps (experimental) +----------------------------------- + +:class:`~clams.app.ClamsGGUFPromptableApp` runs a model in GGUF format +with `llama.cpp `_ instead of +``transformers``. The app owns a private ``llama-server`` process: it +downloads the pinned GGUF file (and the ``mmproj`` projector file of a +vision model) from the Hugging Face Hub, starts the server on a free +localhost port, sends prompts through the server's OpenAI-compatible +chat API, and stops the server when the app exits. The app never +connects to a server it did not start, so model selection stays with +the app and is recorded in the output MMIF as for an HF app. + +This backend is experimental. Its class attributes and the container +image may change between releases. + +When to use +^^^^^^^^^^^ + +Choose :class:`~clams.app.ClamsGGUFPromptableApp` when the app should +run without ``torch``: a quantized model on a small GPU, a GPU that +current ``torch`` wheels no longer support, or CPU-only inference. The +same model in GGUF form gives different output than its ``transformers`` +original, so an app that must reproduce an HF app's output stays with +:class:`~clams.app.ClamsHFPromptableApp`. + +Declaring a GGUF promptable app +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The declaration follows :ref:`hf-promptable-declaring`, with these +differences: + +1. ``analyzer_versions`` keys are ```` or + ``:`` (e.g. + ``ggml-org/gemma-4-E4B-it-GGUF:Q8_0``), and the values are commit + hashes of the GGUF repository. The quantization tag picks the model + file, and the repository's ``mmproj`` file is picked for vision + input. +2. ``MODEL_FILES`` names the files explicitly for a repository where the + tag does not pick exactly one file (several projectors, draft + models, shards). +3. ``metadata.py`` calls + :meth:`ClamsGGUFPromptableApp.inject_promptable_parameters + `. + +The ``model`` parameter, family and singleton handling, and the +``model`` refinement into ``appConfiguration`` work as for HF apps. +Audio input is not supported yet. + +GGUF apps have no ``parallelPrompts`` parameter. ``llama-server`` +decodes at most as many requests together as it has slots, a number +fixed when the server starts, and +:meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched` keeps every +slot busy. The slot count is a server setting (see below), and its +default, like the way slots share the context, belongs to llama.cpp +and changes between its releases; see the llama.cpp documentation. The +startup log reports the slot count and the context of a slot. + +We recommend one slot, set in the app's ``LLAMA_ARGS``. The server +decodes the prompts in its slots together, so with more slots an +output can change with the other prompts in flight. In our tests on +Maxwell and Pascal GPUs, more slots were also no faster. + +Class-attribute hooks +^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 22 60 18 + + * - Attribute + - Meaning + - Required + * - ``MODEL_FILES`` + - File names per model id: ``{'': {'model': '' or + [shards], 'mmproj': ''}}``. Models not listed are + resolved from the repository listing. + - no + * - ``LLAMA_ARGS`` + - Default values of llama.cpp's ``LLAMA_ARG_*`` environment + variables for what the app's models need, e.g. a larger batch + size for the image tokens of a vision model, or the slot count. + The environment of the app process takes precedence (see + below). + - no + * - ``RESTIFIER_PRODUCTION_WORKERS``, + ``RESTIFIER_PRODUCTION_MAX_REQUESTS`` + - Read only by :meth:`~clams.restify.Restifier.serve_production` + (no effect in the CLI): one gunicorn worker, never recycled, so + the server the app owns is neither duplicated nor restarted per + request. A singleton app starts its server before gunicorn + forks, so all workers share it; a family app starts one server + per worker. + - no + +Server settings +^^^^^^^^^^^^^^^ + +The SDK gives ``llama-server`` only the model files, its address and +the reasoning output format on the command line. Every other setting +is one of llama.cpp's own ``LLAMA_ARG_*`` environment variables, which +the server reads for every argument not on its command line. A value +comes from, in increasing precedence: + +1. ``llama-server``'s defaults, which fit the run to the hardware: GPU + offload, context size, batch sizes, threads and the split over + several GPUs. The server places what fits on the GPUs, keeps the + rest in system memory, and sizes the context to the free memory. +1. The app's ``LLAMA_ARGS``, for what its models need. +1. The environment of the app process, set by whoever runs the app. + +The variables and their defaults belong to llama.cpp and change +between its releases; see the llama.cpp documentation. The settings in +effect are logged with their source when the server starts, and the +context size and slot count when it is ready. + +One prompt (system and user text, all its images, and the generated +tokens) must fit in the context of a server slot. A prompt that does +not fails the request with an error naming the size, so a caller can +lower ``maxImagesPerPrompt`` or give the server a larger context. + +Installation and containers +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The python side needs the ``[llamaserver]`` extra: +``pip install clams-python[llamaserver]``. The ``llama-server`` +executable is not a python package: outside a container, put it on the +``PATH`` or name it in the ``LLAMA_SERVER`` environment variable. + +The ``clams-python-llamaserver`` container image provides it. The image +is built on llama.cpp's own CUDA server image, so it has no ``torch`` +and does not derive from the other ``clams-python`` images. Mount a +volume at ``/cache`` for the downloaded model and for the CUDA kernels +the driver compiles at the first start on GPUs the image has no machine +code for. + diff --git a/documentation/appmetadata.rst b/documentation/appmetadata.rst index d0ba20a..d94afb3 100644 --- a/documentation/appmetadata.rst +++ b/documentation/appmetadata.rst @@ -222,8 +222,65 @@ section). Syntax for Parameter Specification in App Metadata -------------------------------------------------- -Metadata Schema -=============== +The ``parameters`` field is a list of objects. Each object must include +``name``, ``description``, ``type``, and ``multivalued``. The ``type`` must be +one of ``integer``, ``number``, ``string``, ``boolean``, or ``map``. The +optional ``choices`` field lists accepted values, and ``default`` sets a +default value (making the parameter optional). Defaults should be JSON values +of the declared type; use a JSON array for a multivalued parameter. + +.. code-block:: json + + { + "parameters": [ + { + "name": "minFrameCount", + "description": "Minimum number of frames to process.", + "type": "integer", + "multivalued": false, + "default": 5, + "choices": [1, 5, 10] + }, + { + "name": "labels", + "description": "Labels to include in the output.", + "type": "string", + "multivalued": true, + "default": ["PERSON", "ORG"] + } + ] + } + +``boolean`` parameters always use ``"multivalued": false``; ``map`` parameters +always use ``"multivalued": true`` and represent each key-value pair as a +colon-separated string (for example, ``"default": ["key1:value1"]``). See +:ref:`runtime-params-detailed` for how parameter values are parsed and passed +to the app. + +Required Environment Variables +============================== + +Some apps cannot run without a value that comes from the machine they run on. +Examples are an access token for a gated model on the Hugging Face Hub, which must stay out of the output MMIF, and the address of a model server (such as Ollama or vLLM) that the app sends requests to. +Declare these with :meth:`~clams.appmetadata.AppMetadata.add_env_requirement`, and read the value from the environment in the app code. + +.. code-block:: python + + metadata.add_env_requirement( + name='HF_TOKEN', + description='token with access to pyannote/speaker-diarization-3.1') + metadata.add_env_requirement( + name='OLLAMA_HOST', + description='URL of the Ollama server that runs the model') + +Being listed in ``env_requirements`` means the variable is required. +Do not list optional settings, variables that have a usable default, or common system variables such as ``PYTHONPATH``. + +Do not take a secret as a runtime parameter: the SDK records runtime parameters in the ``metadata`` of every view the app signs, so the value ends up in the output MMIF. +See :ref:`clamsapp-envvars` for how users pass these variables to an app. + +Full Metadata JSON Schema +========================= The schema for app metadata is as follows. (You can also download the schema in `JSON Schema `_ format from `here `_.) @@ -234,5 +291,3 @@ The schema for app metadata is as follows. :lift_definitions: True :auto_target: True :auto_reference: True - - diff --git a/documentation/clamsapp.rst b/documentation/clamsapp.rst index 76c6c1f..d0eb06b 100644 --- a/documentation/clamsapp.rst +++ b/documentation/clamsapp.rst @@ -124,6 +124,53 @@ Unlike the data directory, the cache directory is not required to be mounted, bu This can include sensitive information such as browser cache, authentication tokens, etc, hence will pose a great security risk. It is recommended to create a separate directory to use as a cache directory for CLAMS containers. +.. _clamsapp-envvars: + +Passing environment variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some apps need environment variables to run, for example an access token for a gated model, or the address of a model server such as Ollama or vLLM. +These apps list the variables in the ``env_requirements`` field of their app metadata. +If one of them is unset or empty, the app stops at startup with an error that names the missing variables. +To provide them, write a file with one ``NAME=value`` per line, usually named ``.env``. + +.. code-block:: bash + + HF_TOKEN= + OLLAMA_HOST=http://host.docker.internal:11434 + +Then pass the file with ``--env-file`` when you start the container. + +.. code-block:: bash + + $ docker run --env-file .env -v /path/to/data/directory:/data -p :5000 + +To set a single variable, use ``-e NAME=value`` in place of ``--env-file``. + +.. note:: + + Inside a container, ``localhost`` is the container itself. + To reach a model server that runs on the host machine, use ``host.docker.internal`` as the host name. + On Linux, Docker also needs ``--add-host=host.docker.internal:host-gateway`` in the ``docker run`` command, and Podman uses ``host.containers.internal`` in its place. + +When you run the app without a container, load the file into your shell first. +``set -a`` is needed because ``source`` alone does not export the variables to the app. + +.. code-block:: bash + + $ set -a; source .env; set +a + $ python app.py + +.. note:: + + Docker and bash read this file differently, so use the plain ``NAME=value`` form to make one file work for both. + Do not add an ``export`` prefix, quotes around the value, or a comment after the value. + A comment on its own line is fine. + +.. warning:: + + Keep the ``.env`` file out of version control. + .. _clamsapp-invoking: Invoking the app server @@ -400,7 +447,7 @@ When running in production mode, the following environment variables can be used - Logging verbosity level (``debug``, ``info``, ``warning``, ``error``) - ``warning`` -By default, the number of workers is calculated as ``(CPU cores x 2) + 1``. For GPU-based apps, see `GPU Memory Management `_ for details on automatic worker scaling and VRAM management. +By default, the number of workers is calculated as ``(CPU cores x 2) + 1``, and each worker is replaced after one request. An app can fix both in code; see `GPU Memory Management `_ for the worker calculation, the recycling and the order of precedence between the environment, the app and the ``serve_production()`` call. ``metadata.py``: Getting app metadata ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/documentation/gpu-apps.rst b/documentation/gpu-apps.rst index 2f2025e..7c844f9 100644 --- a/documentation/gpu-apps.rst +++ b/documentation/gpu-apps.rst @@ -77,6 +77,39 @@ By default, workers are recycled after each request (``max_requests=1``) to full restifier.serve_production(max_requests=0) # Workers persist +Who sets the worker count and recycling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Three parties can set these two values, and the first match wins: + +.. list-table:: + :header-rows: 1 + :widths: 22 39 39 + + * - Party + - Workers + - Recycling (``max_requests``) + * - The deployment (``app.py``, or whoever starts the server) + - ``serve_production(workers=N)``, else the + ``CLAMS_GUNICORN_WORKERS`` environment variable + - ``serve_production(max_requests=N)`` + * - The app class + - ``RESTIFIER_PRODUCTION_WORKERS`` class attribute + - ``RESTIFIER_PRODUCTION_MAX_REQUESTS`` class attribute + * - The SDK default + - the calculation above + - ``1`` + +An app declares the class attributes when it holds something that must +not be forked or restarted with the worker. For example, +:class:`~clams.app.ClamsGGUFPromptableApp` loads its model in a +``llama-server`` child process, so it declares one worker and no +recycling: a fork would duplicate the model, and a worker replacement +would restart it on every request. Any :class:`~clams.app.ClamsApp` +subclass may declare them; only +:meth:`~clams.restify.Restifier.serve_production` reads them. The +attribute names are constants in :mod:`clams.restify`. + NVIDIA Memory Oversubscription ------------------------------ diff --git a/documentation/index.rst b/documentation/index.rst index 5882e11..3747eee 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -17,6 +17,7 @@ CLAMS Python SDK input-output runtime-params app-baseclasses + promptable-backends gpu-apps appmetadata appdirectory diff --git a/documentation/promptable-backends.rst b/documentation/promptable-backends.rst new file mode 100644 index 0000000..41d117f --- /dev/null +++ b/documentation/promptable-backends.rst @@ -0,0 +1,114 @@ +.. _promptable-backends: + +HF and GGUF Promptable Apps +=========================== + +The SDK has two backends for promptable apps: +:class:`~clams.app.ClamsHFPromptableApp` runs a Hugging Face +``transformers`` model in the app process, and +:class:`~clams.app.ClamsGGUFPromptableApp` (experimental) runs a GGUF +model in a ``llama-server`` process that the app starts and owns. Both +share the promptable runtime parameters, the ``model`` parameter +derived from ``analyzer_versions``, and the output format. This page +lists where they differ. For how to declare each, see +:ref:`hf-promptable` and :ref:`gguf-promptable`. + +The primary target is NVIDIA GPUs with CUDA: the container images ship +the latest release of each backend, built with CUDA support. + +.. list-table:: + :header-rows: 1 + :widths: 20 40 40 + + * - + - HF (``ClamsHFPromptableApp``) + - GGUF (``ClamsGGUFPromptableApp``) + * - Inference engine + - ``torch`` and ``transformers``, in the app process + - ``llama-server`` (llama.cpp), a child process on localhost + * - Python extra + - ``[hf]`` + - ``[llamaserver]``, plus a ``llama-server`` executable + * - Container image + - ``clams-python-hf5`` and its ``ffmpeg`` / ``opencv4`` variants + - ``clams-python-llamaserver``, built on llama.cpp's CUDA image + * - Model id in ``analyzer_versions`` + - ````, refined to ``@`` + - ``:``, refined to + ``:@`` + * - Model files + - the repository snapshot, loaded with ``from_pretrained()`` + - one GGUF file (or its shards) and, for vision input, one + ``mmproj`` file, picked by the quantization tag or + ``MODEL_FILES`` + * - Hardware + - NVIDIA GPUs that the latest ``torch`` supports: Turing and + newer, since ``torch`` 2.15 dropped Maxwell, Pascal and Volta. + Falls back to CPU. + - NVIDIA GPUs that llama.cpp's CUDA build supports, including + older ones: tested on Maxwell. Falls back to CPU. + * - Model placement + - one device (the first GPU, or CPU), unless ``MODEL_KWARGS`` + sets a ``device_map`` + - the server places what fits on the visible GPUs and keeps the + rest in system memory + * - Hardware settings + - class attributes (``DTYPE``, ``MODEL_KWARGS``) set by the app + developer + - ``LLAMA_ARG_*`` environment variables: defaults in the app's + ``LLAMA_ARGS``, overridden by the deployment. Note that these + variables are not recorded in the output MMIF; the settings in + effect appear only in the app log. + * - Prompts run together + - ``parallelPrompts`` runtime parameter, per request; the prompts + are padded into one batch + - the server's slot count, fixed when the server starts (one + recommended); no ``parallelPrompts`` parameter + * - Memory for one request + - grows with ``parallelPrompts`` and the prompt size; too much + runs the GPU out of memory + - fixed when the server starts; how the slots share the context + depends on the llama.cpp release + * - Prompt too large + - out-of-memory error + - context-size error (an error view naming the context size) + * - Production mode + - workers by CPU count and ``est_gpu_mem_typ``, each replaced + after one request + - one worker, never replaced + (``RESTIFIER_PRODUCTION_WORKERS``, + ``RESTIFIER_PRODUCTION_MAX_REQUESTS``) + * - ``hwFetch`` profiling + - GPU name, total, available and peak memory + - GPU name and total memory only + * - Audio input + - supported, if the model takes audio + - not supported + * - Status + - stable + - experimental + +Two differences matter most when moving an app from one backend to the +other. + +Who sets the batch size +----------------------- + +An HF app takes ``parallelPrompts`` with each request and stacks that +many prompts into one forward pass. A GGUF app sends as many prompts +at a time as ``llama-server`` has slots, and the server decodes them +together. The slot count is a server setting, fixed when the server +starts; see :ref:`gguf-promptable` for the recommended value. An app +moved to the GGUF backend must stop reading +``parameters['parallelPrompts']`` and stop passing ``parallel_prompts`` to +:meth:`~clams.app.ClamsGGUFPromptableApp.generate_batched`. + +Outputs differ between backends +------------------------------- + +A GGUF file is a converted, usually quantized, copy of the original +model, and llama.cpp computes with different kernels. The same model +gives different outputs on the two backends, even at temperature 0. +With more than one slot, GGUF outputs can also shift slightly with the +other prompts decoded at the same time. Compare outputs before you +replace an HF app with a GGUF one. diff --git a/documentation/target-versions.csv b/documentation/target-versions.csv index 7ed5ca5..d43c580 100644 --- a/documentation/target-versions.csv +++ b/documentation/target-versions.csv @@ -1,4 +1,5 @@ "``clams-python`` version","``mmif-python`` version","Target MMIF Specification" +`1.9.0 `__,`1.8.1 `__,`1.2.0 `__ `1.8.2 `__,`1.7.0 `__,`1.2.0 `__ `1.8.1 `__,`1.6.0 `__,`1.2.0 `__ `1.8.0 `__,`1.5.4 `__,`1.2.0 `__ diff --git a/pyproject.toml b/pyproject.toml index e5728db..4dfd294 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3 :: Only", ] dependencies = [ - "mmif-python==1.7.0", + "mmif-python==1.8.1", "Flask>=2", "Flask-RESTful>=0.3.9", "gunicorn>=20", @@ -36,12 +36,16 @@ homepage = "https://clams.ai" source = "https://github.com/clamsproject/clams-python" [project.optional-dependencies] -dev = ["mypy", "pytest", "pytest-cov", "pillow", "setuptools"] +dev = ["mypy", "pytest", "pytest-cov", "pillow", "numpy", "setuptools"] docs = ["sphinx>=7.0,<8.0", "furo", "m2r2", "sphinx-jsonschema"] -test = ["mypy", "pytest", "pytest-cov", "pillow"] +test = ["mypy", "pytest", "pytest-cov", "pillow", "numpy"] # Required for apps using the HuggingFace transformers backend # (clams.backends.hf). Heavy deps; opt-in only. hf = ["torch", "transformers", "pillow", "tqdm"] +# Required for apps using the llama.cpp llama-server backend +# (clams.backends.llamaserver). The llama-server binary itself is not a +# python package; it comes from the base image or the system. +llamaserver = ["huggingface_hub", "pillow"] [tool.setuptools.packages.find] where = ["."] @@ -65,6 +69,7 @@ module = ["mmif.*", "flask_restful.*", "gunicorn.*", "jsonschema.*"] ignore_missing_imports = true [[tool.mypy.overrides]] -# optional backends (the `hf` extra) and the `metadata.py` of a running app -module = ["torch.*", "transformers.*", "metadata"] +# optional backends (the `hf` and `llamaserver` extras) and the +# `metadata.py` of a running app +module = ["torch.*", "transformers.*", "huggingface_hub.*", "metadata"] ignore_missing_imports = true diff --git a/tests/test_clamsapp.py b/tests/test_clamsapp.py index 9af9c5f..c55bea8 100644 --- a/tests/test_clamsapp.py +++ b/tests/test_clamsapp.py @@ -86,6 +86,54 @@ def _annotate(self, mmif, **kwargs): return mmif +def requiring_env(*names): + """ + Patch :class:`ExampleClamsApp` so that its metadata declares ``names`` + as required environment variables. + """ + original = ExampleClamsApp._load_appmetadata + + def load(self): + metadata = original(self) + for name in names: + metadata.add_env_requirement(name, f'{name} for testing') + return metadata + return patch.object(ExampleClamsApp, '_load_appmetadata', load) + + +class TestEnvRequirements(unittest.TestCase): + + names = ('CLAMS_TEST_TOKEN_A', 'CLAMS_TEST_TOKEN_B') + secret = 'do-not-leak-1234' + + def setUp(self): + patcher = patch.dict(os.environ) + patcher.start() + self.addCleanup(patcher.stop) + for name in self.names: + os.environ.pop(name, None) + + def test_starts_when_all_set(self): + for name in self.names: + os.environ[name] = self.secret + with requiring_env(*self.names): + app = ExampleClamsApp() + self.assertNotIn(self.secret, app.appmetadata()) + + def test_fails_when_unset_or_empty(self): + os.environ['CLAMS_TEST_TOKEN_A'] = self.secret + os.environ['CLAMS_TEST_TOKEN_B'] = '' + with requiring_env(*self.names), self.assertRaises(RuntimeError) as e: + ExampleClamsApp() + message = str(e.exception) + self.assertIn('CLAMS_TEST_TOKEN_B', message) + self.assertNotIn('CLAMS_TEST_TOKEN_A', message) + self.assertNotIn(self.secret, message) + + def test_no_declaration_needs_no_env(self): + ExampleClamsApp() + + class TestClamsApp(unittest.TestCase): def setUp(self): @@ -314,6 +362,17 @@ def test_app_tags_propagated_to_view(self): ['TemporalSegmentation', 'BarsDetection'], ) + def test_add_env_requirement(self): + env = self.app.metadata.add_env_requirement('HF_TOKEN', 'hub token') + self.assertEqual('HF_TOKEN', env.name) + dumped = json.loads(self.app.appmetadata()) + self.assertEqual([{'name': 'HF_TOKEN', 'description': 'hub token'}], + dumped['env_requirements']) + with self.assertRaises(ValueError): + self.app.metadata.add_env_requirement('HF_TOKEN', 'duplicate') + with self.assertRaises(ValueError): + self.app.metadata.add_env_requirement('NOT A NAME', 'invalid') + def test_add_app_tag_dedupes_and_validates(self): self.app.metadata.add_app_tag('foo', 'bar', 'foo') self.assertEqual(self.app.metadata.app_tags, ['foo', 'bar']) diff --git a/tests/test_llamaserver.py b/tests/test_llamaserver.py new file mode 100644 index 0000000..c1359f6 --- /dev/null +++ b/tests/test_llamaserver.py @@ -0,0 +1,676 @@ +""" +Tests for the llama-server backend (``clams.backends.llamaserver``) and +``ClamsGGUFPromptableApp``. A fake ``llama-server`` (a small python +HTTP server with the same command line and endpoints) stands in for +llama.cpp, so no binary, GPU or network is needed. +""" +import contextlib +import json +import os +import shutil +import stat +import sys +import tempfile +import textwrap +import types +import unittest +import urllib.request +from unittest import mock + +import numpy as np +from PIL import Image + +from clams.app import ClamsGGUFPromptableApp +from clams.backends import llamaserver + +from tests.test_promptable import make_metadata + +FAKE_SERVER = textwrap.dedent('''\ + #!{python} + """Fake llama-server: /health, and a chat endpoint that echoes.""" + import argparse + import json + import os + import sys + import threading + import time + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + # chat requests received, for /requests + COUNTS = {{'requests': 0}} + LOCK = threading.Lock() + + if os.environ.get('FAKE_LLAMA_EXIT'): + print('error: failed to load model', flush=True) + sys.exit(int(os.environ['FAKE_LLAMA_EXIT'])) + parser = argparse.ArgumentParser() + parser.add_argument('--host') + parser.add_argument('--port', type=int) + args, _ = parser.parse_known_args() + argv = sys.argv[1:] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self, payload, status=200): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path == '/health': + self._send({{'status': 'ok'}}) + elif self.path == '/props': + # as llama-server: the command line, else the + # LLAMA_ARG_* environment, else the built-in default + # (the "model's trained context", one slot) + def setting(flag, env, default): + if flag in argv: + return int(argv[argv.index(flag) + 1]) + return int(os.environ.get(env, default)) + total = setting('--ctx-size', 'LLAMA_ARG_CTX_SIZE', 32768) + slots = setting('--parallel', 'LLAMA_ARG_N_PARALLEL', 1) + self._send({{'default_generation_settings': {{ + 'n_ctx': total // slots}}, 'total_slots': slots}}) + elif self.path == '/requests': + self._send(COUNTS) + else: + self.send_error(404) + + def do_POST(self): + started = time.time() + with LOCK: + COUNTS['requests'] += 1 + length = int(self.headers['Content-Length']) + request = json.loads(self.rfile.read(length)) + messages = request.pop('messages') + images = sum( + 1 for m in messages if isinstance(m['content'], list) + for p in m['content'] if p['type'] == 'image_url') + last_user = [m for m in messages if m['role'] == 'user'][-1] + content = last_user['content'] + if isinstance(content, list): + content = ' '.join( + p['text'] for p in content if p['type'] == 'text') + if content == 'TOO_LONG': + self._send({{'error': {{ + 'code': 400, 'type': 'exceed_context_size_error', + 'message': 'request exceeds the available context'}}}}, + status=400) + return + # optional decode time, longer for prompts with more images + time.sleep(images * float(os.environ.get( + 'FAKE_LLAMA_SECONDS_PER_IMAGE', 0))) + echo ={{'images': images, 'last_user': content, + 'started': started, 'ended': time.time(), + 'roles': [m['role'] for m in messages], + 'assistant': [m['content'] for m in messages + if m['role'] == 'assistant'], + 'params': request, 'argv': argv, + 'env': sorted(k for k in os.environ + if k.startswith('LLAMA_ARG_'))}} + self._send({{'choices': [ + {{'message': {{'role': 'assistant', + 'content': json.dumps(echo)}}}}]}}) + + print('server is listening', flush=True) + ThreadingHTTPServer((args.host, args.port), Handler).serve_forever() +''') + + +@contextlib.contextmanager +def _environ(values): + """Set environment variables for the block, then restore them.""" + saved = {k: os.environ.get(k) for k in values} + os.environ.update(values) + try: + yield + finally: + for k, v in saved.items(): + if v is None: + del os.environ[k] + else: + os.environ[k] = v + + +class FakeServerMixin: + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp() + cls.fake_binary = os.path.join(cls.tmp, 'llama-server') + with open(cls.fake_binary, 'w') as f: + f.write(FAKE_SERVER.format(python=sys.executable)) + os.chmod(cls.fake_binary, os.stat(cls.fake_binary).st_mode + | stat.S_IXUSR) + cls.model_file = os.path.join(cls.tmp, 'model-Q8_0.gguf') + cls.mmproj_file = os.path.join(cls.tmp, 'mmproj-model-f16.gguf') + for path in (cls.model_file, cls.mmproj_file): + with open(path, 'wb') as f: + f.write(b'GGUF') + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + +class TestPickGGUFFiles(unittest.TestCase): + + FILES = ['README.md', 'model-Q4_K_M.gguf', 'model-Q8_0.gguf', + 'mmproj-model-f16.gguf'] + + def test_variant_picks_model_and_mmproj(self): + self.assertEqual( + llamaserver._pick_gguf_files(self.FILES, 'q8_0'), + (['model-Q8_0.gguf'], 'mmproj-model-f16.gguf')) + + def test_no_projector_gives_none(self): + self.assertEqual( + llamaserver._pick_gguf_files(['m-Q8_0.gguf'], 'Q8_0'), + (['m-Q8_0.gguf'], None)) + + def test_shards_are_returned_in_order(self): + files = ['Q4_K_M/m-Q4_K_M-00002-of-00002.gguf', + 'Q4_K_M/m-Q4_K_M-00001-of-00002.gguf'] + models, _ = llamaserver._pick_gguf_files(files, 'Q4_K_M') + self.assertEqual(models, sorted(files)) + + def test_missing_shard_is_ambiguous(self): + files = ['m-Q4_K_M-00001-of-00003.gguf', + 'm-Q4_K_M-00002-of-00003.gguf'] + with self.assertRaises(ValueError): + llamaserver._pick_gguf_files(files, 'Q4_K_M') + + def test_unmatched_variant_raises(self): + with self.assertRaisesRegex(ValueError, 'no GGUF model file'): + llamaserver._pick_gguf_files(self.FILES, 'Q6_K') + + def test_ambiguous_variant_raises(self): + with self.assertRaisesRegex(ValueError, 'several GGUF files'): + llamaserver._pick_gguf_files(self.FILES, '') + + def test_several_projectors_raise(self): + files = ['m-Q8_0.gguf', 'mmproj-f16.gguf', 'mmproj-f32.gguf'] + with self.assertRaisesRegex(ValueError, 'several mmproj'): + llamaserver._pick_gguf_files(files, 'Q8_0') + + def test_draft_models_are_skipped_and_projector_matches_variant(self): + # file listing of ggml-org/gemma-4-26B-A4B-it-GGUF + files = ['dflash-gemma-4-26B-A4B-it-BF16.gguf', + 'dflash-gemma-4-26B-A4B-it-Q8_0.gguf', + 'gemma-4-26B-A4B-it-BF16.gguf', + 'gemma-4-26B-A4B-it-Q4_0.gguf', + 'gemma-4-26B-A4B-it-Q8_0.gguf', + 'mmproj-gemma-4-26B-A4B-it-BF16.gguf', + 'mmproj-gemma-4-26B-A4B-it-Q8_0.gguf', + 'mtp-gemma-4-26B-A4B-it-BF16.gguf', + 'mtp-gemma-4-26B-A4B-it-Q4_0.gguf', + 'mtp-gemma-4-26B-A4B-it-Q8_0.gguf'] + self.assertEqual( + llamaserver._pick_gguf_files(files, 'Q8_0'), + (['gemma-4-26B-A4B-it-Q8_0.gguf'], + 'mmproj-gemma-4-26B-A4B-it-Q8_0.gguf')) + # no Q4_0 projector: several remain, so the choice is ambiguous + with self.assertRaisesRegex(ValueError, 'several mmproj'): + llamaserver._pick_gguf_files(files, 'Q4_0') + + +class TestMessageConversion(unittest.TestCase): + + def test_pil_and_numpy_images_become_data_urls(self): + pil = Image.new('RGB', (4, 4)) + arr = np.zeros((4, 4, 3), dtype=np.uint8) + for image in (pil, arr): + url = llamaserver._image_to_data_url(image) + self.assertTrue(url.startswith('data:image/png;base64,')) + + def test_urls_pass_through(self): + url = 'https://example.com/frame.png' + self.assertEqual(llamaserver._image_to_data_url(url), url) + + def test_image_parts_become_image_url_parts(self): + conversation = [ + {'role': 'system', 'content': 'be brief'}, + {'role': 'user', 'content': [ + {'type': 'image', 'image': Image.new('RGB', (2, 2))}, + {'type': 'text', 'text': 'describe'}]}, + ] + messages = llamaserver.to_openai_messages(conversation) + self.assertEqual(messages[0], conversation[0]) + parts = messages[1]['content'] + self.assertEqual(parts[0]['type'], 'image_url') + self.assertEqual(parts[1], {'type': 'text', 'text': 'describe'}) + + def test_audio_parts_raise(self): + conversation = [{'role': 'user', 'content': [ + {'type': 'audio', 'audio': b''}]}] + with self.assertRaises(ValueError): + llamaserver.to_openai_messages(conversation) + + +class TestLlamaServer(FakeServerMixin, unittest.TestCase): + + def make_server(self, **kwargs): + kwargs.setdefault('binary', self.fake_binary) + kwargs.setdefault('startup_timeout', 30) + return llamaserver.LlamaServer(self.model_file, **kwargs) + + def test_command_line(self): + server = self.make_server(mmproj_path=self.mmproj_file, port=1234) + cmd = server._command() + self.assertEqual(cmd[0], self.fake_binary) + for flag, value in (('--model', self.model_file), + ('--mmproj', self.mmproj_file), + ('--port', '1234'), + ('--reasoning-format', 'none')): + self.assertEqual(cmd[cmd.index(flag) + 1], value) + # other settings go through the environment + for flag in ('--ctx-size', '--parallel', '--n-gpu-layers', + '--ubatch-size'): + self.assertNotIn(flag, cmd) + + def test_start_chat_stop(self): + with self.make_server() as server: + self.assertTrue(server.is_running()) + self.assertEqual(server.host, '127.0.0.1') + reply = json.loads(server.chat( + [{'role': 'user', 'content': 'hello'}], max_tokens=5)) + self.assertEqual(reply['last_user'], 'hello') + self.assertEqual(reply['params'], {'max_tokens': 5}) + # the server's own defaults are read back + self.assertEqual(server.slot_context, 32768) + self.assertEqual(server.slots, 1) + self.assertFalse(server.is_running()) + + def test_environment_overrides_reach_the_server(self): + # llama.cpp's own variables pass through; the two naming the + # address do not, since the command line sets it + env = {'LLAMA_ARG_CTX_SIZE': '8192', 'LLAMA_ARG_N_PARALLEL': '2', + 'LLAMA_ARG_HOST': '0.0.0.0', 'LLAMA_ARG_PORT': '8080'} + with _environ(env): + with self.make_server() as server: + self.assertEqual(server.slot_context, 4096) + self.assertEqual(server.slots, 2) + reply = json.loads(server.chat( + [{'role': 'user', 'content': 'hello'}])) + self.assertEqual(reply['env'], ['LLAMA_ARG_CTX_SIZE', + 'LLAMA_ARG_N_PARALLEL']) + + def test_llama_args_are_defaults_under_the_environment(self): + llama_args = {'LLAMA_ARG_CTX_SIZE': 8192, 'LLAMA_ARG_UBATCH': 2048} + with self.make_server(llama_args=llama_args) as server: + self.assertEqual(server.slot_context, 8192) + reply = json.loads(server.chat( + [{'role': 'user', 'content': 'hello'}])) + self.assertEqual(reply['env'], ['LLAMA_ARG_CTX_SIZE', + 'LLAMA_ARG_UBATCH']) + # the environment of the app process wins over the defaults + with _environ({'LLAMA_ARG_CTX_SIZE': '4096'}): + server = self.make_server(llama_args=llama_args) + env, settings = server._environment() + self.assertEqual(env['LLAMA_ARG_CTX_SIZE'], '4096') + self.assertEqual(env['LLAMA_ARG_UBATCH'], '2048') + self.assertIn('LLAMA_ARG_CTX_SIZE=4096 (environment)', settings) + self.assertIn('LLAMA_ARG_UBATCH=2048 (app)', settings) + with server: + self.assertEqual(server.slot_context, 4096) + + def test_llama_args_take_only_llama_arg_variables(self): + for key in ('UBATCH', 'LLAMA_ARG_PORT', 'HF_TOKEN'): + with self.assertRaisesRegex(ValueError, 'LLAMA_ARG_'): + self.make_server(llama_args={key: '1'}) + + def test_startup_failure_reports_server_output(self): + os.environ['FAKE_LLAMA_EXIT'] = '3' + try: + with self.assertRaisesRegex(RuntimeError, 'failed to load'): + self.make_server().start() + finally: + del os.environ['FAKE_LLAMA_EXIT'] + + def test_only_owner_process_stops_server(self): + server = self.make_server().start() + try: + owner = server._owner_pid + server._owner_pid = owner + 1 # as seen from a forked child + self.assertFalse(server.owned_here()) + server.stop() + self.assertTrue(server.is_running()) + server._owner_pid = owner + finally: + server.stop() + self.assertFalse(server.is_running()) + + def test_forked_child_sees_parents_server_running(self): + # a gunicorn worker cannot wait on the master's child process; + # it must still see the server as running + import warnings + server = self.make_server().start() + try: + with warnings.catch_warnings(): + # the child only sends one health request, then exits + warnings.filterwarnings('ignore', message='.*use of fork', + category=DeprecationWarning) + pid = os.fork() + if pid == 0: # child + os._exit(0 if server.is_running() else 1) + _, status = os.waitpid(pid, 0) + self.assertEqual(os.waitstatus_to_exitcode(status), 0) + finally: + server.stop() + + def test_log_levels_of_server_output(self): + import logging + level = llamaserver._log_level + self.assertEqual(level('E load_backend: failed to load x.so'), + logging.WARNING) + self.assertEqual(level('common_fit_params: warning: low memory'), + logging.WARNING) + # the server's own log lines carry a timestamp before the level + self.assertEqual(level('0.12.315.223 W srv load_model: cap ' + 'image_max_tokens (original=1120) to 512'), + logging.WARNING) + self.assertEqual(level('slot print_timing: id 0 | prompt eval'), + logging.INFO) + self.assertEqual(level('load_tensors: CPU_Mapped model buffer size'), + logging.DEBUG) + + def test_missing_binary_raises(self): + server = self.make_server(binary='no-such-llama-server-binary') + with self.assertRaises(FileNotFoundError): + server._command() + + def test_sigterm_stops_server(self): + script = textwrap.dedent(f'''\ + import sys, time + sys.path.insert(0, {os.getcwd()!r}) + from clams.backends.llamaserver import LlamaServer + server = LlamaServer({self.model_file!r}, + binary={self.fake_binary!r}, + startup_timeout=30).start() + print(server._process.pid, flush=True) + time.sleep(60) + ''') + import signal + import subprocess + import time + app = subprocess.Popen([sys.executable, '-c', script], + stdout=subprocess.PIPE, text=True) + server_pid = int(app.stdout.readline()) + app.send_signal(signal.SIGTERM) + app.wait(timeout=30) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + try: + os.kill(server_pid, 0) + except ProcessLookupError: + break + time.sleep(0.2) + else: + os.kill(server_pid, signal.SIGKILL) + self.fail('llama-server outlived its SIGTERMed parent') + + +class TestGGUFPromptableApp(FakeServerMixin, unittest.TestCase): + + SINGLETON_AV = {'org/fake-GGUF:Q8_0': 'abc1234'} + MULTI_AV = {'org/fake-GGUF:Q8_0': 'abc1234', + 'org/other-GGUF:Q4_K_M': 'def5678'} + + def setUp(self): + self._env = os.environ.get(llamaserver.LLAMA_SERVER_ENV) + os.environ[llamaserver.LLAMA_SERVER_ENV] = self.fake_binary + self._download = llamaserver.download_gguf + self.downloads = [] + + def fake_download(repo_id, revision, model_files, mmproj_file=None): + self.downloads.append((repo_id, revision, list(model_files), + mmproj_file)) + return ([self.model_file], + self.mmproj_file if mmproj_file else None) + + llamaserver.download_gguf = fake_download + self.apps = [] + + def tearDown(self): + for app in self.apps: + if app.server is not None: + app.server.stop() + llamaserver.download_gguf = self._download + if self._env is None: + del os.environ[llamaserver.LLAMA_SERVER_ENV] + else: + os.environ[llamaserver.LLAMA_SERVER_ENV] = self._env + + def make_app(self, analyzer_versions=None, **attrs): + analyzer_versions = analyzer_versions or self.SINGLETON_AV + + def load_metadata(_self): + metadata = make_metadata( + call_helper=False, + analyzer_versions=dict(analyzer_versions)) + ClamsGGUFPromptableApp.inject_promptable_parameters(metadata) + return metadata + + class_attrs = { + '_load_appmetadata': load_metadata, + '_appmetadata': lambda _self: None, + '_annotate': lambda _self, mmif, **kw: mmif, + 'MODEL_FILES': { + key: {'model': 'model-Q8_0.gguf', + 'mmproj': 'mmproj-model-f16.gguf'} + for key in analyzer_versions}, + } + class_attrs.update(attrs) + cls = type('TestGGUFApp', (ClamsGGUFPromptableApp,), class_attrs) + app = cls() + self.apps.append(app) + return app + + def test_singleton_starts_server_at_init(self): + app = self.make_app() + self.assertTrue(app.server.is_running()) + self.assertEqual(app.loaded_model, ('org/fake-GGUF:Q8_0', 'abc1234')) + self.assertEqual(self.downloads, [ + ('org/fake-GGUF', 'abc1234', ['model-Q8_0.gguf'], + 'mmproj-model-f16.gguf')]) + + def test_multimember_family_starts_on_first_use_and_switches(self): + app = self.make_app(self.MULTI_AV) + self.assertIsNone(app.server) + first = app.load_model('org/fake-GGUF:Q8_0') + self.assertIs(app.load_model('org/fake-GGUF:Q8_0@abc1234'), first) + second = app.load_model('org/other-GGUF:Q4_K_M') + self.assertFalse(first.is_running()) + self.assertTrue(second.is_running()) + self.assertEqual(self.downloads[-1][:2], + ('org/other-GGUF', 'def5678')) + + def test_refine_params_pins_revision(self): + app = self.make_app() + refined = app._refine_params(prompt=['hi']) + self.assertEqual(refined['model'], 'org/fake-GGUF:Q8_0@abc1234') + + def test_missing_model_parameter_raises(self): + def load_metadata(_self): + return make_metadata(call_helper=True, + analyzer_versions=dict(self.SINGLETON_AV)) + + cls = type('TestGGUFAppBad', (ClamsGGUFPromptableApp,), { + '_load_appmetadata': load_metadata, + '_appmetadata': lambda _self: None, + '_annotate': lambda _self, mmif, **kw: mmif, + }) + with self.assertRaisesRegex(ValueError, 'inject_promptable'): + cls() + + def test_generate_keeps_order_and_maps_params(self): + app = self.make_app(LLAMA_ARGS={'LLAMA_ARG_UBATCH': '2048'}) + images = [[Image.new('RGB', (4, 4))], + [Image.new('RGB', (4, 4)), Image.new('RGB', (4, 4))], + [Image.new('RGB', (4, 4))]] + outputs = [json.loads(o) for o in app.generate( + ['describe'], system_prompt='be brief', images=images, + max_new_tokens=64, temperature=0.0, top_p=0.9, top_k=20, + use_reasoning=True)] + self.assertEqual([o['images'] for o in outputs], [1, 2, 1]) + self.assertEqual(outputs[0]['roles'], ['system', 'user']) + self.assertEqual(outputs[0]['params'], { + 'max_tokens': 64, 'temperature': 0.0, 'top_p': 0.9, + 'top_k': 20, 'cache_prompt': False, + 'chat_template_kwargs': {'enable_thinking': True}}) + self.assertEqual(outputs[0]['env'], ['LLAMA_ARG_UBATCH']) + + def test_reasoning_is_off_unless_requested(self): + app = self.make_app() + output = json.loads(app.generate(['hi'])[0]) + self.assertEqual(output['params']['chat_template_kwargs'], + {'enable_thinking': False}) + + def test_template_kwargs_override_is_sent(self): + def template_kwargs(_self, **params): + return {'reasoning_effort': 'low'} + + app = self.make_app(build_template_kwargs=template_kwargs) + output = json.loads(app.generate(['hi'], use_reasoning=True)[0]) + self.assertEqual(output['params']['chat_template_kwargs'], + {'reasoning_effort': 'low'}) + + def test_production_settings_keep_the_server(self): + app = self.make_app() + self.assertEqual(app.RESTIFIER_PRODUCTION_WORKERS, 1) + self.assertEqual(app.RESTIFIER_PRODUCTION_MAX_REQUESTS, 0) + + def test_no_parallel_prompts_parameter(self): + app = self.make_app() + names = {p.name for p in app.metadata.parameters} + self.assertNotIn('parallelPrompts', names) + self.assertIn('maxImagesPerPrompt', names) + with self.assertRaisesRegex(ValueError, 'parallel_prompts'): + app.generate_batched(['hi'], images=[[]], parallel_prompts=2) + + def test_free_slots_take_the_next_prompt(self): + # 2 slots; the first prompt takes three times as long as the + # others, so the third must start before the first ends + with _environ({'FAKE_LLAMA_SECONDS_PER_IMAGE': '0.3'}): + app = self.make_app(LLAMA_ARGS={'LLAMA_ARG_N_PARALLEL': '2'}) + self.assertEqual(app.server.slots, 2) + progress = [] + app.report_progress = lambda done, total: progress.append(done) + images = [[Image.new('RGB', (4, 4))] * k for k in (3, 1, 1)] + outputs = [json.loads(o) for o in app.generate_batched( + ['describe'], images=images)] + self.assertEqual([o['images'] for o in outputs], [3, 1, 1]) + self.assertLess(outputs[2]['started'], outputs[0]['ended']) + # progress before the first and after each finished prompt + self.assertEqual(progress, [0, 1, 2, 3]) + + def test_context_error_leaves_queued_prompts_unsent(self): + app = self.make_app() # one slot + images = [[Image.new('RGB', (4, 4))] for _ in range(5)] + with self.assertRaisesRegex(ValueError, 'does not fit'): + app.generate_batched(['TOO_LONG'], images=images) + with urllib.request.urlopen( + f"{app.server.base_url}/requests") as response: + sent = json.load(response)['requests'] + # the failing prompt, and at most the one the free worker had + # already taken before the others were cancelled + self.assertLessEqual(sent, 2) + + def test_text_only_prompt_runs_once(self): + app = self.make_app() + outputs = app.generate(['hello']) + self.assertEqual(len(outputs), 1) + self.assertEqual(json.loads(outputs[0])['last_user'], 'hello') + + def test_user_only_mode_fills_earlier_replies(self): + app = self.make_app() + output = json.loads(app.generate( + ['first', 'second'], prompt_mode='user-only')[0]) + self.assertEqual(output['last_user'], 'second') + self.assertEqual(output['roles'], ['user', 'assistant', 'user']) + # the first reply was filled into the assistant placeholder + self.assertEqual(json.loads(output['assistant'][0])['last_user'], + 'first') + + def test_turn_taking_mode_sends_one_request(self): + app = self.make_app() + output = json.loads(app.generate( + ['q1', 'a1', 'q2'], prompt_mode='turn-taking')[0]) + self.assertEqual(output['roles'], ['user', 'assistant', 'user']) + self.assertEqual(output['assistant'], ['a1']) + + def test_audio_input_raises(self): + app = self.make_app() + with self.assertRaises(ValueError): + app.generate(['hi'], audios=[[b'']]) + + def test_generate_without_model_raises(self): + app = self.make_app(self.MULTI_AV) + with self.assertRaises(RuntimeError): + app.generate(['hi']) + + def test_failed_prompt_yields_empty_string(self): + app = self.make_app() + app.server.stop() + self.assertEqual(app.generate(['hi']), ['']) + + def test_prompt_over_context_fails_the_request(self): + app = self.make_app() + with self.assertRaisesRegex(ValueError, + r'\(32768 tokens\).*maxImagesPerPrompt'): + app.generate(['TOO_LONG']) + + def test_server_of_another_process_is_not_replaced(self): + app = self.make_app(self.MULTI_AV) + server = app.load_model('org/fake-GGUF:Q8_0') + owner = server._owner_pid + server._owner_pid = owner + 1 # as seen from a gunicorn worker + try: + # the same model: used as it is + self.assertIs(app.load_model('org/fake-GGUF:Q8_0'), server) + # another model: cannot stop the other process's server + with self.assertRaisesRegex(RuntimeError, 'another process'): + app.load_model('org/other-GGUF:Q4_K_M') + self.assertTrue(server.is_running()) + finally: + server._owner_pid = owner + + +class TestOfflineResolve(unittest.TestCase): + + def test_cached_snapshot_is_listed_when_hub_is_unreachable(self): + snapshot = tempfile.mkdtemp() + for name in ('m-Q8_0-00001-of-00002.gguf', + 'm-Q8_0-00002-of-00002.gguf', 'mmproj-m-f16.gguf'): + open(os.path.join(snapshot, name), 'wb').close() + + def offline(*args, **kwargs): + raise OSError('offline') + + # a stand-in for huggingface_hub (not a test dependency): the Hub + # is unreachable, and the revision is in the local cache + hub = types.ModuleType('huggingface_hub') + hub.HfApi = type('HfApi', (), {'list_repo_files': offline}) + hub.snapshot_download = ( + lambda *a, **kw: snapshot if kw.get('local_files_only') else + offline()) + try: + with mock.patch.dict(sys.modules, {'huggingface_hub': hub}): + self.assertEqual( + llamaserver.resolve_gguf_files( + 'org/m-GGUF', 'abc', 'Q8_0'), + (['m-Q8_0-00001-of-00002.gguf', + 'm-Q8_0-00002-of-00002.gguf'], + 'mmproj-m-f16.gguf')) + finally: + shutil.rmtree(snapshot, ignore_errors=True) + shutil.rmtree(snapshot, ignore_errors=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_promptable.py b/tests/test_promptable.py index 2cd2ab6..08d6e93 100644 --- a/tests/test_promptable.py +++ b/tests/test_promptable.py @@ -754,6 +754,50 @@ def test_single_element_prompt_still_list(self): self.assertEqual(refined['prompt'], ['only']) +class TestPinnedModelHelpers(unittest.TestCase): + """ + The opt-in ``model`` parameter helpers on plain + :class:`ClamsPromptableApp`. + """ + + AV = {'org/model-a': 'aaaaaaa', 'org/model-b': 'bbbbbbb'} + + def test_plain_app_gets_no_model_parameter(self): + app = make_test_app( + make_metadata(call_helper=True, analyzer_versions=self.AV)) + self.assertNotIn( + 'model', {p.name for p in app.metadata.parameters}) + + def test_opt_in_model_parameter_pins_revision(self): + m = make_metadata(call_helper=True, analyzer_versions=self.AV) + ClamsPromptableApp._add_model_parameter( + m, helper_name='test', id_kind='model id', description='m') + app = make_test_app(m) + refined = app._refine_params(prompt=['hi'], model=['org/model-b']) + self.assertEqual(refined['model'], 'org/model-b@bbbbbbb') + self.assertEqual(app._split_model_ref('org/model-a'), + ('org/model-a', 'aaaaaaa')) + self.assertEqual(app._split_model_ref('org/model-a@ccccccc'), + ('org/model-a', 'ccccccc')) + + def test_model_value_outside_analyzer_versions_is_untouched(self): + # e.g. a remote-API app with its own ``model`` parameter + m = make_metadata(call_helper=True, + pre_declare={'name': 'model', 'type': 'string', + 'default': 'remote-model', + 'description': 'remote model'}, + analyzer_versions={'local/ocr': 'ddddddd'}) + app = make_test_app(m) + refined = app._refine_params(prompt=['hi']) + self.assertEqual(refined['model'], 'remote-model') + + def test_empty_analyzer_versions_raises(self): + with self.assertRaisesRegex(ValueError, 'analyzer_versions'): + ClamsPromptableApp._add_model_parameter( + make_metadata(call_helper=True), helper_name='test', + id_kind='model id', description='m') + + # --------------------------------------------------------------------------- # ClamsHFPromptableApp class-attribute validation # ---------------------------------------------------------------------------