adding env_requirements app metadata key

clams/app/__init__.py

@@ -141,6 +141,7 @@ class ClamsApp(ABC):
 
     def __init__(self):
         self.metadata: AppMetadata = self._load_appmetadata()
+        self._check_env_requirements()
         super().__init__()
 
         # data type spec to be used in type caster
@@ -191,6 +192,30 @@ def _load_appmetadata(self) -> AppMetadata:
             metadata = self._appmetadata()
         return metadata
 
+    def _check_env_requirements(self) -> None:
+        """
+        A private method to stop the app at startup when an environment
+        variable declared in ``env_requirements`` of the app metadata is unset
+        or empty. Only presence is checked, and the error message names the
+        missing variables. Values are never stored, logged, or included in the
+        message.
+
+        :raises RuntimeError: if any required environment variable is unset
+            or empty
+        """
+        # `os.environ.get` is used only for its truthiness, so no value is
+        # kept in a local variable (some traceback tools print locals)
+        missing = [env for env in self.metadata.env_requirements
+                   if not os.environ.get(env.name)]
+        if missing:
+            listing = '\n'.join(f'  {env.name}: {env.description}'
+                                 for env in missing)
+            raise RuntimeError(
+                f'{self.metadata.name} cannot start because these required '
+                f'environment variables are unset or empty:\n{listing}\n'
+                'See https://clams.ai/clams-python/clamsapp.html'
+                '#clamsapp-envvars for how to pass them.')
+
     @abstractmethod
     def _appmetadata(self) -> AppMetadata:
         """

clams/appmetadata/__init__.py

@@ -268,6 +268,29 @@ def __init__(self, **kwargs):
     }
 
 
+class EnvRequirement(_BaseModel):
+    """
+    Defines a data model that describes an environment variable that a CLAMS
+    app cannot run without, such as an access token for a gated model. Only
+    the name of the variable is part of the app metadata, never its value.
+    """
+    name: str = pydantic.Field(
+        ...,
+        pattern=r'^[A-Za-z_][A-Za-z0-9_]*$',
+        description="Name of the environment variable."
+    )
+    description: str = pydantic.Field(
+        ...,
+        description="Description of what the variable is for and how to "
+                    "obtain its value."
+    )
+
+    model_config = {
+        'title': 'CLAMS App Environment Requirement',
+        'extra': 'forbid',
+    }
+
+
 class AppMetadata(pydantic.BaseModel):
     """
     Data model that describes a CLAMS app. 
@@ -379,6 +402,14 @@ class AppMetadata(pydantic.BaseModel):
                     "The values declared here are propagated by the SDK into the ``appTags`` field of every view "
                     "the app signs."
     )
+    env_requirements: List[EnvRequirement] = pydantic.Field(
+        [],
+        description="(optional) List of environment variables that the app "
+                    "cannot run without. Being listed here means the variable "
+                    "is required, so optional settings are left out. Values "
+                    "of these variables are never part of app metadata or "
+                    "output MMIF."
+    )
     est_gpu_mem_min: int = pydantic.Field(
         0,
         description="(optional) Minimum GPU memory required to run the app, in megabytes (MB). "
@@ -567,6 +598,24 @@ def add_more(self, key: str, value: str):
                 raise ValueError(f"'{key}' is already being used in the appmetadata!")
         else:
             raise ValueError("Key and value should not be empty!")
+
+    def add_env_requirement(self, name: str,
+                            description: str) -> EnvRequirement:
+        """
+        Helper method to declare an environment variable that the app cannot
+        run without.
+
+        :param name: name of the environment variable
+        :param description: what the variable is for and how to get its value
+        :returns: the newly added
+            :class:`~clams.appmetadata.EnvRequirement` object
+        :raises ValueError: if the name is invalid or is already declared
+        """
+        if any(env.name == name for env in self.env_requirements):
+            raise ValueError(f"'{name}' is already declared as required!")
+        new_env = EnvRequirement(name=name, description=description)
+        self.env_requirements.append(new_env)
+        return new_env
 
     def jsonify(self, pretty=False):
         """

clams/develop/templates/app/.dockerignore.template

@@ -8,4 +8,4 @@ bin/
 build/
 *~
 **/*.pyc
-**/__pyche__
+**/__pycache__

documentation/appmetadata.rst

@@ -222,8 +222,65 @@ section).
 Syntax for Parameter Specification in App Metadata
 --------------------------------------------------
 
-Metadata Schema
-===============
+The ``parameters`` field is a list of objects. Each object must include
+``name``, ``description``, ``type``, and ``multivalued``. The ``type`` must be
+one of ``integer``, ``number``, ``string``, ``boolean``, or ``map``. The
+optional ``choices`` field lists accepted values, and ``default`` sets a
+default value (making the parameter optional). Defaults should be JSON values
+of the declared type; use a JSON array for a multivalued parameter.
+
+.. code-block:: json
+
+   {
+     "parameters": [
+       {
+         "name": "minFrameCount",
+         "description": "Minimum number of frames to process.",
+         "type": "integer",
+         "multivalued": false,
+         "default": 5,
+         "choices": [1, 5, 10]
+       },
+       {
+         "name": "labels",
+         "description": "Labels to include in the output.",
+         "type": "string",
+         "multivalued": true,
+         "default": ["PERSON", "ORG"]
+       }
+     ]
+   }
+
+``boolean`` parameters always use ``"multivalued": false``; ``map`` parameters
+always use ``"multivalued": true`` and represent each key-value pair as a
+colon-separated string (for example, ``"default": ["key1:value1"]``). See
+:ref:`runtime-params-detailed` for how parameter values are parsed and passed
+to the app.
+
+Required Environment Variables
+==============================
+
+Some apps cannot run without a value that comes from the machine they run on.
+Examples are an access token for a gated model on the Hugging Face Hub, which must stay out of the output MMIF, and the address of a model server (such as Ollama or vLLM) that the app sends requests to.
+Declare these with :meth:`~clams.appmetadata.AppMetadata.add_env_requirement`, and read the value from the environment in the app code.
+
+.. code-block:: python
+
+   metadata.add_env_requirement(
+       name='HF_TOKEN',
+       description='token with access to pyannote/speaker-diarization-3.1')
+   metadata.add_env_requirement(
+       name='OLLAMA_HOST',
+       description='URL of the Ollama server that runs the model')
+
+Being listed in ``env_requirements`` means the variable is required.
+Do not list optional settings, variables that have a usable default, or common system variables such as ``PYTHONPATH``.
+
+Do not take a secret as a runtime parameter: the SDK records runtime parameters in the ``metadata`` of every view the app signs, so the value ends up in the output MMIF.
+See :ref:`clamsapp-envvars` for how users pass these variables to an app.
+
+Full Metadata JSON Schema
+=========================
 
 The schema for app metadata is as follows. 
 (You can also download the schema in `JSON Schema <https://json-schema.org/>`_ format from `here <appmetadata.jsonschema>`_.)
@@ -234,5 +291,3 @@ The schema for app metadata is as follows.
    :lift_definitions: True
    :auto_target: True
    :auto_reference: True
-
-

documentation/clamsapp.rst

@@ -124,6 +124,53 @@ Unlike the data directory, the cache directory is not required to be mounted, bu
    This can include sensitive information such as browser cache, authentication tokens, etc, hence will pose a great security risk.
    It is recommended to create a separate directory to use as a cache directory for CLAMS containers.
 
+.. _clamsapp-envvars:
+
+Passing environment variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Some apps need environment variables to run, for example an access token for a gated model, or the address of a model server such as Ollama or vLLM.
+These apps list the variables in the ``env_requirements`` field of their app metadata.
+If one of them is unset or empty, the app stops at startup with an error that names the missing variables.
+To provide them, write a file with one ``NAME=value`` per line, usually named ``.env``.
+
+.. code-block:: bash
+
+   HF_TOKEN=<your token>
+   OLLAMA_HOST=http://host.docker.internal:11434
+
+Then pass the file with ``--env-file`` when you start the container.
+
+.. code-block:: bash
+
+   $ docker run --env-file .env -v /path/to/data/directory:/data -p <PORT>:5000 <IMAGE_NAME>
+
+To set a single variable, use ``-e NAME=value`` in place of ``--env-file``.
+
+.. note::
+
+   Inside a container, ``localhost`` is the container itself.
+   To reach a model server that runs on the host machine, use ``host.docker.internal`` as the host name.
+   On Linux, Docker also needs ``--add-host=host.docker.internal:host-gateway`` in the ``docker run`` command, and Podman uses ``host.containers.internal`` in its place.
+
+When you run the app without a container, load the file into your shell first.
+``set -a`` is needed because ``source`` alone does not export the variables to the app.
+
+.. code-block:: bash
+
+   $ set -a; source .env; set +a
+   $ python app.py
+
+.. note::
+
+   Docker and bash read this file differently, so use the plain ``NAME=value`` form to make one file work for both.
+   Do not add an ``export`` prefix, quotes around the value, or a comment after the value.
+   A comment on its own line is fine.
+
+.. warning::
+
+   Keep the ``.env`` file out of version control.
+
 .. _clamsapp-invoking:
 
 Invoking the app server

tests/test_clamsapp.py

@@ -83,6 +83,54 @@ def _annotate(self, mmif, **kwargs):
         return mmif
 
 
+def requiring_env(*names):
+    """
+    Patch :class:`ExampleClamsApp` so that its metadata declares ``names``
+    as required environment variables.
+    """
+    original = ExampleClamsApp._load_appmetadata
+
+    def load(self):
+        metadata = original(self)
+        for name in names:
+            metadata.add_env_requirement(name, f'{name} for testing')
+        return metadata
+    return patch.object(ExampleClamsApp, '_load_appmetadata', load)
+
+
+class TestEnvRequirements(unittest.TestCase):
+
+    names = ('CLAMS_TEST_TOKEN_A', 'CLAMS_TEST_TOKEN_B')
+    secret = 'do-not-leak-1234'
+
+    def setUp(self):
+        patcher = patch.dict(os.environ)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        for name in self.names:
+            os.environ.pop(name, None)
+
+    def test_starts_when_all_set(self):
+        for name in self.names:
+            os.environ[name] = self.secret
+        with requiring_env(*self.names):
+            app = ExampleClamsApp()
+        self.assertNotIn(self.secret, app.appmetadata())
+
+    def test_fails_when_unset_or_empty(self):
+        os.environ['CLAMS_TEST_TOKEN_A'] = self.secret
+        os.environ['CLAMS_TEST_TOKEN_B'] = ''
+        with requiring_env(*self.names), self.assertRaises(RuntimeError) as e:
+            ExampleClamsApp()
+        message = str(e.exception)
+        self.assertIn('CLAMS_TEST_TOKEN_B', message)
+        self.assertNotIn('CLAMS_TEST_TOKEN_A', message)
+        self.assertNotIn(self.secret, message)
+
+    def test_no_declaration_needs_no_env(self):
+        ExampleClamsApp()
+
+
 class TestClamsApp(unittest.TestCase):
 
     def setUp(self):
@@ -311,6 +359,17 @@ def test_app_tags_propagated_to_view(self):
             ['TemporalSegmentation', 'BarsDetection'],
         )
 
+    def test_add_env_requirement(self):
+        env = self.app.metadata.add_env_requirement('HF_TOKEN', 'hub token')
+        self.assertEqual('HF_TOKEN', env.name)
+        dumped = json.loads(self.app.appmetadata())
+        self.assertEqual([{'name': 'HF_TOKEN', 'description': 'hub token'}],
+                         dumped['env_requirements'])
+        with self.assertRaises(ValueError):
+            self.app.metadata.add_env_requirement('HF_TOKEN', 'duplicate')
+        with self.assertRaises(ValueError):
+            self.app.metadata.add_env_requirement('NOT A NAME', 'invalid')
+
     def test_add_app_tag_dedupes_and_validates(self):
         self.app.metadata.add_app_tag('foo', 'bar', 'foo')
         self.assertEqual(self.app.metadata.app_tags, ['foo', 'bar'])