releasing 1.6.0

documentation/introduction.rst

@@ -94,6 +94,6 @@ For a full list of available helper methods, please refer to the API documentati
 MMIF usage in CLAMS Workflows
 -----------------------------
 
-In the context of CLAMS, a **Workflow** refers to the sequence of CLAMS applications that have been executed to generate the views and annotations within a MMIF file.
+In the context of CLAMS, a **Workflow** refers to the sequence of CLAMS applications that have been executed to generate the views and annotations within a MMIF file. Each step of a workflow is one *app execution*: the views produced by a single run of an app, which share the same ``app`` and ``timestamp`` values in their metadata. A step counts whether its views hold annotations, only warnings, or an error, so the workflow records every app that ran, including ones that failed or found nothing to annotate.
 
-When using the ``mmif-python`` SDK, a unique identifier for a workflow (``workflowId``) is calculated based on the applications involved. This identifier is constructed by concatenating the application name, version, and a hash of the runtime parameters for each step in the sequence. This ensures that the identifier uniquely represents not just the apps used, but their specific configurations, aiding in reproducibility.
+When using the ``mmif-python`` SDK, a unique identifier for a workflow (``workflowId``) is calculated based on the applications involved. This identifier is constructed by concatenating a prefix that describes the source documents, followed by the application name, version, and a hash of the runtime parameters for each step in the sequence. This ensures that the identifier uniquely represents not just the apps used, but their specific configurations, aiding in reproducibility. The ``mmif rewind`` command in app mode counts the same steps, so rewinding N apps from a MMIF yields the ``workflowId`` with its last N steps removed.

documentation/target-versions.csv

@@ -1,4 +1,5 @@
 "``mmif-python`` version","Target MMIF Specification"
+`1.6.0 <https://pypi.org/project/mmif-python/1.6.0/>`__,`1.2.0 <https://mmif.clams.ai/1.2.0/>`__
 `1.5.4 <https://pypi.org/project/mmif-python/1.5.4/>`__,`1.2.0 <https://mmif.clams.ai/1.2.0/>`__
 `1.5.3 <https://pypi.org/project/mmif-python/1.5.3/>`__,`1.2.0 <https://mmif.clams.ai/1.2.0/>`__
 `1.5.2 <https://pypi.org/project/mmif-python/1.5.2/>`__,`1.2.0 <https://mmif.clams.ai/1.2.0/>`__

mmif/serialize/annotation.py

@@ -368,6 +368,12 @@ def __init__(self, doc_obj: Optional[Union[bytes, str, dict]] = None, *_) -> Non
         # around the use of these three dictionaries
         # (names changed since, `existing` >> `ephemeral` and `temporary` >> `pending`)
         self._props_original: DocumentProperties = DocumentProperties(None, self)
+        # pending props are never cleared: every `Mmif.serialize()` call
+        # converts them into `Annotation`s again, and
+        # `Mmif.generate_capital_annotations()` relies on its duplicate check
+        # to skip values already written. Calling it more than once (e.g.,
+        # early in clams-python's `ClamsApp.annotate()`) is safe only while
+        # that check holds.
         self._props_pending: AnnotationProperties = AnnotationProperties(None, self)
         self.reserved_names.update(('_props_original', '_props_pending'))
 

mmif/serialize/mmif.py

@@ -15,7 +15,8 @@
 import warnings
 from collections import defaultdict
 from datetime import datetime, timezone
-from typing import Any, List, Union, Optional, Dict, cast, Iterator
+from typing import Any, List, Union, Optional, Dict, cast, Iterable, Iterator
+from urllib.parse import urlparse
 
 import jsonschema.validators
 
@@ -28,6 +29,10 @@
 
 __all__ = ['Mmif']
 
+# base URL of first-party CLAMS app identifiers
+# (e.g., ``http://apps.clams.ai/swt-detection/v8.6``)
+app_directory_baseurl = "http://apps.clams.ai"
+
 
 class MmifMetadata(MmifObject):
     """
@@ -122,11 +127,15 @@ def append(self, value: View, overwrite=False) -> None:
 
     def get_last_contentful_view(self) -> Optional[View]:
         """
-        Returns the last view that is contentful, i.e., has no error or warning .
+        Returns the last view that is contentful, i.e., has at least one
+        annotation (see :meth:`~mmif.serialize.view.View.has_content`).
+        Views with no annotations, including error and warning views, are
+        skipped.
         """
         for view in reversed(self._items.values()):
-            if 'error' not in view.metadata and 'warnings' not in view.metadata:
+            if view.has_content():
                 return view
+        return None
 
     def get_last_view(self) -> Optional[View]:
         """
@@ -336,7 +345,9 @@ def generate_capital_annotations(self):
         behind this behavior and discussion.
         """
         # this view will be the default kitchen sink for all generated annotations.
-        last_view = self.views.get_last_contentful_view()
+        # this can be an empty view, from an app that only adds document properties
+        last_view = next((v for v in reversed(self.views)
+                          if not v.has_error() and not v.has_warnings()), None)
 
         # proceed only when there's at least one view
         if last_view:
@@ -356,18 +367,12 @@ def generate_capital_annotations(self):
             # new properties to record in the current serialization call
             anns_to_write = defaultdict(dict)
             for view in self.views:
-                doc_id = None
-                if AnnotationTypes.Annotation in view.metadata.contains:
-                    if 'document' in view.metadata.contains[AnnotationTypes.Annotation]:
-                        doc_id = view.metadata.contains[AnnotationTypes.Annotation]['document']
-                    for ann in view.get_annotations(AnnotationTypes.Annotation):
-                        if doc_id is None:
-                            # note that in the input MMIF that generated with old implementation,
-                            # this value can be in the "short" form, i.e., without view ID prefix
-                            # however, after deserialization, all document IDs should have the 
-                            # view ID prefix, at least within in-memory representation.
-                            doc_id = ann.get_property('document')
-                        existing_anns[doc_id].update(ann.properties.items())
+                for ann in view.get_annotations(AnnotationTypes.Annotation):
+                    # view-level `contains` defaults are exposed on each
+                    # annotation as ephemeral properties; in MMIF made by old
+                    # SDKs, the id may be in the short form (no view prefix)
+                    existing_anns[ann.get_property('document')].update(
+                        ann.properties.items())
                 for doc in view.get_documents():
                     anns_to_write[doc.id].update(doc._props_pending.items())
             for doc in self.documents:
@@ -763,6 +768,111 @@ def get_view_contains(self, at_types: Union[ThingTypesBase, str, List[Union[str,
                     return view
         return None
 
+    @staticmethod
+    def _full_app_id(app_id: str) -> str:
+        """
+        Expands a first-party app ID shorthand (e.g.,
+        ``swt-detection/v8.6``) into a full URI under
+        :data:`app_directory_baseurl`. An ``http(s)`` URL is returned as is.
+        """
+        if urlparse(app_id).scheme in ('http', 'https'):
+            return app_id
+        return f"{app_directory_baseurl}/{app_id.strip('/')}"
+
+    def get_all_views_by_app(self, *app_ids: str) -> List[View]:
+        """
+        Returns the list of all views produced by any of the given apps.
+
+        Each app ID is either a full app URI (required for third-party
+        apps) or a first-party shorthand such as ``swt-detection/v8.6``,
+        which is expanded under :data:`app_directory_baseurl`. Matching is
+        exact: an ID without a version matches no view, and there are no
+        wildcards, so list every version to cover a version range. To group
+        views into app runs instead, see
+        :func:`mmif.utils.workflow_helper.group_views_by_app`.
+
+        :param app_ids: one or more app IDs; a view matches if it was
+            produced by any of them
+        :return: the list of matching views
+        """
+        full_ids = {self._full_app_id(app_id) for app_id in app_ids}
+        return [view for view in self.views if view.metadata.app in full_ids]
+
+    get_views_by_app = get_all_views_by_app
+
+    def get_view_by_app(self, *app_ids: str) -> Optional[View]:
+        """
+        Returns the last view appended that was produced by any of the given
+        apps. App IDs are matched as in :meth:`get_all_views_by_app`.
+
+        :param app_ids: one or more app IDs
+        :return: the view, or None if no view matches
+        """
+        matches = self.get_all_views_by_app(*app_ids)
+        return matches[-1] if matches else None
+
+    def get_all_views_by_apptag(self, *tags: str) -> List[View]:
+        """
+        Returns the list of all views whose ``appTags`` metadata contains
+        any of the given tags.
+
+        :param tags: one or more tags; a view matches if it carries any of
+            them
+        :return: the list of matching views
+        """
+        return [view for view in self.views
+                if any(tag in view.metadata.appTags for tag in tags)]
+
+    get_views_by_apptag = get_all_views_by_apptag
+
+    def get_view_by_apptag(self, *tags: str) -> Optional[View]:
+        """
+        Returns the last view appended whose ``appTags`` metadata contains
+        any of the given tags.
+
+        :param tags: one or more tags
+        :return: the view, or None if no view matches
+        """
+        matches = self.get_all_views_by_apptag(*tags)
+        return matches[-1] if matches else None
+
+    def get_annotations_of_labels(
+            self, at_type: Union[ThingTypesBase, str],
+            include_labels: Iterable[str] = (),
+            exclude_labels: Iterable[str] = ()) -> Iterator[Annotation]:
+        """
+        Finds annotations of a type across all views, filtered by their
+        ``label`` property.
+
+        With both label lists empty (the default), every annotation of
+        ``at_type`` is returned, including annotations that have no
+        ``label``. A non-empty ``include_labels`` keeps only annotations
+        whose ``label`` is listed, so annotations without a ``label`` are
+        dropped. ``exclude_labels`` is applied after ``include_labels`` and
+        never drops annotations without a ``label``.
+
+        The ``label`` value is read with
+        :py:meth:`~mmif.serialize.annotation.Annotation.get_property`, so
+        the deprecated spellings of the property (``frameType`` on
+        ``TimeFrame``, ``boxType`` on ``BoundingBox``) are honored.
+
+        :param at_type: the type of the annotations to look for
+        :param include_labels: labels to keep; empty means no restriction
+        :param exclude_labels: labels to leave out
+        :return: an iterator of matching annotations, in view order and
+                 then in annotation order within each view
+        """
+        include = tuple(include_labels)
+        exclude = tuple(exclude_labels)
+        for view in self.get_all_views_contain(at_type):
+            for ann in view.get_annotations(at_type):
+                label = ann.get_property('label')
+                if include and label not in include:
+                    continue
+                if label is not None and label in exclude:
+                    continue
+                yield ann
+
     def _is_in_time_range(self, ann: Annotation, range_s: Union[int, float], range_e: Union[int, float]) -> bool:
         """
         Checks if the annotation is anchored within the given time range. Any overlap is considered included. 

mmif/serialize/model.py

@@ -134,13 +134,31 @@ def set_additional_property(self, key: str, value: Any) -> None:
         """
         Method to set values in _unnamed_attributes.
 
+        A key that matches a declared (named) attribute is stored in that
+        attribute instead, with a warning, so that a key never lives in both
+        places. A leading ``@`` in the key is read as the ``_`` prefix of
+        the attribute name, following the JSON-LD mapping used in
+        serialization.
+
         :param key: the attribute name
         :param value: the desired value
         :return: None
         :raise: AttributeError if additional properties are disallowed by :func:`disallow_additional_properties`
+        :raise: KeyError if the key is a reserved name
         """
+        attr = f'_{key[1:]}' if key.startswith('@') else key
         if self._unnamed_attributes is None:
-            raise AttributeError(f"Additional properties are disallowed by {self.__class__}")
+            hint = (f' "{key}" is a declared attribute; use obj["{attr}"] = value.'
+                    if attr in self._named_attributes() else '')
+            raise AttributeError(f"Additional properties are disallowed by {self.__class__}.{hint}")
+        if key in self.reserved_names or attr in self.reserved_names:
+            raise KeyError(f"can't set a reserved name: {key}")
+        if attr in self._named_attributes():
+            warnings.warn(f'"{key}" is a declared attribute of {type(self).__name__}; '
+                          f'storing it as the attribute, not as an additional property.',
+                          stacklevel=2)
+            self[attr] = value
+            return
         self._unnamed_attributes[key] = value  # pytype: disable=unsupported-operands
 
     def _named_attributes(self) -> Generator[str, None, None]:

mmif/serialize/view.py

@@ -418,6 +418,14 @@ def has_error(self) -> bool:
     def has_warnings(self):
         return self.metadata.has_warnings()
 
+    def has_content(self) -> bool:
+        """
+        Returns True if the view has at least one annotation. Per the MMIF
+        specification, error views and warning views carry no annotations,
+        so a view with content has neither an error nor warnings.
+        """
+        return len(self.annotations) > 0
+
 
 class ViewMetadata(MmifObject):
     """
@@ -430,6 +438,7 @@ def __init__(self, viewmetadata_obj: Optional[Union[bytes, str, dict]] = None, *
         self.document: str = ''
         self.timestamp: Optional[datetime] = None
         self.app: str = ''
+        self.appTags: List[str] = []
         self.contains: ContainsDict = ContainsDict()
         self.parameters: Dict[str, str] = {}
         self.appConfiguration: Dict[str, Union[PRMTV_TYPES, List[PRMTV_TYPES]]] = {}

mmif/utils/cli/rewind.py

@@ -31,11 +31,14 @@ def prompt_user(mmif_obj: mmif.Mmif) -> int:
 
 def rewind_mmif(mmif_obj: mmif.Mmif, choice: int, choice_is_viewnum: bool = True) -> mmif.Mmif:
     """
-    Rewind MMIF by deleting the last N views. 
-    The number of views to rewind is given as a number of "views", or number of "producer apps". 
-    By default, the number argument is interpreted as the number of "views". 
-    Note that when the same app is repeatedly run in a CLAMS workflow and produces multiple views in a row,
-    rewinding in "app" mode will rewind all those views at once.
+    Rewind MMIF by deleting the last N views.
+    The number of views to rewind is given as a number of "views", or number of "producer apps".
+    By default, the number argument is interpreted as the number of "views".
+    In "app" mode, one app is one app execution as grouped by
+    :func:`mmif.utils.workflow_helper.group_views_by_app`: all views that
+    share the same ``app`` and ``timestamp``, including warning and error
+    views. Rewinding one app removes the last execution only; earlier runs
+    of the same app are kept.
 
     :param mmif_obj: mmif object
     :param choice: number of views to rewind
@@ -59,7 +62,9 @@ def describe_argparser():
     oneliner = 'provides CLI to rewind a MMIF from a CLAMS workflow.'
     additional = textwrap.dedent("""
     MMIF rewinder rewinds a MMIF by deleting the last N views.
-    N can be specified as a number of views, or a number of producer apps. """)
+    N can be specified as a number of views, or a number of producer apps.
+    In app mode, one app is one app execution: all views that share the
+    same app and timestamp, including warning and error views. """)
     return oneliner, oneliner + '\n\n' + additional