diff --git a/documentation/introduction.rst b/documentation/introduction.rst index c9f63e98..979270af 100644 --- a/documentation/introduction.rst +++ b/documentation/introduction.rst @@ -94,6 +94,6 @@ For a full list of available helper methods, please refer to the API documentati MMIF usage in CLAMS Workflows ----------------------------- -In the context of CLAMS, a **Workflow** refers to the sequence of CLAMS applications that have been executed to generate the views and annotations within a MMIF file. +In the context of CLAMS, a **Workflow** refers to the sequence of CLAMS applications that have been executed to generate the views and annotations within a MMIF file. Each step of a workflow is one *app execution*: the views produced by a single run of an app, which share the same ``app`` and ``timestamp`` values in their metadata. A step counts whether its views hold annotations, only warnings, or an error, so the workflow records every app that ran, including ones that failed or found nothing to annotate. -When using the ``mmif-python`` SDK, a unique identifier for a workflow (``workflowId``) is calculated based on the applications involved. This identifier is constructed by concatenating the application name, version, and a hash of the runtime parameters for each step in the sequence. This ensures that the identifier uniquely represents not just the apps used, but their specific configurations, aiding in reproducibility. +When using the ``mmif-python`` SDK, a unique identifier for a workflow (``workflowId``) is calculated based on the applications involved. This identifier is constructed by concatenating a prefix that describes the source documents, followed by the application name, version, and a hash of the runtime parameters for each step in the sequence. This ensures that the identifier uniquely represents not just the apps used, but their specific configurations, aiding in reproducibility. The ``mmif rewind`` command in app mode counts the same steps, so rewinding N apps from a MMIF yields the ``workflowId`` with its last N steps removed. diff --git a/documentation/target-versions.csv b/documentation/target-versions.csv index 05eaa918..f6df6fd0 100644 --- a/documentation/target-versions.csv +++ b/documentation/target-versions.csv @@ -1,4 +1,5 @@ "``mmif-python`` version","Target MMIF Specification" +`1.6.0 `__,`1.2.0 `__ `1.5.4 `__,`1.2.0 `__ `1.5.3 `__,`1.2.0 `__ `1.5.2 `__,`1.2.0 `__ diff --git a/mmif/serialize/annotation.py b/mmif/serialize/annotation.py index e0bd9072..487a2001 100644 --- a/mmif/serialize/annotation.py +++ b/mmif/serialize/annotation.py @@ -368,6 +368,12 @@ def __init__(self, doc_obj: Optional[Union[bytes, str, dict]] = None, *_) -> Non # around the use of these three dictionaries # (names changed since, `existing` >> `ephemeral` and `temporary` >> `pending`) self._props_original: DocumentProperties = DocumentProperties(None, self) + # pending props are never cleared: every `Mmif.serialize()` call + # converts them into `Annotation`s again, and + # `Mmif.generate_capital_annotations()` relies on its duplicate check + # to skip values already written. Calling it more than once (e.g., + # early in clams-python's `ClamsApp.annotate()`) is safe only while + # that check holds. self._props_pending: AnnotationProperties = AnnotationProperties(None, self) self.reserved_names.update(('_props_original', '_props_pending')) diff --git a/mmif/serialize/mmif.py b/mmif/serialize/mmif.py index 08d1ecf1..3b4f54a9 100644 --- a/mmif/serialize/mmif.py +++ b/mmif/serialize/mmif.py @@ -15,7 +15,8 @@ import warnings from collections import defaultdict from datetime import datetime, timezone -from typing import Any, List, Union, Optional, Dict, cast, Iterator +from typing import Any, List, Union, Optional, Dict, cast, Iterable, Iterator +from urllib.parse import urlparse import jsonschema.validators @@ -28,6 +29,10 @@ __all__ = ['Mmif'] +# base URL of first-party CLAMS app identifiers +# (e.g., ``http://apps.clams.ai/swt-detection/v8.6``) +app_directory_baseurl = "http://apps.clams.ai" + class MmifMetadata(MmifObject): """ @@ -122,11 +127,15 @@ def append(self, value: View, overwrite=False) -> None: def get_last_contentful_view(self) -> Optional[View]: """ - Returns the last view that is contentful, i.e., has no error or warning . + Returns the last view that is contentful, i.e., has at least one + annotation (see :meth:`~mmif.serialize.view.View.has_content`). + Views with no annotations, including error and warning views, are + skipped. """ for view in reversed(self._items.values()): - if 'error' not in view.metadata and 'warnings' not in view.metadata: + if view.has_content(): return view + return None def get_last_view(self) -> Optional[View]: """ @@ -336,7 +345,9 @@ def generate_capital_annotations(self): behind this behavior and discussion. """ # this view will be the default kitchen sink for all generated annotations. - last_view = self.views.get_last_contentful_view() + # this can be an empty view, from an app that only adds document properties + last_view = next((v for v in reversed(self.views) + if not v.has_error() and not v.has_warnings()), None) # proceed only when there's at least one view if last_view: @@ -356,18 +367,12 @@ def generate_capital_annotations(self): # new properties to record in the current serialization call anns_to_write = defaultdict(dict) for view in self.views: - doc_id = None - if AnnotationTypes.Annotation in view.metadata.contains: - if 'document' in view.metadata.contains[AnnotationTypes.Annotation]: - doc_id = view.metadata.contains[AnnotationTypes.Annotation]['document'] - for ann in view.get_annotations(AnnotationTypes.Annotation): - if doc_id is None: - # note that in the input MMIF that generated with old implementation, - # this value can be in the "short" form, i.e., without view ID prefix - # however, after deserialization, all document IDs should have the - # view ID prefix, at least within in-memory representation. - doc_id = ann.get_property('document') - existing_anns[doc_id].update(ann.properties.items()) + for ann in view.get_annotations(AnnotationTypes.Annotation): + # view-level `contains` defaults are exposed on each + # annotation as ephemeral properties; in MMIF made by old + # SDKs, the id may be in the short form (no view prefix) + existing_anns[ann.get_property('document')].update( + ann.properties.items()) for doc in view.get_documents(): anns_to_write[doc.id].update(doc._props_pending.items()) for doc in self.documents: @@ -763,6 +768,111 @@ def get_view_contains(self, at_types: Union[ThingTypesBase, str, List[Union[str, return view return None + @staticmethod + def _full_app_id(app_id: str) -> str: + """ + Expands a first-party app ID shorthand (e.g., + ``swt-detection/v8.6``) into a full URI under + :data:`app_directory_baseurl`. An ``http(s)`` URL is returned as is. + """ + if urlparse(app_id).scheme in ('http', 'https'): + return app_id + return f"{app_directory_baseurl}/{app_id.strip('/')}" + + def get_all_views_by_app(self, *app_ids: str) -> List[View]: + """ + Returns the list of all views produced by any of the given apps. + + Each app ID is either a full app URI (required for third-party + apps) or a first-party shorthand such as ``swt-detection/v8.6``, + which is expanded under :data:`app_directory_baseurl`. Matching is + exact: an ID without a version matches no view, and there are no + wildcards, so list every version to cover a version range. To group + views into app runs instead, see + :func:`mmif.utils.workflow_helper.group_views_by_app`. + + :param app_ids: one or more app IDs; a view matches if it was + produced by any of them + :return: the list of matching views + """ + full_ids = {self._full_app_id(app_id) for app_id in app_ids} + return [view for view in self.views if view.metadata.app in full_ids] + + get_views_by_app = get_all_views_by_app + + def get_view_by_app(self, *app_ids: str) -> Optional[View]: + """ + Returns the last view appended that was produced by any of the given + apps. App IDs are matched as in :meth:`get_all_views_by_app`. + + :param app_ids: one or more app IDs + :return: the view, or None if no view matches + """ + matches = self.get_all_views_by_app(*app_ids) + return matches[-1] if matches else None + + def get_all_views_by_apptag(self, *tags: str) -> List[View]: + """ + Returns the list of all views whose ``appTags`` metadata contains + any of the given tags. + + :param tags: one or more tags; a view matches if it carries any of + them + :return: the list of matching views + """ + return [view for view in self.views + if any(tag in view.metadata.appTags for tag in tags)] + + get_views_by_apptag = get_all_views_by_apptag + + def get_view_by_apptag(self, *tags: str) -> Optional[View]: + """ + Returns the last view appended whose ``appTags`` metadata contains + any of the given tags. + + :param tags: one or more tags + :return: the view, or None if no view matches + """ + matches = self.get_all_views_by_apptag(*tags) + return matches[-1] if matches else None + + def get_annotations_of_labels( + self, at_type: Union[ThingTypesBase, str], + include_labels: Iterable[str] = (), + exclude_labels: Iterable[str] = ()) -> Iterator[Annotation]: + """ + Finds annotations of a type across all views, filtered by their + ``label`` property. + + With both label lists empty (the default), every annotation of + ``at_type`` is returned, including annotations that have no + ``label``. A non-empty ``include_labels`` keeps only annotations + whose ``label`` is listed, so annotations without a ``label`` are + dropped. ``exclude_labels`` is applied after ``include_labels`` and + never drops annotations without a ``label``. + + The ``label`` value is read with + :py:meth:`~mmif.serialize.annotation.Annotation.get_property`, so + the deprecated spellings of the property (``frameType`` on + ``TimeFrame``, ``boxType`` on ``BoundingBox``) are honored. + + :param at_type: the type of the annotations to look for + :param include_labels: labels to keep; empty means no restriction + :param exclude_labels: labels to leave out + :return: an iterator of matching annotations, in view order and + then in annotation order within each view + """ + include = tuple(include_labels) + exclude = tuple(exclude_labels) + for view in self.get_all_views_contain(at_type): + for ann in view.get_annotations(at_type): + label = ann.get_property('label') + if include and label not in include: + continue + if label is not None and label in exclude: + continue + yield ann + def _is_in_time_range(self, ann: Annotation, range_s: Union[int, float], range_e: Union[int, float]) -> bool: """ Checks if the annotation is anchored within the given time range. Any overlap is considered included. diff --git a/mmif/serialize/model.py b/mmif/serialize/model.py index 0f1130e8..827366f3 100644 --- a/mmif/serialize/model.py +++ b/mmif/serialize/model.py @@ -134,13 +134,31 @@ def set_additional_property(self, key: str, value: Any) -> None: """ Method to set values in _unnamed_attributes. + A key that matches a declared (named) attribute is stored in that + attribute instead, with a warning, so that a key never lives in both + places. A leading ``@`` in the key is read as the ``_`` prefix of + the attribute name, following the JSON-LD mapping used in + serialization. + :param key: the attribute name :param value: the desired value :return: None :raise: AttributeError if additional properties are disallowed by :func:`disallow_additional_properties` + :raise: KeyError if the key is a reserved name """ + attr = f'_{key[1:]}' if key.startswith('@') else key if self._unnamed_attributes is None: - raise AttributeError(f"Additional properties are disallowed by {self.__class__}") + hint = (f' "{key}" is a declared attribute; use obj["{attr}"] = value.' + if attr in self._named_attributes() else '') + raise AttributeError(f"Additional properties are disallowed by {self.__class__}.{hint}") + if key in self.reserved_names or attr in self.reserved_names: + raise KeyError(f"can't set a reserved name: {key}") + if attr in self._named_attributes(): + warnings.warn(f'"{key}" is a declared attribute of {type(self).__name__}; ' + f'storing it as the attribute, not as an additional property.', + stacklevel=2) + self[attr] = value + return self._unnamed_attributes[key] = value # pytype: disable=unsupported-operands def _named_attributes(self) -> Generator[str, None, None]: diff --git a/mmif/serialize/view.py b/mmif/serialize/view.py index d72a519b..2cb9fcda 100644 --- a/mmif/serialize/view.py +++ b/mmif/serialize/view.py @@ -418,6 +418,14 @@ def has_error(self) -> bool: def has_warnings(self): return self.metadata.has_warnings() + def has_content(self) -> bool: + """ + Returns True if the view has at least one annotation. Per the MMIF + specification, error views and warning views carry no annotations, + so a view with content has neither an error nor warnings. + """ + return len(self.annotations) > 0 + class ViewMetadata(MmifObject): """ @@ -430,6 +438,7 @@ def __init__(self, viewmetadata_obj: Optional[Union[bytes, str, dict]] = None, * self.document: str = '' self.timestamp: Optional[datetime] = None self.app: str = '' + self.appTags: List[str] = [] self.contains: ContainsDict = ContainsDict() self.parameters: Dict[str, str] = {} self.appConfiguration: Dict[str, Union[PRMTV_TYPES, List[PRMTV_TYPES]]] = {} diff --git a/mmif/utils/cli/rewind.py b/mmif/utils/cli/rewind.py index 8dccc63f..48cfdf01 100644 --- a/mmif/utils/cli/rewind.py +++ b/mmif/utils/cli/rewind.py @@ -31,11 +31,14 @@ def prompt_user(mmif_obj: mmif.Mmif) -> int: def rewind_mmif(mmif_obj: mmif.Mmif, choice: int, choice_is_viewnum: bool = True) -> mmif.Mmif: """ - Rewind MMIF by deleting the last N views. - The number of views to rewind is given as a number of "views", or number of "producer apps". - By default, the number argument is interpreted as the number of "views". - Note that when the same app is repeatedly run in a CLAMS workflow and produces multiple views in a row, - rewinding in "app" mode will rewind all those views at once. + Rewind MMIF by deleting the last N views. + The number of views to rewind is given as a number of "views", or number of "producer apps". + By default, the number argument is interpreted as the number of "views". + In "app" mode, one app is one app execution as grouped by + :func:`mmif.utils.workflow_helper.group_views_by_app`: all views that + share the same ``app`` and ``timestamp``, including warning and error + views. Rewinding one app removes the last execution only; earlier runs + of the same app are kept. :param mmif_obj: mmif object :param choice: number of views to rewind @@ -59,7 +62,9 @@ def describe_argparser(): oneliner = 'provides CLI to rewind a MMIF from a CLAMS workflow.' additional = textwrap.dedent(""" MMIF rewinder rewinds a MMIF by deleting the last N views. - N can be specified as a number of views, or a number of producer apps. """) + N can be specified as a number of views, or a number of producer apps. + In app mode, one app is one app execution: all views that share the + same app and timestamp, including warning and error views. """) return oneliner, oneliner + '\n\n' + additional diff --git a/mmif/utils/video_document_helper.py b/mmif/utils/video_document_helper.py index 65edc93d..e3838bf1 100644 --- a/mmif/utils/video_document_helper.py +++ b/mmif/utils/video_document_helper.py @@ -9,12 +9,12 @@ import warnings from io import StringIO from typing import Iterable # todo: replace with collections.abc.Iterable in Python 3.9 -from typing import List, Optional, Union, Tuple +from typing import Iterator, List, Optional, Union, Tuple import mmif -from mmif import Annotation, Document, Mmif +from mmif import Annotation, Document, Mmif, View from mmif.utils.timeunit_helper import convert -from mmif.vocabulary import DocumentTypes +from mmif.vocabulary import AnnotationTypes, DocumentTypes _CV_DEPS = ('av', 'cv2', 'PIL', 'wurlitzer') _cv_import_warning = ( @@ -699,6 +699,95 @@ def extract_images_by_mode_with_sources( return images, sources +def _to_timepoints( + output_view: View, + video_document: Document, + sources: List[Union[str, int]], +) -> List[str]: + """ + Turns per-image sources, as returned by + :py:func:`extract_images_by_mode_with_sources`, into ``TimePoint`` + annotation ids, parallel to ``sources``. + + A ``str`` source is the id of an existing ``TimePoint`` and is passed + through. An ``int`` source is a timepoint in milliseconds that no + ``TimePoint`` backs; a new ``TimePoint`` is created for it in + ``output_view`` and its id is used. No lookup for an existing + ``TimePoint`` at the same time is done, so two calls on the same + fallback timepoint create two annotations. + + :param output_view: the view new ``TimePoint`` annotations are added to + :param video_document: the video the timepoints belong to + :param sources: per-image sources + :return: ``TimePoint`` ids, parallel to ``sources`` + """ + ids: List[str] = [] + for source in sources: + if isinstance(source, str): + ids.append(source) + else: + tp = output_view.new_annotation( + AnnotationTypes.TimePoint, document=video_document.id, + timePoint=int(source), timeUnit='milliseconds') + ids.append(tp.id) + return ids + + +def iter_timeframe_images( + mmif: Mmif, + output_view: View, + include_labels: Iterable[str] = (), + exclude_labels: Iterable[str] = (), + mode: Union[SamplingMode, None] = None, + as_PIL: bool = False, +) -> Iterator[Tuple[Annotation, List, List[str]]]: + """ + Samples images from every ``TimeFrame`` in the MMIF, filtered by label, + and yields them with the ``TimePoint`` each image was taken from. + + ``TimeFrame`` annotations are selected with + :py:meth:`~mmif.serialize.mmif.Mmif.get_annotations_of_labels`, and + images are sampled from each with + :py:func:`extract_images_by_mode_with_sources`. Images that cannot be + read from the video (``None`` in that function's result, already + reported there in a warning) are left out together with their sources. + A ``TimeFrame`` with no readable image is skipped. + + Every image comes with the id of a ``TimePoint``. When an image was + sampled at a time no ``TimePoint`` backs (``tfSamplingMode`` fallback), + a new ``TimePoint`` is created in ``output_view`` as a side effect of + iterating, so pass the view the calling app writes its own output into. + The ids are meant for the ``origins`` of whatever the app derives from + the images. + + :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance + :param output_view: the view new ``TimePoint`` annotations are added to + :param include_labels: ``TimeFrame`` labels to keep; empty means all + :param exclude_labels: ``TimeFrame`` labels to leave out + :param mode: :py:class:`SamplingMode`, or None to use the context + default (the ``tfSamplingMode`` runtime parameter in CLAMS apps) + :param as_PIL: return :py:class:`PIL.Image.Image` instead of + :py:class:`~numpy.ndarray` + :return: an iterator of ``(time_frame, images, timepoint_ids)``, where + ``images`` and ``timepoint_ids`` are parallel, non-empty lists + """ + # select first, so TimePoints added to ``output_view`` while iterating + # never interfere with the views being read + time_frames = list(mmif.get_annotations_of_labels( + AnnotationTypes.TimeFrame, include_labels, exclude_labels)) + for time_frame in time_frames: + images, sources = extract_images_by_mode_with_sources( + mmif, time_frame, mode=mode, as_PIL=as_PIL) + readable = [(image, source) for image, source in zip(images, sources) + if image is not None] + if not readable: + continue + video_document = _resolve_video_document(mmif, time_frame) + timepoint_ids = _to_timepoints( + output_view, video_document, [source for _, source in readable]) + yield time_frame, [image for image, _ in readable], timepoint_ids + + def extract_images_by_mode( mmif: Mmif, time_frame: Annotation, diff --git a/mmif/utils/workflow_helper.py b/mmif/utils/workflow_helper.py index 512a6cd8..2b54a10b 100644 --- a/mmif/utils/workflow_helper.py +++ b/mmif/utils/workflow_helper.py @@ -14,8 +14,22 @@ def group_views_by_app(views: ViewsList) -> List[List[Any]]: """ Groups views into app executions based on app and timestamp. - An "app" is a set of views produced by the same app at the - exact same timestamp. + An app execution is the set of views produced by a single run of an + app. Apps based on ``clams-python`` give every view from one run the + same ``metadata.timestamp``, so views that share ``app`` and + ``timestamp`` form one execution, and two runs of the same app form + two executions. Every execution counts, whether its views hold + annotations, only warnings, or an error. Views without ``app`` or + ``timestamp`` cannot be grouped and are left out. + + The executions are the common unit of + :func:`generate_workflow_identifier`, :func:`describe_single_mmif`, + and app-mode rewinding (:func:`mmif.utils.cli.rewind.rewind_mmif`), + so all three agree on the number and order of executions. + + :param views: views to group + :return: list of app executions, each a list of views, ordered by + timestamp """ # Filter out views that don't have a timestamp or app, as they can't be grouped. groupable_views = [ @@ -121,9 +135,15 @@ def generate_workflow_identifier(mmif_input: Union[str, Path, Mmif], document mix as ``Type-N`` pairs joined by ``-`` and sorted by type name (e.g. ``TextDocument-1-VideoDocument-1``). + Each app execution, as grouped by :func:`group_views_by_app`, adds + exactly one ``app_name/version/param_hash`` segment, including + executions that produced only warnings, only an error, or no + annotations. Hence, rewinding N apps from a MMIF (see + :func:`mmif.utils.cli.rewind.rewind_mmif`) yields the identifier with + its last N segments removed. + Uses view.metadata.parameters (raw user-passed values) for hashing - to ensure reproducibility. Views with errors or warnings are excluded - from the identifier; empty views are included. + to ensure reproducibility. :param mmif_input: Path to MMIF file (str or Path) or a Mmif object :param return_param_dicts: If True, also return the parameter dictionaries @@ -144,10 +164,6 @@ def generate_workflow_identifier(mmif_input: Union[str, Path, Mmif], # Use the first view in the run as representative for metadata first_view = app_execution[0] - # Skip runs where the representative view has errors or warnings - if first_view.has_error() or first_view.has_warnings(): - continue - app = first_view.metadata.get("app") if app is None: continue @@ -183,7 +199,7 @@ class SingleMmifStats(BaseModel): app_count: int = Field(..., alias="appCount", description="Total number of app executions identified.") error_views: List[str] = Field(default_factory=list, alias="errorViews", description="List of view IDs that contain errors.") warning_views: List[str] = Field(default_factory=list, alias="warningViews", description="List of view IDs that contain warnings.") - empty_views: List[str] = Field(default_factory=list, alias="emptyViews", description="List of view IDs that contain no annotations.") + empty_views: List[str] = Field(default_factory=list, alias="emptyViews", description="List of view IDs that contain no annotations, error, or warnings.") annotation_count_by_type: Dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Total annotation counts across the file.") class AppProfiling(BaseModel): @@ -254,7 +270,15 @@ def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict: conceptual hierarchy is that a **workflow** is a sequence of **apps**, and each **app** execution can produce one or more **views**. This function groups views that share the same ``app`` and ``metadata.timestamp`` into - a single logical "app execution". + a single logical "app execution" (see :func:`group_views_by_app`). + + Every app execution is listed in ``apps``, including executions whose + views hold only warnings or an error; the IDs of such views are also + listed in ``stats.warningViews`` and ``stats.errorViews``. The + ``workflowId`` is computed from the same executions by + :func:`generate_workflow_identifier`, so its segments match the + ``apps`` entries one to one. The only exception is the trailing + ``non-existing-app`` entry that collects views that cannot be grouped. .. note:: For MMIF files generated by apps based on ``clams-python`` <= 1.3.3, all @@ -276,9 +300,17 @@ def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict: """ mmif = _read_mmif_from_path(mmif_input) + # view status is a property of each view, independent of grouping error_view_ids = [] warning_view_ids = [] empty_view_ids = [] + for view in mmif.views: + if view.has_error(): + error_view_ids.append(view.id) + elif view.has_warnings(): + warning_view_ids.append(view.id) + elif not view.has_content(): + empty_view_ids.append(view.id) # Generate the new "apps" list grouped_apps = [] @@ -286,14 +318,9 @@ def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict: view_groups = group_views_by_app(mmif.views) for group in view_groups: first_view = group[0] - # skip executions with errors or warnings - if first_view.has_error() or first_view.has_warnings(): - continue execution_ann_counter = Counter() for view in group: - if len(view.annotations) == 0: - empty_view_ids.append(view.id) execution_ann_counter.update(Counter(str(ann.at_type) for ann in view.annotations)) execution_view_ids = [v.id for v in group] @@ -315,19 +342,8 @@ def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict: annotationCountByType=count_dict )) - # Handle unassigned and problematic views - all_view_ids = set(v.id for v in mmif.views) - - for view in mmif.views: - if view.id not in processed_view_ids: - if view.has_error(): - error_view_ids.append(view.id) - elif view.has_warnings(): - warning_view_ids.append(view.id) - elif len(view.annotations) == 0: - empty_view_ids.append(view.id) - - unassigned_view_ids = all_view_ids - processed_view_ids - set(error_view_ids) - set(warning_view_ids) + # views that cannot be grouped (no app or timestamp), whatever they hold + unassigned_view_ids = set(v.id for v in mmif.views) - processed_view_ids # Store app_count before potentially adding the special entry app_count = len(grouped_apps) diff --git a/tests/test_serialize.py b/tests/test_serialize.py index 0ebe50c0..b1be30ed 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -15,7 +15,7 @@ import mmif as mmifpkg from mmif.serialize import * from mmif.serialize.model import * -from mmif.serialize.view import ContainsDict, ErrorDict +from mmif.serialize.view import ContainsDict, ErrorDict, ViewMetadata from mmif.vocabulary import AnnotationTypes, DocumentTypes, ThingType from tests.mmif_examples import * @@ -349,6 +349,145 @@ def test_get_view_contains(self): view = mmif_obj.get_view_contains(not_existing_attype) self.assertIsNone(view) + def _mmif_with_view_states(self): + mmif_obj = Mmif(MMIF_EXAMPLES['everything']) + states = {} + for state in ('contentful', 'empty', 'warnings', 'error'): + view = mmif_obj.new_view() + view.metadata.app = tester_appname + states[state] = view + states['contentful'].new_annotation(AnnotationTypes.TimeFrame) + states['warnings'].metadata.warnings = ['some warning'] + states['error'].set_error('some error', 'some trace') + return mmif_obj, states + + def test_view_has_content(self): + _, states = self._mmif_with_view_states() + self.assertEqual( + {state: view.has_content() for state, view in states.items()}, + {'contentful': True, 'empty': False, 'warnings': False, 'error': False}) + + def test_get_last_contentful_view(self): + mmif_obj, states = self._mmif_with_view_states() + # empty, warnings, and error views appended later are all skipped + self.assertEqual(states['contentful'].id, + mmif_obj.views.get_last_contentful_view().id) + + def test_capital_annotations_go_to_empty_last_view(self): + # an app that only adds document properties leaves an empty view, + # which must still receive the generated Annotation + mmif_obj = Mmif(MMIF_EXAMPLES['everything']) + doc_props_view = mmif_obj.new_view() + doc_props_view.metadata.app = tester_appname + mmif_obj.documents[0].add_property('author', 'me') + roundtrip = Mmif(mmif_obj.serialize()) + anns = list(roundtrip[doc_props_view.id].get_annotations( + AnnotationTypes.Annotation, author='me')) + self.assertEqual(1, len(anns)) + + def _mmif_with_two_documents(self) -> Mmif: + mmif_obj = Mmif(MMIF_EXAMPLES['everything']) + doc2 = Document() + doc2.at_type = DocumentTypes.TextDocument + doc2.id = 'doc2' + doc2.location = 'aScheme:///data/doc2.txt' + mmif_obj.add_document(doc2) + return mmif_obj + + def test_capital_annotations_repeated_generation(self): + # pending props stay after conversion, so converting again must not + # write them twice, also when a view holds several documents' props + mmif_obj = self._mmif_with_two_documents() + view = mmif_obj.new_view() + view.metadata.app = tester_appname + mmif_obj['m1'].add_property('author', 'me') + mmif_obj['doc2'].add_property('author', 'you') + mmif_obj.generate_capital_annotations() + roundtrip = Mmif(mmif_obj.serialize()) + anns = list(roundtrip[view.id].get_annotations(AnnotationTypes.Annotation)) + self.assertEqual(2, len(anns)) + + def test_capital_annotations_skip_value_in_multi_document_view(self): + # a value already recorded for a document is skipped, even when the + # earlier view also holds Annotations for other documents + mmif_obj = self._mmif_with_two_documents() + earlier = mmif_obj.new_view() + earlier.metadata.app = tester_appname + earlier.new_annotation(AnnotationTypes.Annotation, document='m1', author='me') + earlier.new_annotation(AnnotationTypes.Annotation, document='doc2', author='you') + mmif_obj = Mmif(mmif_obj.serialize()) + current = mmif_obj.new_view() + current.metadata.app = tester_appname + mmif_obj['m1'].add_property('author', 'me') + roundtrip = Mmif(mmif_obj.serialize()) + anns = list(roundtrip[current.id].get_annotations(AnnotationTypes.Annotation)) + self.assertEqual(0, len(anns)) + + def _mmif_with_app_views(self) -> Mmif: + mmif_obj = Mmif(MMIF_EXAMPLES['everything']) + for app, tags in ( + ('http://apps.clams.ai/swt-detection/v8.6', ['TemporalSegmentation']), + ('http://apps.clams.ai/swt-detection/v8.7', ['TemporalSegmentation', 'SceneRecognition']), + ('https://example.com/apps/foo/v1', []), + ('http://apps.clams.ai/swt-detection/v8.6', ['TemporalSegmentation']), + ): + view = mmif_obj.new_view() + view.metadata.app = app + view.metadata.appTags = tags + return mmif_obj + + def test_view_metadata_apptags(self): + mmif_obj = self._mmif_with_app_views() + reloaded = Mmif(mmif_obj.serialize()) + tagged, untagged = reloaded.views[-3], reloaded.views[-2] + self.assertEqual(tagged.metadata.appTags, + ['TemporalSegmentation', 'SceneRecognition']) + self.assertEqual(untagged.metadata.appTags, []) + # an empty list is not serialized + raw = json.loads(reloaded.serialize()) + self.assertNotIn('appTags', raw['views'][-2]['metadata']) + + def test_get_all_views_by_app(self): + mmif_obj = self._mmif_with_app_views() + new_ids = [v.id for v in mmif_obj.views][-4:] + v86_a, v87, foo, v86_b = new_ids + by_full = mmif_obj.get_all_views_by_app('http://apps.clams.ai/swt-detection/v8.6') + self.assertEqual([v86_a, v86_b], [v.id for v in by_full]) + # first-party shorthand expands to the same URI + by_short = mmif_obj.get_views_by_app('swt-detection/v8.6') + self.assertEqual([v86_a, v86_b], [v.id for v in by_short]) + # several IDs are OR-ed, e.g. for a version range + by_range = mmif_obj.get_views_by_app('swt-detection/v8.6', 'swt-detection/v8.7') + self.assertEqual([v86_a, v87, v86_b], [v.id for v in by_range]) + # no wildcard or prefix matching + self.assertEqual([], mmif_obj.get_views_by_app('swt-detection')) + # third-party apps need the full URI + self.assertEqual([], mmif_obj.get_views_by_app('foo/v1')) + by_third = mmif_obj.get_views_by_app('https://example.com/apps/foo/v1') + self.assertEqual([foo], [v.id for v in by_third]) + + def test_get_view_by_app(self): + mmif_obj = self._mmif_with_app_views() + last_id = [v.id for v in mmif_obj.views][-1] + self.assertEqual(last_id, mmif_obj.get_view_by_app('swt-detection/v8.6').id) + self.assertIsNone(mmif_obj.get_view_by_app('not-existing-app/v1')) + + def test_get_all_views_by_apptag(self): + mmif_obj = self._mmif_with_app_views() + v86_a, v87, foo, v86_b = [v.id for v in mmif_obj.views][-4:] + by_tag = mmif_obj.get_all_views_by_apptag('TemporalSegmentation') + self.assertEqual([v86_a, v87, v86_b], [v.id for v in by_tag]) + # several tags are OR-ed + by_tags = mmif_obj.get_views_by_apptag('SceneRecognition', 'NotATag') + self.assertEqual([v87], [v.id for v in by_tags]) + self.assertEqual([], mmif_obj.get_views_by_apptag('NotATag')) + + def test_get_view_by_apptag(self): + mmif_obj = self._mmif_with_app_views() + last_id = [v.id for v in mmif_obj.views][-1] + self.assertEqual(last_id, mmif_obj.get_view_by_apptag('TemporalSegmentation').id) + self.assertIsNone(mmif_obj.get_view_by_apptag('NotATag')) + def test_get_views_for_document(self): mmif_obj = Mmif(MMIF_EXAMPLES['everything']) @@ -405,6 +544,50 @@ def test_new_view_id(self): self.assertEqual(e_view.id, f'{p}4') self.assertEqual(len(mmif_obj.views), 5) + def test_get_annotations_of_labels(self): + mmif_obj = Mmif(validate=False) + v1 = mmif_obj.new_view() + v1.new_contain(AnnotationTypes.TimeFrame, timeUnit='milliseconds') + shot = v1.new_annotation(AnnotationTypes.TimeFrame, aid='tf_1', + label='shot', start=0, end=10) + speech = v1.new_annotation(AnnotationTypes.TimeFrame, aid='tf_2', + label='speech', start=10, end=20) + unlabeled = v1.new_annotation(AnnotationTypes.TimeFrame, aid='tf_3', + start=20, end=30) + v2 = mmif_obj.new_view() + v2.new_contain(AnnotationTypes.TimeFrame, timeUnit='milliseconds') + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + legacy = v2.new_annotation(AnnotationTypes.TimeFrame, aid='tf_1', + frameType='shot', start=30, end=40) + v3 = mmif_obj.new_view() + v3.new_contain(AnnotationTypes.BoundingBox) + box = v3.new_annotation(AnnotationTypes.BoundingBox, aid='bb_1', + label='shot') + + def ids(*args, **kwargs): + return [a.id for a in + mmif_obj.get_annotations_of_labels(*args, **kwargs)] + + tf = AnnotationTypes.TimeFrame + # no filter: every TimeFrame across views, unlabeled included, + # in view order; other types never leak in + self.assertEqual( + [shot.id, speech.id, unlabeled.id, legacy.id], ids(tf)) + # allow list drops unlabeled annotations and reads `frameType` + self.assertEqual([shot.id, legacy.id], ids(tf, ['shot'])) + # ignore list keeps unlabeled annotations + self.assertEqual([speech.id, unlabeled.id], + ids(tf, exclude_labels=['shot'])) + # ignore list is applied after the allow list + self.assertEqual([], ids(tf, ['shot'], ['shot'])) + # label filtering works for any annotation type + self.assertEqual([box.id], + ids(AnnotationTypes.BoundingBox, ['shot'])) + # any iterable works, and a type no view contains yields nothing + self.assertEqual([speech.id], ids(tf, iter(['speech']))) + self.assertEqual([], ids(AnnotationTypes.TimePoint)) + def test_get_annotations_between_time(self): token_type = "http://vocab.lappsgrid.org/Token" # Below tokens are obtained by 'jq' in CLI using command: @@ -696,6 +879,57 @@ def test_under_at_swap(self): self.assertIn('@value', plain_json.keys()) self.assertIn('@language', plain_json.keys()) + def test_set_additional_property_undeclared(self): + metadata = ViewMetadata() + with warnings.catch_warnings(): + warnings.simplefilter('error') + metadata.set_additional_property('someExtra', 1) + self.assertEqual(metadata._unnamed_attributes['someExtra'], 1) + self.assertEqual(metadata.get('someExtra'), 1) + + def test_set_additional_property_declared(self): + # a declared key goes to the attribute, never to the side dict + metadata = ViewMetadata() + with self.assertWarns(UserWarning): + metadata.set_additional_property('appTags', ['A', 'B']) + self.assertEqual(metadata.appTags, ['A', 'B']) + self.assertEqual(metadata.get('appTags'), ['A', 'B']) + self.assertNotIn('appTags', metadata._unnamed_attributes) + # the value is converted to the declared attribute class + with self.assertWarns(UserWarning): + metadata.set_additional_property( + 'error', {'message': 'm', 'stackTrace': 't'}) + self.assertIsInstance(metadata.error, ErrorDict) + + def test_set_additional_property_at_prefix(self): + class WithLdAttribute(MmifObject): + def __init__(self, *args): + self._value = '' + super().__init__(*args) + + obj = WithLdAttribute() + with self.assertWarns(UserWarning): + obj.set_additional_property('@value', 'v') + self.assertEqual(obj._value, 'v') + self.assertNotIn('@value', obj._unnamed_attributes) + + def test_set_additional_property_disallowed(self): + # classes without additional properties refuse every key, + # pointing to item assignment for declared ones + text = Text() + with self.assertRaisesRegex(AttributeError, 'declared attribute'): + text.set_additional_property('@value', 'v') + with self.assertRaises(AttributeError) as cm: + text.set_additional_property('someExtra', 'v') + self.assertNotIn('declared attribute', str(cm.exception)) + self.assertEqual(text.value, '') + + def test_set_additional_property_reserved(self): + metadata = ViewMetadata() + for name in ('reserved_names', '_unnamed_attributes'): + with self.assertRaises(KeyError): + metadata.set_additional_property(name, None) + @pytest.mark.skip("TODO: does not work with text examples that are a mixture of old (short-id) and new (long-id) annotations") def test_print_mmif(self): with patch('sys.stdout', new=StringIO()) as fake_out: @@ -1458,7 +1692,7 @@ def test_document_adding_duplicate_properties(self): self.assertEqual(2, len(doc1._props_pending)) mmif_roundtrip2 = Mmif(mmif_roundtrip1.serialize()) ## but not serialized - self.assertEqual(0, len(list(mmif_roundtrip2.views.get_last_contentful_view().get_annotations(AnnotationTypes.Annotation)))) + self.assertEqual(0, len(list(mmif_roundtrip2[r1_vid].get_annotations(AnnotationTypes.Annotation)))) # adding non-duplicate value should be serialized into a new Annotation object # even when there is a duplicate key in a previous view diff --git a/tests/test_utils.py b/tests/test_utils.py index f4f6e185..936a0dc3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -443,6 +443,79 @@ def test_extract_images_by_mode_with_sources_all_fallback(self): # the sources match the sampled ms values self.assertEqual(sorted(sources), sources) + def test_to_timepoints(self): + out_view = self.mmif_obj.new_view() + # existing TimePoint ids pass through, nothing is created + self.assertEqual(['v_0:tp_1', 'v_0:tp_2'], vdh._to_timepoints( + out_view, self.video_doc, ['v_0:tp_1', 'v_0:tp_2'])) + self.assertEqual(0, len(out_view.annotations)) + self.assertNotIn(AnnotationTypes.TimePoint, + out_view.metadata.contains) + # a millisecond source gets a new TimePoint in the output view, + # and order follows the input + ids = vdh._to_timepoints(out_view, self.video_doc, + ['v_0:tp_1', 300]) + self.assertEqual('v_0:tp_1', ids[0]) + minted = self.mmif_obj[ids[1]] + self.assertEqual(out_view.id, minted.parent) + self.assertEqual(300, minted.get_property('timePoint')) + self.assertEqual(self.video_doc.id, minted.get_property('document')) + self.assertIn(AnnotationTypes.TimePoint, out_view.metadata.contains) + + def test_iter_timeframe_images_representatives(self): + tps = self._make_timepoints(10) + reps = [tps[2].id, tps[5].id] + tf = self.a_view.new_annotation( + AnnotationTypes.TimeFrame, aid='tf_0', label='shot', + targets=[tp.id for tp in tps], representatives=reps) + out_view = self.mmif_obj.new_view() + results = list(vdh.iter_timeframe_images( + self.mmif_obj, out_view, mode=vdh.SamplingMode.REPRESENTATIVES)) + self.assertEqual(1, len(results)) + time_frame, images, tp_ids = results[0] + self.assertEqual(tf.id, time_frame.id) + self.assertEqual(2, len(images)) + self.assertEqual(reps, tp_ids) + # every image is backed by an existing TimePoint + self.assertEqual(0, len(out_view.annotations)) + + def test_iter_timeframe_images_single_fallback(self): + self.a_view.new_annotation( + AnnotationTypes.TimeFrame, aid='tf_0', label='shot', + start=100, end=500, timeUnit='milliseconds', + document=self.video_doc.id) + out_view = self.mmif_obj.new_view() + results = list(vdh.iter_timeframe_images( + self.mmif_obj, out_view, mode=vdh.SamplingMode.SINGLE, + as_PIL=True)) + self.assertEqual(1, len(results)) + _, images, tp_ids = results[0] + self.assertEqual(1, len(images)) + self.assertTrue(hasattr(images[0], 'size')) + minted = self.mmif_obj[tp_ids[0]] + self.assertEqual(out_view.id, minted.parent) + self.assertEqual(300, minted.get_property('timePoint')) + + def test_iter_timeframe_images_labels(self): + for i, label in enumerate(['shot', 'credits', None]): + props = dict(start=100 + i * 1000, end=500 + i * 1000, + timeUnit='milliseconds', document=self.video_doc.id) + if label is not None: + props['label'] = label + self.a_view.new_annotation( + AnnotationTypes.TimeFrame, aid=f'tf_{i}', **props) + out_view = self.mmif_obj.new_view() + + def labels(**kwargs): + return [tf.get_property('label') for tf, _, _ in + vdh.iter_timeframe_images( + self.mmif_obj, out_view, + mode=vdh.SamplingMode.SINGLE, **kwargs)] + + self.assertEqual(['shot', 'credits', None], labels()) + self.assertEqual(['shot'], labels(include_labels=['shot'])) + self.assertEqual(['shot', None], labels(exclude_labels=['credits'])) + def test_renamed_helpers_emit_deprecation_warnings(self): # The three pre-rename names live as deprecation aliases that # delegate to the new names. @@ -830,6 +903,42 @@ def test_none_stays_parallel_with_sources(self): self.assertIsNone(images[1]) self.assertIsNone(images[2]) + def test_iter_timeframe_images_drops_unreadable(self): + mmif_obj = Mmif(validate=False) + doc = self._doc(self.healthy) + mmif_obj.add_document(doc) + vdh.open_container(doc).close() + duration = doc.get_property('duration') + view = mmif_obj.new_view() + tp_ids = [] + for i, ms in enumerate([1000, duration + 10000]): + tp = view.new_annotation( + AnnotationTypes.TimePoint, aid=f'tp_{i}', timePoint=ms, + timeUnit='milliseconds', document=doc.id) + tp_ids.append(tp.id) + # one TimeFrame with a readable and an unreadable image, one with + # only an unreadable image + view.new_annotation( + AnnotationTypes.TimeFrame, aid='tf_0', document=doc.id, + timeUnit='milliseconds', start=1000, end=duration + 10000, + targets=tp_ids, representatives=tp_ids) + view.new_annotation( + AnnotationTypes.TimeFrame, aid='tf_1', document=doc.id, + timeUnit='milliseconds', start=duration + 10000, + end=duration + 20000, targets=[tp_ids[1]], + representatives=[tp_ids[1]]) + out_view = mmif_obj.new_view() + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + results = list(vdh.iter_timeframe_images( + mmif_obj, out_view, mode=vdh.SamplingMode.REPRESENTATIVES)) + self.assertEqual(1, len(results)) + time_frame, images, origins = results[0] + self.assertEqual(f'{view.id}:tf_0', time_frame.id) + self.assertEqual([tp_ids[0]], origins) + self.assertEqual(1, len(images)) + self.assertIsNotNone(images[0]) + class TestSequenceHelper(unittest.TestCase): @@ -1101,9 +1210,69 @@ def test_describe_single_mmif_error_view(self): result = wfh.describe_single_mmif(tmp_file) # Validate against Pydantic model validated = wfh.SingleMmifDesc.model_validate(result) - self.assertEqual(validated.stats.app_count, 0) - self.assertEqual(len(validated.apps), 0) - self.assertEqual(len(validated.stats.error_views), 1) + # a failed run is still an app execution + self.assertEqual(validated.stats.app_count, 1) + self.assertEqual(len(validated.apps), 1) + self.assertEqual(validated.stats.error_views, [view.id]) + self.assertEqual(validated.stats.empty_views, []) + finally: + os.unlink(tmp_file) + + def test_describe_single_mmif_warnings_only_view(self): + """Test describe_single_mmif with a run that produced only warnings.""" + view = self.basic_mmif.new_view() + view.metadata.app = "http://apps.clams.ai/test-app/v1.0.0" + view.metadata.timestamp = "2024-01-01T12:00:00Z" + view.metadata.warnings = ["EmptyOutputWarning: no annotations"] + tmp_file = self.create_temp_mmif_file(self.basic_mmif) + try: + validated = wfh.SingleMmifDesc.model_validate( + wfh.describe_single_mmif(tmp_file)) + self.assertEqual(validated.stats.app_count, 1) + self.assertEqual(validated.stats.warning_views, [view.id]) + self.assertEqual(validated.stats.empty_views, []) + finally: + os.unlink(tmp_file) + + def test_describe_single_mmif_annotations_and_warnings(self): + """Test describe_single_mmif with a run of a contentful view and a warnings view.""" + contentful = self.basic_mmif.new_view() + contentful.metadata.app = "http://apps.clams.ai/test-app/v1.0.0" + contentful.metadata.timestamp = "2024-01-01T12:00:00Z" + contentful.new_annotation(AnnotationTypes.TimeFrame) + warning = self.basic_mmif.new_view() + warning.metadata.app = "http://apps.clams.ai/test-app/v1.0.0" + warning.metadata.timestamp = "2024-01-01T12:00:00Z" + warning.metadata.warnings = ["UserWarning: something odd"] + tmp_file = self.create_temp_mmif_file(self.basic_mmif) + try: + validated = wfh.SingleMmifDesc.model_validate( + wfh.describe_single_mmif(tmp_file)) + self.assertEqual(validated.stats.app_count, 1) + self.assertEqual(validated.apps[0].view_ids, + [contentful.id, warning.id]) + self.assertEqual(validated.stats.warning_views, [warning.id]) + self.assertEqual(validated.stats.error_views, []) + self.assertEqual(validated.stats.empty_views, []) + finally: + os.unlink(tmp_file) + + def test_describe_single_mmif_empty_view(self): + """ + Test describe_single_mmif with an empty view. clams-python 1.7.4 and + later never emit one, so this covers older files and other producers. + """ + view = self.basic_mmif.new_view() + view.metadata.app = "http://apps.clams.ai/test-app/v1.0.0" + view.metadata.timestamp = "2024-01-01T12:00:00Z" + tmp_file = self.create_temp_mmif_file(self.basic_mmif) + try: + validated = wfh.SingleMmifDesc.model_validate( + wfh.describe_single_mmif(tmp_file)) + self.assertEqual(validated.stats.app_count, 1) + self.assertEqual(validated.stats.empty_views, [view.id]) + self.assertEqual(validated.stats.warning_views, []) + self.assertEqual(validated.stats.error_views, []) finally: os.unlink(tmp_file)