diff --git a/documentation/target-versions.csv b/documentation/target-versions.csv
index 38ec0d61..f144fda0 100644
--- a/documentation/target-versions.csv
+++ b/documentation/target-versions.csv
@@ -1,4 +1,5 @@
"``mmif-python`` version","Target MMIF Specification"
+`1.8.1 `__,`1.2.0 `__
`1.7.0 `__,`1.2.0 `__
`1.6.0 `__,`1.2.0 `__
`1.5.4 `__,`1.2.0 `__
diff --git a/mmif/utils/audio_document_helper.py b/mmif/utils/audio_document_helper.py
index f622ee5c..e30bd81d 100644
--- a/mmif/utils/audio_document_helper.py
+++ b/mmif/utils/audio_document_helper.py
@@ -30,28 +30,31 @@ def iter_audio_windows(
min_last_window_ms: int = 0,
) -> Iterator[tuple["numpy.ndarray", int, int]]:
"""
- Decodes an audio document, or the audio track of a video, into windows.
+ Reads the audio of an audio or video document as windows of samples.
- Samples are resampled to mono ``float32`` at ``sample_rate`` and decoded
- incrementally, so memory use is bounded by one window plus its overlap.
- The windows are those of :func:`~mmif.utils.timeunit_helper.iter_time_windows`
- from the start of the audio stream, which can be later than zero. After
- the last full window, one window runs from the next window start to the
- end of the stream, unless it is shorter than ``min_last_window_ms``.
+ The audio is mixed down to mono and resampled to ``sample_rate``. The
+ file is decoded while the windows are read, so a long file is never held
+ in memory. Times are in ms on the media timeline: the first window
+ starts where the audio stream starts, which in a video can be later than
+ 0. The last window ends at the end of the audio and can be shorter than
+ ``window_ms``.
:param document: audio or video document with an audio stream
- :param window_ms: window length in milliseconds
- :param stride_ms: distance between window starts; defaults to ``window_ms``
+ :param window_ms: window length in ms
+ :param stride_ms: distance between window starts in ms; defaults to
+ ``window_ms``, which gives back-to-back windows
:param sample_rate: output sample rate in Hz
- :param min_last_window_ms: shortest final window to keep
- :returns: ``(samples, start_ms, end_ms)`` triples
+ :param min_last_window_ms: a last window shorter than this is left out
+ :returns: ``(samples, start_ms, end_ms)`` per window, with ``samples`` a
+ 1-D ``float32`` array
:raises ValueError: if a size is not positive, ``min_last_window_ms`` is
outside ``[0, window_ms]``, or the document has an unsupported type
:raises FileNotFoundError: if there is no document or no file
:raises RuntimeError: if the file cannot be opened, has no audio stream,
or fails to decode
- :warns UserWarning: if the decoded audio ends more than
- ``TRUNCATION_TOLERANCE_MS`` before the duration the file states
+ :warns UserWarning: if the audio ends more than
+ ``TRUNCATION_TOLERANCE_MS`` before the length the file states, which
+ usually means the file is truncated
"""
if window_ms <= 0:
raise ValueError('window_ms must be positive')
@@ -87,7 +90,6 @@ def _iter_audio_windows(document, window_ms, stride_ms, sample_rate, min_last_wi
chunks = [] # decoded samples from `buffer_start` on, in arrival order
buffer_start = 0
decoded_end = 0
- window_samples = round(window_ms * sample_rate / 1000)
min_last_samples = max(1, round(min_last_window_ms * sample_rate / 1000))
def to_sample(ms):
@@ -160,22 +162,6 @@ def flush():
origin_s = frame_s if frame_s is not None else stream_start_s
origin_ms = round(origin_s * 1000)
windows = iter_time_windows(origin_ms, None, window_ms, stride_ms)
- if frame_s is not None:
- # a timestamp jump is a gap (filled with silence) or an
- # overlap (dropped) on the sample timeline
- frame_start = round((frame_s - origin_s) * sample_rate)
- gap = frame_start - decoded_end - fifo.samples
- if gap:
- yield from flush()
- while gap > 0:
- append(numpy.zeros(min(gap, window_samples), numpy.float32))
- yield from emit()
- gap = frame_start - decoded_end
- if gap < 0:
- samples = frame.to_ndarray().mean(axis=0, dtype=numpy.float32)
- append(samples[-gap:])
- yield from emit()
- continue
# the fifo checks pts against its own count from zero
frame.pts = None
fifo.write(frame)
@@ -202,12 +188,10 @@ def flush():
def _stated_end_s(av, container, stream, stream_start_s):
"""
- The end of the audio stream according to the file's metadata, in
- seconds, or ``None`` when the file does not state it. A container's own
- duration counts only when it holds no other stream, since it covers the
- longest stream (a video's audio can end before its picture). A WAV file
- cut short states its shorter length, since FFmpeg takes it from the file
- size, so its truncation cannot be seen here.
+ End of the audio stream in seconds as the file metadata states it, or
+ ``None``. The container duration is used only when the file has no other
+ stream, because it is the length of the longest stream. A truncated WAV
+ file states its truncated length, so it is not detected.
"""
if stream.duration is not None and stream.time_base is not None:
return stream_start_s + float(stream.duration * stream.time_base)
diff --git a/mmif/utils/av_helper.py b/mmif/utils/av_helper.py
index 5e2f2c94..2add3754 100644
--- a/mmif/utils/av_helper.py
+++ b/mmif/utils/av_helper.py
@@ -40,13 +40,12 @@ def _ffmpeg_op(
media_type: str,
) -> Iterator[None]:
"""
- Turns failed PyAV operations into actionable ``RuntimeError`` exceptions.
+ Turns a PyAV error in the block (or an ``IndexError`` for a missing
+ stream) into a ``RuntimeError`` that names the document and the file and
+ gives the FFmpeg check command.
- :param action: operation attempted, as a verb phrase
- :param document: MMIF document being read
- :param path: resolved local path of the media document
- :param media_type: stream type being read, ``"audio"`` or ``"video"``
- :raises RuntimeError: if the operation fails
+ :param action: verb for the message, such as ``'decode'``
+ :param media_type: ``"audio"`` or ``"video"``
"""
av = _check_av_dep()
try:
@@ -61,8 +60,11 @@ def _ffmpeg_op(
def _check_document(document: Document, document_types: tuple[object, ...]) -> None:
"""
+ Checks that there is a document and that it has one of
+ ``document_types``.
+
:raises FileNotFoundError: if there is no document
- :raises ValueError: if the document has an unsupported type
+ :raises ValueError: if the document has another type
"""
if document is None:
raise FileNotFoundError('The document does not exist.')
@@ -77,12 +79,12 @@ def _open_container_and_stream(
document_types: tuple[object, ...],
) -> tuple[Any, Any, str]:
"""
- Opens a media container and selects its first stream of the requested type.
+ Opens the media file of a document. The caller must close the container.
- :param document: MMIF document holding the media file
- :param media_type: stream type to select, ``"audio"`` or ``"video"``
- :param document_types: accepted MMIF document types
- :returns: open PyAV container, selected stream, and resolved local path
+ :param media_type: ``"audio"`` or ``"video"``; the first stream of this
+ type is returned
+ :param document_types: document types to accept
+ :returns: PyAV container, stream, and local path of the file
:raises FileNotFoundError: if there is no document or no file
:raises ValueError: if the document has an unsupported type
:raises RuntimeError: if the container or requested stream cannot be read
diff --git a/mmif/utils/video_document_helper.py b/mmif/utils/video_document_helper.py
index b560a07f..a5385c61 100644
--- a/mmif/utils/video_document_helper.py
+++ b/mmif/utils/video_document_helper.py
@@ -36,17 +36,9 @@
def _stream_rate(stream):
"""
- Resolves the frame rate of a video stream, as an exact fraction.
-
- Prefers ffmpeg's guessed rate (``av_guess_frame_rate``), which falls back
- to the stream's base rate (``r_frame_rate``) when the average rate
- (``avg_frame_rate``) is unset. Matroska/webm commonly leave
- ``average_rate`` as ``None`` (which would crash ``float()``) or 0, so it
- cannot be used directly; both are falsy, so the chain yields the first
- usable one. The result stays a ``Fraction`` for exact frame-count math.
-
- :param stream: PyAV video stream
- :returns: frame rate, or a falsy value when the stream reports none
+ Frame rate of a PyAV video stream as a ``Fraction``, or a falsy value
+ when the stream has none. Matroska and webm files often have no average
+ rate (``None`` or 0), so FFmpeg's guessed rate is tried first.
"""
return stream.guessed_rate or stream.average_rate or stream.base_rate
@@ -98,22 +90,20 @@ class SamplingMode(Enum):
def open_container(video_document: Document):
"""
- Opens a video file and caches stream metadata on the document.
+ Opens the video file of a document. The caller must close the container.
- Reads ``time_base``, ``start_time``, ``duration``, and ``average_rate``
- from the first video stream and writes ``fps``, ``frameCount``, and
- ``duration`` (in ms) to the document as informational properties.
- These properties are informational only; seek and extraction use
- actual PTS read from decoded frames.
+ Also sets the ``fps``, ``frameCount`` and ``duration`` (ms) properties of
+ the document from the file metadata. These values can be approximate,
+ for instance the frame count of a variable-frame-rate video; the image
+ extraction functions use the frame timestamps instead.
- :param video_document: :py:class:`~mmif.serialize.annotation.Document`
- holding a video document (``"@type": ".../VideoDocument/..."``)
- :returns: open PyAV :py:class:`av.container.InputContainer`
+ :param video_document: video document
+ :returns: open PyAV container
:rtype: av.container.InputContainer
- :raises ValueError: if ``video_document`` is missing or of the wrong type
- :raises RuntimeError: if the file cannot be opened and read as a video,
- for instance an mp4 whose ``moov`` atom is cut off, or a prefix too
- short to hold a video stream
+ :raises FileNotFoundError: if there is no document or no file
+ :raises ValueError: if the document is not a video document
+ :raises RuntimeError: if the file cannot be read as a video, for instance
+ a truncated mp4 that lost its index
"""
av = _check_av_dep()
container, stream, _ = _open_container_and_stream(
@@ -155,11 +145,14 @@ def open_container(video_document: Document):
def get_framerate(video_document: Document) -> float:
"""
- Gets the frame rate of a video document. First by checking the fps
- property of the document, then by opening the video via PyAV.
+ Gets the frame rate of a video document, rounded to 2 decimal places.
- :param video_document: :py:class:`~mmif.serialize.annotation.Document` instance that holds a video document (``"@type": ".../VideoDocument/..."``)
- :return: frames per second as a float, rounded to 2 decimal places
+ Uses the ``fps`` property of the document (or a property with a similar
+ name, such as ``frameRate``) when there is one. Otherwise reads the file
+ with :py:func:`open_container`, which also sets the property.
+
+ :param video_document: video document
+ :return: frames per second
"""
_check_document(video_document, (DocumentTypes.VideoDocument,))
@@ -179,17 +172,9 @@ def get_framerate(video_document: Document) -> float:
def _decode_frames(container, stream, decode_errors: list[str]):
"""
- Iterates decoded frames of a video stream, stopping at the first error.
-
- Decoding stops at the first :py:class:`av.error.FFmpegError`, which a
- truncated or corrupted file can raise at any point, but the decoder is
- flushed first so frames that are held back for reordering are not lost.
- The error text is appended to ``decode_errors`` for the caller to report.
-
- :param container: open PyAV :py:class:`av.container.InputContainer`
- :param stream: video stream to decode
- :param decode_errors: list that collects decoding error texts
- :returns: generator of :py:class:`av.video.frame.VideoFrame`
+ Yields the decoded frames of a video stream until the end or the first
+ decoding error. After an error, the frames that the decoder still holds
+ are yielded, and the error text is appended to ``decode_errors``.
"""
av = _check_av_dep()
try:
@@ -215,20 +200,13 @@ def _warn_missing_images(
decode_errors: list[str],
) -> None:
"""
- Issues a single warning for timepoints that have no image.
+ Issues one warning for the timepoints that have no image. The warning
+ includes the FFmpeg check command when the file seems damaged: there was
+ a decoding error, or an image is missing before the end of the video.
- A missing timepoint that falls before the document's ``duration``, or a
- decoding error, means the file holds fewer images than its container
- metadata promises, hence the media check hint. Timepoints past a known
- duration are simply out of range, and get no hint.
-
- :param video_document: :py:class:`~mmif.serialize.annotation.Document`
- holding a video document
- :param path: resolved local path of the video document
:param missing_ms: sorted distinct timepoints (in ms) that have no image
- :param n_missing_positions: number of positions in the returned list that
- hold ``None``, which exceeds ``len(missing_ms)`` when timepoints repeat
- :param decode_errors: decoding error texts collected while decoding
+ :param n_missing_positions: number of ``None`` entries in the result,
+ which is larger than ``len(missing_ms)`` when timepoints repeat
"""
if len(missing_ms) <= _MAX_LISTED_TIMEPOINTS:
where = ', '.join(f'{ms}ms' for ms in missing_ms)
@@ -255,26 +233,19 @@ def extract_images_from_timepoints(
as_PIL: bool = False,
):
"""
- Extracts images at the given media-timeline timepoints (in milliseconds).
-
- For each requested timepoint, returns the image whose actual
- presentation timestamp (PTS) is closest to it. Duplicate timepoints
- produce duplicate images at the same list positions as the input.
+ Extracts the video frame closest in time to each timepoint.
- The returned list always has one entry per requested timepoint. An entry
- is ``None`` when no image is available, namely when the timepoint is
- more than one frame duration past the last decodable frame. This happens
- for timepoints past the end of the video, and also within a truncated or
- corrupted file, where decoding stops early. Such a call issues a single
- warning naming the missing timepoints.
+ The result has one entry per timepoint, in the given order. An entry is
+ ``None`` when the timepoint is more than one frame past the last frame
+ that can be decoded: past the end of the video, or past the point where
+ a truncated or damaged file stops decoding. One warning lists the
+ timepoints without an image.
- :param video_document: :py:class:`~mmif.serialize.annotation.Document`
- holding a video document (``"@type": ".../VideoDocument/..."``)
- :param timepoints_ms: iterable of timepoint values in milliseconds
+ :param video_document: video document
+ :param timepoints_ms: timepoints in ms on the media timeline
:param as_PIL: return :py:class:`PIL.Image.Image` (RGB) instead of
:py:class:`~numpy.ndarray` (BGR)
- :returns: images in the same order (and with the same multiplicity) as
- ``timepoints_ms``, with ``None`` where no image is available
+ :returns: images, with ``None`` where no image is available
:rtype: list
"""
original_timepoints = list(timepoints_ms)
@@ -366,14 +337,7 @@ def _emit(frame, t_ms):
def _tp_ids_to_timepoints_ms(mmif: Mmif, tp_ids: list[str]) -> list[int]:
- """
- Converts a list of timepoint annotation IDs to media-timeline timepoints in milliseconds.
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param tp_ids: list of timepoint annotation IDs
- :return: list of timepoint values in ms
- :rtype: list
- """
+ """Gets the ``timePoint`` in ms of each of the TimePoint annotations."""
# TODO: when a source annotation has timeUnit='frame', convert_timepoint
# falls back to `frame / fps` ms math that ignores the container's PTS
# start offset. Fully resolving this requires retiring timeUnit='frame'
@@ -386,16 +350,7 @@ def _tp_ids_to_timepoints_ms(mmif: Mmif, tp_ids: list[str]) -> list[int]:
def _timeframe_to_timepoint_range_ms(
mmif: Mmif, time_frame: Annotation
) -> tuple[int, int]:
- """
- Converts a TimeFrame's start/end to media-timeline timepoints in ms.
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: :py:class:`~mmif.serialize.annotation.Annotation`
- instance of a TimeFrame with ``start``, ``end``,
- ``timeUnit``, and ``document`` properties
- :return: tuple of (start_ms, end_ms)
- :rtype: tuple
- """
+ """Gets the ``start`` and ``end`` in ms of a TimeFrame."""
start, end = convert_timeframe(mmif, time_frame, 'ms')
return int(round(cast(float, start))), int(round(cast(float, end)))
@@ -404,17 +359,8 @@ def _sample_all_timepoint_pairs_ms(
mmif: Mmif, time_frame: Annotation
) -> list[tuple[int, str | None]]:
"""
- Samples all (timepoint_ms, source_id) pairs from a TimeFrame. Uses all
- ``targets`` if present (source is the TP annotation id), otherwise
- samples the start/end interval at the stream's average frame rate
- (source is None for each sampled point).
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: :py:class:`~mmif.serialize.annotation.Annotation`
- instance of a TimeFrame
- :return: list of (ms, source_id) pairs; source_id is ``None`` when
- the point came from interval sampling rather than a TP
- :rtype: list
+ ``(ms, TimePoint id)`` for each of the ``targets`` of a TimeFrame. With
+ no ``targets``, every frame from ``start`` to ``end``, as ``(ms, None)``.
"""
if 'targets' in time_frame:
target_ids = time_frame.get_property('targets')
@@ -431,15 +377,8 @@ def _sample_representatives_timepoint_pairs_ms(
mmif: Mmif, time_frame: Annotation
) -> list[tuple[int, str]]:
"""
- Samples (timepoint_ms, source_id) pairs from a TimeFrame's
- representatives. Returns an empty list if ``representatives`` is not
- present (skips the TimeFrame). Source is always the rep TP id.
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: :py:class:`~mmif.serialize.annotation.Annotation`
- instance of a TimeFrame
- :return: list of (ms, rep_id) pairs (empty if no representatives)
- :rtype: list
+ ``(ms, TimePoint id)`` for each of the ``representatives`` of a
+ TimeFrame; empty when it has none.
"""
if 'representatives' in time_frame:
reps = time_frame.get_property('representatives')
@@ -453,17 +392,9 @@ def _sample_single_timepoint_pair_ms(
mmif: Mmif, time_frame: Annotation
) -> list[tuple[int, str | None]]:
"""
- Samples a single (timepoint_ms, source_id) pair from a TimeFrame.
- Uses the middle representative if ``representatives`` is present
- (source is the rep TP id), otherwise falls back to the midpoint of
- the start/end interval (source is None).
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: :py:class:`~mmif.serialize.annotation.Annotation`
- instance of a TimeFrame
- :return: single-element list ``[(ms, source_id)]``; source_id is
- ``None`` when the point came from the interval midpoint fallback
- :rtype: list
+ ``[(ms, TimePoint id)]`` for the middle one of the ``representatives`` of
+ a TimeFrame. With no ``representatives``, ``[(ms, None)]`` for the middle
+ of ``start`` and ``end``.
"""
if 'representatives' in time_frame:
reps = time_frame.get_property('representatives')
@@ -484,28 +415,25 @@ def extract_images_by_count_with_sources(
as_PIL: bool = False
) -> tuple[list, list[str]]:
"""
- Extracts images at a count-controlled subset of the timepoints listed
- in the ``targets`` property of an annotation, alongside the IDs of the
- selected target TPs.
+ Extracts images at some of the ``targets`` TimePoints of an annotation,
+ and returns them with the ids of those TimePoints.
- The number of timepoints chosen is ``max(min_timepoints,
- int(num_targets * fraction))``, clamped to ``max_timepoints`` and to
- the number of available targets. The chosen indices are spread evenly
- across the target list.
+ ``fraction`` of the targets are used, but at least ``min_timepoints``
+ and at most ``max_timepoints``. They are spread evenly over the targets
+ from the first to the last; a single image comes from the middle one.
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param annotation: :py:class:`~mmif.serialize.annotation.Annotation`
- instance containing a ``targets`` property
- :param min_timepoints: minimum number of timepoints to include
- :param max_timepoints: maximum number of timepoints to include
- :param fraction: fraction of targets to include (ideally)
+ :param mmif: MMIF that holds the annotation and its targets
+ :param annotation: annotation with a ``targets`` property
+ :param min_timepoints: least number of images
+ :param max_timepoints: greatest number of images
+ :param fraction: share of the targets to use, from 0 to 1
:param as_PIL: return :py:class:`~PIL.Image.Image` instead of
:py:class:`~numpy.ndarray`
- :return: tuple of (list of images, list of selected target TP IDs);
- the two lists are parallel, and an image is ``None`` when none is
- available for that timepoint (see
+ :return: images and TimePoint ids, as two lists of the same length; an
+ image is ``None`` when none is available (see
:py:func:`extract_images_from_timepoints`)
:rtype: tuple
+ :raises ValueError: if the annotation has no ``targets``
"""
if 'targets' not in annotation:
raise ValueError(
@@ -544,22 +472,8 @@ def extract_images_by_count(
as_PIL: bool = False
) -> list:
"""
- Extracts images at a count-controlled subset of the timepoints listed
- in the ``targets`` property of an annotation. See
- :py:func:`extract_images_by_count_with_sources` for selection details
- and for a variant that also returns the IDs of the selected target TPs.
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param annotation: :py:class:`~mmif.serialize.annotation.Annotation`
- instance containing a ``targets`` property
- :param min_timepoints: minimum number of timepoints to include
- :param max_timepoints: maximum number of timepoints to include
- :param fraction: fraction of targets to include (ideally)
- :param as_PIL: return :py:class:`~PIL.Image.Image` instead of
- :py:class:`~numpy.ndarray`
- :return: list of images, with ``None`` where none is available (see
- :py:func:`extract_images_from_timepoints`)
- :rtype: list
+ :py:func:`extract_images_by_count_with_sources` without the TimePoint
+ ids.
"""
images, _ = extract_images_by_count_with_sources(
mmif, annotation,
@@ -578,26 +492,25 @@ def extract_images_by_mode_with_sources(
as_PIL: bool = False,
) -> tuple[list, list[str | int]]:
"""
- Extracts images from a TimeFrame using a :py:class:`SamplingMode`,
- alongside the per-image source: a TP annotation id (``str``) when the
- image was selected from a TP (a representative or a target), or the
- sampled timepoint in milliseconds (``int``) when a fallback path was
- used (SINGLE with no representatives, or ALL with no targets).
+ Extracts images from a TimeFrame, chosen by a :py:class:`SamplingMode`,
+ and returns them with where each image was taken.
- If ``mode`` is not specified, uses the context-level default (set via
- :py:data:`_sampling_mode` context variable).
+ The source of an image is the id of the TimePoint it was taken at
+ (``str``), or its time in ms (``int``) when the TimeFrame has no
+ TimePoint to use: ``SINGLE`` with no ``representatives``, or ``ALL``
+ with no ``targets``. ``REPRESENTATIVES`` with no ``representatives``
+ gives no images.
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: TimeFrame annotation to sample from
- :param mode: :py:class:`SamplingMode`, or None to use the context
- default
+ :param mmif: MMIF that holds the TimeFrame
+ :param time_frame: TimeFrame to take images from
+ :param mode: sampling mode; ``None`` uses the mode the ``tfSamplingMode``
+ runtime parameter sets in a CLAMS app (``REPRESENTATIVES`` outside
+ an app)
:param as_PIL: return :py:class:`PIL.Image.Image` instead of
:py:class:`~numpy.ndarray`
- :return: tuple of (list of images, list of sources); the two lists
- are parallel, and an image is ``None`` when none is available for
- that timepoint (see :py:func:`extract_images_from_timepoints`).
- May be ``([], [])`` for ``REPRESENTATIVES`` mode when no
- representatives exist.
+ :return: images and sources, as two lists of the same length; an image
+ is ``None`` when none is available (see
+ :py:func:`extract_images_from_timepoints`)
:rtype: tuple
"""
if mode is None:
@@ -627,21 +540,10 @@ def _to_timepoints(
sources: list[str | int],
) -> list[str]:
"""
- Turns per-image sources, as returned by
- :py:func:`extract_images_by_mode_with_sources`, into ``TimePoint``
- annotation ids, parallel to ``sources``.
-
- A ``str`` source is the id of an existing ``TimePoint`` and is passed
- through. An ``int`` source is a timepoint in milliseconds that no
- ``TimePoint`` backs; a new ``TimePoint`` is created for it in
- ``output_view`` and its id is used. No lookup for an existing
- ``TimePoint`` at the same time is done, so two calls on the same
- fallback timepoint create two annotations.
-
- :param output_view: the view new ``TimePoint`` annotations are added to
- :param video_document: the video the timepoints belong to
- :param sources: per-image sources
- :return: ``TimePoint`` ids, parallel to ``sources``
+ Turns the sources from :py:func:`extract_images_by_mode_with_sources`
+ into TimePoint ids. An id is kept as it is. For a time in ms, a new
+ TimePoint is added to ``output_view`` every time, even when one exists
+ at that time already.
"""
ids: list[str] = []
for source in sources:
@@ -664,34 +566,28 @@ def iter_timeframe_images(
as_PIL: bool = False,
) -> Iterator[tuple[Annotation, list, list[str]]]:
"""
- Samples images from every ``TimeFrame`` in the MMIF, filtered by label,
- and yields them with the ``TimePoint`` each image was taken from.
+ Yields images from each TimeFrame in the MMIF that has a selected label,
+ with the id of the TimePoint each image was taken at.
- ``TimeFrame`` annotations are selected with
- :py:meth:`~mmif.serialize.mmif.Mmif.get_annotations_of_labels`, and
- images are sampled from each with
+ The images of a TimeFrame are chosen as in
:py:func:`extract_images_by_mode_with_sources`. Images that cannot be
- read from the video (``None`` in that function's result, already
- reported there in a warning) are left out together with their sources.
- A ``TimeFrame`` with no readable image is skipped.
-
- Every image comes with the id of a ``TimePoint``. When an image was
- sampled at a time no ``TimePoint`` backs (``tfSamplingMode`` fallback),
- a new ``TimePoint`` is created in ``output_view`` as a side effect of
- iterating, so pass the view the calling app writes its own output into.
- The ids are meant for the ``origins`` of whatever the app derives from
- the images.
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param output_view: the view new ``TimePoint`` annotations are added to
- :param include_labels: ``TimeFrame`` labels to keep; empty means all
- :param exclude_labels: ``TimeFrame`` labels to leave out
- :param mode: :py:class:`SamplingMode`, or None to use the context
- default (the ``tfSamplingMode`` runtime parameter in CLAMS apps)
+ read are left out, and a TimeFrame with no readable image is skipped.
+ When an image was taken at a time with no TimePoint, a new TimePoint is
+ added to ``output_view``. Pass the view the app writes its output to,
+ and use the ids as the ``origins`` of what the app makes from the
+ images.
+
+ :param mmif: MMIF to read the TimeFrames from
+ :param output_view: view to add new TimePoints to
+ :param include_labels: TimeFrame labels to use; empty means all
+ :param exclude_labels: TimeFrame labels to leave out
+ :param mode: sampling mode; see
+ :py:func:`extract_images_by_mode_with_sources`
:param as_PIL: return :py:class:`PIL.Image.Image` instead of
:py:class:`~numpy.ndarray`
- :return: an iterator of ``(time_frame, images, timepoint_ids)``, where
- ``images`` and ``timepoint_ids`` are parallel, non-empty lists
+ :return: ``(time_frame, images, timepoint_ids)`` per TimeFrame, where
+ ``images`` and ``timepoint_ids`` have the same length and are not
+ empty
"""
# select first, so TimePoints added to ``output_view`` while iterating
# never interfere with the views being read
@@ -717,23 +613,7 @@ def extract_images_by_mode(
as_PIL: bool = False,
) -> list:
"""
- Extracts images from a TimeFrame using a :py:class:`SamplingMode`.
- See :py:func:`extract_images_by_mode_with_sources` for the variant
- that also returns the per-image source IDs / timepoints.
-
- If ``mode`` is not specified, uses the context-level default (set via
- :py:data:`_sampling_mode` context variable).
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: TimeFrame annotation to sample from
- :param mode: :py:class:`SamplingMode`, or None to use the context
- default
- :param as_PIL: return :py:class:`PIL.Image.Image` instead of
- :py:class:`~numpy.ndarray`
- :return: list of images, with ``None`` where none is available (see
- :py:func:`extract_images_from_timepoints`); may be empty for
- ``REPRESENTATIVES`` mode when no representatives exist
- :rtype: list
+ :py:func:`extract_images_by_mode_with_sources` without the sources.
"""
images, _ = extract_images_by_mode_with_sources(
mmif, time_frame, mode=mode, as_PIL=as_PIL)
@@ -789,7 +669,8 @@ def convert_timepoint(mmif: Mmif, timepoint: Annotation, out_unit: str) -> int |
def convert_timeframe(mmif: Mmif, time_frame: Annotation, out_unit: str) -> tuple[int | float | str, int | float | str]:
"""
- Converts start and end points in a ``TimeFrame`` annotation a different time unit.
+ Converts the start and end of a ``TimeFrame`` annotation to a different
+ time unit.
:param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
:param time_frame: :py:class:`~mmif.serialize.annotation.Annotation` instance that holds a time interval annotation (``"@type": ".../TimeFrame/..."``)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index d5ae0e03..1004afc0 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1033,8 +1033,11 @@ def setUpClass(cls):
for i in range(0, cls.SAMPLES, cls.RATE // 4)]
cls.cut_mkv = cls._cut(cls._write_pcm(tmp / 'whole.mkv', packets),
tmp / 'cut.mkv')
- cls.cut_mp4 = cls._cut(cls._write_aac_mp4(tmp / 'whole.mp4'),
+ cls.cut_mp4 = cls._cut(cls._write_aac(tmp / 'whole.mp4'),
tmp / 'cut.mp4')
+ # at 48 kHz an AAC frame lasts 21.33 ms, which Matroska stores in
+ # whole ms (at 8 kHz it would be exactly 128 ms)
+ cls.aac_mkv = cls._write_aac(tmp / 'tone.mkv', rate=48000)
@classmethod
def _write_pcm(cls, dst, frames, video=None):
@@ -1072,19 +1075,22 @@ def _cut(src, dst):
return dst
@classmethod
- def _write_aac_mp4(cls, dst, seconds=3):
+ def _write_aac(cls, dst, seconds=3, rate=None):
import av
import numpy as np
from fractions import Fraction
- with av.open(str(dst), 'w', options={'movflags': 'faststart'}) as out:
- stream = out.add_stream('aac', rate=cls.RATE, layout='mono')
- t = np.arange(cls.RATE * seconds) / cls.RATE
+ rate = rate or cls.RATE
+ # the mp4 index goes to the front, so a cut copy still opens
+ options = {'movflags': 'faststart'} if dst.suffix == '.mp4' else {}
+ with av.open(str(dst), 'w', options=options) as out:
+ stream = out.add_stream('aac', rate=rate, layout='mono')
+ t = np.arange(rate * seconds) / rate
tone = (np.sin(2 * np.pi * 440 * t) * 0.3).astype('float32')
frame = av.AudioFrame.from_ndarray(tone[None, :], format='fltp',
layout='mono')
- frame.sample_rate = cls.RATE
+ frame.sample_rate = rate
frame.pts = 0
- frame.time_base = Fraction(1, cls.RATE)
+ frame.time_base = Fraction(1, rate)
out.mux(stream.encode(frame))
out.mux(stream.encode(None))
return dst
@@ -1139,28 +1145,31 @@ def test_stream_start_offset(self):
[(s, e) for _, s, e in windows])
self.assertEqual(0, self._first_sample(windows[0][0]))
- def test_gap_is_filled_with_silence(self):
+ def test_samples_are_counted_across_a_gap(self):
+ # 1 s missing between two packets: as with FFmpeg, the second packet
+ # follows the first directly
windows = self._windows(self.gap, window_ms=1000)
- self.assertEqual([(0, 1000), (1000, 2000), (2000, 3000)],
- [(s, e) for _, s, e in windows])
- self.assertFalse(windows[1][0].any())
- self.assertEqual([0, 1], [round(float(v) / self.FULL) for v in windows[2][0][:2]])
+ self.assertEqual([(0, 1000), (1000, 2000)], [(s, e) for _, s, e in windows])
+ self.assertEqual([0, 1], [round(float(v) / self.FULL) for v in windows[1][0][:2]])
def test_short_gap_while_resampling(self):
- # 8 kHz to 16 kHz: the gap falls inside one output frame of the
- # resampler, and must still be placed at 500-750 ms
+ # 8 kHz to 16 kHz with 250 ms missing: nothing is filled
[(samples, start_ms, end_ms)] = self._windows(
self.short_gap, window_ms=2000, sample_rate=16000)
- self.assertEqual((0, 1250), (start_ms, end_ms))
- self.assertEqual(20000, len(samples))
+ self.assertEqual((0, 1000), (start_ms, end_ms))
+ self.assertEqual(16000, len(samples))
- def at(ms):
- return float(samples[ms * 16]) / self.FULL
-
- # away from the edges, where the resampling filter rings
- self.assertAlmostEqual(400 * 8, at(400), delta=2)
- self.assertLess(max(abs(at(ms)) for ms in range(550, 700, 10)), 1)
- self.assertAlmostEqual(50 * 8, at(800), delta=2)
+ def test_millisecond_time_base_keeps_audio(self):
+ # timestamps rounded to whole ms do not change the samples: the
+ # tone has no runs of inserted zeros
+ import numpy as np
+ windows = self._windows(self.aac_mkv, window_ms=1000, sample_rate=16000)
+ samples = np.concatenate([w for w, _, _ in windows])
+ # skip the encoder's start-up and closing samples, which are quiet
+ body = samples[2000:-2000]
+ zero = np.flatnonzero(body == 0)
+ longest = max((len(r) for r in np.split(zero, np.flatnonzero(np.diff(zero) != 1) + 1) if len(r)), default=0)
+ self.assertLess(longest, 3)
def test_rejects_bad_arguments_before_decoding(self):
doc = media_document(self.mono, DocumentTypes.AudioDocument)
@@ -1196,7 +1205,6 @@ def test_truncated_mp4_raises_with_hint(self):
def test_whole_files_do_not_warn_of_truncation(self):
cases = [(self.mono, DocumentTypes.AudioDocument),
(self.stereo, DocumentTypes.AudioDocument),
- (self.gap, DocumentTypes.AudioDocument),
# an audio track inside a longer video
(self.offset, DocumentTypes.VideoDocument)]
for path, at_type in cases: