frame_source/

h264_source.rs

// Copyright (C) The Strand-Braid Authors
// SPDX-License-Identifier: MIT OR Apache-2.0

use std::{
    io::{BufReader, Read, Seek},
    path::Path,
};

use chrono::{DateTime, FixedOffset, Utc};
use h264_reader::{
    Context as H264ParsingContext,
    nal::{
        Nal, RefNal, UnitType,
        sei::{HeaderType, SeiMessage, SeiReader},
    },
    rbsp::BitReaderError,
};
use serde::{Deserialize, Serialize};

use strand_cam_remote_control::{H264_METADATA_UUID, H264_METADATA_VERSION, H264Metadata};

#[cfg(feature = "openh264")]
use machine_vision_formats::owned::OImage;

use crate::{
    EncodedH264, Error, FrameData, FrameDataSource, H264EncodingVariant, ImageData, MyAsStr,
    Result, Timestamp, TimestampSource,
    h264_poc::{self, PocStrategy},
    ntp_timestamp::NtpTimestamp,
    srt_reader::{self, Stanza},
};

struct SrtData {
    stanzas: Vec<Stanza>,
    frame0_time: DateTime<FixedOffset>,
    idx: usize,
    /// Line at which the SRT file's parse stopped early (see
    /// [`srt_reader::SrtParseOutcome::truncated_at_line`]), if that's why
    /// `stanzas` might be shorter than the video's frame count.
    truncated_at_line: Option<usize>,
}

/// Reports that an [`H264Source`] built from an SRT-file timestamp source
/// covers fewer frames than the underlying container, because the SRT ran
/// out of usable stanzas partway through. The source is truncated to the last
/// complete group of pictures (GOP) that a stanza is available for every
/// frame of, so reordered (B-frame) streams remain fully resolvable.
#[derive(Debug, Clone)]
pub struct SrtTruncation {
    /// Number of frames retained (a prefix, rounded down to the last
    /// complete GOP boundary at or before `usable_stanzas`).
    pub kept_frames: usize,
    /// Total number of frames in the underlying container.
    pub total_frames: usize,
    /// Number of stanzas the SRT file yielded successfully.
    pub usable_stanzas: usize,
    /// Line in the SRT file where parsing stopped due to malformed content.
    /// `None` if the SRT file simply had fewer stanzas than the video has
    /// frames (no parse error, just ran out).
    pub malformed_at_line: Option<usize>,
}

#[derive(serde::Deserialize)]
struct SrtMsg {
    timestamp: DateTime<chrono::FixedOffset>,
}

impl SrtData {
    fn parse_time(stanza: &Stanza) -> DateTime<FixedOffset> {
        let msg: SrtMsg = serde_json::from_str(&stanza.lines).unwrap();
        msg.timestamp
    }
    fn next_pts(&mut self) -> Result<std::time::Duration> {
        let stanza = &self.stanzas[self.idx];
        self.idx += 1;
        let tnow = Self::parse_time(stanza);
        Ok(tnow.signed_duration_since(self.frame0_time).to_std()?)
    }
    /// Capture time (relative to frame0) for stanza `idx`, where `idx` is the
    /// frame's presentation (display) rank. Frames are handed to us in decode
    /// order, which differs from presentation order for B-frame streams, so we
    /// must index the (presentation-order) stanzas by display rank rather than
    /// by decode index.
    fn time_at(&self, idx: usize) -> Result<std::time::Duration> {
        let stanza = &self.stanzas[idx];
        let tnow = Self::parse_time(stanza);
        Ok(tnow.signed_duration_since(self.frame0_time).to_std()?)
    }
    fn frame0_time(&self) -> DateTime<FixedOffset> {
        self.frame0_time
    }
    /// Wall-clock span from the first to the last stanza (presentation order).
    fn span(&self) -> Result<std::time::Duration> {
        let first = Self::parse_time(&self.stanzas[0]);
        let last = Self::parse_time(&self.stanzas[self.stanzas.len() - 1]);
        Ok(last.signed_duration_since(first).to_std()?)
    }
}

/// Per-sample timing carried through from an MP4 source, so it can be
/// preserved verbatim when re-muxing (rather than re-derived from
/// presentation-time deltas). This is what lets reordered (B-frame) streams be
/// re-muxed correctly.
#[derive(Debug, Clone, Copy)]
pub struct Mp4SampleTiming {
    /// The sample's own decode duration (stts delta).
    pub decode_duration: std::time::Duration,
    /// The sample's composition time offset (ctts); `presentation = decode +
    /// composition_offset`. Non-zero for reordered streams.
    pub composition_offset: chrono::Duration,
}

pub trait H264Preparser {
    fn put_seq_param_set(&mut self, nalu: &RefNal<'_>) -> eyre::Result<()>;
    fn put_pic_param_set(&mut self, nalu: &RefNal<'_>) -> eyre::Result<()>;
    fn put_sei_nalu(&mut self, nalu: &RefNal<'_>) -> eyre::Result<()>;
    fn put_slice_layer_nalu(&mut self, nalu: &RefNal<'_>, is_i_frame: bool) -> eyre::Result<()>;
    fn set_num_positions(&mut self, num_positions: usize) -> eyre::Result<()>;
    fn set_position(&mut self, pos: usize) -> eyre::Result<()>;
    fn close(self) -> eyre::Result<()>;
}

// Found in libx264-encoded h264 streams. See
// https://code.videolan.org/videolan/x264/-/blob/da14df5535/encoder/set.c#L598
const X264_UUID: &[u8; 16] = uuid::uuid!("dc45e9bd-e6d9-48b7-962c-d820d923eeef").as_bytes();

// Found in videotoolbox-encoded h264 streams.
const VIDEOTOOLBOX_UUID: &[u8; 16] = uuid::uuid!("47564adc-5c4c-433f-94ef-c5113cd143a8").as_bytes();

/// H264 data source. Can come directly from an "Annex B" format .h264 file or
/// from an MP4 file.
///
/// This should be as general purpose H264 file reader as possible.
///
/// Strand Camera specific features are supported if present: metadata at the
/// H264 stream start (UUID 0ba99cc7-f607-3851-b35e-8c7d8c04da0a) is parsed, as
/// are precision time stamps (specified by MISB ST 0604.3).
///
/// ## Timestamp handling:
///
/// ### Case 1, H264 in MP4
///
/// Tracks in MP4 files are composed of samples. Each sample has a presentation
/// timestamp (PTS), the time elapsed from the start. A sample contains multiple
/// (0, 1, 2 or more) H264 NAL units. A sample can contain zero, one, two or
/// more image frames of data. (There exist MP4 files in which a sample contains
/// zero NAL units and thus zero image frames as well as MP4 files in which a
/// single sample contains many NAL units and many image frames.) Samples also
/// carry a duration, which is informational to assist with playback. The
/// duration of frame N should be the PTS of frame N+1 - the PTS of frame N.
/// There seems to be a general assumption that samples should be equi-distant
/// in time and thus that the file has a constant frame rate, although I have
/// not found this in any specification.
///
/// ### Case 2, raw H264
///
/// A raw .h264 file, which is defined as the "Annex B format", (or simply the
/// H264 data inside an MP4 file) may have no explicit timing information in it.
/// Alternatively, for example with the `timing_info_present_flag` in the VUI
/// parameters, timing information may be present. So far we ignore these flags.
/// In this case, the timestamp data for the frame is simply returned as a
/// fraction of complete (in the interval from 0.0 to 1.0).
///
/// ### Case 3, H264 files with `MISPmicrosectime` supplemental enhancement information
///
/// H264 can contain additional NAL units, called supplemental enhancement
/// information (SEI) which is ignored by decoders but can provide additional
/// information such as metadata at the start of H264 data and per-frame
/// timestamps as specified in MISB ST 0604.3.
pub struct H264Source<H: SeekableH264Source> {
    seekable_h264_source: H,
    /// For every NAL unit, the coordinates in the source to read it.
    nal_locations: Vec<H::NalLocation>,
    /// timestamps from MP4 files, one per MP4 sample (which we assume to be one per frame)
    mp4_pts: Option<Vec<std::time::Duration>>,
    frame_time_info: Vec<FrameTimeInfo>,
    pub h264_metadata: Option<H264Metadata>,
    frame0_precision_time: Option<chrono::DateTime<chrono::FixedOffset>>,
    frame0_frameinfo: Option<FrameInfo>,
    width: u32,
    height: u32,
    do_decode_h264: bool,
    timestamp_source: Option<crate::TimestampSource>,
    has_timestamps: bool,
    srt_data: Option<SrtData>,
    /// Per-sample decode duration + composition offset (MP4 sources only), so a
    /// re-mux can preserve the source's timing (stts + ctts) verbatim.
    mp4_sample_timing: Option<Vec<Mp4SampleTiming>>,
    /// For SRT timestamps: the display rank of each decode-order frame, i.e.
    /// `srt_display_rank[decode_index]` is the SRT stanza index (stanzas are in
    /// presentation order) for that frame. `None` when SRT timestamps or
    /// per-sample PTS are unavailable, in which case stanzas are consumed
    /// sequentially.
    srt_display_rank: Option<Vec<usize>>,
    /// The display (presentation) rank of each decode-order frame:
    /// `presentation_rank[decode_index]` is that frame's position in display
    /// order. Derived from per-sample PTS (`mp4_pts`) when present, else from
    /// the bitstream POC. `None` when presentation order cannot be recovered
    /// (no PTS and no decodable POC), which makes [`Self::presentation_order_iter`]
    /// fail loudly.
    presentation_rank: Option<Vec<usize>>,
    /// Whether each decode-order frame is an IDR (starts a new coded video
    /// sequence). Used to bound the presentation-order reorder buffer to one
    /// coded video sequence.
    is_idr: Vec<bool>,
    /// Set when an SRT-file timestamp source ran out of usable stanzas
    /// before covering every frame, and the source was truncated to the last
    /// complete GOP a stanza is available for every frame of.
    srt_truncation: Option<SrtTruncation>,
    average_fps: Option<f64>,
    /// Chroma subsampling of the stream, read from the first SPS. The built-in
    /// OpenH264 decoder only handles 4:2:0, so on a decode failure this is used
    /// to explain that a 4:2:2 / 4:4:4 stream (e.g. from a macOS / FaceTime
    /// camera) is the likely cause, rather than surfacing only OpenH264's opaque
    /// native error code. Only read when compiled with the `openh264` feature.
    #[cfg_attr(not(feature = "openh264"), allow(dead_code))]
    chroma_format: h264_reader::nal::sps::ChromaFormat,
    /// Human-readable H.264 profile of the stream (e.g. "High444"), read from
    /// the first SPS. Used only for diagnostics.
    #[cfg_attr(not(feature = "openh264"), allow(dead_code))]
    profile: String,
}

impl<H: SeekableH264Source> H264Source<H> {
    pub fn as_seekable_h264_source(&self) -> &H {
        &self.seekable_h264_source
    }

    /// Per-sample timing (decode duration + composition offset) for MP4
    /// sources, indexed by frame number (see [`FrameData::idx`]). `None` for
    /// non-MP4 sources.
    pub fn mp4_sample_timing(&self) -> Option<&[Mp4SampleTiming]> {
        self.mp4_sample_timing.as_deref()
    }

    /// Set when an SRT-file timestamp source ran out of usable stanzas
    /// before covering every frame in the container; `None` otherwise.
    pub fn srt_truncation(&self) -> Option<&SrtTruncation> {
        self.srt_truncation.as_ref()
    }

    fn create_iter_unchecked<'a>(
        &'a mut self,
        frame_idx: usize,
    ) -> Box<dyn Iterator<Item = Result<FrameData>> + 'a> {
        let openh264_decoder_state = if self.do_decode_h264 {
            Some(crate::opt_openh264_decoder::new_stream_decoder().unwrap())
        } else {
            None
        };
        // Invert `presentation_rank` to get the decode index of each
        // display-order position, used to pair decoder output pictures (which
        // emerge in display order) with the input frames they decode.
        let display_order = self.presentation_rank.as_ref().map(|rank| {
            let mut order = vec![0usize; rank.len()];
            for (decode_idx, &r) in rank.iter().enumerate() {
                order[r] = decode_idx;
            }
            order
        });
        Box::new(RawH264Iter {
            parent: self,
            frame_idx,
            openh264_decoder_state,
            display_order,
            n_pictures_out: 0,
            pending_meta: std::collections::HashMap::new(),
            decoded_ready: std::collections::BTreeMap::new(),
            next_emit: frame_idx,
            flushed: false,
            poisoned: false,
        })
    }
}

/// Timing information for a frame of video.
pub struct FrameTimeInfo {
    /// The NAL unit location.
    ///
    /// This is an index into the slice &[SeekableH264Source::NalLocation]
    /// returned by [SeekableH264Source::nal_boundaries]. In the case of MP4
    /// files, each index corresponds to multiple NAL units.
    nal_location_index: usize,
    precise_timestamp: Option<DateTime<Utc>>,
    frameinfo: Option<FrameInfo>,
    /// Picture order count reconstructed from the bitstream, when available.
    /// `None` if the POC could not be determined (e.g. `pic_order_cnt_type 1`).
    poc: Option<i64>,
    /// Whether this frame is an IDR picture (starts a new coded video sequence).
    is_idr: bool,
}

impl<H: SeekableH264Source> FrameDataSource for H264Source<H> {
    fn width(&self) -> u32 {
        self.width
    }
    fn height(&self) -> u32 {
        self.height
    }
    fn camera_name(&self) -> Option<&str> {
        self.h264_metadata
            .as_ref()
            .and_then(|x| x.camera_name.as_deref())
    }
    fn gamma(&self) -> Option<f32> {
        self.h264_metadata.as_ref().and_then(|x| x.gamma)
    }
    fn frame0_time(&self) -> Option<chrono::DateTime<chrono::FixedOffset>> {
        match &self.timestamp_source {
            Some(TimestampSource::BestGuess) => unreachable!(),
            Some(TimestampSource::FixedFramerate) => {
                if let Some(t) = &self.frame0_precision_time {
                    Some(*t)
                } else {
                    self.frame0_frameinfo.as_ref().map(|fi| fi.recv.into())
                }
            }
            Some(TimestampSource::MispMicrosectime) => self.frame0_precision_time,
            Some(TimestampSource::FrameInfoRecvTime) | Some(TimestampSource::FrameInfoRtp) => {
                Some(self.frame0_frameinfo.as_ref().unwrap().recv.into())
            }
            Some(TimestampSource::Mp4Pts) | None => None,
            Some(TimestampSource::SrtFile) => self.srt_data.as_ref().map(|x| x.frame0_time()),
        }
    }
    fn average_framerate(&self) -> Option<f64> {
        self.average_fps
    }
    fn skip_n_frames(&mut self, n_frames: usize) -> Result<()> {
        if n_frames > 0 {
            return Err(Error::SkippingFramesNotSupported);
            // Doing so would require finding I frames and only skipping to
            // those (or decoding and interpolating a new I frame).
            // Also: caching SPS and PPS would be required.
            // We do this in the MKV reader, so we should use that
            // implementation for inspiration.
        }
        Ok(())
    }
    fn estimate_luminance_range(&mut self) -> Result<(u16, u16)> {
        Err(Error::NotImplemented("h264 luminance scanning"))
    }
    fn decode_order_iter<'a>(&'a mut self) -> Box<dyn Iterator<Item = Result<FrameData>> + 'a> {
        self.create_iter_unchecked(0)
    }
    fn presentation_order_iter<'a>(
        &'a mut self,
    ) -> Result<Box<dyn Iterator<Item = Result<FrameData>> + 'a>> {
        let rank = self.presentation_rank.clone().ok_or_else(|| {
            Error::H264Poc(
                "cannot recover presentation order: source has neither per-sample \
                 timestamps nor a decodable picture order count"
                    .to_string(),
            )
        })?;
        let is_idr = self.is_idr.clone();
        let total = rank.len();
        // Decode/parse in decode order (required to feed a decoder and to
        // reconstruct POC correctly), then reorder into display order.
        let inner = self.create_iter_unchecked(0);
        Ok(Box::new(PresentationReorderIter {
            inner,
            rank,
            is_idr,
            total,
            pending: Vec::new(),
            ready: std::collections::VecDeque::new(),
            done: false,
        }))
    }
    fn timestamp_source(&self) -> &str {
        self.timestamp_source.as_str()
    }
    fn has_timestamps(&self) -> bool {
        self.has_timestamps
    }
    fn srt_truncation(&self) -> Option<SrtTruncation> {
        self.srt_truncation.clone()
    }
}

pub(crate) struct FromMp4Track {
    pub(crate) sequence_parameter_set: Vec<u8>,
    pub(crate) picture_parameter_set: Vec<u8>,
}

pub trait SeekRead: Seek + Read {}
impl<T> SeekRead for T where T: Seek + Read {}

pub trait SeekableH264Source {
    type NalLocation;
    fn nal_boundaries(&mut self) -> &[Self::NalLocation];
    /// Read multiple NAL units at the specified location
    fn read_nal_units_at_location(&mut self, location: &Self::NalLocation) -> Result<Vec<Vec<u8>>>;
    /// Read multiple NAL units at multiple specified locations
    fn read_nal_units_at_locations(
        &mut self,
        locations: &[Self::NalLocation],
    ) -> Result<Vec<Vec<u8>>> {
        let mut result = Vec::with_capacity(locations.len() * 3);
        for location in locations.iter() {
            let nal_units = self.read_nal_units_at_location(location)?;
            result.extend(nal_units);
        }
        Ok(result)
    }

    /// Return the first SPS
    fn first_sps(&self) -> Option<Vec<u8>>;
    /// Return the first PPS
    fn first_pps(&self) -> Option<Vec<u8>>;
}

#[derive(Debug, PartialEq, Clone)]
pub struct AnnexBLocation {
    pub(crate) start: u64,
    pub(crate) sz: usize,
}

pub struct H264AnnexBSource {
    inner: Box<dyn SeekRead + Send>,
    my_nal_boundaries: Vec<AnnexBLocation>,
}

impl H264AnnexBSource {
    pub fn from_file(fd: std::fs::File) -> Result<Self> {
        let inner = Box::new(BufReader::new(fd));
        Self::from_readseek(inner)
    }
    pub fn from_readseek(mut inner: Box<dyn SeekRead + Send>) -> Result<Self> {
        inner.seek(std::io::SeekFrom::Start(0))?;
        let my_nal_boundaries = crate::h264_annexb_splitter::find_nals(&mut inner)?;
        inner.seek(std::io::SeekFrom::Start(0))?;
        Ok(Self {
            inner,
            my_nal_boundaries,
        })
    }
}

impl SeekableH264Source for H264AnnexBSource {
    type NalLocation = AnnexBLocation;
    fn nal_boundaries(&mut self) -> &[Self::NalLocation] {
        &self.my_nal_boundaries
    }
    fn read_nal_units_at_location(&mut self, location: &Self::NalLocation) -> Result<Vec<Vec<u8>>> {
        self.inner.seek(std::io::SeekFrom::Start(location.start))?;
        let mut buf = vec![0u8; location.sz];
        self.inner.read_exact(&mut buf)?;
        Ok(vec![buf])
    }

    fn first_sps(&self) -> Option<Vec<u8>> {
        None
    }
    fn first_pps(&self) -> Option<Vec<u8>> {
        None
    }
}

impl<H> H264Source<H>
where
    H: SeekableH264Source,
    <H as SeekableH264Source>::NalLocation: Clone,
{
    #[expect(
        clippy::too_many_arguments,
        reason = "we grudgingly accept this ugliness"
    )]
    pub(crate) fn from_seekable_h264_source_with_timestamp_source(
        mut seekable_h264_source: H,
        do_decode_h264: bool,
        mut mp4_pts: Option<Vec<std::time::Duration>>,
        mut mp4_sample_timing: Option<Vec<Mp4SampleTiming>>,
        data_from_mp4_track: Option<FromMp4Track>,
        timestamp_source: crate::TimestampSource,
        srt_file_path: Option<std::path::PathBuf>,
        show_progress: bool,
        mut preparser: Option<Box<dyn H264Preparser>>,
    ) -> Result<Self> {
        let nal_locations: Vec<H::NalLocation> = seekable_h264_source.nal_boundaries().to_vec();

        let mut parsing_ctx = H264ParsingContext::default();

        // open SRT file
        if timestamp_source == crate::TimestampSource::SrtFile && srt_file_path.is_none() {
            return Err(Error::NoSrtPathGiven);
        }

        let mut srt_data = if let Some(srt_file_path) = srt_file_path {
            let outcome = srt_reader::read_srt_file(&srt_file_path)?;
            if outcome.stanzas.is_empty() {
                return Err(Error::SrtParseError {
                    path: srt_file_path,
                    line: outcome.truncated_at_line.unwrap_or(1),
                });
            }
            let frame0_time = SrtData::parse_time(&outcome.stanzas[0]);
            Some(SrtData {
                stanzas: outcome.stanzas,
                idx: 0,
                frame0_time,
                truncated_at_line: outcome.truncated_at_line,
            })
        } else {
            None
        };

        // Use data from container if present
        if let Some(dfc) = data_from_mp4_track {
            tracing::debug!("Using SPS and PPS data from mp4 track.");
            {
                // SPS
                let sps_nal = RefNal::new(&dfc.sequence_parameter_set, &[], true);
                if sps_nal.header().unwrap().nal_unit_type() != UnitType::SeqParameterSet {
                    return Err(Error::ExpectedSpsNotFound);
                }

                let isps =
                    h264_reader::nal::sps::SeqParameterSet::from_bits(sps_nal.rbsp_bits()).unwrap();
                if let Some(preparser) = preparser.as_mut() {
                    preparser
                        .put_seq_param_set(&sps_nal)
                        .map_err(Error::PreParserError)?;
                }
                parsing_ctx.put_seq_param_set(isps);
            }

            {
                // PPS
                let pps_nal = RefNal::new(&dfc.picture_parameter_set, &[], true);
                if pps_nal.header().unwrap().nal_unit_type() != UnitType::PicParameterSet {
                    return Err(Error::ExpectedPpsNotFound);
                }

                let ipps = h264_reader::nal::pps::PicParameterSet::from_bits(
                    &parsing_ctx,
                    pps_nal.rbsp_bits(),
                )
                .unwrap();
                if let Some(preparser) = preparser.as_mut() {
                    preparser
                        .put_pic_param_set(&pps_nal)
                        .map_err(Error::PreParserError)?;
                }
                parsing_ctx.put_pic_param_set(ipps);
            }
        }

        // iterate through all NAL units.
        let timing_data = load_timing_data(
            &nal_locations,
            &mut seekable_h264_source,
            &mut parsing_ctx,
            show_progress,
            preparser,
        )?;

        let TimingData {
            mut frame_time_info,
            frame0_precision_time,
            frame0_frameinfo,
            h264_metadata,
            tz_offset,
        } = timing_data;

        let mut widthheight = None;
        let mut chroma_format = h264_reader::nal::sps::ChromaFormat::YUV420;
        let mut profile = "Unknown".to_string();
        for sps in parsing_ctx.sps() {
            if let Ok(wh) = sps.pixel_dimensions() {
                widthheight = Some(wh);
            }
            chroma_format = sps.chroma_info.chroma_format;
            profile = format!("{:?}", sps.profile());
        }

        let (width, height) = widthheight.ok_or_else(|| crate::Error::ExpectedSpsNotFound)?;

        let timezone = tz_offset.unwrap_or_else(|| chrono::FixedOffset::east_opt(0).unwrap());

        let frame0_precision_time = frame0_precision_time
            .as_ref()
            .map(|dt| dt.with_timezone(&timezone));

        let (timestamp_source, has_timestamps) = match timestamp_source {
            crate::TimestampSource::BestGuess => {
                if frame0_precision_time.is_some() {
                    (Some(crate::TimestampSource::MispMicrosectime), true)
                } else if frame0_frameinfo.is_some() {
                    (Some(crate::TimestampSource::FrameInfoRtp), true)
                } else if mp4_pts.is_some() {
                    (Some(crate::TimestampSource::Mp4Pts), true)
                } else {
                    (None, false)
                }
            }
            crate::TimestampSource::FixedFramerate => (Some(timestamp_source), true),
            crate::TimestampSource::FrameInfoRecvTime | crate::TimestampSource::FrameInfoRtp => {
                if frame0_frameinfo.is_none() {
                    return Err(Error::H264TimestampError(
                        "Requested timestamp that requires FrameInfo, but this information is not present."
                            .into(),
                    ));
                }
                (Some(timestamp_source), true)
            }
            crate::TimestampSource::MispMicrosectime => {
                if frame0_precision_time.is_none() {
                    return Err(Error::H264TimestampError(
                        "Requested timestamp source MispMicrosectime, but frame0_precision_time not present."
                            .into(),
                    ));
                }
                (Some(timestamp_source), true)
            }
            crate::TimestampSource::Mp4Pts => {
                if mp4_pts.is_none() {
                    return Err(Error::H264TimestampError(
                        "Requested timestamp source Mp4Pts, but MP4 PTS not present.".into(),
                    ));
                }
                (Some(timestamp_source), true)
            }
            crate::TimestampSource::SrtFile => (Some(timestamp_source), true),
        };

        if let Some(mp4_pts) = mp4_pts.as_ref()
            && mp4_pts.len() != frame_time_info.len()
        {
            return Err(Error::H264TimestampError(format!(
                "We have {} frames of MP4 PTS timing, but computed {} frames of video.",
                mp4_pts.len(),
                frame_time_info.len()
            )));
        }
        // Precompute the display (presentation) rank of each decode-order frame.
        // Ranking by per-sample PTS is robust to a constant encoder composition
        // delay (unlike matching absolute PTS values against stanza timecodes);
        // when there is no PTS we fall back to the bitstream POC, keyed within
        // each coded video sequence so GOPs stay in order.
        let mut presentation_rank = compute_presentation_rank(mp4_pts.as_deref(), &frame_time_info);

        let mut is_idr: Vec<bool> = frame_time_info.iter().map(|fti| fti.is_idr).collect();

        // If the SRT ran out before covering every frame -- either it was
        // malformed partway through, or it simply has fewer stanzas than the
        // video has frames -- salvage what we can rather than failing
        // outright: keep only whole GOPs fully covered by a usable stanza,
        // dropping the incomplete tail. SRT stanzas are handed to frames by
        // *presentation* rank, so a partial trailing GOP can't be resolved
        // for reordered (B-frame) streams; rounding down to the last
        // complete GOP boundary keeps every kept frame's timing intact.
        let mut srt_truncation = None;
        if let Some(srt) = srt_data.as_mut() {
            let total_frames = frame_time_info.len();
            let usable_stanzas = srt.stanzas.len();
            if usable_stanzas < total_frames {
                let rank = presentation_rank.as_ref().ok_or_else(|| {
                    Error::H264TimestampError(
                        "SRT file has fewer entries than the video has frames, and \
                         presentation order cannot be recovered to safely truncate \
                         (no per-sample PTS and no decodable picture order count)."
                            .to_string(),
                    )
                })?;
                let mut gop_starts: Vec<usize> = std::iter::once(0)
                    .chain((0..total_frames).filter(|&i| is_idr[i]).map(|i| rank[i]))
                    .collect();
                gop_starts.sort_unstable();
                gop_starts.dedup();
                let kept_frames = gop_starts
                    .into_iter()
                    .filter(|&g| g <= usable_stanzas)
                    .max()
                    .unwrap_or(0);

                if kept_frames == 0 {
                    return Err(Error::H264TimestampError(format!(
                        "SRT file only has usable timestamps for {usable_stanzas} of \
                         {total_frames} frames, not even one complete group of pictures."
                    )));
                }

                let keep: Vec<bool> = (0..total_frames).map(|i| rank[i] < kept_frames).collect();
                retain_by_mask(&mut frame_time_info, &keep);
                if let Some(pts) = mp4_pts.as_mut() {
                    retain_by_mask(pts, &keep);
                }
                if let Some(timing) = mp4_sample_timing.as_mut() {
                    retain_by_mask(timing, &keep);
                }
                retain_by_mask(&mut is_idr, &keep);
                if let Some(rank) = presentation_rank.as_mut() {
                    retain_by_mask(rank, &keep);
                }
                srt.stanzas.truncate(kept_frames);

                srt_truncation = Some(SrtTruncation {
                    kept_frames,
                    total_frames,
                    usable_stanzas,
                    malformed_at_line: srt.truncated_at_line,
                });
            }
        }

        let average_fps = calc_avg_fps(&frame_time_info[..]);

        // Rescale the source's per-sample timing (stts/ctts) to the real
        // capture cadence given by the SRT. The intermediate encoder uses a
        // fixed nominal framerate unrelated to the true capture rate, so its
        // absolute durations are meaningless; only its *relative* reorder
        // structure (the ratio of composition offset to frame duration) is.
        // Scaling by real_span/source_span fixes the playback rate while
        // preserving valid decode/composition ordering (and cancels any
        // timescale discrepancy in the source durations).
        let mp4_sample_timing = match (mp4_sample_timing, srt_data.as_ref()) {
            (Some(mut timing), Some(srt)) if timing.len() >= 2 => {
                let source_span: f64 = timing.iter().map(|t| t.decode_duration.as_secs_f64()).sum();
                let real_span = srt.span().map(|d| d.as_secs_f64()).unwrap_or(0.0);
                if source_span > 0.0 && real_span > 0.0 {
                    let scale = real_span / source_span;
                    for t in timing.iter_mut() {
                        t.decode_duration = t.decode_duration.mul_f64(scale);
                        let off_ns = t.composition_offset.num_nanoseconds().unwrap_or(0) as f64;
                        t.composition_offset =
                            chrono::Duration::nanoseconds((off_ns * scale) as i64);
                    }
                }
                Some(timing)
            }
            (other, _) => other,
        };

        // SRT stanzas are in presentation order, so pair them with reordered
        // (B-frame) streams by display rank. Fall back to sequential consumption
        // when presentation order is unknown.
        let srt_display_rank = if srt_data.is_some() {
            presentation_rank.clone()
        } else {
            None
        };

        Ok(Self {
            seekable_h264_source,
            nal_locations,
            mp4_pts,
            mp4_sample_timing,
            srt_display_rank,
            presentation_rank,
            is_idr,
            frame_time_info,
            h264_metadata,
            frame0_precision_time,
            frame0_frameinfo,
            width,
            height,
            do_decode_h264,
            timestamp_source,
            has_timestamps,
            srt_data,
            srt_truncation,
            average_fps,
            chroma_format,
            profile,
        })
    }
}

/// Keep only the elements of `items` at positions where `keep` is `true`,
/// preserving order. `keep` must be the same length as `items`.
fn retain_by_mask<T>(items: &mut Vec<T>, keep: &[bool]) {
    let mut keep = keep.iter();
    items.retain(|_| *keep.next().unwrap());
}

/// Compute the display (presentation) rank of each decode-order frame.
///
/// `presentation_rank[decode_index]` is that frame's position in presentation
/// order. Ranking uses per-sample PTS when available (`mp4_pts`), else the
/// bitstream POC keyed by coded video sequence (so GOPs stay ordered). Returns
/// `None` when neither signal is available for every frame, i.e. presentation
/// order cannot be recovered.
fn compute_presentation_rank(
    mp4_pts: Option<&[std::time::Duration]>,
    frame_time_info: &[FrameTimeInfo],
) -> Option<Vec<usize>> {
    let n = frame_time_info.len();
    if n == 0 {
        return Some(Vec::new());
    }
    // Display sort key per decode index: (coded-video-sequence index, tie-break
    // within that sequence). For PTS the whole stream is one global ordering; for
    // POC we bump the sequence index at each IDR so GOPs stay contiguous and in
    // order even though POC resets to 0 at every IDR.
    let keys: Vec<(i64, i64)> = if let Some(pts) = mp4_pts {
        pts.iter().map(|d| (0i64, d.as_nanos() as i64)).collect()
    } else {
        if !frame_time_info.iter().all(|fti| fti.poc.is_some()) {
            return None;
        }
        let mut cvs = 0i64;
        let mut keys = Vec::with_capacity(n);
        for (i, fti) in frame_time_info.iter().enumerate() {
            if fti.is_idr && i != 0 {
                cvs += 1;
            }
            keys.push((cvs, fti.poc.unwrap()));
        }
        keys
    };
    let mut order: Vec<usize> = (0..n).collect();
    order.sort_by_key(|&i| keys[i]);
    let mut rank = vec![0usize; n];
    for (display_rank, &decode_index) in order.iter().enumerate() {
        rank[decode_index] = display_rank;
    }
    Some(rank)
}

fn calc_avg_fps(fti: &[FrameTimeInfo]) -> Option<f64> {
    if fti.len() <= 1 {
        return None;
    }
    let frames = (fti.len() - 1) as f64;
    if let Some(t0) = fti[0].precise_timestamp {
        // prefer precise_timestamps
        let tend = fti[fti.len() - 1].precise_timestamp.unwrap();
        let secs = (tend - t0).to_std().unwrap().as_secs_f64();
        Some(frames / secs)
    } else if let Some(fi) = &fti[0].frameinfo {
        // else use FrameInfo
        let t0: chrono::DateTime<chrono::Utc> = fi.recv.into();
        let tend: chrono::DateTime<chrono::Utc> =
            fti[fti.len() - 1].frameinfo.as_ref().unwrap().recv.into();
        let secs = (tend - t0).to_std().unwrap().as_secs_f64();
        Some(frames / secs)
    } else {
        // final resort
        None
    }
}

struct TimingData {
    frame_time_info: Vec<FrameTimeInfo>,
    frame0_precision_time: Option<DateTime<Utc>>,
    frame0_frameinfo: Option<FrameInfo>,
    h264_metadata: Option<H264Metadata>,
    tz_offset: Option<FixedOffset>,
}

fn load_timing_data<H>(
    nal_locations: &[H::NalLocation],
    seekable_h264_source: &mut H,
    parsing_ctx: &mut H264ParsingContext,
    show_progress: bool,
    mut preparser: Option<Box<dyn H264Preparser>>,
) -> Result<TimingData>
where
    H: SeekableH264Source,
    <H as SeekableH264Source>::NalLocation: Clone,
{
    let mut scratch = Vec::new();

    let mut tz_offset = None;

    let mut h264_metadata = None;

    // One entry per frame. Can refer to multiple multiple NAL units, e.g.
    // in MP4 files where a frame is an MP4 sample containing multiple NAL
    // units.
    let mut frame_time_info = Vec::new();

    let mut frame0_precision_time = None;
    let mut frame0_frameinfo = None;

    tracing::debug!(
        "Iterating through NAL units at {} locations to load timing data.",
        nal_locations.len()
    );

    let mut pb = if show_progress {
        // Custom progress bar with space at right end to prevent obscuring last
        // digit with cursor.
        let style = indicatif::ProgressStyle::with_template(
            "Iterating NAL units in h264 source {wide_bar} {pos}/{len} ETA: {eta} ",
        )
        .unwrap();
        Some(indicatif::ProgressBar::new(nal_locations.len().try_into().unwrap()).with_style(style))
    } else {
        None
    };

    if let Some(preparser) = preparser.as_mut() {
        preparser
            .set_num_positions(nal_locations.len())
            .map_err(Error::PreParserError)?;
    }
    // Cached value of MISP time data for the frame whose data is being accumulated.
    let mut precise_timestamp = None;
    // Cached value of frame number as we accumluate data.
    let mut next_frame_num = 0;

    // POC (picture order count) reconstruction. The strategy is fixed once the
    // first SPS is seen; `None` means the POC could not be determined (e.g.
    // `pic_order_cnt_type 1`), in which case per-frame `poc` stays `None`.
    let mut poc_strategy: Option<PocStrategy> = None;

    // Cached value of FrameInfo time data for the frame whose data is being
    // accumulated.
    let mut frameinfo = None;

    for (nal_location_index, nal_location) in nal_locations.iter().enumerate() {
        if let Some(preparser) = preparser.as_mut() {
            preparser
                .set_position(nal_location_index)
                .map_err(Error::PreParserError)?;
        }

        if let Some(pb) = pb.as_mut() {
            pb.set_position(nal_location_index.try_into().unwrap());
        }

        // Read all NAL units from this location. (For MP4 files, this means
        // read all NAL units from this sample. For H264 AnnexB files, this
        // will read a single NAL unit.)
        let nal_units = seekable_h264_source.read_nal_units_at_location(nal_location)?;
        for nal_unit in nal_units.iter() {
            // Note, there are multiple NAL units per `nal_location_index`
            // in MP4 files because in that case, `nal_location_index`
            // refers to the MP4 sample which has multiple NAL units.
            let nal = RefNal::new(nal_unit.as_slice(), &[], true);
            let nal_unit_type = nal.header().unwrap().nal_unit_type();
            tracing::trace!("NAL unit location index {nal_location_index}, {nal_unit_type:?}");
            match nal_unit_type {
                UnitType::SEI => {
                    if let Some(preparser) = preparser.as_mut() {
                        preparser
                            .put_sei_nalu(&nal)
                            .map_err(Error::PreParserError)?;
                    }
                    let mut sei_reader = SeiReader::from_rbsp_bytes(nal.rbsp_bytes(), &mut scratch);
                    loop {
                        match sei_reader.next() {
                            Ok(Some(sei_message)) => {
                                tracing::trace!("SEI payload type: {:?}", sei_message.payload_type);
                                match &sei_message.payload_type {
                                    HeaderType::UserDataUnregistered => {
                                        let udu = UserDataUnregistered::read(&sei_message)?;
                                        match udu.uuid {
                                            &H264_METADATA_UUID => {
                                                let md: H264Metadata =
                                                    serde_json::from_slice(udu.payload)?;
                                                if md.version != H264_METADATA_VERSION {
                                                    return Err(Error::H264Error(
                                                        "unexpected version in h264 metadata",
                                                    ));
                                                }
                                                if h264_metadata.is_some() {
                                                    return Err(Error::H264Error(
                                                        "multiple SEI messages, but expected exactly one",
                                                    ));
                                                }

                                                tracing::debug!("Found H264_METADATA_UUID: {md:?}");
                                                tz_offset = Some(*md.creation_time.offset());
                                                h264_metadata = Some(md);
                                            }
                                            X264_UUID => {
                                                let payload_str =
                                                    String::from_utf8_lossy(udu.payload);
                                                tracing::trace!(
                                                    "Ignoring SEI UserDataUnregistered x264 payload: {}",
                                                    payload_str,
                                                );
                                            }
                                            VIDEOTOOLBOX_UUID => {
                                                tracing::trace!(
                                                    "Ignoring SEI UserDataUnregistered from videotoolbox."
                                                );
                                            }
                                            b"MISPmicrosectime" => {
                                                let ts = parse_precision_time(udu.payload)?;
                                                tracing::trace!("Found MISPmicrosectime: {ts:?}");
                                                precise_timestamp = Some(ts);
                                                if next_frame_num == 0 {
                                                    frame0_precision_time = Some(ts);
                                                }
                                            }
                                            b"strawlab.org/89H" => {
                                                let fi: FrameInfo =
                                                    serde_json::from_slice(udu.payload)?;
                                                tracing::trace!("Found 89H FrameInfo: {fi:?}");
                                                frameinfo = Some(fi.clone());
                                                if next_frame_num == 0 {
                                                    frame0_frameinfo = Some(fi);
                                                }
                                            }
                                            _uuid => {
                                                tracing::trace!(
                                                    "Ignoring SEI UserDataUnregistered uuid: {}",
                                                    uuid::Uuid::from_bytes(*udu.uuid).to_string(),
                                                );
                                            }
                                        }
                                    }
                                    _ => {
                                        // handle other SEI types.
                                    }
                                }
                            }
                            Ok(None) => {
                                break;
                            }
                            Err(BitReaderError::ReaderErrorFor(what, io_err)) => {
                                tracing::error!(
                                    "Ignoring error when reading SEI NAL unit {what}: {io_err:?}"
                                );
                                // We do not process this NAL unit but nor do we
                                // propagate the error further. FFMPEG also
                                // skips this error except writing "SEI type 5
                                // size X truncated at Y" where Y is less than
                                // X.
                            }
                            Err(e) => {
                                return Err(Error::H264Nal {
                                    nal_location_index,
                                    e,
                                });
                            }
                        }
                    }
                }
                UnitType::SeqParameterSet => {
                    let isps =
                        h264_reader::nal::sps::SeqParameterSet::from_bits(nal.rbsp_bits()).unwrap();
                    if let Some(preparser) = preparser.as_mut() {
                        preparser
                            .put_seq_param_set(&nal)
                            .map_err(Error::PreParserError)?;
                    }
                    // Fix the POC strategy from the first SPS. `strategy_from_sps`
                    // errors on the unsupported `pic_order_cnt_type 1`; treat that
                    // as "POC unavailable" rather than a hard failure.
                    if poc_strategy.is_none() {
                        poc_strategy = h264_poc::strategy_from_sps(&isps).ok();
                    }
                    parsing_ctx.put_seq_param_set(isps);
                }
                UnitType::PicParameterSet => {
                    match h264_reader::nal::pps::PicParameterSet::from_bits(
                        parsing_ctx,
                        nal.rbsp_bits(),
                    ) {
                        Ok(ipps) => {
                            if let Some(preparser) = preparser.as_mut() {
                                preparser
                                    .put_pic_param_set(&nal)
                                    .map_err(Error::PreParserError)?;
                            }
                            parsing_ctx.put_pic_param_set(ipps);
                        }
                        Err(h264_reader::nal::pps::PpsError::BadPicParamSetId(
                            h264_reader::nal::pps::PicParamSetIdError::IdTooLarge(_id),
                        )) => {
                            // While this is open, ignore the error.
                            // https://github.com/dholroyd/h264-reader/issues/56
                        }
                        Err(e) => {
                            return Err(Error::H264Pps(format!("reading PPS: {e:?}")));
                        }
                    }
                }
                UnitType::SliceLayerWithoutPartitioningIdr
                | UnitType::SliceLayerWithoutPartitioningNonIdr => {
                    let is_i_frame = nal_unit_type == UnitType::SliceLayerWithoutPartitioningIdr;
                    if let Some(preparser) = preparser.as_mut() {
                        preparser
                            .put_slice_layer_nalu(&nal, is_i_frame)
                            .map_err(Error::PreParserError)?;
                    }
                    // Reconstruct this frame's picture order count from the
                    // bitstream. Advancing the strategy is stateful and must
                    // happen once per frame in decode order.
                    let poc = poc_strategy.as_mut().and_then(|strategy| {
                        h264_poc::advance_poc(strategy, parsing_ctx, std::slice::from_ref(nal_unit))
                            .ok()
                    });
                    // The NAL unit with the video frames comes after the
                    // timing into NAL unit(s) so we gather them now.
                    frame_time_info.push(FrameTimeInfo {
                        nal_location_index,
                        precise_timestamp,
                        frameinfo,
                        poc,
                        is_idr: is_i_frame,
                    });
                    // Reset temporary values.
                    precise_timestamp = None;
                    frameinfo = None;
                    next_frame_num += 1;
                }
                _nal_unit_type => {}
            }
        }
    }

    if let Some(pb) = pb.as_mut() {
        pb.finish_and_clear();
    }

    tracing::debug!("Done iterating through all NAL units.");

    Ok(TimingData {
        frame_time_info,
        frame0_precision_time,
        frame0_frameinfo,
        h264_metadata,
        tz_offset,
    })
}

struct RawH264Iter<'parent, H: SeekableH264Source> {
    parent: &'parent mut H264Source<H>,
    /// frame index (not NAL unit index)
    frame_idx: usize,
    openh264_decoder_state: Option<crate::opt_openh264_decoder::DecoderType>,
    /// `display_order[k]` is the decode index of the k-th frame in display
    /// order (the inverse of `parent.presentation_rank`). The decoder outputs
    /// pictures in display order, so this pairs each output picture with the
    /// input frame it decodes. `None` when presentation order is unknown, in
    /// which case output order is assumed to equal decode order (true for
    /// streams without B-frames). Only used when decoding.
    display_order: Option<Vec<usize>>,
    /// Number of pictures the decoder has output so far, i.e. the display rank
    /// of the next picture it will output. Only used when decoding.
    n_pictures_out: usize,
    /// Metadata of frames already fed to the decoder whose picture has not yet
    /// been output, keyed by decode index. Bounded by the stream's reorder
    /// depth (the decoded picture buffer, at most 16 frames). Only used when
    /// decoding.
    pending_meta: std::collections::HashMap<usize, PendingMeta>,
    /// Decoded frames awaiting emission in decode order, keyed by decode
    /// index. Bounded by the stream's reorder depth. Only used when decoding.
    decoded_ready: std::collections::BTreeMap<usize, FrameData>,
    /// Decode index of the next frame to emit. Only used when decoding.
    next_emit: usize,
    /// Whether the end-of-stream drain of the decoder has run.
    flushed: bool,
    /// Set after a decode error: the picture↔frame pairing may be lost, so
    /// stop iterating rather than emit frames with wrong timestamps.
    poisoned: bool,
}

/// Metadata of an input frame fed to the decoder, held until the decoder
/// outputs the corresponding picture.
#[cfg_attr(not(feature = "openh264"), allow(dead_code))]
struct PendingMeta {
    timestamp: Timestamp,
    poc: Option<i64>,
    buf_len: usize,
}

/// One frame read from the source in decode order: its raw NAL units plus
/// timestamp and POC metadata.
struct InputFrame {
    nal_units: Vec<Vec<u8>>,
    timestamp: Timestamp,
    poc: Option<i64>,
    is_idr: bool,
}

impl<H: SeekableH264Source> RawH264Iter<'_, H> {
    /// Read input frame `frame_number` (decode order).
    fn read_input_frame(&mut self, frame_number: usize) -> Result<InputFrame> {
        let nti = &self.parent.frame_time_info[frame_number];
        // Slice of all NAL units belonging to this frame: everything after
        // the previous frame's slice NAL up to and including this frame's.
        // `nal_location_index` is indexed per NAL unit (Annex B) or per
        // sample (MP4); for MP4 each frame is one location so this reduces
        // to `[frame_number..=frame_number]`, but for Annex B a frame spans
        // several NAL units (SEI/SPS/PPS + slice), so we must start after
        // the previous slice rather than at `frame_number`.
        let start = if frame_number == 0 {
            0
        } else {
            self.parent.frame_time_info[frame_number - 1].nal_location_index + 1
        };
        let nal_locations = &self.parent.nal_locations[start..=(nti.nal_location_index)];
        let mp4_pts = self.parent.mp4_pts.as_ref().map(|x| x[frame_number]); // one per mp4 sample
        let fraction_done = frame_number as f32 / self.parent.nal_locations.len() as f32;

        let frame_timestamp = match self.parent.timestamp_source {
            Some(TimestampSource::BestGuess) => unreachable!(),
            Some(TimestampSource::MispMicrosectime) => {
                let f0 = self.parent.frame0_precision_time.as_ref().unwrap();
                Timestamp::Duration(
                    nti.precise_timestamp
                        .unwrap()
                        .signed_duration_since(*f0)
                        .to_std()
                        .unwrap(),
                )
            }
            Some(TimestampSource::FrameInfoRecvTime) => {
                let t0 = self.parent.frame0_frameinfo.as_ref().unwrap().recv;
                let t0: chrono::DateTime<chrono::Utc> = t0.into();
                let this_frame: chrono::DateTime<chrono::Utc> =
                    nti.frameinfo.as_ref().unwrap().recv.into();
                Timestamp::Duration(this_frame.signed_duration_since(t0).to_std().unwrap())
            }
            Some(TimestampSource::FrameInfoRtp) => {
                let fi0 = self.parent.frame0_frameinfo.as_ref().unwrap();
                let rtp0 = fi0.rtp;
                let rtp_now = nti.frameinfo.as_ref().unwrap().rtp;
                let rtp_dur = rtp_now.wrapping_sub(rtp0);
                let rtp_dur_secs = rtp_dur as f64 / 90000.0; // nominally 90 kHz
                Timestamp::Duration(std::time::Duration::from_secs_f64(rtp_dur_secs))
            }
            Some(TimestampSource::FixedFramerate) => {
                let dur_secs = nti.nal_location_index as f64 / self.parent.average_fps.unwrap();
                Timestamp::Duration(std::time::Duration::from_secs_f64(dur_secs))
            }
            Some(TimestampSource::Mp4Pts) => Timestamp::Duration(mp4_pts.unwrap()),
            Some(TimestampSource::SrtFile) => {
                // Pair this decode-order frame with the SRT stanza at its
                // display rank, so B-frame reordering (decode order !=
                // presentation order) does not misassign timestamps. For raw
                // Annex B (no per-sample PTS) fall back to sequential order.
                let rank_idx = self
                    .parent
                    .srt_display_rank
                    .as_ref()
                    .map(|rank| rank[frame_number]);
                let srt_data = self.parent.srt_data.as_mut().unwrap();
                let pts = match rank_idx {
                    Some(idx) => srt_data.time_at(idx).unwrap(),
                    None => srt_data.next_pts().unwrap(),
                };
                Timestamp::Duration(pts)
            }
            None => Timestamp::Fraction(fraction_done),
        };

        let nal_units = self
            .parent
            .seekable_h264_source
            .read_nal_units_at_locations(nal_locations)?;

        Ok(InputFrame {
            nal_units,
            timestamp: frame_timestamp,
            poc: nti.poc,
            is_idr: nti.is_idr,
        })
    }

    /// Feed input frame `frame_number` to the decoder and, if a picture comes
    /// out, pair it with its input frame and store it in `decoded_ready`.
    fn feed_decoder(&mut self, frame_number: usize) -> Result<()> {
        let InputFrame {
            nal_units,
            timestamp,
            poc,
            is_idr,
        } = self.read_input_frame(frame_number)?;

        // MP4 stores the SPS/PPS in the `avcC` box rather than inline in
        // the sample data, so the NAL units for an IDR frame contain only
        // the coded slice. Without parameter sets OpenH264 fails with
        // `dsNoParamSets` (native error 16). Prepend the stream's SPS/PPS
        // ahead of each IDR so the decoder is configured. Annex B sources
        // carry SPS/PPS inline and return `None` here, so this is a no-op
        // for them (their slice already includes the parameter sets).
        let annex_b = if is_idr {
            let mut prefix = Vec::with_capacity(nal_units.len() + 2);
            prefix.extend(self.parent.seekable_h264_source.first_sps());
            prefix.extend(self.parent.seekable_h264_source.first_pps());
            prefix.extend_from_slice(nal_units.as_slice());
            copy_nalus_to_annex_b(&prefix)
        } else {
            copy_nalus_to_annex_b(nal_units.as_slice())
        };

        let buf_len = nal_units.iter().map(|x| x.len()).sum();
        self.pending_meta.insert(
            frame_number,
            PendingMeta {
                timestamp,
                poc,
                buf_len,
            },
        );

        // Attempt the decode first, so that if OpenH264 ever gains
        // 4:2:2 / 4:4:4 support the stream simply works. Only when the
        // decode actually fails do we enrich its opaque native error
        // with the stream's chroma/profile as the likely cause (the
        // built-in decoder handles only 4:2:0).
        let decoder = self.openh264_decoder_state.as_mut().unwrap();
        let decode_result = decoder.decode(&annex_b[..]);
        #[cfg(feature = "openh264")]
        let decode_result = decode_result.map_err(|source| Error::H264DecodeFailed {
            frame: frame_number,
            hint: decode_failure_hint(self.parent.chroma_format, &self.parent.profile),
            source,
        });

        // A decode call returning no picture is not an error: the decoder
        // buffers frames to reorder streams with B-frames into display order,
        // and the picture comes out on a later call (or the end-of-stream
        // drain).
        if let Some(decoded_yuv) = decode_result? {
            accept_decoded_picture(
                decoded_yuv,
                self.display_order.as_deref(),
                &mut self.n_pictures_out,
                &mut self.pending_meta,
                &mut self.decoded_ready,
            )?;
        }
        Ok(())
    }

    /// Drain the pictures still buffered in the decoder at end of stream (for
    /// B-frame streams the decoder holds back frames to reorder them).
    fn drain_decoder(&mut self) -> Result<()> {
        let decoder = self.openh264_decoder_state.as_mut().unwrap();
        let pictures = decoder.flush_remaining();
        #[cfg(feature = "openh264")]
        let pictures = pictures.map_err(|source| Error::H264DecodeFailed {
            frame: self.next_emit,
            hint: decode_failure_hint(self.parent.chroma_format, &self.parent.profile),
            source,
        });
        for decoded_yuv in pictures? {
            accept_decoded_picture(
                decoded_yuv,
                self.display_order.as_deref(),
                &mut self.n_pictures_out,
                &mut self.pending_meta,
                &mut self.decoded_ready,
            )?;
        }
        Ok(())
    }

    /// `next()` for the decoding path: feed input frames (decode order) until
    /// the frame with the next decode index has been decoded, draining the
    /// decoder at end of stream.
    fn next_decoded(&mut self) -> Option<Result<FrameData>> {
        loop {
            if let Some(frame_data) = self.decoded_ready.remove(&self.next_emit) {
                self.next_emit += 1;
                return Some(Ok(frame_data));
            }
            if self.poisoned {
                return None;
            }
            if self.frame_idx < self.parent.frame_time_info.len() {
                let frame_number = self.frame_idx;
                self.frame_idx += 1;
                if let Err(e) = self.feed_decoder(frame_number) {
                    self.poisoned = true;
                    return Some(Err(e));
                }
            } else if !self.flushed {
                self.flushed = true;
                if let Err(e) = self.drain_decoder() {
                    self.poisoned = true;
                    return Some(Err(e));
                }
            } else {
                if !self.pending_meta.is_empty() {
                    self.poisoned = true;
                    return Some(Err(Error::H264Error(
                        "decoder produced fewer pictures than input frames",
                    )));
                }
                return None;
            }
        }
    }
}

impl<H: SeekableH264Source> Iterator for RawH264Iter<'_, H> {
    type Item = Result<FrameData>;
    fn next(&mut self) -> Option<Self::Item> {
        if self.openh264_decoder_state.is_some() {
            return self.next_decoded();
        }
        let frame_number = self.frame_idx;
        if frame_number >= self.parent.frame_time_info.len() {
            return None;
        }
        self.frame_idx += 1;

        Some(self.read_input_frame(frame_number).map(|input| {
            let buf_len = input.nal_units.iter().map(|x| x.len()).sum();
            let buf = EncodedH264 {
                data: H264EncodingVariant::RawEbsp(input.nal_units),
                has_precision_timestamp: self.parent.frame0_precision_time.is_some(),
            };
            FrameData {
                timestamp: input.timestamp,
                image: ImageData::EncodedH264(buf),
                buf_len,
                idx: frame_number,
                poc: input.poc,
            }
        }))
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let total = self.parent.frame_time_info.len();
        // The cursor advances past the end once the iterator is exhausted, so
        // saturate to avoid underflow. When decoding, frames are emitted by
        // `next_emit` (input frames can be consumed ahead of emission while
        // the decoder reorders); otherwise by `frame_idx`.
        let remaining = if self.openh264_decoder_state.is_some() {
            if self.poisoned {
                0
            } else {
                total.saturating_sub(self.next_emit)
            }
        } else {
            total.saturating_sub(self.frame_idx)
        };
        (remaining, Some(remaining))
    }
}

/// Pair a picture output by the decoder with the input frame it decodes and
/// queue the resulting [`FrameData`] for emission.
///
/// The decoder outputs pictures in display order, so the picture's display
/// rank (`n_pictures_out`) is mapped back to a decode index via
/// `display_order`.
#[cfg(feature = "openh264")]
fn accept_decoded_picture(
    decoded_yuv: openh264::decoder::DecodedYUV<'_>,
    display_order: Option<&[usize]>,
    n_pictures_out: &mut usize,
    pending_meta: &mut std::collections::HashMap<usize, PendingMeta>,
    decoded_ready: &mut std::collections::BTreeMap<usize, FrameData>,
) -> Result<()> {
    let display_rank = *n_pictures_out;
    *n_pictures_out += 1;
    let decode_idx = match display_order {
        Some(order) => *order.get(display_rank).ok_or(Error::H264Error(
            "decoder produced more pictures than input frames",
        ))?,
        None => display_rank,
    };
    let meta = pending_meta.remove(&decode_idx).ok_or(Error::H264Error(
        "decoder output picture does not correspond to a pending input frame",
    ))?;
    let frame_data = yuv2rgb(
        decoded_yuv,
        decode_idx,
        meta.poc,
        meta.buf_len,
        meta.timestamp,
    )?;
    decoded_ready.insert(decode_idx, frame_data);
    Ok(())
}

#[cfg(not(feature = "openh264"))]
fn accept_decoded_picture(
    _decoded_yuv: (),
    _display_order: Option<&[usize]>,
    _n_pictures_out: &mut usize,
    _pending_meta: &mut std::collections::HashMap<usize, PendingMeta>,
    _decoded_ready: &mut std::collections::BTreeMap<usize, FrameData>,
) -> Result<()> {
    Err(Error::H264Error("No H264 decoder support at compile time"))
}

/// Reorders a decode-order H.264 frame iterator into presentation (display)
/// order by buffering one coded video sequence at a time.
///
/// Frames arrive from `inner` in decode order. Each coded video sequence
/// (delimited by IDR pictures) is buffered in `pending`; when the next IDR
/// arrives (or the stream ends) the buffered frames are sorted by their display
/// rank and moved to `ready` for emission. Because coded video sequences are
/// contiguous in display order for the closed GOPs produced by strand-cam /
/// braid / ffmpeg, this yields globally correct presentation order with memory
/// bounded by one GOP.
///
/// Open-GOP streams (leading pictures following a non-IDR recovery point) are
/// not handled specially; the encode paths in this workspace use closed GOPs.
struct PresentationReorderIter<'a> {
    inner: Box<dyn Iterator<Item = Result<FrameData>> + 'a>,
    /// `rank[decode_idx]` is the frame's position in display order.
    rank: Vec<usize>,
    /// `is_idr[decode_idx]` marks the start of a coded video sequence.
    is_idr: Vec<bool>,
    /// Total number of frames, used to restamp fraction-done timestamps.
    total: usize,
    /// Frames of the current coded video sequence, buffered in decode order.
    pending: Vec<FrameData>,
    /// Frames flushed and ready to emit, already in display order.
    ready: std::collections::VecDeque<FrameData>,
    /// Whether `inner` has been exhausted (or errored).
    done: bool,
}

impl PresentationReorderIter<'_> {
    /// Sort the buffered coded video sequence by display rank and move it to
    /// `ready`, restamping fraction-done timestamps (which count decode
    /// position) to display position so they stay monotonic.
    fn flush(&mut self) {
        let rank = &self.rank;
        self.pending
            .sort_by_key(|f| rank.get(f.idx()).copied().unwrap_or(usize::MAX));
        let denom = self.total.max(1) as f32;
        for mut frame in self.pending.drain(..).collect::<Vec<_>>() {
            if let Timestamp::Fraction(_) = frame.timestamp {
                let pos = self.rank.get(frame.idx()).copied().unwrap_or(0);
                frame.timestamp = Timestamp::Fraction(pos as f32 / denom);
            }
            self.ready.push_back(frame);
        }
    }
}

impl Iterator for PresentationReorderIter<'_> {
    type Item = Result<FrameData>;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if let Some(frame) = self.ready.pop_front() {
                return Some(Ok(frame));
            }
            if self.done {
                return None;
            }
            match self.inner.next() {
                Some(Ok(frame)) => {
                    // A new coded video sequence begins: flush the previous one
                    // (now complete) before buffering this IDR.
                    if self.is_idr.get(frame.idx()).copied().unwrap_or(false)
                        && !self.pending.is_empty()
                    {
                        self.flush();
                    }
                    self.pending.push(frame);
                }
                Some(Err(e)) => {
                    self.done = true;
                    self.pending.clear();
                    return Some(Err(e));
                }
                None => {
                    self.done = true;
                    self.flush();
                }
            }
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        // Reordering neither adds nor drops frames, so the total is the inner
        // iterator's remaining count plus whatever we are currently holding.
        let (lo, hi) = self.inner.size_hint();
        let buffered = self.ready.len() + self.pending.len();
        (lo + buffered, hi.map(|h| h + buffered))
    }
}

/// Build a likely-cause hint appended to an OpenH264 decode failure. The
/// built-in decoder handles only 4:2:0 (YUV420) chroma, so a stream with any
/// other chroma subsampling is a likely cause. Names the unsupported chroma
/// the stream uses, the profile, and how to re-encode. Returns an empty
/// string when the stream is 4:2:0 (the failure lies elsewhere).
#[cfg(feature = "openh264")]
fn decode_failure_hint(chroma: h264_reader::nal::sps::ChromaFormat, profile: &str) -> String {
    use h264_reader::nal::sps::ChromaFormat::*;
    let feature = match chroma {
        YUV420 => return String::new(),
        Monochrome => "4:0:0 (monochrome) chroma subsampling".to_string(),
        YUV422 => "4:2:2 chroma subsampling".to_string(),
        YUV444 => "4:4:4 chroma subsampling".to_string(),
        Invalid(idc) => format!("unknown chroma subsampling (idc={idc})"),
    };
    format!(
        " This stream uses {feature} (profile {profile}), which the built-in OpenH264 decoder \
         does not support — it decodes only 4:2:0 (YUV420). Re-encode first, \
         e.g. `ffmpeg -i INPUT -c:v libx264 -pix_fmt yuv420p OUTPUT.mp4`."
    )
}

#[cfg(feature = "openh264")]
fn yuv2rgb(
    decoded_yuv: openh264::decoder::DecodedYUV<'_>,
    frame_number: usize,
    poc: Option<i64>,
    buf_len: usize,
    timestamp: Timestamp,
) -> Result<FrameData> {
    use openh264::formats::YUVSource;
    let dim = decoded_yuv.dimensions();

    let stride = dim.0 * 3;
    let mut image_data = vec![0u8; stride * dim.1];
    decoded_yuv.write_rgb8(&mut image_data);

    let dynamic_frame = strand_dynamic_frame::DynamicFrameOwned::from_static(
        OImage::<machine_vision_formats::pixel_format::RGB8>::new(
            dim.0.try_into().unwrap(),
            dim.1.try_into().unwrap(),
            stride,
            image_data,
        )
        .unwrap(),
    );

    let idx = frame_number;
    let image = ImageData::Decoded(dynamic_frame);
    Ok(FrameData {
        timestamp,
        image,
        buf_len,
        idx,
        poc,
    })
}

pub(crate) fn from_annexb_path_with_timestamp_source<P: AsRef<Path>>(
    path: P,
    do_decode_h264: bool,
    timestamp_source: crate::TimestampSource,
    srt_file_path: Option<std::path::PathBuf>,
    show_progress: bool,
) -> Result<H264Source<H264AnnexBSource>> {
    let rdr = std::fs::File::open(path.as_ref())?;
    let seekable_h264_source = H264AnnexBSource::from_file(rdr)?;
    from_annexb_reader_with_timestamp_source(
        seekable_h264_source,
        do_decode_h264,
        timestamp_source,
        srt_file_path,
        show_progress,
    )
}

fn from_annexb_reader_with_timestamp_source(
    annex_b_source: H264AnnexBSource,
    do_decode_h264: bool,
    timestamp_source: crate::TimestampSource,
    srt_file_path: Option<std::path::PathBuf>,
    show_progress: bool,
) -> Result<H264Source<H264AnnexBSource>> {
    H264Source::from_seekable_h264_source_with_timestamp_source(
        annex_b_source,
        do_decode_h264,
        None, // mp4_pts
        None, // mp4_sample_timing
        None, // data_from_mp4_track
        timestamp_source,
        srt_file_path,
        show_progress,
        None,
    )
}

pub(crate) struct UserDataUnregistered<'a> {
    pub uuid: &'a [u8; 16],
    pub payload: &'a [u8],
}

impl<'a> UserDataUnregistered<'a> {
    pub fn read(msg: &SeiMessage<'a>) -> Result<UserDataUnregistered<'a>> {
        if msg.payload_type != HeaderType::UserDataUnregistered {
            return Err(Error::UduError(format!(
                "expected UserDataUnregistered message, found {:?}",
                msg.payload_type
            )));
        }
        if msg.payload.len() < 16 {
            return Err(Error::UduError(
                "SEI payload too short to contain UserDataUnregistered message".to_string(),
            ));
        }
        let uuid = (&msg.payload[0..16]).try_into().unwrap();

        let payload = &msg.payload[16..];
        Ok(UserDataUnregistered { uuid, payload })
    }
}

pub(crate) fn parse_precision_time(payload: &[u8]) -> Result<chrono::DateTime<chrono::Utc>> {
    if payload.len() != 12 {
        return Err(Error::UnexpectedPayloadLength);
    }

    // // Time Stamp Status byte from MISB Standard 0603.
    // // Could parse Locked/Unlocked (bit 7), Normal/Discontinuity (bit 6),
    // // Forward/Reverse (bit 5).

    // let time_stamp_status = payload[0];
    // if time_stamp_status & 0x1F != 0x1F {
    //     anyhow::bail!(
    //         "unexpected time stamp status byte. Full payload: {{{}}}",
    //         pretty_hex::simple_hex(&payload),
    //     );
    // }

    let mut precision_time_stamp_bytes = [0u8; 8];
    for i in &[3, 6, 9] {
        if payload[*i] != 0xFF {
            return Err(Error::UnexpectedStartCodeByte);
        }
    }
    precision_time_stamp_bytes[0..2].copy_from_slice(&payload[1..3]);
    precision_time_stamp_bytes[2..4].copy_from_slice(&payload[4..6]);
    precision_time_stamp_bytes[4..6].copy_from_slice(&payload[7..9]);
    precision_time_stamp_bytes[6..8].copy_from_slice(&payload[10..12]);
    let precision_time_stamp: i64 = i64::from_be_bytes(precision_time_stamp_bytes);
    let dur = chrono::Duration::microseconds(precision_time_stamp);

    let epoch_start = chrono::NaiveDate::from_ymd_opt(1970, 1, 1)
        .unwrap()
        .and_hms_micro_opt(0, 0, 0, 0)
        .unwrap()
        .and_local_timezone(chrono::Utc)
        .unwrap();

    Ok(epoch_start + dur)
}

/// Copy raw headerless EBSP NAL units to Annex B
fn copy_nalus_to_annex_b(nalus: &[Vec<u8>]) -> Vec<u8> {
    let sz = nalus.iter().fold(0, |acc, x| acc + x.len() + 4);
    let mut result = vec![0u8; sz];
    let mut start_idx = 0;
    for src in nalus.iter() {
        let dest = &mut result[start_idx..start_idx + 4 + src.len()];
        dest[3] = 0x01;
        dest[4..].copy_from_slice(src);
        start_idx += src.len() + 4;
    }
    result
}

/// Timing information associated with each video frame
///
/// UUID strawlab.org/89H
#[derive(Debug, Clone, Serialize, Deserialize)]
struct FrameInfo {
    /// Receive timestamp as NTP (Network Time Protocol) timestamp
    recv: NtpTimestamp,
    /// RTP (Real Time Protocol) timestamp as reported by the sender
    rtp: u32,
}

// ----

#[cfg(test)]
mod test {
    /// The decode-failure hint names the OpenH264-unsupported chroma the
    /// stream uses, and is empty for a 4:2:0 stream (where the failure lies
    /// elsewhere).
    #[cfg(feature = "openh264")]
    #[test]
    fn decode_failure_hint_flags_features() {
        use super::decode_failure_hint;
        use h264_reader::nal::sps::ChromaFormat::*;

        // The unsupported feature named in the "uses ... (profile" clause.
        fn features(chroma: h264_reader::nal::sps::ChromaFormat) -> String {
            let h = decode_failure_hint(chroma, "P");
            if h.is_empty() {
                return String::new();
            }
            let start = h.find("uses ").unwrap() + "uses ".len();
            let end = h.find(" (profile").unwrap();
            h[start..end].to_string()
        }

        // 4:2:0: nothing to explain.
        assert!(decode_failure_hint(YUV420, "High").is_empty());

        // Non-4:2:0 chroma: flag it.
        assert_eq!(features(YUV444), "4:4:4 chroma subsampling");
        assert_eq!(features(YUV422), "4:2:2 chroma subsampling");
    }

    #[cfg(feature = "openh264")]
    #[test]
    fn parse_h264() -> crate::Result<()> {
        use super::*;

        {
            let file_buf = include_bytes!("test-data/test_less-avc_mono8_15x14.h264");
            let cursor = std::io::Cursor::new(file_buf);
            let seekable_h264_source = H264AnnexBSource::from_readseek(Box::new(cursor))?;

            let do_decode_h264 = true;
            let mut h264_src = from_annexb_reader_with_timestamp_source(
                seekable_h264_source,
                do_decode_h264,
                TimestampSource::BestGuess,
                None,
                false,
            )?;
            assert_eq!(h264_src.width(), 15);
            assert_eq!(h264_src.height(), 14);
            let frames: Vec<_> = h264_src.decode_order_iter().collect();
            assert_eq!(frames.len(), 1);
        }

        {
            let file_buf = include_bytes!("test-data/test_less-avc_rgb8_16x16.h264");
            let cursor = std::io::Cursor::new(file_buf);
            let seekable_h264_source = H264AnnexBSource::from_readseek(Box::new(cursor))?;
            let do_decode_h264 = true;
            let mut h264_src = from_annexb_reader_with_timestamp_source(
                seekable_h264_source,
                do_decode_h264,
                TimestampSource::BestGuess,
                None,
                false,
            )?;
            assert_eq!(h264_src.width(), 16);
            assert_eq!(h264_src.height(), 16);
            let frames: Vec<_> = h264_src.decode_order_iter().collect();
            assert_eq!(frames.len(), 1);
        }
        Ok(())
    }
}