From 0b0a0a15c821c6b176b878637408af4a9df6a88f Mon Sep 17 00:00:00 2001 From: Jordan Rose Date: Tue, 23 Jan 2024 18:11:41 -0800 Subject: [PATCH] Add mp4san-dump utility for additional testing This dumps every sample in the first track to a file, assuming (without checking) that that track is an H.264 video track with one NAL unit per sample. This can be compared against the output from ffmpeg -i input.mp4 -vcodec copy output.h264 It *will* have differences because it doesn't include the configuration info ("SPS" and "PPS"), but it should be clear that those are the only differences. --- Cargo.toml | 2 +- mp4san-dump/Cargo.toml | 16 +++ mp4san-dump/src/lib.rs | 218 ++++++++++++++++++++++++++++++++++++++++ mp4san-dump/src/main.rs | 41 ++++++++ 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 mp4san-dump/Cargo.toml create mode 100644 mp4san-dump/src/lib.rs create mode 100644 mp4san-dump/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index eea23da..4799494 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["cli", "common", "mp4san", "mp4san-derive", "mp4san-test", "mp4san-test-gen", "webpsan", "webpsan-test"] +members = ["cli", "common", "mp4san", "mp4san-derive", "mp4san-dump", "mp4san-test", "mp4san-test-gen", "webpsan", "webpsan-test"] resolver = "2" [workspace.package] diff --git a/mp4san-dump/Cargo.toml b/mp4san-dump/Cargo.toml new file mode 100644 index 0000000..00086ef --- /dev/null +++ b/mp4san-dump/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "mp4san-dump" +description = "Sample utilities built on top of mp4san to synchronously dump frames" +version.workspace = true +edition.workspace = true + +publish = false + +[dependencies] +mp4san = { path = "../mp4san" } +mediasan-common = { path = "../common" } + +bytes = "1.3.0" +env_logger = "0.10.0" +futures-util = { version = "0.3.28", default-features = false, features = ["io"] } +log = "0.4.17" diff --git a/mp4san-dump/src/lib.rs b/mp4san-dump/src/lib.rs new file mode 100644 index 0000000..5a61528 --- /dev/null +++ b/mp4san-dump/src/lib.rs @@ -0,0 +1,218 @@ +//! WARNING: This is not a 100% correct implementation of a frame dumper for H.264 in MPEG-4. +//! +//! Many many things were skipped and/or hardcoded. Do not use this as a reference, only a starting +//! point. + +use std::io; +use std::ops::Range; + +use bytes::Buf as _; +use futures_util::{pin_mut, FutureExt as _}; +use mediasan_common::{bail_attach, ensure_attach, report_attach, SeekSkipAdapter}; +use mp4san::parse::{ + ArrayEntry, BoxHeader, BoxType, FtypBox, MoovBox, Mp4Box, ParseBox, ParseError, ParsedBox, StscEntry, TrakBox, +}; +use mp4san::{Error, COMPATIBLE_BRAND}; + +// Note: modified version of mp4san::sanitize_async_with_config; should eventually be folded back +// together with that. +pub fn parse(input: &[u8]) -> Result, Error> { + let mut reader = io::Cursor::new(input); + + let mut mdat_seen = false; + let mut ftyp_seen = false; + let mut moov: Option> = None; + + while reader.position() != reader.get_ref().len() as u64 { + let start_pos = reader.position(); + + let header = BoxHeader::parse(&mut reader).expect("valid header"); + + match header.box_type() { + name @ (BoxType::FREE | BoxType::SKIP) => { + skip_box(&mut reader, &header).expect("can skip in Cursor"); + log::info!("{name} @ 0x{start_pos:08x}"); + } + + BoxType::FTYP => { + assert!(!ftyp_seen, "multiple ftyp boxes"); + let mut read_ftyp = read_data_sync(&mut reader, header, 1024).expect("valid box"); + let ftyp_data: &mut FtypBox = read_ftyp.data.parse().expect("valid ftyp"); + let compatible_brand_count = ftyp_data.compatible_brands().len(); + let FtypBox { major_brand, minor_version, .. } = ftyp_data; + log::info!("ftyp @ 0x{start_pos:08x}: {major_brand} version {minor_version}, {compatible_brand_count} compatible brands"); + + ensure_attach!( + ftyp_data.compatible_brands().any(|b| b == COMPATIBLE_BRAND), + ParseError::UnsupportedFormat(ftyp_data.major_brand) + ); + + ftyp_seen = true; + } + + // NB: ISO 14496-12-2012 specifies a default ftyp, but we don't currently use it. The spec says that it + // contains a single compatible brand, "mp41", and notably not "isom" which is the ISO spec we follow for + // parsing now. This implies that there's additional stuff in "mp41" which is not in "isom". "mp41" is also + // very old at this point, so it'll require additional research/work to be able to parse/remux it. + _ if !ftyp_seen => { + bail_attach!(ParseError::InvalidBoxLayout, "ftyp is not the first significant box"); + } + + BoxType::MDAT => { + mdat_seen = true; + skip_box(&mut reader, &header).expect("can skip in Cursor"); + log::info!("mdat @ 0x{start_pos:08x}"); + } + + BoxType::MOOV => { + let mut read_moov: Mp4Box = + read_data_sync(&mut reader, header, 1024 * 1024).expect("can read box"); + let children = read_moov.data.parse().expect("valid moov").parsed_mut(); + + let chunk_count = children.tracks.iter().map(|trak| trak.co().entry_count()).sum::(); + let trak_count = children.tracks.len(); + + log::info!("moov @ 0x{start_pos:08x}: {trak_count} traks {chunk_count} chunks"); + moov = Some(read_moov); + } + + name => { + skip_box(&mut reader, &header).expect("can skip in Cursor"); + log::info!("{name} @ 0x{start_pos:08x}"); + } + } + } + + if !ftyp_seen { + bail_attach!(ParseError::MissingRequiredBox(BoxType::FTYP)); + } + if !mdat_seen { + bail_attach!(ParseError::MissingRequiredBox(BoxType::MDAT)); + } + let Some(moov) = moov else { + bail_attach!(ParseError::MissingRequiredBox(BoxType::MOOV)); + }; + + Ok(moov) +} + +/// Skip a box's data assuming its header has already been read. +fn skip_box(reader: &mut impl io::Seek, header: &BoxHeader) -> Result<(), io::Error> { + match header.box_data_size().expect("valid header") { + Some(box_size) => reader.seek(io::SeekFrom::Current(box_size as i64))?, + None => reader.seek(io::SeekFrom::End(0))?, + }; + Ok(()) +} + +/// Read a box's data. +fn read_data_sync(reader: &mut R, header: BoxHeader, max_size: u64) -> Result, Error> +where + R: std::io::Read + std::io::Seek, + T: ParseBox + ParsedBox, +{ + let async_reader = SeekSkipAdapter(futures_util::io::AllowStdIo::new(reader)); + pin_mut!(async_reader); + Mp4Box::read_data(async_reader, header, max_size) + .now_or_never() + .expect("only awaits reader") +} + +/// Iterates through every sample in the track in order, ignoring timecodes and edit lists. +pub fn for_each_sample( + track: &TrakBox, + full_data: &[u8], + mut process: impl FnMut(&[u8]) -> Result<(), Error>, +) -> Result<(), Error> { + let samples = track.parsed().media.parsed().info.parsed().samples; + let mut sample_to_chunk_walker = SampleToChunkWalker::new(samples.parsed().sample_to_chunk.entries()); + + let offset_for_chunk = |i| { + Ok::(match samples.parsed().chunk_offsets { + mp4san::parse::StblCoRef::Stco(stco) => stco + .entries() + .nth(i as usize - 1) + .ok_or_else(|| report_attach!(ParseError::InvalidInput))? + .get()? as usize, + mp4san::parse::StblCoRef::Co64(co64) => co64 + .entries() + .nth(i as usize - 1) + .ok_or_else(|| report_attach!(ParseError::InvalidInput))? + .get()? as usize, + }) + }; + + let mut prev_chunk_index = 0; + let mut current_chunk_data: &[u8] = &[]; + for (sample_size, sample_index) in samples.parsed().sample_sizes.sample_sizes().zip(0..) { + let sample_size = sample_size?; + let ChunkInfo { chunk_index, num_samples_in_chunk: _ } = sample_to_chunk_walker.chunk_info_for(sample_index); + if chunk_index != prev_chunk_index { + current_chunk_data = &full_data[offset_for_chunk(chunk_index)?..]; + prev_chunk_index = chunk_index; + } + let current_sample = ¤t_chunk_data[..sample_size as usize]; + current_chunk_data.advance(sample_size as usize); + + process(current_sample)?; + } + + Ok(()) +} + +#[derive(PartialEq, Eq)] +struct ChunkInfo { + pub chunk_index: u32, + pub num_samples_in_chunk: u32, +} + +/// Wraps an iterator over stsc ("sample-to-chunk") entries to allow looking up chunks for given +/// sample indexes. +/// +/// Samples must be accessed in increasing order, but skipping is allowed. This type expects sample +/// indexes to be zero-indexed. +struct SampleToChunkWalker { + entry_iter: std::iter::Peekable, + chunks_in_entry: Range, + num_samples_in_chunk: u32, + samples_seen_so_far: usize, +} + +impl<'a, T> SampleToChunkWalker +where + T: Iterator>, +{ + fn new(iter: T) -> Self { + Self { entry_iter: iter.peekable(), chunks_in_entry: 0..0, num_samples_in_chunk: 0, samples_seen_so_far: 0 } + } + + fn advance_one_chunk(&mut self) { + self.samples_seen_so_far += self.num_samples_in_chunk as usize; + _ = self.chunks_in_entry.next(); + if self.chunks_in_entry.is_empty() { + let next_entry = self + .entry_iter + .next() + .expect("has more entries") + .get() + .expect("valid sample-to-chunk entry"); + let upper_bound = self.entry_iter.peek().map_or(u32::MAX, |following_entry| { + following_entry.get().expect("valid sample-to-chunk entry").first_chunk + }); + self.chunks_in_entry = next_entry.first_chunk..upper_bound; + self.num_samples_in_chunk = next_entry.samples_per_chunk; + } + } + + fn chunk_info_for(&mut self, sample_index: usize) -> ChunkInfo { + assert!( + sample_index >= self.samples_seen_so_far, + "searching backwards is not implemented" + ); + while sample_index - self.samples_seen_so_far >= self.num_samples_in_chunk as usize { + self.advance_one_chunk(); + } + + ChunkInfo { chunk_index: self.chunks_in_entry.start, num_samples_in_chunk: self.num_samples_in_chunk } + } +} diff --git a/mp4san-dump/src/main.rs b/mp4san-dump/src/main.rs new file mode 100644 index 0000000..7ae9b2d --- /dev/null +++ b/mp4san-dump/src/main.rs @@ -0,0 +1,41 @@ +//! WARNING: This is not a 100% correct implementation of a frame dumper for H.264 in MPEG-4. +//! +//! Many many things were skipped and/or hardcoded. Do not use this as a reference, only a starting +//! point. + +use std::io::Read as _; +use std::io::{self, Write}; + +use bytes::Buf; +use mp4san::parse::MoovBox; + +const NAL_HEADER: &[u8] = &[0, 0, 0, 1]; + +pub fn main() { + env_logger::init(); + + let mut input = Vec::with_capacity(100 * 1024); + io::stdin().read_to_end(&mut input).expect("can read stdin"); + + let moov = mp4san_dump::parse(&input).expect("valid input"); + let moov_children = moov.data.parsed::().expect("parsed moov box already").parsed(); + + // FIXME: The first track isn't always the video track. + if let Some(track) = moov_children.tracks.get(0) { + mp4san_dump::for_each_sample(track, &input, |mut sample| { + // FIXME: not all NAL lengths use four bytes + let nal_length = sample.get_u32() as usize; + assert_eq!( + nal_length, + sample.len(), + "NAL split across samples, or multiple NALs in a sample" + ); + + std::io::stdout().write_all(NAL_HEADER).expect("can write to stdout"); + std::io::stdout().write_all(sample).expect("can write to stdout"); + + Ok(()) + }) + .expect("valid parsed input"); + } +}