Skip to content

Commit

Permalink
fix(TTML): Correctly handle multiple samples in a segment
Browse files Browse the repository at this point in the history 
Fixes shaka-project#8087

Implements handling of multiple samples in a segment. Also fixes the testdata for the multiple MDAT test, as the prior data was invalid (not conforming to ISO14496-12).
  • Loading branch information
@julijane
julijane committed Feb 15, 2025
1 parent c42169e commit c1ffbd5
Show file tree
Hide file tree
Showing 6 changed files with 110 additions and 34 deletions.
1 change: 1 addition & 0 deletions AUTHORS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ Jesper Haug Karsrud <[email protected]>
Johan Sundström <[email protected]>
Jonas Birmé <[email protected]>
Jozef Chúťka <[email protected]>
Juliane Holzt <[email protected]>
Jun Hong Chong <[email protected]>
Jürgen Kartnaller <[email protected]>
Justin Swaney <[email protected]>
Expand Down
1 change: 1 addition & 0 deletions CONTRIBUTORS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ Jono Ward <[email protected]>
Jozef Chúťka <[email protected]>
Juan Manuel Tomás <[email protected]>
Julian Domingo <[email protected]>
Juliane Holzt <[email protected]>
Jun Hong Chong <[email protected]>
Jürgen Kartnaller <[email protected]>
Justin Swaney <[email protected]>
Expand Down
110 changes: 81 additions & 29 deletions lib/text/mp4_ttml_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@

goog.provide('shaka.text.Mp4TtmlParser');

goog.require('goog.asserts');
goog.require('shaka.text.TextEngine');
goog.require('shaka.text.TtmlTextParser');
goog.require('shaka.util.BufferUtils');
goog.require('shaka.util.Error');
goog.require('shaka.util.Mp4Parser');
goog.require('shaka.util.Mp4BoxParsers');
goog.require('shaka.util.Uint8ArrayUtils');


Expand Down Expand Up @@ -80,67 +82,117 @@ shaka.text.Mp4TtmlParser = class {
parseMedia(data, time, uri) {
const Mp4Parser = shaka.util.Mp4Parser;

let sawMDAT = false;
let payload = [];
let defaultSampleSize = null;

/** @type {!Array<Uint8Array>} */
const mdats = [];

/* @type {!Map<number,!Array<number>>} */
const subSampleSizesPerSample = new Map();

/** @type {!Array<number>} */
let subSizes = [];
const sampleSizes = [];

const parser = new Mp4Parser()
.box('moof', Mp4Parser.children)
.box('traf', Mp4Parser.children)
.fullBox('tfhd', (box) => {
goog.asserts.assert(
box.flags != null,
'A TFHD box should have a valid flags value');
const parsedTFHDBox = shaka.util.Mp4BoxParsers.parseTFHD(
box.reader, box.flags);
defaultSampleSize = parsedTFHDBox.defaultSampleSize;
})
.fullBox('trun', (box) => {
goog.asserts.assert(
box.version != null,
'A TRUN box should have a valid version value');
goog.asserts.assert(
box.flags != null,
'A TRUN box should have a valid flags value');

const parsedTRUNBox = shaka.util.Mp4BoxParsers.parseTRUN(
box.reader, box.version, box.flags);

for (const sample of parsedTRUNBox.sampleData) {
const sampleSize =
sample.sampleSize || defaultSampleSize || 0;
sampleSizes.push(sampleSize);
}
})
.fullBox('subs', (box) => {
subSizes = [];
const reader = box.reader;
const entryCount = reader.readUint32();
let currentSampleNum = -1;
for (let i = 0; i < entryCount; i++) {
reader.readUint32(); // sample_delta
const sampleDelta = reader.readUint32();
currentSampleNum += sampleDelta;
const subsampleCount = reader.readUint16();
const subsampleSizes = [];
for (let j = 0; j < subsampleCount; j++) {
if (box.version == 1) {
subSizes.push(reader.readUint32());
subsampleSizes.push(reader.readUint32());
} else {
subSizes.push(reader.readUint16());
subsampleSizes.push(reader.readUint16());
}
reader.readUint8(); // priority
reader.readUint8(); // discardable
reader.readUint32(); // codec_specific_parameters
}
subSampleSizesPerSample.set(currentSampleNum, subsampleSizes);
}
})
.box('mdat', Mp4Parser.allData((data) => {
sawMDAT = true;
// Join this to any previous payload, in case the mp4 has multiple
// mdats.
if (subSizes.length) {
const contentData =
shaka.util.BufferUtils.toUint8(data, 0, subSizes[0]);
const images = [];
let offset = subSizes[0];
for (let i = 1; i < subSizes.length; i++) {
const imageData =
shaka.util.BufferUtils.toUint8(data, offset, subSizes[i]);
const raw =
shaka.util.Uint8ArrayUtils.toStandardBase64(imageData);
images.push('data:image/png;base64,' + raw);
offset += subSizes[i];
}
payload = payload.concat(
this.parser_.parseMedia(contentData, time, uri, images));
} else {
payload = payload.concat(
this.parser_.parseMedia(data, time, uri, /* images= */ []));
}
// We collect all of the mdats first, before parsing any of them.
// This is necessary in case the mp4 has multiple mdats.
mdats.push(data);
}));
parser.parse(data, /* partialOkay= */ false);

if (!sawMDAT) {
if (mdats.length == 0) {
throw new shaka.util.Error(
shaka.util.Error.Severity.CRITICAL,
shaka.util.Error.Category.TEXT,
shaka.util.Error.Code.INVALID_MP4_TTML);
}

const fullData =
shaka.util.Uint8ArrayUtils.concat(...mdats);

let sampleOffset = 0;
for (let sampleNum = 0; sampleNum < sampleSizes.length; sampleNum++) {
const sampleData =
shaka.util.BufferUtils.toUint8(fullData, sampleOffset,
sampleSizes[sampleNum]);
sampleOffset += sampleSizes[sampleNum];

const subSampleSizes = subSampleSizesPerSample.get(sampleNum);

if (subSampleSizes && subSampleSizes.length) {
const contentData =
shaka.util.BufferUtils.toUint8(sampleData, 0, subSampleSizes[0]);
const images = [];
let subOffset = subSampleSizes[0];
for (let i = 1; i < subSampleSizes.length; i++) {
const imageData =
shaka.util.BufferUtils.toUint8(data, subOffset,
subSampleSizes[i]);
const raw =
shaka.util.Uint8ArrayUtils.toStandardBase64(imageData);
images.push('data:image/png;base64,' + raw);
subOffset += subSampleSizes[sampleNum][i];
}
payload = payload.concat(
this.parser_.parseMedia(contentData, time, uri, images));
} else {
payload = payload.concat(
this.parser_.parseMedia(sampleData, time, uri,
/* images= */ []));
}
}

return payload;
}
};
Expand Down
Binary file modified test/test/assets/ttml-segment-multiple-mdat.mp4
View file
Binary file not shown.
Binary file added test/test/assets/ttml-segment-multiple-sample.mp4
View file
Binary file not shown.
32 changes: 27 additions & 5 deletions test/text/mp4_ttml_parser_unit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ describe('Mp4TtmlParser', () => {
const ttmlSegmentUri = '/base/test/test/assets/ttml-segment.mp4';
const ttmlSegmentMultipleMDATUri =
'/base/test/test/assets/ttml-segment-multiple-mdat.mp4';
const ttmlSegmentMultipleSampleUri =
'/base/test/test/assets/ttml-segment-multiple-sample.mp4';
const imscImageInitSegmentUri =
'/base/test/test/assets/imsc-image-init.cmft';
const imscImageSegmentUri =
Expand All @@ -22,6 +24,8 @@ describe('Mp4TtmlParser', () => {
/** @type {!Uint8Array} */
let ttmlSegmentMultipleMDAT;
/** @type {!Uint8Array} */
let ttmlSegmentMultipleSample;
/** @type {!Uint8Array} */
let imscImageInitSegment;
/** @type {!Uint8Array} */
let imscImageSegment;
Expand All @@ -33,16 +37,18 @@ describe('Mp4TtmlParser', () => {
shaka.test.Util.fetch(ttmlInitSegmentUri),
shaka.test.Util.fetch(ttmlSegmentUri),
shaka.test.Util.fetch(ttmlSegmentMultipleMDATUri),
shaka.test.Util.fetch(ttmlSegmentMultipleSampleUri),
shaka.test.Util.fetch(imscImageInitSegmentUri),
shaka.test.Util.fetch(imscImageSegmentUri),
shaka.test.Util.fetch(audioInitSegmentUri),
]);
ttmlInitSegment = shaka.util.BufferUtils.toUint8(responses[0]);
ttmlSegment = shaka.util.BufferUtils.toUint8(responses[1]);
ttmlSegmentMultipleMDAT = shaka.util.BufferUtils.toUint8(responses[2]);
imscImageInitSegment = shaka.util.BufferUtils.toUint8(responses[3]);
imscImageSegment = shaka.util.BufferUtils.toUint8(responses[4]);
audioInitSegment = shaka.util.BufferUtils.toUint8(responses[5]);
ttmlSegmentMultipleSample = shaka.util.BufferUtils.toUint8(responses[3]);
imscImageInitSegment = shaka.util.BufferUtils.toUint8(responses[4]);
imscImageSegment = shaka.util.BufferUtils.toUint8(responses[5]);
audioInitSegment = shaka.util.BufferUtils.toUint8(responses[6]);
});

it('parses init segment', () => {
Expand All @@ -62,8 +68,24 @@ describe('Mp4TtmlParser', () => {
expect(ret[0].nestedCues.length).toBe(1);
expect(ret[1].nestedCues.length).toBe(1);
// Cues.
expect(ret[0].nestedCues[0].nestedCues.length).toBe(10);
expect(ret[1].nestedCues[0].nestedCues.length).toBe(10);
expect(ret[0].nestedCues[0].nestedCues.length).toBe(5);
expect(ret[1].nestedCues[0].nestedCues.length).toBe(5);
});

it('handles media segments with multiple sample', () => {
const parser = new shaka.text.Mp4TtmlParser();
parser.parseInit(ttmlInitSegment);
const time =
{periodStart: 0, segmentStart: 0, segmentEnd: 60, vttOffset: 0};
const ret = parser.parseMedia(ttmlSegmentMultipleSample, time, null);
// Bodies.
expect(ret.length).toBe(2);
// Divs.
expect(ret[0].nestedCues.length).toBe(1);
expect(ret[1].nestedCues.length).toBe(1);
// Cues.
expect(ret[0].nestedCues[0].nestedCues.length).toBe(5);
expect(ret[1].nestedCues[0].nestedCues.length).toBe(5);
});

it('accounts for offset', () => {
Expand Down

0 comments on commit c1ffbd5

Please sign in to comment.