From b4c7472a13cc320bb9a177b3241a6c26e262a0e4 Mon Sep 17 00:00:00 2001 From: Pierre-Hugues Husson Date: Fri, 21 Feb 2020 13:11:31 +0100 Subject: [PATCH] Parse TFRF for continuous SmoothStreaming segments refresh This fixes issues #855 --- .../extractor/mp4/FragmentedMp4Extractor.java | 53 +++++-- .../smoothstreaming/DefaultSsChunkSource.java | 144 +++++++++++++++--- .../source/smoothstreaming/SsChunkSource.java | 3 +- .../source/smoothstreaming/SsMediaPeriod.java | 9 +- .../source/smoothstreaming/SsMediaSource.java | 9 +- .../smoothstreaming/manifest/SsManifest.java | 46 +++++- .../smoothstreaming/SsMediaPeriodTest.java | 3 +- 7 files changed, 227 insertions(+), 40 deletions(-) diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java index 7db61fc9e12..edb8bc65c39 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java @@ -65,6 +65,10 @@ public class FragmentedMp4Extractor implements Extractor { public static final ExtractorsFactory FACTORY = () -> new Extractor[] {new FragmentedMp4Extractor()}; + public interface SsAtomCallback { + void onTfrfAtom(long startTime, long duration); + } + /** * Flags controlling the behavior of the extractor. Possible flag values are {@link * #FLAG_WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME}, {@link #FLAG_WORKAROUND_IGNORE_TFDT_BOX}, @@ -115,6 +119,8 @@ public class FragmentedMp4Extractor implements Extractor { new byte[] {-94, 57, 79, 82, 90, -101, 79, 20, -94, 68, 108, 66, 124, 100, -115, -12}; private static final Format EMSG_FORMAT = Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG); + private static final byte[] TFRF_UUID = + new byte[] { -44, -128, 126, -14, -54, 57, 70, -107, -114, 84, 38, -53, -98, 70, -89, -97 }; // Parser states. private static final int STATE_READING_ATOM_HEADER = 0; @@ -151,6 +157,8 @@ public class FragmentedMp4Extractor implements Extractor { private final ArrayDeque pendingMetadataSampleInfos; @Nullable private final TrackOutput additionalEmsgTrackOutput; + private SsAtomCallback ssAtomCallback; + private int parserState; private int atomType; private long atomSize; @@ -267,6 +275,10 @@ public FragmentedMp4Extractor( enterReadingAtomHeaderState(); } + public void setSsAtomCallback(SsAtomCallback cb) { + ssAtomCallback = cb; + } + @Override public boolean sniff(ExtractorInput input) throws IOException, InterruptedException { return Sniffer.sniffFragmented(input); @@ -685,7 +697,7 @@ private static long parseMehd(ParsableByteArray mehd) { return version == 0 ? mehd.readUnsignedInt() : mehd.readUnsignedLongToLong(); } - private static void parseMoof(ContainerAtom moof, SparseArray trackBundleArray, + private void parseMoof(ContainerAtom moof, SparseArray trackBundleArray, @Flags int flags, byte[] extendedTypeScratch) throws ParserException { int moofContainerChildrenSize = moof.containerChildren.size(); for (int i = 0; i < moofContainerChildrenSize; i++) { @@ -700,7 +712,7 @@ private static void parseMoof(ContainerAtom moof, SparseArray track /** * Parses a traf atom (defined in 14496-12). */ - private static void parseTraf(ContainerAtom traf, SparseArray trackBundleArray, + private void parseTraf(ContainerAtom traf, SparseArray trackBundleArray, @Flags int flags, byte[] extendedTypeScratch) throws ParserException { LeafAtom tfhd = traf.getLeafAtomOfType(Atom.TYPE_tfhd); @Nullable TrackBundle trackBundle = parseTfhd(tfhd.data, trackBundleArray); @@ -1004,20 +1016,41 @@ private static int parseTrun(TrackBundle trackBundle, int index, long decodeTime return trackRunEnd; } - private static void parseUuid(ParsableByteArray uuid, TrackFragment out, + private void parseUuid(ParsableByteArray uuid, TrackFragment out, byte[] extendedTypeScratch) throws ParserException { uuid.setPosition(Atom.HEADER_SIZE); uuid.readBytes(extendedTypeScratch, 0, 16); - // Currently this parser only supports Microsoft's PIFF SampleEncryptionBox. - if (!Arrays.equals(extendedTypeScratch, PIFF_SAMPLE_ENCRYPTION_BOX_EXTENDED_TYPE)) { - return; + if (Arrays.equals(extendedTypeScratch, PIFF_SAMPLE_ENCRYPTION_BOX_EXTENDED_TYPE)) { + // Except for the extended type, this box is identical to a SENC box. See "Portable encoding of + // audio-video objects: The Protected Interoperable File Format (PIFF), John A. Bocharov et al, + // Section 5.3.2.1." + parseSenc(uuid, 16, out); + return; } - // Except for the extended type, this box is identical to a SENC box. See "Portable encoding of - // audio-video objects: The Protected Interoperable File Format (PIFF), John A. Bocharov et al, - // Section 5.3.2.1." - parseSenc(uuid, 16, out); + if (Arrays.equals(extendedTypeScratch, TFRF_UUID)) { + ParsableByteArray data = uuid; + data.setPosition(Atom.HEADER_SIZE+16); + int fullAtom = data.readInt(); + int version = Atom.parseFullAtomVersion(fullAtom); + + int fragmentCount = data.readUnsignedByte(); + for(int i=0; i { + //Times are in timescale base + public final long startTimeTs; + public final long durationTs; + public final int chunkId; + public ChunkInfo(long startTimeTs, long durationTs, int chunkId) { + this.startTimeTs = startTimeTs; + this.durationTs=durationTs; + this.chunkId=chunkId; + } + + @Override + public int compareTo(ChunkInfo chunkInfo) { + if(this.startTimeTs > chunkInfo.startTimeTs) + return 1; + else if(this.startTimeTs < chunkInfo.startTimeTs) + return -1; + return 0; + } + } + private final LoaderErrorThrower manifestLoaderErrorThrower; private final int streamElementIndex; private final ChunkExtractorWrapper[] extractorWrappers; @@ -82,6 +108,9 @@ public SsChunkSource createChunkSource( private int currentManifestChunkOffset; @Nullable private IOException fatalError; + private final TreeSet ssChunks = new TreeSet(); + @Nullable private final SsMediaSource mediaSource; + private final long tsDeltaUs; /** * @param manifestLoaderErrorThrower Throws errors affecting loading of manifests. @@ -95,12 +124,14 @@ public DefaultSsChunkSource( SsManifest manifest, int streamElementIndex, TrackSelection trackSelection, - DataSource dataSource) { + DataSource dataSource, + SsMediaSource mediaSource) { this.manifestLoaderErrorThrower = manifestLoaderErrorThrower; this.manifest = manifest; this.streamElementIndex = streamElementIndex; this.trackSelection = trackSelection; this.dataSource = dataSource; + this.mediaSource = mediaSource; StreamElement streamElement = manifest.streamElements[streamElementIndex]; extractorWrappers = new ChunkExtractorWrapper[trackSelection.length()]; @@ -123,7 +154,63 @@ public DefaultSsChunkSource( /* timestampAdjuster= */ null, track); extractorWrappers[i] = new ChunkExtractorWrapper(extractor, streamElement.type, format); + extractor.setSsAtomCallback(this); + } + for(int i=0; i toRemove = new ArrayList<>(); + for (ChunkInfo i : ssChunks) { + if (i.startTimeTs < tsOld) + toRemove.add(i); } + ssChunks.removeAll(toRemove); + return toRemove.isEmpty(); + } + + private synchronized void updateTimeline() { + ChunkInfo end = ssChunks.last(); + ChunkInfo head = ssChunks.first(); + + long chunksWindowDuration = (end.durationTs + end.startTimeTs - head.startTimeTs)/10L; + long startTime = head.startTimeTs/10L; + long defaultStart = manifest.dvrWindowLengthUs - 10*1000L*1000L; + + SinglePeriodTimeline timeline = new SinglePeriodTimeline(C.TIME_UNSET, chunksWindowDuration, startTime, + defaultStart, true /* isSeekable */, true /* isDynamic */, true, null, null); + if(mediaSource != null) { + mediaSource.sourceInfoRefreshed(timeline); + } + } + + public synchronized void onTfrfAtom(long start, long duration) { + if(!manifest.isLive) return; + + boolean ret = + ssChunks.add(new ChunkInfo( + start, + duration, + currentManifestChunkOffset++)); + //If we were already aware of this chunk, don't do anything + if(!ret) return; + clearOldChunks(); + updateTimeline(); } @Override @@ -185,6 +272,25 @@ public int getPreferredQueueSize(long playbackPositionUs, List + Math.abs(chunkFloor.startTimeTs - loadPositionUs)) + chunk = chunkFloor; + else + chunk = chunkCeiling; + } + return chunk; + } + @Override public final void getNextChunk( long playbackPositionUs, @@ -202,21 +308,12 @@ public final void getNextChunk( return; } - int chunkIndex; - if (queue.isEmpty()) { - chunkIndex = streamElement.getChunkIndex(loadPositionUs); - } else { - chunkIndex = - (int) (queue.get(queue.size() - 1).getNextChunkIndex() - currentManifestChunkOffset); - if (chunkIndex < 0) { - // This is before the first chunk in the current manifest. - fatalError = new BehindLiveWindowException(); - return; - } - } + clearOldChunks(); - if (chunkIndex >= streamElement.chunkCount) { - // This is beyond the last chunk in the current manifest. + MediaChunk previous = queue.isEmpty() ? null : queue.get(queue.size() - 1); + ChunkInfo chunk = bestChunk(previous, loadPositionUs); + if(chunk == null) { + // This is before the first chunk in the current manifest. out.endOfStream = !manifest.isLive; return; } @@ -227,28 +324,27 @@ public final void getNextChunk( MediaChunkIterator[] chunkIterators = new MediaChunkIterator[trackSelection.length()]; for (int i = 0; i < chunkIterators.length; i++) { int trackIndex = trackSelection.getIndexInTrackGroup(i); - chunkIterators[i] = new StreamElementIterator(streamElement, trackIndex, chunkIndex); + chunkIterators[i] = new StreamElementIterator(streamElement, trackIndex, chunk.chunkId); } trackSelection.updateSelectedTrack( playbackPositionUs, bufferedDurationUs, timeToLiveEdgeUs, queue, chunkIterators); - long chunkStartTimeUs = streamElement.getStartTimeUs(chunkIndex); - long chunkEndTimeUs = chunkStartTimeUs + streamElement.getChunkDurationUs(chunkIndex); + long chunkStartTimeUs = chunk.startTimeTs/10L; + long chunkEndTimeUs = chunkStartTimeUs + chunk.durationTs/10L; long chunkSeekTimeUs = queue.isEmpty() ? loadPositionUs : C.TIME_UNSET; - int currentAbsoluteChunkIndex = chunkIndex + currentManifestChunkOffset; int trackSelectionIndex = trackSelection.getSelectedIndex(); ChunkExtractorWrapper extractorWrapper = extractorWrappers[trackSelectionIndex]; int manifestTrackIndex = trackSelection.getIndexInTrackGroup(trackSelectionIndex); - Uri uri = streamElement.buildRequestUri(manifestTrackIndex, chunkIndex); + Uri uri = streamElement.buildRequestUriFromStartTime(manifestTrackIndex, chunk.startTimeTs); out.chunk = newMediaChunk( trackSelection.getSelectedFormat(), dataSource, uri, - currentAbsoluteChunkIndex, + chunk.chunkId, chunkStartTimeUs, chunkEndTimeUs, chunkSeekTimeUs, diff --git a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java index 111393140e5..18b15cbb644 100644 --- a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java +++ b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java @@ -46,7 +46,8 @@ SsChunkSource createChunkSource( SsManifest manifest, int streamElementIndex, TrackSelection trackSelection, - @Nullable TransferListener transferListener); + @Nullable TransferListener transferListener, + SsMediaSource mediaSource); } /** diff --git a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriod.java b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriod.java index f7940fed1b6..5544dc83d5b 100644 --- a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriod.java +++ b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriod.java @@ -23,6 +23,7 @@ import com.google.android.exoplayer2.offline.StreamKey; import com.google.android.exoplayer2.source.CompositeSequenceableLoaderFactory; import com.google.android.exoplayer2.source.MediaPeriod; +import com.google.android.exoplayer2.source.MediaSource; import com.google.android.exoplayer2.source.MediaSourceEventListener.EventDispatcher; import com.google.android.exoplayer2.source.SampleStream; import com.google.android.exoplayer2.source.SequenceableLoader; @@ -59,6 +60,7 @@ private ChunkSampleStream[] sampleStreams; private SequenceableLoader compositeSequenceableLoader; private boolean notifiedReadingStarted; + private SsMediaSource mediaSource; public SsMediaPeriod( SsManifest manifest, @@ -69,7 +71,8 @@ public SsMediaPeriod( LoadErrorHandlingPolicy loadErrorHandlingPolicy, EventDispatcher eventDispatcher, LoaderErrorThrower manifestLoaderErrorThrower, - Allocator allocator) { + Allocator allocator, + SsMediaSource mediaSource) { this.manifest = manifest; this.chunkSourceFactory = chunkSourceFactory; this.transferListener = transferListener; @@ -84,6 +87,7 @@ public SsMediaPeriod( compositeSequenceableLoader = compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams); eventDispatcher.mediaPeriodCreated(); + this.mediaSource = mediaSource; } public void updateManifest(SsManifest manifest) { @@ -244,7 +248,8 @@ private ChunkSampleStream buildSampleStream(TrackSelection select manifest, streamElementIndex, selection, - transferListener); + transferListener, + mediaSource); return new ChunkSampleStream<>( manifest.streamElements[streamElementIndex].type, null, diff --git a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaSource.java b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaSource.java index 6836b5cd427..22e90ba86f8 100644 --- a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaSource.java +++ b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaSource.java @@ -567,7 +567,8 @@ public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long star loadErrorHandlingPolicy, eventDispatcher, manifestLoaderErrorThrower, - allocator); + allocator, + this); mediaPeriods.add(period); return period; } @@ -725,13 +726,19 @@ private void processManifest() { refreshSourceInfo(timeline); } + public void sourceInfoRefreshed(Timeline timeline) { + refreshSourceInfo(timeline); + } + private void scheduleManifestRefresh() { if (!manifest.isLive) { return; } + /* long nextLoadTimestamp = manifestLoadStartTimestamp + MINIMUM_MANIFEST_REFRESH_PERIOD_MS; long delayUntilNextLoad = Math.max(0, nextLoadTimestamp - SystemClock.elapsedRealtime()); manifestRefreshHandler.postDelayed(this::startLoadingManifest, delayUntilNextLoad); + */ } private void startLoadingManifest() { diff --git a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifest.java b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifest.java index b91bfc8f675..2794169634a 100644 --- a/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifest.java +++ b/library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifest.java @@ -80,6 +80,7 @@ public static class StreamElement { private final List chunkStartTimes; private final long[] chunkStartTimesUs; private final long lastChunkDurationUs; + private final long lastChunkDuration; public StreamElement( String baseUri, @@ -110,6 +111,7 @@ public StreamElement( language, formats, chunkStartTimes, + lastChunkDuration, Util.scaleLargeTimestamps(chunkStartTimes, C.MICROS_PER_SECOND, timescale), Util.scaleLargeTimestamp(lastChunkDuration, C.MICROS_PER_SECOND, timescale)); } @@ -128,6 +130,7 @@ private StreamElement( @Nullable String language, Format[] formats, List chunkStartTimes, + long lastChunkDuration, long[] chunkStartTimesUs, long lastChunkDurationUs) { this.baseUri = baseUri; @@ -143,6 +146,7 @@ private StreamElement( this.language = language; this.formats = formats; this.chunkStartTimes = chunkStartTimes; + this.lastChunkDuration = lastChunkDuration; this.chunkStartTimesUs = chunkStartTimesUs; this.lastChunkDurationUs = lastChunkDurationUs; chunkCount = chunkStartTimes.size(); @@ -158,7 +162,7 @@ private StreamElement( public StreamElement copy(Format[] formats) { return new StreamElement(baseUri, chunkTemplate, type, subType, timescale, name, maxWidth, maxHeight, displayWidth, displayHeight, language, formats, chunkStartTimes, - chunkStartTimesUs, lastChunkDurationUs); + lastChunkDuration, chunkStartTimesUs, lastChunkDurationUs); } /** @@ -181,6 +185,16 @@ public long getStartTimeUs(int chunkIndex) { return chunkStartTimesUs[chunkIndex]; } + /** + * Returns the unscaled start time of the specified chunk. + * + * @param chunkIndex The index of the chunk. + * @return The start time of the chunk, in original manifest timebase. + */ + public long getStartTime(int chunkIndex) { + return chunkStartTimes.get(chunkIndex); + } + /** * Returns the duration of the specified chunk. * @@ -192,6 +206,17 @@ public long getChunkDurationUs(int chunkIndex) { : chunkStartTimesUs[chunkIndex + 1] - chunkStartTimesUs[chunkIndex]; } + /** + * Returns the unscaled duration of the specified chunk. + * + * @param chunkIndex The index of the chunk. + * @return The duration of the chunk, in original manifest timebase. + */ + public long getChunkDuration(int chunkIndex) { + return (chunkIndex == chunkCount - 1) ? lastChunkDuration + : chunkStartTimes.get(chunkIndex + 1) - chunkStartTimes.get(chunkIndex); + } + /** * Builds a uri for requesting the specified chunk of the specified track. * @@ -212,6 +237,25 @@ public Uri buildRequestUri(int track, int chunkIndex) { .replace(URL_PLACEHOLDER_START_TIME_2, startTimeString); return UriUtil.resolveToUri(baseUri, chunkUrl); } + + /** + * Builds a uri for requesting the chunk at the specified time of the specified track. + * + * @param track The index of the track for which to build the URL. + * @param startTime The unscaled timestamp of the chunk + * @return The request uri. + */ + public Uri buildRequestUriFromStartTime(int track, long startTime) { + Assertions.checkState(formats != null); + Assertions.checkState(chunkStartTimes != null); + String bitrateString = Integer.toString(formats[track].bitrate); + String chunkUrl = chunkTemplate + .replace(URL_PLACEHOLDER_BITRATE_1, bitrateString) + .replace(URL_PLACEHOLDER_BITRATE_2, bitrateString) + .replace(URL_PLACEHOLDER_START_TIME_1, Long.toString(startTime)) + .replace(URL_PLACEHOLDER_START_TIME_2, Long.toString(startTime)); + return UriUtil.resolveToUri(baseUri, chunkUrl); + } } public static final int UNSET_LOOKAHEAD = -1; diff --git a/library/smoothstreaming/src/test/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriodTest.java b/library/smoothstreaming/src/test/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriodTest.java index b9c63f843db..7ce2679a003 100644 --- a/library/smoothstreaming/src/test/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriodTest.java +++ b/library/smoothstreaming/src/test/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriodTest.java @@ -75,7 +75,8 @@ public void getSteamKeys_isCompatibleWithSsManifestFilter() { /* mediaPeriodId= */ new MediaPeriodId(/* periodUid= */ new Object()), /* mediaTimeOffsetMs= */ 0), mock(LoaderErrorThrower.class), - mock(Allocator.class)); + mock(Allocator.class), + null); MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration( mediaPeriodFactory, testManifest);