diff --git a/demo/common/assets.js b/demo/common/assets.js index c9de58057f..a7abf4223b 100644 --- a/demo/common/assets.js +++ b/demo/common/assets.js @@ -1175,6 +1175,7 @@ shakaAssets.testAssets = [ .addFeature(shakaAssets.Feature.HIGH_DEFINITION) .addFeature(shakaAssets.Feature.MP4) .addFeature(shakaAssets.Feature.THUMBNAILS) + .addFeature(shakaAssets.Feature.OFFLINE) .addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'), new ShakaDemoAssetInfo( /* name= */ 'Art of Motion (HLS) (external thumbnails)', @@ -1185,6 +1186,7 @@ shakaAssets.testAssets = [ .addFeature(shakaAssets.Feature.HIGH_DEFINITION) .addFeature(shakaAssets.Feature.MP2TS) .addFeature(shakaAssets.Feature.THUMBNAILS) + .addFeature(shakaAssets.Feature.OFFLINE) .addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'), new ShakaDemoAssetInfo( /* name= */ 'Art of Motion (MP4) (external thumbnails)', diff --git a/demo/main.js b/demo/main.js index fb37e1791e..d66a4be99c 100644 --- a/demo/main.js +++ b/demo/main.js @@ -586,7 +586,8 @@ shakaDemo.Main = class { asset.storedProgress = 0; this.dispatchEventWithName_('shaka-main-offline-progress'); const start = Date.now(); - const stored = await storage.store(asset.manifestUri, metadata).promise; + const stored = await storage.store(asset.manifestUri, metadata, + /* mimeType= */ null, asset.extraThumbnail).promise; const end = Date.now(); console.log('Download time:', end - start); asset.storedContent = stored; @@ -1381,8 +1382,10 @@ shakaDemo.Main = class { } } - for (const extraThumbnail of asset.extraThumbnail) { - this.player_.addThumbnailsTrack(extraThumbnail); + if (!(asset.storedContent && asset.storedContent.offlineUri)) { + for (const extraThumbnail of asset.extraThumbnail) { + this.player_.addThumbnailsTrack(extraThumbnail); + } } for (const extraChapter of asset.extraChapter) { diff --git a/externs/shaka/manifest.js b/externs/shaka/manifest.js index f8570ae940..4d39a42ddb 100644 --- a/externs/shaka/manifest.js +++ b/externs/shaka/manifest.js @@ -562,3 +562,25 @@ shaka.extern.Stream; * @exportDoc */ shaka.extern.MssPrivateData; + + +/** + * @typedef {{ + * height: number, + * positionX: number, + * positionY: number, + * width: number + * }} + * + * @property {number} height + * The thumbnail height in px. + * @property {number} positionX + * The thumbnail left position in px. + * @property {number} positionY + * The thumbnail top position in px. + * @property {number} width + * The thumbnail width in px. + * + * @exportDoc + */ +shaka.extern.ThumbnailSprite; diff --git a/externs/shaka/offline.js b/externs/shaka/offline.js index 57e72dd50e..fa9f13731c 100644 --- a/externs/shaka/offline.js +++ b/externs/shaka/offline.js @@ -235,7 +235,8 @@ shaka.extern.StreamDB; * pendingInitSegmentRefId: (string|undefined), * dataKey: number, * mimeType: ?string, - * codecs: ?string + * codecs: ?string, + * thumbnailSprite: ?shaka.extern.ThumbnailSprite * }} * * @property {?number} initSegmentKey @@ -272,6 +273,8 @@ shaka.extern.StreamDB; * The mimeType of the segment. * @property {?string} codecs * The codecs of the segment. + * @property {?shaka.extern.ThumbnailSprite} thumbnailSprite + * The segment's thumbnail sprite. */ shaka.extern.SegmentDB; diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index 6bf1e8784c..fa41865e4f 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -291,7 +291,7 @@ shaka.media.SegmentReference = class { /** @type {?shaka.extern.aesKey} */ this.aesKey = aesKey; - /** @type {?shaka.media.SegmentReference.ThumbnailSprite} */ + /** @type {?shaka.extern.ThumbnailSprite} */ this.thumbnailSprite = null; /** @type {number} */ @@ -550,7 +550,7 @@ shaka.media.SegmentReference = class { /** * Set the segment's thumbnail sprite. * - * @param {shaka.media.SegmentReference.ThumbnailSprite} thumbnailSprite + * @param {shaka.extern.ThumbnailSprite} thumbnailSprite * @export */ setThumbnailSprite(thumbnailSprite) { @@ -560,7 +560,7 @@ shaka.media.SegmentReference = class { /** * Returns the segment's thumbnail sprite. * - * @return {?shaka.media.SegmentReference.ThumbnailSprite} + * @return {?shaka.extern.ThumbnailSprite} * @export */ getThumbnailSprite() { diff --git a/lib/offline/download_info.js b/lib/offline/download_info.js index 0c5ce747aa..45073787b7 100644 --- a/lib/offline/download_info.js +++ b/lib/offline/download_info.js @@ -45,7 +45,11 @@ shaka.offline.DownloadInfo = class { static idForSegmentRef(ref) { // Escape the URIs using encodeURI, to make sure that a weirdly formed URI // cannot cause two unrelated refs to be considered equivalent. - return ref.getUris().map((uri) => '{' + encodeURI(uri) + '}').join('') + + const removeSprites = (uri) => { + return uri.split('#xywh=')[0]; + }; + return ref.getUris().map( + (uri) => '{' + encodeURI(removeSprites(uri)) + '}').join('') + ':' + ref.startByte + ':' + ref.endByte; } diff --git a/lib/offline/indexeddb/v1_storage_cell.js b/lib/offline/indexeddb/v1_storage_cell.js index bab8f7878b..1e206709d5 100644 --- a/lib/offline/indexeddb/v1_storage_cell.js +++ b/lib/offline/indexeddb/v1_storage_cell.js @@ -214,6 +214,7 @@ shaka.offline.indexeddb.V1StorageCell = class tilesLayout: '', mimeType: null, codecs: null, + thumbnailSprite: null, }; } diff --git a/lib/offline/indexeddb/v2_storage_cell.js b/lib/offline/indexeddb/v2_storage_cell.js index 6fc7e4f864..854ee9596d 100644 --- a/lib/offline/indexeddb/v2_storage_cell.js +++ b/lib/offline/indexeddb/v2_storage_cell.js @@ -158,6 +158,7 @@ shaka.offline.indexeddb.V2StorageCell = class tilesLayout: '', mimeType: null, codecs: null, + thumbnailSprite: null, }; } }; diff --git a/lib/offline/manifest_converter.js b/lib/offline/manifest_converter.js index 77fbe3cd12..65705e51cd 100644 --- a/lib/offline/manifest_converter.js +++ b/lib/offline/manifest_converter.js @@ -255,6 +255,9 @@ shaka.offline.ManifestConverter = class { segmentDB.tilesLayout || ''); ref.mimeType = segmentDB.mimeType || streamDB.mimeType || ''; ref.codecs = segmentDB.codecs || streamDB.codecs || ''; + if (segmentDB.thumbnailSprite) { + ref.setThumbnailSprite(segmentDB.thumbnailSprite); + } return ref; } diff --git a/lib/offline/storage.js b/lib/offline/storage.js index eaefc4da05..4fcfd71501 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -11,6 +11,8 @@ goog.require('shaka.Player'); goog.require('shaka.log'); goog.require('shaka.media.DrmEngine'); goog.require('shaka.media.ManifestParser'); +goog.require('shaka.media.SegmentIndex'); +goog.require('shaka.media.SegmentReference'); goog.require('shaka.net.NetworkingEngine'); goog.require('shaka.net.NetworkingUtils'); goog.require('shaka.offline.DownloadInfo'); @@ -20,13 +22,16 @@ goog.require('shaka.offline.SessionDeleter'); goog.require('shaka.offline.StorageMuxer'); goog.require('shaka.offline.StoredContentUtils'); goog.require('shaka.offline.StreamBandwidthEstimator'); +goog.require('shaka.text.TextEngine'); goog.require('shaka.util.AbortableOperation'); goog.require('shaka.util.ArrayUtils'); +goog.require('shaka.util.BufferUtils'); goog.require('shaka.util.ConfigUtils'); goog.require('shaka.util.Destroyer'); goog.require('shaka.util.Error'); goog.require('shaka.util.IDestroyable'); goog.require('shaka.util.Iterables'); +goog.require('shaka.util.ManifestParserUtils'); goog.require('shaka.util.MimeUtils'); goog.require('shaka.util.Platform'); goog.require('shaka.util.PlayerConfiguration'); @@ -142,6 +147,13 @@ shaka.offline.Storage = class { this.config_ = null; this.networkingEngine_ = null; }); + + /** + * Contains an ID for use with creating streams. The manifest parser should + * start with small IDs, so this starts with a large one. + * @private {number} + */ + this.nextExternalStreamId_ = 1e9; } @@ -249,8 +261,10 @@ shaka.offline.Storage = class { * application-specific metadata you need associated with the stored * content. For details on the data types that can be stored here, please * refer to {@link https://bit.ly/StructClone} - * @param {string=} mimeType + * @param {?string=} mimeType * The mime type for the content |manifestUri| points to. + * @param {?Array.=} externalThumbnails + * The external thumbnails to store along the main content. * @return {!shaka.extern.IAbortableOperation.} * An AbortableOperation that resolves with a structure representing what * was stored. The "offlineUri" member is the URI that should be given to @@ -260,7 +274,7 @@ shaka.offline.Storage = class { * AbortableOperation. * @export */ - store(uri, appMetadata, mimeType) { + store(uri, appMetadata, mimeType, externalThumbnails) { goog.asserts.assert( this.networkingEngine_, 'Cannot call |store| after calling |destroy|.'); @@ -290,7 +304,8 @@ shaka.offline.Storage = class { this.openDownloadManagers_.push(downloader); const storeOp = this.store_( - uri, appMetadata || {}, getParser, config, downloader); + uri, appMetadata || {}, externalThumbnails || [], + getParser, config, downloader); const abortableStoreOp = new shaka.util.AbortableOperation(storeOp, () => { return downloader.abortAll(); }); @@ -306,13 +321,15 @@ shaka.offline.Storage = class { * * @param {string} uri * @param {!Object} appMetadata + * @param {!Array.} externalThumbnails * @param {function():!Promise.} getParser * @param {shaka.extern.PlayerConfiguration} config * @param {!shaka.offline.DownloadManager} downloader * @return {!Promise.} * @private */ - async store_(uri, appMetadata, getParser, config, downloader) { + async store_(uri, appMetadata, externalThumbnails, + getParser, config, downloader) { this.requireSupport_(); // Since we will need to use |parser|, |drmEngine|, |activeHandle|, and @@ -357,6 +374,14 @@ shaka.offline.Storage = class { uri); } + for (const thumbnailUri of externalThumbnails) { + const imageStream = + // eslint-disable-next-line no-await-in-loop + await this.createExternalImageStream_(thumbnailUri, manifest); + manifest.imageStreams.push(imageStream); + this.ensureNotDestroyed_(); + } + // Create the DRM engine, and load the keys in the manifest. drmEngine = await this.createDrmEngine( manifest, @@ -1216,6 +1241,171 @@ shaka.offline.Storage = class { return manifest; } + /** + * @param {string} uri + * @param {shaka.extern.Manifest} manifest + * @return {!Promise.} + * @private + */ + async createExternalImageStream_(uri, manifest) { + const mimeType = await this.getTextMimetype_(uri); + + if (mimeType != 'text/vtt') { + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.TEXT, + shaka.util.Error.Code.UNSUPPORTED_EXTERNAL_THUMBNAILS_URI, + uri); + } + + goog.asserts.assert( + this.networkingEngine_, 'Need networking engine.'); + const buffer = await this.getTextData_(uri, + this.networkingEngine_, + this.config_.streaming.retryParameters); + + const factory = shaka.text.TextEngine.findParser(mimeType); + if (!factory) { + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.TEXT, + shaka.util.Error.Code.MISSING_TEXT_PLUGIN, + mimeType); + } + const TextParser = factory(); + const time = { + periodStart: 0, + segmentStart: 0, + segmentEnd: manifest.presentationTimeline.getDuration(), + vttOffset: 0, + }; + const data = shaka.util.BufferUtils.toUint8(buffer); + const cues = TextParser.parseMedia(data, time, uri, /* images= */ []); + + const references = []; + for (const cue of cues) { + let uris = null; + const getUris = () => { + if (uris == null) { + uris = shaka.util.ManifestParserUtils.resolveUris( + [uri], [cue.payload]); + } + return uris || []; + }; + const reference = new shaka.media.SegmentReference( + cue.startTime, + cue.endTime, + getUris, + /* startByte= */ 0, + /* endByte= */ null, + /* initSegmentReference= */ null, + /* timestampOffset= */ 0, + /* appendWindowStart= */ 0, + /* appendWindowEnd= */ Infinity, + ); + if (cue.payload.includes('#xywh')) { + const spriteInfo = cue.payload.split('#xywh=')[1].split(','); + if (spriteInfo.length === 4) { + reference.setThumbnailSprite({ + height: parseInt(spriteInfo[3], 10), + positionX: parseInt(spriteInfo[0], 10), + positionY: parseInt(spriteInfo[1], 10), + width: parseInt(spriteInfo[2], 10), + }); + } + } + references.push(reference); + } + + let segmentMimeType = mimeType; + if (references.length) { + segmentMimeType = await shaka.net.NetworkingUtils.getMimeType( + references[0].getUris()[0], + this.networkingEngine_, this.config_.manifest.retryParameters); + } + + return { + id: this.nextExternalStreamId_++, + originalId: null, + groupId: null, + createSegmentIndex: () => Promise.resolve(), + segmentIndex: new shaka.media.SegmentIndex(references), + mimeType: segmentMimeType || '', + codecs: '', + kind: '', + encrypted: false, + drmInfos: [], + keyIds: new Set(), + language: 'und', + originalLanguage: null, + label: null, + type: shaka.util.ManifestParserUtils.ContentType.IMAGE, + primary: false, + trickModeVideo: null, + emsgSchemeIdUris: null, + roles: [], + forced: false, + channelsCount: null, + audioSamplingRate: null, + spatialAudio: false, + closedCaptions: null, + tilesLayout: '1x1', + accessibilityPurpose: null, + external: true, + fastSwitching: false, + fullMimeTypes: new Set([shaka.util.MimeUtils.getFullType( + segmentMimeType || '', '')]), + }; + } + + /** + * @param {string} uri + * @return {!Promise.} + * @private + */ + async getTextMimetype_(uri) { + let mimeType; + try { + goog.asserts.assert( + this.networkingEngine_, 'Need networking engine.'); + // eslint-disable-next-line require-atomic-updates + mimeType = await shaka.net.NetworkingUtils.getMimeType(uri, + this.networkingEngine_, + this.config_.streaming.retryParameters); + } catch (error) {} + + if (mimeType) { + return mimeType; + } + + shaka.log.error( + 'The mimeType has not been provided and it could not be deduced ' + + 'from its uri.'); + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.TEXT, + shaka.util.Error.Code.TEXT_COULD_NOT_GUESS_MIME_TYPE, + uri); + } + + /** + * @param {string} uri + * @param {!shaka.net.NetworkingEngine} netEngine + * @param {shaka.extern.RetryParameters} retryParams + * @return {!Promise.} + * @private + */ + async getTextData_(uri, netEngine, retryParams) { + const type = shaka.net.NetworkingEngine.RequestType.SEGMENT; + + const request = shaka.net.NetworkingEngine.makeRequest([uri], retryParams); + request.method = 'GET'; + + const response = await netEngine.request(type, request).promise; + + return response.data; + } + /** * This method is public so that it can be override in testing. * @@ -1396,6 +1586,7 @@ shaka.offline.Storage = class { dataKey: 0, mimeType: segment.mimeType, codecs: segment.codecs, + thumbnailSprite: segment.thumbnailSprite, }; streamDb.segments.push(segmentDB); groupId = (groupId + 1) % numberOfParallelDownloads; diff --git a/test/offline/manifest_convert_unit.js b/test/offline/manifest_convert_unit.js index 8bb3698543..8fefb023fb 100644 --- a/test/offline/manifest_convert_unit.js +++ b/test/offline/manifest_convert_unit.js @@ -347,6 +347,7 @@ describe('ManifestConverter', () => { tilesLayout: '', mimeType, codecs, + thumbnailSprite: null, }; return segment;