Fix/webvtt hbbtv (#4089)

* First working rendering using parts of vtt.js * First working version using video.js version of vtt.js including coloring and highlighting * Adjust vtt library * Remove tracks on cue exit * Add basic test page * Working version on LG * Manual rendering independent of onenter and onexit * Manual rendering for multiple tracks * Fix wrong ref to vtt caption container * Fix a bug that causes wrong selection of initial WebVTT track and therefor mapping of wrong text data to a certain track * Move dependencies to contrib folder * Enable/disable custom VTT rendering via settings flag * Update the setting for custom vtt rendering before initializing the player. This is important otherwise the setting has no effect. * Remove remote parsing server setting for now as this is not implemented * Fix unit tests * Add vttjs demo to sample section * Add missing parameters to Typescript and Settins files * Remove vtt.js from reference page * Set log level to 4 * Use minified version of vtt library * Remove non required css attributes
Dash-Industry-Forum · Nov 22, 2022 · 56b31c0 · 56b31c0
1 parent 3c91333
commit 56b31c0
Show file tree

Hide file tree

Showing 15 changed files with 2,467 additions and 92 deletions.
diff --git a/contrib/videojs-vtt.js/vtt.js b/contrib/videojs-vtt.js/vtt.js
diff --git a/contrib/videojs-vtt.js/vtt.min.js b/contrib/videojs-vtt.js/vtt.min.js
diff --git a/index.d.ts b/index.d.ts
@@ -226,7 +226,10 @@ declare namespace dashjs {
                 scheduleWhilePaused?: boolean
             },
             text?: {
-                defaultEnabled?: boolean
+                defaultEnabled?: boolean,
+                webvtt?: {
+                    customRenderingEnabled?: number
+                }
             },
             liveCatchup?: {
                 maxDrift?: number;

diff --git a/samples/captioning/vttjs.html b/samples/captioning/vttjs.html
@@ -0,0 +1,134 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>WebVTT Custom Rendering</title>
+    <script src="../../contrib/videojs-vtt.js/vtt.min.js"></script>
+    <script class="code" src="../../contrib/akamai/controlbar/ControlBar.js"></script>
+    <script src="../../dist/dash.all.debug.js"></script>
+
+    <!-- Bootstrap core CSS -->
+    <link href="../lib/bootstrap/bootstrap.min.css" rel="stylesheet">
+    <link href="../lib/main.css" rel="stylesheet">
+    <link rel="stylesheet" href="../../contrib/akamai/controlbar/controlbar.css">
+
+    <style>
+        video {
+            width: 100%;
+        }
+
+        .dash-video-player {
+            position: relative; /* This position relative is needed to position the menus */
+            margin: 0 auto;
+            line-height: 1.0;
+        }
+    </style>
+
+    <script class="code">
+        function init() {
+            var url = 'https://dash.akamaized.net/akamai/test/caption_test/ElephantsDream/elephants_dream_480p_heaac5_1_https.mpd',
+                video = document.querySelector('video'),
+                vttRenderingDiv,
+                player;
+
+            vttRenderingDiv = document.querySelector("#vtt-rendering-div");
+            player = dashjs.MediaPlayer({}).create();
+            player.updateSettings({
+                debug: {
+                    logLevel: 5
+                },
+                streaming: {
+                    text: {
+                        webvtt: {
+                            customRenderingEnabled: true
+                        }
+                    }
+                }
+            })
+            player.initialize(video, url, true);
+            player.attachVttRenderingDiv(vttRenderingDiv)
+            controlbar = new ControlBar(player);
+            controlbar.initialize();
+        }
+    </script>
+</head>
+<body>
+
+<main>
+    <div class="container py-4">
+        <header class="pb-3 mb-4 border-bottom">
+            <img class=""
+                 src="../lib/img/dashjs-logo.png"
+                 width="200">
+        </header>
+        <div class="row">
+            <div class="col-md-4">
+                <div class="h-100 p-5 bg-light border rounded-3">
+                    <h3>WebVTT Custom Rendering Demo</h3>
+                    <p>This example shows how content with VTT captions can be played back by the dash.js player using
+                        the external vtt.js library. This enables VTT support on devices that do not provide native VTT
+                        support. <p>The first
+                        captions appear at the 15s mark.</p>
+                    <p>Note: This sample will only work when using http as the subtitles are not hosted via https.</p>
+                </div>
+            </div>
+            <div class="col-md-8">
+                <div class="dash-video-player code">
+                    <div class="videoContainer" id="videoContainer">
+                        <video preload="auto" muted></video>
+                        <div style="position: relative">
+                            <div id="vtt-rendering-div" style="min-width: 600px; min-height: 100px;"></div>
+                        </div>
+                        <div id="videoController" class="video-controller unselectable">
+                            <div id="playPauseBtn" class="btn-play-pause" title="Play/Pause">
+                                <span id="iconPlayPause" class="icon-play"></span>
+                            </div>
+                            <span id="videoTime" class="time-display">00:00:00</span>
+                            <div id="fullscreenBtn" class="btn-fullscreen control-icon-layout" title="Fullscreen">
+                                <span class="icon-fullscreen-enter"></span>
+                            </div>
+                            <div id="bitrateListBtn" class="control-icon-layout" title="Bitrate List">
+                                <span class="icon-bitrate"></span>
+                            </div>
+                            <input type="range" id="volumebar" class="volumebar" value="1" min="0" max="1" step=".01"/>
+                            <div id="muteBtn" class="btn-mute control-icon-layout" title="Mute">
+                                <span id="iconMute" class="icon-mute-off"></span>
+                            </div>
+                            <div id="trackSwitchBtn" class="control-icon-layout" title="A/V Tracks">
+                                <span class="icon-tracks"></span>
+                            </div>
+                            <div id="captionBtn" class="btn-caption control-icon-layout" title="Closed Caption">
+                                <span class="icon-caption"></span>
+                            </div>
+                            <span id="videoDuration" class="duration-display">00:00:00</span>
+                            <div class="seekContainer">
+                                <div id="seekbar" class="seekbar seekbar-complete">
+                                    <div id="seekbar-buffer" class="seekbar seekbar-buffer"></div>
+                                    <div id="seekbar-play" class="seekbar seekbar-play"></div>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="row">
+            <div class="col-md-12">
+                <div id="code-output"></div>
+            </div>
+        </div>
+        <footer class="pt-3 mt-4 text-muted border-top">
+            &copy; DASH-IF
+        </footer>
+    </div>
+</main>
+
+
+<script>
+    document.addEventListener('DOMContentLoaded', function () {
+        init();
+    });
+</script>
+<script src="../highlighter.js"></script>
+</body>
+</html>
diff --git a/samples/samples.json b/samples/samples.json
@@ -500,6 +500,18 @@
                     "Video",
                     "Audio"
                 ]
+            },
+            {
+                "title": "WebVTT Custom Rendering",
+                "description": "This example shows how content with VTT captions can be played back by the dash.js player using the external vtt.js library. This enables VTT support on devices that do not provide native VTT support.",
+                "href": "captioning/vttjs.html",
+                "image": "lib/img/sintel-1.jpg",
+                "labels": [
+                    "VoD",
+                    "External caption",
+                    "Video",
+                    "Audio"
+                ]
             }
         ]
     },

diff --git a/src/core/Settings.js b/src/core/Settings.js
@@ -137,7 +137,10 @@ import Events from './events/Events';
  *                scheduleWhilePaused: true
  *            },
  *            text: {
- *                defaultEnabled: true
+ *                defaultEnabled: true,
+ *                webvtt: {
+ *                    customRenderingEnabled: false
+ *                }
  *            },
  *            liveCatchup: {
  *                maxDrift: NaN,
@@ -448,6 +451,9 @@ import Events from './events/Events';
  * @typedef {Object} Text
  * @property {number} [defaultEnabled=true]
  * Enable/disable subtitle rendering by default.
+ * @property {object} [webvtt={customRenderingEnabled=false}]
+ * Enables the custom rendering for WebVTT captions. For details refer to the "Subtitles and Captions" sample section of dash.js.
+ * Custom WebVTT rendering requires the external library vtt.js that can be found in the contrib folder.
  */
 
 /**
@@ -854,7 +860,10 @@ function Settings() {
                 scheduleWhilePaused: true
             },
             text: {
-                defaultEnabled: true
+                defaultEnabled: true,
+                webvtt: {
+                    customRenderingEnabled: false
+                }
             },
             liveCatchup: {
                 maxDrift: NaN,

diff --git a/src/streaming/MediaPlayer.js b/src/streaming/MediaPlayer.js
@@ -1425,6 +1425,13 @@ function MediaPlayer() {
         videoModel.setTTMLRenderingDiv(div);
     }
 
+    function attachVttRenderingDiv(div) {
+        if (!videoModel.getElement()) {
+            throw ELEMENT_NOT_ATTACHED_ERROR;
+        }
+        videoModel.setVttRenderingDiv(div);
+    }
+
     /*
     ---------------------------------------------------------------------------
 
@@ -2441,6 +2448,7 @@ function MediaPlayer() {
         setCustomInitialTrackSelectionFunction,
         resetCustomInitialTrackSelectionFunction,
         attachTTMLRenderingDiv,
+        attachVttRenderingDiv,
         getCurrentTextTrackIndex,
         provideThumbnail,
         getDashAdapter,

diff --git a/src/streaming/models/VideoModel.js b/src/streaming/models/VideoModel.js
@@ -50,6 +50,7 @@ function VideoModel() {
         element,
         _currentTime,
         TTMLRenderingDiv,
+        vttRenderingDiv,
         previousPlaybackRate;
 
     const VIDEO_MODEL_WRONG_ELEMENT_TYPE = 'element is not video or audio DOM type!';
@@ -186,6 +187,10 @@ function VideoModel() {
         return TTMLRenderingDiv;
     }
 
+    function getVttRenderingDiv() {
+        return vttRenderingDiv;
+    }
+
     function setTTMLRenderingDiv(div) {
         TTMLRenderingDiv = div;
         // The styling will allow the captions to match the video window size and position.
@@ -197,6 +202,10 @@ function VideoModel() {
         TTMLRenderingDiv.style.left = 0;
     }
 
+    function setVttRenderingDiv(div) {
+        vttRenderingDiv = div;
+    }
+
     function setStallState(type, state) {
         stallStream(type, state);
     }
@@ -448,6 +457,8 @@ function VideoModel() {
         getSource,
         getTTMLRenderingDiv,
         setTTMLRenderingDiv,
+        getVttRenderingDiv,
+        setVttRenderingDiv,
         getPlaybackQuality,
         addEventListener,
         removeEventListener,

diff --git a/src/streaming/text/TextController.js b/src/streaming/text/TextController.js
@@ -33,6 +33,7 @@ import FactoryMaker from '../../core/FactoryMaker';
 import TextSourceBuffer from './TextSourceBuffer';
 import TextTracks from './TextTracks';
 import VTTParser from '../utils/VTTParser';
+import VttCustomRenderingParser from '../utils/VttCustomRenderingParser';
 import TTMLParser from '../utils/TTMLParser';
 import EventBus from '../../core/EventBus';
 import Events from '../../core/events/Events';
@@ -55,6 +56,7 @@ function TextController(config) {
         textSourceBuffers,
         textTracks,
         vttParser,
+        vttCustomRenderingParser,
         ttmlParser,
         eventBus,
         allTracksAreDisabled,
@@ -68,6 +70,7 @@ function TextController(config) {
         disableTextBeforeTextTracksAdded = false;
 
         vttParser = VTTParser(context).getInstance();
+        vttCustomRenderingParser = VttCustomRenderingParser(context).getInstance();
         ttmlParser = TTMLParser(context).getInstance();
         eventBus = EventBus(context).getInstance();
 
@@ -76,12 +79,17 @@ function TextController(config) {
 
     function initialize() {
         eventBus.on(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance);
+        if (settings.get().streaming.text.webvtt.customRenderingEnabled) {
+            eventBus.on(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance);
+            eventBus.on(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance);
+        }
     }
 
     function initializeForStream(streamInfo) {
         const streamId = streamInfo.id;
         const tracks = TextTracks(context).create({
             videoModel,
+            settings,
             streamInfo
         });
         tracks.initialize();
@@ -95,8 +103,10 @@ function TextController(config) {
             videoModel,
             textTracks: tracks,
             vttParser,
+            vttCustomRenderingParser,
             ttmlParser,
-            streamInfo
+            streamInfo,
+            settings
         });
         textSourceBuffer.initialize();
         textSourceBuffers[streamId] = textSourceBuffer;
@@ -200,6 +210,31 @@ function TextController(config) {
         textTracksAdded = true;
     }
 
+    function _onPlaybackTimeUpdated(e) {
+        try {
+            const streamId = e.streamId;
+
+            if (!textTracks[streamId] || isNaN(e.time)) {
+                return;
+            }
+            textTracks[streamId].manualCueProcessing(e.time);
+        } catch (err) {
+        }
+    }
+
+    function _onPlaybackSeeking(e) {
+        try {
+            const streamId = e.streamId;
+
+            if (!textTracks[streamId]) {
+                return;
+            }
+            textTracks[streamId].disableManualTracks();
+        } catch (e) {
+
+        }
+    }
+
     function enableText(streamId, enable) {
         checkParameterType(enable, 'boolean');
         if (isTextEnabled() !== enable) {
@@ -251,6 +286,9 @@ function TextController(config) {
             return;
         }
 
+
+        textTracks[streamId].disableManualTracks();
+
         textTracks[streamId].setModeForTrackIdx(oldTrackIdx, Constants.TEXT_HIDDEN);
         textTracks[streamId].setCurrentTrackIdx(idx);
         textTracks[streamId].setModeForTrackIdx(idx, Constants.TEXT_SHOWING);
@@ -283,7 +321,7 @@ function TextController(config) {
                 if (mediaInfo.id ? currentFragTrack.id !== mediaInfo.id : currentFragTrack.index !== mediaInfo.index) {
                     textTracks[streamId].deleteCuesFromTrackIdx(oldTrackIdx);
                     textSourceBuffers[streamId].setCurrentFragmentedTrackIdx(i);
-                }  else if (oldTrackIdx === -1) {
+                } else if (oldTrackIdx === -1) {
                     // in fragmented use case, if the user selects the older track (the one selected before disabled text track)
                     // no CURRENT_TRACK_CHANGED event will be triggered because the mediaInfo in the StreamProcessor is equal to the one we are selecting
                     // For that reason we reactivate the StreamProcessor and the ScheduleController
@@ -336,6 +374,10 @@ function TextController(config) {
     function reset() {
         resetInitialSettings();
         eventBus.off(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance);
+        if (settings.get().streaming.text.webvtt.customRenderingEnabled) {
+            eventBus.off(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance);
+            eventBus.off(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance)
+        }
 
         Object.keys(textSourceBuffers).forEach((key) => {
             textSourceBuffers[key].resetEmbedded();