diff --git a/src/renderer/components/ft-video-player/ft-video-player.js b/src/renderer/components/ft-video-player/ft-video-player.js index 338d244500d28..d56a7789eb6a4 100644 --- a/src/renderer/components/ft-video-player/ft-video-player.js +++ b/src/renderer/components/ft-video-player/ft-video-player.js @@ -36,7 +36,7 @@ const EXPECTED_PLAY_RELATED_ERROR_MESSAGES = [ // videojs-http-streaming calls this hook everytime it makes a request, // so we can use it to convert the Range header into the range query parameter for the streaming URLs videojs.Vhs.xhr.beforeRequest = (options) => { - if (store.getters.getProxyVideos) { + if (store.getters.getProxyVideos && !options.uri.startsWith('data:application/dash+xml')) { const { uri } = options options.uri = getProxyUrl(uri) } diff --git a/src/renderer/helpers/api/invidious.js b/src/renderer/helpers/api/invidious.js index c8e4883bc5fa5..daea27629cc47 100644 --- a/src/renderer/helpers/api/invidious.js +++ b/src/renderer/helpers/api/invidious.js @@ -2,6 +2,7 @@ import store from '../../store/index' import { stripHTML, toLocalePublicationString } from '../utils' import { isNullOrEmpty } from '../strings' import autolinker from 'autolinker' +import { FormatUtils, Misc, Player } from 'youtubei.js' function getCurrentInstance() { return store.getters.getCurrentInvidiousInstance @@ -318,11 +319,13 @@ export function filterInvidiousFormats(formats, allowAv1 = false) { // Which is caused by Invidious API limitation on AV1 formats (see related issues) // Commented code to be restored after Invidious issue fixed // - // if (allowAv1 && av1Formats.length > 0) { - // return [...audioFormats, ...av1Formats] - // } else { - // return [...audioFormats, ...h264Formats] - // } + // As we generate our own DASH manifest (using YouTube.js) for multiple audio track support in Electron, + // we can allow AV1 in that situation. If we aren't in electron, + // we still can't use them until Invidious fixes the issue on their side + if (process.env.IS_ELECTRON && allowAv1 && av1Formats.length > 0) { + return [...audioFormats, ...av1Formats] + } + return [...audioFormats, ...h264Formats] } @@ -337,3 +340,80 @@ export async function getHashtagInvidious(hashtag, page) { const response = await invidiousAPICall(payload) return response.results } + +/** + * Generates a DASH manifest locally from Invidious' adaptive formats and manifest, + * doing so allows us to support multiple audio tracks, which Invidious doesn't support yet + * @param {import('youtubei.js').Misc.Format[]} formats + * @param {string=} invidiousInstance the formats will be proxied through the specified instance, when one is provided + */ +export async function generateInvidiousDashManifestLocally(formats, invidiousInstance) { + // create a dummy player, as deciphering requires making requests to YouTube, + // which we want to avoid when Invidious is selected as the backend + const player = new Player() + player.decipher = (url) => url + + let urlTransformer + + if (invidiousInstance) { + /** + * @param {URL} url + */ + urlTransformer = (url) => { + return new URL(url.toString().replace(url.origin, invidiousInstance)) + } + } + + return await FormatUtils.toDash({ + adaptive_formats: formats + }, urlTransformer, undefined, undefined, player) +} + +export function convertInvidiousToLocalFormat(format) { + const [initStart, initEnd] = format.init.split('-') + const [indexStart, indexEnd] = format.index.split('-') + + const duration = parseInt(parseFloat(new URL(format.url).searchParams.get('dur')) * 1000) + + // only converts the properties that are needed to generate a DASH manifest with YouTube.js + // audioQuality and qualityLabel don't go inside the DASH manifest, but are used by YouTube.js + // to determine whether a format is an audio or video stream respectively. + + /** @type {import('./local').LocalFormat} */ + const localFormat = new Misc.Format({ + itag: format.itag, + mimeType: format.type, + bitrate: format.bitrate, + width: format.width, + height: format.height, + initRange: { + start: initStart, + end: initEnd + }, + indexRange: { + start: indexStart, + end: indexEnd + }, + // lastModified: format.lmt, + // contentLength: format.clen, + url: format.url, + approxDurationMs: duration, + ...(format.type.startsWith('audio/') + ? { + audioQuality: format.audioQuality, + audioSampleRate: format.audioSampleRate, + audioChannels: format.audioChannels + } + : { + fps: format.fps, + qualityLabel: format.qualityLabel, + colorInfo: format.colorInfo + }) + }) + + // Adding freeTubeUrl allows us to reuse the code, + // to generate the audio tracks for audio only mode, that we use for the local API + localFormat.freeTubeUrl = format.url + + return localFormat +} diff --git a/src/renderer/views/Watch/Watch.js b/src/renderer/views/Watch/Watch.js index eae5ca44cf508..4799f5188f97e 100644 --- a/src/renderer/views/Watch/Watch.js +++ b/src/renderer/views/Watch/Watch.js @@ -29,7 +29,13 @@ import { parseLocalTextRuns, parseLocalWatchNextVideo } from '../../helpers/api/local' -import { filterInvidiousFormats, invidiousGetVideoInformation, youtubeImageUrlToInvidious } from '../../helpers/api/invidious' +import { + convertInvidiousToLocalFormat, + filterInvidiousFormats, + generateInvidiousDashManifestLocally, + invidiousGetVideoInformation, + youtubeImageUrlToInvidious +} from '../../helpers/api/invidious' /** * @typedef {object} AudioSource @@ -594,63 +600,11 @@ export default defineComponent({ const hasMultipleAudioTracks = audioFormats.some(format => format.audio_track) if (hasMultipleAudioTracks) { - /** @type {string[]} */ - const ids = [] - - /** @type {AudioTrack[]} */ - const audioTracks = [] - - /** @type {import('youtubei.js').Misc.Format[][]} */ - const sourceLists = [] - - for (const format of audioFormats) { - // Some videos with multiple audio tracks, have a broken one, that doesn't have any audio track information - // It seems to be the same as default audio track but broken - // At the time of writing, this video has a broken audio track: https://youtu.be/UJeSWbR6W04 - if (!format.audio_track) { - continue - } - - const index = ids.indexOf(format.audio_track.id) - if (index === -1) { - ids.push(format.audio_track.id) - - let kind - - if (format.audio_track.audio_is_default) { - kind = 'main' - } else if (format.is_dubbed) { - kind = 'translation' - } else if (format.is_descriptive) { - kind = 'descriptions' - } else { - kind = 'alternative' - } - - audioTracks.push({ - id: format.audio_track.id, - kind, - label: format.audio_track.display_name, - language: format.language, - isDefault: format.audio_track.audio_is_default, - sourceList: [] - }) - - sourceLists.push([ - format - ]) - } else { - sourceLists[index].push(format) - } - } - - for (let i = 0; i < audioTracks.length; i++) { - audioTracks[i].sourceList = this.createLocalAudioSourceList(sourceLists[i]) - } + const audioTracks = this.createAudioTracksFromLocalFormats(audioFormats) this.audioTracks = audioTracks - this.audioSourceList = this.audioTracks.find(track => track.isDefault).sourceList + this.audioSourceList = audioTracks.find(track => track.isDefault).sourceList } else { this.audioTracks = [] @@ -713,7 +667,6 @@ export default defineComponent({ this.isLoading = true } - this.dashSrc = this.createInvidiousDashManifest() this.videoStoryboardSrc = `${this.currentInvidiousInstance}/api/v1/storyboards/${this.videoId}?height=90` invidiousGetVideoInformation(this.videoId) @@ -853,18 +806,28 @@ export default defineComponent({ return object })) - this.audioSourceList = result.adaptiveFormats.filter((format) => { - return format.type.includes('audio') - }).map((format) => { - return { - url: format.url, - type: format.type, - label: 'Audio', - qualityLabel: parseInt(format.bitrate) - } - }).sort((a, b) => { - return a.qualityLabel - b.qualityLabel - }) + this.audioTracks = [] + this.dashSrc = await this.createInvidiousDashManifest() + + if (process.env.IS_ELECTRON && this.audioTracks.length > 0) { + // when we are in Electron and the video has multiple audio tracks, + // we populate the list inside createInvidiousDashManifest + // as we need to work out the different audio tracks for the DASH manifest anyway + this.audioSourceList = this.audioTracks.find(track => track.isDefault).sourceList + } else { + this.audioSourceList = result.adaptiveFormats.filter((format) => { + return format.type.includes('audio') + }).map((format) => { + return { + url: format.url, + type: format.type, + label: 'Audio', + qualityLabel: parseInt(format.bitrate) + } + }).sort((a, b) => { + return a.qualityLabel - b.qualityLabel + }) + } if (this.activeFormat === 'audio') { this.activeSourceList = this.audioSourceList @@ -953,6 +916,68 @@ export default defineComponent({ } }, + /** + * @param {import('../../helpers/api/local').LocalFormat[]} audioFormats + * @returns {AudioTrack[]} + */ + createAudioTracksFromLocalFormats: function (audioFormats) { + /** @type {string[]} */ + const ids = [] + + /** @type {AudioTrack[]} */ + const audioTracks = [] + + /** @type {import('youtubei.js').Misc.Format[][]} */ + const sourceLists = [] + + for (const format of audioFormats) { + // Some videos with multiple audio tracks, have a broken one, that doesn't have any audio track information + // It seems to be the same as default audio track but broken + // At the time of writing, this video has a broken audio track: https://youtu.be/UJeSWbR6W04 + if (!format.audio_track) { + continue + } + + const index = ids.indexOf(format.audio_track.id) + if (index === -1) { + ids.push(format.audio_track.id) + + let kind + + if (format.audio_track.audio_is_default) { + kind = 'main' + } else if (format.is_dubbed) { + kind = 'translation' + } else if (format.is_descriptive) { + kind = 'descriptions' + } else { + kind = 'alternative' + } + + audioTracks.push({ + id: format.audio_track.id, + kind, + label: format.audio_track.display_name, + language: format.language, + isDefault: format.audio_track.audio_is_default, + sourceList: [] + }) + + sourceLists.push([ + format + ]) + } else { + sourceLists[index].push(format) + } + } + + for (let i = 0; i < audioTracks.length; i++) { + audioTracks[i].sourceList = this.createLocalAudioSourceList(sourceLists[i]) + } + + return audioTracks + }, + /** * @param {import('../../helpers/api/local').LocalFormat[]} audioFormats * @returns {AudioSource[]} @@ -1383,10 +1408,72 @@ export default defineComponent({ ] }, - createInvidiousDashManifest: function () { + createInvidiousDashManifest: async function () { let url = `${this.currentInvidiousInstance}/api/manifest/dash/id/${this.videoId}` - if (!process.env.IS_ELECTRON || this.proxyVideos) { + // If we are in Electron, + // we can use YouTube.js' DASH manifest generator to generate the manifest. + // Using YouTube.js' gives us support for multiple audio tracks (currently not supported by Invidious) + if (process.env.IS_ELECTRON) { + // Invidious' API response doesn't include the height and width (and fps and qualityLabel for AV1) of video streams + // so we need to extract them from Invidious' manifest + const response = await fetch(url) + const originalText = await response.text() + + const parsedManifest = new DOMParser().parseFromString(originalText, 'application/xml') + + /** @type {import('youtubei.js').Misc.Format[]} */ + const formats = [] + + /** @type {import('youtubei.js').Misc.Format[]} */ + const audioFormats = [] + + let hasMultipleAudioTracks = false + + for (const format of this.adaptiveFormats) { + if (format.type.startsWith('video/')) { + const representation = parsedManifest.querySelector(`Representation[id="${format.itag}"][bandwidth="${format.bitrate}"]`) + + format.height = parseInt(representation.getAttribute('height')) + format.width = parseInt(representation.getAttribute('width')) + format.fps = parseInt(representation.getAttribute('frameRate')) + + // the quality label is missing for AV1 formats + if (!format.qualityLabel) { + format.qualityLabel = format.width > format.height ? `${format.height}p` : `${format.width}p` + } + } + + const localFormat = convertInvidiousToLocalFormat(format) + + if (localFormat.has_audio) { + audioFormats.push(localFormat) + + if (localFormat.is_dubbed || localFormat.is_descriptive) { + hasMultipleAudioTracks = true + } + } + + formats.push(localFormat) + } + + if (hasMultipleAudioTracks) { + // match YouTube's local API response with English + const languageNames = new Intl.DisplayNames('en-US', { type: 'language' }) + for (const format of audioFormats) { + this.generateAudioTrackFieldInvidious(format, languageNames) + } + + this.audioTracks = this.createAudioTracksFromLocalFormats(audioFormats) + } + + const manifest = await generateInvidiousDashManifestLocally( + formats, + this.proxyVideos ? this.currentInvidiousInstance : undefined + ) + + url = `data:application/dash+xml;charset=UTF-8,${encodeURIComponent(manifest)}` + } else if (this.proxyVideos) { url += '?local=true' } @@ -1400,6 +1487,39 @@ export default defineComponent({ ] }, + /** + * @param {import('youtubei.js').Misc.Format} format + * @param {Intl.DisplayNames} languageNames + */ + generateAudioTrackFieldInvidious: function (format, languageNames) { + let type = '' + + // use the same id numbers as YouTube (except -1, when we aren't sure what it is) + let idNumber = '' + + if (format.is_descriptive) { + type = ' descriptive' + idNumber = 2 + } else if (format.is_dubbed) { + type = '' + idNumber = 3 + } else if (format.is_original) { + type = ' original' + idNumber = 4 + } else { + type = ' alternative' + idNumber = -1 + } + + const languageName = languageNames.of(format.language) + + format.audio_track = { + audio_is_default: !!format.is_original, + id: `${format.language}.${idNumber}`, + display_name: `${languageName}${type}` + } + }, + getAdaptiveFormatsInvidious: async function (existingInfoResult = null) { let result if (existingInfoResult) {