diff --git a/data/com.github.rafostar.Clapper.gschema.xml b/data/com.github.rafostar.Clapper.gschema.xml index 141c0440..c1828c72 100644 --- a/data/com.github.rafostar.Clapper.gschema.xml +++ b/data/com.github.rafostar.Clapper.gschema.xml @@ -103,14 +103,14 @@ Set PlayFlags for playbin - + false - Enable to use adaptive streaming for YouTube + Enable to use adaptive streaming 1 - Max YouTube video quality type + Max online video quality type diff --git a/src/assets/node-ytdl-core/sig.js b/src/assets/node-ytdl-core/sig.js deleted file mode 100644 index 6ecb161f..00000000 --- a/src/assets/node-ytdl-core/sig.js +++ /dev/null @@ -1,151 +0,0 @@ -/* Copyright (C) 2012-present by fent - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -const jsVarStr = '[a-zA-Z_\\$][a-zA-Z_0-9]*'; -const jsSingleQuoteStr = `'[^'\\\\]*(:?\\\\[\\s\\S][^'\\\\]*)*'`; -const jsDoubleQuoteStr = `"[^"\\\\]*(:?\\\\[\\s\\S][^"\\\\]*)*"`; -const jsQuoteStr = `(?:${jsSingleQuoteStr}|${jsDoubleQuoteStr})`; -const jsKeyStr = `(?:${jsVarStr}|${jsQuoteStr})`; -const jsPropStr = `(?:\\.${jsVarStr}|\\[${jsQuoteStr}\\])`; -const jsEmptyStr = `(?:''|"")`; -const reverseStr = ':function\\(a\\)\\{' + - '(?:return )?a\\.reverse\\(\\)' + -'\\}'; -const sliceStr = ':function\\(a,b\\)\\{' + - 'return a\\.slice\\(b\\)' + -'\\}'; -const spliceStr = ':function\\(a,b\\)\\{' + - 'a\\.splice\\(0,b\\)' + -'\\}'; -const swapStr = ':function\\(a,b\\)\\{' + - 'var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?' + -'\\}'; -const actionsObjRegexp = new RegExp( - `var (${jsVarStr})=\\{((?:(?:${ - jsKeyStr}${reverseStr}|${ - jsKeyStr}${sliceStr}|${ - jsKeyStr}${spliceStr}|${ - jsKeyStr}${swapStr - }),?\\r?\\n?)+)\\};`); -const actionsFuncRegexp = new RegExp(`${`function(?: ${jsVarStr})?\\(a\\)\\{` + - `a=a\\.split\\(${jsEmptyStr}\\);\\s*` + - `((?:(?:a=)?${jsVarStr}`}${ - jsPropStr -}\\(a,\\d+\\);)+)` + - `return a\\.join\\(${jsEmptyStr}\\)` + - `\\}`); -const reverseRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${reverseStr}`, 'm'); -const sliceRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${sliceStr}`, 'm'); -const spliceRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${spliceStr}`, 'm'); -const swapRegexp = new RegExp(`(?:^|,)(${jsKeyStr})${swapStr}`, 'm'); - -const swapHeadAndPosition = (arr, position) => { - const first = arr[0]; - arr[0] = arr[position % arr.length]; - arr[position] = first; - - return arr; -} - -function decipher(sig, tokens) { - sig = sig.split(''); - tokens = tokens.split(','); - - for(let i = 0, len = tokens.length; i < len; i++) { - let token = tokens[i], pos; - switch (token[0]) { - case 'r': - sig = sig.reverse(); - break; - case 'w': - pos = ~~token.slice(1); - sig = swapHeadAndPosition(sig, pos); - break; - case 's': - pos = ~~token.slice(1); - sig = sig.slice(pos); - break; - case 'p': - pos = ~~token.slice(1); - sig.splice(0, pos); - break; - } - } - - return sig.join(''); -}; - -function extractActions(body) { - const objResult = actionsObjRegexp.exec(body); - const funcResult = actionsFuncRegexp.exec(body); - - if(!objResult || !funcResult) - return null; - - const obj = objResult[1].replace(/\$/g, '\\$'); - const objBody = objResult[2].replace(/\$/g, '\\$'); - const funcBody = funcResult[1].replace(/\$/g, '\\$'); - - let result = reverseRegexp.exec(objBody); - const reverseKey = result && result[1] - .replace(/\$/g, '\\$') - .replace(/\$|^'|^"|'$|"$/g, ''); - result = sliceRegexp.exec(objBody); - const sliceKey = result && result[1] - .replace(/\$/g, '\\$') - .replace(/\$|^'|^"|'$|"$/g, ''); - result = spliceRegexp.exec(objBody); - const spliceKey = result && result[1] - .replace(/\$/g, '\\$') - .replace(/\$|^'|^"|'$|"$/g, ''); - result = swapRegexp.exec(objBody); - const swapKey = result && result[1] - .replace(/\$/g, '\\$') - .replace(/\$|^'|^"|'$|"$/g, ''); - - const keys = `(${[reverseKey, sliceKey, spliceKey, swapKey].join('|')})`; - const myreg = `(?:a=)?${obj - }(?:\\.${keys}|\\['${keys}'\\]|\\["${keys}"\\])` + - `\\(a,(\\d+)\\)`; - const tokenizeRegexp = new RegExp(myreg, 'g'); - const tokens = []; - - while((result = tokenizeRegexp.exec(funcBody)) !== null) { - const key = result[1] || result[2] || result[3]; - const pos = result[4]; - switch (key) { - case swapKey: - tokens.push(`w${result[4]}`); - break; - case reverseKey: - tokens.push('r'); - break; - case sliceKey: - tokens.push(`s${result[4]}`); - break; - case spliceKey: - tokens.push(`p${result[4]}`); - break; - } - } - - return tokens.join(','); -} diff --git a/src/debug.js b/src/debug.js index d5d0eb35..4a814593 100644 --- a/src/debug.js +++ b/src/debug.js @@ -14,22 +14,13 @@ const clapperDebugger = new Debug.Debugger('Clapper', { }), high_precision: true, }); -clapperDebugger.enabled = ( + +var enabled = ( clapperDebugger.enabled || G_DEBUG_ENV != null && G_DEBUG_ENV.includes('Clapper') ); - -const ytDebugger = new Debug.Debugger('YouTube', { - name_printer: new Ink.Printer({ - font: Ink.Font.BOLD, - color: Ink.Color.RED - }), - time_printer: new Ink.Printer({ - color: Ink.Color.LIGHT_BLUE - }), - high_precision: true, -}); +clapperDebugger.enabled = enabled; function _logStructured(debuggerName, msg, level) { @@ -43,23 +34,12 @@ function _logStructured(debuggerName, msg, level) function _debug(debuggerName, msg) { if(msg.message) { - _logStructured( - debuggerName, - msg.message, + _logStructured(debuggerName, msg.message, GLib.LogLevelFlags.LEVEL_CRITICAL ); - return; } - - switch(debuggerName) { - case 'Clapper': - clapperDebugger.debug(msg); - break; - case 'YouTube': - ytDebugger.debug(msg); - break; - } + clapperDebugger.debug(msg); } function debug(msg) @@ -67,12 +47,12 @@ function debug(msg) _debug('Clapper', msg); } -function ytDebug(msg) -{ - _debug('YouTube', msg); -} - function warn(msg) { _logStructured('Clapper', msg, GLib.LogLevelFlags.LEVEL_WARNING); } + +function message(msg) +{ + _logStructured('Clapper', msg, GLib.LogLevelFlags.LEVEL_MESSAGE); +} diff --git a/src/gtuber.js b/src/gtuber.js new file mode 100644 index 00000000..50ad5900 --- /dev/null +++ b/src/gtuber.js @@ -0,0 +1,297 @@ +const { Gio, GstClapper } = imports.gi; +const Debug = imports.src.debug; +const Misc = imports.src.misc; +const FileOps = imports.src.fileOps; +const Gtuber = Misc.tryImport('Gtuber'); + +const { debug, warn } = Debug; +const { settings } = Misc; + +const best = { + video: null, + audio: null, + video_audio: null, +}; +const codecPairs = []; +const qualityType = { + 0: 30, // normal + 1: 60, // hfr +}; + +var isAvailable = (Gtuber != null); +var cancellable = null; +let client = null; + +function resetBestStreams() +{ + best.video = null; + best.audio = null; + best.video_audio = null; +} + +function isStreamAllowed(stream, opts) +{ + const vcodec = stream.video_codec; + const acodec = stream.audio_codec; + + if( + vcodec + && (!vcodec.startsWith(opts.vcodec) + || (stream.height < 240 || stream.height > opts.height) + || stream.fps > qualityType[opts.quality]) + ) { + return false; + } + + if( + acodec + && (!acodec.startsWith(opts.acodec)) + ) { + return false; + } + + return (vcodec != null || acodec != null); +} + +function updateBestStreams(streams, opts) +{ + for(let stream of streams) { + if(!isStreamAllowed(stream, opts)) + continue; + + const type = (stream.video_codec && stream.audio_codec) + ? 'video_audio' + : (stream.video_codec) + ? 'video' + : 'audio'; + + if(!best[type] || best[type].bitrate < stream.bitrate) + best[type] = stream; + } +} + +function _streamFilter(opts, stream) +{ + switch(stream) { + case best.video: + return (best.audio != null || best.video_audio == null); + case best.audio: + return (best.video != null || best.video_audio == null); + case best.video_audio: + return (best.video == null || best.audio == null); + default: + return (opts.adaptive) + ? isStreamAllowed(stream, opts) + : false; + } +} + +function generateManifest(info, opts) +{ + const gen = new Gtuber.ManifestGenerator({ + pretty: Debug.enabled, + }); + gen.set_media_info(info); + gen.set_filter_func(_streamFilter.bind(this, opts)); + + debug('trying to get manifest'); + + for(let pair of codecPairs) { + opts.vcodec = pair[0]; + opts.acodec = pair[1]; + + /* Find best streams among adaptive ones */ + if (!opts.adaptive) + updateBestStreams(info.get_adaptive_streams(), opts); + + const data = gen.to_data(); + + /* Release our ref */ + if (!opts.adaptive) + resetBestStreams(); + + if(data) { + debug('got manifest'); + return data; + } + } + + debug('manifest not generated'); + + return null; +} + +function getBestCombinedUri(info, opts) +{ + const streams = info.get_streams(); + + debug('searching for best combined URI'); + + for(let pair of codecPairs) { + opts.vcodec = pair[0]; + opts.acodec = pair[1]; + + /* Find best non-adaptive stream */ + updateBestStreams(streams, opts); + + const bestUri = (best.video_audio) + ? best.video_audio.get_uri() + : (best.audio) + ? best.audio.get_uri() + : (best.video) + ? best.video.get_uri() + : null; + + /* Release our ref */ + resetBestStreams(); + + if(bestUri) { + debug('got best possible URI'); + return bestUri; + } + } + + /* If still nothing find stream by height */ + for(let stream of streams) { + const height = stream.get_height(); + if(!height || height > opts.height) + continue; + + if(!best.video_audio || best.video_audio.height < stream.height) + best.video_audio = stream; + } + + const anyUri = (best.video_audio) + ? best.video_audio.get_uri() + : null; + + /* Release our ref */ + resetBestStreams(); + + if (anyUri) + debug('got any URI'); + + return anyUri; +} + +async function _parseMediaInfoAsync(info, player) +{ + const resp = { + uri: null, + title: info.title, + }; + + const { root } = player.widget; + const surface = root.get_surface(); + const monitor = root.display.get_monitor_at_surface(surface); + + const opts = { + width: monitor.geometry.width * monitor.scale_factor, + height: monitor.geometry.height * monitor.scale_factor, + vcodec: null, + acodec: null, + quality: settings.get_int('yt-quality-type'), + adaptive: settings.get_boolean('yt-adaptive-enabled'), + }; + + if(info.has_adaptive_streams) { + const data = generateManifest(info, opts); + if(data) { + const manifestFile = await FileOps.saveFilePromise( + 'tmp', null, 'manifest', data + ).catch(debug); + + if(!manifestFile) + throw new Error('Gtuber: no manifest file was generated'); + + resp.uri = manifestFile.get_uri(); + + return resp; + } + } + + resp.uri = getBestCombinedUri(info, opts); + + if(!resp.uri) + throw new Error("Gtuber: no compatible stream found"); + + return resp; +} + +function _createClient(player) +{ + client = new Gtuber.Client(); + debug('created new gtuber client'); + + /* TODO: config based on what HW supports */ + //codecPairs.push(['vp9', 'opus']); + + codecPairs.push(['avc', 'mp4a']); +} + +function mightHandleUri(uri) +{ + const unsupported = [ + 'file', 'fd', 'dvd', 'cdda', + 'dvb', 'v4l2', 'gs' + ]; + return !unsupported.includes(Misc.getUriProtocol(uri)); +} + +function cancelFetching() +{ + if(cancellable && !cancellable.is_cancelled()) + cancellable.cancel(); +} + +function parseUriPromise(uri, player) +{ + return new Promise((resolve, reject) => { + if(!client) { + if(!isAvailable) { + debug('gtuber is not installed'); + return resolve({ uri, title: null }); + } + _createClient(player); + } + + /* Stop to show reaction and restore internet bandwidth */ + if(player.state !== GstClapper.ClapperState.STOPPED) + player.stop(); + + cancellable = new Gio.Cancellable(); + debug('gtuber is fetching media info...'); + + client.fetch_media_info_async(uri, cancellable, (client, task) => { + cancellable = null; + let info = null; + + try { + info = client.fetch_media_info_finish(task); + debug('gtuber successfully fetched media info'); + } + catch(err) { + const taskCancellable = task.get_cancellable(); + + if(taskCancellable.is_cancelled()) + return reject(err); + + const gtuberNoPlugin = ( + err.domain === Gtuber.ClientError.quark() + && err.code === Gtuber.ClientError.NO_PLUGIN + ); + if(!gtuberNoPlugin) + return reject(err); + + warn(`Gtuber: ${err.message}, trying URI as is...`); + + /* Allow handling URI as is via GStreamer plugins */ + return resolve({ uri, title: null }); + } + + _parseMediaInfoAsync(info, player) + .then(resp => resolve(resp)) + .catch(err => reject(err)); + }); + }); +} diff --git a/src/main.js b/src/main.js index 71f8a554..cb10edab 100644 --- a/src/main.js +++ b/src/main.js @@ -1,6 +1,7 @@ imports.gi.versions.Gdk = '4.0'; imports.gi.versions.Gtk = '4.0'; imports.gi.versions.Soup = '2.4'; +imports.gi.versions.Gtuber = '0.0'; pkg.initGettext(); pkg.initFormat(); diff --git a/src/mainRemote.js b/src/mainRemote.js index 216af851..cc412f7f 100644 --- a/src/mainRemote.js +++ b/src/mainRemote.js @@ -1,6 +1,7 @@ imports.gi.versions.Gdk = '4.0'; imports.gi.versions.Gtk = '4.0'; imports.gi.versions.Soup = '2.4'; +imports.gi.versions.Gtuber = '0.0'; pkg.initGettext(); diff --git a/src/misc.js b/src/misc.js index 8b390089..cf443038 100644 --- a/src/misc.js +++ b/src/misc.js @@ -1,7 +1,8 @@ const { Gio, GLib, Gdk, Gtk } = imports.gi; const Debug = imports.src.debug; -const { debug } = Debug; +const { debug, message } = Debug; +const failedImports = []; var appName = 'Clapper'; var appId = 'com.github.rafostar.Clapper'; @@ -28,6 +29,23 @@ const subsKeys = Object.keys(subsTitles); let inhibitCookie; +function tryImport(libName) +{ + let lib = null; + + try { + lib = imports.gi[libName]; + } + catch(err) { + if(!failedImports.includes(libName)) { + failedImports.push(libName); + message(err.message); + } + } + + return lib; +} + function getResourceUri(path) { const res = `file://${pkg.pkgdatadir}/${path}`; @@ -224,22 +242,3 @@ function getIsTouch(gesture) return false; } } - -function encodeHTML(text) -{ - return text.replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} - -function decodeURIPlus(uri) -{ - return decodeURI(uri.replace(/\+/g, ' ')); -} - -function isHex(num) -{ - return Boolean(num.match(/[0-9a-f]+$/i)); -} diff --git a/src/player.js b/src/player.js index be0aeae3..b11a1f8d 100644 --- a/src/player.js +++ b/src/player.js @@ -2,7 +2,7 @@ const { Gdk, Gio, GObject, Gst, GstClapper, Gtk } = imports.gi; const ByteArray = imports.byteArray; const Debug = imports.src.debug; const Misc = imports.src.misc; -const YouTube = imports.src.youtube; +const Gtuber = imports.src.gtuber; const { PlaylistWidget } = imports.src.playlist; const { WebApp } = imports.src.webApp; @@ -45,7 +45,6 @@ class ClapperPlayer extends GstClapper.Clapper this.webserver = null; this.webapp = null; - this.ytClient = null; this.playlistWidget = new PlaylistWidget(); this.seekDone = true; @@ -142,24 +141,13 @@ class ClapperPlayer extends GstClapper.Clapper set_uri(uri) { this.customVideoTitle = null; + Gtuber.cancelFetching(); - if(Misc.getUriProtocol(uri) !== 'file') { - const [isYouTubeUri, videoId] = YouTube.checkYouTubeUri(uri); - - if(!isYouTubeUri) - return super.set_uri(uri); - - if(!this.ytClient) - this.ytClient = new YouTube.YouTubeClient(); - - const { root } = this.widget; - const surface = root.get_surface(); - const monitor = root.display.get_monitor_at_surface(surface); - - this.ytClient.getPlaybackDataAsync(videoId, monitor) - .then(data => { - this.customVideoTitle = data.title; - super.set_uri(data.uri); + if(Gtuber.mightHandleUri(uri)) { + Gtuber.parseUriPromise(uri, this) + .then(res => { + this.customVideoTitle = res.title; + super.set_uri(res.uri); }) .catch(debug); diff --git a/src/prefs.js b/src/prefs.js index a4acaa8f..415ba1c4 100644 --- a/src/prefs.js +++ b/src/prefs.js @@ -1,6 +1,7 @@ const { Adw, GObject, Gio, Gst, Gtk } = imports.gi; const Debug = imports.src.debug; const Misc = imports.src.misc; +const Gtuber = imports.src.gtuber; const { debug } = Debug; const { settings } = Misc; @@ -537,6 +538,7 @@ class ClapperPrefsPluginRankingSubpage extends Gtk.Box var PrefsWindow = GObject.registerClass({ GTypeName: 'ClapperPrefsWindow', Template: Misc.getResourceUri('ui/preferences-window.ui'), + InternalChildren: ['gtuber_group'], }, class ClapperPrefsWindow extends Adw.PreferencesWindow { @@ -546,6 +548,7 @@ class ClapperPrefsWindow extends Adw.PreferencesWindow transient_for: window, }); + this._gtuber_group.visible = Gtuber.isAvailable; this.show(); } }); diff --git a/src/widget.js b/src/widget.js index c709573b..693e3265 100644 --- a/src/widget.js +++ b/src/widget.js @@ -4,7 +4,6 @@ const Debug = imports.src.debug; const Dialogs = imports.src.dialogs; const Misc = imports.src.misc; const { Player } = imports.src.player; -const YouTube = imports.src.youtube; const Revealers = imports.src.revealers; const { debug } = Debug; @@ -803,12 +802,10 @@ class ClapperWidget extends Gtk.Grid { const dropTarget = new Gtk.DropTarget({ actions: Gdk.DragAction.COPY | Gdk.DragAction.MOVE, - preload: true, }); dropTarget.set_gtypes([GObject.TYPE_STRING]); dropTarget.connect('motion', this._onDataMotion.bind(this)); dropTarget.connect('drop', this._onDataDrop.bind(this)); - dropTarget.connect('notify::value', this._onDropValueNotify.bind(this)); return dropTarget; } @@ -1009,36 +1006,6 @@ class ClapperWidget extends Gtk.Grid this.posY = posY; } - _onDropValueNotify(dropTarget) - { - if(!dropTarget.value) - return; - - const uris = dropTarget.value.split(/\r?\n/); - const firstUri = uris[0]; - - if(uris.length > 1 || !Gst.uri_is_valid(firstUri)) - return; - - /* Check if user is dragging a YouTube link */ - const [isYouTubeUri, videoId] = YouTube.checkYouTubeUri(firstUri); - if(!isYouTubeUri) return; - - /* Since this is a YouTube video, - * create YT client if it was not created yet */ - if(!this.player.ytClient) - this.player.ytClient = new YouTube.YouTubeClient(); - - const { ytClient } = this.player; - - /* Speed up things by prefetching new video info before drop */ - if( - !ytClient.compareLastVideoId(videoId) - && ytClient.downloadingVideoId !== videoId - ) - ytClient.getVideoInfoPromise(videoId).catch(debug); - } - _onDataMotion(dropTarget, x, y) { return Gdk.DragAction.MOVE; diff --git a/src/youtube.js b/src/youtube.js deleted file mode 100644 index 7cd7d8e1..00000000 --- a/src/youtube.js +++ /dev/null @@ -1,1003 +0,0 @@ -const { GObject, Gst, Soup } = imports.gi; -const Dash = imports.src.dash; -const Debug = imports.src.debug; -const FileOps = imports.src.fileOps; -const Misc = imports.src.misc; -const YTItags = imports.src.youtubeItags; -const YTDL = imports.src.assets['node-ytdl-core']; - -const debug = Debug.ytDebug; -const { settings } = Misc; - -const InitAsyncState = { - NONE: 0, - IN_PROGRESS: 1, - DONE: 2, -}; - -var YouTubeClient = GObject.registerClass({ - Signals: { - 'info-resolved': { - param_types: [GObject.TYPE_BOOLEAN] - } - } -}, class ClapperYouTubeClient extends Soup.Session -{ - _init() - { - super._init({ - timeout: 7, - max_conns_per_host: 1, - /* TODO: share this with GstClapper lib (define only once) */ - user_agent: 'Mozilla/5.0 (X11; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0', - }); - this.initAsyncState = InitAsyncState.NONE; - - /* videoID of current active download */ - this.downloadingVideoId = null; - - this.lastInfo = null; - this.postInfo = { - clientVersion: "2.20210605.09.00", - visitorData: "", - }; - - this.cachedSig = { - id: null, - actions: null, - timestamp: "", - }; - } - - getVideoInfoPromise(videoId) - { - /* If in middle of download and same videoID, - * resolve to current download */ - if( - this.downloadingVideoId - && this.downloadingVideoId === videoId - ) - return this._getCurrentDownloadPromise(); - - return new Promise(async (resolve, reject) => { - /* Do not redownload info for the same video */ - if(this.compareLastVideoId(videoId)) - return resolve(this.lastInfo); - - this.abort(); - - /* Prevent doing this code more than once at a time */ - if(this.initAsyncState === InitAsyncState.NONE) { - this.initAsyncState = InitAsyncState.IN_PROGRESS; - - debug('loading cookies DB'); - const cacheDir = await FileOps.createCacheDirPromise().catch(debug); - if(!cacheDir) { - this.initAsyncState = InitAsyncState.NONE; - return reject(new Error('could not create cookies DB')); - } - - const cookiesDB = new Soup.CookieJarDB({ - filename: cacheDir.get_child('cookies.sqlite').get_path(), - read_only: false, - }); - this.add_feature(cookiesDB); - debug('successfully loaded cookies DB'); - - this.initAsyncState = InitAsyncState.DONE; - } - - /* Too many tries might trigger 429 ban, - * leave while with break as a "goto" replacement */ - let tries = 1; - while(tries--) { - debug(`obtaining YouTube video info: ${videoId}`); - this.downloadingVideoId = videoId; - - let result; - let isFoundInTemp = false; - let isUsingPlayerResp = false; - - const tempInfo = await FileOps.getFileContentsPromise('tmp', 'yt-info', videoId).catch(debug); - if(tempInfo) { - debug('checking temp info for requested video'); - let parsedTempInfo; - - try { parsedTempInfo = JSON.parse(tempInfo); } - catch(err) { debug(err); } - - if(parsedTempInfo) { - const nowSeconds = Math.floor(Date.now() / 1000); - const { expireDate } = parsedTempInfo.streamingData; - - if(expireDate && expireDate > nowSeconds) { - debug(`found usable info, remaining live: ${expireDate - nowSeconds}`); - - isFoundInTemp = true; - result = { data: parsedTempInfo }; - } - else - debug('temp info expired'); - } - } - - if(!result) - result = await this._getPlayerInfoPromise(videoId).catch(debug); - if(!result || !result.data) { - if(result && result.isAborted) { - debug(new Error('download aborted')); - break; - } - } - isUsingPlayerResp = (result != null); - - if(!result) - result = await this._getInfoPromise(videoId).catch(debug); - if(!result || !result.data) { - if(result && result.isAborted) - debug(new Error('download aborted')); - - break; - } - - if(!isFoundInTemp) { - const [isPlayable, reason] = this._getPlayabilityStatus(result.data); - - if(!isPlayable) { - debug(new Error(reason)); - break; - } - } - - let info = this._getReducedInfo(result.data); - - if(this._getIsCipher(info.streamingData)) { - debug('video requires deciphering'); - - /* Decipher actions do not change too often, so try - * to reuse without triggering too many requests ban */ - let actions = this.cachedSig.actions; - - if(actions) - debug('using remembered decipher actions'); - else { - let sts = ""; - const embedUri = `https://www.youtube.com/embed/${videoId}`; - result = await this._downloadDataPromise(embedUri).catch(debug); - - if(result && result.isAborted) - break; - else if(!result || !result.data) { - debug(new Error('could not download embed body')); - break; - } - - let ytPath = result.data.match(/jsUrl\":\"(.*?)\.js/g); - if(ytPath) { - ytPath = (ytPath[0] && ytPath[0].length > 16) - ? ytPath[0].substring(8) : null; - } - if(!ytPath) { - debug(new Error('could not find YouTube player URI')); - break; - } - const ytUri = `https://www.youtube.com${ytPath}`; - if( - /* check if site has "/" after ".com" */ - ytUri[23] !== '/' - || !Gst.Uri.is_valid(ytUri) - ) { - debug(`misformed player URI: ${ytUri}`); - break; - } - debug(`found player URI: ${ytUri}`); - - const ytId = ytPath.split('/').find(el => Misc.isHex(el)); - let ytSigData = await FileOps.getFileContentsPromise( - 'user_cache', 'yt-sig', ytId - ).catch(debug); - - if(ytSigData) { - ytSigData = ytSigData.split(';'); - - if(ytSigData[0] && ytSigData[0] > 0) { - sts = ytSigData[0]; - debug(`found local sts: ${sts}`); - } - - const actionsIndex = (ytSigData.length > 1) ? 1 : 0; - actions = ytSigData[actionsIndex]; - } - - if(!actions) { - result = await this._downloadDataPromise(ytUri).catch(debug); - - if(result && result.isAborted) - break; - else if(!result || !result.data) { - debug(new Error('could not download player body')); - break; - } - - const stsArr = result.data.match(/signatureTimestamp[=\:]\d+/g); - if(stsArr) { - sts = (stsArr[0] && stsArr[0].length > 19) - ? stsArr[0].substring(19) : null; - - if(isNaN(sts) || sts <= 0) - sts = ""; - else - debug(`extracted player sts: ${sts}`); - } - - actions = YTDL.sig.extractActions(result.data); - if(actions) { - debug('deciphered, saving cipher actions to cache file'); - const saveData = sts + ';' + actions; - /* We do not need to wait for it */ - FileOps.saveFilePromise('user_cache', 'yt-sig', ytId, saveData); - } - } - if(!actions || !actions.length) { - debug(new Error('could not extract decipher actions')); - break; - } - if(this.cachedSig.id !== ytId) { - this.cachedSig.id = ytId; - this.cachedSig.actions = actions; - this.cachedSig.timestamp = sts; - - /* Cipher info from player without timestamp is invalid - * so download it again now that we have a timestamp */ - if(isUsingPlayerResp && sts > 0) { - debug(`redownloading player info with sts: ${sts}`); - - result = await this._getPlayerInfoPromise(videoId).catch(debug); - if(!result || !result.data) { - if(result && result.isAborted) - debug(new Error('download aborted')); - - break; - } - info = this._getReducedInfo(result.data); - } - } - } - debug(`successfully obtained decipher actions: ${actions}`); - - const isDeciphered = this._decipherStreamingData( - info.streamingData, actions - ); - if(!isDeciphered) { - debug('streaming data could not be deciphered'); - break; - } - } - - if(!isFoundInTemp) { - const exp = info.streamingData.expiresInSeconds || 0; - const dateSeconds = Math.floor(Date.now() / 1000); - - /* Estimated safe time for rewatching video */ - info.streamingData.expireDate = dateSeconds + Number(exp); - - /* Last info is stored in variable, so don't wait here */ - FileOps.saveFilePromise( - 'tmp', 'yt-info', videoId, JSON.stringify(info) - ); - } - - this.lastInfo = info; - debug('video info is ready to use'); - - this.emit('info-resolved', true); - this.downloadingVideoId = null; - - return resolve(info); - } - - /* Do not clear video info here, as we might still have - * valid info from last video that can be reused */ - this.emit('info-resolved', false); - this.downloadingVideoId = null; - - reject(new Error('could not obtain YouTube video info')); - }); - } - - async getPlaybackDataAsync(videoId, monitor) - { - const info = await this.getVideoInfoPromise(videoId).catch(debug); - - if(!info) - throw new Error('no YouTube video info'); - - let uri = null; - const itagOpts = { - width: monitor.geometry.width * monitor.scale_factor, - height: monitor.geometry.height * monitor.scale_factor, - codec: 'h264', - type: YTItags.QualityType[settings.get_int('yt-quality-type')], - adaptive: settings.get_boolean('yt-adaptive-enabled'), - }; - - uri = await this.getHLSUriAsync(info, itagOpts); - - if(!uri) { - const dashInfo = await this.getDashInfoAsync(info, itagOpts).catch(debug); - - if(dashInfo) { - debug('parsed video info to dash info'); - const dash = Dash.generateDash(dashInfo); - - if(dash) { - debug('got dash data'); - - const dashFile = await FileOps.saveFilePromise( - 'tmp', null, 'clapper.mpd', dash - ).catch(debug); - - if(dashFile) - uri = dashFile.get_uri(); - - debug('got dash file'); - } - } - } - - if(!uri) - uri = this.getBestCombinedUri(info, itagOpts); - - if(!uri) - throw new Error('no YouTube video URI'); - - debug(`final URI: ${uri}`); - - const title = (info.videoDetails && info.videoDetails.title) - ? Misc.decodeURIPlus(info.videoDetails.title) - : videoId; - - debug(`title: ${title}`); - - return { uri, title }; - } - - async getHLSUriAsync(info, itagOpts) - { - const isLive = ( - info.videoDetails.isLiveContent - && (!info.videoDetails.lengthSeconds - || Number(info.videoDetails.lengthSeconds) <= 0) - ); - debug(`video is live: ${isLive}`); - - /* YouTube only uses HLS for live content */ - if(!isLive) - return null; - - const hlsUri = info.streamingData.hlsManifestUrl; - if(!hlsUri) { - /* HLS may be unavailable on finished live streams */ - debug('no HLS manifest URL'); - return null; - } - - if(!itagOpts.adaptive) { - const result = await this._downloadDataPromise(hlsUri).catch(debug); - if(!result || !result.data) { - debug(new Error('HLS manifest download failed')); - return hlsUri; - } - - const hlsArr = result.data.split('\n'); - const hlsStreams = []; - - let index = hlsArr.length; - while(index--) { - const url = hlsArr[index]; - if(!Gst.Uri.is_valid(url)) - continue; - - const itagIndex = url.indexOf('/itag/') + 6; - const itag = url.substring(itagIndex, itagIndex + 2); - - hlsStreams.push({ itag, url }); - } - - debug(`obtaining HLS itags for resolution: ${itagOpts.width}x${itagOpts.height}`); - const hlsItags = YTItags.getHLSItags(itagOpts); - debug(`HLS itags: ${JSON.stringify(hlsItags)}`); - - const hlsStream = this.getBestStreamFromItags(hlsStreams, hlsItags); - if(hlsStream) - return hlsStream.url; - } - - return hlsUri; - } - - async getDashInfoAsync(info, itagOpts) - { - if( - !info.streamingData - || !info.streamingData.adaptiveFormats - || !info.streamingData.adaptiveFormats.length - ) - return null; - - debug(`obtaining DASH itags for resolution: ${itagOpts.width}x${itagOpts.height}`); - const dashItags = YTItags.getDashItags(itagOpts); - debug(`DASH itags: ${JSON.stringify(dashItags)}`); - - const filteredStreams = { - video: [], - audio: [], - }; - - for(let fmt of ['video', 'audio']) { - debug(`filtering ${fmt} streams`); - let index = dashItags[fmt].length; - - while(index--) { - const itag = dashItags[fmt][index]; - const foundStream = info.streamingData.adaptiveFormats.find(stream => stream.itag == itag); - if(foundStream) { - /* Parse and convert mimeType string into object */ - foundStream.mimeInfo = this._getMimeInfo(foundStream.mimeType); - - /* Sanity check */ - if(!foundStream.mimeInfo || foundStream.mimeInfo.content !== fmt) { - debug(new Error(`mimeType parsing failed on stream: ${itag}`)); - continue; - } - - /* Sort from worst to best */ - filteredStreams[fmt].unshift(foundStream); - debug(`added ${fmt} itag: ${foundStream.itag}`); - - if(!itagOpts.adaptive) - break; - } - } - if(!filteredStreams[fmt].length) { - debug(`dash info ${fmt} streams list is empty`); - return null; - } - } - - debug('following redirects'); - - for(let fmtArr of Object.values(filteredStreams)) { - for(let stream of fmtArr) { - debug(`initial URL: ${stream.url}`); - - /* Errors in some cases are to be expected here, - * so be quiet about them and use fallback methods */ - const result = await this._downloadDataPromise( - stream.url, 'HEAD' - ).catch(err => debug(err.message)); - - if(!result || !result.uri) { - debug('redirect could not be resolved'); - return null; - } - - stream.url = Misc.encodeHTML(result.uri) - .replace('?', '/') - .replace(/&/g, '/') - .replace(/=/g, '/'); - - debug(`resolved URL: ${stream.url}`); - } - } - - debug('all redirects resolved'); - - return { - duration: info.videoDetails.lengthSeconds, - adaptations: [ - filteredStreams.video, - filteredStreams.audio, - ] - }; - } - - getBestCombinedUri(info, itagOpts) - { - debug(`obtaining best combined URL for resolution: ${itagOpts.width}x${itagOpts.height}`); - const streams = info.streamingData.formats; - - if(!streams.length) - return null; - - const combinedItags = YTItags.getCombinedItags(itagOpts); - let combinedStream = this.getBestStreamFromItags(streams, combinedItags); - - if(!combinedStream) { - debug('trying any combined stream as last resort'); - combinedStream = streams[streams.length - 1]; - } - - if(!combinedStream || !combinedStream.url) - return null; - - return combinedStream.url; - } - - getBestStreamFromItags(streams, itags) - { - let index = itags.length; - - while(index--) { - const itag = itags[index]; - const stream = streams.find(stream => stream.itag == itag); - if(stream) { - debug(`found preferred stream itag: ${stream.itag}`); - return stream; - } - } - debug('could not find preferred stream for itags'); - - return null; - } - - compareLastVideoId(videoId) - { - if(!this.lastInfo) - return false; - - if( - !this.lastInfo - || !this.lastInfo.videoDetails - || this.lastInfo.videoDetails.videoId !== videoId - /* TODO: check if video expired */ - ) - return false; - - return true; - } - - _downloadDataPromise(url, method, reqData) - { - method = method || 'GET'; - - return new Promise((resolve, reject) => { - const message = Soup.Message.new(method, url); - const result = { - data: null, - isAborted: false, - uri: null, - }; - - if(reqData) { - message.set_request( - "application/json", - Soup.MemoryUse.COPY, - reqData - ); - } - - this.queue_message(message, (session, msg) => { - debug('got message response'); - const statusCode = msg.status_code; - - if(statusCode === 200) { - result.data = msg.response_body.data; - - if(method === 'HEAD') - result.uri = msg.uri.to_string(false); - - return resolve(result); - } - - debug(`response code: ${statusCode}`); - - /* Internal Soup codes mean download aborted - * or some other error that cannot be handled - * and we do not want to retry in such case */ - if(statusCode < 10 || statusCode === 429) { - result.isAborted = true; - return resolve(result); - } - - return reject(new Error('could not download data')); - }); - }); - } - - _getCurrentDownloadPromise() - { - debug('resolving after current download finishes'); - - return new Promise((resolve, reject) => { - const infoResolvedSignal = this.connect('info-resolved', (self, success) => { - this.disconnect(infoResolvedSignal); - - debug('current download finished, resolving'); - - if(!success) - return reject(new Error('info resolve was unsuccessful')); - - /* At this point new video info is set */ - resolve(this.lastInfo); - }); - }); - } - - _getPlayabilityStatus(info) - { - if( - !info.playabilityStatus - || !info.playabilityStatus.status === 'OK' - ) - return [false, 'video is not playable']; - - if(!info.streamingData) - return [false, 'video response data is missing streaming data']; - - return [true, null]; - } - - _getReducedInfo(info) - { - const reduced = { - videoDetails: { - videoId: info.videoDetails.videoId, - title: info.videoDetails.title, - lengthSeconds: info.videoDetails.lengthSeconds, - isLiveContent: info.videoDetails.isLiveContent - }, - streamingData: info.streamingData - }; - - /* Make sure we have all formats arrays, - * so we will not have to keep checking */ - if(!reduced.streamingData.formats) - reduced.streamingData.formats = []; - if(!reduced.streamingData.adaptiveFormats) - reduced.streamingData.adaptiveFormats = []; - - return reduced; - } - - _getMimeInfo(mimeType) - { - debug(`parsing mimeType: ${mimeType}`); - - const mimeArr = mimeType.split(';'); - - let codecs = mimeArr.find(info => info.includes('codecs')).split('=')[1]; - codecs = codecs.substring(1, codecs.length - 1); - - const mimeInfo = { - content: mimeArr[0].split('/')[0], - type: mimeArr[0], - codecs, - }; - - debug(`parsed mimeType: ${JSON.stringify(mimeInfo)}`); - - return mimeInfo; - } - - _getPlayerInfoPromise(videoId) - { - const data = this._getPlayerPostData(videoId); - const apiKey = 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; - const url = `https://www.youtube.com/youtubei/v1/player?key=${apiKey}`; - - return new Promise((resolve, reject) => { - if(!data) { - debug('not using player info due to missing data'); - return resolve(null); - } - debug('downloading info from player'); - - this._downloadDataPromise(url, 'POST', data).then(result => { - if(result.isAborted) - return resolve(result); - - debug('parsing player info JSON'); - - let info = null; - - try { info = JSON.parse(result.data); } - catch(err) { debug(err.message); } - - if(!info) - return reject(new Error('could not parse video info JSON')); - - debug('successfully parsed video info JSON'); - - /* Update post info values from response */ - if(info && info.responseContext && info.responseContext.visitorData) { - const visData = info.responseContext.visitorData; - - this.postInfo.visitorData = visData; - debug(`new visitor ID: ${visData}`); - } - - result.data = info; - resolve(result); - }) - .catch(err => reject(err)); - }); - } - - _getInfoPromise(videoId) - { - return new Promise((resolve, reject) => { - const query = [ - `video_id=${videoId}`, - `html5=1`, - `el=embedded`, - `eurl=https://youtube.googleapis.com/v/${videoId}`, - `sts=${this.cachedSig.timestamp}`, - ].join('&'); - const url = `https://www.youtube.com/get_video_info?${query}`; - - debug('downloading info from video'); - - this._downloadDataPromise(url).then(result => { - if(result.isAborted) - return resolve(result); - - debug('parsing video info JSON'); - - const gstUri = Gst.Uri.from_string('?' + result.data); - - if(!gstUri) - return reject(new Error('could not convert query to URI')); - - const playerResponse = gstUri.get_query_value('player_response'); - const cliVer = gstUri.get_query_value('cver'); - - if(cliVer && cliVer !== this.postInfo.clientVersion) { - this.postInfo.clientVersion = cliVer; - debug(`updated client version: ${cliVer}`); - } - - if(!playerResponse) - return reject(new Error('no player response in query')); - - let info = null; - - try { info = JSON.parse(playerResponse); } - catch(err) { debug(err.message); } - - if(!info) - return reject(new Error('could not parse video info JSON')); - - debug('successfully parsed video info JSON'); - result.data = info; - - resolve(result); - }) - .catch(err => reject(err)); - }); - } - - _getIsCipher(data) - { - const stream = (data.formats.length) - ? data.formats[0] - : data.adaptiveFormats[0]; - - if(!stream) { - debug(new Error('no streams')); - return false; - } - - if(stream.url) - return false; - - if( - stream.signatureCipher - || stream.cipher - ) - return true; - - /* FIXME: no URLs and no cipher, what now? */ - debug(new Error('no url or cipher in streams')); - - return false; - } - - _decipherStreamingData(data, actions) - { - debug('checking cipher query keys'); - - /* Cipher query keys should be the same for all - * streams, so parse any stream to get their names */ - const anyStream = data.formats[0] || data.adaptiveFormats[0]; - const sigQuery = anyStream.signatureCipher || anyStream.cipher; - - if(!sigQuery) - return false; - - const gstUri = Gst.Uri.from_string('?' + sigQuery); - const queryKeys = gstUri.get_query_keys(); - - const cipherKey = queryKeys.find(key => { - const value = gstUri.get_query_value(key); - /* A long value that is not URI */ - return ( - value.length > 32 - && !Gst.Uri.is_valid(value) - ); - }); - if(!cipherKey) { - debug('no stream cipher key name'); - return false; - } - - const sigKey = queryKeys.find(key => { - const value = gstUri.get_query_value(key); - /* A short value that is not URI */ - return ( - value.length < 32 - && !Gst.Uri.is_valid(value) - ); - }); - if(!sigKey) { - debug('no stream signature key name'); - return false; - } - - const urlKey = queryKeys.find(key => - Gst.Uri.is_valid(gstUri.get_query_value(key)) - ); - if(!urlKey) { - debug('no stream URL key name'); - return false; - } - - const cipherKeys = { - url: urlKey, - sig: sigKey, - cipher: cipherKey, - }; - - debug('deciphering streams'); - - for(let format of [data.formats, data.adaptiveFormats]) { - for(let stream of format) { - const formatUrl = this._getDecipheredUrl( - stream, actions, cipherKeys - ); - if(!formatUrl) { - debug('undecipherable stream'); - debug(stream); - - return false; - } - stream.url = formatUrl; - - /* Remove unneeded data */ - if(stream.signatureCipher) - delete stream.signatureCipher; - if(stream.cipher) - delete stream.cipher; - } - } - debug('all streams deciphered'); - - return true; - } - - _getDecipheredUrl(stream, actions, queryKeys) - { - debug(`deciphering stream id: ${stream.itag}`); - - const sigQuery = stream.signatureCipher || stream.cipher; - if(!sigQuery) return null; - - const gstUri = Gst.Uri.from_string('?' + sigQuery); - - const url = gstUri.get_query_value(queryKeys.url); - const cipher = gstUri.get_query_value(queryKeys.cipher); - const sig = gstUri.get_query_value(queryKeys.sig); - - const key = YTDL.sig.decipher(cipher, actions); - if(!key) return null; - - debug('stream deciphered'); - - return `${url}&${sig}=${encodeURIComponent(key)}`; - } - - _getPlayerPostData(videoId) - { - const cliVer = this.postInfo.clientVersion; - if(!cliVer) return null; - - const visitor = this.postInfo.visitorData; - const sts = this.cachedSig.timestamp || null; - - const ua = this.user_agent; - const browserVer = ua.substring(ua.lastIndexOf('/') + 1); - - if(!visitor) - debug('visitor ID is unknown'); - - const data = { - videoId: videoId, - context: { - client: { - visitorData: visitor, - userAgent: `${ua},gzip(gfe)`, - clientName: "WEB", - clientVersion: cliVer, - osName: "X11", - osVersion: "", - originalUrl: `https://www.youtube.com/watch?v=${videoId}`, - browserName: "Firefox", - browserVersion: browserVer, - playerType: "UNIPLAYER" - }, - user: { - lockedSafetyMode: false - }, - request: { - useSsl: true, - internalExperimentFlags: [], - consistencyTokenJars: [] - } - }, - playbackContext: { - contentPlaybackContext: { - html5Preference: "HTML5_PREF_WANTS", - lactMilliseconds: "1069", - referer: `https://www.youtube.com/watch?v=${videoId}`, - signatureTimestamp: sts, - autoCaptionsDefaultOn: false, - liveContext: { - startWalltime: "0" - } - } - }, - captionParams: {} - }; - - return JSON.stringify(data); - } -}); - -function checkYouTubeUri(uri) -{ - const gstUri = Gst.Uri.from_string(uri); - const originalHost = gstUri.get_host(); - gstUri.normalize(); - - const host = gstUri.get_host(); - let videoId = null; - - switch(host) { - case 'www.youtube.com': - case 'youtube.com': - case 'm.youtube.com': - videoId = gstUri.get_query_value('v'); - if(!videoId) { - /* Handle embedded videos */ - const segments = gstUri.get_path_segments(); - if(segments && segments.length) - videoId = segments[segments.length - 1]; - } - break; - case 'youtu.be': - videoId = gstUri.get_path_segments()[1]; - break; - default: - const scheme = gstUri.get_scheme(); - if(scheme === 'yt' || scheme === 'youtube') { - /* ID is case sensitive */ - videoId = originalHost; - break; - } - break; - } - - const success = (videoId != null); - - return [success, videoId]; -} diff --git a/src/youtubeItags.js b/src/youtubeItags.js deleted file mode 100644 index 09b69326..00000000 --- a/src/youtubeItags.js +++ /dev/null @@ -1,85 +0,0 @@ -var QualityType = { - 0: 'normal', - 1: 'hfr', -}; - -const Itags = { - video: { - h264: { - normal: { - 240: 133, - 360: 134, - 480: 135, - 720: 136, - 1080: 137, - }, - hfr: { - 720: 298, - 1080: 299, - }, - }, - }, - audio: { - aac: [140], - opus: [249, 250, 251], - }, - combined: { - 360: 18, - 720: 22, - }, - hls: { - 240: 92, - 360: 93, - 480: 94, - 720: 95, - 1080: 96, - } -}; - -function _appendItagArray(arr, opts, formats) -{ - const keys = Object.keys(formats); - - for(let fmt of keys) { - arr.push(formats[fmt]); - - if( - fmt >= opts.height - || Math.floor(fmt * 16 / 9) >= opts.width - ) - break; - } - - return arr; -} - -function getDashItags(opts) -{ - const allowed = { - video: [], - audio: (opts.codec === 'h264') - ? Itags.audio.aac - : Itags.audio.opus - }; - const types = Object.keys(Itags.video[opts.codec]); - - for(let type of types) { - const formats = Itags.video[opts.codec][type]; - _appendItagArray(allowed.video, opts, formats); - - if(type === opts.type) - break; - } - - return allowed; -} - -function getCombinedItags(opts) -{ - return _appendItagArray([], opts, Itags.combined); -} - -function getHLSItags(opts) -{ - return _appendItagArray([], opts, Itags.hls); -} diff --git a/ui/preferences-window.ui b/ui/preferences-window.ui index fac189a9..81d71bc0 100644 --- a/ui/preferences-window.ui +++ b/ui/preferences-window.ui @@ -197,8 +197,8 @@ - - YouTube + + Gtuber Prefer adaptive streaming