From 06f8e5d2591a16961975e245f7b3103f4b30cc01 Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 14 Mar 2021 21:00:18 +0100 Subject: [PATCH 1/4] YT: cache current decipher actions --- src/misc.js | 5 +++++ src/youtube.js | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/misc.js b/src/misc.js index 69f0f5cf..5493c55e 100644 --- a/src/misc.js +++ b/src/misc.js @@ -109,3 +109,8 @@ function decodeURIPlus(uri) { return decodeURI(uri.replace(/\+/g, ' ')); } + +function isHex(num) +{ + return Boolean(num.match(/[0-9a-f]+$/i)); +} diff --git a/src/youtube.js b/src/youtube.js index 1c1b49a1..3c877488 100644 --- a/src/youtube.js +++ b/src/youtube.js @@ -1,6 +1,7 @@ -const { GLib, GObject, Gst, Soup } = imports.gi; +const { Gio, GLib, GObject, Gst, Soup } = imports.gi; const ByteArray = imports.byteArray; const Debug = imports.src.debug; +const Misc = imports.src.misc; const YTDL = imports.src.assets['node-ytdl-core']; const { debug } = Debug; @@ -23,6 +24,10 @@ var YouTubeClient = GObject.registerClass({ this.downloadingVideoId = null; this.lastInfo = null; + this.cachedSig = { + id: null, + actions: null, + }; } getVideoInfoPromise(videoId) @@ -110,9 +115,16 @@ var YouTubeClient = GObject.registerClass({ } debug(`found player URI: ${ytUri}`); - /* TODO: cache */ + const ytId = ytPath.split('/').find(el => Misc.isHex(el)); let actions; + if(this.cachedSig.id === ytId) { + debug('reusing cached cipher actions'); + actions = this.cachedSig.actions; + } + + /* TODO: load cache from file */ + if(!actions) { const [pBody, isAbortedPlayer] = await this._downloadDataPromise(ytUri).catch(debug); @@ -121,6 +133,7 @@ var YouTubeClient = GObject.registerClass({ break; } actions = YTDL.sig.extractActions(pBody); + this._createCacheFileAsync(ytId, actions); } if(!actions || !actions.length) { @@ -128,6 +141,13 @@ var YouTubeClient = GObject.registerClass({ break; } debug('successfully obtained decipher actions'); + + if(this.cachedSig.id !== ytId) { + this.cachedSig.id = ytId; + this.cachedSig.actions = actions; + debug('set current decipher actions for reuse'); + } + const isDeciphered = this._decipherStreamingData( info.streamingData, actions ); @@ -394,6 +414,24 @@ var YouTubeClient = GObject.registerClass({ return `${url}&${sig}=${key}`; } + + _createCacheFileAsync(ytId, actions) + { + const cachePath = GLib.get_user_cache_dir() + '/' + ytId; + const cacheFile = Gio.File.new_for_path(cachePath); + + debug('saving cipher actions to cache file'); + + cacheFile.replace_contents_bytes_async( + GLib.Bytes.new_take(actions), + null, + false, + Gio.FileCreateFlags.NONE, + null + ) + .then(() => debug('saved cache file')) + .catch(debug); + } }); function checkYouTubeUri(uri) From 5b6141ee8c389f5ceec2b27c242af1071f02d048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Dzi=C4=99giel?= Date: Mon, 15 Mar 2021 13:07:12 +0100 Subject: [PATCH 2/4] YT: do not check player ID if actions are cached --- src/youtube.js | 110 ++++++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 56 deletions(-) diff --git a/src/youtube.js b/src/youtube.js index 3c877488..9753a8ed 100644 --- a/src/youtube.js +++ b/src/youtube.js @@ -52,7 +52,8 @@ var YouTubeClient = GObject.registerClass({ debug(`obtaining YouTube video info: ${videoId}`); this.downloadingVideoId = videoId; - const [info, isAborted] = await this._getInfoPromise(videoId).catch(debug); + let [info, isAborted] = await this._getInfoPromise(videoId).catch(debug); + if(!info) { if(isAborted) return reject(new Error('download aborted')); @@ -84,68 +85,63 @@ var YouTubeClient = GObject.registerClass({ if(!info.streamingData.adaptiveFormats) info.streamingData.adaptiveFormats = []; - const isCipher = this._getIsCipher(info.streamingData); - if(isCipher) { + if(this._getIsCipher(info.streamingData)) { debug('video requires deciphering'); - const embedUri = `https://www.youtube.com/embed/${videoId}`; - const [body, isAbortedBody] = - await this._downloadDataPromise(embedUri).catch(debug); - - if(isAbortedBody) - break; - - /* We need matching info, so start from beginning */ - if(!body) - continue; - - const ytPath = body.match(/(?<=jsUrl\":\").*?(?=\")/gs)[0]; - if(!ytPath) { - debug(new Error('could not find YouTube player URI')); - break; - } - const ytUri = `https://www.youtube.com${ytPath}`; - if( - /* check if site has "/" after ".com" */ - ytUri[23] !== '/' - || !Gst.Uri.is_valid(ytUri) - ) { - debug(`misformed player URI: ${ytUri}`); - break; - } - debug(`found player URI: ${ytUri}`); - - const ytId = ytPath.split('/').find(el => Misc.isHex(el)); - let actions; - - if(this.cachedSig.id === ytId) { - debug('reusing cached cipher actions'); - actions = this.cachedSig.actions; - } - - /* TODO: load cache from file */ - + /* Decipher actions do not change too often, so try + * to reuse without triggering too many requests ban */ + let actions = this.cachedSig.actions; if(!actions) { - const [pBody, isAbortedPlayer] = - await this._downloadDataPromise(ytUri).catch(debug); - if(!pBody || isAbortedPlayer) { - debug(new Error('could not download player body')); + const embedUri = `https://www.youtube.com/embed/${videoId}`; + const [body, isAbortedBody] = + await this._downloadDataPromise(embedUri).catch(debug); + + if(isAbortedBody) + break; + if(!body) + continue; + + const ytPath = body.match(/(?<=jsUrl\":\").*?(?=\")/gs)[0]; + if(!ytPath) { + debug(new Error('could not find YouTube player URI')); break; } - actions = YTDL.sig.extractActions(pBody); - this._createCacheFileAsync(ytId, actions); - } + const ytUri = `https://www.youtube.com${ytPath}`; + if( + /* check if site has "/" after ".com" */ + ytUri[23] !== '/' + || !Gst.Uri.is_valid(ytUri) + ) { + debug(`misformed player URI: ${ytUri}`); + break; + } + debug(`found player URI: ${ytUri}`); - if(!actions || !actions.length) { - debug(new Error('could not extract decipher actions')); - break; - } - debug('successfully obtained decipher actions'); + const ytId = ytPath.split('/').find(el => Misc.isHex(el)); - if(this.cachedSig.id !== ytId) { - this.cachedSig.id = ytId; - this.cachedSig.actions = actions; - debug('set current decipher actions for reuse'); + /* TODO: load cache from file */ + + if(!actions) { + const [pBody, isAbortedPlayer] = + await this._downloadDataPromise(ytUri).catch(debug); + if(!pBody || isAbortedPlayer) { + debug(new Error('could not download player body')); + break; + } + actions = YTDL.sig.extractActions(pBody); + this._createCacheFileAsync(ytId, actions); + } + if(!actions || !actions.length) { + debug(new Error('could not extract decipher actions')); + break; + } + debug('successfully obtained decipher actions'); + + if(this.cachedSig.id !== ytId) { + this.cachedSig.id = ytId; + this.cachedSig.actions = actions; + debug('remembered current decipher actions for reuse'); + } } const isDeciphered = this._decipherStreamingData( @@ -214,6 +210,8 @@ var YouTubeClient = GObject.registerClass({ debug(`got chunk of data, length: ${chunk.length}`); const chunkData = chunk.get_data(); + if(!chunkData) return; + data += (chunkData instanceof Uint8Array) ? ByteArray.toString(chunkData) : chunkData; From 8c307dc90fcfe3ef9436e69ff06ab6b7fe59071e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Dzi=C4=99giel?= Date: Mon, 15 Mar 2021 13:14:41 +0100 Subject: [PATCH 3/4] YT: save decipher actions only after successful deciphering --- src/youtube.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/youtube.js b/src/youtube.js index 9753a8ed..13dc808e 100644 --- a/src/youtube.js +++ b/src/youtube.js @@ -129,7 +129,10 @@ var YouTubeClient = GObject.registerClass({ break; } actions = YTDL.sig.extractActions(pBody); - this._createCacheFileAsync(ytId, actions); + if(actions) { + debug('deciphered'); + this._createCacheFileAsync(ytId, actions); + } } if(!actions || !actions.length) { debug(new Error('could not extract decipher actions')); From a4d55f81149ce9644e55f47a488dcfb5ae92f5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Dzi=C4=99giel?= Date: Mon, 15 Mar 2021 16:35:36 +0100 Subject: [PATCH 4/4] YT: store and load decipher actions from Clapper cache dir --- src/main.js | 2 ++ src/youtube.js | 71 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/src/main.js b/src/main.js index 54e20d2a..31c16c66 100644 --- a/src/main.js +++ b/src/main.js @@ -4,6 +4,8 @@ imports.gi.versions.Gtk = '4.0'; const { Gio, Gst } = imports.gi; Gst.init(null); +Gio._promisify(Gio._LocalFilePrototype, 'load_bytes_async', 'load_bytes_finish'); +Gio._promisify(Gio._LocalFilePrototype, 'make_directory_async', 'make_directory_finish'); Gio._promisify(Gio._LocalFilePrototype, 'replace_contents_bytes_async', 'replace_contents_finish'); const { App } = imports.src.app; diff --git a/src/youtube.js b/src/youtube.js index 13dc808e..9525f360 100644 --- a/src/youtube.js +++ b/src/youtube.js @@ -91,7 +91,10 @@ var YouTubeClient = GObject.registerClass({ /* Decipher actions do not change too often, so try * to reuse without triggering too many requests ban */ let actions = this.cachedSig.actions; - if(!actions) { + + if(actions) + debug('using remembered decipher actions'); + else { const embedUri = `https://www.youtube.com/embed/${videoId}`; const [body, isAbortedBody] = await this._downloadDataPromise(embedUri).catch(debug); @@ -118,8 +121,7 @@ var YouTubeClient = GObject.registerClass({ debug(`found player URI: ${ytUri}`); const ytId = ytPath.split('/').find(el => Misc.isHex(el)); - - /* TODO: load cache from file */ + actions = await this._getCacheFileActionsPromise(ytId).catch(debug); if(!actions) { const [pBody, isAbortedPlayer] = @@ -138,14 +140,12 @@ var YouTubeClient = GObject.registerClass({ debug(new Error('could not extract decipher actions')); break; } - debug('successfully obtained decipher actions'); - if(this.cachedSig.id !== ytId) { this.cachedSig.id = ytId; this.cachedSig.actions = actions; - debug('remembered current decipher actions for reuse'); } } + debug(`successfully obtained decipher actions: ${actions}`); const isDeciphered = this._decipherStreamingData( info.streamingData, actions @@ -416,13 +416,32 @@ var YouTubeClient = GObject.registerClass({ return `${url}&${sig}=${key}`; } - _createCacheFileAsync(ytId, actions) + async _createCacheFileAsync(ytId, actions) { - const cachePath = GLib.get_user_cache_dir() + '/' + ytId; - const cacheFile = Gio.File.new_for_path(cachePath); - debug('saving cipher actions to cache file'); + const ytCacheDir = Gio.File.new_for_path([ + GLib.get_user_cache_dir(), + Misc.appId, + 'yt-sig' + ].join('/')); + + for(let dir of [ytCacheDir.get_parent(), ytCacheDir]) { + if(dir.query_exists(null)) + continue; + + const dirCreated = await dir.make_directory_async( + GLib.PRIORITY_DEFAULT, + null, + ).catch(debug); + + if(!dirCreated) { + debug(new Error(`could not create dir: ${dir.get_path()}`)); + return; + } + } + + const cacheFile = ytCacheDir.get_child(ytId); cacheFile.replace_contents_bytes_async( GLib.Bytes.new_take(actions), null, @@ -433,6 +452,38 @@ var YouTubeClient = GObject.registerClass({ .then(() => debug('saved cache file')) .catch(debug); } + + _getCacheFileActionsPromise(ytId) + { + return new Promise((resolve, reject) => { + debug('checking decipher actions from cache file'); + + const ytActionsFile = Gio.File.new_for_path([ + GLib.get_user_cache_dir(), + Misc.appId, + 'yt-sig', + ytId + ].join('/')); + + if(!ytActionsFile.query_exists(null)) { + debug(`no such cache file: ${ytId}`); + return resolve(null); + } + + ytActionsFile.load_bytes_async(null) + .then(result => { + const data = result[0].get_data(); + if(!data || !data.length) + return reject(new Error('actions cache file is empty')); + + if(data instanceof Uint8Array) + resolve(ByteArray.toString(data)); + else + resolve(data); + }) + .catch(err => reject(err)); + }); + } }); function checkYouTubeUri(uri)