YT: decipher videos with signatures

Increase amount of playable YouTube videos by deciphering the ones that require to do so.

Many thanks to "node-ytdl-core" devs for JS regular expressions needed for YouTube player parsing.

src/youtube.js

@@ -1,6 +1,7 @@
 const { GLib, GObject, Gst, Soup } = imports.gi;
 const ByteArray = imports.byteArray;
 const Debug = imports.src.debug;
+const YTDL = imports.src.assets['node-ytdl-core'];
 
 const { debug } = Debug;
 
@@ -21,7 +22,6 @@ var YouTubeClient = GObject.registerClass({
         /* videoID of current active download */
         this.downloadingVideoId = null;
 
-        this.downloadAborted = false;
         this.lastInfo = null;
     }
 
@@ -47,9 +47,9 @@ var YouTubeClient = GObject.registerClass({
                 debug(`obtaining YouTube video info: ${videoId}`);
                 this.downloadingVideoId = videoId;
 
-                const info = await this._getInfoPromise(videoId).catch(debug);
+                const [info, isAborted] = await this._getInfoPromise(videoId).catch(debug);
                 if(!info) {
-                    if(this.downloadAborted)
+                    if(isAborted)
                         return reject(new Error('download aborted'));
 
                     debug(`failed, remaining tries: ${tries}`);
@@ -62,15 +62,71 @@ var YouTubeClient = GObject.registerClass({
                 )
                     ? 'video is not playable'
                     : (!info.streamingData)
-                    ? 'video response data is missing URIs'
+                    ? 'video response data is missing streaming data'
                     : null;
 
                 if(invalidInfoMsg) {
                     this.lastInfo = null;
-                    this.emit('info-resolved', false);
-                    this.downloadingVideoId = null;
 
-                    return reject(new Error(invalidInfoMsg));
+                    debug(new Error(invalidInfoMsg));
+                    break;
+                }
+
+                /* Make sure we have all formats arrays,
+                 * so we will not have to keep checking */
+                if(!info.streamingData.formats)
+                    info.streamingData.formats = [];
+                if(!info.streamingData.adaptiveFormats)
+                    info.streamingData.adaptiveFormats = [];
+
+                const isCipher = this._getIsCipher(info.streamingData);
+                if(isCipher) {
+                    debug('video requires deciphering');
+
+                    const embedUri = `https://www.youtube.com/embed/${videoId}`;
+                    const [body, isAbortedBody] =
+                        await this._downloadDataPromise(embedUri).catch(debug);
+
+                    if(isAbortedBody)
+                        break;
+
+                    /* We need matching info, so start from beginning */
+                    if(!body)
+                        continue;
+
+                    const ytPath = body.match(/(?<=jsUrl\":\").*?(?=\")/gs)[0];
+                    if(!ytPath) {
+                        debug(new Error('could not find YouTube player URI'));
+                        break;
+                    }
+                    const ytUri = `https://www.youtube.com${ytPath}`;
+                    debug(`found player URI: ${ytUri}`);
+
+                    /* TODO: cache */
+                    let actions;
+
+                    if(!actions) {
+                        const [pBody, isAbortedPlayer] =
+                            await this._downloadDataPromise(ytUri).catch(debug);
+                        if(!pBody || isAbortedPlayer) {
+                            debug(new Error('could not download player body'));
+                            break;
+                        }
+                        actions = YTDL.sig.extractActions(pBody);
+                    }
+
+                    if(!actions || !actions.length) {
+                        debug(new Error('could not extract decipher actions'));
+                        break;
+                    }
+                    debug('successfully obtained decipher actions');
+                    const isDeciphered = this._decipherStreamingData(
+                        info.streamingData, actions
+                    );
+                    if(!isDeciphered) {
+                        debug('streaming data could not be deciphered');
+                        break;
+                    }
                 }
 
                 this.lastInfo = info;
@@ -80,7 +136,7 @@ var YouTubeClient = GObject.registerClass({
                 return resolve(info);
             }
 
-            /* Do not clear video info here, as we still have
+            /* Do not clear video info here, as we might still have
              * valid info from last video that can be reused */
             this.emit('info-resolved', false);
             this.downloadingVideoId = null;
@@ -91,10 +147,7 @@ var YouTubeClient = GObject.registerClass({
 
     getBestCombinedUri(info)
     {
-        if(
-            !info.streamingData.formats
-            || !info.streamingData.formats.length
-        )
+        if(!info.streamingData.formats.length)
             return null;
 
         const combinedStream = info.streamingData.formats[
@@ -123,6 +176,41 @@ var YouTubeClient = GObject.registerClass({
         return true;
     }
 
+    _downloadDataPromise(url)
+    {
+        return new Promise((resolve, reject) => {
+            const message = Soup.Message.new('GET', url);
+            let data = '';
+
+            const chunkSignal = message.connect('got-chunk', (msg, chunk) => {
+                debug(`got chunk of data, length: ${chunk.length}`);
+
+                const chunkData = chunk.get_data();
+                data += (chunkData instanceof Uint8Array)
+                    ? ByteArray.toString(chunkData)
+                    : chunkData;
+            });
+
+            this.queue_message(message, (session, msg) => {
+                msg.disconnect(chunkSignal);
+
+                debug('got message response');
+                const statusCode = msg.status_code;
+
+                /* Internal Soup codes mean download aborted
+                 * or some other error that cannot be handled
+                 * and we do not want to retry in such case */
+                if(statusCode < 10)
+                    return resolve([null, true]);
+
+                if(statusCode !== 200)
+                    return reject(new Error(`response code: ${statusCode}`));
+
+                resolve([data, false]);
+            });
+        });
+    }
+
     _getCurrentDownloadPromise()
     {
         debug('resolving after current download finishes');
@@ -145,36 +233,20 @@ var YouTubeClient = GObject.registerClass({
     _getInfoPromise(videoId)
     {
         return new Promise((resolve, reject) => {
-            const url = `https://www.youtube.com/get_video_info?video_id=${videoId}&el=embedded`;
-            const message = Soup.Message.new('GET', url);
-            let data = '';
+            const query = [
+                `video_id=${videoId}`,
+                `el=embedded`,
+                `eurl=https://youtube.googleapis.com/v/${videoId}`,
+            ].join('&');
+            const url = `https://www.youtube.com/get_video_info?${query}`;
 
-            const chunkSignal = message.connect('got-chunk', (msg, chunk) => {
-                debug(`got chunk of data, length: ${chunk.length}`);
-
-                const chunkData = chunk.get_data();
-                data += (chunkData instanceof Uint8Array)
-                    ? ByteArray.toString(chunkData)
-                    : chunkData;
-            });
-
-            this.queue_message(message, (session, msg) => {
-                msg.disconnect(chunkSignal);
-
-                debug('got message response');
-
-                const statusCode = msg.status_code;
-
-                /* Internal Soup codes mean download abort
-                 * or some other error that cannot be handled */
-                this.downloadAborted = (statusCode < 10);
-
-                if(statusCode !== 200)
-                    return reject(new Error(`response code: ${statusCode}`));
+            this._downloadDataPromise(url).then(res => {
+                if(res[1])
+                    return resolve([null, true]);
 
                 debug('parsing video info JSON');
 
-                const gstUri = Gst.Uri.from_string('?' + data);
+                const gstUri = Gst.Uri.from_string('?' + res[0]);
 
                 if(!gstUri)
                     return reject(new Error('could not convert query to URI'));
@@ -193,11 +265,126 @@ var YouTubeClient = GObject.registerClass({
                     return reject(new Error('could not parse video info JSON'));
 
                 debug('successfully parsed video info JSON');
-
-                resolve(info);
-            });
+                resolve([info, false]);
+            })
+            .catch(err => reject(err));
         });
     }
+
+    _getIsCipher(data)
+    {
+        /* Check only first best combined,
+         * AFAIK there are no videos without it */
+        if(data.formats[0].url)
+            return false;
+
+        if(
+            data.formats[0].signatureCipher
+            || data.formats[0].cipher
+        )
+            return true;
+
+        /* FIXME: no URLs and no cipher, what now? */
+        debug(new Error('no url or cipher in streams'));
+
+        return false;
+    }
+
+    _decipherStreamingData(data, actions)
+    {
+        debug('checking cipher query keys');
+
+        /* Cipher query keys should be the same for all
+         * streams, so parse any stream to get their names */
+        const anyStream = data.formats[0] || data.adaptiveFormats[0];
+        const sigQuery = anyStream.signatureCipher || anyStream.cipher;
+
+        if(!sigQuery)
+            return false;
+
+        const gstUri = Gst.Uri.from_string('?' + sigQuery);
+        const queryKeys = gstUri.get_query_keys();
+
+        const cipherKey = queryKeys.find(key => {
+            const value = gstUri.get_query_value(key);
+            /* A long value that is not URI */
+            return (
+                value.length > 32
+                && !Gst.Uri.is_valid(value)
+            );
+        });
+        if(!cipherKey) {
+            debug('no stream cipher key name');
+            return false;
+        }
+
+        const sigKey = queryKeys.find(key => {
+            const value = gstUri.get_query_value(key);
+            /* A short value that is not URI */
+            return (
+                value.length < 32
+                && !Gst.Uri.is_valid(value)
+            );
+        });
+        if(!sigKey) {
+            debug('no stream signature key name');
+            return false;
+        }
+
+        const urlKey = queryKeys.find(key =>
+            Gst.Uri.is_valid(gstUri.get_query_value(key))
+        );
+        if(!urlKey) {
+            debug('no stream URL key name');
+            return false;
+        }
+
+        const cipherKeys = {
+            url: urlKey,
+            sig: sigKey,
+            cipher: cipherKey,
+        };
+
+        debug('deciphering streams');
+
+        for(let format of [data.formats, data.adaptiveFormats]) {
+            for(let stream of format) {
+                const formatUrl = this._getDecipheredUrl(
+                    stream, actions, cipherKeys
+                );
+                if(!formatUrl) {
+                    debug('undecipherable stream');
+                    debug(stream);
+
+                    return false;
+                }
+                stream.url = formatUrl;
+            }
+        }
+        debug('all streams deciphered');
+
+        return true;
+    }
+
+    _getDecipheredUrl(stream, actions, queryKeys)
+    {
+        debug(`deciphering stream id: ${stream.itag}`);
+
+        const sigQuery = stream.signatureCipher || stream.cipher;
+        if(!sigQuery) return null;
+
+        const gstUri = Gst.Uri.from_string('?' + sigQuery);
+
+        const url = gstUri.get_query_value(queryKeys.url);
+        const cipher = gstUri.get_query_value(queryKeys.cipher);
+        const sig = gstUri.get_query_value(queryKeys.sig);
+
+        const key = YTDL.sig.decipher(cipher, actions);
+
+        debug('stream deciphered');
+
+        return `${url}&${sig}=${key}`;
+    }
 });
 
 function checkYouTubeUri(uri)