From 3fbaff704571293be83e2b56d36b761f42cce1ec Mon Sep 17 00:00:00 2001
From: Biswa Kalyan Bhuyan <biswa.bhuyan@vegastack.com>
Date: Sat, 29 Mar 2025 12:35:50 +0530
Subject: Update version v0.3.2
---
README.md | 2 +-
requirements-dev.txt | 26 +----
requirements.txt | 25 ++--
settings.py | 20 ++--
youtube/__init__.py | 6 +-
youtube/channel.py | 2 +-
youtube/comments.py | 8 +-
youtube/get_app_version/get_app_version.py | 48 ++++----
youtube/opensearch.xml | 2 +-
youtube/playlist.py | 2 +-
youtube/static/js/av-merge.js | 30 ++++-
youtube/static/js/plyr-start.js | 17 ++-
youtube/static/js/watch.js | 3 +-
youtube/static/modules/plyr/custom_plyr.css | 38 +++++++
youtube/static/watch.css | 26 +++++
youtube/templates/base.html | 6 +-
youtube/templates/error.html | 2 +-
youtube/util.py | 171 ++++++++++++++++++----------
youtube/version.py | 2 +-
youtube/watch.py | 134 +++++++++++-----------
20 files changed, 345 insertions(+), 225 deletions(-)
diff --git a/README.md b/README.md
index be1a473..67e21ee 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# yt-local
+# Biswa's Youtube
Fork of [youtube-local](https://github.com/user234683/youtube-local)
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 8208c56..7174282 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,21 +1,5 @@
-blinker==1.7.0
-Brotli==1.1.0
-cachetools==5.3.3
-click==8.1.7
-defusedxml==0.7.1
-Flask==3.0.2
-gevent==24.2.1
-greenlet==3.0.3
-iniconfig==2.0.0
-itsdangerous==2.1.2
-Jinja2==3.1.4
-MarkupSafe==2.1.5
-packaging==24.0
-pluggy==1.4.0
-PySocks==1.7.1
-pytest==8.1.1
-stem==1.8.2
-urllib3==2.2.2
-Werkzeug==3.0.3
-zope.event==5.0
-zope.interface==6.2
+# Include all production requirements
+-r requirements.txt
+
+# Development requirements
+pytest>=6.2.1
diff --git a/requirements.txt b/requirements.txt
index 95f8f12..b54a1f2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,17 +1,8 @@
-blinker==1.7.0
-Brotli==1.1.0
-cachetools==5.3.3
-click==8.1.7
-defusedxml==0.7.1
-Flask==3.0.2
-gevent==24.2.1
-greenlet==3.0.3
-itsdangerous==2.1.2
-Jinja2==3.1.4
-MarkupSafe==2.1.5
-PySocks==1.7.1
-stem==1.8.2
-urllib3==2.2.2
-Werkzeug==3.0.3
-zope.event==5.0
-zope.interface==6.2
+Flask>=1.0.3
+gevent>=1.2.2
+Brotli>=1.0.7
+PySocks>=1.6.8
+urllib3>=1.24.1
+defusedxml>=0.5.0
+cachetools>=4.0.0
+stem>=1.8.0
diff --git a/settings.py b/settings.py
index eb210c5..36ed141 100644
--- a/settings.py
+++ b/settings.py
@@ -61,7 +61,7 @@ SETTINGS_INFO = collections.OrderedDict([
('allow_foreign_addresses', {
'type': bool,
'default': False,
- 'comment': '''This will allow others to connect to your YT Local instance as a website.
+ 'comment': '''This will allow others to connect to your Biswa's Youtube instance as a website.
For security reasons, enabling this is not recommended.''',
'hidden': True,
'category': 'network',
@@ -322,13 +322,6 @@ Archive: https://archive.ph/OZQbN''',
'comment': '',
}),
- ('gather_googlevideo_domains', {
- 'type': bool,
- 'default': False,
- 'comment': '''Developer use to debug 403s''',
- 'hidden': True,
- }),
-
('debugging_save_responses', {
'type': bool,
'default': False,
@@ -338,7 +331,7 @@ Archive: https://archive.ph/OZQbN''',
('settings_version', {
'type': int,
- 'default': 5,
+ 'default': 6,
'comment': '''Do not change, remove, or comment out this value, or else your settings may be lost or corrupted''',
'hidden': True,
}),
@@ -419,11 +412,20 @@ def upgrade_to_5(settings_dict):
return new_settings
+def upgrade_to_6(settings_dict):
+ new_settings = settings_dict.copy()
+ if 'gather_googlevideo_domains' in new_settings:
+ del new_settings['gather_googlevideo_domains']
+ new_settings['settings_version'] = 6
+ return new_settings
+
+
upgrade_functions = {
1: upgrade_to_2,
2: upgrade_to_3,
3: upgrade_to_4,
4: upgrade_to_5,
+ 5: upgrade_to_6,
}
diff --git a/youtube/__init__.py b/youtube/__init__.py
index 64aed56..f122704 100644
--- a/youtube/__init__.py
+++ b/youtube/__init__.py
@@ -19,14 +19,14 @@ yt_app.add_url_rule('/settings', 'settings_page', settings.settings_page, method
@yt_app.route('/')
def homepage():
- return flask.render_template('home.html', title="YT Local")
+ return flask.render_template('home.html', title="Biswa's Youtube")
@yt_app.route('/licenses')
def licensepage():
return flask.render_template(
'licenses.html',
- title="Licenses - YT Local"
+ title="Licenses - Biswa's Youtube"
)
@@ -121,7 +121,7 @@ def error_page(e):
elif (exc_info()[0] == util.FetchError
and exc_info()[1].code == '404'
):
- error_message = ('Error: The page you are looking for isn\'t here. ¯\_(ツ)_/¯')
+ error_message = ('Error: The page you are looking for isn\'t here.')
return flask.render_template('error.html',
error_code=exc_info()[1].code,
error_message=error_message,
diff --git a/youtube/channel.py b/youtube/channel.py
index b520121..81881eb 100644
--- a/youtube/channel.py
+++ b/youtube/channel.py
@@ -292,7 +292,7 @@ def get_number_of_videos_channel(channel_id):
try:
response = util.fetch_url(url, headers_mobile,
debug_name='number_of_videos', report_text='Got number of videos')
- except urllib.error.HTTPError as e:
+ except (urllib.error.HTTPError, util.FetchError) as e:
traceback.print_exc()
print("Couldn't retrieve number of videos")
return 1000
diff --git a/youtube/comments.py b/youtube/comments.py
index 92a89e1..1ff1a21 100644
--- a/youtube/comments.py
+++ b/youtube/comments.py
@@ -53,7 +53,7 @@ def request_comments(ctoken, replies=False):
'hl': 'en',
'gl': 'US',
'clientName': 'MWEB',
- 'clientVersion': '2.20240328.08.00',
+ 'clientVersion': '2.20210804.02.00',
},
},
'continuation': ctoken.replace('=', '%3D'),
@@ -78,7 +78,7 @@ def single_comment_ctoken(video_id, comment_id):
def post_process_comments_info(comments_info):
for comment in comments_info['comments']:
- comment['author'] = strip_non_ascii(comment['author'])
+ comment['author'] = strip_non_ascii(comment['author']) if comment.get('author') else ""
comment['author_url'] = concat_or_none(
'/', comment['author_url'])
comment['author_avatar'] = concat_or_none(
@@ -189,10 +189,10 @@ def video_comments(video_id, sort=0, offset=0, lc='', secret_key=''):
comments_info['error'] += '\n\n' + e.error_message
comments_info['error'] += '\n\nExit node IP address: %s' % e.ip
else:
- comments_info['error'] = 'YouTube blocked the request. IP address: %s' % e.ip
+ comments_info['error'] = 'YouTube blocked the request. Error: %s' % str(e)
except Exception as e:
- comments_info['error'] = 'YouTube blocked the request. IP address: %s' % e.ip
+ comments_info['error'] = 'YouTube blocked the request. Error: %s' % str(e)
if comments_info.get('error'):
print('Error retrieving comments for ' + str(video_id) + ':\n' +
diff --git a/youtube/get_app_version/get_app_version.py b/youtube/get_app_version/get_app_version.py
index 9852359..4995bb7 100644
--- a/youtube/get_app_version/get_app_version.py
+++ b/youtube/get_app_version/get_app_version.py
@@ -11,17 +11,10 @@ import subprocess
def app_version():
def minimal_env_cmd(cmd):
# make minimal environment
- env = {}
- for k in ['SYSTEMROOT', 'PATH']:
- v = os.environ.get(k)
- if v is not None:
- env[k] = v
-
- env['LANGUAGE'] = 'C'
- env['LANG'] = 'C'
- env['LC_ALL'] = 'C'
- out = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
+ env = {k: os.environ[k] for k in ['SYSTEMROOT', 'PATH'] if k in os.environ}
+ env.update({'LANGUAGE': 'C', 'LANG': 'C', 'LC_ALL': 'C'})
+
+ out = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
return out
subst_list = {
@@ -31,24 +24,21 @@ def app_version():
}
if os.system("command -v git > /dev/null 2>&1") != 0:
- subst_list
- else:
- if call(["git", "branch"], stderr=STDOUT,
- stdout=open(os.devnull, 'w')) != 0:
- subst_list
- else:
- # version
- describe = minimal_env_cmd(["git", "describe", "--always"])
- git_revision = describe.strip().decode('ascii')
- # branch
- branch = minimal_env_cmd(["git", "branch"])
- git_branch = branch.strip().decode('ascii').replace('* ', '')
-
- subst_list = {
- "version": __version__,
- "branch": git_branch,
- "commit": git_revision
- }
+ return subst_list
+
+ if call(["git", "branch"], stderr=STDOUT, stdout=open(os.devnull, 'w')) != 0:
+ return subst_list
+
+ describe = minimal_env_cmd(["git", "describe", "--tags", "--always"])
+ git_revision = describe.strip().decode('ascii')
+
+ branch = minimal_env_cmd(["git", "branch"])
+ git_branch = branch.strip().decode('ascii').replace('* ', '')
+
+ subst_list.update({
+ "branch": git_branch,
+ "commit": git_revision
+ })
return subst_list
diff --git a/youtube/opensearch.xml b/youtube/opensearch.xml
index 09d1cb7..cde9beb 100644
--- a/youtube/opensearch.xml
+++ b/youtube/opensearch.xml
@@ -1,5 +1,5 @@
<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/">
-<ShortName>YT Local</ShortName>
+<ShortName>Biswa's Youtube</ShortName>
<Description>no CIA shit in the background</Description>
<InputEncoding>UTF-8</InputEncoding>
<Image width="16" height="16">data:image/x-icon;base64,AAABAAEAEBAAAAEACAAlAgAAFgAAAIlQTkcNChoKAAAADUlIRFIAAAAQAAAAEAgGAAAAH/P/YQAAAexJREFUOI2lkzFPmlEUhp/73fshtCUCRtvQkJoKMrDQJvoHnBzUhc3EH0DUQf+As6tujo4M6mTiIDp0kGiMTRojTRNSW6o12iD4YYXv3g7Qr4O0ScM7npz7vOe+J0fk83lDF7K6eQygwkdHhI+P0bYNxmBXq5RmZui5vGQgn0f7fKi7O4oLC1gPD48BP9JpnpRKJFZXcQMB3m1u4vr9NHp76d/bo39/n4/z84ROThBa4/r91OJxMKb9BSn5mskAIOt1eq6uEFpjVyrEcjk+T0+TXlzkbTZLuFDAur9/nIFRipuREQCe7+zgBgK8mZvj/fIylVTKa/6UzXKbSnnuHkA0GnwbH/cA0a0takND3IyOEiwWAXBiMYTWjzLwtvB9bAyAwMUF8ZUVPiwtYTWbHqA6PIxoNv8OMLbN3eBga9TZWYQxaKX+AJJJhOv+AyAlT0slAG6TSX5n8+zszJugkzxA4PzcK9YSCQCk42DXaq1aGwqgfT5ebG9jpMQyUjKwu8vrtbWWqxC83NjAd31NsO2uleJnX58HCJ6eEjk8BGNQAA+RCOXJScpTU2AMwnUxlkXk4ACA+2iUSKGArNeRjkMsl6M8MYHQGtHpmIxSvFpfRzoORinQGqvZBCEwQoAxfMlkaIRCnQH/o66v8Re19MavaDNLfgAAAABJRU5ErkJggg==</Image>
diff --git a/youtube/playlist.py b/youtube/playlist.py
index 83d530c..28b8149 100644
--- a/youtube/playlist.py
+++ b/youtube/playlist.py
@@ -115,7 +115,7 @@ def get_playlist_page():
video_count = yt_data_extract.deep_get(info, 'metadata', 'video_count')
if video_count is None:
- video_count = 40
+ video_count = 1000
return flask.render_template(
'playlist.html',
diff --git a/youtube/static/js/av-merge.js b/youtube/static/js/av-merge.js
index e00f440..cfe9574 100644
--- a/youtube/static/js/av-merge.js
+++ b/youtube/static/js/av-merge.js
@@ -20,6 +20,29 @@
// TODO: Call abort to cancel in-progress appends?
+// Buffer sizes for different systems
+const BUFFER_CONFIG = {
+ default: 50 * 10**6, // 50 megabytes
+ webOS: 20 * 10**6, // 20 megabytes WebOS (LG)
+ samsungTizen: 20 * 10**6, // 20 megabytes Samsung Tizen OS
+ androidTV: 30 * 10**6, // 30 megabytes Android TV
+ desktop: 50 * 10**6, // 50 megabytes PC/Mac
+};
+
+function detectSystem() {
+ const userAgent = navigator.userAgent.toLowerCase();
+ if (/webos|lg browser/i.test(userAgent)) {
+ return "webOS";
+ } else if (/tizen/i.test(userAgent)) {
+ return "samsungTizen";
+ } else if (/android tv|smart-tv/i.test(userAgent)) {
+ return "androidTV";
+ } else if (/firefox|chrome|safari|edge/i.test(userAgent)) {
+ return "desktop";
+ } else {
+ return "default";
+ }
+}
function AVMerge(video, srcInfo, startTime){
this.audioSource = null;
@@ -164,6 +187,8 @@ AVMerge.prototype.printDebuggingInfo = function() {
}
function Stream(avMerge, source, startTime, avRatio) {
+ const selectedSystem = detectSystem();
+ let baseBufferTarget = BUFFER_CONFIG[selectedSystem] || BUFFER_CONFIG.default;
this.avMerge = avMerge;
this.video = avMerge.video;
this.url = source['url'];
@@ -173,10 +198,11 @@ function Stream(avMerge, source, startTime, avRatio) {
this.mimeCodec = source['mime_codec']
this.streamType = source['acodec'] ? 'audio' : 'video';
if (this.streamType == 'audio') {
- this.bufferTarget = avRatio*50*10**6;
+ this.bufferTarget = avRatio * baseBufferTarget;
} else {
- this.bufferTarget = 50*10**6; // 50 megabytes
+ this.bufferTarget = baseBufferTarget;
}
+ console.info(`Detected system: ${selectedSystem}. Applying bufferTarget of ${this.bufferTarget} bytes to ${this.streamType}.`);
this.initRange = source['init_range'];
this.indexRange = source['index_range'];
diff --git a/youtube/static/js/plyr-start.js b/youtube/static/js/plyr-start.js
index 56068f0..3838acc 100644
--- a/youtube/static/js/plyr-start.js
+++ b/youtube/static/js/plyr-start.js
@@ -58,7 +58,7 @@
},
});
- const player = new Plyr(document.getElementById('js-video-player'), {
+ const playerOptions = {
// Learning about autoplay permission https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Permissions-Policy/autoplay#syntax
autoplay: autoplayActive,
disableContextMenu: false,
@@ -117,5 +117,20 @@
tooltips: {
controls: true,
},
+ }
+
+ const player = new Plyr(document.getElementById('js-video-player'), playerOptions);
+
+ // disable double click to fullscreen
+ // https://github.com/sampotts/plyr/issues/1370#issuecomment-528966795
+ player.eventListeners.forEach(function(eventListener) {
+ if(eventListener.type === 'dblclick') {
+ eventListener.element.removeEventListener(eventListener.type, eventListener.callback, eventListener.options);
+ }
});
+
+ // Add .started property, true after the playback has been started
+ // Needed so controls won't be hidden before playback has started
+ player.started = false;
+ player.once('playing', function(){this.started = true});
})();
diff --git a/youtube/static/js/watch.js b/youtube/static/js/watch.js
index 95d9fa7..00803cf 100644
--- a/youtube/static/js/watch.js
+++ b/youtube/static/js/watch.js
@@ -5,8 +5,9 @@ function changeQuality(selection) {
let videoPaused = video.paused;
let videoSpeed = video.playbackRate;
let srcInfo;
- if (avMerge)
+ if (avMerge && typeof avMerge.close === 'function') {
avMerge.close();
+ }
if (selection.type == 'uni'){
srcInfo = data['uni_sources'][selection.index];
video.src = srcInfo.url;
diff --git a/youtube/static/modules/plyr/custom_plyr.css b/youtube/static/modules/plyr/custom_plyr.css
index 0fd3c52..7a9f0f3 100644
--- a/youtube/static/modules/plyr/custom_plyr.css
+++ b/youtube/static/modules/plyr/custom_plyr.css
@@ -37,3 +37,41 @@ e.g. Firefox playback speed options */
max-height: 320px;
overflow-y: auto;
}
+
+/*
+* Custom styles similar to youtube
+*/
+.plyr__controls {
+ display: flex;
+ justify-content: center;
+}
+
+.plyr__progress__container {
+ position: absolute;
+ bottom: 0;
+ width: 100%;
+ margin-bottom: -10px;
+}
+
+.plyr__controls .plyr__controls__item:first-child {
+ margin-left: 0;
+ margin-right: 0;
+ z-index: 5;
+}
+
+.plyr__controls .plyr__controls__item.plyr__volume {
+ margin-left: auto;
+}
+
+.plyr__controls .plyr__controls__item.plyr__progress__container {
+ padding-left: 10px;
+ padding-right: 10px;
+}
+
+.plyr__progress input[type="range"] {
+ margin-bottom: 50px;
+}
+
+/*
+* End custom styles
+*/
diff --git a/youtube/static/watch.css b/youtube/static/watch.css
index 460bba3..c0bdec6 100644
--- a/youtube/static/watch.css
+++ b/youtube/static/watch.css
@@ -128,6 +128,29 @@ header {
background-color: var(--buttom-hover);
}
+.live-url-choices {
+ background-color: var(--thumb-background);
+ margin: 1rem 0;
+ padding: 1rem;
+}
+
+.playability-error {
+ position: relative;
+ box-sizing: border-box;
+ height: 30vh;
+ margin: 1rem 0;
+}
+
+.playability-error > span {
+ display: flex;
+ background-color: var(--thumb-background);
+ height: 100%;
+ object-fit: cover;
+ justify-content: center;
+ align-items: center;
+ text-align: center;
+}
+
.playlist {
display: grid;
grid-gap: 4px;
@@ -622,6 +645,9 @@ figure.sc-video {
max-height: 80vh;
overflow-y: scroll;
}
+ .playability-error {
+ height: 60vh;
+ }
.playlist {
display: grid;
grid-gap: 1px;
diff --git a/youtube/templates/base.html b/youtube/templates/base.html
index 393cc52..d90c61f 100644
--- a/youtube/templates/base.html
+++ b/youtube/templates/base.html
@@ -10,7 +10,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="Content-Security-Policy" content="default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' blob: {{ app_url }}/* data: https://*.googlevideo.com; {{ "img-src 'self' https://*.googleusercontent.com https://*.ggpht.com https://*.ytimg.com;" if not settings.proxy_images else "" }}">
<title>{{ page_title }}</title>
- <link title="YT Local" href="/youtube.com/opensearch.xml" rel="search" type="application/opensearchdescription+xml">
+ <link title="Biswa's Youtube" href="/youtube.com/opensearch.xml" rel="search" type="application/opensearchdescription+xml">
<link href="/youtube.com/static/favicon.ico" type="image/x-icon" rel="icon">
<link href="/youtube.com/static/normalize.css" rel="stylesheet">
<link href="{{ theme_path }}" rel="stylesheet">
@@ -31,7 +31,7 @@
<body>
<header class="header">
<nav class="home">
- <a href="/youtube.com" id="home-link">YT Local</a>
+ <a href="/youtube.com" id="home-link">Biswa's Youtube</a>
</nav>
<form class="form" id="site-search" action="/youtube.com/results">
<input type="search" name="search_query" class="search-box" value="{{ search_box_value }}"
@@ -157,7 +157,7 @@
</main>
<footer class="footer">
<div>
- <a href="https://git.sr.ht/~heckyel/yt-local"
+ <a href="https://git.surgot.in/yt-local"
rel="noopener noreferrer" target="_blank">
Released under the AGPLv3 or later
</a>
diff --git a/youtube/templates/error.html b/youtube/templates/error.html
index 97f8ca9..8e67861 100644
--- a/youtube/templates/error.html
+++ b/youtube/templates/error.html
@@ -19,7 +19,7 @@
<div class="code-error" id="error-box">
<h1>500 Uncaught exception:</h1>
<div class="code-box"><code>{{ traceback }}</code></div>
- <p>Please report this issue at <a href="https://todo.sr.ht/~heckyel/yt-local" target="_blank" rel="noopener noreferrer">https://todo.sr.ht/~heckyel/yt-local</a></p>
+ <p>Please report this issue at <a href="https://git.surgot.in/yt-local" target="_blank" rel="noopener noreferrer">https://git.surgot.in/yt-local</a></p>
<p>Remember to include the traceback in your issue and redact any information in it you do not want to share</p>
</div>
{% else %}
diff --git a/youtube/util.py b/youtube/util.py
index b9225d2..c59fae8 100644
--- a/youtube/util.py
+++ b/youtube/util.py
@@ -318,10 +318,11 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
cleanup_func(response) # release_connection for urllib3
content = decode_content(
content,
- response.getheader('Content-Encoding', default='identity'))
+ response.headers.get('Content-Encoding', default='identity'))
if (settings.debugging_save_responses
- and debug_name is not None and content):
+ and debug_name is not None
+ and content):
save_dir = os.path.join(settings.data_dir, 'debug')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
@@ -394,23 +395,22 @@ def head(url, use_tor=False, report_text=None, max_redirects=10):
round(time.monotonic() - start_time, 3))
return response
-
-mobile_user_agent = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.80 Mobile Safari/537.36'
+mobile_user_agent = 'Mozilla/5.0 (Linux; Android 7.0; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36'
mobile_ua = (('User-Agent', mobile_user_agent),)
-desktop_user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0'
+desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0'
desktop_ua = (('User-Agent', desktop_user_agent),)
json_header = (('Content-Type', 'application/json'),)
desktop_xhr_headers = (
('Accept', '*/*'),
('Accept-Language', 'en-US,en;q=0.5'),
('X-YouTube-Client-Name', '1'),
- ('X-YouTube-Client-Version', '2.20240327.00.00'),
+ ('X-YouTube-Client-Version', '2.20240304.00.00'),
) + desktop_ua
mobile_xhr_headers = (
('Accept', '*/*'),
('Accept-Language', 'en-US,en;q=0.5'),
- ('X-YouTube-Client-Name', '1'),
- ('X-YouTube-Client-Version', '2.20240328.08.00'),
+ ('X-YouTube-Client-Name', '2'),
+ ('X-YouTube-Client-Version', '2.20240304.08.00'),
) + mobile_ua
@@ -431,29 +431,34 @@ class RateLimitedQueue(gevent.queue.Queue):
gevent.queue.Queue.__init__(self)
def get(self):
- with self.lock: # blocks if another greenlet currently has the lock
- if ((self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial) or
- (self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial)):
- self.surpassed_initial = True
- gevent.sleep(self.waiting_period)
- self.count_since_last_wait = 0
+ self.lock.acquire() # blocks if another greenlet currently has the lock
+ if self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial:
+ gevent.sleep(self.waiting_period)
+ self.count_since_last_wait = 0
- self.count_since_last_wait += 1
+ elif self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial:
+ self.surpassed_initial = True
+ gevent.sleep(self.waiting_period)
+ self.count_since_last_wait = 0
- if not self.currently_empty and self.empty():
- self.currently_empty = True
- self.empty_start = time.monotonic()
+ self.count_since_last_wait += 1
- item = gevent.queue.Queue.get(self) # blocks when nothing left
+ if not self.currently_empty and self.empty():
+ self.currently_empty = True
+ self.empty_start = time.monotonic()
- if self.currently_empty:
- if time.monotonic() - self.empty_start >= self.waiting_period:
- self.count_since_last_wait = 0
- self.surpassed_initial = False
+ item = gevent.queue.Queue.get(self) # blocks when nothing left
- self.currently_empty = False
+ if self.currently_empty:
+ if time.monotonic() - self.empty_start >= self.waiting_period:
+ self.count_since_last_wait = 0
+ self.surpassed_initial = False
- return item
+ self.currently_empty = False
+
+ self.lock.release()
+
+ return item
def download_thumbnail(save_directory, video_id):
@@ -662,19 +667,19 @@ def to_valid_filename(name):
# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72
INNERTUBE_CLIENTS = {
- 'android-test-suite': {
+ 'android': {
'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
'INNERTUBE_CONTEXT': {
'client': {
'hl': 'en',
'gl': 'US',
- 'clientName': 'ANDROID_TESTSUITE',
- 'clientVersion': '1.9',
+ 'clientName': 'ANDROID',
+ 'clientVersion': '19.09.36',
'osName': 'Android',
'osVersion': '12',
'androidSdkVersion': 31,
'platform': 'MOBILE',
- 'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip'
+ 'userAgent': 'com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip'
},
# https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
#'thirdParty': {
@@ -685,57 +690,42 @@ INNERTUBE_CLIENTS = {
'REQUIRE_JS_PLAYER': False,
},
- 'ios': {
- 'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'hl': 'en',
- 'gl': 'US',
- 'clientName': 'IOS',
- 'clientVersion': '19.12.3',
- 'deviceModel': 'iPhone14,3',
- 'userAgent': 'com.google.ios.youtube/19.12.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
- }
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
- 'REQUIRE_JS_PLAYER': False
- },
-
- 'android': {
+ 'android-test-suite': {
'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
'INNERTUBE_CONTEXT': {
'client': {
'hl': 'en',
'gl': 'US',
- 'clientName': 'ANDROID',
- 'clientVersion': '19.15.35',
+ 'clientName': 'ANDROID_TESTSUITE',
+ 'clientVersion': '1.9',
'osName': 'Android',
- 'osVersion': '14',
- 'androidSdkVersion': 34,
+ 'osVersion': '12',
+ 'androidSdkVersion': 31,
'platform': 'MOBILE',
- 'userAgent': 'com.google.android.youtube/19.15.35 (Linux; U; Android 14; en_US; Google Pixel 6 Pro) gzip'
- }
+ 'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip'
+ },
+ # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287
+ #'thirdParty': {
+ # 'embedUrl': 'https://google.com', # Can be any valid URL
+ #}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
'REQUIRE_JS_PLAYER': False,
},
- 'android_music': {
- 'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI',
+ 'ios': {
+ 'INNERTUBE_API_KEY': 'AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc',
'INNERTUBE_CONTEXT': {
'client': {
'hl': 'en',
'gl': 'US',
- 'clientName': 'ANDROID_MUSIC',
- 'clientVersion': '6.48.51',
- 'osName': 'Android',
- 'osVersion': '14',
- 'androidSdkVersion': 34,
- 'platform': 'MOBILE',
- 'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip'
+ 'clientName': 'IOS',
+ 'clientVersion': '19.09.3',
+ 'deviceModel': 'iPhone14,3',
+ 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)'
}
},
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
'REQUIRE_JS_PLAYER': False
},
@@ -766,14 +756,62 @@ INNERTUBE_CLIENTS = {
'INNERTUBE_CONTEXT': {
'client': {
'clientName': 'WEB',
- 'clientVersion': '2.20240327.00.00',
+ 'clientVersion': '2.20220801.00.00',
'userAgent': desktop_user_agent,
}
},
'INNERTUBE_CONTEXT_CLIENT_NAME': 1
},
+ 'android_vr': {
+ 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'ANDROID_VR',
+ 'clientVersion': '1.60.19',
+ 'deviceMake': 'Oculus',
+ 'deviceModel': 'Quest 3',
+ 'androidSdkVersion': 32,
+ 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
+ 'osName': 'Android',
+ 'osVersion': '12L',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
+ 'REQUIRE_JS_PLAYER': False,
+ },
}
+def get_visitor_data():
+ visitor_data = None
+ visitor_data_cache = os.path.join(settings.data_dir, 'visitorData.txt')
+ if not os.path.exists(settings.data_dir):
+ os.makedirs(settings.data_dir)
+ if os.path.isfile(visitor_data_cache):
+ with open(visitor_data_cache, 'r') as file:
+ print('Getting visitor_data from cache')
+ visitor_data = file.read()
+ max_age = 12*3600
+ file_age = time.time() - os.path.getmtime(visitor_data_cache)
+ if file_age > max_age:
+ print('visitor_data cache is too old. Removing file...')
+ os.remove(visitor_data_cache)
+ return visitor_data
+
+ print('Fetching youtube homepage to get visitor_data')
+ yt_homepage = 'https://www.youtube.com'
+ yt_resp = fetch_url(yt_homepage, headers={'User-Agent': mobile_user_agent}, report_text='Getting youtube homepage')
+ visitor_data_re = r'''"visitorData":\s*?"(.+?)"'''
+ visitor_data_match = re.search(visitor_data_re, yt_resp.decode())
+ if visitor_data_match:
+ visitor_data = visitor_data_match.group(1)
+ print(f'Got visitor_data: {len(visitor_data)}')
+ with open(visitor_data_cache, 'w') as file:
+ print('Saving visitor_data cache...')
+ file.write(visitor_data)
+ return visitor_data
+ else:
+ print('Unable to get visitor_data value')
+ return visitor_data
def call_youtube_api(client, api, data):
client_params = INNERTUBE_CLIENTS[client]
@@ -781,12 +819,17 @@ def call_youtube_api(client, api, data):
key = client_params['INNERTUBE_API_KEY']
host = client_params.get('INNERTUBE_HOST') or 'www.youtube.com'
user_agent = context['client'].get('userAgent') or mobile_user_agent
+ visitor_data = get_visitor_data()
url = 'https://' + host + '/youtubei/v1/' + api + '?key=' + key
+ if visitor_data:
+ context['client'].update({'visitorData': visitor_data})
data['context'] = context
data = json.dumps(data)
headers = (('Content-Type', 'application/json'),('User-Agent', user_agent))
+ if visitor_data:
+ headers = ( *headers, ('X-Goog-Visitor-Id', visitor_data ))
response = fetch_url(
url, data=data, headers=headers,
debug_name='youtubei_' + api + '_' + client,
@@ -797,6 +840,8 @@ def call_youtube_api(client, api, data):
def strip_non_ascii(string):
''' Returns the string without non ASCII characters'''
+ if string is None:
+ return ""
stripped = (c for c in string if 0 < ord(c) < 127)
return ''.join(stripped)
diff --git a/youtube/version.py b/youtube/version.py
index 1d74fdd..1ffb850 100644
--- a/youtube/version.py
+++ b/youtube/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '0.2.18'
+__version__ = 'v0.3.2'
diff --git a/youtube/watch.py b/youtube/watch.py
index 2cfece5..0274cd0 100644
--- a/youtube/watch.py
+++ b/youtube/watch.py
@@ -2,7 +2,6 @@ import youtube
from youtube import yt_app
from youtube import util, comments, local_playlist, yt_data_extract
from youtube.util import time_utc_isoformat
-from youtube.util import INNERTUBE_CLIENTS
import settings
from flask import request
@@ -344,7 +343,6 @@ def _add_to_error(info, key, additional_message):
def fetch_player_response(client, video_id):
return util.call_youtube_api(client, 'player', {
'videoId': video_id,
- 'params': 'CgIIAdgDAQ==',
})
@@ -369,84 +367,94 @@ def fetch_watch_page_info(video_id, playlist_id, index):
watch_page = watch_page.decode('utf-8')
return yt_data_extract.extract_watch_info_from_html(watch_page)
+
def extract_info(video_id, use_invidious, playlist_id=None, index=None):
- for client in INNERTUBE_CLIENTS:
- tasks = (
- gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
- gevent.spawn(fetch_player_response, client, video_id) # Use client from INNERTUBE_CLIENTS
- )
- gevent.joinall(tasks)
- util.check_gevent_exceptions(*tasks)
- info, player_response = tasks[0].value, tasks[1].value
+ primary_client = 'android_vr'
+ fallback_client = 'ios'
+ last_resort_client = 'tv_embedded'
+
+ tasks = (
+ # Get video metadata from here
+ gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index),
+ gevent.spawn(fetch_player_response, primary_client, video_id)
+ )
+ gevent.joinall(tasks)
+ util.check_gevent_exceptions(*tasks)
+ info = tasks[0].value or {}
+ player_response = tasks[1].value or {}
+
+ yt_data_extract.update_with_new_urls(info, player_response)
+
+ # Fallback to 'ios' if no valid URLs are found
+ if not info.get('formats') or info.get('player_urls_missing'):
+ print(f"No URLs found in '{primary_client}', attempting with '{fallback_client}'.")
+ player_response = fetch_player_response(fallback_client, video_id) or {}
yt_data_extract.update_with_new_urls(info, player_response)
- # Age restricted video, retry
- if info['age_restricted'] or info['player_urls_missing']:
- if info['age_restricted']:
- print('Age restricted video, retrying')
- else:
- print('Player urls missing, retrying')
- player_response = fetch_player_response('tv_embedded', video_id)
- yt_data_extract.update_with_new_urls(info, player_response)
+ # Final attempt with 'tv_embedded' if there are still no URLs
+ if not info.get('formats') or info.get('player_urls_missing'):
+ print(f"No URLs found in '{fallback_client}', attempting with '{last_resort_client}'")
+ player_response = fetch_player_response(last_resort_client, video_id) or {}
+ yt_data_extract.update_with_new_urls(info, player_response)
- # signature decryption
+ # signature decryption
+ if info.get('formats'):
decryption_error = decrypt_signatures(info, video_id)
if decryption_error:
- decryption_error = 'Error decrypting url signatures: ' + decryption_error
- info['playability_error'] = decryption_error
+ info['playability_error'] = 'Error decrypting url signatures: ' + decryption_error
- # check if urls ready (non-live format) in former livestream
- # urls not ready if all of them have no filesize
- if info['was_live']:
- info['urls_ready'] = False
- for fmt in info['formats']:
- if fmt['file_size'] is not None:
- info['urls_ready'] = True
- else:
- info['urls_ready'] = True
+ # check if urls ready (non-live format) in former livestream
+ # urls not ready if all of them have no filesize
+ if info['was_live']:
+ info['urls_ready'] = False
+ for fmt in info['formats']:
+ if fmt['file_size'] is not None:
+ info['urls_ready'] = True
+ else:
+ info['urls_ready'] = True
- # livestream urls
- # sometimes only the livestream urls work soon after the livestream is over
- if (info['hls_manifest_url']
- and (info['live'] or not info['formats'] or not info['urls_ready'])
- ):
+ # livestream urls
+ # sometimes only the livestream urls work soon after the livestream is over
+ info['hls_formats'] = []
+ if info.get('hls_manifest_url') and (info.get('live') or not info.get('formats') or not info['urls_ready']):
+ try:
manifest = util.fetch_url(info['hls_manifest_url'],
debug_name='hls_manifest.m3u8',
report_text='Fetched hls manifest'
).decode('utf-8')
-
info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest)
if not err:
info['playability_error'] = None
for fmt in info['hls_formats']:
fmt['video_quality'] = video_quality_string(fmt)
- else:
+ except Exception as e:
+ print(f"Error obteniendo HLS manifest: {e}")
info['hls_formats'] = []
- # check for 403. Unnecessary for tor video routing b/c ip address is same
- info['invidious_used'] = False
- info['invidious_reload_button'] = False
- info['tor_bypass_used'] = False
- if (settings.route_tor == 1
- and info['formats'] and info['formats'][0]['url']):
- try:
- response = util.head(info['formats'][0]['url'],
- report_text='Checked for URL access')
- except urllib3.exceptions.HTTPError:
- print('Error while checking for URL access:\n')
- traceback.print_exc()
- return info
-
- if response.status == 403:
- print('Access denied (403) for video urls.')
- print('Routing video through Tor')
- info['tor_bypass_used'] = True
- for fmt in info['formats']:
- fmt['url'] += '&use_tor=1'
- elif 300 <= response.status < 400:
- print('Error: exceeded max redirects while checking video URL')
- return info
+ # check for 403. Unnecessary for tor video routing b/c ip address is same
+ info['invidious_used'] = False
+ info['invidious_reload_button'] = False
+ info['tor_bypass_used'] = False
+ if (settings.route_tor == 1
+ and info['formats'] and info['formats'][0]['url']):
+ try:
+ response = util.head(info['formats'][0]['url'],
+ report_text='Checked for URL access')
+ except urllib3.exceptions.HTTPError:
+ print('Error while checking for URL access:\n')
+ traceback.print_exc()
+ return info
+
+ if response.status == 403:
+ print('Access denied (403) for video urls.')
+ print('Routing video through Tor')
+ info['tor_bypass_used'] = True
+ for fmt in info['formats']:
+ fmt['url'] += '&use_tor=1'
+ elif 300 <= response.status < 400:
+ print('Error: exceeded max redirects while checking video URL')
+ return info
def video_quality_string(format):
@@ -651,12 +659,6 @@ def get_watch_page(video_id=None):
'/videoplayback',
'/videoplayback/name/' + filename)
- if settings.gather_googlevideo_domains:
- with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f:
- url = info['formats'][0]['url']
- subdomain = url[0:url.find(".googlevideo.com")]
- f.write(subdomain + "\n")
-
download_formats = []
for format in (info['formats'] + info['hls_formats']):
--
cgit v1.2.3-59-g8ed1b