From a4e01da27c08e43a67b2618ad1e71c1f8f86d5cd Mon Sep 17 00:00:00 2001 From: Biswakalyan Bhuyan Date: Thu, 19 Sep 2024 15:33:11 +0530 Subject: youtube fronend --- generate_release.py | 247 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 generate_release.py (limited to 'generate_release.py') diff --git a/generate_release.py b/generate_release.py new file mode 100644 index 0000000..8bec3a0 --- /dev/null +++ b/generate_release.py @@ -0,0 +1,247 @@ +# Generate a windows release and a generated embedded distribution of python +# Latest python version is the argument of the script (or oldwin for +# vista, 7 and 32-bit versions) +# Requirements: 7z, git +# wine is required in order to build on Linux + +import sys +import urllib +import urllib.request +import subprocess +import shutil +import os +import hashlib + +latest_version = sys.argv[1] +if len(sys.argv) > 2: + bitness = sys.argv[2] +else: + bitness = '64' + +if latest_version == 'oldwin': + bitness = '32' + latest_version = '3.7.9' + suffix = 'windows-vista-7-only' +else: + suffix = 'windows' + +def check(code): + if code != 0: + raise Exception('Got nonzero exit code from command') +def check_subp(x): + if x.returncode != 0: + raise Exception('Got nonzero exit code from command') + +def log(line): + print('[generate_release.py] ' + line) + +# https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions +def remove_files_with_extensions(path, extensions): + for root, dirs, files in os.walk(path): + for file in files: + if os.path.splitext(file)[1] in extensions: + os.remove(os.path.join(root, file)) + +def download_if_not_exists(file_name, url, sha256=None): + if not os.path.exists('./' + file_name): + log('Downloading ' + file_name + '..') + data = urllib.request.urlopen(url).read() + log('Finished downloading ' + file_name) + with open('./' + file_name, 'wb') as f: + f.write(data) + if sha256: + digest = hashlib.sha256(data).hexdigest() + if digest != sha256: + log('Error: ' + file_name + ' has wrong hash: ' + digest) + sys.exit(1) + else: + log('Using existing ' + file_name) + +def wine_run_shell(command): + if os.name == 'posix': + check(os.system('wine ' + command.replace('\\', '/'))) + elif os.name == 'nt': + check(os.system(command)) + else: + raise Exception('Unsupported OS') + +def wine_run(command_parts): + if os.name == 'posix': + command_parts = ['wine',] + command_parts + if subprocess.run(command_parts).returncode != 0: + raise Exception('Got nonzero exit code from command') + +# ---------- Get current release version, for later ---------- +log('Getting current release version') +describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE) +if describe_result.returncode != 0: + raise Exception('Git describe failed') + +release_tag = describe_result.stdout.strip().decode('ascii') + + +# ----------- Make copy of yt-local files using git ----------- + +if os.path.exists('./yt-local'): + log('Removing old release') + shutil.rmtree('./yt-local') + +# Export git repository - this will ensure .git and things in gitignore won't +# be included. Git only supports exporting archive formats, not into +# directories, so pipe into 7z to put it into .\yt-local (not to be +# confused with working directory. I'm calling it the same thing so it will +# have that name when extracted from the final release zip archive) +log('Making copy of yt-local files') +check(os.system('git archive --format tar master | 7z x -si -ttar -oyt-local')) + +if len(os.listdir('./yt-local')) == 0: + raise Exception('Failed to copy yt-local files') + + +# ----------- Generate embedded python distribution ----------- +os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # *.pyc files double the size of the distribution +get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' +latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version +if bitness == '32': + latest_dist_url += '-embed-win32.zip' +else: + latest_dist_url += '-embed-amd64.zip' + +# I've verified that all the dlls in the following are signed by Microsoft. +# Using this because Microsoft only provides installers whose files can't be +# extracted without a special tool. +if bitness == '32': + visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x86.7z' + visual_c_runtime_sha256 = '2549eb4d2ce4cf3a87425ea01940f74368bf1cda378ef8a8a1f1a12ed59f1547' + visual_c_name = 'vc15_(14.10.25017.0)_2017_x86.7z' + visual_c_path_to_dlls = 'runtime_minimum/System' +else: + visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x64.7z' + visual_c_runtime_sha256 = '4f00b824c37e1017a93fccbd5775e6ee54f824b6786f5730d257a87a3d9ce921' + visual_c_name = 'vc15_(14.10.25017.0)_2017_x64.7z' + visual_c_path_to_dlls = 'runtime_minimum/System64' + +download_if_not_exists('get-pip.py', get_pip_url) + +python_dist_name = 'python-dist-' + latest_version + '-' + bitness + '.zip' + +download_if_not_exists(python_dist_name, latest_dist_url) +download_if_not_exists(visual_c_name, + visual_c_runtime_url, sha256=visual_c_runtime_sha256) + +if os.path.exists('./python'): + log('Removing old python distribution') + shutil.rmtree('./python') + + +log('Extracting python distribution') + +check(os.system(r'7z -y x -opython ' + python_dist_name)) + +log('Executing get-pip.py') +wine_run(['./python/python.exe', '-I', 'get-pip.py']) + +''' +# Explanation of .pth, ._pth, and isolated mode + +## Isolated mode + We want to run in what is called isolated mode, given by the switch -I. +This mode prevents the embedded python distribution from searching in +global directories for imports + + For example, if a user has `C:\Python37` and the embedded distribution is +the same version, importing something using the embedded distribution will +search `C:\Python37\Libs\site-packages`. This is not desirable because it +means I might forget to distribute a dependency if I have it installed +globally and I don't see any import errors. It also means that an outdated +package might override the one being distributed and cause other problems. + + Isolated mode also means global environment variables and registry +entries will be ignored + +## The trouble with isolated mode + Isolated mode also prevents the current working directory (cwd) from +being added to `sys.path`. `sys.path` is the list of directories python will +search in for imports. In non-isolated mode this is automatically populated +with the cwd, `site-packages`, the directory of the python executable, etc. + +# How to get the cwd into sys.path in isolated mode + The hack to get this to work is to use a .pth file. Normally, these files +are just an additional list of directories to be added to `sys.path`. +However, they also allow arbitrary code execution on lines beginning with +`import ` (see https://docs.python.org/3/library/site.html). So, we simply +add `import sys; sys.path.insert(0, '')` to add the cwd to path. `''` is +shorthand for the cwd. See https://bugs.python.org/issue33698#msg318272 + +# ._pth files in the embedded distribution +A python37._pth file is included in the embedded distribution. The presence +of tis file causes the embedded distribution to always use isolated mode +(which we want). They are like .pth files, except they do not allow the +arbitrary code execution trick. In my experimentation, I found that they +prevent .pth files from loading. So the ._pth file will have to be removed +and replaced with a .pth. Isolated mode will have to be specified manually. +''' + +log('Removing ._pth') +major_release = latest_version.split('.')[1] +os.remove(r'./python/python3' + major_release + '._pth') + +log('Adding path_fixes.pth') +with open(r'./python/path_fixes.pth', 'w', encoding='utf-8') as f: + f.write("import sys; sys.path.insert(0, '')\n") + + +'''# python3x._pth file tells the python executable where to look for files +# Need to add the directory where packages are installed, +# and the parent directory (which is where the yt-local files are) +major_release = latest_version.split('.')[1] +with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f: + f.write('.\\Lib\\site-packages\n') + f.write('..\n')''' + +log('Inserting Microsoft C Runtime') +check_subp(subprocess.run([r'7z', '-y', 'e', '-opython', visual_c_name, visual_c_path_to_dlls])) + +log('Installing dependencies') +wine_run(['./python/python.exe', '-I', '-m', 'pip', 'install', '--no-compile', '-r', './requirements.txt']) + +log('Uninstalling unnecessary gevent stuff') +wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'cffi', 'pycparser']) +shutil.rmtree(r'./python/Lib/site-packages/gevent/tests') +shutil.rmtree(r'./python/Lib/site-packages/gevent/testing') +remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation + +log('Uninstalling pip and others') +wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'pip', 'wheel']) + +log('Removing pyc files') # Have to do this because get-pip and some packages don't respect --no-compile +remove_files_with_extensions(r'./python', ['.pyc']) + +log('Removing dist-info and __pycache__') +for root, dirs, files in os.walk(r'./python'): + for dir in dirs: + if dir == '__pycache__' or dir.endswith('.dist-info'): + shutil.rmtree(os.path.join(root, dir)) + + +'''log('Removing get-pip.py and zipped distribution') +os.remove(r'.\get-pip.py') +os.remove(r'.\latest-dist.zip')''' + +print() +log('Finished generating python distribution') + +# ----------- Copy generated distribution into release folder ----------- +log('Copying python distribution into release folder') +shutil.copytree(r'./python', r'./yt-local/python') + +# ----------- Create release zip ----------- +output_filename = 'yt-local-' + release_tag + '-' + suffix + '.zip' +if os.path.exists('./' + output_filename): + log('Removing previous zipped release') + os.remove('./' + output_filename) +log('Zipping release') +check(os.system(r'7z -mx=9 a ' + output_filename + ' ./yt-local')) + +print('\n') +log('Finished') -- cgit v1.2.3-59-g8ed1b