youtube fronendHEADmaster

1 files changed, 247 insertions, 0 deletions

diff --git a/generate_release.py b/generate_release.py
new file mode 100644
index 0000000..8bec3a0
--- /dev/null
+++ b/generate_release.py
@@ -0,0 +1,247 @@
+# Generate a windows release and a generated embedded distribution of python
+# Latest python version is the argument of the script (or oldwin for
+# vista, 7 and 32-bit versions)
+# Requirements: 7z, git
+# wine is required in order to build on Linux
+
+import sys
+import urllib
+import urllib.request
+import subprocess
+import shutil
+import os
+import hashlib
+
+latest_version = sys.argv[1]
+if len(sys.argv) > 2:
+    bitness = sys.argv[2]
+else:
+    bitness = '64'
+
+if latest_version == 'oldwin':
+    bitness = '32'
+    latest_version = '3.7.9'
+    suffix = 'windows-vista-7-only'
+else:
+    suffix = 'windows'
+
+def check(code):
+    if code != 0:
+        raise Exception('Got nonzero exit code from command')
+def check_subp(x):
+    if x.returncode != 0:
+        raise Exception('Got nonzero exit code from command')
+
+def log(line):
+    print('[generate_release.py] ' + line)
+
+# https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions
+def remove_files_with_extensions(path, extensions):
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            if os.path.splitext(file)[1] in extensions:
+                os.remove(os.path.join(root, file))
+
+def download_if_not_exists(file_name, url, sha256=None):
+    if not os.path.exists('./' + file_name):
+        log('Downloading ' + file_name + '..')
+        data = urllib.request.urlopen(url).read()
+        log('Finished downloading ' + file_name)
+        with open('./' + file_name, 'wb') as f:
+            f.write(data)
+        if sha256:
+            digest = hashlib.sha256(data).hexdigest()
+            if digest != sha256:
+                log('Error: ' + file_name + ' has wrong hash: ' + digest)
+                sys.exit(1)
+    else:
+        log('Using existing ' + file_name)
+
+def wine_run_shell(command):
+    if os.name == 'posix':
+        check(os.system('wine ' + command.replace('\\', '/')))
+    elif os.name == 'nt':
+        check(os.system(command))
+    else:
+        raise Exception('Unsupported OS')
+
+def wine_run(command_parts):
+    if os.name == 'posix':
+        command_parts = ['wine',] + command_parts
+    if subprocess.run(command_parts).returncode != 0:
+        raise Exception('Got nonzero exit code from command')
+
+# ---------- Get current release version, for later ----------
+log('Getting current release version')
+describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE)
+if describe_result.returncode != 0:
+    raise Exception('Git describe failed')
+
+release_tag = describe_result.stdout.strip().decode('ascii')
+
+
+# ----------- Make copy of yt-local files using git -----------
+
+if os.path.exists('./yt-local'):
+    log('Removing old release')
+    shutil.rmtree('./yt-local')
+
+# Export git repository - this will ensure .git and things in gitignore won't
+# be included. Git only supports exporting archive formats, not into
+# directories, so pipe into 7z to put it into .\yt-local (not to be
+# confused with working directory. I'm calling it the same thing so it will
+# have that name when extracted from the final release zip archive)
+log('Making copy of yt-local files')
+check(os.system('git archive --format tar master | 7z x -si -ttar -oyt-local'))
+
+if len(os.listdir('./yt-local')) == 0:
+    raise Exception('Failed to copy yt-local files')
+
+
+# ----------- Generate embedded python distribution -----------
+os.environ['PYTHONDONTWRITEBYTECODE'] = '1'     # *.pyc files double the size of the distribution
+get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
+latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version
+if bitness == '32':
+    latest_dist_url += '-embed-win32.zip'
+else:
+    latest_dist_url += '-embed-amd64.zip'
+
+# I've verified that all the dlls in the following are signed by Microsoft.
+# Using this because Microsoft only provides installers whose files can't be
+# extracted without a special tool.
+if bitness == '32':
+    visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x86.7z'
+    visual_c_runtime_sha256 = '2549eb4d2ce4cf3a87425ea01940f74368bf1cda378ef8a8a1f1a12ed59f1547'
+    visual_c_name = 'vc15_(14.10.25017.0)_2017_x86.7z'
+    visual_c_path_to_dlls = 'runtime_minimum/System'
+else:
+    visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x64.7z'
+    visual_c_runtime_sha256 = '4f00b824c37e1017a93fccbd5775e6ee54f824b6786f5730d257a87a3d9ce921'
+    visual_c_name = 'vc15_(14.10.25017.0)_2017_x64.7z'
+    visual_c_path_to_dlls = 'runtime_minimum/System64'
+
+download_if_not_exists('get-pip.py', get_pip_url)
+
+python_dist_name = 'python-dist-' + latest_version + '-' + bitness + '.zip'
+
+download_if_not_exists(python_dist_name, latest_dist_url)
+download_if_not_exists(visual_c_name,
+    visual_c_runtime_url, sha256=visual_c_runtime_sha256)
+
+if os.path.exists('./python'):
+    log('Removing old python distribution')
+    shutil.rmtree('./python')
+
+
+log('Extracting python distribution')
+
+check(os.system(r'7z -y x -opython ' + python_dist_name))
+
+log('Executing get-pip.py')
+wine_run(['./python/python.exe', '-I', 'get-pip.py'])
+
+'''
+# Explanation of .pth, ._pth, and isolated mode
+
+## Isolated mode
+    We want to run in what is called isolated mode, given by the switch -I.
+This mode prevents the embedded python distribution from searching in
+global directories for imports
+
+    For example, if a user has `C:\Python37` and the embedded distribution is
+the same version, importing something using the embedded distribution will
+search `C:\Python37\Libs\site-packages`. This is not desirable because it
+means I might forget to distribute a dependency if I have it installed
+globally and I don't see any import errors. It also means that an outdated
+package might override the one being distributed and cause other problems.
+
+    Isolated mode also means global environment variables and registry
+entries will be ignored
+
+## The trouble with isolated mode
+    Isolated mode also prevents the current working directory (cwd) from
+being added to `sys.path`. `sys.path` is the list of directories python will
+search in for imports. In non-isolated mode this is automatically populated
+with the cwd, `site-packages`, the directory of the python executable, etc.
+
+# How to get the cwd into sys.path in isolated mode
+    The hack to get this to work is to use a .pth file. Normally, these files
+are just an additional list of directories to be added to `sys.path`.
+However, they also allow arbitrary code execution on lines beginning with
+`import ` (see https://docs.python.org/3/library/site.html). So, we simply
+add `import sys; sys.path.insert(0, '')` to add the cwd to path. `''` is
+shorthand for the cwd. See https://bugs.python.org/issue33698#msg318272
+
+# ._pth files in the embedded distribution
+A python37._pth file is included in the embedded distribution. The presence
+of tis file causes the embedded distribution to always use isolated mode
+(which we want). They are like .pth files, except they do not allow the
+arbitrary code execution trick. In my experimentation, I found that they
+prevent .pth files from loading. So the ._pth file will have to be removed
+and replaced with a .pth. Isolated mode will have to be specified manually.
+'''
+
+log('Removing ._pth')
+major_release = latest_version.split('.')[1]
+os.remove(r'./python/python3' + major_release + '._pth')
+
+log('Adding path_fixes.pth')
+with open(r'./python/path_fixes.pth', 'w', encoding='utf-8') as f:
+    f.write("import sys; sys.path.insert(0, '')\n")
+
+
+'''# python3x._pth file tells the python executable where to look for files
+#  Need to add the directory where packages are installed,
+# and the parent directory (which is where the yt-local files are)
+major_release = latest_version.split('.')[1]
+with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f:
+    f.write('.\\Lib\\site-packages\n')
+    f.write('..\n')'''
+
+log('Inserting Microsoft C Runtime')
+check_subp(subprocess.run([r'7z', '-y', 'e', '-opython', visual_c_name, visual_c_path_to_dlls]))
+
+log('Installing dependencies')
+wine_run(['./python/python.exe', '-I', '-m', 'pip', 'install', '--no-compile', '-r', './requirements.txt'])
+
+log('Uninstalling unnecessary gevent stuff')
+wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'cffi', 'pycparser'])
+shutil.rmtree(r'./python/Lib/site-packages/gevent/tests')
+shutil.rmtree(r'./python/Lib/site-packages/gevent/testing')
+remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation
+
+log('Uninstalling pip and others')
+wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'pip', 'wheel'])
+
+log('Removing pyc files')   # Have to do this because get-pip and some packages don't respect --no-compile
+remove_files_with_extensions(r'./python', ['.pyc'])
+
+log('Removing dist-info and __pycache__')
+for root, dirs, files in os.walk(r'./python'):
+    for dir in dirs:
+        if dir == '__pycache__' or dir.endswith('.dist-info'):
+            shutil.rmtree(os.path.join(root, dir))
+
+
+'''log('Removing get-pip.py and zipped distribution')
+os.remove(r'.\get-pip.py')
+os.remove(r'.\latest-dist.zip')'''
+
+print()
+log('Finished generating python distribution')
+
+# ----------- Copy generated distribution into release folder -----------
+log('Copying python distribution into release folder')
+shutil.copytree(r'./python', r'./yt-local/python')
+
+# ----------- Create release zip -----------
+output_filename = 'yt-local-' + release_tag + '-' + suffix + '.zip'
+if os.path.exists('./' + output_filename):
+    log('Removing previous zipped release')
+    os.remove('./' + output_filename)
+log('Zipping release')
+check(os.system(r'7z -mx=9 a ' + output_filename + ' ./yt-local'))
+
+print('\n')
+log('Finished')