From a0c6217d95b3ba4d8869c7e65969057303318c29 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 19:58:45 +0100 Subject: [PATCH] Sistemazione build. --- loaih/build.py | 167 +++++++++++++++++++++++++++++++++++-------------- test.yml | 2 +- 2 files changed, 120 insertions(+), 49 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index be90f64..071127d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -1,21 +1,35 @@ #!/usr/bin/env python3 +# encoding: utf-8 +"""Classes and functions to build an AppImage.""" +import os +import glob +import subprocess +import shutil +import re +import shlex +import tempfile import urllib.request -import loaih from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re, shlex +import loaih class Collection(list): + """Aggregates metadata on a collection of builds.""" def __init__(self, query, arch = ['x86', 'x86_64']): """Build a list of version to check/build for this round.""" super().__init__() - self.extend([ Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) ]) + self.extend([ + Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) + ]) class Build(loaih.RemoteBuild): - LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] + """Builds a single version.""" + + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', + 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGBASIC = [ 'en-GB' ] - ARCHSTD = [ u'x86', u'x86_64' ] + ARCHSTD = [ 'x86', 'x86_64' ] def __init__(self, query, arch, version = None): super().__init__(query, version) @@ -37,17 +51,24 @@ class Build(loaih.RemoteBuild): self.remote_path = '' self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' + self.appnamedir = '' # Specific build version + self.appname = 'LibreOffice' self.appversion = '' + self.appimagedir = '' self.appimagefilename = {} self.zsyncfilename = {} + # Other variables by build + self.languagepart = '.' + self.language + self.helppart = '' + # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} - self.built = { u'x86': False, u'x86_64': False } - + self.built = { 'x86': False, 'x86_64': False } + # Preparing the default for the relative path on the storage for # different versions. # The path will evaluated as part of the check() function, as it is @@ -59,7 +80,8 @@ class Build(loaih.RemoteBuild): def calculate(self): """Calculate exclusions and other variables.""" # AppName - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' + if self.query in { 'prerelease', 'daily' }: + self.appname = 'LibreOfficeDev' # Calculating languagepart self.languagepart = "." @@ -69,7 +91,8 @@ class Build(loaih.RemoteBuild): self.languagepart += self.language # Calculating help part - self.helppart = '.help' if self.offline_help else '' + if self.offline_help: + self.helppart = '.help' # Building the required names for arch in Build.ARCHSTD: @@ -125,9 +148,11 @@ class Build(loaih.RemoteBuild): path_arr = [ self.storage_path, '' ] path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") - matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( - f"//a[contains(@href, '{name}')]/@href" - ) + matching = [] + with urllib.request.urlopen(path) as url: + matching = etree.HTML(url.read()).xpath( + f"//a[contains(@href, '{name}')]/@href" + ) if len(matching) > 0: # Already built. @@ -136,7 +161,11 @@ class Build(loaih.RemoteBuild): else: # Repo is local print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + command = f"find {self.full_path} -name {self.appimagefilename[arch]}" + res = subprocess.run(shlex.split(command), + capture_output=True, + env={ "LC_ALL": "C" }, + text=True, encoding='utf-8', check=True) if "No such file or directory" in res.stderr: # Folder is not existent: so the version was not built @@ -168,10 +197,15 @@ class Build(loaih.RemoteBuild): continue # Identifying downloads - contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] + contents = [] + with urllib.request.urlopen(self.url[arch]) as url: + contents = etree.HTML(url.read()).xpath("//td/a") + + self.tarballs[arch] = [ x.text + for x in contents + if x.text.endswith('tar.gz') and 'deb' in x.text + ] tarballs = self.tarballs[arch] - maintarball = tarballs[0] # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) @@ -184,8 +218,8 @@ class Build(loaih.RemoteBuild): # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) - except: - print(f"Failed to download {archive}.") + except Exception as error: + print(f"Failed to download {archive}: {error}.") print(f"Finished downloads for {self.version}.") @@ -213,7 +247,7 @@ class Build(loaih.RemoteBuild): def __unpackbuild__(self, arch): # We start by filtering out tarballs from the list - buildtarballs = [ self.tarballs[arch][0] ] + buildtarballs = [ self.tarballs[arch][0] ] # Let's process standard languages and append results to the # buildtarball @@ -239,50 +273,75 @@ class Build(loaih.RemoteBuild): # Looping for each language in self.language for lang in self.language.split(","): if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] + if 'pack' + lang in x ]) else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - + buildtarballs.extend([ x for x in self.tarballs[arch] + if 'langpack' + lang in x ]) + os.chdir(self.appnamedir) # Unpacking the tarballs for archive in buildtarballs: - subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + subprocess.run(shlex.split( + f"tar xzf {self.download_path}/{archive}"), check=True) # create appimagedir self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages - subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find .. -iname '*.deb' -exec dpkg -x {} . \;" + ), cwd=self.appimagedir, check=True) if self.portable: - subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find . -type f -iname 'bootstraprc' " + + r"-exec sed -i 's|^UserInstallation=.*|" + + r"UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version + ), cwd=self.appimagedir, check=True) # Changing desktop file - subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) - subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find . -iname startcenter.desktop -exec cp {} . \;" + ), cwd=self.appimagedir, check=True) - subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split( + f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " + + r"startcenter.desktop > startcenter.desktop" + ), cwd=self.appimagedir, check=True) + + subprocess.run(shlex.split( + r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " + + r"-exec cp {} . \;" + ), cwd=self.appimagedir, check=True) # Find the name of the binary called in the desktop file. binaryname = '' - with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: - a = d.readlines() - for line in a: + with open( + os.path.join(self.appimagedir, 'startcenter.desktop'), + 'r', encoding="utf-8" + ) as desktopfile: + for line in desktopfile.readlines(): if re.match(r'^Exec', line): binaryname = line.split('=')[-1].split(' ')[0] # Esci al primo match break + #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') #binaryname = binary_exec.stdout.strip("\n") bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) + subprocess.run(shlex.split( + r"find ../../opt -iname soffice -path '*program*' " + + r"-exec ln -sf {} ./%s \;" % binaryname + ), cwd=bindir, check=True) # Download AppRun from github - apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" + apprunurl = r"https://github.com/AppImage/AppImageKit/releases/" + apprunurl += f"download/continuous/AppRun-{arch}" dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) @@ -298,14 +357,19 @@ class Build(loaih.RemoteBuild): buildopts_str = str.join(' ', buildopts) # Build the number-specific build - subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) - + subprocess.run(shlex.split( + f"{self.appnamedir}/appimagetool {buildopts_str} -v " + + f"./{self.appname}.AppDir/" + ), env={ "VERSION": self.appversion }, check=True) + print(f"Built AppImage version {self.appversion}") # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) + subprocess.run(shlex.split( + r"find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+" + ), check=True) def checksums(self): @@ -323,10 +387,11 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') + checksum = subprocess.run(shlex.split(f"md5sum {file}"), + capture_output=True, text=True, encoding='utf-8', check=True) if checksum.stdout: - with open(f"{file}.md5", 'w') as c: - c.write(checksum.stdout) + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" @@ -345,11 +410,10 @@ class Build(loaih.RemoteBuild): else: remotepath = str.join('/', [ self.remote_path, '' ]) try: - subprocess.run( - shlex.split( - f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}" - ) - ) + subprocess.run(shlex.split( + r"rsync -rlIvz --munge-links *.AppImage* " + + f"{self.remote_host}:{remotepath}" + ), check=True) finally: pass @@ -358,7 +422,9 @@ class Build(loaih.RemoteBuild): # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + subprocess.run(shlex.split( + f"cp -f {file} {self.full_path}" + ), check=True) def generalize_and_link(self, chdir = 'default'): @@ -368,7 +434,7 @@ class Build(loaih.RemoteBuild): return # If a prerelease or a daily version, either. - if self.query == 'daily' or self.query == 'prerelease': + if self.query in { 'daily', 'prerelease' }: return if chdir == 'default': @@ -392,7 +458,9 @@ class Build(loaih.RemoteBuild): # Doing it both for short_name and for branchname for version in versions: - appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + appimagefilename[arch] = self.appname + '-' + version + appimagefilename[arch] += self.languagepart + self.helppart + appimagefilename[arch] += f'-{arch}.AppImage' zsyncfilename[arch] = appimagefilename[arch] + '.zsync' # Create the symlink @@ -411,7 +479,10 @@ class Build(loaih.RemoteBuild): os.unlink(zsyncfilename[arch]) shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile - subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) + subprocess.run(shlex.split( + r"sed --in-place 's/^Filename:.*$/Filename: " + + f"{appimagefilename[arch]}/' {zsyncfilename[arch]}" + ), check=True) self.__create_checksum__(zsyncfilename[arch]) diff --git a/test.yml b/test.yml index 768cdd6..8fcf1d7 100644 --- a/test.yml +++ b/test.yml @@ -8,7 +8,7 @@ data: sign: true builds: - - query: fresh + - query: 7.2.3 language: basic offline_help: false portable: false