Fix tentativo per creazione di link corretti e zsync truccati.

This commit is contained in:
Emiliano Vavassori 2022-04-08 00:56:28 +02:00
parent 6387ad1ac0
commit 413b84a1c3
2 changed files with 134 additions and 145 deletions

View File

@ -3,7 +3,7 @@
import urllib.request
import loaih.versions as versions
from lxml import etree
import tempfile, os, sys, glob, subprocess, shutil, re
import tempfile, os, sys, glob, subprocess, shutil, re, shlex
class Build(object):
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
@ -13,32 +13,34 @@ class Build(object):
def __init__(self, query, arch):
"""Build all versions that can be found in the indicated repo."""
self.query = query
self.queried_name = False if '.' in self.query else True
self.arch = arch
self.url = {}
# Getting versions and so on
v = versions.BuildVersion(self.query)
self.version = v.version
self.short_version = str.join('.', self.version.split('.')[0:2])
self.branch_version = None
if not '.' in self.query:
self.branch_version = self.query
self.url = v.basedirurl
# Other default values
self.language = 'basic'
self.offline_help = False
self.portable = False
self.updatable = True
self.sign = False
self.storage_path = '/srv/http/appimage.sys42.eu'
self.sign = True
self.storage_path = '/mnt/appimage'
self.download_path = '/var/tmp/downloads'
# Specific build version
self.appversion = ''
self.genappversion = ''
self.appimagefilename = {}
self.genappimagefilename = {}
# Getting versions and so on
v = versions.BuildVersion(self.query)
self.zsyncfilename = {}
# Creating a tempfile
self.builddir = tempfile.mkdtemp()
self.tarballs = {}
self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
self.version = v.version
self.url = v.basedirurl
self.built = { u'x86': False, u'x86_64': False }
# Preparing the default for the relative path on the storage for
@ -51,73 +53,32 @@ class Build(object):
def calculate(self):
"""Calculate exclusions and other variables."""
# Incompatibilities - if portable and updatable are asked together,
# only portable will be built.
if self.portable and self.updatable:
print("Upgradable and portable options were required together. Building only portable.")
self.updatable = False
# AppName
self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
if self.updatable and not self.queried_name:
# If the queried version was a numbered version, doesn't make sense
# to build an updatable version.
self.updatable = False
# Mandate to the private function to calculate the full_path available
# for the storage and the checks.
self.__calculate_full_path__()
# Building expected AppImageName
# Calculating languagepart
self.languagepart = "."
if ',' in self.language:
self.languagepart += self.language.replace(',', '-')
else:
self.languagepart += self.language
# Calculating help part
self.helppart = '.help' if self.offline_help else ''
# If the build was called by queried name, build from latest release available but build with the most generic name
self.appversion = self.version + self.languagepart + self.helppart
myver = str.join('.', self.version.split('.')[0:2])
self.genappversion = myver + self.languagepart + self.helppart
# Building the required names
for arch in Build.ARCHSTD:
self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage'
self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage'
self.appimagefilename[arch] = __genappimagefilename__(self.version, arch)
self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync'
# Mandate to the private function to calculate the full_path available
# for the storage and the checks.
self.__calculate_full_path__()
def check(self):
"""Checking if the requested AppImage has been already built."""
for arch in self.arch:
# For generalized builds, we need to check if there are .ver file
# and it contains the specific version found.
print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver'))
res = subprocess.run("find {path} -name {appimage}'".format(
path = self.full_path,
appimage = self.genappimagefilename[arch] + '.ver'
), shell=True, capture_output=True, env={ "LC_ALL": "C" })
if "No such file or directory" in res.stderr.decode('utf-8'):
# Folder is not existent: so the version was not built
# Build stays false, and we go to the next arch
continue
if res.stdout:
# All good, the command was executed fine.
for file in res.stdout.decode('utf-8').strip('\n').split('\n'):
if self.version in open(file, 'r').read():
self.built[arch] = True
print("Debug: searching for {file}".format(file = self.appimagefilename[arch]))
res = subprocess.run("find {path} -name '{appimage}'".format(
path = self.full_path,
appimage = self.appimagefilename[arch]
), shell=True, capture_output=True)
if res.stdout:
if len(res.stdout.decode('utf-8').strip('\n')) > 1:
self.built[arch] = True
if self.built[arch]:
print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch))
def __gen_appimagefilename__(self, version, arch):
"""Generalize the construction of the name of the app."""
return self.appname + f'-{version}' + self.languagepart + self.helppart + f'-{arch}.AppImage'
def __calculate_full_path__(self):
@ -139,49 +100,70 @@ class Build(object):
self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr))
def check(self):
"""Checking if the requested AppImage has been already built."""
if not len(self.appimagefilename) == 2:
self.calculate()
for arch in self.arch:
print(f"Searching for {self.appimagefilename[arch]}")
res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}'"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8')
if "No such file or directory" in res.stderr:
# Folder is not existent: so the version was not built
# Build stays false, and we go to the next arch
continue
if res.stdout:
# All good, the command was executed fine.
for file in res.stdout.strip('\n').split('\n'):
if self.version in open(file, 'r').read():
print(f"Build for {self.version} found.")
self.built[arch] = True
if self.built[arch]:
print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.")
def download(self):
"""Downloads the contents of the URL as it was a folder."""
print(f"Started downloads for {self.version}. Please wait.")
for arch in self.arch:
# Checking if a valid path has been provided
if self.url[arch] == '-':
print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch))
print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.")
# Faking already built it so to skip other checks.
self.built[arch] = True
continue
if self.built[arch]:
print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch))
print(f"A build for {arch} was already found. Skipping specific packages.")
continue
# Identifying downloads
contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a")
self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ]
tarballs = self.tarballs[arch]
maintarball = tarballs[0]
# Create and change directory to the download location
os.makedirs(self.download_path, exist_ok = True)
os.chdir(self.download_path)
for archive in tarballs:
# If the archive is already there, do not do anything.
if os.path.exists(os.path.join(self.download_path, archive)):
print("Archive %s is already there! Sweet" % archive)
if os.path.exists(archive):
continue
# Download the archive
try:
urllib.request.urlretrieve(self.url[arch] + archive, archive)
except:
print("Failed to download {archive}.".format(archive = archive))
print("Got %s." % archive)
print(f"Failed to download {archive}.")
print(f"Finished downloads for {self.version}.")
def build(self):
"""Building all the versions."""
# We have 4 builds to do:
# * standard languages, no help
# * standard languages + offline help
# * all languages, no help
# * all languages + offline help
for arch in self.arch:
if self.built[arch]:
@ -190,28 +172,18 @@ class Build(object):
# Preparation tasks
self.appnamedir = os.path.join(self.builddir, self.appname)
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
os.makedirs(self.appimagedir, exist_ok = True)
# And then cd to the appname folder.
os.chdir(self.appnamedir)
# Download appimagetool from github
appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch)
appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage"
urllib.request.urlretrieve(appimagetoolurl, 'appimagetool')
os.chmod('appimagetool', 0o755)
# Build the requested version.
if self.queried_name and not self.portable:
# If it is portable, do not generate a generalized version
self.__unpackbuild__(arch, True)
self.__unpackbuild__(arch)
def __unpackbuild__(self, arch, generalize = False):
if generalize and self.portable:
# Doesn't particularly make sense to build a generic portable
# version. Just skipping the specific generic build
return
def __unpackbuild__(self, arch):
# We start by filtering out tarballs from the list
buildtarballs = [ self.tarballs[arch][0] ]
@ -243,35 +215,38 @@ class Build(object):
else:
buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ])
# Unpacking the tarballs
for archive in buildtarballs:
subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True)
os.chdir(self.appnamedir)
# Unpacking the tarballs
for archive in buildtarballs:
subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}"))
# create appimagedir
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
os.makedirs(self.appimagedir, exist_ok = True)
# At this point, let's decompress the deb packages
subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir)
subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir)
if self.portable:
shortversion = str.join('.', self.version.split('.')[:3])
subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir)
subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir)
# Changing desktop file
subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir)
subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir)
subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir)
subprocess.run(shlex.split("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname), cwd=self.appimagedir)
subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir)
subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir)
# Find the name of the binary called in the desktop file.
binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n')
binary_exec = subprocess.run(shlex.split("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
binaryname = binary_exec.stdout.strip("\n")
bindir=os.path.join(self.appimagedir, 'usr', 'bin')
os.makedirs(bindir, exist_ok = True)
subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir)
subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir)
# Download AppRun from github
apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch)
apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}"
dest = os.path.join(self.appimagedir, 'AppRun')
urllib.request.urlretrieve(apprunurl, dest)
os.chmod(dest, 0o755)
@ -280,57 +255,36 @@ class Build(object):
buildopts = []
if self.sign:
buildopts.append('--sign')
# Building app
# adding zsync build if updatable
if self.updatable:
# Updatable make sense only for generic images for fresh, still,
# daily. If a request was for a specific version, I'd not build an
# updatable version.
# zsync name was generated already
buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'")
# If asked to do a generalized build:
if generalize:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
# Build version file management
with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
v.write(self.version)
else:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
else:
if generalize:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
v.write(self.version)
else:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
print("Built AppImage version {version}".format(version = self.appversion))
buildopts_str = str.join(' ', buildopts)
# Build the number-specific build
subprocess.run(shlex.split(f"VERSION={self.appversion} ./appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"))
print(f"Built AppImage version {self.appversion}")
# Cleanup phase, before new run.
for deb in glob.glob(self.appnamedir + '/*.deb'):
os.remove(deb)
subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True)
subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+"))
def checksums(self):
"""Create checksums of the built versions."""
# Skip checksum if initally the build was already found in the storage directory
if all(self.built.values()):
# All checksums are already created.
return
# On the contrary, checksums will be in any case overwritten if
# existent, but generated only for built packages anyways
return
os.chdir(self.appnamedir)
for appimage in glob.glob('*.AppImage*'):
if appimage.endswith('.ver'):
# Skipping checksums for .ver files.
continue
# See if a checksum already exist
if not os.path.exists(appimage + '.md5'):
subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True)
for arch in self.arch:
for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]:
# For any built arch, find out if a file exist.
if len(glob.glob(self.appimagefilename[arch] + '.md5')) == 0:
# Build checksum
subprocess.run(shlex.split(f"md5sum {item} > {item}.md5"))
def publish(self):
@ -342,7 +296,41 @@ class Build(object):
os.chdir(self.appnamedir)
# Forcing creation of subfolders, in case there is a new build
os.makedirs(self.full_path, exist_ok = True)
subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True)
for file in glob.glob("*.AppImage*"):
subprocess.run(shlex.split(f"cp -f {file} {self.fullpath}"))
def generalize_and_link(self):
"""Creates the needed generalized files if needed."""
# If called with a pointed version, no generalize and link necessary.
if not self.branch_version:
return
# Creating versions for short version and query text
versions = [ self.short_version, self.branch_version ]
for arch in Build.ARCHSTD:
# if the appimage for the reported arch is not found, skip to next
# arch
if not os.path.exists(self.appimagefilename[arch]):
continue
# Doing it both for short_name and for branchname
for version in versions:
appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage'
zsyncfilename[arch] = appimagefilename[arch] + '.zsync'
os.chdir(self.full_path)
# Create the symlink
os.symlink(self.appimagefilename[arch], appimagefilename[arch])
# Create the checksum for the AppImage
subprocess.run(shlex.split("md5sum {item} > {item}.md5".format(item=appimagefilename[arch])))
# Do not continue if no zsync file is provided.
if not self.updatable:
continue
os.copy(self.zsyncfilename[arch], zsyncfilename[arch])
# Editing the zsyncfile
subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}"))
def __del__(self):

View File

@ -12,7 +12,7 @@ import loaih
@click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic")
@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help")
@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable")
@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu")
@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage")
@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign")
@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable")
@click.argument('query')
@ -57,6 +57,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
obj.build()
obj.checksums()
obj.publish()
obj.generalize_and_link()
del obj
else: