libreitalia
/
loaih
7
0
Fork 0
Code Issues 6 Pull Requests Packages Releases 5 Wiki Activity

Sistemazione build.

This commit is contained in:
Emiliano Vavassori 2023-01-05 19:58:45 +01:00
parent f95c4d4b1d
commit a0c6217d95
2 changed files with 120 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
loaih/build.py
View File

@ -1,21 +1,35 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# encoding: utf-8
"""Classes and functions to build an AppImage."""
import os
import glob
import subprocess
import shutil
import re
import shlex
import tempfile
import urllib.request import urllib.request
import loaih
from lxml import etree from lxml import etree
import tempfile, os, sys, glob, subprocess, shutil, re, shlex import loaih
class Collection(list): class Collection(list):
"""Aggregates metadata on a collection of builds."""
def __init__(self, query, arch = ['x86', 'x86_64']): def __init__(self, query, arch = ['x86', 'x86_64']):
"""Build a list of version to check/build for this round.""" """Build a list of version to check/build for this round."""
super().__init__() super().__init__()
self.extend([ Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) ]) self.extend([
Build(query, arch, version) for version in loaih.Base.collectedbuilds(query)
])
class Build(loaih.RemoteBuild): class Build(loaih.RemoteBuild):
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] """Builds a single version."""
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt',
'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
LANGBASIC = [ 'en-GB' ] LANGBASIC = [ 'en-GB' ]
ARCHSTD = [ u'x86', u'x86_64' ] ARCHSTD = [ 'x86', 'x86_64' ]
def __init__(self, query, arch, version = None): def __init__(self, query, arch, version = None):
super().__init__(query, version) super().__init__(query, version)
@ -37,17 +51,24 @@ class Build(loaih.RemoteBuild):
self.remote_path = '' self.remote_path = ''
self.storage_path = '/mnt/appimage' self.storage_path = '/mnt/appimage'
self.download_path = '/var/tmp/downloads' self.download_path = '/var/tmp/downloads'
self.appnamedir = ''
# Specific build version # Specific build version
self.appname = 'LibreOffice'
self.appversion = '' self.appversion = ''
self.appimagedir = ''
self.appimagefilename = {} self.appimagefilename = {}
self.zsyncfilename = {} self.zsyncfilename = {}
# Other variables by build
self.languagepart = '.' + self.language
self.helppart = ''
# Creating a tempfile # Creating a tempfile
self.builddir = tempfile.mkdtemp() self.builddir = tempfile.mkdtemp()
self.tarballs = {} self.tarballs = {}
self.built = { u'x86': False, u'x86_64': False } self.built = { 'x86': False, 'x86_64': False }
# Preparing the default for the relative path on the storage for # Preparing the default for the relative path on the storage for
# different versions. # different versions.
# The path will evaluated as part of the check() function, as it is # The path will evaluated as part of the check() function, as it is
@ -59,7 +80,8 @@ class Build(loaih.RemoteBuild):
def calculate(self): def calculate(self):
"""Calculate exclusions and other variables.""" """Calculate exclusions and other variables."""
# AppName # AppName
self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' if self.query in { 'prerelease', 'daily' }:
self.appname = 'LibreOfficeDev'
# Calculating languagepart # Calculating languagepart
self.languagepart = "." self.languagepart = "."
@ -69,7 +91,8 @@ class Build(loaih.RemoteBuild):
self.languagepart += self.language self.languagepart += self.language
# Calculating help part # Calculating help part
self.helppart = '.help' if self.offline_help else '' if self.offline_help:
self.helppart = '.help'
# Building the required names # Building the required names
for arch in Build.ARCHSTD: for arch in Build.ARCHSTD:
@ -125,9 +148,11 @@ class Build(loaih.RemoteBuild):
path_arr = [ self.storage_path, '' ] path_arr = [ self.storage_path, '' ]
path = str.join('/', path_arr) path = str.join('/', path_arr)
print(f"DEBUG - Name: {name}, URL: {path}") print(f"DEBUG - Name: {name}, URL: {path}")
matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( matching = []
f"//a[contains(@href, '{name}')]/@href" with urllib.request.urlopen(path) as url:
) matching = etree.HTML(url.read()).xpath(
f"//a[contains(@href, '{name}')]/@href"
)
if len(matching) > 0: if len(matching) > 0:
# Already built. # Already built.
@ -136,7 +161,11 @@ class Build(loaih.RemoteBuild):
else: else:
# Repo is local # Repo is local
print(f"Searching for {self.appimagefilename[arch]}") print(f"Searching for {self.appimagefilename[arch]}")
res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') command = f"find {self.full_path} -name {self.appimagefilename[arch]}"
res = subprocess.run(shlex.split(command),
capture_output=True,
env={ "LC_ALL": "C" },
text=True, encoding='utf-8', check=True)
if "No such file or directory" in res.stderr: if "No such file or directory" in res.stderr:
# Folder is not existent: so the version was not built # Folder is not existent: so the version was not built
@ -168,10 +197,15 @@ class Build(loaih.RemoteBuild):
continue continue
# Identifying downloads # Identifying downloads
contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") contents = []
self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] with urllib.request.urlopen(self.url[arch]) as url:
contents = etree.HTML(url.read()).xpath("//td/a")
self.tarballs[arch] = [ x.text
for x in contents
if x.text.endswith('tar.gz') and 'deb' in x.text
]
tarballs = self.tarballs[arch] tarballs = self.tarballs[arch]
maintarball = tarballs[0]
# Create and change directory to the download location # Create and change directory to the download location
os.makedirs(self.download_path, exist_ok = True) os.makedirs(self.download_path, exist_ok = True)
@ -184,8 +218,8 @@ class Build(loaih.RemoteBuild):
# Download the archive # Download the archive
try: try:
urllib.request.urlretrieve(self.url[arch] + archive, archive) urllib.request.urlretrieve(self.url[arch] + archive, archive)
except: except Exception as error:
print(f"Failed to download {archive}.") print(f"Failed to download {archive}: {error}.")
print(f"Finished downloads for {self.version}.") print(f"Finished downloads for {self.version}.")
@ -213,7 +247,7 @@ class Build(loaih.RemoteBuild):
def __unpackbuild__(self, arch): def __unpackbuild__(self, arch):
# We start by filtering out tarballs from the list # We start by filtering out tarballs from the list
buildtarballs = [ self.tarballs[arch][0] ] buildtarballs = [ self.tarballs[arch][0] ]
# Let's process standard languages and append results to the # Let's process standard languages and append results to the
# buildtarball # buildtarball
@ -239,50 +273,75 @@ class Build(loaih.RemoteBuild):
# Looping for each language in self.language # Looping for each language in self.language
for lang in self.language.split(","): for lang in self.language.split(","):
if self.offline_help: if self.offline_help:
buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) buildtarballs.extend([ x for x in self.tarballs[arch]
if 'pack' + lang in x ])
else: else:
buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) buildtarballs.extend([ x for x in self.tarballs[arch]
if 'langpack' + lang in x ])
os.chdir(self.appnamedir) os.chdir(self.appnamedir)
# Unpacking the tarballs # Unpacking the tarballs
for archive in buildtarballs: for archive in buildtarballs:
subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) subprocess.run(shlex.split(
f"tar xzf {self.download_path}/{archive}"), check=True)
# create appimagedir # create appimagedir
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
os.makedirs(self.appimagedir, exist_ok = True) os.makedirs(self.appimagedir, exist_ok = True)
# At this point, let's decompress the deb packages # At this point, let's decompress the deb packages
subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) subprocess.run(shlex.split(
r"find .. -iname '*.deb' -exec dpkg -x {} . \;"
), cwd=self.appimagedir, check=True)
if self.portable: if self.portable:
subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) subprocess.run(shlex.split(
r"find . -type f -iname 'bootstraprc' " +
r"-exec sed -i 's|^UserInstallation=.*|" +
r"UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version
), cwd=self.appimagedir, check=True)
# Changing desktop file # Changing desktop file
subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) subprocess.run(shlex.split(
subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) r"find . -iname startcenter.desktop -exec cp {} . \;"
), cwd=self.appimagedir, check=True)
subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) subprocess.run(shlex.split(
f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " +
r"startcenter.desktop > startcenter.desktop"
), cwd=self.appimagedir, check=True)
subprocess.run(shlex.split(
r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " +
r"-exec cp {} . \;"
), cwd=self.appimagedir, check=True)
# Find the name of the binary called in the desktop file. # Find the name of the binary called in the desktop file.
binaryname = '' binaryname = ''
with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: with open(
a = d.readlines() os.path.join(self.appimagedir, 'startcenter.desktop'),
for line in a: 'r', encoding="utf-8"
) as desktopfile:
for line in desktopfile.readlines():
if re.match(r'^Exec', line): if re.match(r'^Exec', line):
binaryname = line.split('=')[-1].split(' ')[0] binaryname = line.split('=')[-1].split(' ')[0]
# Esci al primo match # Esci al primo match
break break
#binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
#binaryname = binary_exec.stdout.strip("\n") #binaryname = binary_exec.stdout.strip("\n")
bindir=os.path.join(self.appimagedir, 'usr', 'bin') bindir=os.path.join(self.appimagedir, 'usr', 'bin')
os.makedirs(bindir, exist_ok = True) os.makedirs(bindir, exist_ok = True)
subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) subprocess.run(shlex.split(
r"find ../../opt -iname soffice -path '*program*' " +
r"-exec ln -sf {} ./%s \;" % binaryname
), cwd=bindir, check=True)
# Download AppRun from github # Download AppRun from github
apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" apprunurl = r"https://github.com/AppImage/AppImageKit/releases/"
apprunurl += f"download/continuous/AppRun-{arch}"
dest = os.path.join(self.appimagedir, 'AppRun') dest = os.path.join(self.appimagedir, 'AppRun')
urllib.request.urlretrieve(apprunurl, dest) urllib.request.urlretrieve(apprunurl, dest)
os.chmod(dest, 0o755) os.chmod(dest, 0o755)
@ -298,14 +357,19 @@ class Build(loaih.RemoteBuild):
buildopts_str = str.join(' ', buildopts) buildopts_str = str.join(' ', buildopts)
# Build the number-specific build # Build the number-specific build
subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) subprocess.run(shlex.split(
f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
f"./{self.appname}.AppDir/"
), env={ "VERSION": self.appversion }, check=True)
print(f"Built AppImage version {self.appversion}") print(f"Built AppImage version {self.appversion}")
# Cleanup phase, before new run. # Cleanup phase, before new run.
for deb in glob.glob(self.appnamedir + '/*.deb'): for deb in glob.glob(self.appnamedir + '/*.deb'):
os.remove(deb) os.remove(deb)
subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) subprocess.run(shlex.split(
r"find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+"
), check=True)
def checksums(self): def checksums(self):
@ -323,10 +387,11 @@ class Build(loaih.RemoteBuild):
def __create_checksum__(self, file): def __create_checksum__(self, file):
"""Internal function to create checksum file.""" """Internal function to create checksum file."""
checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') checksum = subprocess.run(shlex.split(f"md5sum {file}"),
capture_output=True, text=True, encoding='utf-8', check=True)
if checksum.stdout: if checksum.stdout:
with open(f"{file}.md5", 'w') as c: with open(f"{file}.md5", 'w', encoding='utf-8') as csfile:
c.write(checksum.stdout) csfile.write(checksum.stdout)
def publish(self): def publish(self):
"""Moves built versions to definitive storage.""" """Moves built versions to definitive storage."""
@ -345,11 +410,10 @@ class Build(loaih.RemoteBuild):
else: else:
remotepath = str.join('/', [ self.remote_path, '' ]) remotepath = str.join('/', [ self.remote_path, '' ])
try: try:
subprocess.run( subprocess.run(shlex.split(
shlex.split( r"rsync -rlIvz --munge-links *.AppImage* " +
f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}" f"{self.remote_host}:{remotepath}"
) ), check=True)
)
finally: finally:
pass pass
@ -358,7 +422,9 @@ class Build(loaih.RemoteBuild):
# Forcing creation of subfolders, in case there is a new build # Forcing creation of subfolders, in case there is a new build
os.makedirs(self.full_path, exist_ok = True) os.makedirs(self.full_path, exist_ok = True)
for file in glob.glob("*.AppImage*"): for file in glob.glob("*.AppImage*"):
subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) subprocess.run(shlex.split(
f"cp -f {file} {self.full_path}"
), check=True)
def generalize_and_link(self, chdir = 'default'): def generalize_and_link(self, chdir = 'default'):
@ -368,7 +434,7 @@ class Build(loaih.RemoteBuild):
return return
# If a prerelease or a daily version, either. # If a prerelease or a daily version, either.
if self.query == 'daily' or self.query == 'prerelease': if self.query in { 'daily', 'prerelease' }:
return return
if chdir == 'default': if chdir == 'default':
@ -392,7 +458,9 @@ class Build(loaih.RemoteBuild):
# Doing it both for short_name and for branchname # Doing it both for short_name and for branchname
for version in versions: for version in versions:
appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' appimagefilename[arch] = self.appname + '-' + version
appimagefilename[arch] += self.languagepart + self.helppart
appimagefilename[arch] += f'-{arch}.AppImage'
zsyncfilename[arch] = appimagefilename[arch] + '.zsync' zsyncfilename[arch] = appimagefilename[arch] + '.zsync'
# Create the symlink # Create the symlink
@ -411,7 +479,10 @@ class Build(loaih.RemoteBuild):
os.unlink(zsyncfilename[arch]) os.unlink(zsyncfilename[arch])
shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch])
# Editing the zsyncfile # Editing the zsyncfile
subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) subprocess.run(shlex.split(
r"sed --in-place 's/^Filename:.*$/Filename: " +
f"{appimagefilename[arch]}/' {zsyncfilename[arch]}"
), check=True)
self.__create_checksum__(zsyncfilename[arch]) self.__create_checksum__(zsyncfilename[arch])

2
test.yml
View File

@ -8,7 +8,7 @@ data:
sign: true sign: true
builds: builds:
- query: fresh - query: 7.2.3
language: basic language: basic
offline_help: false offline_help: false
portable: false portable: false