Sistemazione build.
This commit is contained in:
Emiliano Vavassori
parent
f95c4d4b1d
commit
a0c6217d95
167
loaih/build.py
167
loaih/build.py
|
|
@ -1,21 +1,35 @@
|
|||
#!/usr/bin/env python3
|
||||
# encoding: utf-8
|
||||
"""Classes and functions to build an AppImage."""
|
||||
|
||||
import os
|
||||
import glob
|
||||
import subprocess
|
||||
import shutil
|
||||
import re
|
||||
import shlex
|
||||
import tempfile
|
||||
import urllib.request
|
||||
import loaih
|
||||
from lxml import etree
|
||||
import tempfile, os, sys, glob, subprocess, shutil, re, shlex
|
||||
import loaih
|
||||
|
||||
class Collection(list):
|
||||
"""Aggregates metadata on a collection of builds."""
|
||||
|
||||
def __init__(self, query, arch = ['x86', 'x86_64']):
|
||||
"""Build a list of version to check/build for this round."""
|
||||
super().__init__()
|
||||
self.extend([ Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) ])
|
||||
self.extend([
|
||||
Build(query, arch, version) for version in loaih.Base.collectedbuilds(query)
|
||||
])
|
||||
|
||||
class Build(loaih.RemoteBuild):
|
||||
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
|
||||
"""Builds a single version."""
|
||||
|
||||
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt',
|
||||
'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
|
||||
LANGBASIC = [ 'en-GB' ]
|
||||
ARCHSTD = [ u'x86', u'x86_64' ]
|
||||
ARCHSTD = [ 'x86', 'x86_64' ]
|
||||
|
||||
def __init__(self, query, arch, version = None):
|
||||
super().__init__(query, version)
|
||||
|
|
@ -37,17 +51,24 @@ class Build(loaih.RemoteBuild):
|
|||
self.remote_path = ''
|
||||
self.storage_path = '/mnt/appimage'
|
||||
self.download_path = '/var/tmp/downloads'
|
||||
self.appnamedir = ''
|
||||
|
||||
# Specific build version
|
||||
self.appname = 'LibreOffice'
|
||||
self.appversion = ''
|
||||
self.appimagedir = ''
|
||||
self.appimagefilename = {}
|
||||
self.zsyncfilename = {}
|
||||
|
||||
# Other variables by build
|
||||
self.languagepart = '.' + self.language
|
||||
self.helppart = ''
|
||||
|
||||
# Creating a tempfile
|
||||
self.builddir = tempfile.mkdtemp()
|
||||
self.tarballs = {}
|
||||
self.built = { u'x86': False, u'x86_64': False }
|
||||
|
||||
self.built = { 'x86': False, 'x86_64': False }
|
||||
|
||||
# Preparing the default for the relative path on the storage for
|
||||
# different versions.
|
||||
# The path will evaluated as part of the check() function, as it is
|
||||
|
|
@ -59,7 +80,8 @@ class Build(loaih.RemoteBuild):
|
|||
def calculate(self):
|
||||
"""Calculate exclusions and other variables."""
|
||||
# AppName
|
||||
self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
|
||||
if self.query in { 'prerelease', 'daily' }:
|
||||
self.appname = 'LibreOfficeDev'
|
||||
|
||||
# Calculating languagepart
|
||||
self.languagepart = "."
|
||||
|
|
@ -69,7 +91,8 @@ class Build(loaih.RemoteBuild):
|
|||
self.languagepart += self.language
|
||||
|
||||
# Calculating help part
|
||||
self.helppart = '.help' if self.offline_help else ''
|
||||
if self.offline_help:
|
||||
self.helppart = '.help'
|
||||
|
||||
# Building the required names
|
||||
for arch in Build.ARCHSTD:
|
||||
|
|
@ -125,9 +148,11 @@ class Build(loaih.RemoteBuild):
|
|||
path_arr = [ self.storage_path, '' ]
|
||||
path = str.join('/', path_arr)
|
||||
print(f"DEBUG - Name: {name}, URL: {path}")
|
||||
matching = etree.HTML(urllib.request.urlopen(path).read()).xpath(
|
||||
f"//a[contains(@href, '{name}')]/@href"
|
||||
)
|
||||
matching = []
|
||||
with urllib.request.urlopen(path) as url:
|
||||
matching = etree.HTML(url.read()).xpath(
|
||||
f"//a[contains(@href, '{name}')]/@href"
|
||||
)
|
||||
|
||||
if len(matching) > 0:
|
||||
# Already built.
|
||||
|
|
@ -136,7 +161,11 @@ class Build(loaih.RemoteBuild):
|
|||
else:
|
||||
# Repo is local
|
||||
print(f"Searching for {self.appimagefilename[arch]}")
|
||||
res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8')
|
||||
command = f"find {self.full_path} -name {self.appimagefilename[arch]}"
|
||||
res = subprocess.run(shlex.split(command),
|
||||
capture_output=True,
|
||||
env={ "LC_ALL": "C" },
|
||||
text=True, encoding='utf-8', check=True)
|
||||
|
||||
if "No such file or directory" in res.stderr:
|
||||
# Folder is not existent: so the version was not built
|
||||
|
|
@ -168,10 +197,15 @@ class Build(loaih.RemoteBuild):
|
|||
continue
|
||||
|
||||
# Identifying downloads
|
||||
contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a")
|
||||
self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ]
|
||||
contents = []
|
||||
with urllib.request.urlopen(self.url[arch]) as url:
|
||||
contents = etree.HTML(url.read()).xpath("//td/a")
|
||||
|
||||
self.tarballs[arch] = [ x.text
|
||||
for x in contents
|
||||
if x.text.endswith('tar.gz') and 'deb' in x.text
|
||||
]
|
||||
tarballs = self.tarballs[arch]
|
||||
maintarball = tarballs[0]
|
||||
|
||||
# Create and change directory to the download location
|
||||
os.makedirs(self.download_path, exist_ok = True)
|
||||
|
|
@ -184,8 +218,8 @@ class Build(loaih.RemoteBuild):
|
|||
# Download the archive
|
||||
try:
|
||||
urllib.request.urlretrieve(self.url[arch] + archive, archive)
|
||||
except:
|
||||
print(f"Failed to download {archive}.")
|
||||
except Exception as error:
|
||||
print(f"Failed to download {archive}: {error}.")
|
||||
|
||||
print(f"Finished downloads for {self.version}.")
|
||||
|
||||
|
|
@ -213,7 +247,7 @@ class Build(loaih.RemoteBuild):
|
|||
|
||||
def __unpackbuild__(self, arch):
|
||||
# We start by filtering out tarballs from the list
|
||||
buildtarballs = [ self.tarballs[arch][0] ]
|
||||
buildtarballs = [ self.tarballs[arch][0] ]
|
||||
|
||||
# Let's process standard languages and append results to the
|
||||
# buildtarball
|
||||
|
|
@ -239,50 +273,75 @@ class Build(loaih.RemoteBuild):
|
|||
# Looping for each language in self.language
|
||||
for lang in self.language.split(","):
|
||||
if self.offline_help:
|
||||
buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ])
|
||||
buildtarballs.extend([ x for x in self.tarballs[arch]
|
||||
if 'pack' + lang in x ])
|
||||
else:
|
||||
buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ])
|
||||
|
||||
buildtarballs.extend([ x for x in self.tarballs[arch]
|
||||
if 'langpack' + lang in x ])
|
||||
|
||||
os.chdir(self.appnamedir)
|
||||
|
||||
# Unpacking the tarballs
|
||||
for archive in buildtarballs:
|
||||
subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}"))
|
||||
subprocess.run(shlex.split(
|
||||
f"tar xzf {self.download_path}/{archive}"), check=True)
|
||||
|
||||
# create appimagedir
|
||||
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
|
||||
os.makedirs(self.appimagedir, exist_ok = True)
|
||||
|
||||
# At this point, let's decompress the deb packages
|
||||
subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir)
|
||||
subprocess.run(shlex.split(
|
||||
r"find .. -iname '*.deb' -exec dpkg -x {} . \;"
|
||||
), cwd=self.appimagedir, check=True)
|
||||
|
||||
if self.portable:
|
||||
subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir)
|
||||
subprocess.run(shlex.split(
|
||||
r"find . -type f -iname 'bootstraprc' " +
|
||||
r"-exec sed -i 's|^UserInstallation=.*|" +
|
||||
r"UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version
|
||||
), cwd=self.appimagedir, check=True)
|
||||
|
||||
# Changing desktop file
|
||||
subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir)
|
||||
subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir)
|
||||
subprocess.run(shlex.split(
|
||||
r"find . -iname startcenter.desktop -exec cp {} . \;"
|
||||
), cwd=self.appimagedir, check=True)
|
||||
|
||||
subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir)
|
||||
subprocess.run(shlex.split(
|
||||
f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " +
|
||||
r"startcenter.desktop > startcenter.desktop"
|
||||
), cwd=self.appimagedir, check=True)
|
||||
|
||||
subprocess.run(shlex.split(
|
||||
r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " +
|
||||
r"-exec cp {} . \;"
|
||||
), cwd=self.appimagedir, check=True)
|
||||
|
||||
# Find the name of the binary called in the desktop file.
|
||||
binaryname = ''
|
||||
with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d:
|
||||
a = d.readlines()
|
||||
for line in a:
|
||||
with open(
|
||||
os.path.join(self.appimagedir, 'startcenter.desktop'),
|
||||
'r', encoding="utf-8"
|
||||
) as desktopfile:
|
||||
for line in desktopfile.readlines():
|
||||
if re.match(r'^Exec', line):
|
||||
binaryname = line.split('=')[-1].split(' ')[0]
|
||||
# Esci al primo match
|
||||
break
|
||||
|
||||
#binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
|
||||
#binaryname = binary_exec.stdout.strip("\n")
|
||||
|
||||
bindir=os.path.join(self.appimagedir, 'usr', 'bin')
|
||||
os.makedirs(bindir, exist_ok = True)
|
||||
subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir)
|
||||
subprocess.run(shlex.split(
|
||||
r"find ../../opt -iname soffice -path '*program*' " +
|
||||
r"-exec ln -sf {} ./%s \;" % binaryname
|
||||
), cwd=bindir, check=True)
|
||||
|
||||
# Download AppRun from github
|
||||
apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}"
|
||||
apprunurl = r"https://github.com/AppImage/AppImageKit/releases/"
|
||||
apprunurl += f"download/continuous/AppRun-{arch}"
|
||||
dest = os.path.join(self.appimagedir, 'AppRun')
|
||||
urllib.request.urlretrieve(apprunurl, dest)
|
||||
os.chmod(dest, 0o755)
|
||||
|
|
@ -298,14 +357,19 @@ class Build(loaih.RemoteBuild):
|
|||
|
||||
buildopts_str = str.join(' ', buildopts)
|
||||
# Build the number-specific build
|
||||
subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion })
|
||||
|
||||
subprocess.run(shlex.split(
|
||||
f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
|
||||
f"./{self.appname}.AppDir/"
|
||||
), env={ "VERSION": self.appversion }, check=True)
|
||||
|
||||
print(f"Built AppImage version {self.appversion}")
|
||||
|
||||
# Cleanup phase, before new run.
|
||||
for deb in glob.glob(self.appnamedir + '/*.deb'):
|
||||
os.remove(deb)
|
||||
subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+"))
|
||||
subprocess.run(shlex.split(
|
||||
r"find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+"
|
||||
), check=True)
|
||||
|
||||
|
||||
def checksums(self):
|
||||
|
|
@ -323,10 +387,11 @@ class Build(loaih.RemoteBuild):
|
|||
|
||||
def __create_checksum__(self, file):
|
||||
"""Internal function to create checksum file."""
|
||||
checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8')
|
||||
checksum = subprocess.run(shlex.split(f"md5sum {file}"),
|
||||
capture_output=True, text=True, encoding='utf-8', check=True)
|
||||
if checksum.stdout:
|
||||
with open(f"{file}.md5", 'w') as c:
|
||||
c.write(checksum.stdout)
|
||||
with open(f"{file}.md5", 'w', encoding='utf-8') as csfile:
|
||||
csfile.write(checksum.stdout)
|
||||
|
||||
def publish(self):
|
||||
"""Moves built versions to definitive storage."""
|
||||
|
|
@ -345,11 +410,10 @@ class Build(loaih.RemoteBuild):
|
|||
else:
|
||||
remotepath = str.join('/', [ self.remote_path, '' ])
|
||||
try:
|
||||
subprocess.run(
|
||||
shlex.split(
|
||||
f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}"
|
||||
)
|
||||
)
|
||||
subprocess.run(shlex.split(
|
||||
r"rsync -rlIvz --munge-links *.AppImage* " +
|
||||
f"{self.remote_host}:{remotepath}"
|
||||
), check=True)
|
||||
finally:
|
||||
pass
|
||||
|
||||
|
|
@ -358,7 +422,9 @@ class Build(loaih.RemoteBuild):
|
|||
# Forcing creation of subfolders, in case there is a new build
|
||||
os.makedirs(self.full_path, exist_ok = True)
|
||||
for file in glob.glob("*.AppImage*"):
|
||||
subprocess.run(shlex.split(f"cp -f {file} {self.full_path}"))
|
||||
subprocess.run(shlex.split(
|
||||
f"cp -f {file} {self.full_path}"
|
||||
), check=True)
|
||||
|
||||
|
||||
def generalize_and_link(self, chdir = 'default'):
|
||||
|
|
@ -368,7 +434,7 @@ class Build(loaih.RemoteBuild):
|
|||
return
|
||||
|
||||
# If a prerelease or a daily version, either.
|
||||
if self.query == 'daily' or self.query == 'prerelease':
|
||||
if self.query in { 'daily', 'prerelease' }:
|
||||
return
|
||||
|
||||
if chdir == 'default':
|
||||
|
|
@ -392,7 +458,9 @@ class Build(loaih.RemoteBuild):
|
|||
|
||||
# Doing it both for short_name and for branchname
|
||||
for version in versions:
|
||||
appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage'
|
||||
appimagefilename[arch] = self.appname + '-' + version
|
||||
appimagefilename[arch] += self.languagepart + self.helppart
|
||||
appimagefilename[arch] += f'-{arch}.AppImage'
|
||||
zsyncfilename[arch] = appimagefilename[arch] + '.zsync'
|
||||
|
||||
# Create the symlink
|
||||
|
|
@ -411,7 +479,10 @@ class Build(loaih.RemoteBuild):
|
|||
os.unlink(zsyncfilename[arch])
|
||||
shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch])
|
||||
# Editing the zsyncfile
|
||||
subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}"))
|
||||
subprocess.run(shlex.split(
|
||||
r"sed --in-place 's/^Filename:.*$/Filename: " +
|
||||
f"{appimagefilename[arch]}/' {zsyncfilename[arch]}"
|
||||
), check=True)
|
||||
self.__create_checksum__(zsyncfilename[arch])
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue