libreitalia
/
loaih
7
0
Fork 0
Code Issues 7 Pull Requests Packages Releases 5 Wiki Activity

Implementing partial downloads. Implementing dependency checks.

This commit is contained in:
emiliano.vavassori 2024-08-22 22:45:02 +02:00
parent ad1ef60947
commit 0c40661251
4 changed files with 104 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
Dockerfile-dev Normal file
View File

@ -0,0 +1,19 @@
# vim:sts=4:sw=4
FROM python:3.9-slim-bullseye
RUN mkdir /build && \
apt update && apt install -y git && \
rm -rf /var/lib/apt/lists/* && \
cd /root && \
git clone https://git.libreitalia.org/libreitalia/loaih.git && \
cd loaih && \
python3 -m venv venv && \
. venv/bin/activate && \
pip install build && \
python3 -m build && \
pip install dist/loaih*.whl && \
deactivate && \
ln -sf /root/loaih/venv/bin/loaih /usr/local/bin/loaih
WORKDIR /build
ENTRYPOINT [ "/usr/local/bin/loaih" ]
CMD [ "--help" ]

1
pyproject.toml
View File

@ -18,6 +18,7 @@ dependencies = [
"lxml",
"pyyaml",
"requests",
"lddcollect"
]
classifiers = [
"Development Status :: 5 - Production/Stable",

99
src/loaih/build.py
View File

@ -40,6 +40,7 @@ class Build():
self.version = version
self.tidy_folder = True
self.verbose = True
self.check_dependencies = False
self.arch = arch
self.short_version = str.join('.', self.version.version.split('.')[0:2])
self.branch_version = self.version.branch
@ -88,6 +89,16 @@ class Build():
def calculate(self):
"""Calculate exclusions and other variables."""
if self.verbose:
print("--- Preliminary system checks ---")
if isinstance(shutil.which('apt'), str):
# APT is found in path. We assume we can find dependencies.
self.check_dependencies = True
else:
print("CAUTION: your system seems not to include a working version of apt.\nThis will cause the AppImage to leverage system libraries when run.")
self.check_dependencies = False
if self.verbose:
print("--- Calculate Phase ---")
@ -174,7 +185,7 @@ class Build():
print(f"Found requested AppImage: {self.appimagefilename}.")
def download(self):
def download(self, compact = False):
"""Downloads the contents of the URL as it was a folder."""
if self.verbose:
@ -196,6 +207,15 @@ class Build():
# Identifying downloads
self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x ]
self.download_tarballs = []
self.download_tarballs.extend(self.tarballs[0])
# Issue #5: manage a limited number of downloads and not the full set.
if compact:
self.download_tarballs = self.__select_tarballs__()
else:
self.download_tarballs = self.tarballs
# Create and change directory to the download location
os.makedirs(self.download_path, exist_ok = True)
os.chdir(self.download_path)
@ -435,40 +455,46 @@ class Build():
file.write(chunk)
return filename
def __unpackbuild__(self):
# We start by filtering out tarballs from the list
buildtarballs = [ self.tarballs[0] ]
def __select_tarballs__(self):
retval = [ self.tarballs[0] ]
# Let's process standard languages and append results to the
# buildtarball
if self.language == 'basic':
if self.offline_help:
buildtarballs.extend([ x for x in self.tarballs if 'pack_en-GB' in x ])
retval.extend([ x for x in self.tarballs if 'pack_en-GB' in x ])
else:
buildtarballs.extend([ x for x in self.tarballs if 'langpack_en-GB' in x])
retval.extend([ x for x in self.tarballs if 'langpack_en-GB' in x])
elif self.language == 'standard':
for lang in Build.LANGSTD:
if self.offline_help:
buildtarballs.extend([ x for x in self.tarballs if 'pack_' + lang in x ])
retval.extend([ x for x in self.tarballs if 'pack_' + lang in x ])
else:
buildtarballs.extend([ x for x in self.tarballs if 'langpack_' + lang in x ])
retval.extend([ x for x in self.tarballs if 'langpack_' + lang in x ])
elif self.language == 'full':
if self.offline_help:
# We need also all help. Let's replace buildtarball with the
# whole bunch
buildtarballs = self.tarballs
retval = self.tarballs
else:
buildtarballs.extend([ x for x in self.tarballs if 'langpack' in x ])
retval.extend([ x for x in self.tarballs if 'langpack' in x ])
else:
# Looping for each language in self.language
for lang in self.language.split(","):
if self.offline_help:
buildtarballs.extend([ x for x in self.tarballs
retval.extend([ x for x in self.tarballs
if 'pack' + lang in x ])
else:
buildtarballs.extend([ x for x in self.tarballs
retval.extend([ x for x in self.tarballs
if 'langpack' + lang in x ])
return retval
def __unpackbuild__(self):
# We start by filtering out tarballs from the list
buildtarballs = self.__select_tarballs__()
os.chdir(self.appnamedir)
# Unpacking the tarballs
@ -482,7 +508,7 @@ class Build():
# create appimagedir
if self.verbose:
print("---- Preparing the build ----")
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
self.appimagedir = os.path.join(self.appnamedir, self.appname + '.AppDir')
os.makedirs(self.appimagedir, exist_ok = True)
# At this point, let's decompress the deb packages
@ -512,6 +538,46 @@ class Build():
r"-exec cp {} . \;"
), cwd=self.appimagedir, check=True)
# Finding path to main executable
cmd = subprocess.run(shlex.split(
r"find -iname soffice.bin -print"
), cwd=self.appimagedir, check = True, capture_output=True)
main_executable = os.path.abspath(os.path.join(
self.appimagedir,
cmd.stdout.strip().decode('utf-8')))
# If the system permits it, we leverage lddcollect
# to find the packages that contain .so dependencies in the main build.
if self.check_dependencies:
if self.verbose:
print("Checking for dependent libraries")
import lddcollect
# We first process the ELF
raw = lddcollect.process_elf(main_executable, verbose = False, dpkg = True)
# If all works as expected, we obtain a tuple of:
# (debian_packages, all_libraries, files_not_found)
(debian_packages, all_libraries, not_found) = raw
if len(debian_packages) != 0:
# We need, first, to download those packages.
debs = [ x.split(":")[0] for x in debian_packages ]
downloadpath = os.path.abspath(os.path.join(self.builddir, 'dependencies'))
if self.verbose:
print("Downloading missing dependencies, please wait.")
# We download the missing dependencies leveraging apt
subprocess.run(shlex.split(
r"apt download " + debs.join(' ')),
cwd=downloadpath, check=True)
# then we install them inside the AppDir
subprocess.run(shlex.split(
r"find " + downloadpath + r" -iname \*.deb -exec dpkg -x {} " + self.appimagedir + r" \;"
), cwd=self.builddir, check=True)
# Find the name of the binary called in the desktop file.
binaryname = ''
with open(
@ -527,10 +593,11 @@ class Build():
#binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
#binaryname = binary_exec.stdout.strip("\n")
# Creating a soft link so the executable in the desktop file is present
bindir=os.path.join(self.appimagedir, 'usr', 'bin')
os.makedirs(bindir, exist_ok = True)
subprocess.run(shlex.split(
r"find ../../opt -iname soffice -path '*program*' " +
r"find ../../opt -iname soffice.bin -path '*program*' " +
r"-exec ln -sf {} ./%s \;" % binaryname
), cwd=bindir, check=True)
@ -557,13 +624,13 @@ class Build():
print("---- Start building ----")
subprocess.run(shlex.split(
f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
f"./{self.appname}.AppDir/"
f"{self.appimagedir}"
), env={ "VERSION": self.appversion }, check=True)
print("---- End building ----")
else:
subprocess.run(shlex.split(
f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
f"./{self.appname}.AppDir/"
f"{self.appimagedir}"
), env={ "VERSION": self.appversion }, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, check=True)

2
src/loaih/script.py
View File

@ -104,7 +104,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
if check:
appbuild.check()
appbuild.download()
appbuild.download(compact = True)
appbuild.build()
if checksums:
appbuild.checksums()