From 0c4066125108769c1caddf2052c507a5dc445786 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 22 Aug 2024 22:45:02 +0200 Subject: [PATCH] Implementing partial downloads. Implementing dependency checks. --- Dockerfile-dev | 19 +++++++++ pyproject.toml | 1 + src/loaih/build.py | 99 +++++++++++++++++++++++++++++++++++++-------- src/loaih/script.py | 2 +- 4 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 Dockerfile-dev diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 0000000..efe57e6 --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,19 @@ +# vim:sts=4:sw=4 +FROM python:3.9-slim-bullseye + +RUN mkdir /build && \ + apt update && apt install -y git && \ + rm -rf /var/lib/apt/lists/* && \ + cd /root && \ + git clone https://git.libreitalia.org/libreitalia/loaih.git && \ + cd loaih && \ + python3 -m venv venv && \ + . venv/bin/activate && \ + pip install build && \ + python3 -m build && \ + pip install dist/loaih*.whl && \ + deactivate && \ + ln -sf /root/loaih/venv/bin/loaih /usr/local/bin/loaih +WORKDIR /build +ENTRYPOINT [ "/usr/local/bin/loaih" ] +CMD [ "--help" ] diff --git a/pyproject.toml b/pyproject.toml index 8adcd73..89488f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "lxml", "pyyaml", "requests", + "lddcollect" ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/src/loaih/build.py b/src/loaih/build.py index c9a0697..2525f55 100644 --- a/src/loaih/build.py +++ b/src/loaih/build.py @@ -40,6 +40,7 @@ class Build(): self.version = version self.tidy_folder = True self.verbose = True + self.check_dependencies = False self.arch = arch self.short_version = str.join('.', self.version.version.split('.')[0:2]) self.branch_version = self.version.branch @@ -88,6 +89,16 @@ class Build(): def calculate(self): """Calculate exclusions and other variables.""" + if self.verbose: + print("--- Preliminary system checks ---") + + if isinstance(shutil.which('apt'), str): + # APT is found in path. We assume we can find dependencies. + self.check_dependencies = True + else: + print("CAUTION: your system seems not to include a working version of apt.\nThis will cause the AppImage to leverage system libraries when run.") + self.check_dependencies = False + if self.verbose: print("--- Calculate Phase ---") @@ -174,7 +185,7 @@ class Build(): print(f"Found requested AppImage: {self.appimagefilename}.") - def download(self): + def download(self, compact = False): """Downloads the contents of the URL as it was a folder.""" if self.verbose: @@ -196,6 +207,15 @@ class Build(): # Identifying downloads self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x ] + self.download_tarballs = [] + self.download_tarballs.extend(self.tarballs[0]) + + # Issue #5: manage a limited number of downloads and not the full set. + if compact: + self.download_tarballs = self.__select_tarballs__() + else: + self.download_tarballs = self.tarballs + # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) @@ -435,40 +455,46 @@ class Build(): file.write(chunk) return filename - def __unpackbuild__(self): - # We start by filtering out tarballs from the list - buildtarballs = [ self.tarballs[0] ] + def __select_tarballs__(self): + retval = [ self.tarballs[0] ] # Let's process standard languages and append results to the # buildtarball if self.language == 'basic': if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs if 'pack_en-GB' in x ]) + retval.extend([ x for x in self.tarballs if 'pack_en-GB' in x ]) else: - buildtarballs.extend([ x for x in self.tarballs if 'langpack_en-GB' in x]) + retval.extend([ x for x in self.tarballs if 'langpack_en-GB' in x]) elif self.language == 'standard': for lang in Build.LANGSTD: if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs if 'pack_' + lang in x ]) + retval.extend([ x for x in self.tarballs if 'pack_' + lang in x ]) else: - buildtarballs.extend([ x for x in self.tarballs if 'langpack_' + lang in x ]) + retval.extend([ x for x in self.tarballs if 'langpack_' + lang in x ]) elif self.language == 'full': if self.offline_help: # We need also all help. Let's replace buildtarball with the # whole bunch - buildtarballs = self.tarballs + retval = self.tarballs else: - buildtarballs.extend([ x for x in self.tarballs if 'langpack' in x ]) + retval.extend([ x for x in self.tarballs if 'langpack' in x ]) else: # Looping for each language in self.language for lang in self.language.split(","): if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs + retval.extend([ x for x in self.tarballs if 'pack' + lang in x ]) else: - buildtarballs.extend([ x for x in self.tarballs + retval.extend([ x for x in self.tarballs if 'langpack' + lang in x ]) + return retval + + + def __unpackbuild__(self): + # We start by filtering out tarballs from the list + buildtarballs = self.__select_tarballs__() + os.chdir(self.appnamedir) # Unpacking the tarballs @@ -482,7 +508,7 @@ class Build(): # create appimagedir if self.verbose: print("---- Preparing the build ----") - self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') + self.appimagedir = os.path.join(self.appnamedir, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages @@ -512,6 +538,46 @@ class Build(): r"-exec cp {} . \;" ), cwd=self.appimagedir, check=True) + # Finding path to main executable + cmd = subprocess.run(shlex.split( + r"find -iname soffice.bin -print" + ), cwd=self.appimagedir, check = True, capture_output=True) + main_executable = os.path.abspath(os.path.join( + self.appimagedir, + cmd.stdout.strip().decode('utf-8'))) + + # If the system permits it, we leverage lddcollect + # to find the packages that contain .so dependencies in the main build. + if self.check_dependencies: + if self.verbose: + print("Checking for dependent libraries") + + import lddcollect + # We first process the ELF + raw = lddcollect.process_elf(main_executable, verbose = False, dpkg = True) + + # If all works as expected, we obtain a tuple of: + # (debian_packages, all_libraries, files_not_found) + (debian_packages, all_libraries, not_found) = raw + + if len(debian_packages) != 0: + # We need, first, to download those packages. + debs = [ x.split(":")[0] for x in debian_packages ] + downloadpath = os.path.abspath(os.path.join(self.builddir, 'dependencies')) + + if self.verbose: + print("Downloading missing dependencies, please wait.") + + # We download the missing dependencies leveraging apt + subprocess.run(shlex.split( + r"apt download " + debs.join(' ')), + cwd=downloadpath, check=True) + + # then we install them inside the AppDir + subprocess.run(shlex.split( + r"find " + downloadpath + r" -iname \*.deb -exec dpkg -x {} " + self.appimagedir + r" \;" + ), cwd=self.builddir, check=True) + # Find the name of the binary called in the desktop file. binaryname = '' with open( @@ -527,10 +593,11 @@ class Build(): #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') #binaryname = binary_exec.stdout.strip("\n") + # Creating a soft link so the executable in the desktop file is present bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) subprocess.run(shlex.split( - r"find ../../opt -iname soffice -path '*program*' " + + r"find ../../opt -iname soffice.bin -path '*program*' " + r"-exec ln -sf {} ./%s \;" % binaryname ), cwd=bindir, check=True) @@ -557,13 +624,13 @@ class Build(): print("---- Start building ----") subprocess.run(shlex.split( f"{self.appnamedir}/appimagetool {buildopts_str} -v " + - f"./{self.appname}.AppDir/" + f"{self.appimagedir}" ), env={ "VERSION": self.appversion }, check=True) print("---- End building ----") else: subprocess.run(shlex.split( f"{self.appnamedir}/appimagetool {buildopts_str} -v " + - f"./{self.appname}.AppDir/" + f"{self.appimagedir}" ), env={ "VERSION": self.appversion }, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) diff --git a/src/loaih/script.py b/src/loaih/script.py index 536bf90..2133a2c 100644 --- a/src/loaih/script.py +++ b/src/loaih/script.py @@ -104,7 +104,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path if check: appbuild.check() - appbuild.download() + appbuild.download(compact = True) appbuild.build() if checksums: appbuild.checksums()