From 50e019262a0f63608b7ac851e99e617b62dc3edd Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 23 Aug 2024 11:38:58 +0200 Subject: [PATCH] Tentative fix to include package dependencies. --- src/loaih/build.py | 207 +++++++++++++++++++++++++++++---------------- 1 file changed, 133 insertions(+), 74 deletions(-) diff --git a/src/loaih/build.py b/src/loaih/build.py index fda5a83..77f8f97 100644 --- a/src/loaih/build.py +++ b/src/loaih/build.py @@ -91,11 +91,22 @@ class Build(): """Calculate exclusions and other variables.""" if self.verbose: - print("--- Preliminary system checks ---") + print("--- Preliminary Phase ---") if isinstance(shutil.which('apt'), str): # APT is found in path. We assume we can find dependencies. self.check_dependencies = True + + + if self.verbose: + print("Updating system packages cache.") + # Updating package cache + subprocess.run(['sudo', 'apt', 'update'], check=True, stdout=subprocess.DEVNULL) + if self.verbose: + print("Ensuring apt-file is installed and updated.") + # Updating apt-file cache + subprocess.run(['sudo', 'apt', 'install', 'apt-file', '-y'], check=True, stdout=subprocess.DEVNULL) + subprocess.run(['sudo', 'apt-file', 'update'], check=True, stdout=subprocess.DEVNULL) else: print("CAUTION: your system seems not to include a working version of apt.\nThis will cause the AppImage to leverage system libraries when run.") self.check_dependencies = False @@ -237,12 +248,12 @@ class Build(): def build(self): """Building all the versions.""" - if self.verbose: - print("--- Building Phase ---") - if self.found: return + if self.verbose: + print("--- Building Phase ---") + # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) os.makedirs(self.appnamedir, exist_ok=True) @@ -256,6 +267,13 @@ class Build(): # Build the requested version. self.__unpackbuild__() + self.__prepare_contents__() + if self.check_dependencies: + if self.verbose: + print("Searching for dependent libraries, it might take a while.") + + self.__missing_dependencies__() + self.__finalize_build__() def checksums(self): @@ -505,6 +523,7 @@ class Build(): subprocess.run(shlex.split( f"tar xzf {self.download_path}/{archive}"), check=True) + def __prepare_contents__(self): # create appimagedir if self.verbose: print("---- Preparing the build ----") @@ -512,6 +531,9 @@ class Build(): os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages + if self.verbose: + print("Unpacking main archives") + subprocess.run(shlex.split( r"find .. -iname '*.deb' -exec dpkg -x {} . \;" ), cwd=self.appimagedir, check=True) @@ -524,6 +546,9 @@ class Build(): ), cwd=self.appimagedir, check=True) # Changing desktop file + if self.verbose: + print("Preparing .desktop file.") + subprocess.run(shlex.split( r"find . -iname startcenter.desktop -exec cp {} . \;" ), cwd=self.appimagedir, check=True) @@ -533,6 +558,8 @@ class Build(): r"startcenter.desktop" ), cwd=self.appimagedir, check=False) + if self.verbose: + print("Preparing icon file.") subprocess.run(shlex.split( r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " + r"-exec cp {} . \;" @@ -542,62 +569,10 @@ class Build(): cmd = subprocess.run(shlex.split( r"find -iname soffice.bin -print" ), cwd=self.appimagedir, check = True, capture_output=True) - main_executable = os.path.abspath(os.path.join( + self.main_executable = os.path.abspath(os.path.join( self.appimagedir, cmd.stdout.strip().decode('utf-8'))) - # If the system permits it, we leverage lddcollect - # to find the packages that contain .so dependencies in the main build. - if self.check_dependencies: - if self.verbose: - print("Checking for dependent libraries") - - import lddcollect - # We first process the ELF - raw = lddcollect.process_elf(main_executable, verbose = False, dpkg = True) - - # If all works as expected, we obtain a tuple of: - # (debian_packages, all_libraries, files_not_found) - (debian_packages, all_libraries, not_found) = raw - - if len(debian_packages) != 0: - # We need, first, to download those packages. - debs = [ x.split(':')[0] for x in debian_packages ] - downloadpath = os.path.abspath(os.path.join(self.builddir, 'dependencies')) - os.makedirs(downloadpath) - - if self.verbose: - print("Downloading missing dependencies, please wait.") - - # Updating package cache - subprocess.run(['sudo', 'apt', 'update'], check=True) - # Updating apt-file cache - subprocess.run(['sudo', 'apt-file', 'update'], check=True) - - # Let's try to find and install also other libraries - additional = list(dict.fromkeys([ libfinderhelper(x) for x in not_found ])) - debs.extend(additional) - - # We download the missing dependencies leveraging apt - subprocess.run(shlex.split( - r"apt download " + " ".join(debs) - ), cwd=downloadpath, check=True) - - # then we install them inside a temporary path - temporary = os.path.abspath(os.path.join(downloadpath, 'temp')) - os.makedirs(temporary) - subprocess.run(shlex.split( - r"find " + downloadpath + r" -iname \*.deb -exec dpkg -x {} " + temporary + r" \;" - ), cwd=self.builddir, check=True) - - # We are finally copying the .so files in the same path as main_executable - libdirs = [ 'lib/x86_64-linux-gnu', 'usr/lib/x86_64-linux-gnu' ] - for libdir in libdirs: - fulllibdir = os.path.abspath(os.path.join(temporary, libdir)) - subprocess.run(shlex.split( - f"cp -Ra {fulllibdir}/. {os.path.dirname(main_executable)}/" - ), cwd=temporary, check=True) - # Find the name of the binary called in the desktop file. binaryname = '' with open( @@ -609,10 +584,10 @@ class Build(): binaryname = line.split('=')[-1].split(' ')[0] # Esci al primo match break - + #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') #binaryname = binary_exec.stdout.strip("\n") - + # Creating a soft link so the executable in the desktop file is present bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) @@ -621,29 +596,93 @@ class Build(): r"-exec ln -sf {} ./%s \;" % binaryname ), cwd=bindir, check=True) + def __missing_dependencies__(self): + """Finds and copy in the appimagedir any missing libraries.""" + # If the system permits it, we leverage lddcollect + # to find the packages that contain .so dependencies in the main build. + import lddcollect + # We first process the ELF + raw = lddcollect.process_elf(self.main_executable, verbose = False, dpkg = True) + + # If all works as expected, we obtain a tuple of: + # (debian_packages, all_libraries, files_not_found) + (debian_packages, all_libraries, not_found) = raw + + if len(debian_packages) != 0: + # Creating temporary folders + debs = [ x.split(':')[0] for x in debian_packages ] + downloadpath = os.path.abspath(os.path.join(self.builddir, 'dependencies')) + os.makedirs(downloadpath) + + + if self.verbose: + print("Downloading missing dependencies, please wait.") + + # Let's try to find and install also other libraries + additional = list(dict.fromkeys([ Helpers.lib_to_deb(x) for x in not_found ])) + debs.extend(additional) + + # It seems the download command does not download dependencies of + # the packages. + if self.verbose: + print("Constructing the dependency tree.") + + for deb in debian_packages: + debs.extend(Helpers.deb_dependencies(deb)) + + # Re-cleaning up the dependency tree + debs = list(dict.fromkeys(debs)) + + # We download the missing dependencies leveraging apt + subprocess.run(shlex.split( + r"apt download " + " ".join(debs) + ), cwd=downloadpath, check=True) + + # then we install them inside a temporary path + temporary = os.path.abspath(os.path.join(downloadpath, 'temp')) + os.makedirs(temporary) + subprocess.run(shlex.split( + r"find " + downloadpath + r" -iname \*.deb -exec dpkg -x {} " + temporary + r" \;" + ), cwd=self.builddir, check=True) + + # We are finally copying the .so files in the same path as main_executable + libdirs = [ 'lib/x86_64-linux-gnu', 'usr/lib/x86_64-linux-gnu' ] + for libdir in libdirs: + fulllibdir = os.path.abspath(os.path.join(temporary, libdir)) + subprocess.run(shlex.split( + f"cp -Ra {fulllibdir}/. {os.path.dirname(self.main_executable)}/" + ), cwd=temporary, check=True) + + if self.debug: + with open(os.path.abspath(os.builddir, 'dependencies.lst'), 'w', encoding="utf-8") as deplist: + deplist.write("\n".join(debs)) + + def __finalize_build__(self): + if self.verbose: + print("Finalizing build...") # Cleaning up AppDir cleanup_dirs = [ 'etc', 'lib', 'lib64', 'usr/lib', 'usr/local' ] for local in cleanup_dirs: shutil.rmtree(os.path.abspath(os.path.join(self.appimagedir, local)), ignore_errors=True) - + # Download AppRun from github apprunurl = r"https://github.com/AppImage/AppImageKit/releases/" apprunurl += f"download/continuous/AppRun-{self.arch}" dest = os.path.join(self.appimagedir, 'AppRun') self.__download__(apprunurl, dest) os.chmod(dest, 0o755) - + # Dealing with extra options buildopts = [] if self.sign: buildopts.append('--sign') - + # adding zsync build if updatable if self.updatable: buildopts.append(f"-u 'zsync|{self.zsyncfilename}'") - + buildopts_str = str.join(' ', buildopts) - + # Build the number-specific build if self.verbose: print("---- Start building ----") @@ -658,17 +697,17 @@ class Build(): f"{self.appimagedir}" ), env={ "VERSION": self.appversion }, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) - + if self.verbose: print(f"Built AppImage version {self.appversion}") - + # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) subprocess.run(shlex.split( r"find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+" ), check=True) - + self.built = True def __del__(self): @@ -678,11 +717,31 @@ class Build(): shutil.rmtree(self.builddir) -def libfinderhelper(libraryname): - """Uses system tools to identify the missing package.""" +class Helpers: - libsearch = subprocess.run(shlex.split( - f"sudo apt-file find -lx {libraryname}$" - ), check=True, capture_output=True) - candidate = [ x for x in libsearch.stdout.decode('utf-8').split('\n') if 'lib' in x ][0] - return candidate + @staticmethod + def deb_dependencies(package_name): + """Returns the array of the dependencies of that package.""" + + # First pass: find dependency of that package in raw output + pass1 = subprocess.Popen(shlex.split( + f"apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances --no-pre-depends {package_name}" + ), stdout=subprocess.PIPE) + + # Second pass: only grep interesting lines. + pass2 = subprocess.Popen(shlex.split( + r"grep '^\w'" + ), stdin=pass1.stdout, stdout=subprocess.PIPE, encoding='utf-8') + stdout, stderr = pass2.communicate() + + return stdout.strip().split("\n") + + @staticmethod + def lib_to_deb(libraryname): + """Uses system tools to identify the missing package.""" + + libsearch = subprocess.run(shlex.split( + f"sudo apt-file find -lx {libraryname}$" + ), check=True, capture_output=True) + candidate = [ x for x in libsearch.stdout.decode('utf-8').split('\n') if 'lib' in x ][0] + return candidate