Tentative fix for issue #5. Not thoroughly tested, though.

Dockerfile-dev

@@ -0,0 +1,19 @@
+# vim:sts=4:sw=4
+FROM python:3.9-slim-bullseye
+
+RUN mkdir /build && \
+    apt update && apt install -y git && \
+    rm -rf /var/lib/apt/lists/* && \
+    cd /root && \
+    git clone https://git.libreitalia.org/libreitalia/loaih.git && \
+    cd loaih && git checkout dependencies && \
+    python3 -m venv venv && \
+    . venv/bin/activate && \
+    pip install build && \
+    python3 -m build && \
+    pip install dist/loaih*.whl && \
+    deactivate && \
+    ln -sf /root/loaih/venv/bin/loaih /usr/local/bin/loaih
+WORKDIR /build
+ENTRYPOINT [ "/usr/local/bin/loaih" ]
+CMD [ "--help" ]

check_updates.sh

@@ -16,8 +16,8 @@ if [[ ${retval} -ne 0 ]]; then
   fi
   pip3 uninstall -y loaih
   # build the actual toolkit
-  python3 setup.py bdist_wheel
-  pip3 install dist/*.whl; rv=$?
+  python3 -m build -w
+  pip3 install dist/loaih*.whl; rv=$?
   if [[ -d venv ]]; then 
     deactivate
   fi

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "lxml",
     "pyyaml",
     "requests",
+    "lddcollect"
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",

src/loaih/__init__.py

@@ -134,7 +134,7 @@ class Solver():
         retval.query = self.text
         retval.branch = self.branch
         retval.version = self.version
-        if retval.branch != 'daily':
+        if retval.branch != 'daily' and retval.branch != 'prerelease':
             retval.urls['x86_64'] = self.baseurl + 'x86_64/'
 
             try:
@@ -234,6 +234,8 @@ class NamedSolver(Solver):
             fullversion: str = str(archived_versions[-1])
             self.baseurl = ARCHIVE + fullversion + 'deb/'
             self.version = fullversion.rstrip('/')
+            if self.branch == 'prerelease':
+                self.baseurl = PRERELEASE
 
         return self.version
 

src/loaih/build.py

@@ -36,10 +36,12 @@ class Build():
     LANGBASIC = [ 'en-GB' ]
     ARCHSTD = [ 'x86', 'x86_64' ]
 
-    def __init__(self, version: loaih.Version, arch):
+    def __init__(self, version: loaih.Version, arch, debug=False):
+        self.debug = debug
         self.version = version
         self.tidy_folder = True
         self.verbose = True
+        self.check_dependencies = False
         self.arch = arch
         self.short_version = str.join('.', self.version.version.split('.')[0:2])
         self.branch_version = self.version.branch
@@ -88,6 +90,27 @@ class Build():
     def calculate(self):
         """Calculate exclusions and other variables."""
 
+        if self.verbose:
+            print("--- Preliminary Phase ---")
+
+        if isinstance(shutil.which('apt'), str):
+            # APT is found in path. We assume we can find dependencies.
+            self.check_dependencies = True
+
+
+            if self.verbose:
+                print("Updating system packages cache.")
+            # Updating package cache
+            subprocess.run(['sudo', 'apt', 'update'], check=True, stdout=subprocess.DEVNULL)
+            if self.verbose:
+                print("Ensuring apt-file is installed and updated.")
+            # Updating apt-file cache
+            subprocess.run(['sudo', 'apt', 'install', 'apt-file', '-y'], check=True, stdout=subprocess.DEVNULL)
+            subprocess.run(['sudo', 'apt-file', 'update'], check=True, stdout=subprocess.DEVNULL)
+        else:
+            print("CAUTION: your system seems not to include a working version of apt.\nThis will cause the AppImage to leverage system libraries when run.")
+            self.check_dependencies = False
+
         if self.verbose:
             print("--- Calculate Phase ---")
 
@@ -174,7 +197,7 @@ class Build():
                 print(f"Found requested AppImage: {self.appimagefilename}.")
 
 
-    def download(self):
+    def download(self, compact=False):
         """Downloads the contents of the URL as it was a folder."""
 
         if self.verbose:
@@ -194,12 +217,20 @@ class Build():
             self.found = True
 
         # Identifying downloads
-        self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x ]
+        self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x and self.version.version in x ]
+
+        self.download_tarballs = []
+
+        # Issue #5: manage a limited number of downloads and not the full set.
+        if compact:
+            self.download_tarballs = self.__select_tarballs__()
+        else:
+            self.download_tarballs = self.tarballs
 
         # Create and change directory to the download location
         os.makedirs(self.download_path, exist_ok = True)
         os.chdir(self.download_path)
-        for archive in self.tarballs:
+        for archive in self.download_tarballs:
             # If the archive is already there, do not do anything.
             if os.path.exists(archive):
                 continue
@@ -217,12 +248,12 @@ class Build():
     def build(self):
         """Building all the versions."""
 
-        if self.verbose:
-            print("--- Building Phase ---")
-
         if self.found:
             return
 
+        if self.verbose:
+            print("--- Building Phase ---")
+
         # Preparation tasks
         self.appnamedir = os.path.join(self.builddir, self.appname)
         os.makedirs(self.appnamedir, exist_ok=True)
@@ -236,6 +267,13 @@ class Build():
 
         # Build the requested version.
         self.__unpackbuild__()
+        self.__prepare_contents__()
+        if self.check_dependencies:
+            if self.verbose:
+                print("Searching for dependent libraries, it might take a while.")
+
+            self.__missing_dependencies__()
+        self.__finalize_build__()
 
 
     def checksums(self):
@@ -435,40 +473,46 @@ class Build():
                     file.write(chunk)
         return filename
 
-    def __unpackbuild__(self):
-        # We start by filtering out tarballs from the list
-        buildtarballs = [ self.tarballs[0] ]
+    def __select_tarballs__(self):
+        retval = [ self.tarballs[0] ]
 
         # Let's process standard languages and append results to the
         # buildtarball
         if self.language == 'basic':
             if self.offline_help:
-                buildtarballs.extend([ x for x in self.tarballs if 'pack_en-GB' in x ])
+                retval.extend([ x for x in self.tarballs if 'pack_en-GB' in x ])
             else:
-                buildtarballs.extend([ x for x in self.tarballs if 'langpack_en-GB' in x])
+                retval.extend([ x for x in self.tarballs if 'langpack_en-GB' in x])
         elif self.language == 'standard':
             for lang in Build.LANGSTD:
                 if self.offline_help:
-                    buildtarballs.extend([ x for x in self.tarballs if 'pack_' + lang in x ])
+                    retval.extend([ x for x in self.tarballs if 'pack_' + lang in x ])
                 else:
-                    buildtarballs.extend([ x for x in self.tarballs if 'langpack_' + lang in x ])
+                    retval.extend([ x for x in self.tarballs if 'langpack_' + lang in x ])
         elif self.language == 'full':
             if self.offline_help:
                 # We need also all help. Let's replace buildtarball with the
                 # whole bunch
-                buildtarballs = self.tarballs
+                retval = self.tarballs
             else:
-                buildtarballs.extend([ x for x in self.tarballs if 'langpack' in x ])
+                retval.extend([ x for x in self.tarballs if 'langpack' in x ])
         else:
             # Looping for each language in self.language
             for lang in self.language.split(","):
                 if self.offline_help:
-                    buildtarballs.extend([ x for x in self.tarballs
+                    retval.extend([ x for x in self.tarballs
                         if 'pack' + lang in x ])
                 else:
-                    buildtarballs.extend([ x for x in self.tarballs
+                    retval.extend([ x for x in self.tarballs
                         if 'langpack' + lang in x ])
 
+        return retval
+
+
+    def __unpackbuild__(self):
+        # We start by filtering out tarballs from the list
+        buildtarballs = self.__select_tarballs__()
+
         os.chdir(self.appnamedir)
 
         # Unpacking the tarballs
@@ -479,13 +523,17 @@ class Build():
             subprocess.run(shlex.split(
                 f"tar xzf {self.download_path}/{archive}"), check=True)
 
+    def __prepare_contents__(self):
         # create appimagedir
         if self.verbose:
             print("---- Preparing the build ----")
-        self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
+        self.appimagedir = os.path.join(self.appnamedir, self.appname + '.AppDir')
         os.makedirs(self.appimagedir, exist_ok = True)
 
         # At this point, let's decompress the deb packages
+        if self.verbose:
+            print("Unpacking main archives")
+
         subprocess.run(shlex.split(
             r"find .. -iname '*.deb' -exec dpkg -x {} . \;"
         ), cwd=self.appimagedir, check=True)
@@ -498,6 +546,9 @@ class Build():
             ), cwd=self.appimagedir, check=True)
 
         # Changing desktop file
+        if self.verbose:
+            print("Preparing .desktop file.")
+
         subprocess.run(shlex.split(
             r"find . -iname startcenter.desktop -exec cp {} . \;"
         ), cwd=self.appimagedir, check=True)
@@ -507,11 +558,21 @@ class Build():
             r"startcenter.desktop"
         ), cwd=self.appimagedir, check=False)
 
+        if self.verbose:
+            print("Preparing icon file.")
         subprocess.run(shlex.split(
             r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " +
             r"-exec cp {} . \;"
         ), cwd=self.appimagedir, check=True)
 
+        # Finding path to main executable
+        cmd = subprocess.run(shlex.split(
+            r"find -iname soffice.bin -print"
+        ), cwd=self.appimagedir, check = True, capture_output=True)
+        self.main_executable = os.path.abspath(os.path.join(
+            self.appimagedir, 
+            cmd.stdout.strip().decode('utf-8')))
+
         # Find the name of the binary called in the desktop file.
         binaryname = ''
         with open(
@@ -527,13 +588,83 @@ class Build():
         #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
         #binaryname = binary_exec.stdout.strip("\n")
     
+        # Creating a soft link so the executable in the desktop file is present
         bindir=os.path.join(self.appimagedir, 'usr', 'bin')
         os.makedirs(bindir, exist_ok = True)
         subprocess.run(shlex.split(
-            r"find ../../opt -iname soffice -path '*program*' " +
+            r"find ../../opt -iname soffice.bin -path '*program*' " +
             r"-exec ln -sf {} ./%s \;" % binaryname
         ), cwd=bindir, check=True)
 
+    def __missing_dependencies__(self):
+        """Finds and copy in the appimagedir any missing libraries."""
+        # If the system permits it, we leverage lddcollect
+        # to find the packages that contain .so dependencies in the main build.
+        import lddcollect
+        # We first process the ELF
+        raw = lddcollect.process_elf(self.main_executable, verbose = False, dpkg = True)
+
+        # If all works as expected, we obtain a tuple of:
+        # (debian_packages, all_libraries, files_not_found)
+        (debian_packages, all_libraries, not_found) = raw
+
+        if len(debian_packages) != 0:
+            # Creating temporary folders
+            debs = [ x.split(':')[0] for x in debian_packages ]
+            downloadpath = os.path.abspath(os.path.join(self.builddir, 'dependencies'))
+            os.makedirs(downloadpath)
+            
+
+            if self.verbose:
+                print("Downloading missing dependencies, please wait.")
+
+            # Let's try to find and install also other libraries
+            additional = list(dict.fromkeys([ Helpers.lib_to_deb(x) for x in not_found ]))
+            debs.extend(additional)
+
+            # It seems the download command does not download dependencies of
+            # the packages.
+            if self.verbose:
+                print("Constructing the dependency tree.")
+
+            for deb in debian_packages:
+                debs.extend(Helpers.deb_dependencies(deb))
+
+            # Re-cleaning up the dependency tree
+            debs = list(dict.fromkeys(debs))
+
+            # We download the missing dependencies leveraging apt
+            subprocess.run(shlex.split(
+                r"apt download " + " ".join(debs)
+            ), cwd=downloadpath, check=True)
+
+            # then we install them inside a temporary path
+            temporary = os.path.abspath(os.path.join(downloadpath, 'temp'))
+            os.makedirs(temporary)
+            subprocess.run(shlex.split(
+                r"find " + downloadpath + r" -iname \*.deb -exec dpkg -x {} " + temporary + r" \;"
+            ), cwd=self.builddir, check=True)
+
+            # We are finally copying the .so files in the same path as main_executable
+            libdirs = [ 'lib/x86_64-linux-gnu', 'usr/lib/x86_64-linux-gnu' ]
+            for libdir in libdirs:
+                fulllibdir = os.path.abspath(os.path.join(temporary, libdir))
+                subprocess.run(shlex.split(
+                    f"cp -Ra {fulllibdir}/. {os.path.dirname(self.main_executable)}/"
+                ), cwd=temporary, check=True)
+
+            if self.debug:
+                with open(os.path.abspath(os.storage_path, 'dependencies.lst'), 'w', encoding="utf-8") as deplist:
+                    deplist.write("\n".join(debs))
+
+    def __finalize_build__(self):
+        if self.verbose:
+            print("Finalizing build...")
+        # Cleaning up AppDir
+        cleanup_dirs = [ 'etc', 'lib', 'lib64', 'usr/lib', 'usr/local' ]
+        for local in cleanup_dirs:
+            shutil.rmtree(os.path.abspath(os.path.join(self.appimagedir, local)), ignore_errors=True)
+    
         # Download AppRun from github
         apprunurl = r"https://github.com/AppImage/AppImageKit/releases/"
         apprunurl += f"download/continuous/AppRun-{self.arch}"
@@ -557,13 +688,13 @@ class Build():
             print("---- Start building ----")
             subprocess.run(shlex.split(
                 f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
-                f"./{self.appname}.AppDir/"
+                f"{self.appimagedir}"
             ), env={ "VERSION": self.appversion }, check=True)
             print("---- End building ----")
         else:
             subprocess.run(shlex.split(
                 f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
-                f"./{self.appname}.AppDir/"
+                f"{self.appimagedir}"
             ), env={ "VERSION": self.appversion }, stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL, check=True)
     
@@ -581,5 +712,36 @@ class Build():
 
     def __del__(self):
         """Destructor"""
+        if not self.debug:
             # Cleaning up build directory
             shutil.rmtree(self.builddir)
+
+
+class Helpers:
+
+    @staticmethod
+    def deb_dependencies(package_name):
+        """Returns the array of the dependencies of that package."""
+
+        # First pass: find dependency of that package in raw output
+        pass1 = subprocess.Popen(shlex.split(
+            f"apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances --no-pre-depends {package_name}"
+        ), stdout=subprocess.PIPE)
+
+        # Second pass: only grep interesting lines.
+        pass2 = subprocess.Popen(shlex.split(
+            r"grep '^\w'"
+        ), stdin=pass1.stdout, stdout=subprocess.PIPE, encoding='utf-8')
+        stdout, stderr = pass2.communicate()
+
+        return stdout.strip().split("\n")
+
+    @staticmethod
+    def lib_to_deb(libraryname):
+        """Uses system tools to identify the missing package."""
+
+        libsearch = subprocess.run(shlex.split(
+            f"sudo apt-file find -lx {libraryname}$"
+        ), check=True, capture_output=True)
+        candidate = [ x for x in libsearch.stdout.decode('utf-8').split('\n') if 'lib' in x ][0]
+        return candidate

src/loaih/script.py

@@ -59,8 +59,9 @@ def getversion(query, jsonout, default_to_current):
 @click.option('--updatable', '-u', is_flag=True, default=False, help="Create an updatable AppImage (compatible with zsync2). Default: not updatable")
 @click.option('--download-path', '-d', default='./downloads', type=str, help="Path to the download folder. Default: ./downloads")
 @click.option('--repo-path', '-r', default='.', type=str, help="Path to the final storage of the AppImage. Default: current directory")
+@click.option('--debug', 'debug', is_flag=True, default=False, help="Activate debug options.")
 @click.argument('query')
-def build(arch, language, offline, portable, updatable, download_path, repo_path, check, checksums, sign, keep, query):
+def build(arch, language, offline, portable, updatable, download_path, repo_path, check, checksums, sign, keep, query, debug):
     """Builds an Appimage with the provided options."""
 
     # Multiple query support
@@ -89,6 +90,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
     for myquery in queries:
         for appbuild in loaih.build.Collection(myquery, arches):
             # Configuration phase
+            appbuild.debug = debug
             appbuild.tidy_folder = False
             appbuild.language = language
             appbuild.offline_help = offline
@@ -104,7 +106,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
             if check:
                 appbuild.check()
 
-            appbuild.download()
+            appbuild.download(compact = True)
             appbuild.build()
             if checksums:
                 appbuild.checksums()

src/loaih/version.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python3
 # encoding: utf-8
 
-version = "1.3.3"
+version = "1.4.0rc0"