Implementing partial downloads. Implementing dependency checks.

Dockerfile-dev

@@ -0,0 +1,19 @@
+# vim:sts=4:sw=4
+FROM python:3.9-slim-bullseye
+
+RUN mkdir /build && \
+    apt update && apt install -y git && \
+    rm -rf /var/lib/apt/lists/* && \
+    cd /root && \
+    git clone https://git.libreitalia.org/libreitalia/loaih.git && \
+    cd loaih && \
+    python3 -m venv venv && \
+    . venv/bin/activate && \
+    pip install build && \
+    python3 -m build && \
+    pip install dist/loaih*.whl && \
+    deactivate && \
+    ln -sf /root/loaih/venv/bin/loaih /usr/local/bin/loaih
+WORKDIR /build
+ENTRYPOINT [ "/usr/local/bin/loaih" ]
+CMD [ "--help" ]

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "lxml",
     "pyyaml",
     "requests",
+    "lddcollect"
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",

src/loaih/build.py

@@ -40,6 +40,7 @@ class Build():
         self.version = version
         self.tidy_folder = True
         self.verbose = True
+        self.check_dependencies = False
         self.arch = arch
         self.short_version = str.join('.', self.version.version.split('.')[0:2])
         self.branch_version = self.version.branch
@@ -88,6 +89,16 @@ class Build():
     def calculate(self):
         """Calculate exclusions and other variables."""
 
+        if self.verbose:
+            print("--- Preliminary system checks ---")
+
+        if isinstance(shutil.which('apt'), str):
+            # APT is found in path. We assume we can find dependencies.
+            self.check_dependencies = True
+        else:
+            print("CAUTION: your system seems not to include a working version of apt.\nThis will cause the AppImage to leverage system libraries when run.")
+            self.check_dependencies = False
+
         if self.verbose:
             print("--- Calculate Phase ---")
 
@@ -174,7 +185,7 @@ class Build():
                 print(f"Found requested AppImage: {self.appimagefilename}.")
 
 
-    def download(self):
+    def download(self, compact = False):
         """Downloads the contents of the URL as it was a folder."""
 
         if self.verbose:
@@ -196,6 +207,15 @@ class Build():
         # Identifying downloads
         self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x ]
 
+        self.download_tarballs = []
+        self.download_tarballs.extend(self.tarballs[0])
+
+        # Issue #5: manage a limited number of downloads and not the full set.
+        if compact:
+            self.download_tarballs = self.__select_tarballs__()
+        else:
+            self.download_tarballs = self.tarballs
+
         # Create and change directory to the download location
         os.makedirs(self.download_path, exist_ok = True)
         os.chdir(self.download_path)
@@ -435,40 +455,46 @@ class Build():
                     file.write(chunk)
         return filename
 
-    def __unpackbuild__(self):
+    def __select_tarballs__(self):
-        # We start by filtering out tarballs from the list
+        retval = [ self.tarballs[0] ]
-        buildtarballs = [ self.tarballs[0] ]
 
         # Let's process standard languages and append results to the
         # buildtarball
         if self.language == 'basic':
             if self.offline_help:
-                buildtarballs.extend([ x for x in self.tarballs if 'pack_en-GB' in x ])
+                retval.extend([ x for x in self.tarballs if 'pack_en-GB' in x ])
             else:
-                buildtarballs.extend([ x for x in self.tarballs if 'langpack_en-GB' in x])
+                retval.extend([ x for x in self.tarballs if 'langpack_en-GB' in x])
         elif self.language == 'standard':
             for lang in Build.LANGSTD:
                 if self.offline_help:
-                    buildtarballs.extend([ x for x in self.tarballs if 'pack_' + lang in x ])
+                    retval.extend([ x for x in self.tarballs if 'pack_' + lang in x ])
                 else:
-                    buildtarballs.extend([ x for x in self.tarballs if 'langpack_' + lang in x ])
+                    retval.extend([ x for x in self.tarballs if 'langpack_' + lang in x ])
         elif self.language == 'full':
             if self.offline_help:
                 # We need also all help. Let's replace buildtarball with the
                 # whole bunch
-                buildtarballs = self.tarballs
+                retval = self.tarballs
             else:
-                buildtarballs.extend([ x for x in self.tarballs if 'langpack' in x ])
+                retval.extend([ x for x in self.tarballs if 'langpack' in x ])
         else:
             # Looping for each language in self.language
             for lang in self.language.split(","):
                 if self.offline_help:
-                    buildtarballs.extend([ x for x in self.tarballs
+                    retval.extend([ x for x in self.tarballs
                         if 'pack' + lang in x ])
                 else:
-                    buildtarballs.extend([ x for x in self.tarballs
+                    retval.extend([ x for x in self.tarballs
                         if 'langpack' + lang in x ])
 
+        return retval
+
+
+    def __unpackbuild__(self):
+        # We start by filtering out tarballs from the list
+        buildtarballs = self.__select_tarballs__()
+
         os.chdir(self.appnamedir)
 
         # Unpacking the tarballs
@@ -482,7 +508,7 @@ class Build():
         # create appimagedir
         if self.verbose:
             print("---- Preparing the build ----")
-        self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
+        self.appimagedir = os.path.join(self.appnamedir, self.appname + '.AppDir')
         os.makedirs(self.appimagedir, exist_ok = True)
 
         # At this point, let's decompress the deb packages
@@ -512,6 +538,46 @@ class Build():
             r"-exec cp {} . \;"
         ), cwd=self.appimagedir, check=True)
 
+        # Finding path to main executable
+        cmd = subprocess.run(shlex.split(
+            r"find -iname soffice.bin -print"
+        ), cwd=self.appimagedir, check = True, capture_output=True)
+        main_executable = os.path.abspath(os.path.join(
+            self.appimagedir, 
+            cmd.stdout.strip().decode('utf-8')))
+
+        # If the system permits it, we leverage lddcollect
+        # to find the packages that contain .so dependencies in the main build.
+        if self.check_dependencies:
+            if self.verbose:
+                print("Checking for dependent libraries")
+
+            import lddcollect
+            # We first process the ELF
+            raw = lddcollect.process_elf(main_executable, verbose = False, dpkg = True)
+
+            # If all works as expected, we obtain a tuple of:
+            # (debian_packages, all_libraries, files_not_found)
+            (debian_packages, all_libraries, not_found) = raw
+
+            if len(debian_packages) != 0:
+                # We need, first, to download those packages.
+                debs = [ x.split(":")[0] for x in debian_packages ]
+                downloadpath = os.path.abspath(os.path.join(self.builddir, 'dependencies'))
+                
+                if self.verbose:
+                    print("Downloading missing dependencies, please wait.")
+
+                # We download the missing dependencies leveraging apt
+                subprocess.run(shlex.split(
+                    r"apt download " + debs.join(' ')),
+                    cwd=downloadpath, check=True)
+
+                # then we install them inside the AppDir
+                subprocess.run(shlex.split(
+                    r"find " + downloadpath + r" -iname \*.deb -exec dpkg -x {} " + self.appimagedir + r" \;"
+                ), cwd=self.builddir, check=True)
+
         # Find the name of the binary called in the desktop file.
         binaryname = ''
         with open(
@@ -527,10 +593,11 @@ class Build():
         #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
         #binaryname = binary_exec.stdout.strip("\n")
 
+        # Creating a soft link so the executable in the desktop file is present
         bindir=os.path.join(self.appimagedir, 'usr', 'bin')
         os.makedirs(bindir, exist_ok = True)
         subprocess.run(shlex.split(
-            r"find ../../opt -iname soffice -path '*program*' " +
+            r"find ../../opt -iname soffice.bin -path '*program*' " +
             r"-exec ln -sf {} ./%s \;" % binaryname
         ), cwd=bindir, check=True)
 
@@ -557,13 +624,13 @@ class Build():
             print("---- Start building ----")
             subprocess.run(shlex.split(
                 f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
-                f"./{self.appname}.AppDir/"
+                f"{self.appimagedir}"
             ), env={ "VERSION": self.appversion }, check=True)
             print("---- End building ----")
         else:
             subprocess.run(shlex.split(
                 f"{self.appnamedir}/appimagetool {buildopts_str} -v " +
-                f"./{self.appname}.AppDir/"
+                f"{self.appimagedir}"
             ), env={ "VERSION": self.appversion }, stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL, check=True)
 

src/loaih/script.py

@@ -104,7 +104,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
             if check:
                 appbuild.check()
 
-            appbuild.download()
+            appbuild.download(compact = True)
             appbuild.build()
             if checksums:
                 appbuild.checksums()