Cleaning up downloads directory before Daily and Prerelease, but leave downloaded versions in place already.

loaih/__init__.py

@@ -4,10 +4,11 @@
 
 import datetime
 import json
-from re import I
+import re
 import requests
+import subprocess
+import shlex
 from lxml import html
-from packaging.version import parse as parse_version
 
 # Constants
 DOWNLOADPAGE = "https://www.libreoffice.org/download/download/"
@@ -57,6 +58,24 @@ class Version():
             'x86_64': '-'
         }
 
+    def appname(self):
+        """Determines the app name based on the query branch determined."""
+        datematch = re.match(r'[0-9]{8}', self.query)
+        retval = 'LibreOffice'
+        if self.query in {'prerelease', 'daily', 'current', 'yesterday'} or datematch:
+            retval = 'LibreOfficeDev'
+
+        return retval
+
+    def cleanup_downloads(self, path, verbose=False) -> None:
+        """Cleanups the downloads folder to assure new versions are built."""
+        search_name = self.appname() + '_' + self.version
+        cmd = f"find {path} -iname {search_name}\\*.tar.gz -delete"
+        if verbose:
+            subprocess.run(shlex.split(cmd), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        else:
+            subprocess.run(shlex.split(cmd))
+
     def to_dict(self):
         """Returns a dictionary of versions."""
         return {

loaih/build.py

@@ -60,7 +60,7 @@ class Build():
         self.appnamedir = ''
 
         # Specific build version
-        self.appname = 'LibreOffice'
+        self.appname = self.version.appname()
         self.appversion = ''
         self.appimagedir = ''
         self.appimagefilename = ''
@@ -102,10 +102,6 @@ class Build():
             if self.verbose:
                 print("Repo is local.")
 
-        # AppName
-        if self.branch_version in { 'prerelease', 'daily' }:
-            self.appname = 'LibreOfficeDev'
-
         # Calculating languagepart
         self.languagepart = "."
         if ',' in self.language:
@@ -196,17 +192,14 @@ class Build():
             self.found = True
 
         # Identifying downloads
-        self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()")
+        self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x ]
-            if x.endswith('tar.gz') and 'deb' in x ]
 
         # Create and change directory to the download location
         os.makedirs(self.download_path, exist_ok = True)
         os.chdir(self.download_path)
         for archive in self.tarballs:
             # If the archive is already there, do not do anything.
-            # If it is a daily build or a pre-release, due to filename
+            if os.path.exists(archive):
-            # clashes, redownload the whole build.
-            if os.path.exists(archive) and self.version.query not in { 'daily', 'prerelease' }:
                 continue
 
             # Download the archive

loaih/script.py

@@ -8,6 +8,7 @@ import sys
 import json
 import click
 import yaml
+from dotmap import DotMap
 import loaih
 import loaih.build
 
@@ -57,15 +58,47 @@ def batch(yamlfile, verbose):
 
     # This is a buildfile. So we have to load the file and pass the build options ourselves.
     config = {}
-    with open(os.path.abspath(yamlfile), 'r', encoding= 'utf-8') as file:
+    with open(os.path.abspath(yamlfile), 'r', encoding='utf-8') as file:
         config = yaml.safe_load(file)
 
+    # Globals for yamlfile
+    gvars = DotMap()
+    gvars.storage_path = "/srv/http/appimage.sys42.eu"
+    if 'repo' in config['data'] and config['data']['repo']:
+        gvars.storage_path = config['data']['repo']
+
+    gvars.download_path = "/var/tmp/downloads"
+    if 'download' in config['data'] and config['data']['download']:
+        gvars.download_path = config['data']['download']
+
+    if 'http' in gvars.storage_path:
+        gvars.remoterepo = True
+        gvars.remote_host = "ciccio.libreitalia.org"
+        if 'remote_host' in config['data'] and config['data']['remote_host']:
+            gvars.remote_host = config['data']['remote_host']
+
+        gvars.remote_path = "/var/lib/nethserver/vhost/appimages"
+        if 'remote_path' in config['data'] and config['data']['remote_path']:
+            gvars.remote_path = config['data']['remote_path']
+
+    if 'sign' in config['data'] and config['data']['sign']:
+        gvars.sign = True
+
     # With the config file, we ignore all the command line options and set
     # generic default.
     for cbuild in config['builds']:
         # Loop a run for each build.
         collection = loaih.build.Collection(cbuild['query'])
 
+        # Generic run for the query
+        generic = loaih.Solver.parse(cbuild['query'])
+
+        # In case prerelease or daily branches are used, cleanup the download
+        # folder before first run (to make sure the contents are effectively
+        # fresh).
+        if generic.branch in {'prerelease', 'daily'}:
+            generic.cleanup_downloads(gvars.download_path, verbose)
+
         for obj in collection:
             # Configuration phase
             obj.verbose = verbose
@@ -73,23 +106,12 @@ def batch(yamlfile, verbose):
             obj.offline_help = cbuild['offline_help']
             obj.portable = cbuild['portable']
             obj.updatable = True
-            obj.storage_path = "/srv/http/appimage.sys42.eu"
+            obj.storage_path = gvars.storage_path
-            if 'repo' in config['data'] and config['data']['repo']:
+            obj.download_path = gvars.download_path
-                obj.storage_path = config['data']['repo']
+            obj.remoterepo = gvars.remoterepo
-            obj.download_path = "/var/tmp/downloads"
+            obj.remote_host = gvars.remote_host
-            if 'download' in config['data'] and config['data']['download']:
+            obj.remote_path = gvars.remote_path
-                obj.download_path = config['data']['download']
+            obj.sign = gvars.sign
-            if 'http' in obj.storage_path:
-                obj.remoterepo = True
-                obj.remote_host = "ciccio.libreitalia.org"
-                if 'remote_host' in config['data'] and config['data']['remote_host']:
-                    obj.remote_host = config['data']['remote_host']
-                obj.remote_path = "/var/lib/nethserver/vhost/appimages"
-                if 'remote_path' in config['data'] and config['data']['remote_path']:
-                    obj.remote_path = config['data']['remote_path']
-
-            if 'sign' in config['data'] and config['data']['sign']:
-                obj.sign = True
 
             # Build phase
             obj.calculate()