From 3dbfef9fbef3da22a6de017393004e67ec686050 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 9 Dec 2023 23:26:03 +0100 Subject: [PATCH] Cleaning up downloads directory before Daily and Prerelease, but leave downloaded versions in place already. --- loaih/__init__.py | 23 +++++++++++++++++-- loaih/build.py | 13 +++-------- loaih/script.py | 58 ++++++++++++++++++++++++++++++++--------------- 3 files changed, 64 insertions(+), 30 deletions(-) diff --git a/loaih/__init__.py b/loaih/__init__.py index b272313..7b733b0 100644 --- a/loaih/__init__.py +++ b/loaih/__init__.py @@ -4,10 +4,11 @@ import datetime import json -from re import I +import re import requests +import subprocess +import shlex from lxml import html -from packaging.version import parse as parse_version # Constants DOWNLOADPAGE = "https://www.libreoffice.org/download/download/" @@ -57,6 +58,24 @@ class Version(): 'x86_64': '-' } + def appname(self): + """Determines the app name based on the query branch determined.""" + datematch = re.match(r'[0-9]{8}', self.query) + retval = 'LibreOffice' + if self.query in {'prerelease', 'daily', 'current', 'yesterday'} or datematch: + retval = 'LibreOfficeDev' + + return retval + + def cleanup_downloads(self, path, verbose=False) -> None: + """Cleanups the downloads folder to assure new versions are built.""" + search_name = self.appname() + '_' + self.version + cmd = f"find {path} -iname {search_name}\\*.tar.gz -delete" + if verbose: + subprocess.run(shlex.split(cmd), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + subprocess.run(shlex.split(cmd)) + def to_dict(self): """Returns a dictionary of versions.""" return { diff --git a/loaih/build.py b/loaih/build.py index 6732d72..da62458 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -60,7 +60,7 @@ class Build(): self.appnamedir = '' # Specific build version - self.appname = 'LibreOffice' + self.appname = self.version.appname() self.appversion = '' self.appimagedir = '' self.appimagefilename = '' @@ -102,10 +102,6 @@ class Build(): if self.verbose: print("Repo is local.") - # AppName - if self.branch_version in { 'prerelease', 'daily' }: - self.appname = 'LibreOfficeDev' - # Calculating languagepart self.languagepart = "." if ',' in self.language: @@ -196,17 +192,14 @@ class Build(): self.found = True # Identifying downloads - self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") - if x.endswith('tar.gz') and 'deb' in x ] + self.tarballs = [ x for x in loaih.match_xpath(self.url, "//td/a/text()") if x.endswith('tar.gz') and 'deb' in x ] # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) for archive in self.tarballs: # If the archive is already there, do not do anything. - # If it is a daily build or a pre-release, due to filename - # clashes, redownload the whole build. - if os.path.exists(archive) and self.version.query not in { 'daily', 'prerelease' }: + if os.path.exists(archive): continue # Download the archive diff --git a/loaih/script.py b/loaih/script.py index 49d8011..6b878e6 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -8,6 +8,7 @@ import sys import json import click import yaml +from dotmap import DotMap import loaih import loaih.build @@ -57,15 +58,47 @@ def batch(yamlfile, verbose): # This is a buildfile. So we have to load the file and pass the build options ourselves. config = {} - with open(os.path.abspath(yamlfile), 'r', encoding= 'utf-8') as file: + with open(os.path.abspath(yamlfile), 'r', encoding='utf-8') as file: config = yaml.safe_load(file) + # Globals for yamlfile + gvars = DotMap() + gvars.storage_path = "/srv/http/appimage.sys42.eu" + if 'repo' in config['data'] and config['data']['repo']: + gvars.storage_path = config['data']['repo'] + + gvars.download_path = "/var/tmp/downloads" + if 'download' in config['data'] and config['data']['download']: + gvars.download_path = config['data']['download'] + + if 'http' in gvars.storage_path: + gvars.remoterepo = True + gvars.remote_host = "ciccio.libreitalia.org" + if 'remote_host' in config['data'] and config['data']['remote_host']: + gvars.remote_host = config['data']['remote_host'] + + gvars.remote_path = "/var/lib/nethserver/vhost/appimages" + if 'remote_path' in config['data'] and config['data']['remote_path']: + gvars.remote_path = config['data']['remote_path'] + + if 'sign' in config['data'] and config['data']['sign']: + gvars.sign = True + # With the config file, we ignore all the command line options and set # generic default. for cbuild in config['builds']: # Loop a run for each build. collection = loaih.build.Collection(cbuild['query']) + # Generic run for the query + generic = loaih.Solver.parse(cbuild['query']) + + # In case prerelease or daily branches are used, cleanup the download + # folder before first run (to make sure the contents are effectively + # fresh). + if generic.branch in {'prerelease', 'daily'}: + generic.cleanup_downloads(gvars.download_path, verbose) + for obj in collection: # Configuration phase obj.verbose = verbose @@ -73,23 +106,12 @@ def batch(yamlfile, verbose): obj.offline_help = cbuild['offline_help'] obj.portable = cbuild['portable'] obj.updatable = True - obj.storage_path = "/srv/http/appimage.sys42.eu" - if 'repo' in config['data'] and config['data']['repo']: - obj.storage_path = config['data']['repo'] - obj.download_path = "/var/tmp/downloads" - if 'download' in config['data'] and config['data']['download']: - obj.download_path = config['data']['download'] - if 'http' in obj.storage_path: - obj.remoterepo = True - obj.remote_host = "ciccio.libreitalia.org" - if 'remote_host' in config['data'] and config['data']['remote_host']: - obj.remote_host = config['data']['remote_host'] - obj.remote_path = "/var/lib/nethserver/vhost/appimages" - if 'remote_path' in config['data'] and config['data']['remote_path']: - obj.remote_path = config['data']['remote_path'] - - if 'sign' in config['data'] and config['data']['sign']: - obj.sign = True + obj.storage_path = gvars.storage_path + obj.download_path = gvars.download_path + obj.remoterepo = gvars.remoterepo + obj.remote_host = gvars.remote_host + obj.remote_path = gvars.remote_path + obj.sign = gvars.sign # Build phase obj.calculate()