From df5012eedd1ca3ac79ae2c35f95125c535880af6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:07:12 +0100 Subject: [PATCH] Prima implementazione controllo build remota e caricamento con rsync + ssh. --- loaih/build.py | 80 +++++++++++++++++++++++++++++++++++++------------ loaih/script.py | 9 +++++- 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index bbd0631..a7a2a04 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -32,6 +32,9 @@ class Build(loaih.RemoteBuild): self.portable = False self.updatable = True self.sign = True + self.remoterepo = False + self.remote_host = '' + self.remote_path = '' self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' @@ -90,7 +93,7 @@ class Build(loaih.RemoteBuild): if self.query == 'daily': self.relative_path.append('daily') elif self.query == 'prerelease': - self.relative_path.append('prerelease') + self.relative_path.append('prerelease') # Not the same check, an additional one if self.portable: @@ -105,25 +108,44 @@ class Build(loaih.RemoteBuild): def check(self): """Checking if the requested AppImage has been already built.""" - if not len(self.appimagefilename) == 2: + if len(self.appimagefilename) != 2: self.calculate() for arch in self.arch: - print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') - if "No such file or directory" in res.stderr: - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue + # First, check if by metadata the repo is remote or not. + if self.remoterepo or 'http' in self.storage_path: + self.remoterepo = True + # Remote storage. I have to query a remote site to know if it + # was already built. + name = self.appimagefilename[arch] + matching = etree.HTML(urllib.request.urlopen(str.join('/', + self.relative_path.insert(0, self.storage_path) + )).read()).xpath( + f"//a[contains(@href, '{name}')/@href" + ) - if res.stdout and len(res.stdout.strip("\n")) > 0: - # All good, the command was executed fine. - print(f"Build for {self.version} found.") - self.built[arch] = True + if len(matching) > 0: + # Already built. + self.built[arch] = True + + else: + # Repo is local + print(f"Searching for {self.appimagefilename[arch]}") + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + + if "No such file or directory" in res.stderr: + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout and len(res.stdout.strip("\n")) > 0: + # All good, the command was executed fine. + print(f"Build for {self.version} found.") + self.built[arch] = True if self.built[arch]: - print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") def download(self): @@ -309,13 +331,33 @@ class Build(loaih.RemoteBuild): return os.chdir(self.appnamedir) - # Forcing creation of subfolders, in case there is a new build - os.makedirs(self.full_path, exist_ok = True) - for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + + # Two cases here: local and remote storage_path. + if self.remoterepo: + # Remote first. + # Build destination directory + if len(self.relative_path) > 0: + remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) + else: + remotepath = str.join('/', [ self.remote_path, '' ]) + try: + subprocess.run( + shlex.split( + f"rsync -avz -e ssh *.AppImage* {self.remote_host}:{remotepath}" + ) + ) + finally: + pass + + else: + # Local + # Forcing creation of subfolders, in case there is a new build + os.makedirs(self.full_path, exist_ok = True) + for file in glob.glob("*.AppImage*"): + subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - def generalize_and_link(self): + def generalize_and_link(self, chdir = self.full_path): """Creates the needed generalized files if needed.""" # If called with a pointed version, no generalize and link necessary. if not self.branch_version: @@ -335,7 +377,7 @@ class Build(loaih.RemoteBuild): if self.built[arch]: continue - os.chdir(self.full_path) + os.chdir(chdir) # if the appimage for the reported arch is not found, skip to next # arch if not os.path.exists(self.appimagefilename[arch]): diff --git a/loaih/script.py b/loaih/script.py index 40ddf36..46afbe0 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -71,6 +71,10 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.updatable = True obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + if 'http' in obj.storage_path: + obj.remoterepo = True + obj.remote_host = config['data']['remote_host'] if 'remote_host' in config['data'] and config['data']['remote_host'] else 'ciccio.libreitalia.org' + obj.remote_path = config['data']['remote_path'] if 'remote_path' in config['data'] and config['data']['remote_path'] else '/var/lib/nethserver/vhost/appimages' if 'sign' in config['data'] and config['data']['sign']: obj.sign = True @@ -83,8 +87,11 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.download() obj.build() obj.checksums() + if obj.remoterepo: + obj.generalize_and_link(obj.appnamedir) obj.publish() - obj.generalize_and_link() + if not obj.remoterepo: + obj.generalize_and_link() del obj else: