diff --git a/pyproject.toml b/pyproject.toml index 89488f6..0ee2472 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,10 +15,11 @@ license = "MIT" requires-python = ">= 3.6" dependencies = [ "click", + "lddcollect", "lxml", + "python-magic", "pyyaml", - "requests", - "lddcollect" + "requests" ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/src/loaih/build.py b/src/loaih/build.py index 7bd0535..6461493 100644 --- a/src/loaih/build.py +++ b/src/loaih/build.py @@ -3,6 +3,7 @@ """Classes and functions to build an AppImage.""" import os +import datetime import glob import subprocess import shutil @@ -11,6 +12,7 @@ import shlex import tempfile import hashlib import requests +import magic import loaih @@ -27,6 +29,7 @@ class Collection(list): arch = [ x for x in arch if version.urls[x] != '-' ] self.extend([ Build(version, ar) for ar in arch ]) +class BuildException(Exception): pass class Build(): """Builds a single version.""" @@ -237,7 +240,7 @@ class Build(): # Download the archive try: - self.__download_archive__(archive) + self.__download_archive_debug__(archive) except Exception as error: print(f"Failed to download {archive}: {error}.") @@ -453,9 +456,33 @@ class Build(): with open(f"{file}.md5", 'w', encoding='utf-8') as checkfile: checkfile.write(f"{retval.hexdigest()} {os.path.basename(file)}") - def __download_archive__(self, archive): + def __download_archive__(self, archive) -> str: return self.__download__(self.url, archive) + def __download_archive_debug__(self, archive) -> str: + """Analyses the downloaded archive to prevent issues with unpacking.""" + break_control = 0 + testedfilename = "" + while break_control <= 5: + timenow = datetime.datetime.now() + testedfilename, resource = self.__download_debug__(self.url, archive) + mime = magic.Magic(mime=True) + mimetype = mime.from_file(testedfilename) + if mimetype == 'application/gzip': + return testedfilename + + # On the contrary, we will dump a logfile, remove the download and + # redo the download. + with open(os.path.join(self.downloadpath, 'downloadfailure.log'), 'wa') as logfile: + logfile.write(f"{timenow.isoformat()};{resource.url};{mimetype}") + + file.unlink(testedfilename) + break_control += 1 + + # If it arrives here, 5 attempts to download the archive have failed. + raise BuildException(f"All downloads failed for {archive}. Exiting.") + + def __download__(self, url: str, filename: str): basename = filename if '/' in filename: @@ -473,6 +500,23 @@ class Build(): file.write(chunk) return filename + def __download_debug__(self, url: str, filename: str) -> (str, requests.Response): + basename = filename + if '/' in filename: + basename = filename.split('/')[-1] + + full_url = url + if url.endswith('/'): + # URL has to be completed with basename of filename + full_url = url + basename + + with requests.get(full_url, stream=True, timeout=10) as resource: + resource.raise_for_status() + with open(filename, 'wb') as file: + for chunk in resource.iter_content(chunk_size=8192): + file.write(chunk) + return (filename, resource) + def __select_tarballs__(self): retval = [ self.tarballs[0] ]