First steps to refactoring the code and giving it a clear structure.

loaih/__init__.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python
 # encoding: utf-8
-"""Machinery for compiling new versions of AppImages."""
+"""machinery for compiling new versions of appimages."""
 
-import urllib.request
 import datetime
-import re
+import json
-from lxml import etree
+import loaih.solvers
 from packaging.version import parse as parse_version
 
-
 class Definitions(): # pylint: disable=too-few-public-methods
     """Definitions for the module."""
 
@@ -37,13 +35,74 @@ class Definitions(): # pylint: disable=too-few-public-methods
         }
     }
 
+
+class Version():
+    """Represent the skeleton of each queried version."""
+
+    def __init__(self):
+        self.query = ''
+        self.version = ''
+        self.urls = {
+            'x86': '-',
+            'x86_64': '-'
+        }
+
+    def to_dict(self):
+        """Returns a dictionary of versions."""
+        return {
+            'query': self.query,
+            'version': self.version,
+            'basedirurl': self.urls
+        }
+
+    def to_json(self):
+        """Returns a json representation of the version."""
+        return json.dumps(self.to_dict())
+
+    def __str__(self):
+        return f"""query: {self.query}
+version: {self.version}
+x86: {self.urls['x86']}
+x86_64: {self.urls['x86_64']}"""
+
+class QueryError(Exception): pass
+
+class Query():
+    """Represents each query and helps determining the aspects of a version."""
+
+    def __init__(self, query: str, default_to_current = False):
+        self.text = query
+        self.type = ''
+        self.default_to_current = default_to_current
+
+        # Let's first determine which type of query we are doing.
+        if self.text in { 'current', 'yesterday', 'daily' }:
+            self.type = 'daily'
+        elif self.text in { 'still', 'fresh', 'prerelease' }:
+            self.type = 'named'
+        elif '.' in self.text:
+            self.type = 'exact_version'
+        else:
+            try:
+                int(query)
+            except ValueError:
+                raise QueryError("The queried version does not exist.")
+
+        # Since the query is number only, let's let it leave with the
+        # number, but set the type to 'daily'
+        self.type = 'daily'
+
+        # call Base.parse_query to popolate the results
+        self.results = loaih.solvers.Solver.parse_query(self)
+
+
 class Base():
     """Contains methods that might be useful outside class."""
     # Class for static methods which might be useful even outside the build
     # scripts.
 
     @staticmethod
-    def dailyurl(date = datetime.datetime.today()):
+    def dailyurl(date = datetime.datetime.today(), default_current=False):
         """Returns the URL for the latest valid daily build."""
         # As per other parts of the build, we need to maintain an URL also for
         # x86 versions that it isn't really provided.
@@ -66,18 +125,24 @@ class Base():
         results = raw_page.xpath(
             f"""//td/a[contains(text(), "{date.strftime('%Y-%m-%d')}")]/text()"""
         )
-        if len(results) == 0:
+        if len(results) > 0:
-            # No results found, no version found, let's return a
+            # On the contrary, more than a version is found. let's order the
+            # list and get the latest item
+            return { 'x86': '-', 'x86_64': f"{daily_selector}{sorted(results)[-1]}" }
+        # No results found, no version found.
+        # But if default_current is true, redo the search with 'current'
+        if not default_current:
             return { 'x86': '-', 'x86_64': '-' }
 
-        # On the contrary, more than a version is found. let's order the
+        # default_current is true - redo all the queries for current
-        # list and get the latest item
+        current_link = raw_page.xpath("//td/a[contains(text(), 'current']/text()")[0]
-        return { 'x86': '-', 'x86_64': f"{daily_selector}{sorted(results)[-1]}" }
+        return {'x86': '-', 'x86_64': f"{daily_selector}{current_link}" }
+
 
     @staticmethod
-    def dailyver(date = datetime.datetime.today()):
+    def dailyver(date = datetime.datetime.today(), force_current = False):
         """Returns versions present on the latest daily build."""
-        url = Base.dailyurl(date)['x86_64']
+        url = Base.dailyurl(date, force_current)['x86_64']
         # If no daily releases has been provided yet, return empty
         if url == '-':
             return []
@@ -88,33 +153,38 @@ class Base():
         tarball = etree.HTML(fullpage).xpath(archive_path)#pylint: disable=c-extension-no-member
         # This should have returned the main package for a version, but can
         # have returned multiple ones, so let's treat it as a list
-        return [ x.split('_')[1] for x in tarball ]
+        tarball_versions = [ x.split('_')[1] for x in tarball ]
+        if len(tarball_versions) == 1:
+            return tarball_versions[0]
 
     @staticmethod
     def namedver(query):
         """Gets the version for a specific named version."""
 
+        if 'current' in query:
+            # Should return the daily version with the current link
+            return Base.dailyver(datetime.datetime.today(), True)
+
         if 'yesterday' in query:
             return Base.dailyver(datetime.datetime.now() + datetime.timedelta(days=-1))
 
-        if query.startswith('daily'):
+        if 'daily' in query:
-            print("daily")
-            if '-' in query or '_'  in query:
-                # Splitting the string
-                stringdate = re.sub(r'daily[-_:]', '', query)
-
-                print(f"Stringdate: {stringdate}")
-                print(f"strptime: {datetime.datetime.strptime(stringdate, '%Y%m%d')}")
-                return Base.dailyver(datetime.datetime.strptime(stringdate, "%Y%m%d"))
-
             return Base.dailyver()
 
-        # In case the query isn't for daily
+        # If the name is convertible to integer, it means it is written as
-        return etree.HTML(#pylint: disable=c-extension-no-member
+        # <year><month><date> and we ask for a daily build.
-            urllib.request.urlopen(
+        try:
-                Definitions.SELECTORS[query]['URL']
+            int(query)
-            ).read()
+        except ValueError:
-        ).xpath(Definitions.SELECTORS[query]['xpath'])
+            # All other options - fresh, prerelease, still
+            return etree.HTML(#pylint: disable=c-extension-no-member
+                urllib.request.urlopen(
+                    Definitions.SELECTORS[query]['URL']
+                ).read()
+            ).xpath(Definitions.SELECTORS[query]['xpath'])
+
+        # Lets restart with integer version
+        return Base.dailyver(datetime.datetime.strptime(query, "%Y%m%d"))
 
     @staticmethod
     def fullversion(version):
@@ -165,34 +235,62 @@ class Base():
 
         return retval
 
-    @staticmethod
+    #@staticmethod
-    def collectedbuilds(query):
+    #def collectedbuilds(query, default_current=False):
-        """Creates a list of Builds based on each queried version found."""
+    #    """Creates a list of Builds based on each queried version found."""
-        retval = []
+    #    retval = []
-        if '.' in query:
+    #    if '.' in query:
-            # Called with a numeric query. Pass it to RemoteBuild
+    #        # Called with a numeric query. Pass it to RemoteBuild
-            retval.append(RemoteBuild(query))
+    #        retval.append(RemoteBuild(query))
-        elif '-' in query or '_' in query:
+    #    else:
-            # daily but with date
+    #        try:
-            # The date is the last part.
+    #            int(query)
-            stringdate = re.split(r'[_-]', query)[-1]
+    #        except ValueError:
-            querydate = datetime.datetime.strptime(stringdate, "%Y%m%d").date()
+    #            # Named query
+    #            named_version = Base.namedver(query)
+    #            # named_version should be just one version anyway
+    #            if isinstance(named_version, list) and len(named_version) > 1:
+    #                for ver in named_version:
+    #                    if 'daily' in query:
 
-            dailyver = Base.dailyver(querydate)
+    #            if 'daily' in query and specs == {}:
-        else:
+    #                # No daily build has been found.
-            # Named query
+    #                # Rerun the query but with 'current'
-            dailyver = Base.namedver(query)
+    #                named_version = Base.namedver('current')
 
-            if not dailyver:
+    #            if specs != {}:
-                # a is empty
+    #                if isinstance(specs, list) and len(specs) > 1:
-                return retval
+    #                    for spec in specs:
+    #                        remotebuild = RemoteBuild(query)
+    #                        if 'daily' in query:
+    #                            remotebuild.daily(spec)
+    #                        else:
+    #                            remotebuild.from_version(spec['version'])
 
-            if isinstance(dailyver, list) and len(dailyver) > 1:
+    #                        retval.append(remotebuild)
-                retval.extend([ RemoteBuild(query, version) for version in dailyver ])
-            else:
-                retval.append(RemoteBuild(query))
 
-        return sorted(retval, key=lambda x: x.version)
+    #                else:
+    #                    # Possibly single instance of dict
+    #                    remotebuild = RemoteBuild(query)
+    #                    remotebuild.from_version(specs['version'])
+    #                    retval.append(remotebuild)
+    #            else:
+    #                return retval
+
+    #        # If the query is convertible in integer, we are still asking for a
+    #        # daily build in a specific day
+    #        specs = Base.dailyver(datetime.datetime.strptime(query, '%Y%m%d'))
+    #        if isinstance(specs, list) and len(specs) > 1:
+    #            for spec in specs:
+    #                remotebuild = RemoteBuild('daily')
+    #                remotebuild.daily(spec)
+    #                retval.append(remotebuild)
+    #        else:
+    #            remotebuild = RemoteBuild('daily')
+    #            remotebuild.daily(specs)
+    #            retval.append(remotebuild)
+
+    #    return sorted(retval, key=lambda x: x.version)
 
 
 class RemoteBuild():
@@ -201,9 +299,10 @@ class RemoteBuild():
     def __init__(self, query, version = None):
         """Should simplify the single builded version."""
         self.query = query
-        self.version = ''
+        self.version = version or ''
         self.basedirurl = { 'x86': '-', 'x86_64': '-' }
 
+    def from_version(self, version):
         if version and isinstance(version, str):
             self.version = version
 
@@ -241,16 +340,9 @@ class RemoteBuild():
 
         self.basedirurl = Base.urlfromqueryandver(self.query, self.version)
 
-    def todict(self):
+    def daily(self, dailyspecs):
-        """Returns a dictionary of versions."""
+        """Builds a remote version starting from dailyver dictionary."""
-        return {
+        self.version = dailyspecs['version']
-            'query': self.query,
+        self.basedirurl = dailyspecs['basedirurl']
-            'version': self.version,
+
-            'basedirurl': self.basedirurl
-        }
 
-    def __str__(self):
-        return f"""query: {self.query}
-version: {self.version}
-x86: {self.basedirurl['x86']}
-x86_64: {self.basedirurl['x86_64']}"""

loaih/script.py

@@ -2,7 +2,7 @@
 # encoding: utf-8
 """Helps with command line commands."""
 
-import os
+import os, sys
 import json
 import click
 import yaml
@@ -14,13 +14,12 @@ def cli():
     """Helps with command line commands."""
 
 @cli.command()
-@click.option('-j', '--json', 'jsonout', default=False, is_flag=True,
+@click.option('-j', '--json', 'jsonout', default=False, is_flag=True, help="Output format in json.")
-    help="Output format in json.")
 @click.argument('query')
 def getversion(query, jsonout):
-    """Get the numeral version from a named version."""
+    """Get download information for named or numbered versions."""
 
-    batch = []
+    batchlist = []
     queries = []
     if ',' in query:
         queries.extend(query.split(','))
@@ -28,21 +27,85 @@ def getversion(query, jsonout):
         queries.append(query)
 
     for singlequery in queries:
-        batch.extend(loaih.Base.collectedbuilds(singlequery))
+        elem = loaih.RemoteBuild(singlequery)
+        batchlist.extend(loaih.Base.collectedbuilds(singlequery))
 
-    if len(batch) > 0:
+    if len(batchlist) > 0:
         if jsonout:
-            click.echo(json.dumps([x.todict() for x in batch]))
+            click.echo(json.dumps([x.todict() for x in batchlist]))
         else:
-            for value in batch:
+            for value in batchlist:
                 click.echo(value)
 
+@cli.command()
+@click.option("--verbose", '-v', is_flag=True, default=False, help="Show building phases.", show_default=True)
+@click.argument("yamlfile")
+def batch(yamlfile, verbose):
+    """Builds a collection of AppImages based on YAML file."""
+    # Defaults for a batch building is definitely more different than a manual
+    # one. To reflect this behaviour, I decided to split the commands between
+    # batch (bulk creation) and build (manual building).
+
+    # Check if yamlfile exists.
+    if not os.path.exists(os.path.abspath(yamlfile)):
+        click.echo(f"YAML file {yamlfile} does not exists or is unreadable.")
+        sys.exit(1)
+
+    # This is a buildfile. So we have to load the file and pass the build options ourselves.
+    config = {}
+    with open(os.path.abspath(yamlfile), 'r', encoding= 'utf-8') as file:
+        config = yaml.safe_load(file)
+
+    # With the config file, we ignore all the command line options and set
+    # generic default.
+    for cbuild in config['builds']:
+        # Loop a run for each build.
+        collection = loaih.build.Collection(cbuild['query'])
+
+        for obj in collection:
+            # Configuration phase
+            obj.language = cbuild['language']
+            obj.offline_help = cbuild['offline_help']
+            obj.portable = cbuild['portable']
+            obj.updatable = True
+            obj.storage_path = "/srv/http/appimage.sys42.eu"
+            if 'repo' in config['data'] and config['data']['repo']:
+                obj.storage_path = config['data']['repo']
+            obj.download_path = "/var/tmp/downloads"
+            if 'download' in config['data'] and config['data']['download']:
+                obj.download_path = config['data']['download']
+            if 'http' in obj.storage_path:
+                obj.remoterepo = True
+                obj.remote_host = "ciccio.libreitalia.org"
+                if 'remote_host' in config['data'] and config['data']['remote_host']:
+                    obj.remote_host = config['data']['remote_host']
+                obj.remote_path = "/var/lib/nethserver/vhost/appimages"
+                if 'remote_path' in config['data'] and config['data']['remote_path']:
+                    obj.remote_path = config['data']['remote_path']
+
+            if 'sign' in config['data'] and config['data']['sign']:
+                obj.sign = True
+
+            # Build phase
+            obj.calculate()
+            obj.check()
+            obj.download()
+            obj.build()
+            obj.checksums()
+            if obj.remoterepo and obj.appnamedir:
+                obj.generalize_and_link(obj.appnamedir)
+            obj.publish()
+            if not obj.remoterepo:
+                obj.generalize_and_link()
+            del obj
+
+
 @cli.command()
 @click.option('-a', '--arch', 'arch', default='all',
-    type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False),
+    type=click.Choice(['x86_64', 'x86', 'all'], case_sensitive=False),
-    help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all")
+    help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: x86_64")
-@click.option('-c/-C', '--check/--no-check', 'check', default=True,
+@click.option('--check', '-c', is_flag=True, default=False,
-    help="Check in the final storage if the queried version is existent. Default: check")
+    help="Check in the repository path if the queried version is existent. Default: do not check")
 @click.option('-d', '--download-path', 'download_path',
     default = '/var/tmp/downloads', type=str,
     help="Path to the download folder. Default: /var/tmp/downloads")
@@ -70,82 +133,31 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
     else:
         arches = [ arch.lower() ]
 
-    if query.endswith('.yml') or query.endswith('.yaml'):
-        # This is a buildfile. So we have to load the file and pass the build options ourselves.
-        config = {}
-        with open(query, 'r', encoding= 'utf-8') as file:
-            config = yaml.safe_load(file)
 
-        # With the config file, we ignore all the command line options and set
+    collection = loaih.build.Collection(query, arches)
-        # generic default.
+    for obj in collection:
-        for cbuild in config['builds']:
+        # Configuration phase
-            # Loop a run for each build.
+        obj.language = language
-            collection = loaih.build.Collection(cbuild['query'], arches)
+        obj.offline_help = offline
+        obj.portable = portable
+        obj.updatable = updatable
+        if repo_path == '.':
+            repo_path = os.getcwd()
+        obj.storage_path = repo_path
+        obj.download_path = download_path
 
-            for obj in collection:
+        if sign:
-                # Configuration phase
+            obj.sign = True
-                obj.language = cbuild['language']
-                obj.offline_help = cbuild['offline_help']
-                obj.portable = cbuild['portable']
-                obj.updatable = True
-                obj.storage_path = "/srv/http/appimage.sys42.eu"
-                if 'repo' in config['data'] and config['data']['repo']:
-                    obj.storage_path = config['data']['repo']
-                obj.download_path = "/var/tmp/downloads"
-                if 'download' in config['data'] and config['data']['download']:
-                    obj.download_path = config['data']['download']
-                if 'http' in obj.storage_path:
-                    obj.remoterepo = True
-                    obj.remote_host = "ciccio.libreitalia.org"
-                    if 'remote_host' in config['data'] and config['data']['remote_host']:
-                        obj.remote_host = config['data']['remote_host']
-                    obj.remote_path = "/var/lib/nethserver/vhost/appimages"
-                    if 'remote_path' in config['data'] and config['data']['remote_path']:
-                        obj.remote_path = config['data']['remote_path']
 
-                if 'sign' in config['data'] and config['data']['sign']:
+        # Running phase
-                    obj.sign = True
+        obj.calculate()
 
-                # Build phase
+        if check:
-                obj.calculate()
+            obj.check()
-                if not 'force' in config['data'] or not config['data']['force']:
-                    obj.check()
 
-                obj.download()
+        obj.download()
-                obj.build()
+        obj.build()
-                obj.checksums()
+        obj.checksums()
-                if obj.remoterepo and obj.appnamedir:
+        obj.publish()
-                    obj.generalize_and_link(obj.appnamedir)
+        obj.generalize_and_link()
-                obj.publish()
+        del obj
-                if not obj.remoterepo:
-                    obj.generalize_and_link()
-                del obj
-
-    else:
-        collection = loaih.build.Collection(query, arches)
-        for obj in collection:
-            # Configuration phase
-            obj.language = language
-            obj.offline_help = offline
-            obj.portable = portable
-            obj.updatable = updatable
-            if repo_path == '.':
-                repo_path = os.getcwd()
-            obj.storage_path = repo_path
-            obj.download_path = download_path
-
-            if sign:
-                obj.sign = True
-
-            # Running phase
-            obj.calculate()
-
-            if check:
-                obj.check()
-
-            obj.download()
-            obj.build()
-            obj.checksums()
-            obj.publish()
-            obj.generalize_and_link()
-            del obj

loaih/solvers.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""Solvers for the queries."""
+
+import datetime
+import requests
+from lxml import html
+
+class Solver():
+    """Helps solving the queries to collections of versions."""
+
+    @staticmethod
+    def parse_query(query: Query):
+        """Returns a list of versions for the query."""
+
+        retval = []
+        if query.type == 'daily':
+            solver = DailySolver(query.text)
+            retval.append(solver.to_version())
+        elif query.type == 'named':
+            solver = NamedSolver(query.text)
+        else:
+            solver = NumberedSolver(query.text)
+
+        return retval
+
+
+class DailySolver():
+    def __init__(self, query, default_to_current = False):
+        self.query = query
+        self.default_to_current = default_to_current
+        self.version = ''
+        self.url = ''
+        self.__get_url__()
+        self.__get_version__()
+
+    def __get_url__(self):
+        """Get daily urls based on query."""
+        # The base URL for daily releases is already determined. Let's define
+        # the definitive one.
+
+        if self.query != 'current':
+            baseurl = requests.get(Definitions.DAILY)
+            pageobj = html.fromstring(baseurl.content)
+            xpath_string = "//td/a[starts-with(text(), 'Linux-rpm_deb-x86') and contains(text(), 'TDF/')]/text()"
+            tinderbox_segment = str(pageobj.xpath(xpath_string)[-1])
+            baseurl= f"{Definitions.DAILY}{tinderbox_segment}"
+
+            # Reiterate now to search for the dated version
+            base_page = requests.get(baseurl)
+            baseobj = html.fromstring(base_page.content)
+            daily_set = baseobj.xpath("//td/a/text()")
+            
+            searchdate = datetime.datetime.today()
+            if self.query == 'yesterday':
+                searchdate = searchdate + datetime.timedelta(days=-1)
+            else:
+                searchdate = datetime.datetime.strptime(self.query, '%Y%m%d')
+
+            search_results = [ x for x in daily_set if searchdate.strftime('%Y-%m-%d') in x ][-1]
+
+            if len(search_results) < 1:
+                # Searched date do not exist. if default_to_current is set,
+                # let's re-run the solver with 'current' query.
+                if self.default_to_current:
+                    current = DailySolver('current')
+                    current.query = self.query
+                    self.url = current.url
+
+            else:
+                # We'll presume there will be just one result anyways.
+                self.url = f"{baseurl}{search_results[-1]}"
+
+        else:
+            # Current.
+            current_page = requests.get(f"{Definitions.DAILY}current.html")
+            current_obj = html.fromstring(current_page.content)
+            xpath_string = "//td/a[contains(@href, 'Linux-rpm_deb-x86') and contains(@href, 'TDF/') and contains(@href, 'deb.tar')]/@href"
+            current_link = str(current_obj.xpath(xpath_string)[-1])
+
+            split_link = str(current_link).split('/')
+            return '/'.join(split_link[1:-1]) + '/'
+
+    def __get_version__(self):
+        page = requests.get(self.baseurl)
+        obj = html.fromstring(page.content)
+        xpath_string = "//td/a[contains(text(), '_deb.tar.gz')]/text()"
+        link = str(obj.xpath(xpath_string)[-1])
+        self.version = link.split('/')[-1].split('_')[1]
+
+    def to_version(self):
+        version = Version()
+        version.query = self.query
+        version.version = self.version
+        version.urls['x86_64'] = self.url
+        return version