Code for passing also a date for daily builds. Possibly blocked by cli command.

This commit is contained in:
emiliano.vavassori 2023-12-02 03:00:46 +01:00
parent 71a81b6a8e
commit 142a09df14
1 changed files with 62 additions and 36 deletions

View File

@ -1,12 +1,15 @@
#!/usr/bin/env python
# encoding: utf-8
"""Machinery for compiling new versions of AppImages."""
import urllib.request
import datetime
import re
from lxml import etree
from packaging.version import parse as parse_version
class Definitions():
class Definitions(): # pylint: disable=too-few-public-methods
"""Definitions for the module."""
@ -49,14 +52,15 @@ class Base():
# Fixing daily selector
# As seen, the number of the tinderbox building the daily version can
# change. We try to fulfill the void by adding a step.
tinderboxpage = etree.HTML(urllib.request.urlopen(Definitions.DAILY).read())
tburl = tinderboxpage.xpath(
"//td/a[starts-with(text(), 'Linux-rpm_deb-x86') and contains(text(), 'TDF/')]/text()"
tinderboxpage = etree.HTML(#pylint: disable=c-extension-no-member
urllib.request.urlopen(Definitions.DAILY).read() #pylint: disable=consider-using-with
xpath = "//td/a[starts-with(text(), 'Linux-rpm_deb-x86') and contains(text(), 'TDF/')]/text()" #pylint: disable=line-too-long
tburl = str(tinderboxpage.xpath(xpath)[0])
daily_selector = f"{Definitions.DAILY}{tburl}"
# Get the anchor for today's builds
raw_page = etree.HTML(urllib.request.urlopen(daily_selector).read())
raw_page = etree.HTML(urllib.request.urlopen(daily_selector).read())# pylint: disable=c-extension-no-member,consider-using-with
results = raw_page.xpath(
@ -79,26 +83,38 @@ class Base():
return []
# Rerun the page parsing, this time to find out the versions built
b = etree.HTML(urllib.request.urlopen(url).read()).xpath("//td/a[contains(text(), '_deb.tar.gz')]/text()")
fullpage = urllib.request.urlopen(url).read() #pylint: disable=consider-using-with
archive_path = "//td/a[contains(text(), '_deb.tar.gz')]/text()"
tarball = etree.HTML(fullpage).xpath(archive_path)#pylint: disable=c-extension-no-member
# This should have returned the main package for a version, but can
# have returned multiple ones, so let's treat it as a list
return [ x.split('_')[1] for x in b ]
return [ x.split('_')[1] for x in tarball ]
def namedver(query):
"""Gets the version for a specific named version."""
if query == 'daily' or query == 'yesterday':
# Daily needs double parsing for the same result to apply.
# We first select today's build anchor:
date =
if query == 'yesterday':
# Use yesterdays' date for testing purposes.
date += datetime.timedelta(days=-1)
return Base.dailyver(date)
if 'yesterday' in query:
return Base.dailyver( + datetime.timedelta(days=-1))
if query.startswith('daily'):
if '-' in query or '_' in query:
# Splitting the string
stringdate = re.sub(r'daily[-_:]', '', query)
print(f"Stringdate: {stringdate}")
print(f"strptime: {datetime.datetime.strptime(stringdate, '%Y%m%d')}")
return Base.dailyver(datetime.datetime.strptime(stringdate, "%Y%m%d"))
return Base.dailyver()
# In case the query isn't for daily
return etree.HTML(urllib.request.urlopen(Definitions.SELECTORS[query]['URL']).read()).xpath(Definitions.SELECTORS[query]['xpath'])
return etree.HTML(#pylint: disable=c-extension-no-member
def fullversion(version):
@ -144,9 +160,9 @@ class Base():
retval['x86'] = baseurl + 'x86/'
retval['x86'] = '-'
retval['x86_64'] = baseurl + 'x86_64/'
return retval
@ -156,23 +172,31 @@ class Base():
if '.' in query:
# Called with a numeric query. Pass it to RemoteBuild
elif '-' in query or '_' in query:
# daily but with date
# The date is the last part.
stringdate = re.split(r'[_-]', query)[-1]
querydate = datetime.datetime.strptime(stringdate, "%Y%m%d").date()
dailyver = Base.dailyver(querydate)
# Named query
a = Base.namedver(query)
dailyver = Base.namedver(query)
if not a:
if not dailyver:
# a is empty
return retval
if isinstance(a, list) and len(a) > 1:
retval.extend([ RemoteBuild(query, version) for version in a ])
if isinstance(dailyver, list) and len(dailyver) > 1:
retval.extend([ RemoteBuild(query, version) for version in dailyver ])
return sorted(retval, key=lambda x: x.version)
class RemoteBuild(object):
class RemoteBuild():
"""Builds a version with checking remotely if it was not already built."""
def __init__(self, query, version = None):
"""Should simplify the single builded version."""
@ -188,21 +212,22 @@ class RemoteBuild(object):
# Let's check if a specific version was requested.
if self.version == '':
# In case it was not requested, we will carry on the generic
# namedver() query.
# If the results are more than one, we'll take the latest (since we are requested to provide a single build).
a = Base.namedver(self.query)
if isinstance(a, list):
# if the number of versions is zero, return and exit
if not a:
return None
# namedver() query.
# If the results are more than one, we'll take the latest
# (since we are requested to provide a single build).
a_version = Base.namedver(self.query)
if len(a) == 1:
if isinstance(a_version, list):
# if the number of versions is zero, return and exit
if not a_version:
self.version = None
if len(a_version) == 1:
# version is a single one.
self.version = a[0]
self.version = a_version[0]
# In this case, we will select the latest release.
self.version = sorted(a)[-1]
self.version = sorted(a_version)[-1]
# If the version has already a version, as requested by user,
# continue using that version
@ -213,10 +238,11 @@ class RemoteBuild(object):
if len(str(self.version).split('.')) < 4:
# If not 4 dotted, let's search for the 4 dotted version
self.version = Base.fullversion(self.version)
self.basedirurl = Base.urlfromqueryandver(self.query, self.version)
def todict(self):
"""Returns a dictionary of versions."""
return {
'query': self.query,
'version': self.version,