394 lines
14 KiB
Python
394 lines
14 KiB
Python
# distinfo.py - provide meta information for distro repositories
|
|
#
|
|
# Copyright (c) 2005 Gustavo Noronha Silva <kov@debian.org>
|
|
# Copyright (c) 2006-2007 Sebastian Heinlein <glatzor@ubuntu.com>
|
|
#
|
|
# Authors: Gustavo Noronha Silva <kov@debian.org>
|
|
# Sebastian Heinlein <glatzor@ubuntu.com>
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License as
|
|
# published by the Free Software Foundation; either version 2 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
|
|
# USA
|
|
|
|
from __future__ import print_function
|
|
|
|
import csv
|
|
import errno
|
|
import logging
|
|
import os
|
|
from subprocess import Popen, PIPE
|
|
import re
|
|
|
|
import apt_pkg
|
|
|
|
from apt_pkg import gettext as _
|
|
|
|
|
|
def _expand_template(template, csv_path):
|
|
"""Expand the given template.
|
|
|
|
A template file consists of a header, followed by paragraphs
|
|
of templated suites, followed by a footer. A templated suite
|
|
is any paragraph where the Suite field contains {.
|
|
|
|
This function expands all templated suites using the information
|
|
found in the CSV file supplied by distro-info-data.
|
|
|
|
It yields lines of template info.
|
|
"""
|
|
|
|
known_suites = set()
|
|
|
|
# Copy out any header, and gather all hardcoded suites
|
|
with apt_pkg.TagFile(template) as tmpl:
|
|
for section in tmpl:
|
|
if "X-Exclude-Suites" in section:
|
|
known_suites.update(section["X-Exclude-Suites"].split(", "))
|
|
if "Suite" in section:
|
|
if "{" in section["Suite"]:
|
|
break
|
|
|
|
known_suites.add(section["Suite"])
|
|
|
|
yield from str(section).splitlines()
|
|
else:
|
|
# We did not break, so we did copy all of them
|
|
return
|
|
|
|
for section in tmpl:
|
|
if "Suite" in section:
|
|
known_suites.add(section["Suite"])
|
|
|
|
with open(csv_path) as csv_object:
|
|
releases = reversed(list(csv.DictReader(csv_object)))
|
|
|
|
# Perform template substitution on the middle of the list
|
|
for rel in releases:
|
|
if rel["series"] in known_suites:
|
|
continue
|
|
yield ""
|
|
rel["version"] = rel["version"].replace(" LTS", "")
|
|
with apt_pkg.TagFile(template) as tmpl:
|
|
for section in tmpl:
|
|
# Only work on template sections, this skips head and tails
|
|
if "Suite" not in section or "{" not in section["Suite"]:
|
|
continue
|
|
if "X-Version" in section:
|
|
# Version requirements. Maybe should be made nicer
|
|
ver = rel["version"]
|
|
if any(
|
|
(field.startswith("le") and
|
|
apt_pkg.version_compare(field[3:], ver) < 0) or
|
|
(field.startswith("ge") and
|
|
apt_pkg.version_compare(field[3:], ver) > 0)
|
|
for field in section["X-Version"].split(", ")):
|
|
continue
|
|
|
|
for line in str(section).format(**rel).splitlines():
|
|
if line.startswith("X-Version"):
|
|
continue
|
|
yield line
|
|
|
|
# Copy out remaining suites
|
|
with apt_pkg.TagFile(template) as tmpl:
|
|
# Skip the head again, we don't want to copy it twice
|
|
for section in tmpl:
|
|
if "Suite" in section and "{" in section["Suite"]:
|
|
break
|
|
|
|
for section in tmpl:
|
|
# Ignore any template parts and copy the rest out,
|
|
# this is the inverse of the template substitution loop
|
|
if "Suite" in section and "{" in section["Suite"]:
|
|
continue
|
|
|
|
yield from str(section).splitlines()
|
|
|
|
|
|
class Template(object):
|
|
|
|
def __init__(self):
|
|
self.name = None
|
|
self.child = False
|
|
self.parents = [] # ref to parent template(s)
|
|
self.match_name = None
|
|
self.description = None
|
|
self.base_uri = None
|
|
self.type = None
|
|
self.components = []
|
|
self.children = []
|
|
self.match_uri = None
|
|
self.mirror_set = {}
|
|
self.distribution = None
|
|
self.available = True
|
|
self.official = True
|
|
|
|
def has_component(self, comp):
|
|
''' Check if the distribution provides the given component '''
|
|
return comp in (c.name for c in self.components)
|
|
|
|
def is_mirror(self, url):
|
|
''' Check if a given url of a repository is a valid mirror '''
|
|
proto, hostname, dir = split_url(url)
|
|
if hostname in self.mirror_set:
|
|
return self.mirror_set[hostname].has_repository(proto, dir)
|
|
else:
|
|
return False
|
|
|
|
|
|
class Component(object):
|
|
|
|
def __init__(self, name, desc=None, long_desc=None, parent_component=None):
|
|
self.name = name
|
|
self.description = desc
|
|
self.description_long = long_desc
|
|
self.parent_component = parent_component
|
|
|
|
def get_parent_component(self):
|
|
return self.parent_component
|
|
|
|
def set_parent_component(self, parent):
|
|
self.parent_component = parent
|
|
|
|
def get_description(self):
|
|
if self.description_long is not None:
|
|
return self.description_long
|
|
elif self.description is not None:
|
|
return self.description
|
|
else:
|
|
return None
|
|
|
|
def set_description(self, desc):
|
|
self.description = desc
|
|
|
|
def set_description_long(self, desc):
|
|
self.description_long = desc
|
|
|
|
def get_description_long(self):
|
|
return self.description_long
|
|
|
|
|
|
class Mirror(object):
|
|
''' Storage for mirror related information '''
|
|
|
|
def __init__(self, proto, hostname, dir, location=None):
|
|
self.hostname = hostname
|
|
self.repositories = []
|
|
self.add_repository(proto, dir)
|
|
self.location = location
|
|
|
|
def add_repository(self, proto, dir):
|
|
self.repositories.append(Repository(proto, dir))
|
|
|
|
def get_repositories_for_proto(self, proto):
|
|
return [r for r in self.repositories if r.proto == proto]
|
|
|
|
def has_repository(self, proto, dir):
|
|
if dir is None:
|
|
return False
|
|
for r in self.repositories:
|
|
if r.proto == proto and dir in r.dir:
|
|
return True
|
|
return False
|
|
|
|
def get_repo_urls(self):
|
|
return [r.get_url(self.hostname) for r in self.repositories]
|
|
|
|
def get_location(self):
|
|
return self.location
|
|
|
|
def set_location(self, location):
|
|
self.location = location
|
|
|
|
|
|
class Repository(object):
|
|
|
|
def __init__(self, proto, dir):
|
|
self.proto = proto
|
|
self.dir = dir
|
|
|
|
def get_info(self):
|
|
return self.proto, self.dir
|
|
|
|
def get_url(self, hostname):
|
|
return "%s://%s/%s" % (self.proto, hostname, self.dir)
|
|
|
|
|
|
def split_url(url):
|
|
''' split a given URL into the protocoll, the hostname and the dir part '''
|
|
split = re.split(":*\\/+", url, maxsplit=2)
|
|
while len(split) < 3:
|
|
split.append(None)
|
|
return split
|
|
|
|
|
|
class DistInfo(object):
|
|
|
|
def __init__(self, dist=None, base_dir="/usr/share/python-apt/templates"):
|
|
self.metarelease_uri = ''
|
|
self.templates = []
|
|
self.arch = apt_pkg.config.find("APT::Architecture")
|
|
|
|
location = None
|
|
match_loc = re.compile(r"^#LOC:(.+)$")
|
|
match_mirror_line = re.compile(
|
|
r"^(#LOC:.+)|(((http)|(ftp)|(rsync)|(file)|(mirror)|(https))://"
|
|
r"[A-Za-z0-9/\.:\-_@]+)$")
|
|
#match_mirror_line = re.compile(r".+")
|
|
|
|
if not dist:
|
|
try:
|
|
dist = Popen(["lsb_release", "-i", "-s"],
|
|
universal_newlines=True,
|
|
stdout=PIPE).communicate()[0].strip()
|
|
except (OSError, IOError) as exc:
|
|
if exc.errno != errno.ENOENT:
|
|
logging.warning(
|
|
'lsb_release failed, using defaults:' % exc)
|
|
dist = "Debian"
|
|
|
|
self.dist = dist
|
|
|
|
map_mirror_sets = {}
|
|
|
|
dist_fname = "%s/%s.info" % (base_dir, dist)
|
|
csv_fname = "/usr/share/distro-info/{}.csv".format(dist.lower())
|
|
|
|
template = None
|
|
component = None
|
|
for line in _expand_template(dist_fname, csv_fname):
|
|
tokens = line.split(':', 1)
|
|
if len(tokens) < 2:
|
|
continue
|
|
field = tokens[0].strip()
|
|
value = tokens[1].strip()
|
|
if field == 'ChangelogURI':
|
|
self.changelogs_uri = _(value)
|
|
elif field == 'MetaReleaseURI':
|
|
self.metarelease_uri = value
|
|
elif field == 'Suite':
|
|
self.finish_template(template, component)
|
|
component = None
|
|
template = Template()
|
|
template.name = value
|
|
template.distribution = dist
|
|
template.match_name = "^%s$" % value
|
|
elif field == 'MatchName':
|
|
template.match_name = value
|
|
elif field == 'ParentSuite':
|
|
template.child = True
|
|
for nanny in self.templates:
|
|
# look for parent and add back ref to it
|
|
if nanny.name == value:
|
|
template.parents.append(nanny)
|
|
nanny.children.append(template)
|
|
elif field == 'Available':
|
|
template.available = apt_pkg.string_to_bool(value)
|
|
elif field == 'Official':
|
|
template.official = apt_pkg.string_to_bool(value)
|
|
elif field == 'RepositoryType':
|
|
template.type = value
|
|
elif field == 'BaseURI' and not template.base_uri:
|
|
template.base_uri = value
|
|
elif field == 'BaseURI-%s' % self.arch:
|
|
template.base_uri = value
|
|
elif field == 'MatchURI' and not template.match_uri:
|
|
template.match_uri = value
|
|
elif field == 'MatchURI-%s' % self.arch:
|
|
template.match_uri = value
|
|
elif (field == 'MirrorsFile' or
|
|
field == 'MirrorsFile-%s' % self.arch):
|
|
# Make the path absolute.
|
|
value = os.path.isabs(value) and value or \
|
|
os.path.abspath(os.path.join(base_dir, value))
|
|
if value not in map_mirror_sets:
|
|
mirror_set = {}
|
|
try:
|
|
with open(value) as value_f:
|
|
mirror_data = list(filter(
|
|
match_mirror_line.match,
|
|
[x.strip() for x in value_f]))
|
|
except Exception:
|
|
print("WARNING: Failed to read mirror file")
|
|
mirror_data = []
|
|
for line in mirror_data:
|
|
if line.startswith("#LOC:"):
|
|
location = match_loc.sub(r"\1", line)
|
|
continue
|
|
(proto, hostname, dir) = split_url(line)
|
|
if hostname in mirror_set:
|
|
mirror_set[hostname].add_repository(proto, dir)
|
|
else:
|
|
mirror_set[hostname] = Mirror(
|
|
proto, hostname, dir, location)
|
|
map_mirror_sets[value] = mirror_set
|
|
template.mirror_set = map_mirror_sets[value]
|
|
elif field == 'Description':
|
|
template.description = _(value)
|
|
elif field == 'Component':
|
|
if (component and not
|
|
template.has_component(component.name)):
|
|
template.components.append(component)
|
|
component = Component(value)
|
|
elif field == 'CompDescription':
|
|
component.set_description(_(value))
|
|
elif field == 'CompDescriptionLong':
|
|
component.set_description_long(_(value))
|
|
elif field == 'ParentComponent':
|
|
component.set_parent_component(value)
|
|
self.finish_template(template, component)
|
|
template = None
|
|
component = None
|
|
|
|
def finish_template(self, template, component):
|
|
" finish the current tempalte "
|
|
if not template:
|
|
return
|
|
# reuse some properties of the parent template
|
|
if template.match_uri is None and template.child:
|
|
for t in template.parents:
|
|
if t.match_uri:
|
|
template.match_uri = t.match_uri
|
|
break
|
|
if template.mirror_set == {} and template.child:
|
|
for t in template.parents:
|
|
if t.match_uri:
|
|
template.mirror_set = t.mirror_set
|
|
break
|
|
if component and not template.has_component(component.name):
|
|
template.components.append(component)
|
|
component = None
|
|
# the official attribute is inherited
|
|
for t in template.parents:
|
|
template.official = t.official
|
|
self.templates.append(template)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
d = DistInfo("Ubuntu", "/usr/share/python-apt/templates")
|
|
logging.info(d.changelogs_uri)
|
|
for template in d.templates:
|
|
logging.info("\nSuite: %s" % template.name)
|
|
logging.info("Desc: %s" % template.description)
|
|
logging.info("BaseURI: %s" % template.base_uri)
|
|
logging.info("MatchURI: %s" % template.match_uri)
|
|
if template.mirror_set != {}:
|
|
logging.info("Mirrors: %s" % list(template.mirror_set.keys()))
|
|
for comp in template.components:
|
|
logging.info(" %s -%s -%s" % (comp.name,
|
|
comp.description,
|
|
comp.description_long))
|
|
for child in template.children:
|
|
logging.info(" %s" % child.description)
|