580 lines
22 KiB
Python
Executable File
580 lines
22 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
import apt
|
|
import argparse
|
|
import distro_info
|
|
import json
|
|
import os
|
|
import sys
|
|
import gettext
|
|
import shutil
|
|
import subprocess
|
|
|
|
from UpdateManager.Core.utils import get_dist
|
|
|
|
from datetime import datetime
|
|
from textwrap import wrap
|
|
from urllib.error import URLError, HTTPError
|
|
from urllib.request import urlopen
|
|
|
|
# TODO make DEBUG an environmental variable
|
|
DEBUG = False
|
|
UA_STATUS_FILE = "/var/lib/ubuntu-advantage/status.json"
|
|
|
|
|
|
class PatchStats:
|
|
"""Tracks overall patch status
|
|
|
|
The relationship between archives enabled and whether a patch is eligible
|
|
for receiving updates is non-trivial. We track here all the important
|
|
buckets a package can be in:
|
|
|
|
- Whether it is set to expire with no ESM coverage
|
|
- Whether it is in an archive covered by ESM
|
|
- Whether it received LTS patches
|
|
- whether it received ESM patches
|
|
|
|
We also track the total packages covered and uncovered, and for the
|
|
uncovered packages, we track where they originate from.
|
|
|
|
The Ubuntu main archive receives patches for 5 years.
|
|
Canonical-owned archives (excluding partner) receive patches for 10 years.
|
|
patches for 10 years.
|
|
"""
|
|
def __init__(self):
|
|
# TODO no-update FIPS is never patched
|
|
self.pkgs_uncovered_fips = set()
|
|
|
|
# list of package names available in ESM
|
|
self.pkgs_updated_in_esmi = set()
|
|
self.pkgs_updated_in_esma = set()
|
|
|
|
self.pkgs_mr = set()
|
|
self.pkgs_um = set()
|
|
self.pkgs_unavailable = set()
|
|
self.pkgs_thirdparty = set()
|
|
# the bin of unknowns
|
|
self.pkgs_uncategorized = set()
|
|
|
|
|
|
def print_debug(s):
|
|
if DEBUG:
|
|
print(s)
|
|
|
|
|
|
def whats_in_esm(url):
|
|
pkgs = set()
|
|
# return a set of package names in an esm archive
|
|
try:
|
|
response = urlopen(url)
|
|
except (URLError, HTTPError):
|
|
print_debug('failed to load: %s' % url)
|
|
return pkgs
|
|
try:
|
|
content = response.read().decode('utf-8')
|
|
except IOError:
|
|
print('failed to read data at: %s' % url)
|
|
sys.exit(1)
|
|
for line in content.split('\n'):
|
|
if not line.startswith('Package:'):
|
|
continue
|
|
else:
|
|
pkg = line.split(': ')[1]
|
|
pkgs.add(pkg)
|
|
return pkgs
|
|
|
|
|
|
def get_ua_status():
|
|
"""Return dict of active ua status information from ubuntu-advantage-tools
|
|
|
|
Prefer to obtain status information from cache on disk to avoid costly
|
|
roundtrips to contracts.canonical.com to check on available services.
|
|
|
|
Fallback to call ua status --format=json.
|
|
|
|
If there are errors running: ua status --format=json or if the status on
|
|
disk is unparseable, return an empty dict.
|
|
"""
|
|
if os.path.exists(UA_STATUS_FILE):
|
|
with open(UA_STATUS_FILE, "r") as stream:
|
|
status = stream.read()
|
|
else:
|
|
try:
|
|
status = subprocess.check_output(
|
|
['ua', 'status', '--format=json']
|
|
).decode()
|
|
except subprocess.CalledProcessError as e:
|
|
print_debug('failed to run ua status: %s' % e)
|
|
return {}
|
|
try:
|
|
return json.loads(status)
|
|
except json.decoder.JSONDecodeError as e:
|
|
print_debug('failed to parse JSON from ua status output: %s' % e)
|
|
return {}
|
|
|
|
|
|
def is_ua_service_enabled(service_name: str) -> bool:
|
|
"""Check to see if named esm service is enabled.
|
|
|
|
:return: True if UA status reports service as enabled.
|
|
"""
|
|
status = get_ua_status()
|
|
for service in status.get("services", []):
|
|
if service["name"] == service_name:
|
|
# Machines unattached to UA will not provide service 'status' key.
|
|
return service.get("status") == "enabled"
|
|
return False
|
|
|
|
|
|
def trim_archive(archive):
|
|
return archive.split("-")[-1]
|
|
|
|
|
|
def trim_site(host):
|
|
# *.ec2.archive.ubuntu.com -> archive.ubuntu.com
|
|
if host.endswith("archive.ubuntu.com"):
|
|
return "archive.ubuntu.com"
|
|
return host
|
|
|
|
|
|
def mirror_list():
|
|
m_file = '/usr/share/ubuntu-release-upgrader/mirrors.cfg'
|
|
if not os.path.exists(m_file):
|
|
print("Official mirror list not found.")
|
|
with open(m_file) as f:
|
|
items = [x.strip() for x in f]
|
|
mirrors = [s.split('//')[1].split('/')[0] for s in items
|
|
if not s.startswith("#") and not s == ""]
|
|
# ddebs.ubuntu.com isn't in mirrors.cfg for every release
|
|
mirrors.append('ddebs.ubuntu.com')
|
|
return mirrors
|
|
|
|
|
|
def origins_for(ver: apt.package.Version) -> str:
|
|
s = []
|
|
for origin in ver.origins:
|
|
if not origin.site:
|
|
# When the package is installed, site is empty, archive/component
|
|
# are "now/now"
|
|
continue
|
|
site = trim_site(origin.site)
|
|
s.append("%s %s/%s" % (site, origin.archive, origin.component))
|
|
return ",".join(s)
|
|
|
|
|
|
def print_wrapped(str):
|
|
print("\n".join(wrap(str, break_on_hyphens=False)))
|
|
|
|
|
|
def print_thirdparty_count():
|
|
print(gettext.dngettext("update-manager",
|
|
"%s package is from a third party",
|
|
"%s packages are from third parties",
|
|
len(pkgstats.pkgs_thirdparty)) %
|
|
"{:>{width}}".format(len(pkgstats.pkgs_thirdparty), width=width))
|
|
|
|
|
|
def print_unavailable_count():
|
|
print(gettext.dngettext("update-manager",
|
|
"%s package is no longer available for "
|
|
"download",
|
|
"%s packages are no longer available for "
|
|
"download",
|
|
len(pkgstats.pkgs_unavailable)) %
|
|
"{:>{width}}".format(len(pkgstats.pkgs_unavailable), width=width))
|
|
|
|
|
|
def parse_options():
|
|
'''Parse command line arguments.
|
|
|
|
Return parser
|
|
'''
|
|
parser = argparse.ArgumentParser(
|
|
description='Return information about security support for packages')
|
|
parser.add_argument('--thirdparty', action='store_true')
|
|
parser.add_argument('--unavailable', action='store_true')
|
|
return parser
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Prefer redirecting to 'pro security-status' if it exists
|
|
if shutil.which("/usr/bin/pro"):
|
|
print("This command has been replaced with 'pro security-status'.")
|
|
subprocess.run(["/usr/bin/pro", "security-status"])
|
|
sys.exit(0)
|
|
|
|
# gettext
|
|
APP = "update-manager"
|
|
DIR = "/usr/share/locale"
|
|
gettext.bindtextdomain(APP, DIR)
|
|
gettext.textdomain(APP)
|
|
|
|
parser = parse_options()
|
|
args = parser.parse_args()
|
|
|
|
esm_site = "esm.ubuntu.com"
|
|
|
|
try:
|
|
dpkg = subprocess.check_output(['dpkg', '--print-architecture'])
|
|
arch = dpkg.decode().strip()
|
|
except subprocess.CalledProcessError:
|
|
print("failed getting dpkg architecture")
|
|
sys.exit(1)
|
|
|
|
cache = apt.Cache()
|
|
pkgstats = PatchStats()
|
|
codename = get_dist()
|
|
di = distro_info.UbuntuDistroInfo()
|
|
lts = di.is_lts(codename)
|
|
release_expired = True
|
|
if codename in di.supported():
|
|
release_expired = False
|
|
# distro-info-data in Ubuntu 16.04 LTS does not have eol-esm data
|
|
if codename != 'xenial':
|
|
eol_data = [(r.eol, r.eol_esm)
|
|
for r in di._releases if r.series == codename][0]
|
|
elif codename == 'xenial':
|
|
eol_data = (datetime.strptime('2021-04-21', '%Y-%m-%d'),
|
|
datetime.strptime('2024-04-21', '%Y-%m-%d'))
|
|
eol = eol_data[0]
|
|
eol_esm = eol_data[1]
|
|
|
|
all_origins = set()
|
|
origins_by_package = {}
|
|
official_mirrors = mirror_list()
|
|
|
|
# N.B. only the security pocket is checked because this tool displays
|
|
# information about security updates
|
|
esm_url = \
|
|
'https://%s/%s/ubuntu/dists/%s-%s-%s/main/binary-%s/Packages'
|
|
pkgs_in_esma = whats_in_esm(esm_url %
|
|
(esm_site, 'apps', codename, 'apps',
|
|
'security', arch))
|
|
pkgs_in_esmi = whats_in_esm(esm_url %
|
|
(esm_site, 'infra', codename, 'infra',
|
|
'security', arch))
|
|
|
|
for pkg in cache:
|
|
pkgname = pkg.name
|
|
|
|
downloadable = True
|
|
if not pkg.is_installed:
|
|
continue
|
|
if not pkg.candidate or not pkg.candidate.downloadable:
|
|
downloadable = False
|
|
pkg_sites = []
|
|
origins_by_package[pkgname] = set()
|
|
|
|
for ver in pkg.versions:
|
|
# Loop through origins and store all of them. The idea here is that
|
|
# we don't care where the installed package comes from, provided
|
|
# there is at least one repository we identify as being
|
|
# security-assured under either LTS or ESM.
|
|
for origin in ver.origins:
|
|
# TODO: in order to handle FIPS and other archives which have
|
|
# root-level path names, we'll need to loop over ver.uris
|
|
# instead
|
|
if not origin.site:
|
|
continue
|
|
site = trim_site(origin.site)
|
|
archive = origin.archive
|
|
component = origin.component
|
|
origin = origin.origin
|
|
official_mirror = False
|
|
thirdparty = True
|
|
# thirdparty providers like dl.google.com don't set "Origin"
|
|
if origin != "Ubuntu":
|
|
thirdparty = False
|
|
if site in official_mirrors:
|
|
site = "official_mirror"
|
|
if "MY_MIRROR" in os.environ:
|
|
if site in os.environ["MY_MIRROR"]:
|
|
site = "official_mirror"
|
|
t = (site, archive, component, thirdparty)
|
|
if not site:
|
|
continue
|
|
all_origins.add(t)
|
|
origins_by_package[pkgname].add(t)
|
|
|
|
if DEBUG:
|
|
pkg_sites.append("%s %s/%s" %
|
|
(site, archive, component))
|
|
|
|
print_debug("available versions for %s" % pkgname)
|
|
print_debug(",".join(pkg_sites))
|
|
|
|
# This tracks suites we care about. Sadly, it appears that the way apt
|
|
# stores origins truncates away the path that comes after the
|
|
# domainname in the site portion, or maybe I am just clueless, but
|
|
# there's no way to tell FIPS apart from ESM, for instance.
|
|
# See 00REPOS.txt for examples
|
|
|
|
# 2020-03-18 ver.filename has the path so why is that no good?
|
|
|
|
# TODO Need to handle:
|
|
# MAAS, lxd, juju PPAs
|
|
# other PPAs
|
|
# other repos
|
|
|
|
# TODO handle partner.c.c
|
|
|
|
# main and restricted from release, -updates, -proposed, or -security
|
|
# pockets
|
|
suite_main = ("official_mirror", codename, "main", True)
|
|
suite_main_updates = ("official_mirror", codename + "-updates",
|
|
"main", True)
|
|
suite_main_security = ("official_mirror", codename + "-security",
|
|
"main", True)
|
|
suite_main_proposed = ("official_mirror", codename + "-proposed",
|
|
"main", True)
|
|
|
|
suite_restricted = ("official_mirror", codename, "restricted",
|
|
True)
|
|
suite_restricted_updates = ("official_mirror",
|
|
codename + "-updates",
|
|
"restricted", True)
|
|
suite_restricted_security = ("official_mirror",
|
|
codename + "-security",
|
|
"restricted", True)
|
|
suite_restricted_proposed = ("official_mirror",
|
|
codename + "-proposed",
|
|
"restricted", True)
|
|
|
|
# universe and multiverse from release, -updates, -proposed, or -security
|
|
# pockets
|
|
suite_universe = ("official_mirror", codename, "universe", True)
|
|
suite_universe_updates = ("official_mirror", codename + "-updates",
|
|
"universe", True)
|
|
suite_universe_security = ("official_mirror",
|
|
codename + "-security",
|
|
"universe", True)
|
|
suite_universe_proposed = ("official_mirror",
|
|
codename + "-proposed",
|
|
"universe", True)
|
|
|
|
suite_multiverse = ("official_mirror", codename, "multiverse",
|
|
True)
|
|
suite_multiverse_updates = ("official_mirror",
|
|
codename + "-updates",
|
|
"multiverse", True)
|
|
suite_multiverse_security = ("official_mirror",
|
|
codename + "-security",
|
|
"multiverse", True)
|
|
suite_multiverse_proposed = ("official_mirror",
|
|
codename + "-proposed",
|
|
"multiverse", True)
|
|
|
|
# packages from the esm respositories
|
|
# N.B. Origin: Ubuntu is not set for esm
|
|
suite_esm_main = (esm_site, "%s-infra-updates" % codename,
|
|
"main")
|
|
suite_esm_main_security = (esm_site,
|
|
"%s-infra-security" % codename, "main")
|
|
suite_esm_universe = (esm_site,
|
|
"%s-apps-updates" % codename, "main")
|
|
suite_esm_universe_security = (esm_site,
|
|
"%s-apps-security" % codename,
|
|
"main")
|
|
|
|
esm_infra_enabled = is_ua_service_enabled("esm-infra")
|
|
esm_apps_enabled = is_ua_service_enabled("esm-apps")
|
|
ua_attached = get_ua_status().get("attached", False)
|
|
|
|
# Now do the final loop through
|
|
for pkg in cache:
|
|
if not pkg.is_installed:
|
|
continue
|
|
if not pkg.candidate or not pkg.candidate.downloadable:
|
|
pkgstats.pkgs_unavailable.add(pkg.name)
|
|
continue
|
|
pkgname = pkg.name
|
|
pkg_origins = origins_by_package[pkgname]
|
|
|
|
# This set of is_* booleans tracks specific situations we care about in
|
|
# the logic below; for instance, if the package has a main origin, or
|
|
# if the esm repos are enabled.
|
|
|
|
# Some packages get added in -updates and don't exist in the release
|
|
# pocket e.g. ubuntu-advantage-tools and libdrm-updates. To be safe all
|
|
# pockets are allowed.
|
|
is_mr_pkg_origin = (suite_main in pkg_origins) or \
|
|
(suite_main_updates in pkg_origins) or \
|
|
(suite_main_security in pkg_origins) or \
|
|
(suite_main_proposed in pkg_origins) or \
|
|
(suite_restricted in pkg_origins) or \
|
|
(suite_restricted_updates in pkg_origins) or \
|
|
(suite_restricted_security in pkg_origins) or \
|
|
(suite_restricted_proposed in pkg_origins)
|
|
is_um_pkg_origin = (suite_universe in pkg_origins) or \
|
|
(suite_universe_updates in pkg_origins) or \
|
|
(suite_universe_security in pkg_origins) or \
|
|
(suite_universe_proposed in pkg_origins) or \
|
|
(suite_multiverse in pkg_origins) or \
|
|
(suite_multiverse_updates in pkg_origins) or \
|
|
(suite_multiverse_security in pkg_origins) or \
|
|
(suite_multiverse_proposed in pkg_origins)
|
|
|
|
is_esm_infra_pkg_origin = (suite_esm_main in pkg_origins) or \
|
|
(suite_esm_main_security in pkg_origins)
|
|
is_esm_apps_pkg_origin = (suite_esm_universe in pkg_origins) or \
|
|
(suite_esm_universe_security in pkg_origins)
|
|
|
|
# A third party one won't appear in any of the above origins
|
|
if not is_mr_pkg_origin and not is_um_pkg_origin \
|
|
and not is_esm_infra_pkg_origin and not is_esm_apps_pkg_origin:
|
|
pkgstats.pkgs_thirdparty.add(pkgname)
|
|
|
|
if False: # TODO package has ESM fips origin
|
|
# TODO package has ESM fips-updates origin: OK
|
|
# If user has enabled FIPS, but not updates, BAD, but need some
|
|
# thought on how to display it, as it can't be patched at all
|
|
pass
|
|
elif is_mr_pkg_origin:
|
|
pkgstats.pkgs_mr.add(pkgname)
|
|
elif is_um_pkg_origin:
|
|
pkgstats.pkgs_um.add(pkgname)
|
|
else:
|
|
# TODO print information about packages in this category if in
|
|
# debugging mode
|
|
pkgstats.pkgs_uncategorized.add(pkgname)
|
|
|
|
# Check to see if the package is available in esm-infra or esm-apps
|
|
# and add it to the right pkgstats category
|
|
# NB: apps is ordered first for testing the hello package which is both
|
|
# in esmi and esma
|
|
if pkgname in pkgs_in_esma:
|
|
pkgstats.pkgs_updated_in_esma.add(pkgname)
|
|
elif pkgname in pkgs_in_esmi:
|
|
pkgstats.pkgs_updated_in_esmi.add(pkgname)
|
|
|
|
total_packages = (len(pkgstats.pkgs_mr) +
|
|
len(pkgstats.pkgs_um) +
|
|
len(pkgstats.pkgs_thirdparty) +
|
|
len(pkgstats.pkgs_unavailable))
|
|
width = len(str(total_packages))
|
|
print("%s packages installed, of which:" %
|
|
"{:>{width}}".format(total_packages, width=width))
|
|
|
|
# filters first as they provide less information
|
|
if args.thirdparty:
|
|
if pkgstats.pkgs_thirdparty:
|
|
pkgs_thirdparty = sorted(p for p in pkgstats.pkgs_thirdparty)
|
|
print_thirdparty_count()
|
|
print_wrapped(' '.join(pkgs_thirdparty))
|
|
msg = ("Packages from third parties are not provided by the "
|
|
"official Ubuntu archive, for example packages from "
|
|
"Personal Package Archives in Launchpad.")
|
|
print("")
|
|
print_wrapped(msg)
|
|
print("")
|
|
print_wrapped("Run 'apt-cache policy %s' to learn more about "
|
|
"that package." % pkgs_thirdparty[0])
|
|
sys.exit(0)
|
|
else:
|
|
print_wrapped("You have no packages installed from a third party.")
|
|
sys.exit(0)
|
|
if args.unavailable:
|
|
if pkgstats.pkgs_unavailable:
|
|
pkgs_unavailable = sorted(p for p in pkgstats.pkgs_unavailable)
|
|
print_unavailable_count()
|
|
print_wrapped(' '.join(pkgs_unavailable))
|
|
msg = ("Packages that are not available for download "
|
|
"may be left over from a previous release of "
|
|
"Ubuntu, may have been installed directly from "
|
|
"a .deb file, or are from a source which has "
|
|
"been disabled.")
|
|
print("")
|
|
print_wrapped(msg)
|
|
print("")
|
|
print_wrapped("Run 'apt-cache show %s' to learn more about "
|
|
"that package." % pkgs_unavailable[0])
|
|
sys.exit(0)
|
|
else:
|
|
print_wrapped("You have no packages installed that are no longer "
|
|
"available.")
|
|
sys.exit(0)
|
|
# Only show LTS patches and expiration notices if the release is not
|
|
# yet expired; showing LTS patches would give a false sense of
|
|
# security.
|
|
if not release_expired:
|
|
print("%s receive package updates%s until %d/%d" %
|
|
("{:>{width}}".format(len(pkgstats.pkgs_mr),
|
|
width=width),
|
|
" with LTS" if lts else "",
|
|
eol.month, eol.year))
|
|
elif release_expired and lts:
|
|
receive_text = "could receive"
|
|
if esm_infra_enabled:
|
|
if len(pkgstats.pkgs_mr) == 1:
|
|
receive_text = "is receiving"
|
|
else:
|
|
receive_text = "are receiving"
|
|
print("%s %s security updates with ESM Infra "
|
|
"until %d/%d" %
|
|
("{:>{width}}".format(len(pkgstats.pkgs_mr),
|
|
width=width),
|
|
receive_text, eol_esm.month, eol_esm.year))
|
|
if lts and pkgstats.pkgs_um and is_ua_service_enabled("esm-apps"):
|
|
if len(pkgstats.pkgs_um) == 1:
|
|
receive_text = "is receiving"
|
|
else:
|
|
receive_text = "are receiving"
|
|
print("%s %s security updates with ESM Apps "
|
|
"until %d/%d" %
|
|
("{:>{width}}".format(len(pkgstats.pkgs_um),
|
|
width=width),
|
|
receive_text, eol_esm.month, eol_esm.year))
|
|
if pkgstats.pkgs_thirdparty:
|
|
print_thirdparty_count()
|
|
if pkgstats.pkgs_unavailable:
|
|
print_unavailable_count()
|
|
# print the detail messages after the count of packages
|
|
if pkgstats.pkgs_thirdparty:
|
|
msg = ("Packages from third parties are not provided by the "
|
|
"official Ubuntu archive, for example packages from "
|
|
"Personal Package Archives in Launchpad.")
|
|
print("")
|
|
print_wrapped(msg)
|
|
action = ("For more information on the packages, run "
|
|
"'ubuntu-security-status --thirdparty'.")
|
|
print_wrapped(action)
|
|
if pkgstats.pkgs_unavailable:
|
|
msg = ("Packages that are not available for download "
|
|
"may be left over from a previous release of "
|
|
"Ubuntu, may have been installed directly from "
|
|
"a .deb file, or are from a source which has "
|
|
"been disabled.")
|
|
print("")
|
|
print_wrapped(msg)
|
|
action = ("For more information on the packages, run "
|
|
"'ubuntu-security-status --unavailable'.")
|
|
print_wrapped(action)
|
|
# print the ESM calls to action last
|
|
if lts and not esm_infra_enabled:
|
|
if release_expired and pkgstats.pkgs_mr:
|
|
pkgs_updated_in_esmi = pkgstats.pkgs_updated_in_esmi
|
|
print("")
|
|
msg = gettext.dngettext(
|
|
"update-manager",
|
|
"Enable Extended Security "
|
|
"Maintenance (ESM Infra) to "
|
|
"get %i security update (so far) ",
|
|
"Enable Extended Security "
|
|
"Maintenance (ESM Infra) to "
|
|
"get %i security updates (so far) ",
|
|
len(pkgs_updated_in_esmi)) % len(pkgs_updated_in_esmi)
|
|
msg += gettext.dngettext(
|
|
"update-manager",
|
|
"and enable coverage of %i "
|
|
"package.",
|
|
"and enable coverage of %i "
|
|
"packages.",
|
|
len(pkgstats.pkgs_mr)) % len(pkgstats.pkgs_mr)
|
|
print_wrapped(msg)
|
|
if ua_attached:
|
|
print("\nEnable ESM Infra with: ua enable esm-infra")
|
|
|
|
if lts and not ua_attached:
|
|
print("\nThis machine is not attached to an Ubuntu Advantage "
|
|
"subscription.\nSee https://ubuntu.com/advantage")
|