File deps.py of Package base-installer

Overview Repositories Revisions Requests Users Attributes Meta

File deps.py of Package base-installer (Revision f50e075b1923b469fac76635ce9b86e9)

Currently displaying revision f50e075b1923b469fac76635ce9b86e9 , Show latest

#!/usr/bin/python

import string
import re
import glob
import sys
import getopt

def help ():
	print "Usage: ", sys.argv[0], "[opts] package..."
	print "  --help         help"
	print "  --install      compute install dependencies"
	print "  --basepacks    group basepacks"
	print "  --group PKG    group install/build set for PKG"
	print "  --ignore PKG   ignore PKG as rpm provider"
	sys.exit(0)

# command line processing
opts, args = getopt.getopt(sys.argv[1:], 'x', ['help', 'group=', 'basepacks', 'speconly', 'install', 'ignore='])
groups = set()
ignorepkgs = set()
speconly = 0
install = 0
for o, a in opts:
	if o == "--help":
		help()
	if o == "--group":
		groups.add(a)
	if o == "--basepacks":
		groups.add("BASEPACKS")
	if o == "--speconly":
		speconly = 1
	if o == "--install":
		install = 1
	if o == "--ignore":
		ignorepkgs.add(a)

if len(args) == 0:
	help()

# Read in dependency information
colonlistre = re.compile("([^:]*):(.*)")

# Set of (partially recursively resolved) build requires
# (bdeps[spec]) of a spec file, set of spec files bspecs.
bspecs = set()
bdeps = dict()
fh = open("/work/built/info/depend/x86_64.depend", 'r')
for line in fh:
	mo = colonlistre.search(line)
	if mo:
		pack = mo.group(1)
		deps = mo.group(2)
	bspecs.add(pack)
	bdeps[pack] = set()
	for dep in deps.rsplit():
		bdeps[pack].add(dep)
fh.close
# There is a set of special packages that do not get build
# dependencies resolved correctly:
#   autoconf automake binutils bzip2 db gcc gcc41 gdbm gettext glibc
#   libtool ncurses perl rpm zlib
# Fix that here by manually adding BuildRequires from them
bdeps["autoconf"].add("help2man")
bdeps["automake"].add("help2man")
bdeps["binutils"].add("dejagnu")
bdeps["binutils"].add("gcc-c++")
bdeps["gcc43"].add("bison")
bdeps["gcc43"].add("flex")
bdeps["gcc43"].add("gettext-devel")
bdeps["gcc43"].add("mpfr-devel")
bdeps["gcc43"].add("libunwind-devel")
bdeps["gcc43"].add("gcc-ada")
bdeps["libgcj43"].add("gtk2-devel")
bdeps["libgcj43"].add("libart_lgpl-devel")
bdeps["libgcj43"].add("update-desktop-files")
bdeps["gettext-runtime"].add("expat")
bdeps["glibc"].add("gcc-c++")
bdeps["ncurses"].add("ed")
bdeps["ncurses"].add("gcc-c++")
#bdeps["perl"].add("db-devel")
#bdeps["rpm-python"].add("python-devel")

# Set of rpm provides of a package (prov[pack]), set of
# packages providing a rpm provide (rprov[prov]).
prov = dict()
rprov = dict()
fh = open("/work/built/info/rpmdeps/x86_64.provides", 'r')
for line in fh:
	mo = colonlistre.search(line)
	if mo:
		pack = mo.group(1)
		deps = mo.group(2)
	else:
		continue
	if pack in ignorepkgs:
		continue
	# filter out versions
	deps = re.sub("([^ ]*) = [^ ]*", "\\1", deps)
	prov[pack] = set()
	for dep in deps.rsplit():
		prov[pack].add(dep)
		rprov.setdefault(dep, set()).add(pack)
fh.close

# Set of rpm requires for a package (req[pack]), set of
# packages rpm requiring a package (rreq[dep]).
req = dict()
rreq = dict()
fh = open("/work/built/info/rpmdeps/x86_64.requires", 'r')
for line in fh:
	mo = colonlistre.search(line)
	if mo:
		pack = mo.group(1)
		deps = mo.group(2)
	else:
		continue
	# filter out versions and file-requires
	deps = re.sub("([^ ]*) [<>=]* [^ ]*", "\\1", deps)
	deps = re.sub("/[^ ]*", "", deps)
	req[pack] = set()
	for dep in deps.rsplit():
		req[pack].add(dep)
		rreq.setdefault(dep, set()).add(pack)
fh.close
req["rpm"].add("perl")

# Set of spec files (specs), set of subpackages of
# a spec file (subpacks[spec]), spec file for package
# (rsubpack[pack]).
specs = set()
subpacks = dict()
rsubpack = dict()
fh = open("/work/built/info/subpacks/x86_64", 'r')
for line in fh:
	mo = colonlistre.search(line)
	if mo:
		pack = mo.group(1)
		deps = mo.group(2)
	else:
		continue
	specs.add(pack)
	rsubpack[pack] = pack
	for dep in deps.rsplit():
		rsubpack[dep] = pack
		rsubpack[dep + "-32bit"] = pack
		rsubpack[dep + "-64bit"] = pack
		subpacks.setdefault(pack, set()).add(dep)
fh.close
# some missing magic stuff
rsubpack["dev86"] = "glibc"
rsubpack["bin86"] = "glibc"

# The set of basepacks (minus convenience)
basepacks = set()
# /work/abuild/lib/abuild/init_buildsystem/glibc/...
# .../base-files
for dep in "aaa_base acl attr bash bzip2 coreutils libdb-4_5 diffutils filesystem fillup glibc grep insserv libacl libattr libgcc41 libnscd libxcrypt m4 ncurses pam permissions popt readline rpm sed tar zlib".rsplit():
	basepacks.add(dep)
# .../compiler-default
basepacks.add("gcc")
# .../files:
for dep in "audit-libs bison cpio cpp cpp41 cracklib file findutils flex gawk gettext-devel glibc-devel glibc-locale groff gzip info libltdl libmudflap41 libstdc++41 libzio linux-kernel-headers make man mktemp net-tools netcfg pam-modules patch sysvinit texinfo timezone util-linux".rsplit():
	basepacks.add(dep)
# .../libs:
for dep in "autoconf automake binutils bzip2 libdb-4_5 gcc gcc41 gdbm gettext glibc libtool ncurses perl rpm zlib".rsplit():
	basepacks.add(dep)
# remove what is convenience only:
#for dep in "vim".rsplit():
#	basepacks.remove(dep)

# Candidates for removal:
#  procinfo, bind-utils, bind-libs, klogd, cvs, module-init-tools, gpm, gawk
# Convenience:
#  man, less, vim, strace, psmisc

# Helpers for solver functions.
def build_dependencies (pack, ignoreset):
  deps = set()
  # add build-requires of PACKs spec file to dependencies
  if not pack in rsubpack:
	return set()
  pkg = rsubpack[pack]
  if not pkg in bdeps:
	return set()
  if pkg in ignoreset:
	return bdeps[pkg].difference(ignoreset[pkg])
  return bdeps[pkg]

def install_dependencies (pack, ignoreset, avail):
  deps = set()
  if not pack in req:
	return set()
  if pack in ignoreset:
	ignores = ignoreset[pack]
  else:
	ignores = set()
  # for each rpm requires, check what packages provide it and add
  # the first, if another provider is not already in AVAIL or DEPS
  for r in req[pack]:
	if r in rprov and len(deps.union(avail).intersection(rprov[r])) == 0:
	  if len(rprov[r].difference(ignores)) > 0:
		deps.add (rprov[r].difference(ignores).copy().pop())
  return deps

# The actual solvers.

def compute_rec_build_set (packs, ignoreset):
  """Compute recursively the set of packages to build and install if packages in PACKS are supposed to be built"""
  # start iteration with packs U basepacks, because basepacks
  # are implicit build dependencies of packs
  bdeps = set()
  ideps = set()
  bworklist = packs.copy()
  iworklist = basepacks.copy()
  while len(bworklist) > 0 or len(iworklist) > 0:
    if len(bworklist) > 0:
      tobuild = bworklist.pop()
      iworklist = iworklist.union (build_dependencies (tobuild, ignoreset).difference (ideps))
      bdeps.add (tobuild)
    if len(iworklist) > 0:
      toinstall = iworklist.pop()
      if toinstall in rsubpack:
        tobuild = rsubpack[toinstall]
      if not tobuild in bdeps:
	bworklist.add(tobuild)
      iworklist = iworklist.union (install_dependencies (toinstall, ignoreset, ideps).difference (ideps))
      ideps.add (toinstall)
  return ( bdeps, ideps )

def compute_rec_install_set (packs, ignoreset):
  """Compute recursively which packages are required to be installed if packages in PACKS are supposed to be installed"""
  ideps = set()
  iworklist = packs.copy()
  while len(iworklist) > 0:
    if len(iworklist) > 0:
      toinstall = iworklist.pop()
      iworklist = iworklist.union (install_dependencies (toinstall, ignoreset, ideps).difference (ideps))
      ideps.add (toinstall)
  return ( set(), ideps )

def compute_build_set (packs, ignoreset):
  """Compute which packages are required to be installed for building packages in PACKS"""
  # start iteration with packs U basepacks, because basepacks
  # are implicit build dependencies of packs
  bdeps = set()
  ideps = set()
  bworklist = packs.copy()
  iworklist = basepacks.copy()
  while len(bworklist) > 0 or len(iworklist) > 0:
    if len(bworklist) > 0:
      tobuild = bworklist.pop()
      iworklist = iworklist.union (build_dependencies (tobuild, ignoreset).difference (ideps))
      bdeps.add (tobuild)
    if len(iworklist) > 0:
      toinstall = iworklist.pop()
      iworklist = iworklist.union (install_dependencies (toinstall, ignoreset, ideps).difference (ideps))
      ideps.add (toinstall)
  return ( bdeps, ideps )

# Add build-dependencies that get stripped because of autobuild
#bdeps["libtool"].add("gcc-java")
#bdeps["gettext"].add("gcc-java")
# For walking the chain ignore the following edges
ignoreset = dict()
#ignoreset.setdefault("gettext", set()).add("libgcj41")
#ignoreset.setdefault("gettext-devel", set()).add("libgcj41")
#ignoreset.setdefault("gettext-devel", set()).add("libgcj.so.7")

# basepacks dependencies
basedeps = compute_rec_build_set (basepacks, ignoreset)
if install:
	basedeps = basedeps[1]
else:
	basedeps = basedeps[0].union(basedeps[1])
print "# basepacks: ", len(basepacks), basepacks
print "# basedeps: ", len(basedeps), basedeps

# groups requested
depslist = list()
for group in groups:
	if group == "BASEPACKS":
		depslist.append ( (group, basedeps) )
	else:
		d = set()
		d.add(group)
		if install:
			deps = compute_rec_install_set (d, ignoreset)[1]
		else:
			deps = compute_rec_build_set (d, ignoreset)
			deps = deps[0].union(deps[1])
		depslist.append ( (group, deps) )
		print "# ", group, " deps: ", len(deps), deps

# individual packages
deps = set()
for arg in args:
	d = set()
	d.add(arg)
	if install:
		d = compute_rec_install_set (d, ignoreset)
		d = d[1]
	else:
		d = compute_rec_build_set (d, ignoreset)
		d = d[0].union(d[1])
	print "# ", arg, " deps: ", len(d), d
	deps = deps.union(d)

print "# overall deps (w/o basedeps): ", len(deps.difference(basedeps)), deps.difference(basedeps)
print "# ", len(deps), " packages (w/ basedeps)", len(deps.difference(basedeps)), " (w/o)"
if not install:
	print "# ", len(deps.intersection(specs)), " source packages (w/ basedeps)", len(deps.difference(basedeps).intersection(specs)), " (w/o)"

# Graph printing stuff

def reduce_to_base(edge, base, label):
	from_ = edge[0]
	to = edge[1]
	if speconly:
		if from_ in rsubpack:
			from_ = rsubpack[from_]
		if to in rsubpack:
			to = rsubpack[to]
	if from_ in base:
		from_ = label
	if to in base:
		to = label
	return (from_, to)

def reduce_edge (edge, basesets):
	for label, base in basesets:
		edge = reduce_to_base (edge, base, label)
	return edge

def print_full():
	# Construct the set of edges that represent build dependencies
	# of the main packages in the dependency set
	bdep_edges = set()
	if not install:
	  for dep in deps.intersection(bdeps):
		for bdep in deps.intersection(bdeps[dep]):
			bdep_edges.add( (dep, bdep) )

# Construct the set of edges that represent install dependenices
	# of the packages
	idep_edges = set()
	for dep in deps:
		for idep in req[dep]:
			if idep in rprov:
				idep_edges.add( (dep, rprov[idep].copy().pop()) )

# Edge set representing sub-package -> package build triggers
	subp_edges = set()
	if not install:
	  for dep in deps.difference(bdeps):
		subp_edges.add ( (dep, rsubpack[dep]) )

# Output a dot graph file
	print "digraph dependencies {"
	print "node [shape=box]"
	for dep in basepacks:
		if dep in deps:
			print "\"%s\" [color=green];" % dep
	for edge in bdep_edges:
		print "\"%s\" -> \"%s\"" % (edge[0], edge[1])
	for edge in subp_edges.difference(bdep_edges):
		print "\"%s\" -> \"%s\" [style=dotted];" % (edge[0], edge[1])
	for edge in idep_edges.difference(bdep_edges).difference(subp_edges):
		print "\"%s\" -> \"%s\" [style=dashed];" % (edge[0], edge[1])
	print "}"

def print_reduced(redsets):
	# Construct the set of edges that represent build dependencies
	# of the main packages in the dependency set
	bdep_edges = set()
	if not install:
	  for dep in deps.intersection(bdeps):
		for bdep in deps.intersection(bdeps[dep]):
			edge = reduce_edge ( (dep, bdep), redsets)
			if edge[0] != edge[1]:
				bdep_edges.add(edge)
	# Install dependency edges
	idep_edges = set()
	for dep in deps:
	    if dep in req:
		for idep in req[dep]:
			if idep in rprov:
				to = rprov[idep].copy().pop()
				edge = reduce_edge( (dep, to), redsets)
				if edge[0] != edge[1]:
					idep_edges.add(edge)
	print "digraph dependencies {"
	print "node [shape=box]"
	for arg in args:
		print "\"%s\" [color=red];" % arg
	for label, rset in redsets:
		print "\"%s\" [color=blue];" % label
	for edge in bdep_edges:
		print "\"%s\" -> \"%s\"" % (edge[0], edge[1])
	for edge in idep_edges:
		if not edge in bdep_edges:
			print "\"%s\" -> \"%s\" [style=dotted];" % (edge[0], edge[1])
	print "}"

# Generate and print the graph
if len(depslist) == 0:
	print_full()
else:
	print_reduced(depslist)

Places

File deps.py of Package base-installer (Revision f50e075b1923b469fac76635ce9b86e9)

Places