File Manager

Editing File: /usr/lib64/python2.7/site-packages/reportclient/debuginfo.py

"""
    This module provides classes and functions used to download and manage
    debuginfos.
"""

import sys
import os
import time
import errno
import shutil
from subprocess import Popen

from yum import _, YumBase
from yum.callbacks import DownloadBaseCallback
from yum.Errors import YumBaseError

from reportclient import (_, log1, log2, RETURN_OK, RETURN_FAILURE,
                          RETURN_CANCEL_BY_USER, verbose, ask_yes_no,
                          error_msg)

TMPDIR=""

def ensure_abrt_uid(fn):
    """
    Ensures that the function is called using abrt's uid and gid

Returns:
        Either an unchanged function object or a wrapper function object for
        the function.
    """

import pwd
    current_uid = os.getuid()
    current_gid = os.getgid()
    abrt = pwd.getpwnam("abrt")

# if we're are already running as abrt, don't do anything
    if abrt.pw_uid == current_uid and abrt.pw_gid == current_gid:
        return fn

def wrapped(*args, **kwargs):
        """
        Wrapper function around the called function.

Sets up uid and gid to match abrt's and after the function finishes
        rolls its uid and gid back.

Returns:
            Return value of the wrapped function.
        """

# switch to abrt
        os.setegid(abrt.pw_gid)
        os.seteuid(abrt.pw_uid)
        # extract the files as abrt:abrt
        retval = fn(*args, **kwargs)
        # switch back to whatever we were
        os.seteuid(current_uid)
        os.setegid(current_gid)
        return retval

return wrapped

# TODO: unpack just required debuginfo and not entire rpm?
# ..that can lead to: foo.c No such file and directory
# files is not used...
@ensure_abrt_uid
def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm, exact_files=False):
    """
    Unpacks a single rpm located in tmp_dir into destdir.

Arguments:
        package_file_name - name of the rpm file
        files - files to extract from the rpm
        tmp_dir - temporary directory where the rpm file is located
        destdir - destination directory for the rpm package extraction
        keeprpm - check if the user wants to delete rpms from the tmp directory
        exact_files - extract only specified files

Returns:
        RETURN_FAILURE in case of a serious problem
    """

package_full_path = tmp_dir + "/" + package_file_name
    log1("Extracting %s to %s", package_full_path, destdir)
    log2("%s", files)
    print _("Extracting cpio from {0}").format(package_full_path)
    unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
    try:
        unpacked_cpio = open(unpacked_cpio_path, 'wb')
    except IOError, ex:
        print _("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex)
        return RETURN_FAILURE

rpm2cpio = Popen(["rpm2cpio", package_full_path],
                       stdout = unpacked_cpio, bufsize = -1)
    retcode = rpm2cpio.wait()

if retcode == 0:
        log1("cpio written OK")
        if not keeprpm:
            log1("keeprpms = False, removing %s", package_full_path)
            #print _("Removing temporary rpm file")
            os.unlink(package_full_path)
    else:
        unpacked_cpio.close()
        print _("Can't extract package '{0}'").format(package_full_path)
        return RETURN_FAILURE

# close the file
    unpacked_cpio.close()
    # and open it for reading
    unpacked_cpio = open(unpacked_cpio_path, 'rb')

print _("Caching files from {0} made from {1}").format("unpacked.cpio", package_file_name)

file_patterns = ""
    cpio_args = ["cpio", "-idu"]
    if exact_files:
        for filename in files:
            file_patterns += "." + filename + " "
        cpio_args = ["cpio", "-idu", file_patterns.strip()]

with open("/dev/null", "w") as null:
        cpio = Popen(cpio_args, cwd=destdir, bufsize=-1,
                     stdin=unpacked_cpio, stdout=null, stderr=null)
        retcode = cpio.wait()

if retcode == 0:
        log1("files extracted OK")
        #print _("Removing temporary cpio file")
        os.unlink(unpacked_cpio_path)
    else:
        print _("Can't extract files from '{0}'").format(unpacked_cpio_path)
        return RETURN_FAILURE

def clean_up():
    """
    Removes the temporary directory.
    """

if TMPDIR:
        try:
            shutil.rmtree(TMPDIR)
        except OSError, ex:
            if ex.errno != errno.ENOENT:
                error_msg(_("Can't remove '{0}': {1}").format(TMPDIR, ex))

class MyDownloadCallback(DownloadBaseCallback):
    """
    This class serves as a download progress handler for yum's progress bar.
    """

def __init__(self, total_pkgs):
        """
        Sets up instance variables

Arguments:
            total_pkgs - number of packages to download
        """

self.total_pkgs = total_pkgs
        self.downloaded_pkgs = 0
        self.last_pct = 0
        self.last_time = 0
        DownloadBaseCallback.__init__(self)

def updateProgress(self, name, frac, fread, ftime):
        """
        A method used to update the progress

Arguments:
            name - filename
            frac - progress fracment (0 -> 1)
            fread - formated string containing BytesRead
            ftime - formated string containing remaining or elapsed time
        """

pct = int(frac * 100)
        if pct == self.last_pct:
            log2("percentage is the same, not updating progress")
            return

self.last_pct = pct
        # if run from terminal we can have fancy output
        if sys.stdout.isatty():
            sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u"
                    % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
            )
            if pct == 100:
                #print (_("Downloading (%i of %i) %s: %3u%%")
                #        % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
                #)
                print (_("Downloading ({0} of {1}) {2}: {3:3}%").format(
                        self.downloaded_pkgs + 1, self.total_pkgs, name, pct
                        )
                )
        # but we want machine friendly output when spawned from abrt-server
        else:
            t = time.time()
            if self.last_time == 0:
                self.last_time = t
            # update only every 5 seconds
            if pct == 100 or self.last_time > t or t - self.last_time >= 5:
                print (_("Downloading ({0} of {1}) {2}: {3:3}%").format(
                        self.downloaded_pkgs + 1, self.total_pkgs, name, pct
                        )
                )
                self.last_time = t
                if pct == 100:
                    self.last_time = 0

sys.stdout.flush()

def downloadErrorCallback(callBackObj):
    """
    A callback function for mirror errors.
    """

print _("Problem '{0!s}' occured while downloading from mirror: '{1!s}'. Trying next one").format(
        callBackObj.exception, callBackObj.mirror)
    # explanation of the return value can be found here:
    # /usr/lib/python2.7/site-packages/urlgrabber/mirror.py
    return {'fail':0}

class DebugInfoDownload(YumBase):
    """
    This class is used to manage download of debuginfos.
    """

def __init__(self, cache, tmp, repo_pattern="*debug*", keep_rpms=False,
                 noninteractive=True):
        self.old_stdout = -1
        self.cachedir = cache
        self.tmpdir = tmp
        global TMPDIR
        TMPDIR = tmp
        self.keeprpms = keep_rpms
        self.noninteractive = noninteractive
        self.repo_pattern=repo_pattern
        YumBase.__init__(self)
        self.mute_stdout()
        #self.conf.cache = os.geteuid() != 0
        # Setup yum (Ts, RPM db, Repo & Sack)
        # doConfigSetup() takes some time, let user know what we are doing
        print _("Initializing yum")
        try:
            # Saw this exception here:
            # cannot open Packages index using db3 - Permission denied (13)
            # yum.Errors.YumBaseError: Error: rpmdb open failed
            self.doConfigSetup()
        except YumBaseError, ex:
            self.unmute_stdout()
            print _("Error initializing yum (YumBase.doConfigSetup): '{0!s}'").format(ex)
            #return 1 - can't do this in constructor
            exit(1)
        self.unmute_stdout()

def mute_stdout(self):
        """
        Links sys.stdout with /dev/null and saves the old stdout
        """

if verbose < 2:
            self.old_stdout = sys.stdout
            sys.stdout = open("/dev/null", "w")

def unmute_stdout(self):
        """
        Replaces sys.stdout by stdout saved using mute
        """

if verbose < 2:
            if self.old_stdout != -1:
                sys.stdout = self.old_stdout
            else:
                print "ERR: unmute called without mute?"

@ensure_abrt_uid
    def setup_tmp_dirs(self):
        if not os.path.exists(self.tmpdir):
            try:
                os.makedirs(self.tmpdir)
            except OSError, ex:
                print "Can't create tmpdir: %s" % ex
                return RETURN_FAILURE
        if not os.path.exists(self.cachedir):
            try:
                os.makedirs(self.cachedir)
            except OSError, ex:
                print "Can't create cachedir: %s" % ex
                return RETURN_FAILURE

return RETURN_OK

# return value will be used as exitcode. So 0 = ok, !0 - error
    def download(self, files, download_exact_files=False):
        """
        Downloads rpms into a temporary directory

Arguments:
            package_files_dict - a dict containing {pkg: file list} entries
            total_pkgs - total number of packages to download
            download_exact_files - extract only specified files

Returns:
            RETURN_OK if all goes well.
            RETURN_FAILURE in case it cannot set up either of the directories.
        """

installed_size = 0
        total_pkgs = 0
        todownload_size = 0
        downloaded_pkgs = 0
        # nothing to download?
        if not files:
            return RETURN_FAILURE

#if verbose == 0:
        #    # this suppress yum messages about setting up repositories
        #    mute_stdout()

# make yumdownloader work as non root user
        if not self.setCacheDir():
            print _("Error: can't make cachedir, exiting")
            return RETURN_FAILURE

# disable all not needed
        for repo in self.repos.listEnabled():
            try:
                repo.close()
                self.repos.disableRepo(repo.id)
            except YumBaseError, ex:
                print _("Can't disable repository '{0!s}': {1!s}").format(repo.id, str(ex))

# This takes some time, let user know what we are doing
        print _("Setting up yum repositories")
        # setting-up repos one-by-one, so we can skip the broken ones...
        # this helps when users are using 3rd party repos like rpmfusion
        # in rawhide it results in: Can't find valid base url...
        for r in self.repos.findRepos(pattern=self.repo_pattern):
            try:
                rid = self.repos.enableRepo(r.id)
                self.repos.doSetup(thisrepo=str(r.id))
                log1("enabled repo %s", rid)
                setattr(r, "skip_if_unavailable", True)
                # yes, we want async download, otherwise our progressCallback
                # is not called and the internal yum's one  is used,
                # which causes artifacts on output
                try:
                    setattr(r, "_async", False)
                except (NameError, AttributeError), ex:
                    print ex
                    print _("Can't disable async download, the output might contain artifacts!")
            except YumBaseError, ex:
                print _("Can't setup {0}: {1}, disabling").format(r.id, ex)
                self.repos.disableRepo(r.id)

# This is somewhat "magic", it unpacks the metadata making it usable.
        # Looks like this is the moment when yum talks to remote servers,
        # which takes time (sometimes minutes), let user know why
        # we have "paused":
        print _("Looking for needed packages in repositories")
        try:
            self.repos.populateSack(mdtype='metadata', cacheonly=1)
        except YumBaseError, ex:
            print _("Error retrieving metadata: '{0!s}'").format(ex)
            #we don't want to die here, some metadata might be already retrieved
            # so there is a chance we already have what we need
            #return 1

try:
            # Saw this exception here:
            # raise Errors.NoMoreMirrorsRepoError, errstr
            # NoMoreMirrorsRepoError: failure:
            # repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
            # from fedora-debuginfo: [Errno 256] No more mirrors to try.
            self.repos.populateSack(mdtype='filelists', cacheonly=1)
        except YumBaseError, ex:
            print _("Error retrieving filelists: '{0!s}'").format(ex)
            # we don't want to die here, some repos might be already processed
            # so there is a chance we already have what we need
            #return 1

#if verbose == 0:
        #    # re-enable the output to stdout
        #    unmute_stdout()

not_found = []
        package_files_dict = {}
        for debuginfo_path in files:
            log2("yum whatprovides %s", debuginfo_path)
            pkg = self.pkgSack.searchFiles(debuginfo_path)
            # sometimes one file is provided by more rpms, we can use either of
            # them, so let's use the first match
            if pkg:
                if pkg[0] in package_files_dict.keys():
                    package_files_dict[pkg[0]].append(debuginfo_path)
                else:
                    package_files_dict[pkg[0]] = [debuginfo_path]
                    todownload_size += float(pkg[0].size)
                    installed_size += float(pkg[0].installedsize)
                    total_pkgs += 1

log2("found pkg for %s: %s", debuginfo_path, pkg[0])
            else:
                log2("not found pkg for %s", debuginfo_path)
                not_found.append(debuginfo_path)

# connect our progress update callback
        dnlcb = MyDownloadCallback(total_pkgs)
        self.repos.setProgressBar(dnlcb)
        self.repos.setMirrorFailureCallback(downloadErrorCallback)

if verbose != 0 or len(not_found) != 0:
            print _("Can't find packages for {0} debuginfo files").format(len(not_found))
        if verbose != 0 or total_pkgs != 0:
            print _("Packages to download: {0}").format(total_pkgs)
            question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format(
                         todownload_size / (1024*1024),
                         installed_size / (1024*1024)
                        )
            if self.noninteractive == False and not ask_yes_no(question):
                print _("Download cancelled by user")
                return RETURN_CANCEL_BY_USER
            # set up tmp and cache dirs so that we can check free space in both
            retval = self.setup_tmp_dirs()
            if retval != RETURN_OK:
                return retval
            # check if there is enough free space in both tmp and cache
            res = os.statvfs(self.tmpdir)
            tmp_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
            if (todownload_size / (1024*1024)) > tmp_space:
                question = _("Warning: Not enough free space in tmp dir '{0}'"
                             " ({1:.2f}Mb left). Continue?").format(
                    self.tmpdir, tmp_space)
                if not self.noninteractive and not ask_yes_no(question):
                    print _("Download cancelled by user")
                    return RETURN_CANCEL_BY_USER
            res = os.statvfs(self.cachedir)
            cache_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
            if (installed_size / (1024*1024)) > cache_space:
                question = _("Warning: Not enough free space in cache dir "
                             "'{0}' ({1:.2f}Mb left). Continue?").format(
                    self.cachedir, cache_space)
                if not self.noninteractive and not ask_yes_no(question):
                    print _("Download cancelled by user")
                    return RETURN_CANCEL_BY_USER

for pkg, files in package_files_dict.iteritems():
            dnlcb.downloaded_pkgs = downloaded_pkgs
            repo.cache = 0
            remote = pkg.returnSimple('relativepath')
            local = os.path.basename(remote)
            retval = self.setup_tmp_dirs()
            # continue only if the tmp dirs are ok
            if retval != RETURN_OK:
                return retval

remote_path = pkg.returnSimple('remote_url')
            # check if the pkg is in a local repo and copy it if it is
            err = None
            if remote_path.startswith('file:///'):
                pkg_path = remote_path[7:]
                log2("copying from local repo: %s", remote)
                try:
                    shutil.copy(pkg_path, self.tmpdir)
                except OSError, ex:
                    print _("Cannot copy file '{0}': {1}").format(pkg_path, ex)
                    continue
            else:
                # pkg is in a remote repo, we need to download it to tmpdir
                local = os.path.join(self.tmpdir, local)
                pkg.localpath = local # Hack: to set the localpath we want
                err = self.downloadPkgs(pkglist=[pkg])
            # normalize the name
            # just str(pkg) doesn't work because it can have epoch
            pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch
            package_file_name = pkg_nvra + ".rpm"
            if err:
                # I observed a zero-length file left on error,
                # which prevents cleanup later. Fix it:
                try:
                    os.unlink(self.tmpdir + "/" + package_file_name)
                except OSError:
                    pass
                print (_("Downloading package {0} failed").format(pkg))
            else:
                unpack_result = unpack_rpm(package_file_name, files, self.tmpdir,
                                           self.cachedir, self.keeprpms,
                                           exact_files=download_exact_files)
                if unpack_result == RETURN_FAILURE:
                    # recursively delete the temp dir on failure
                    print _("Unpacking failed, aborting download...")
                    clean_up()
                    return RETURN_FAILURE

downloaded_pkgs += 1

if not self.keeprpms and os.path.exists(self.tmpdir):
            # Was: "All downloaded packages have been extracted, removing..."
            # but it was appearing even if no packages were in fact extracted
            # (say, when there was one package, and it has download error).
            print (_("Removing {0}").format(self.tmpdir))
            try:
                os.rmdir(self.tmpdir)
            except OSError:
                error_msg(_("Can't remove %s, probably contains an error log").format(self.tmpdir))

return RETURN_OK

def build_ids_to_path(pfx, build_ids):
    """
    Transforms build ids into a path.

build_id1=${build_id:0:2}
    build_id2=${build_id:2}
    file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug"
    """

return ["%s/usr/lib/debug/.build-id/%s/%s.debug" % (pfx, b_id[:2], b_id[2:]) for b_id in build_ids]

# beware this finds only missing libraries, but not the executable itself ..

def filter_installed_debuginfos(build_ids, cache_dirs):
    """
    Checks for installed debuginfos.

Arguments:
        build_ids - string containing build ids
        cache_dirs - list of cache directories

Returns:
        List of missing debuginfo files.
    """

files = build_ids_to_path("", build_ids)
    missing = []

# 1st pass -> search in /usr/lib
    for debuginfo_path in files:
        log2("looking: %s", debuginfo_path)
        if os.path.exists(debuginfo_path):
            log2("found: %s", debuginfo_path)
            continue
        log2("not found: %s", debuginfo_path)
        missing.append(debuginfo_path)

if missing:
        files = missing
        missing = []
    else: # nothing is missing, we can stop looking
        return missing

for cache_dir in cache_dirs:
        log2("looking in %s" % cache_dir)
        for debuginfo_path in files:
            cache_debuginfo_path = cache_dir + debuginfo_path
            log2("looking: %s", cache_debuginfo_path)
            if os.path.exists(cache_debuginfo_path):
                log2("found: %s", cache_debuginfo_path)
                continue
            log2("not found: %s", debuginfo_path)
            missing.append(debuginfo_path)
        # in next iteration look only for files missing
        # from previous iterations
        if missing:
            files = missing
            missing = []
        else: # nothing is missing, we can stop looking
            return missing

return files

Back to Directory File Manager