| import os, logging, datetime, glob, shutil |
| from autotest_lib.client.bin import utils, os_dep |
| from autotest_lib.client.common_lib import error |
| import virt_utils, virt_installer |
| |
| |
| def kill_qemu_processes(): |
| """ |
| Kills all qemu processes, also kills all processes holding /dev/kvm down. |
| """ |
| logging.debug("Killing any qemu processes that might be left behind") |
| utils.system("pkill qemu", ignore_status=True) |
| # Let's double check to see if some other process is holding /dev/kvm |
| if os.path.isfile("/dev/kvm"): |
| utils.system("fuser -k /dev/kvm", ignore_status=True) |
| |
| |
| def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None): |
| """ |
| Create symbolic links for the appropriate qemu and qemu-img commands on |
| the kvm test bindir. |
| |
| @param test_bindir: KVM test bindir |
| @param prefix: KVM prefix path |
| @param bin_list: List of qemu binaries to link |
| @param unittest: Path to configuration file unittests.cfg |
| """ |
| qemu_path = os.path.join(test_bindir, "qemu") |
| qemu_img_path = os.path.join(test_bindir, "qemu-img") |
| qemu_unittest_path = os.path.join(test_bindir, "unittests") |
| if os.path.lexists(qemu_path): |
| os.unlink(qemu_path) |
| if os.path.lexists(qemu_img_path): |
| os.unlink(qemu_img_path) |
| if unittest and os.path.lexists(qemu_unittest_path): |
| os.unlink(qemu_unittest_path) |
| |
| logging.debug("Linking qemu binaries") |
| |
| if bin_list: |
| for bin in bin_list: |
| if os.path.basename(bin) == 'qemu-kvm': |
| os.symlink(bin, qemu_path) |
| elif os.path.basename(bin) == 'qemu-img': |
| os.symlink(bin, qemu_img_path) |
| |
| elif prefix: |
| kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64") |
| if not os.path.isfile(kvm_qemu): |
| raise error.TestError('Invalid qemu path') |
| kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img") |
| if not os.path.isfile(kvm_qemu_img): |
| raise error.TestError('Invalid qemu-img path') |
| os.symlink(kvm_qemu, qemu_path) |
| os.symlink(kvm_qemu_img, qemu_img_path) |
| |
| if unittest: |
| logging.debug("Linking unittest dir") |
| os.symlink(unittest, qemu_unittest_path) |
| |
| |
| def install_roms(rom_dir, prefix): |
| logging.debug("Path to roms specified. Copying roms to install prefix") |
| rom_dst_dir = os.path.join(prefix, 'share', 'qemu') |
| for rom_src in glob.glob('%s/*.bin' % rom_dir): |
| rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src)) |
| logging.debug("Copying rom file %s to %s", rom_src, rom_dst) |
| shutil.copy(rom_src, rom_dst) |
| |
| |
| class KvmInstallException(Exception): |
| pass |
| |
| |
| class FailedKvmInstall(KvmInstallException): |
| pass |
| |
| |
| class KvmNotInstalled(KvmInstallException): |
| pass |
| |
| |
| class BaseInstaller(object): |
| def __init__(self, mode=None): |
| self.install_mode = mode |
| self._full_module_list = None |
| |
| def set_install_params(self, test, params): |
| self.params = params |
| |
| load_modules = params.get('load_modules', 'no') |
| if not load_modules or load_modules == 'yes': |
| self.should_load_modules = True |
| elif load_modules == 'no': |
| self.should_load_modules = False |
| default_extra_modules = str(None) |
| self.extra_modules = eval(params.get("extra_modules", |
| default_extra_modules)) |
| |
| self.cpu_vendor = virt_utils.get_cpu_vendor() |
| |
| self.srcdir = test.srcdir |
| if not os.path.isdir(self.srcdir): |
| os.makedirs(self.srcdir) |
| |
| self.test_bindir = test.bindir |
| self.results_dir = test.resultsdir |
| |
| # KVM build prefix, for the modes that do need it |
| prefix = os.path.join(test.bindir, 'build') |
| self.prefix = os.path.abspath(prefix) |
| |
| # Current host kernel directory |
| default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2] |
| self.host_kernel_srcdir = params.get('host_kernel_source', |
| default_host_kernel_source) |
| |
| # Extra parameters that can be passed to the configure script |
| self.extra_configure_options = params.get('extra_configure_options', |
| None) |
| |
| # Do we want to save the result of the build on test.resultsdir? |
| self.save_results = True |
| save_results = params.get('save_results', 'no') |
| if save_results == 'no': |
| self.save_results = False |
| |
| self._full_module_list = list(self._module_list()) |
| |
| |
| def install_unittests(self): |
| userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace") |
| test_repo = self.params.get("test_git_repo") |
| test_branch = self.params.get("test_branch", "master") |
| test_commit = self.params.get("test_commit", None) |
| test_lbranch = self.params.get("test_lbranch", "master") |
| |
| if test_repo: |
| test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests") |
| virt_utils.get_git_branch(test_repo, test_branch, test_srcdir, |
| test_commit, test_lbranch) |
| unittest_cfg = os.path.join(test_srcdir, 'x86', |
| 'unittests.cfg') |
| self.test_srcdir = test_srcdir |
| else: |
| unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86', |
| 'unittests.cfg') |
| self.unittest_cfg = None |
| if os.path.isfile(unittest_cfg): |
| self.unittest_cfg = unittest_cfg |
| else: |
| if test_repo: |
| logging.error("No unittest config file %s found, skipping " |
| "unittest build", self.unittest_cfg) |
| |
| self.unittest_prefix = None |
| if self.unittest_cfg: |
| logging.info("Building and installing unittests") |
| os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg))) |
| utils.system('./configure --prefix=%s' % self.prefix) |
| utils.system('make') |
| utils.system('make install') |
| self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu', |
| 'tests') |
| |
| |
| def full_module_list(self): |
| """Return the module list used by the installer |
| |
| Used by the module_probe test, to avoid using utils.unload_module(). |
| """ |
| if self._full_module_list is None: |
| raise KvmNotInstalled("KVM modules not installed yet (installer: %s)" % (type(self))) |
| return self._full_module_list |
| |
| |
| def _module_list(self): |
| """Generate the list of modules that need to be loaded |
| """ |
| yield 'kvm' |
| yield 'kvm-%s' % (self.cpu_vendor) |
| if self.extra_modules: |
| for module in self.extra_modules: |
| yield module |
| |
| |
| def _load_modules(self, mod_list): |
| """ |
| Load the KVM modules |
| |
| May be overridden by subclasses. |
| """ |
| logging.info("Loading KVM modules") |
| for module in mod_list: |
| utils.system("modprobe %s" % module) |
| |
| |
| def load_modules(self, mod_list=None): |
| if mod_list is None: |
| mod_list = self.full_module_list() |
| self._load_modules(mod_list) |
| |
| |
| def _unload_modules(self, mod_list=None): |
| """ |
| Just unload the KVM modules, without trying to kill Qemu |
| """ |
| if mod_list is None: |
| mod_list = self.full_module_list() |
| logging.info("Unloading previously loaded KVM modules") |
| for module in reversed(mod_list): |
| utils.unload_module(module) |
| |
| |
| def unload_modules(self, mod_list=None): |
| """ |
| Kill Qemu and unload the KVM modules |
| """ |
| kill_qemu_processes() |
| self._unload_modules(mod_list) |
| |
| |
| def reload_modules(self): |
| """ |
| Reload the KVM modules after killing Qemu and unloading the current modules |
| """ |
| self.unload_modules() |
| self.load_modules() |
| |
| |
| def reload_modules_if_needed(self): |
| if self.should_load_modules: |
| self.reload_modules() |
| |
| |
| class YumInstaller(BaseInstaller): |
| """ |
| Class that uses yum to install and remove packages. |
| """ |
| def set_install_params(self, test, params): |
| super(YumInstaller, self).set_install_params(test, params) |
| # Checking if all required dependencies are available |
| os_dep.command("rpm") |
| os_dep.command("yum") |
| |
| default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools']) |
| default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img']) |
| default_pkg_path_list = str(None) |
| self.pkg_list = eval(params.get("pkg_list", default_pkg_list)) |
| self.pkg_path_list = eval(params.get("pkg_path_list", |
| default_pkg_path_list)) |
| self.qemu_bin_paths = eval(params.get("qemu_bin_paths", |
| default_qemu_bin_paths)) |
| |
| |
| def _clean_previous_installs(self): |
| kill_qemu_processes() |
| removable_packages = "" |
| for pkg in self.pkg_list: |
| removable_packages += " %s" % pkg |
| |
| utils.system("yum remove -y %s" % removable_packages) |
| |
| |
| def _get_packages(self): |
| for pkg in self.pkg_path_list: |
| utils.get_file(pkg, os.path.join(self.srcdir, |
| os.path.basename(pkg))) |
| |
| |
| def _install_packages(self): |
| """ |
| Install all downloaded packages. |
| """ |
| os.chdir(self.srcdir) |
| utils.system("yum install --nogpgcheck -y *.rpm") |
| |
| |
| def install(self): |
| self.install_unittests() |
| self._clean_previous_installs() |
| self._get_packages() |
| self._install_packages() |
| create_symlinks(test_bindir=self.test_bindir, |
| bin_list=self.qemu_bin_paths, |
| unittest=self.unittest_prefix) |
| self.reload_modules_if_needed() |
| if self.save_results: |
| virt_utils.archive_as_tarball(self.srcdir, self.results_dir) |
| |
| |
| class KojiInstaller(YumInstaller): |
| """ |
| Class that handles installing KVM from the fedora build service, koji. |
| |
| It uses yum to install and remove packages. Packages are specified |
| according to the syntax defined in the PkgSpec class. |
| """ |
| def set_install_params(self, test, params): |
| """ |
| Gets parameters and initializes the package downloader. |
| |
| @param test: kvm test object |
| @param params: Dictionary with test arguments |
| """ |
| super(KojiInstaller, self).set_install_params(test, params) |
| self.tag = params.get("koji_tag", None) |
| self.koji_cmd = params.get("koji_cmd", None) |
| if self.tag is not None: |
| virt_utils.set_default_koji_tag(self.tag) |
| self.koji_pkgs = eval(params.get("koji_pkgs", "[]")) |
| |
| |
| def _get_packages(self): |
| """ |
| Downloads the specific arch RPMs for the specific build name. |
| """ |
| koji_client = virt_utils.KojiClient(cmd=self.koji_cmd) |
| for pkg_text in self.koji_pkgs: |
| pkg = virt_utils.KojiPkgSpec(pkg_text) |
| if pkg.is_valid(): |
| koji_client.get_pkgs(pkg, dst_dir=self.srcdir) |
| else: |
| logging.error('Package specification (%s) is invalid: %s', pkg, |
| pkg.describe_invalid()) |
| |
| |
| def _clean_previous_installs(self): |
| kill_qemu_processes() |
| removable_packages = " ".join(self._get_rpm_names()) |
| utils.system("yum -y remove %s" % removable_packages) |
| |
| |
| def install(self): |
| self._clean_previous_installs() |
| self._get_packages() |
| self._install_packages() |
| self.install_unittests() |
| create_symlinks(test_bindir=self.test_bindir, |
| bin_list=self.qemu_bin_paths, |
| unittest=self.unittest_prefix) |
| self.reload_modules_if_needed() |
| if self.save_results: |
| virt_utils.archive_as_tarball(self.srcdir, self.results_dir) |
| |
| |
| def _get_rpm_names(self): |
| all_rpm_names = [] |
| koji_client = virt_utils.KojiClient(cmd=self.koji_cmd) |
| for pkg_text in self.koji_pkgs: |
| pkg = virt_utils.KojiPkgSpec(pkg_text) |
| rpm_names = koji_client.get_pkg_rpm_names(pkg) |
| all_rpm_names += rpm_names |
| return all_rpm_names |
| |
| |
| def _get_rpm_file_names(self): |
| all_rpm_file_names = [] |
| koji_client = virt_utils.KojiClient(cmd=self.koji_cmd) |
| for pkg_text in self.koji_pkgs: |
| pkg = virt_utils.KojiPkgSpec(pkg_text) |
| rpm_file_names = koji_client.get_pkg_rpm_file_names(pkg) |
| all_rpm_file_names += rpm_file_names |
| return all_rpm_file_names |
| |
| |
| def _install_packages(self): |
| """ |
| Install all downloaded packages. |
| """ |
| os.chdir(self.srcdir) |
| rpm_file_names = " ".join(self._get_rpm_file_names()) |
| utils.system("yum --nogpgcheck -y localinstall %s" % rpm_file_names) |
| |
| |
| class SourceDirInstaller(BaseInstaller): |
| """ |
| Class that handles building/installing KVM directly from a tarball or |
| a single source code dir. |
| """ |
| def set_install_params(self, test, params): |
| """ |
| Initializes class attributes, and retrieves KVM code. |
| |
| @param test: kvm test object |
| @param params: Dictionary with test arguments |
| """ |
| super(SourceDirInstaller, self).set_install_params(test, params) |
| |
| self.mod_install_dir = os.path.join(self.prefix, 'modules') |
| |
| srcdir = params.get("srcdir", None) |
| self.path_to_roms = params.get("path_to_rom_images", None) |
| |
| if self.install_mode == 'localsrc': |
| if srcdir is None: |
| raise error.TestError("Install from source directory specified" |
| "but no source directory provided on the" |
| "control file.") |
| else: |
| shutil.copytree(srcdir, self.srcdir) |
| |
| elif self.install_mode == 'localtar': |
| tarball = params.get("tarball") |
| if not tarball: |
| raise error.TestError("KVM Tarball install specified but no" |
| " tarball provided on control file.") |
| logging.info("Installing KVM from a local tarball") |
| logging.info("Using tarball %s") |
| tarball = utils.unmap_url("/", params.get("tarball"), "/tmp") |
| utils.extract_tarball_to_dir(tarball, self.srcdir) |
| |
| if self.install_mode in ['localtar', 'srcdir']: |
| self.repo_type = virt_utils.check_kvm_source_dir(self.srcdir) |
| p = os.path.join(self.srcdir, 'configure') |
| self.configure_options = virt_installer.check_configure_options(p) |
| |
| |
| def _build(self): |
| make_jobs = utils.count_cpus() |
| os.chdir(self.srcdir) |
| # For testing purposes, it's better to build qemu binaries with |
| # debugging symbols, so we can extract more meaningful stack traces. |
| cfg = "./configure --prefix=%s" % self.prefix |
| if "--disable-strip" in self.configure_options: |
| cfg += " --disable-strip" |
| steps = [cfg, "make clean", "make -j %s" % make_jobs] |
| logging.info("Building KVM") |
| for step in steps: |
| utils.system(step) |
| |
| |
| def _install(self): |
| os.chdir(self.srcdir) |
| logging.info("Installing KVM userspace") |
| if self.repo_type == 1: |
| utils.system("make -C qemu install") |
| elif self.repo_type == 2: |
| utils.system("make install") |
| if self.path_to_roms: |
| install_roms(self.path_to_roms, self.prefix) |
| self.install_unittests() |
| create_symlinks(test_bindir=self.test_bindir, |
| prefix=self.prefix, |
| unittest=self.unittest_prefix) |
| |
| |
| def install(self): |
| self._build() |
| self._install() |
| self.reload_modules_if_needed() |
| if self.save_results: |
| virt_utils.archive_as_tarball(self.srcdir, self.results_dir) |
| |
| class GitRepo(object): |
| def __init__(self, installer, prefix, |
| srcdir, build_steps=[], repo_param=None): |
| params = installer.params |
| self.installer = installer |
| self.repo = params.get(repo_param or (prefix + '_repo')) |
| self.branch = params.get(prefix + '_branch', 'master') |
| self.lbranch = params.get(prefix + '_lbranch', 'master') |
| self.commit = params.get(prefix + '_commit', None) |
| # The config system yields strings, which have to be evalued |
| self.patches = eval(params.get(prefix + '_patches', "[]")) |
| self.build_steps = build_steps |
| self.srcdir = os.path.join(self.installer.srcdir, srcdir) |
| |
| |
| def fetch_and_patch(self): |
| if not self.repo: |
| return |
| virt_utils.get_git_branch(self.repo, self.branch, self.srcdir, |
| self.commit, self.lbranch) |
| os.chdir(self.srcdir) |
| for patch in self.patches: |
| utils.get_file(patch, os.path.join(self.srcdir, |
| os.path.basename(patch))) |
| utils.system('patch -p1 < %s' % os.path.basename(patch)) |
| |
| |
| def build(self): |
| os.chdir(self.srcdir) |
| for step in self.build_steps: |
| logging.info(step) |
| utils.run(step) |
| |
| |
| class GitInstaller(SourceDirInstaller): |
| def _pull_code(self): |
| """ |
| Retrieves code from git repositories. |
| """ |
| params = self.params |
| make_jobs = utils.count_cpus() |
| cfg = 'PKG_CONFIG_PATH="%s/lib/pkgconfig:%s/share/pkgconfig" ./configure' % ( |
| self.prefix, self.prefix) |
| |
| self.spice_protocol = GitRepo(installer=self, prefix='spice_protocol', |
| srcdir='spice-protocol', |
| build_steps= ['./autogen.sh', |
| './configure --prefix=%s' % self.prefix, |
| 'make clean', |
| 'make -j %s' % (make_jobs), |
| 'make install']) |
| |
| self.spice = GitRepo(installer=self, prefix='spice', srcdir='spice', |
| build_steps= ['PKG_CONFIG_PATH="%s/lib/pkgconfig:%s/share/pkgconfig" CXXFLAGS=-Wl,--add-needed ./autogen.sh --prefix=%s' % (self.prefix, self.prefix, self.prefix), |
| 'make clean', |
| 'make -j %s' % (make_jobs), |
| 'make install']) |
| |
| self.userspace = GitRepo(installer=self, prefix='user', |
| repo_param='user_git_repo', srcdir='kvm_userspace') |
| |
| p = os.path.join(self.userspace.srcdir, 'configure') |
| self.configure_options = virt_installer.check_configure_options(p) |
| |
| cfg = cfg + ' --prefix=%s' % self.prefix |
| if "--disable-strip" in self.configure_options: |
| cfg += ' --disable-strip' |
| if self.extra_configure_options: |
| cfg += ' %s' % self.extra_configure_options |
| |
| self.userspace.build_steps=[cfg, 'make clean', 'make -j %s' % make_jobs] |
| |
| if not self.userspace.repo: |
| message = "KVM user git repository path not specified" |
| logging.error(message) |
| raise error.TestError(message) |
| |
| for repo in [self.userspace, self.spice_protocol, self.spice]: |
| if not repo.repo: |
| continue |
| repo.fetch_and_patch() |
| |
| def _build(self): |
| if self.spice_protocol.repo: |
| logging.info('Building Spice-protocol') |
| self.spice_protocol.build() |
| |
| if self.spice.repo: |
| logging.info('Building Spice') |
| self.spice.build() |
| |
| logging.info('Building KVM userspace code') |
| self.userspace.build() |
| |
| |
| def _install(self): |
| os.chdir(self.userspace.srcdir) |
| utils.system('make install') |
| |
| if self.path_to_roms: |
| install_roms(self.path_to_roms, self.prefix) |
| self.install_unittests() |
| create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix, |
| bin_list=None, |
| unittest=self.unittest_prefix) |
| |
| |
| def install(self): |
| self._pull_code() |
| self._build() |
| self._install() |
| self.reload_modules_if_needed() |
| if self.save_results: |
| virt_utils.archive_as_tarball(self.srcdir, self.results_dir) |
| |
| |
| class PreInstalledKvm(BaseInstaller): |
| def install(self): |
| logging.info("Expecting KVM to be already installed. Doing nothing") |
| |
| |
| class FailedInstaller: |
| """ |
| Class used to be returned instead of the installer if a installation fails |
| |
| Useful to make sure no installer object is used if KVM installation fails. |
| """ |
| def __init__(self, msg="KVM install failed"): |
| self._msg = msg |
| |
| |
| def load_modules(self): |
| """Will refuse to load the KVM modules as install failed""" |
| raise FailedKvmInstall("KVM modules not available. reason: %s" % (self._msg)) |
| |
| |
| installer_classes = { |
| 'localsrc': SourceDirInstaller, |
| 'localtar': SourceDirInstaller, |
| 'git': GitInstaller, |
| 'yum': YumInstaller, |
| 'koji': KojiInstaller, |
| 'preinstalled': PreInstalledKvm, |
| } |
| |
| |
| def _installer_class(install_mode): |
| c = installer_classes.get(install_mode) |
| if c is None: |
| raise error.TestError('Invalid or unsupported' |
| ' install mode: %s' % install_mode) |
| return c |
| |
| |
| def make_installer(params): |
| # priority: |
| # - 'install_mode' param |
| # - 'mode' param |
| mode = params.get("install_mode", params.get("mode")) |
| klass = _installer_class(mode) |
| return klass(mode) |
