| import logging, os, re |
| from autotest_lib.client.common_lib import error |
| from autotest_lib.client.virt import virt_utils, rss_client, aexpect |
| |
| |
| def run_whql_submission(test, params, env): |
| """ |
| WHQL submission test: |
| 1) Log into the client machines and into a DTM server machine |
| 2) Copy the automation program binary (dsso_test_binary) to the server machine |
| 3) Run the automation program |
| 4) Pass the program all relevant parameters (e.g. device_data) |
| 5) Wait for the program to terminate |
| 6) Parse and report job results |
| (logs and HTML reports are placed in test.debugdir) |
| |
| @param test: kvm test object |
| @param params: Dictionary with the test parameters |
| @param env: Dictionary with test environment. |
| """ |
| # Log into all client VMs |
| login_timeout = int(params.get("login_timeout", 360)) |
| vms = [] |
| sessions = [] |
| for vm_name in params.objects("vms"): |
| vms.append(env.get_vm(vm_name)) |
| vms[-1].verify_alive() |
| sessions.append(vms[-1].wait_for_login(timeout=login_timeout)) |
| |
| # Make sure all NICs of all client VMs are up |
| for vm in vms: |
| nics = vm.params.objects("nics") |
| for nic_index in range(len(nics)): |
| s = vm.wait_for_login(nic_index, 600) |
| s.close() |
| |
| # Collect parameters |
| server_address = params.get("server_address") |
| server_shell_port = int(params.get("server_shell_port")) |
| server_file_transfer_port = int(params.get("server_file_transfer_port")) |
| server_studio_path = params.get("server_studio_path", "%programfiles%\\ " |
| "Microsoft Driver Test Manager\\Studio") |
| dsso_test_binary = params.get("dsso_test_binary", |
| "deps/whql_submission_15.exe") |
| dsso_test_binary = virt_utils.get_path(test.bindir, dsso_test_binary) |
| dsso_delete_machine_binary = params.get("dsso_delete_machine_binary", |
| "deps/whql_delete_machine_15.exe") |
| dsso_delete_machine_binary = virt_utils.get_path(test.bindir, |
| dsso_delete_machine_binary) |
| test_timeout = float(params.get("test_timeout", 600)) |
| |
| # Copy dsso binaries to the server |
| for filename in dsso_test_binary, dsso_delete_machine_binary: |
| rss_client.upload(server_address, server_file_transfer_port, |
| filename, server_studio_path, timeout=60) |
| |
| # Open a shell session with the server |
| server_session = virt_utils.remote_login("nc", server_address, |
| server_shell_port, "", "", |
| sessions[0].prompt, |
| sessions[0].linesep) |
| server_session.set_status_test_command(sessions[0].status_test_command) |
| |
| # Get the computer names of the server and clients |
| cmd = "echo %computername%" |
| server_name = server_session.cmd_output(cmd).strip() |
| client_names = [session.cmd_output(cmd).strip() for session in sessions] |
| |
| # Delete all client machines from the server's data store |
| server_session.cmd("cd %s" % server_studio_path) |
| for client_name in client_names: |
| cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary), |
| server_name, client_name) |
| server_session.cmd(cmd, print_func=logging.debug) |
| |
| # Reboot the client machines |
| sessions = virt_utils.parallel((vm.reboot, (session,)) |
| for vm, session in zip(vms, sessions)) |
| |
| # Check the NICs again |
| for vm in vms: |
| nics = vm.params.objects("nics") |
| for nic_index in range(len(nics)): |
| s = vm.wait_for_login(nic_index, 600) |
| s.close() |
| |
| # Run whql_pre_command and close the sessions |
| if params.get("whql_pre_command"): |
| for session in sessions: |
| session.cmd(params.get("whql_pre_command"), |
| int(params.get("whql_pre_command_timeout", 600))) |
| session.close() |
| |
| # Run the automation program on the server |
| pool_name = "%s_pool" % client_names[0] |
| submission_name = "%s_%s" % (client_names[0], |
| params.get("submission_name")) |
| cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary), |
| server_name, pool_name, submission_name, |
| test_timeout, " ".join(client_names)) |
| server_session.sendline(cmd) |
| |
| # Helper function: wait for a given prompt and raise an exception if an |
| # error occurs |
| def find_prompt(prompt): |
| m, o = server_session.read_until_last_line_matches( |
| [prompt, server_session.prompt], print_func=logging.info, |
| timeout=600) |
| if m != 0: |
| errors = re.findall("^Error:.*$", o, re.I | re.M) |
| if errors: |
| raise error.TestError(errors[0]) |
| else: |
| raise error.TestError("Error running automation program: " |
| "could not find '%s' prompt" % prompt) |
| |
| # Tell the automation program which device to test |
| find_prompt("Device to test:") |
| server_session.sendline(params.get("test_device")) |
| |
| # Tell the automation program which jobs to run |
| find_prompt("Jobs to run:") |
| server_session.sendline(params.get("job_filter", ".*")) |
| |
| # Set submission DeviceData |
| find_prompt("DeviceData name:") |
| for dd in params.objects("device_data"): |
| dd_params = params.object_params(dd) |
| if dd_params.get("dd_name") and dd_params.get("dd_data"): |
| server_session.sendline(dd_params.get("dd_name")) |
| server_session.sendline(dd_params.get("dd_data")) |
| server_session.sendline() |
| |
| # Set submission descriptors |
| find_prompt("Descriptor path:") |
| for desc in params.objects("descriptors"): |
| desc_params = params.object_params(desc) |
| if desc_params.get("desc_path"): |
| server_session.sendline(desc_params.get("desc_path")) |
| server_session.sendline() |
| |
| # Set machine dimensions for each client machine |
| for vm_name in params.objects("vms"): |
| vm_params = params.object_params(vm_name) |
| find_prompt(r"Dimension name\b.*:") |
| for dp in vm_params.objects("dimensions"): |
| dp_params = vm_params.object_params(dp) |
| if dp_params.get("dim_name") and dp_params.get("dim_value"): |
| server_session.sendline(dp_params.get("dim_name")) |
| server_session.sendline(dp_params.get("dim_value")) |
| server_session.sendline() |
| |
| # Set extra parameters for tests that require them (e.g. NDISTest) |
| for vm_name in params.objects("vms"): |
| vm_params = params.object_params(vm_name) |
| find_prompt(r"Parameter name\b.*:") |
| for dp in vm_params.objects("device_params"): |
| dp_params = vm_params.object_params(dp) |
| if dp_params.get("dp_name") and dp_params.get("dp_regex"): |
| server_session.sendline(dp_params.get("dp_name")) |
| server_session.sendline(dp_params.get("dp_regex")) |
| # Make sure the prompt appears again (if the device isn't found |
| # the automation program will terminate) |
| find_prompt(r"Parameter name\b.*:") |
| server_session.sendline() |
| |
| # Wait for the automation program to terminate |
| try: |
| o = server_session.read_up_to_prompt(print_func=logging.info, |
| timeout=test_timeout + 300) |
| # (test_timeout + 300 is used here because the automation program is |
| # supposed to terminate cleanly on its own when test_timeout expires) |
| done = True |
| except aexpect.ExpectError, e: |
| o = e.output |
| done = False |
| server_session.close() |
| |
| # Look for test results in the automation program's output |
| result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL) |
| if not result_summaries: |
| raise error.TestError("The automation program did not return any " |
| "results") |
| results = result_summaries[-1].strip("-") |
| results = eval("".join(results.splitlines())) |
| |
| # Download logs and HTML reports from the server |
| for i, r in enumerate(results): |
| if "report" in r: |
| try: |
| rss_client.download(server_address, |
| server_file_transfer_port, |
| r["report"], test.debugdir) |
| except rss_client.FileTransferNotFoundError: |
| pass |
| if "logs" in r: |
| try: |
| rss_client.download(server_address, |
| server_file_transfer_port, |
| r["logs"], test.debugdir) |
| except rss_client.FileTransferNotFoundError: |
| pass |
| else: |
| try: |
| # Create symlinks to test log dirs to make it easier |
| # to access them (their original names are not human |
| # readable) |
| link_name = "logs_%s" % r["report"].split("\\")[-1] |
| link_name = link_name.replace(" ", "_") |
| link_name = link_name.replace("/", "_") |
| os.symlink(r["logs"].split("\\")[-1], |
| os.path.join(test.debugdir, link_name)) |
| except (KeyError, OSError): |
| pass |
| |
| # Print result summary (both to the regular logs and to a file named |
| # 'summary' in test.debugdir) |
| def print_summary_line(f, line): |
| logging.info(line) |
| f.write(line + "\n") |
| if results: |
| # Make sure all results have the required keys |
| for r in results: |
| r["id"] = str(r.get("id")) |
| r["job"] = str(r.get("job")) |
| r["status"] = str(r.get("status")) |
| r["pass"] = int(r.get("pass", 0)) |
| r["fail"] = int(r.get("fail", 0)) |
| r["notrun"] = int(r.get("notrun", 0)) |
| r["notapplicable"] = int(r.get("notapplicable", 0)) |
| # Sort the results by failures and total test count in descending order |
| results = [(r["fail"], |
| r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"], |
| r) for r in results] |
| results.sort(reverse=True) |
| results = [r[-1] for r in results] |
| # Print results |
| logging.info("") |
| logging.info("Result summary:") |
| name_length = max(len(r["job"]) for r in results) |
| fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length |
| f = open(os.path.join(test.debugdir, "summary"), "w") |
| print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail", |
| "NotRun", "NotApplicable")) |
| print_summary_line(f, fmt % ("--", "---", "------", "----", "----", |
| "------", "-------------")) |
| for r in results: |
| print_summary_line(f, fmt % (r["id"], r["job"], r["status"], |
| r["pass"], r["fail"], r["notrun"], |
| r["notapplicable"])) |
| f.close() |
| logging.info("(see logs and HTML reports in %s)", test.debugdir) |
| |
| # Kill the client VMs and fail if the automation program did not terminate |
| # on time |
| if not done: |
| virt_utils.parallel(vm.destroy for vm in vms) |
| raise error.TestFail("The automation program did not terminate " |
| "on time") |
| |
| # Fail if there are failed or incomplete jobs (kill the client VMs if there |
| # are incomplete jobs) |
| failed_jobs = [r["job"] for r in results |
| if r["status"].lower() == "investigate"] |
| running_jobs = [r["job"] for r in results |
| if r["status"].lower() == "inprogress"] |
| errors = [] |
| if failed_jobs: |
| errors += ["Jobs failed: %s." % failed_jobs] |
| if running_jobs: |
| for vm in vms: |
| vm.destroy() |
| errors += ["Jobs did not complete on time: %s." % running_jobs] |
| if errors: |
| raise error.TestFail(" ".join(errors)) |
