Robustify starting a VM to ensure it is ssh-able.

This CL adds the logic from retry_until_ssh in cros_vm_lib.sh in
in crosutils. This is added there and needs to be added here because
when sshing into a VM while a machine is under heavy load ssh can hang
due to an SSH bug.

BUG=chromium:209719
TEST=Ran it multiple times + pyflakes + pylint

Change-Id: I0228a540eb531b41f9e69e3e5a4f8df416da1935
Reviewed-on: https://gerrit.chromium.org/gerrit/60921
Tested-by: Chris Sosa <sosa@chromium.org>
Reviewed-by: Don Garrett <dgarrett@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
diff --git a/devmode-test/devinstall_test.py b/devmode-test/devinstall_test.py
index ca8549e..0e498fb 100755
--- a/devmode-test/devinstall_test.py
+++ b/devmode-test/devinstall_test.py
@@ -19,7 +19,6 @@
 import socket
 import sys
 import tempfile
-import time
 
 import constants
 sys.path.append(constants.SOURCE_ROOT)
@@ -35,6 +34,7 @@
 _LOCALHOST = 'localhost'
 _PRIVATE_KEY = os.path.join(constants.CROSUTILS_DIR, 'mod_for_test_scripts',
                             'ssh_keys', 'testing_rsa')
+_MAX_SSH_ATTEMPTS = 3
 
 class TestError(Exception):
   """Raised on any error during testing. It being raised is a test failure."""
@@ -74,10 +74,7 @@
         self.devserver.Stop()
 
       self.devserver = None
-
-      cmd = ['%s/bin/cros_stop_vm' % constants.CROSUTILS_DIR,
-             '--kvm_pid', self.tmpkvmpid]
-      cros_build_lib.RunCommand(cmd, debug_level=logging.DEBUG)
+      self._StopVM()
 
       if self.tmpdir:
         shutil.rmtree(self.tmpdir, ignore_errors=True)
@@ -113,6 +110,38 @@
     s.close()
     return port
 
+  def _RobustlyStartVMWithSSH(self):
+    """Start test copy of VM and ensure we can ssh into it.
+
+    This command is more robust than just naively starting the VM as it will
+    try to start the VM multiple times if the VM fails to start up. This is
+    inspired by retry_until_ssh in crosutils/lib/cros_vm_lib.sh.
+    """
+    for _ in range(_MAX_SSH_ATTEMPTS):
+      try:
+        cmd = ['%s/bin/cros_start_vm' % constants.CROSUTILS_DIR,
+               '--ssh_port', str(self.port),
+               '--image_path', self.working_image_path,
+               '--no_graphics',
+               '--kvm_pid', self.tmpkvmpid]
+        cros_build_lib.RunCommand(cmd, debug_level=logging.DEBUG)
+
+        # Ping the VM to ensure we can SSH into it.
+        self.remote_access.RemoteSh(['true'])
+        return
+      except cros_build_lib.RunCommandError as e:
+        logging.warning('Failed to connect to VM')
+        logging.debug(e)
+        self._StopVM()
+    else:
+      raise TestError('Max attempts to connect to VM exceeded')
+
+  def _StopVM(self):
+    """Stops a running VM set up using _RobustlyStartVMWithSSH."""
+    cmd = ['%s/bin/cros_stop_vm' % constants.CROSUTILS_DIR,
+           '--kvm_pid', self.tmpkvmpid]
+    cros_build_lib.RunCommand(cmd, debug_level=logging.DEBUG)
+
   def PrepareTest(self):
     """Pre-test modification to the image and env to setup test."""
     logging.info('Setting up the image %s for vm testing.',
@@ -131,16 +160,7 @@
     self._WipeStatefulPartition()
 
     logging.info('Starting the vm on port %d.', self.port)
-    cmd = ['%s/bin/cros_start_vm' % constants.CROSUTILS_DIR,
-           '--ssh_port', str(self.port),
-           '--image_path', self.working_image_path,
-           '--no_graphics',
-           '--kvm_pid', self.tmpkvmpid]
-    cros_build_lib.RunCommand(cmd, debug_level=logging.DEBUG)
-
-    # TODO(sosa): Super hack, fix with retry_ssh-like functionality.
-    # crbug.com/209719
-    time.sleep(30)
+    self._RobustlyStartVMWithSSH()
 
     if not self.binhost:
       logging.info('Starting the devserver.')