Adding Ash Install to Provision

Adding Ash install capability from cr/2966529.
Also added tests and reloaded mocks.

BUG=None
TEST=unit

Change-Id: I5124d7fa25b35d9096423851ad1ddb5c3e21ce4e
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/3021806
Reviewed-by: C Shapiro <shapiroc@chromium.org>
Tested-by: Jaques Clapauch <jaquesc@google.com>
Tested-by: C Shapiro <shapiroc@chromium.org>
Auto-Submit: Jaques Clapauch <jaquesc@google.com>
Commit-Queue: C Shapiro <shapiroc@chromium.org>
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/mock_services/serviceadaptermock.go b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/mock_services/serviceadaptermock.go
index b869115..f022e4d 100644
--- a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/mock_services/serviceadaptermock.go
+++ b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/mock_services/serviceadaptermock.go
@@ -8,8 +8,6 @@
 // Package mock_services is a generated GoMock package.
 package mock_services
 
-// Removing until gomock gets updated internally
-
 import (
 	context "context"
 	reflect "reflect"
@@ -55,6 +53,34 @@
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyData", reflect.TypeOf((*MockServiceAdapterInterface)(nil).CopyData), ctx, url)
 }
 
+// CreateDirectories mocks base method.
+func (m *MockServiceAdapterInterface) CreateDirectories(ctx context.Context, dirs []string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateDirectories", ctx, dirs)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CreateDirectories indicates an expected call of CreateDirectories.
+func (mr *MockServiceAdapterInterfaceMockRecorder) CreateDirectories(ctx, dirs interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateDirectories", reflect.TypeOf((*MockServiceAdapterInterface)(nil).CreateDirectories), ctx, dirs)
+}
+
+// DeleteDirectory mocks base method.
+func (m *MockServiceAdapterInterface) DeleteDirectory(ctx context.Context, dir string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteDirectory", ctx, dir)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteDirectory indicates an expected call of DeleteDirectory.
+func (mr *MockServiceAdapterInterfaceMockRecorder) DeleteDirectory(ctx, dir interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDirectory", reflect.TypeOf((*MockServiceAdapterInterface)(nil).DeleteDirectory), ctx, dir)
+}
+
 // PathExists mocks base method.
 func (m *MockServiceAdapterInterface) PathExists(ctx context.Context, path string) (bool, error) {
 	m.ctrl.T.Helper()
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/ashservice.go b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/ashservice.go
new file mode 100644
index 0000000..3fcc3d0
--- /dev/null
+++ b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/ashservice.go
@@ -0,0 +1,227 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// AshInstall state machine construction and helper
+
+package ashservice
+
+import (
+	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services"
+	"context"
+	"fmt"
+	"log"
+	"path/filepath"
+	"time"
+
+	conf "go.chromium.org/chromiumos/config/go"
+	"go.chromium.org/chromiumos/config/go/test/api"
+	"google.golang.org/grpc"
+)
+
+// File specific consts
+const (
+	autotestDir      = "/usr/local/autotest/deps/chrome_test/test_src/out/Release/"
+	stagingDirectory = "/tmp/_tls_chrome_deploy"
+	targetDir        = "/opt/google/chrome"
+	tastDir          = "/usr/local/libexec/chrome-binary-tests/"
+)
+
+// Time specific consts
+const (
+	twoSeconds = 2 * time.Second
+	tenSeconds = 10 * time.Second
+)
+
+// binaries to be copied in installation
+var copyPaths = [...]string{
+	"ash_shell",
+	"aura_demo",
+	"chrome",
+	"chrome-wrapper",
+	"chrome.pak",
+	"chrome_100_percent.pak",
+	"chrome_200_percent.pak",
+	"content_shell",
+	"content_shell.pak",
+	"extensions/",
+	"lib/*.so",
+	"libffmpegsumo.so",
+	"libpdf.so",
+	"libppGoogleNaClPluginChrome.so",
+	"libosmesa.so",
+	"libwidevinecdmadapter.so",
+	"libwidevinecdm.so",
+	"locales/",
+	"nacl_helper_bootstrap",
+	"nacl_irt_*.nexe",
+	"nacl_helper",
+	"resources/",
+	"resources.pak",
+	"xdg-settings",
+	"*.png",
+}
+
+// test binaries to be copied in installation
+var testPaths = [...]string{
+	"*test",
+	"*tests",
+}
+
+// AshService inherits ServiceInterface
+type AshService struct {
+	connection services.ServiceAdapterInterface
+	imagePath  *conf.StoragePath
+}
+
+func NewAshService(dutName string, dutClient api.DutServiceClient, wiringConn *grpc.ClientConn, req *api.InstallAshRequest) AshService {
+	service := AshService{
+		connection: services.NewServiceAdapter(dutName, dutClient, wiringConn),
+		imagePath:  req.AshImagePath,
+	}
+
+	return service
+}
+
+// NewAshServiceFromExistingConnection is equivalent to the above constructor,
+// but recycles a ServiceAdapter. Generally useful for tests.
+func NewAshServiceFromExistingConnection(conn services.ServiceAdapterInterface, imagePath *conf.StoragePath) AshService {
+	return AshService{
+		connection: conn,
+		imagePath:  imagePath,
+	}
+}
+
+// GetFirstState returns the first state of this state machine
+func (a *AshService) GetFirstState() services.ServiceState {
+	return AshPrepareState{
+		service: *a,
+	}
+}
+
+// CleanUpStagingDirectory simply deletes the staging directory
+func (a *AshService) CleanUpStagingDirectory(ctx context.Context) error {
+	return a.connection.DeleteDirectory(ctx, stagingDirectory)
+}
+
+// CreateStagingDirectory ensures a clean staging directory is present
+func (a AshService) CreateStagingDirectory(ctx context.Context) error {
+	if err := a.CleanUpStagingDirectory(ctx); err != nil {
+		return err
+	}
+
+	return a.connection.CreateDirectories(ctx, []string{stagingDirectory})
+}
+
+// CreateBinaryDirectories creates all directories which will house the binaries for the install
+func (a *AshService) CreateBinaryDirectories(ctx context.Context) error {
+	return a.connection.CreateDirectories(ctx, []string{targetDir, autotestDir, tastDir})
+}
+
+// CopyImageToDUT copies the desired image to the DUT, passing through the caching layer.
+func (a *AshService) CopyImageToDUT(ctx context.Context) error {
+	if a.imagePath.HostType == conf.StoragePath_LOCAL || a.imagePath.HostType == conf.StoragePath_HOSTTYPE_UNSPECIFIED {
+		return fmt.Errorf("only GS copying is implemented")
+	}
+	url, err := a.connection.CopyData(ctx, a.imagePath.GetPath())
+	if err != nil {
+		return fmt.Errorf("failed to cache ash compressed, %w", err)
+	}
+	if _, err := a.connection.RunCmd(ctx, "", []string{
+		"curl", url,
+		"|",
+		"tar", "--ignore-command-error", "--overwrite", "--preserve-permissions", fmt.Sprintf("--directory=%s", stagingDirectory), "-xf", "-",
+	}); err != nil {
+		return fmt.Errorf("failed to copy ash compressed, %w", err)
+	}
+
+	return nil
+}
+
+// MountRootFS mounts the root filesystem as a read/write
+func (a *AshService) MountRootFS(ctx context.Context) error {
+	if _, err := a.connection.RunCmd(ctx, "mount", []string{"-o", "remount,rw", "/"}); err != nil {
+		return fmt.Errorf("could not mount root file system, %w", err)
+	}
+	return nil
+}
+
+// isChromeInUse determines if chrome is currently running
+func (a *AshService) isChromeInUse(ctx context.Context) bool {
+	_, err := a.connection.RunCmd(ctx, "lsof", []string{fmt.Sprintf("%s/chrome", targetDir)})
+	return err != nil
+}
+
+// StopChrome stops the UI
+func (a *AshService) StopChrome(ctx context.Context) error {
+	if _, err := a.connection.RunCmd(ctx, "stop", []string{"ui"}); err != nil {
+		// stop ui returns error when UI is terminated, so ignore error here
+		log.Printf("failed to stop chrome, %s", err)
+	}
+	return nil
+}
+
+// KillChrome tries to pkill chrome, retrying/re-polling every two seconds
+func (a *AshService) KillChrome(ctx context.Context) error {
+	for start := time.Now(); time.Since(start) < tenSeconds; time.Sleep(twoSeconds) {
+		if !a.isChromeInUse(ctx) {
+			return nil
+		}
+		log.Printf("chrome binary is still running, killing...")
+		if _, err := a.connection.RunCmd(ctx, "pkill", []string{"'chrome|session_manager'"}); err != nil {
+			return fmt.Errorf("failed run pkill, %s", err)
+		}
+	}
+	return fmt.Errorf("failed to kill chrome")
+}
+
+// Deploy rsyncs files relevant to the install to the correct bin locations
+func (a *AshService) Deploy(ctx context.Context) error {
+	for _, file := range copyPaths {
+		if err := a.deployFile(ctx, file, targetDir); err != nil {
+			return fmt.Errorf("could not deploy copy file, %w", err)
+		}
+	}
+	for _, file := range testPaths {
+		if err := a.deployFile(ctx, file, autotestDir); err != nil {
+			return fmt.Errorf("could not deploy autotest file, %w", err)
+		}
+		if err := a.deployFile(ctx, file, tastDir); err != nil {
+			return fmt.Errorf("could not deploy tast file, %w", err)
+		}
+	}
+	return nil
+}
+
+// deployFile rsyncs one specific file to the desired bin dir
+func (a *AshService) deployFile(ctx context.Context, file string, destination string) error {
+	source := fmt.Sprintf("%s/%s", stagingDirectory, file)
+	target := filepath.Dir(fmt.Sprintf("%s/%s", destination, file))
+
+	if exists, err := a.connection.PathExists(ctx, source); err != nil {
+		return fmt.Errorf("failed to determine file existance, %s", err)
+	} else if !exists {
+		return nil
+	}
+
+	if _, err := a.connection.RunCmd(ctx, "rsync", []string{"-av", source, target}); err != nil {
+		return fmt.Errorf("failed run rsync, %s", err)
+	}
+	return nil
+}
+
+// ReloadBus kill the bus daemon with a SIGHUP
+func (a *AshService) ReloadBus(ctx context.Context) error {
+	if _, err := a.connection.RunCmd(ctx, "killall", []string{"-HUP", "dbus-daemon"}); err != nil {
+		return fmt.Errorf("failed to reload dbus, %s", err)
+	}
+	return nil
+}
+
+// StartChrome restarts the ui
+func (a *AshService) StartChrome(ctx context.Context) error {
+	if _, err := a.connection.RunCmd(ctx, "start", []string{"ui"}); err != nil {
+		return fmt.Errorf("failed to start ui, %s", err)
+	}
+	return nil
+}
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/installstate.go b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/installstate.go
new file mode 100644
index 0000000..2a16561
--- /dev/null
+++ b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/installstate.go
@@ -0,0 +1,36 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package ashservice
+
+import (
+	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services"
+	"context"
+	"fmt"
+)
+
+// Second step of AshInstall State Machine. Responsible for installation
+type AshInstallState struct {
+	service AshService
+}
+
+func (s AshInstallState) Execute(ctx context.Context) error {
+	if err := s.service.MountRootFS(ctx); err != nil {
+		return fmt.Errorf("could not mount root file system, %w", err)
+	}
+	if err := s.service.Deploy(ctx); err != nil {
+		return fmt.Errorf("could not deploy ash files, %w", err)
+	}
+	return nil
+}
+
+func (s AshInstallState) Next() services.ServiceState {
+	return AshPostInstallState{
+		service: s.service,
+	}
+}
+
+func (s AshInstallState) Name() string {
+	return "Ash Install"
+}
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/postinstallstate.go b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/postinstallstate.go
new file mode 100644
index 0000000..cc74a75
--- /dev/null
+++ b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/postinstallstate.go
@@ -0,0 +1,37 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package ashservice
+
+import (
+	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services"
+	"context"
+	"fmt"
+)
+
+// Third step of AshInstall State Machine. Responsible for general clean-up
+type AshPostInstallState struct {
+	service AshService
+}
+
+func (s AshPostInstallState) Execute(ctx context.Context) error {
+	if err := s.service.ReloadBus(ctx); err != nil {
+		return fmt.Errorf("could not reload bus, %w", err)
+	}
+	if err := s.service.StartChrome(ctx); err != nil {
+		return fmt.Errorf("could not start UI, %w", err)
+	}
+	if err := s.service.CleanUpStagingDirectory(ctx); err != nil {
+		return fmt.Errorf("could not delete staging directory, %w", err)
+	}
+	return nil
+}
+
+func (s AshPostInstallState) Next() services.ServiceState {
+	return nil
+}
+
+func (s AshPostInstallState) Name() string {
+	return "Ash PostInstall"
+}
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/preparestate.go b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/preparestate.go
new file mode 100644
index 0000000..3e32063
--- /dev/null
+++ b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice/preparestate.go
@@ -0,0 +1,45 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package ashservice
+
+import (
+	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services"
+	"context"
+	"fmt"
+)
+
+// First step of AshInstall State Machine. Responsible for preparing the machine for the installation
+type AshPrepareState struct {
+	service AshService
+}
+
+func (s AshPrepareState) Execute(ctx context.Context) error {
+	if err := s.service.CreateStagingDirectory(ctx); err != nil {
+		return fmt.Errorf("could not recreate staging directory, %w", err)
+	}
+	if err := s.service.CopyImageToDUT(ctx); err != nil {
+		return fmt.Errorf("could not copy image to DUT, %w", err)
+	}
+	if err := s.service.CreateBinaryDirectories(ctx); err != nil {
+		return fmt.Errorf("could not create target directories, %w", err)
+	}
+	if err := s.service.StopChrome(ctx); err != nil {
+		return fmt.Errorf("could not stop chrome, %w", err)
+	}
+	if err := s.service.KillChrome(ctx); err != nil {
+		return fmt.Errorf("could not kill chrome, %w", err)
+	}
+	return nil
+}
+
+func (s AshPrepareState) Next() services.ServiceState {
+	return AshInstallState{
+		service: s.service,
+	}
+}
+
+func (s AshPrepareState) Name() string {
+	return "Ash Prepare"
+}
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/serviceadapterinterface.go b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/serviceadapterinterface.go
index c413a90..b4fdc37 100644
--- a/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/serviceadapterinterface.go
+++ b/src/chromiumos/test/provision/cmd/provisionserver/bootstrap/services/serviceadapterinterface.go
@@ -24,6 +24,8 @@
 	Restart(ctx context.Context) error
 	PathExists(ctx context.Context, path string) (bool, error)
 	CopyData(ctx context.Context, url string) (string, error)
+	DeleteDirectory(ctx context.Context, dir string) error
+	CreateDirectories(ctx context.Context, dirs []string) error
 }
 
 type ServiceAdapter struct {
@@ -134,3 +136,21 @@
 
 	return resp.GetUrl(), nil
 }
+
+// DeleteDirectory is a thin wrapper around an rm command. Done here as it is
+// expected to be reused often by many services.
+func (s ServiceAdapter) DeleteDirectory(ctx context.Context, dir string) error {
+	if _, err := s.RunCmd(ctx, "rm", []string{"-rf", dir}); err != nil {
+		return fmt.Errorf("could not delete directory, %w", err)
+	}
+	return nil
+}
+
+// Create directories is a thin wrapper around an mkdir command. Done here as it
+// is expected to be reused often by many services.
+func (s ServiceAdapter) CreateDirectories(ctx context.Context, dirs []string) error {
+	if _, err := s.RunCmd(ctx, "mkdir", append([]string{"-p"}, dirs...)); err != nil {
+		return fmt.Errorf("could not create directory, %w", err)
+	}
+	return nil
+}
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/provisionserver.go b/src/chromiumos/test/provision/cmd/provisionserver/provisionserver.go
index 29ee349..fcf4bd7 100644
--- a/src/chromiumos/test/provision/cmd/provisionserver/provisionserver.go
+++ b/src/chromiumos/test/provision/cmd/provisionserver/provisionserver.go
@@ -14,6 +14,7 @@
 
 	"chromiumos/lro"
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services"
+	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice"
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services/crosservice"
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services/lacrosservice"
 
@@ -62,7 +63,7 @@
 	op := s.Manager.NewOperation()
 	cs := crosservice.NewCrOSService(s.dutName, s.dutClient, s.wiringConn, req)
 	response := api.InstallCrosResponse{}
-	if s.provisionOS(ctx, &cs, op) == nil {
+	if s.provision(ctx, &cs, op) == nil {
 		response.Outcome = &api.InstallCrosResponse_Success{}
 	} else {
 		response.Outcome = &api.InstallCrosResponse_Failure{}
@@ -85,10 +86,14 @@
 			tls.ProvisionDutResponse_REASON_PROVISIONING_FAILED.String(),
 		)
 	} else {
-		if s.provisionOS(ctx, &ls, op) == nil {
+		if s.provision(ctx, &ls, op) == nil {
 			response.Outcome = &api.InstallLacrosResponse_Success{}
 		} else {
-			response.Outcome = &api.InstallLacrosResponse_Failure{}
+			response.Outcome = &api.InstallLacrosResponse_Failure{
+				Failure: &api.InstallFailure{
+					Reason: api.InstallFailure_REASON_PROVISIONING_FAILED,
+				},
+			}
 		}
 	}
 	s.Manager.SetResult(op.Name, &response)
@@ -102,7 +107,14 @@
 func (s *ProvisionServer) InstallAsh(ctx context.Context, req *api.InstallAshRequest) (*longrunning.Operation, error) {
 	s.logger.Println("Received api.InstallAshRequest: ", *req)
 	op := s.Manager.NewOperation()
-	s.Manager.SetResult(op.Name, &api.InstallAshResponse{})
+	cs := ashservice.NewAshService(s.dutName, s.dutClient, s.wiringConn, req)
+	response := api.InstallAshResponse{}
+	if s.provision(ctx, &cs, op) == nil {
+		response.Outcome = &api.InstallAshResponse_Success{}
+	} else {
+		response.Outcome = &api.InstallAshResponse_Failure{}
+	}
+	s.Manager.SetResult(op.Name, &response)
 	return op, nil
 }
 
@@ -117,11 +129,11 @@
 	return op, nil
 }
 
-// provisionOS effectively acts as a state transition runner for each of the
+// provision effectively acts as a state transition runner for each of the
 // installation services, transitioning between states as required, and
 // executing each state. Operation status is also set at this state in case of
 // error.
-func (s *ProvisionServer) provisionOS(ctx context.Context, si services.ServiceInterface, operation *longrunning.Operation) error {
+func (s *ProvisionServer) provision(ctx context.Context, si services.ServiceInterface, operation *longrunning.Operation) error {
 	// Set a timeout for provisioning.
 	ctx, cancel := context.WithTimeout(ctx, time.Hour)
 	defer cancel()
@@ -132,7 +144,7 @@
 			codes.DeadlineExceeded,
 			"provision: timed out before provisioning OS",
 			tls.ProvisionDutResponse_REASON_PROVISIONING_TIMEDOUT.String())
-		return fmt.Errorf("deadline failure.")
+		return fmt.Errorf("deadline failure")
 	default:
 	}
 
diff --git a/src/chromiumos/test/provision/cmd/provisionserver/provisionserver_test.go b/src/chromiumos/test/provision/cmd/provisionserver/provisionserver_test.go
index 4b20598..9ee6d18 100644
--- a/src/chromiumos/test/provision/cmd/provisionserver/provisionserver_test.go
+++ b/src/chromiumos/test/provision/cmd/provisionserver/provisionserver_test.go
@@ -4,10 +4,10 @@
 
 package main
 
-// Removing until internal gomock gets updated
 import (
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/info"
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/mock_services"
+	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services/ashservice"
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services/crosservice"
 	"chromiumos/test/provision/cmd/provisionserver/bootstrap/services/lacrosservice"
 	"context"
@@ -511,3 +511,143 @@
 		t.Fatalf("failed install state: %v", err)
 	}
 }
+
+func TestAshInstallStateTransitions(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	sam := mock_services.NewMockServiceAdapterInterface(ctrl)
+
+	as := ashservice.NewAshServiceFromExistingConnection(
+		sam,
+		&conf.StoragePath{
+			HostType: conf.StoragePath_GS,
+			Path:     "path/to/image",
+		},
+	)
+
+	ctx := context.Background()
+
+	// PREPARE
+	st := as.GetFirstState()
+
+	gomock.InOrder(
+		sam.EXPECT().DeleteDirectory(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy")).Return(nil),
+		sam.EXPECT().CreateDirectories(gomock.Any(), gomock.Eq([]string{"/tmp/_tls_chrome_deploy"})).Return(nil),
+		sam.EXPECT().CopyData(gomock.Any(), "path/to/image").Return("image_url", nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "", []string{"curl", "image_url", "|", "tar", "--ignore-command-error", "--overwrite", "--preserve-permissions", "--directory=/tmp/_tls_chrome_deploy", "-xf", "-"}).Return("", nil),
+		sam.EXPECT().CreateDirectories(gomock.Any(), gomock.Eq([]string{"/opt/google/chrome", "/usr/local/autotest/deps/chrome_test/test_src/out/Release/", "/usr/local/libexec/chrome-binary-tests/"})).Return(nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "stop", []string{"ui"}).Return("", nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "lsof", []string{"/opt/google/chrome/chrome"}).Return("", errors.New("chrome is in use!")),
+		sam.EXPECT().RunCmd(gomock.Any(), "pkill", []string{"'chrome|session_manager'"}).Return("", nil),
+		// Make first kill soft fail so we test retry:
+		sam.EXPECT().RunCmd(gomock.Any(), "lsof", []string{"/opt/google/chrome/chrome"}).Return("", errors.New("chrome is ***still*** in use!")),
+		sam.EXPECT().RunCmd(gomock.Any(), "pkill", []string{"'chrome|session_manager'"}).Return("", nil),
+		// Now we let it progress
+		sam.EXPECT().RunCmd(gomock.Any(), "lsof", []string{"/opt/google/chrome/chrome"}).Return("", nil),
+	)
+
+	if err := st.Execute(ctx); err != nil {
+		t.Fatalf("failed prepare state: %v", err)
+	}
+
+	// INSTALL
+	st = st.Next()
+
+	gomock.InOrder(
+		sam.EXPECT().RunCmd(gomock.Any(), "mount", []string{"-o", "remount,rw", "/"}).Return("", nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/ash_shell")).Return(true, nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "rsync", []string{"-av", "/tmp/_tls_chrome_deploy/ash_shell", "/opt/google/chrome"}).Return("", nil),
+		// For all items after we make them exist so we don't need to double every item (we assume that the test isn't breakable here):
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/aura_demo")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/chrome")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/chrome-wrapper")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/chrome.pak")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/chrome_100_percent.pak")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/chrome_200_percent.pak")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/content_shell")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/content_shell.pak")).Return(false, nil),
+		// Testing this one specifically as it should map to the designated folder rather than the top-most:
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/extensions/")).Return(true, nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "rsync", []string{"-av", "/tmp/_tls_chrome_deploy/extensions/", "/opt/google/chrome/extensions"}).Return("", nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/lib/*.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/libffmpegsumo.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/libpdf.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/libppGoogleNaClPluginChrome.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/libosmesa.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/libwidevinecdmadapter.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/libwidevinecdm.so")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/locales/")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/nacl_helper_bootstrap")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/nacl_irt_*.nexe")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/nacl_helper")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/resources/")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/resources.pak")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/xdg-settings")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/*.png")).Return(false, nil),
+
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/*test")).Return(true, nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "rsync", []string{"-av", "/tmp/_tls_chrome_deploy/*test", "/usr/local/autotest/deps/chrome_test/test_src/out/Release"}).Return("", nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/*test")).Return(true, nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "rsync", []string{"-av", "/tmp/_tls_chrome_deploy/*test", "/usr/local/libexec/chrome-binary-tests"}).Return("", nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/*tests")).Return(false, nil),
+		sam.EXPECT().PathExists(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy/*tests")).Return(false, nil),
+	)
+
+	if err := st.Execute(ctx); err != nil {
+		t.Fatalf("failed install state: %v", err)
+	}
+
+	// POST INSTALL
+	st = st.Next()
+
+	gomock.InOrder(
+		sam.EXPECT().RunCmd(gomock.Any(), "killall", []string{"-HUP", "dbus-daemon"}).Return("", nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "start", []string{"ui"}).Return("", nil),
+		sam.EXPECT().DeleteDirectory(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy")).Return(nil),
+	)
+
+	if err := st.Execute(ctx); err != nil {
+		t.Fatalf("failed post-install state: %v", err)
+	}
+
+	if st.Next() != nil {
+		t.Fatalf("provision should be the last step")
+	}
+}
+
+func TestPkillOnlyRunsForTenSeconds(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	sam := mock_services.NewMockServiceAdapterInterface(ctrl)
+
+	as := ashservice.NewAshServiceFromExistingConnection(
+		sam,
+		&conf.StoragePath{
+			HostType: conf.StoragePath_GS,
+			Path:     "path/to/image",
+		},
+	)
+
+	ctx := context.Background()
+
+	// PREPARE
+	st := as.GetFirstState()
+
+	gomock.InOrder(
+		sam.EXPECT().DeleteDirectory(gomock.Any(), gomock.Eq("/tmp/_tls_chrome_deploy")).Return(nil),
+		sam.EXPECT().CreateDirectories(gomock.Any(), gomock.Eq([]string{"/tmp/_tls_chrome_deploy"})).Return(nil),
+		sam.EXPECT().CopyData(gomock.Any(), "path/to/image").Return("image_url", nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "", []string{"curl", "image_url", "|", "tar", "--ignore-command-error", "--overwrite", "--preserve-permissions", "--directory=/tmp/_tls_chrome_deploy", "-xf", "-"}).Return("", nil),
+		sam.EXPECT().CreateDirectories(gomock.Any(), gomock.Eq([]string{"/opt/google/chrome", "/usr/local/autotest/deps/chrome_test/test_src/out/Release/", "/usr/local/libexec/chrome-binary-tests/"})).Return(nil),
+		sam.EXPECT().RunCmd(gomock.Any(), "stop", []string{"ui"}).Return("", nil),
+	)
+
+	sam.EXPECT().RunCmd(gomock.Any(), "lsof", []string{"/opt/google/chrome/chrome"}).Return("", errors.New("chrome is in use!")).AnyTimes()
+	sam.EXPECT().RunCmd(gomock.Any(), "pkill", []string{"'chrome|session_manager'"}).Return("", nil).AnyTimes()
+
+	if err := st.Execute(ctx); err == nil {
+		t.Fatalf("prepare should've failed!")
+	}
+}