Refactor the structure of the publish (upload) service
Refactor the structure to align with the existing provision service (https://source.corp.google.com/chromeos_public/src/platform/dev/src/chromiumos/test/provision/cmd/).
BUG=b:204473963
TEST=Manually run unit tests
Change-Id: I86851f7cdd889a1a8f10337221055d6f0ae86b46
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/3274095
Commit-Queue: Zhihui Xie <zhihuixie@chromium.org>
Tested-by: Zhihui Xie <zhihuixie@chromium.org>
Auto-Submit: Zhihui Xie <zhihuixie@chromium.org>
Reviewed-by: Otabek Kasimov <otabek@google.com>
Reviewed-by: Jaques Clapauch <jaquesc@google.com>
diff --git a/src/chromiumos/test/publish/cmd/cros-publish/main.go b/src/chromiumos/test/publish/cmd/cros-publish/main.go
new file mode 100644
index 0000000..e5b96f7
--- /dev/null
+++ b/src/chromiumos/test/publish/cmd/cros-publish/main.go
@@ -0,0 +1,179 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package main implements the cros-publish used to upload artifacts to GCS
+// bucket.
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "chromiumos/test/publish/cmd/publishserver"
+ "go.chromium.org/luci/common/errors"
+)
+
+const (
+ // version is the version info of this command. It is filled in during emerge.
+ version = "<unknown>"
+ helpDescription = `cros-publish tool
+
+The tool allows to upload test result artifacts to GCS buckets for testing
+needs. Please read go/cros-upload-to-gs-tko-design-proposal for mode details.
+
+Commands:
+ cli Running publish server in CLI mode result will be printed to output
+ file.
+ usage: cros-publish cli -service_account_creds service_account_file
+
+ server Starting server and allow work with server by RPC calls. Mostly
+ used for tests.
+ usage: cros-publish server -service_account_creds service_account_file [-port 80]
+
+ -version Print version of lib.
+ -help Print this help.`
+ defaultLogDirectory = "/tmp/publish/"
+ defaultPort = 80
+)
+
+// createLogFile creates a file and its parent directory for logging purpose.
+func createLogFile() (*os.File, error) {
+ t := time.Now()
+ fullPath := filepath.Join(defaultLogDirectory, t.Format("20060102-150405"))
+ if err := os.MkdirAll(fullPath, 0755); err != nil {
+ return nil, fmt.Errorf("failed to create directory %v: %v", fullPath, err)
+ }
+
+ logFullPathName := filepath.Join(fullPath, "log.txt")
+
+ // Log the full output of the command to disk.
+ logFile, err := os.Create(logFullPathName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create file %v: %v", fullPath, err)
+ }
+ return logFile, nil
+}
+
+// newLogger creates a logger. Using go default logger for now.
+func newLogger(logFile *os.File) *log.Logger {
+ mw := io.MultiWriter(logFile, os.Stderr)
+ return log.New(mw, "", log.LstdFlags|log.LUTC)
+}
+
+type args struct {
+ // Common input params.
+ servviceAccountCreds string
+
+ // Server mode params
+ port int
+}
+
+func validate(a args) error {
+ if a.servviceAccountCreds == "" {
+ return errors.Reason("Service account file not specified").Err()
+ }
+
+ _, err := os.Open(a.servviceAccountCreds)
+ if err != nil {
+ return errors.Reason("Failed to read service account file").Err()
+ }
+ return nil
+}
+
+func startServer(d []string) int {
+ a := args{}
+ fs := flag.NewFlagSet("Start publish publishService", flag.ExitOnError)
+ fs.StringVar(&a.servviceAccountCreds, "service_account_creds", "", "path to service account file containing gcp credentials")
+ fs.IntVar(&a.port, "port", defaultPort, fmt.Sprintf("Specify the port for the publishService. Default value %d.", defaultPort))
+ fs.Parse(d)
+
+ logFile, err := createLogFile()
+ if err != nil {
+ log.Fatalln("Failed to create log file", err)
+ return 2
+ }
+ defer logFile.Close()
+
+ logger := newLogger(logFile)
+ err = validate(a)
+ if err != nil {
+ log.Fatalf("Validate arguments fail: %s", err)
+ return 2
+ }
+
+ publishService, destructor, err := publishserver.NewPublishService(logger, "")
+ defer destructor()
+ if err != nil {
+ logger.Fatalln("Failed to create publish: ", err)
+ return 2
+ }
+
+ err = publishService.StartServer(a.port)
+ if err != nil {
+ logger.Fatalln("Failed to perform publish: ", err)
+ return 1
+ }
+ return 0
+}
+
+// Specify run mode for CLI.
+type runMode string
+
+const (
+ runCli runMode = "cli"
+ runServer runMode = "server"
+ runVersion runMode = "version"
+ runHelp runMode = "help"
+)
+
+func getRunMode() (runMode, error) {
+ if len(os.Args) > 1 {
+ for _, a := range os.Args {
+ if a == "-version" {
+ return runVersion, nil
+ }
+ }
+ switch strings.ToLower(os.Args[1]) {
+ case "cli":
+ return runCli, nil
+ case "server":
+ return runServer, nil
+ }
+ }
+ // If we did not find special run mode then just print help for user.
+ return runHelp, nil
+}
+
+func mainInternal() int {
+ rm, err := getRunMode()
+ if err != nil {
+ log.Fatalln(err)
+ return 2
+ }
+
+ switch rm {
+ case runCli:
+ log.Fatalln("CLI mode is not implemented yet!")
+ return 2
+ case runServer:
+ log.Printf("Running server mode!")
+ return startServer(os.Args[2:])
+ case runVersion:
+ log.Printf("cros-publish version: %s", version)
+ return 0
+ }
+
+ log.Printf(helpDescription)
+ return 0
+}
+
+func main() {
+ os.Exit(mainInternal())
+}
diff --git a/src/chromiumos/test/publish/cmd/publishserver/main.go b/src/chromiumos/test/publish/cmd/publishserver/main.go
deleted file mode 100644
index 2e2a1b5..0000000
--- a/src/chromiumos/test/publish/cmd/publishserver/main.go
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2021 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package main
-
-import (
- "flag"
- "fmt"
- "io"
- "log"
- "net"
- "os"
- "path/filepath"
- "time"
-)
-
-// Version is the version info of this command. It is filled in during emerge.
-var Version = "<unknown>"
-
-// createLogFile creates a file and its parent directory for logging purpose.
-func createLogFile() (*os.File, error) {
- t := time.Now()
- fullPath := filepath.Join("/tmp/publishserver/", t.Format("20060102-150405"))
- if err := os.MkdirAll(fullPath, 0755); err != nil {
- return nil, fmt.Errorf("failed to create directory %v: %v", fullPath, err)
- }
-
- logFullPathName := filepath.Join(fullPath, "log.txt")
-
- // Log the full output of the command to disk.
- logFile, err := os.Create(logFullPathName)
- if err != nil {
- return nil, fmt.Errorf("failed to create file %v: %v", fullPath, err)
- }
- return logFile, nil
-}
-
-// newLogger creates a logger. Using go default logger for now.
-func newLogger(logFile *os.File) *log.Logger {
- mw := io.MultiWriter(logFile, os.Stderr)
- return log.New(mw, "", log.LstdFlags|log.LUTC)
-}
-
-func main() {
- os.Exit(func() int {
- flag.NewFlagSet("version", flag.ExitOnError)
- gcpCredentials := flag.String("gcp-creds", "", "path to file containing gcp credentials")
- flag.Parse()
-
- if os.Args[1] == "version" {
- fmt.Println("publishservice version ", Version)
- return 0
- }
-
- if *gcpCredentials == "" {
- fmt.Println("gcp-creds must be defined.")
- return 1
- }
-
- logFile, err := createLogFile()
- if err != nil {
- log.Fatalln("Failed to create log file: ", err)
- }
- defer logFile.Close()
-
- logger := newLogger(logFile)
- logger.Println("Starting publishservice version ", Version)
- l, err := net.Listen("tcp", ":0")
- if err != nil {
- logger.Fatalln("Failed to create a net listener: ", err)
- return 2
- }
-
- server, destructor, err := newPublishServiceServer(l, logger, *gcpCredentials)
- if err != nil {
- logger.Fatalln("Failed to create server: ", err)
- return 2
- }
- defer destructor()
-
- err = server.Serve(l)
- if err != nil {
- logger.Fatalln("Failed to initialize server: ", err)
- }
- return 0
- }())
-}
diff --git a/src/chromiumos/test/publish/cmd/publishserver/mock_storage/storagemock.go b/src/chromiumos/test/publish/cmd/publishserver/mock_storage/storagemock.go
index e443a04..d0d50e8 100644
--- a/src/chromiumos/test/publish/cmd/publishserver/mock_storage/storagemock.go
+++ b/src/chromiumos/test/publish/cmd/publishserver/mock_storage/storagemock.go
@@ -8,10 +8,11 @@
package mock_storage
import (
- storage "chromiumos/test/publish/cmd/publishserver/storage"
context "context"
reflect "reflect"
+ storage "chromiumos/test/publish/cmd/publishserver/storage"
+
gomock "github.com/golang/mock/gomock"
)
diff --git a/src/chromiumos/test/publish/cmd/publishserver/publishserver.go b/src/chromiumos/test/publish/cmd/publishserver/publishserver.go
index 181eead..36a7919 100644
--- a/src/chromiumos/test/publish/cmd/publishserver/publishserver.go
+++ b/src/chromiumos/test/publish/cmd/publishserver/publishserver.go
@@ -3,53 +3,49 @@
// found in the LICENSE file.
// Implements publish_service.proto (see proto for details)
-package main
+package publishserver
import (
- "chromiumos/lro"
- "chromiumos/test/publish/cmd/publishserver/storage"
"context"
"log"
- "net"
+
+ "chromiumos/lro"
+ "chromiumos/test/publish/cmd/publishserver/storage"
"go.chromium.org/chromiumos/config/go/longrunning"
"go.chromium.org/chromiumos/config/go/test/api"
- "google.golang.org/grpc"
)
-// PublishServiceServer implementation of publish_service.proto
-type PublishServiceServer struct {
+// PublishService implementation of publish_service.proto
+type PublishService struct {
manager *lro.Manager
logger *log.Logger
gsClient storage.GSClientInterface
}
-// newPublishServiceServer creates a new publish service server to listen to rpc requests.
-func newPublishServiceServer(l net.Listener, logger *log.Logger, gcpCredentials string) (*grpc.Server, func(), error) {
+// NewPublishService creates a new publish service with the GCP storage client.
+func NewPublishService(logger *log.Logger, gcpCredentials string) (*PublishService, func(), error) {
gsClient, err := storage.NewGSClient(context.Background(), gcpCredentials)
if err != nil {
return nil, nil, err
}
- s := &PublishServiceServer{
+ publishService := &PublishService{
manager: lro.New(),
logger: logger,
gsClient: gsClient,
}
- server := grpc.NewServer()
destructor := func() {
- s.manager.Close()
- s.gsClient.Close()
+ publishService.manager.Close()
+ publishService.gsClient.Close()
}
- api.RegisterPublishServiceServer(server, s)
- logger.Println("publishservice listen to request at ", l.Addr().String())
- return server, destructor, nil
+ return publishService, destructor, nil
}
// UploadToGS uploads the designated folder to the provided Google Cloud Storage
// bucket/object
-func (s *PublishServiceServer) UploadToGS(ctx context.Context, req *api.UploadToGSRequest) (*longrunning.Operation, error) {
+func (s *PublishService) UploadToGS(ctx context.Context, req *api.UploadToGSRequest) (*longrunning.Operation, error) {
s.logger.Println("Received api.UploadToGSRequest: ", *req)
op := s.manager.NewOperation()
if err := s.gsClient.Upload(ctx, req.LocalDirectory, req.GsDirectory); err != nil {
diff --git a/src/chromiumos/test/publish/cmd/publishserver/publishserver_server.go b/src/chromiumos/test/publish/cmd/publishserver/publishserver_server.go
new file mode 100644
index 0000000..aed7b5d
--- /dev/null
+++ b/src/chromiumos/test/publish/cmd/publishserver/publishserver_server.go
@@ -0,0 +1,35 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package publishserver
+
+import (
+ "fmt"
+ "net"
+
+ "go.chromium.org/chromiumos/config/go/longrunning"
+ "go.chromium.org/chromiumos/config/go/test/api"
+ "go.chromium.org/luci/common/errors"
+ "google.golang.org/grpc"
+
+ "chromiumos/lro"
+)
+
+// StartServer starts publish server on requested port
+func (s *PublishService) StartServer(port int) error {
+ l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+ if err != nil {
+ return errors.Annotate(err, "Start publish server: failed to create listener at %d", port).Err()
+ }
+
+ s.manager = lro.New()
+ defer s.manager.Close()
+ server := grpc.NewServer()
+
+ api.RegisterPublishServiceServer(server, s)
+ longrunning.RegisterOperationsServer(server, s.manager)
+
+ s.logger.Println("Publish server is listening to request at ", l.Addr().String())
+ return server.Serve(l)
+}
diff --git a/src/chromiumos/test/publish/cmd/publishserver/tests/publishserver_test.go b/src/chromiumos/test/publish/cmd/publishserver/tests/publishserver_test.go
index f29062c..ec4a41a 100644
--- a/src/chromiumos/test/publish/cmd/publishserver/tests/publishserver_test.go
+++ b/src/chromiumos/test/publish/cmd/publishserver/tests/publishserver_test.go
@@ -5,8 +5,6 @@
package tests
import (
- "chromiumos/test/publish/cmd/publishserver/mock_storage"
- "chromiumos/test/publish/cmd/publishserver/storage"
"context"
"fmt"
"io/ioutil"
@@ -14,6 +12,9 @@
"path"
"testing"
+ "chromiumos/test/publish/cmd/publishserver/mock_storage"
+ "chromiumos/test/publish/cmd/publishserver/storage"
+
"github.com/golang/mock/gomock"
)
