pkg/cos: Access cos-tools with geo-redundancy
For geo-redundancy, now there are three GCS buckets for COS
artifacts. This commit changes the default bucket from
cos-tools to one of cos-tools, cos-tools-asia and cos-tools-eu
based on the zone the VM is running in.
cos_gpu_installer_v2 is also updated to use geo-redundant
buckets by default.
BUG=b/178641293
TEST=test cos-gpu-installer-v2 in GCE instances
Change-Id: Ibcf0c6dd7f04d80e163b8a22c43d9d144cd1416a
Reviewed-on: https://cos-review.googlesource.com/c/cos/tools/+/24053
Reviewed-by: Arnav Kansal <rnv@google.com>
Cloud-Build: GCB Service account <228075978874@cloudbuild.gserviceaccount.com>
Tested-by: He Gao <hegao@google.com>
diff --git a/src/cmd/cos_gpu_installer/internal/commands/install.go b/src/cmd/cos_gpu_installer/internal/commands/install.go
index 66bf6c6..19a6f70 100644
--- a/src/cmd/cos_gpu_installer/internal/commands/install.go
+++ b/src/cmd/cos_gpu_installer/internal/commands/install.go
@@ -64,9 +64,10 @@
"Whether to allow load unsigned GPU drivers. "+
"If this flag is set to true, module signing security features must be disabled on the host for driver installation to succeed. "+
"This flag is only for debugging.")
- f.StringVar(&c.gcsDownloadBucket, "gcs-download-bucket", "cos-tools",
+ f.StringVar(&c.gcsDownloadBucket, "gcs-download-bucket", "",
"The GCS bucket to download COS artifacts from. "+
- "For example, the default value is 'cos-tools' which is the public COS artifacts bucket.")
+ "The default bucket is one of 'cos-tools', 'cos-tools-asia' and 'cos-tools-eu' based on where the VM is running. "+
+ "Those are the public COS artifacts buckets.")
f.StringVar(&c.gcsDownloadPrefix, "gcs-download-prefix", "",
"The GCS path prefix when downloading COS artifacts."+
"If not set then the COS version build number (e.g. 13310.1041.38) will be used.")
diff --git a/src/pkg/cos/artifacts.go b/src/pkg/cos/artifacts.go
index 0cf6433..4ba47d6 100644
--- a/src/pkg/cos/artifacts.go
+++ b/src/pkg/cos/artifacts.go
@@ -6,7 +6,9 @@
"os"
"path"
"path/filepath"
+ "strings"
+ "cloud.google.com/go/compute/metadata"
log "github.com/golang/glog"
"github.com/pkg/errors"
@@ -15,6 +17,8 @@
const (
cosToolsGCS = "cos-tools"
+ cosToolsGCSAsia = "cos-tools-asia"
+ cosToolsGCSEU = "cos-tools-eu"
chromiumOSSDKGCS = "chromiumos-sdk"
kernelInfo = "kernel_info"
kernelSrcArchive = "kernel-src.tar.gz"
@@ -25,6 +29,16 @@
crosKernelRepo = "https://chromium.googlesource.com/chromiumos/third_party/kernel"
)
+// Map VM zone prefix to specific cos-tools bucket for geo-redundancy.
+var cosToolsPrefixMap = map[string]string{
+ "us": cosToolsGCS,
+ "northamerica": cosToolsGCS,
+ "southamerica": cosToolsGCS,
+ "europe": cosToolsGCSEU,
+ "asia": cosToolsGCSAsia,
+ "australia": cosToolsGCSAsia,
+}
+
// ArtifactsDownloader defines the interface to download COS artifacts.
type ArtifactsDownloader interface {
DownloadKernelSrc(destDir string) error
@@ -44,9 +58,23 @@
// NewGCSDownloader creates a GCSDownloader instance.
func NewGCSDownloader(e *EnvReader, bucket, prefix string) *GCSDownloader {
- // Use cos-tools as the default GCS bucket.
+ // If bucket is not set, use cos-tools, cos-tools-asia or cos-tools-eu
+ // according to the zone the VM is running in for geo-redundancy.
+ // If cannot fetch zone from metadata or get an unknown zone prefix,
+ // use cos-tools as the default GCS bucket.
if bucket == "" {
- bucket = cosToolsGCS
+ zone, err := metadata.Zone()
+ if err != nil {
+ log.Warningf("failed to get zone from metadata, will use 'gs://cos-tools' as artifact bucket, err: %v", err)
+ bucket = cosToolsGCS
+ } else {
+ zonePrefix := strings.Split(zone, "-")[0]
+ if geoBucket, found := cosToolsPrefixMap[zonePrefix]; found {
+ bucket = geoBucket
+ } else {
+ bucket = cosToolsGCS
+ }
+ }
}
// Use build number as the default GCS download prefix.
if prefix == "" {
