Add network/subnet option to allow running in the non-default
This will enable the users to use private subnets based on their
requirement.
BUG=b/218525570
TEST=manual and ./run_tests
Change-Id: Iec5be6d46d073ce51d8696825d1cb559ceefe010
Reviewed-on: https://cos-review.googlesource.com/c/cos/tools/+/31043
Reviewed-by: Robert Kolchmeyer <rkolchmeyer@google.com>
Tested-by: Varsha Teratipally <teratipally@google.com>
Cloud-Build: GCB Service account <228075978874@cloudbuild.gserviceaccount.com>
diff --git a/src/cmd/cos_customizer/README.md b/src/cmd/cos_customizer/README.md
index 2f45930..e9d772e 100644
--- a/src/cmd/cos_customizer/README.md
+++ b/src/cmd/cos_customizer/README.md
@@ -264,6 +264,17 @@
separate from the Cloud Build machine type option, which sets the machine type
of the Cloud Build VM, which is different from the COS Customizer preload VM.
+-`network`: The network/VPC to use for the COS Customizer preload VM.
+The network must have access to Google Cloud Storage. Defaults to
+default network `global/networks/default`. If -subnet is also specified subnet
+must be a subnetwork of network specified by -network.
+
+-`subnet`: The subnet to use for the COS Customizer preload VM. Defaults to
+default network `global/networks/default`. If the network is in auto subnet mode,
+the subnetwork is optional. If the network is in custom subnet mode, then this
+field should be specified. Zone should be specified if this field is specified.
+
+
An example `finish-image-build` step looks like the following:
- name: 'gcr.io/cos-cloud/cos-customizer'
@@ -273,6 +284,18 @@
'-image-name=my-custom-image',
'-image-project=$PROJECT_ID']
+An example `finish-image-build` step with `network` and `subnet` looks like the following:
+
+ - name: 'gcr.io/cos-cloud/cos-customizer'
+ args: ['finish-image-build',
+ '-zone=us-west1-b',
+ '-project=$PROJECT_ID',
+ '-network=global/networks/auto-vpc',
+ '-subnet=regions/us-west1/subnetworks/auto-vpc-subnet-us-west1',
+ '-image-name=my-custom-image',
+ '-image-project=$PROJECT_ID']
+
+
### Optional build steps
The rest of the build steps provided by COS Customizer are optional; if they are
diff --git a/src/cmd/cos_customizer/finish_image_build.go b/src/cmd/cos_customizer/finish_image_build.go
index e69870b..51232f5 100644
--- a/src/cmd/cos_customizer/finish_image_build.go
+++ b/src/cmd/cos_customizer/finish_image_build.go
@@ -45,6 +45,8 @@
imageName string
imageSuffix string
imageFamily string
+ network string
+ subnet string
deprecateOld bool
oldImageTTLSec int
labels *mapVar
@@ -88,6 +90,14 @@
flags.StringVar(&f.zone, "zone", "", "Zone to make GCE resources in.")
flags.StringVar(&f.project, "project", "", "Project to make GCE resources in.")
flags.StringVar(&f.machineType, "machine-type", "n1-standard-1", "Machine type to use during the build.")
+ flags.StringVar(&f.network, "network", "", "Network to use"+
+ " during the build. The network must have access to Google Cloud Storage."+
+ " If not specified, the network named default is used."+
+ " If -subnet is also specified subnet must be a subnetwork of network specified by -network.")
+ flags.StringVar(&f.subnet, "subnet", "", "SubNetwork to use"+
+ " during the build. If not provided, default subnet would be used."+
+ " If the network is in auto subnet mode, the subnetwork is optional. "+
+ " If the network is in custom subnet mode, then this field should be specified. Zone should be specified if this field is specified.")
if f.labels == nil {
f.labels = newMapVar()
}
@@ -157,6 +167,8 @@
buildConfig.Zone = f.zone
buildConfig.MachineType = f.machineType
buildConfig.DiskSize = f.diskSize
+ buildConfig.Network = f.network
+ buildConfig.Subnet = f.subnet
buildConfig.Timeout = f.timeout.String()
provConfig := &provisioner.Config{}
if err := config.LoadFromFile(files.ProvConfig, provConfig); err != nil {
diff --git a/src/cmd/provisioner/embeds_linux_amd64.go b/src/cmd/provisioner/embeds_linux_amd64.go
index f72e1b8..427f337 100644
--- a/src/cmd/provisioner/embeds_linux_amd64.go
+++ b/src/cmd/provisioner/embeds_linux_amd64.go
@@ -24,4 +24,3 @@
//go:embed docker-credential-gcr_amd64
var dockerCredentialGCR []byte
-
diff --git a/src/cmd/provisioner/embeds_linux_arm64.go b/src/cmd/provisioner/embeds_linux_arm64.go
index 65b88cb..b3c56b2 100644
--- a/src/cmd/provisioner/embeds_linux_arm64.go
+++ b/src/cmd/provisioner/embeds_linux_arm64.go
@@ -24,4 +24,3 @@
//go:embed docker-credential-gcr_arm64
var dockerCredentialGCR []byte
-
diff --git a/src/data/build_image.wf.json b/src/data/build_image.wf.json
index 8b280ae..4c96e3d 100644
--- a/src/data/build_image.wf.json
+++ b/src/data/build_image.wf.json
@@ -8,7 +8,9 @@
"cidata_img": {"Required": true, "Description": "Path to CIDATA vfat image containing cloud-init user-data and the provisioner program. Must be in .tar.gz format."},
"disk_size_gb": {"Value": "10", "Description": "The disk size to use for preloading."},
"host_maintenance": {"Value": "MIGRATE", "Description": "VM behavior when there is maintenance."},
- "machine_type": {"Required": true, "Description": "Machine type of the preload VM."}
+ "machine_type": {"Required": true, "Description": "Machine type of the preload VM."},
+ "network": {"Value": "", "Description": "Network to use for preload VM."},
+ "subnet": {"Value": "", "Description": "Subnetwork used for the preload VM."}
},
"Sources": {
"cloud-config": "/data/startup.yaml",
@@ -56,6 +58,12 @@
"scheduling": {
"onHostMaintenance": "${host_maintenance}"
},
+ "networkInterfaces": [
+ {
+ "network": "${network}",
+ "subnetwork": "${subnet}"
+ }
+ ],
"Metadata": {
"user-data": "${SOURCE:cloud-config}",
"block-project-ssh-keys": "TRUE",
diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go
index 498882b..57debde 100644
--- a/src/pkg/config/config.go
+++ b/src/pkg/config/config.go
@@ -72,6 +72,8 @@
Timeout string
GCSFiles []string
GCEEndpoint string
+ Network string
+ Subnet string
}
// SaveConfigToFile clears the target config file and then saves the new config
diff --git a/src/pkg/preloader/preload.go b/src/pkg/preloader/preload.go
index f135031..1e8973f 100644
--- a/src/pkg/preloader/preload.go
+++ b/src/pkg/preloader/preload.go
@@ -310,6 +310,10 @@
buildSpec.MachineType,
"-var:host_maintenance",
hostMaintenance,
+ "-var:network",
+ buildSpec.Network,
+ "-var:subnet",
+ buildSpec.Subnet,
"-gcs_path",
gcs.managedDirURL(),
"-project",
diff --git a/src/pkg/preloader/preload_test.go b/src/pkg/preloader/preload_test.go
index 09510e7..2f7f1ee 100644
--- a/src/pkg/preloader/preload_test.go
+++ b/src/pkg/preloader/preload_test.go
@@ -351,6 +351,20 @@
want: []string{"-var:machine_type", "n1-standard-1"},
},
{
+ testName: "Network",
+ inputImage: config.NewImage("", ""),
+ outputImage: config.NewImage("", ""),
+ buildConfig: &config.Build{Network: "global/networks/vpc", GCSBucket: "bucket", GCSDir: "dir"},
+ want: []string{"-var:network", "global/networks/vpc"},
+ },
+ {
+ testName: "Subnet",
+ inputImage: config.NewImage("", ""),
+ outputImage: config.NewImage("", ""),
+ buildConfig: &config.Build{Subnet: "regions/us-west1/subnetworks/auto-vpc-subnet-us-west1", GCSBucket: "bucket", GCSDir: "dir"},
+ want: []string{"-var:subnet", "regions/us-west1/subnetworks/auto-vpc-subnet-us-west1"},
+ },
+ {
testName: "Timeout",
inputImage: config.NewImage("", ""),
outputImage: config.NewImage("", ""),
diff --git a/testing/network_subnet_test.yaml b/testing/network_subnet_test.yaml
new file mode 100644
index 0000000..4c39fc6
--- /dev/null
+++ b/testing/network_subnet_test.yaml
@@ -0,0 +1,60 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the License);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This test uses subnetwork `cos-customizer-test` in default VPC for the preload VM.
+
+substitutions:
+ "_TEST": "network_subnet_test"
+ "_INPUT_IMAGE": "cos-85-13310-1260-8"
+ "_INPUT_PROJECT": "cos-cloud"
+steps:
+- name: "gcr.io/cloud-builders/bazel"
+ args: ["run", "--spawn_strategy=standalone", ":cos_customizer", "--", "--norun"]
+- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+ args: [ 'gcloud', 'compute', 'networks', 'subnets',
+ 'create', 'cos-customizer-test',
+ '--project', '${PROJECT_ID}', '--network', 'default', '--region', 'us-central1',
+ '--range', '10.124.0.0/20'
+ ]
+- name: "bazel:cos_customizer"
+ args: ["start-image-build",
+ "-build-context=testing/${_TEST}",
+ "-image-name=${_INPUT_IMAGE}",
+ "-image-project=${_INPUT_PROJECT}",
+ "-gcs-bucket=${PROJECT_ID}_cloudbuild",
+ "-gcs-workdir=customizer-$BUILD_ID"]
+- name: "bazel:cos_customizer"
+ args: ["run-script",
+ "-script=preload.sh"]
+- name: "bazel:cos_customizer"
+ args: ["finish-image-build",
+ "-machine-type=n1-standard-8",
+ "-zone=us-central1-a",
+ "-project=$PROJECT_ID",
+ "-subnet=regions/us-central1/subnetworks/cos-customizer-test",
+ "-image-name=preload-test-$BUILD_ID",
+ "-image-project=$PROJECT_ID"]
+- name: "gcr.io/compute-image-tools/daisy"
+ args: ["-project=$PROJECT_ID", "-zone=us-west1-b", "-var:image_name",
+ "preload-test-$BUILD_ID", "-var:image_project", "$PROJECT_ID",
+ "-var:test_cfg", "../${_TEST}/preload_test.cfg", "testing/util/run_test.wf.json"]
+- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+ args: [ 'gcloud', 'compute', 'networks', 'subnets',
+ 'delete', 'cos-customizer-test',
+ '--project', '${PROJECT_ID}', '--region', 'us-central1',
+ ]
+options:
+ machineType: "N1_HIGHCPU_8"
+timeout: "7200s"
diff --git a/testing/network_subnet_test/preload.sh b/testing/network_subnet_test/preload.sh
new file mode 100644
index 0000000..fa6feac
--- /dev/null
+++ b/testing/network_subnet_test/preload.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+echo "Getting auth token."
+AUTH_DATA="$(curl -s -f -m 10 "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" -H "Metadata-Flavor: Google")"
+R=$?
+if [ ${R} -ne 0 ]; then
+ echo "Getting auth token error, exited with status ${R}" >&2
+ exit ${R}
+fi
+
+AUTH="$(echo "${AUTH_DATA}" \
+| tr -d '{}' \
+| sed 's/,/\n/g' \
+| awk -F ':' '/access_token/ { print $2 }' \
+| tr -d '"\n')"
+
+if [ -z "${AUTH}" ]; then
+ echo "Auth token not found in AUTH_DATA ${AUTH_DATA}" >&2
+ exit 1
+fi
+
+echo "Getting instance project and zone."
+PROJ_ZONE="$(curl -s -f -m 10 "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")"
+
+R=$?
+if [ ${R} -ne 0 ]; then
+ echo "Getting project and zone error, exited with status ${R}" >&2
+ exit ${R}
+fi
+
+echo "Getting instance name."
+INSTANCE_NAME="$(curl -s -f -m 10 "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")"
+R=$?
+if [ ${R} -ne 0 ]; then
+ echo "Getting instance name error, exited with status ${R}" >&2
+ exit ${R}
+fi
+
+# Save the output of the instance detail to a local destination
+curl "https://www.googleapis.com/compute/v1/${PROJ_ZONE}/instances/${INSTANCE_NAME}" -H "Authorization":"Bearer ${AUTH}" --header 'Accept: application/json' --compressed > /var/lib/instance_info.json
diff --git a/testing/network_subnet_test/preload_test.cfg b/testing/network_subnet_test/preload_test.cfg
new file mode 100644
index 0000000..b078a0e
--- /dev/null
+++ b/testing/network_subnet_test/preload_test.cfg
@@ -0,0 +1,80 @@
+#cloud-config
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+write_files:
+ - path: /tmp/preloader-test/test.sh
+ permissions: 0644
+ owner: root
+ content: |
+ set -o errexit
+ set -o pipefail
+
+ trap 'fail exiting due to errors' EXIT
+
+ fail() {
+ echo "TestFail: $@"
+ }
+
+ testNetworkSubnet() {
+ if [[ ! -f "/var/lib/instance_info.json" ]]; then
+ echo "/var/lib/instance_info.json is missing"
+ echo "testNetworkSubnet failed"
+ RESULT="fail"
+ return
+ fi
+ textPatternFound=$(grep -R regions/us-central1/subnetworks/cos-customizer-test "/var/lib/instance_info.json")
+ if [[ -z ${textPatternFound} ]]; then
+ echo "/var/lib/instance_info.json: got $(cat "/var/lib/instance_info.json")"
+ echo "testNetworkSubnet fail"
+ RESULT="fail"
+ return
+ fi
+ echo "testNetworkSubnet pass"
+ }
+
+ main() {
+ RESULT="pass"
+ testNetworkSubnet
+ if [[ "${RESULT}" == "fail" ]]; then
+ exit 1
+ fi
+ }
+
+ main 2>&1 | sed "s/^/TestStatus: /"
+ trap - EXIT
+ echo "TestPass: all tests passed"
+
+ - path: /etc/systemd/system/preloader-test.service
+ permissions: 0644
+ owner: root
+ content: |
+ [Unit]
+ Description=Preloader test
+ Wants=network-online.target gcr-online.target docker.service
+ After=network-online.target gcr-online.target docker.service
+
+ [Service]
+ Type=oneshot
+ RemainAfterExit=yes
+ User=root
+ ExecStart=/bin/bash /tmp/preloader-test/test.sh
+ StandardOutput=tty
+ StandardError=tty
+ TTYPath=/dev/ttyS1
+
+runcmd:
+ - systemctl daemon-reload
+ - systemctl --no-block start preloader-test.service
