blob: e69870b0bc0a8adaff29ba2cc0d67b8789411b3a [file] [log] [blame]
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"flag"
"fmt"
"log"
"os/exec"
"strconv"
"strings"
"time"
"cos.googlesource.com/cos/tools.git/src/pkg/config"
"cos.googlesource.com/cos/tools.git/src/pkg/fs"
"cos.googlesource.com/cos/tools.git/src/pkg/gce"
"cos.googlesource.com/cos/tools.git/src/pkg/preloader"
"cos.googlesource.com/cos/tools.git/src/pkg/provisioner"
"cos.googlesource.com/cos/tools.git/src/pkg/tools/partutil"
"github.com/google/subcommands"
)
// FinishImageBuild implements subcommands.Command for the "finish-image-build" command.
// This command finishes an image build by converting saved image configurations into
// an actual GCE image.
type FinishImageBuild struct {
imageProject string
zone string
project string
machineType string
imageName string
imageSuffix string
imageFamily string
deprecateOld bool
oldImageTTLSec int
labels *mapVar
licenses *listVar
inheritLabels bool
oemSize string
oemFSSize4K uint64
diskSize int
timeout time.Duration
}
// Name implements subcommands.Command.Name.
func (f *FinishImageBuild) Name() string {
return "finish-image-build"
}
// Synopsis implements subcommands.Command.Synopsis.
func (f *FinishImageBuild) Synopsis() string {
return "Complete the COS image build and generate a GCE image."
}
// Usage implements subcommands.Command.Usage.
func (f *FinishImageBuild) Usage() string {
return `finish-image-build [flags]
`
}
// SetFlags implements subcommands.Command.SetFlags.
func (f *FinishImageBuild) SetFlags(flags *flag.FlagSet) {
flags.StringVar(&f.imageProject, "image-project", "", "Output image project.")
flags.StringVar(&f.imageName, "image-name", "", "Output image name. Mutually exclusive with 'image-suffix'.")
flags.StringVar(&f.imageSuffix, "image-suffix", "", "Construct the output image name from the input image "+
"name and this suffix. Mutually exclusive with 'image-name'.")
flags.StringVar(&f.imageFamily, "image-family", "", "Output image family.")
flags.BoolVar(&f.deprecateOld, "deprecate-old-images", false, "Deprecate old images in the output image "+
"family. Can only be used if 'image-family' is set.")
flags.IntVar(&f.oldImageTTLSec, "old-image-ttl", 0, "Time-to-live in seconds for old images that are "+
"deprecated. After this period of time, old images will enter the deleted state. Can only be used if "+
"'deprecate-old-images' is set. '0' indicates no time-to-live (images won't be configured to enter "+
"the deleted state).")
flags.StringVar(&f.zone, "zone", "", "Zone to make GCE resources in.")
flags.StringVar(&f.project, "project", "", "Project to make GCE resources in.")
flags.StringVar(&f.machineType, "machine-type", "n1-standard-1", "Machine type to use during the build.")
if f.labels == nil {
f.labels = newMapVar()
}
flags.Var(f.labels, "labels", "Image labels to apply to the result image. Format is "+
"'key1=value1,key2=value2,...'. Example: -labels=hello=world,foo=bar")
if f.licenses == nil {
f.licenses = &listVar{}
}
flags.Var(f.licenses, "licenses", "Image licenses to apply to the result image. Format is "+
"'license1,license2,...' or '-licenses=license1 -licenses=license2'.")
flags.BoolVar(&f.inheritLabels, "inherit-labels", false, "Indicates if the result image should inherit labels "+
"from the source image. Labels specified through the '-labels' flag take precedence over inherited "+
"labels.")
flags.StringVar(&f.oemSize, "oem-size", "", "Size of the new OEM partition, "+
"can be a number with unit like 10G, 10M, 10K or 10B, "+
"or without unit indicating the number of 512B sectors.")
flags.IntVar(&f.diskSize, "disk-size-gb", 0, "The disk size to use when creating the image in GB. Value of '0' "+
"indicates the default size.")
flags.DurationVar(&f.timeout, "timeout", time.Hour, "Timeout value of the image build process. Must be formatted "+
"according to Golang's time.Duration string format.")
}
func (f *FinishImageBuild) validate() error {
// The default size of the OEM partition in a COS image is assumed to be 16MB.
const defaultOEMSizeMB = 16
if f.oemSize != "" {
oemSizeBytes, err := partutil.ConvertSizeToBytes(f.oemSize)
if err != nil {
return fmt.Errorf("invalid format of oem-size: %q, error msg:(%v)", f.oemSize, err)
}
if oemSizeBytes < (defaultOEMSizeMB << 20) {
return fmt.Errorf("oem-size must be at least %dM", defaultOEMSizeMB)
}
}
switch {
case f.imageName == "" && f.imageSuffix == "":
return fmt.Errorf("one of 'image-name' or 'image-suffix' must be set")
case f.imageName != "" && f.imageSuffix != "":
return fmt.Errorf("'image-name' and 'image-suffix' are mutually exclusive")
case f.deprecateOld && f.imageFamily == "":
return fmt.Errorf("'deprecate-old-images' can only be used if 'image-family' is set")
case f.oldImageTTLSec != 0 && !f.deprecateOld:
return fmt.Errorf("'old-image-ttl' can only be used if 'deprecate-old-images' is set")
case f.zone == "":
return fmt.Errorf("'zone' must be set")
case f.project == "":
return fmt.Errorf("'project' must be set")
default:
return nil
}
}
func (f *FinishImageBuild) loadConfigs(files *fs.Files) (*config.Image, *config.Build, *config.Image, *provisioner.Config, error) {
sourceImageConfig := &config.Image{}
if err := config.LoadFromFile(files.SourceImageConfig, sourceImageConfig); err != nil {
return nil, nil, nil, nil, err
}
imageName := f.imageName
if f.imageSuffix != "" {
imageName = sourceImageConfig.Name + f.imageSuffix
}
buildConfig := &config.Build{}
if err := config.LoadFromFile(files.BuildConfig, buildConfig); err != nil {
return nil, nil, nil, nil, err
}
buildConfig.Project = f.project
buildConfig.Zone = f.zone
buildConfig.MachineType = f.machineType
buildConfig.DiskSize = f.diskSize
buildConfig.Timeout = f.timeout.String()
provConfig := &provisioner.Config{}
if err := config.LoadFromFile(files.ProvConfig, provConfig); err != nil {
return nil, nil, nil, nil, err
}
provConfig.BootDisk.OEMSize = f.oemSize
outputImageConfig := config.NewImage(imageName, f.imageProject)
outputImageConfig.Labels = f.labels.m
outputImageConfig.Licenses = f.licenses.l
outputImageConfig.Family = f.imageFamily
return sourceImageConfig, buildConfig, outputImageConfig, provConfig, nil
}
func hasSealOEM(provConfig *provisioner.Config) bool {
for _, s := range provConfig.Steps {
if s.Type == "SealOEM" {
return true
}
}
return false
}
func validateOEM(buildConfig *config.Build, provConfig *provisioner.Config) error {
// The default size of a COS image (imgSize) is assumed to be 10GB.
const imgSize uint64 = 10
// If auto-update is disabled, 2046MB will be reclaimed.
// The size of sda3 is 2GB. We don't want to delete the partition,
// so we need to leave some space in the sda3. And `sfdisk --move-data`
// in some situations requires 1MB free space in the moving direction.
// Therefore, leaving 2MB after the start of sda3 is a safe choice.
// Also, this will make sure the start point of the next partition is
// 4K aligned.
const reclaimedMB uint64 = 2046
const reclaimedBytes uint64 = reclaimedMB << 20
var sizeError error
var oemSizeBytes uint64
var err error
if !hasSealOEM(provConfig) {
if provConfig.BootDisk.OEMSize == "" {
return nil
}
// no need to seal the OEM partition.
// If the OEM partition is to be extended, the following must be true:
// disk-size >= imgSize + oem-size - reclaimed-size.
if provConfig.BootDisk.ReclaimSDA3 {
sizeError = fmt.Errorf("'disk-size-gb' must be at least 'oem-size'- reclaimed space "+
"(%dMB) + image size (%dGB)", reclaimedMB, imgSize)
} else {
sizeError = fmt.Errorf("'disk-size-gb' must be at least 'oem-size' + image size (%dGB)", imgSize)
}
oemSizeBytes, err = partutil.ConvertSizeToBytes(provConfig.BootDisk.OEMSize)
if err != nil {
return fmt.Errorf("invalid format of oem-size: %q, error msg:(%v)", provConfig.BootDisk.OEMSize, err)
}
} else {
// `seal-oem` will automatically disable auto-update and reclaim sda3.
if provConfig.BootDisk.OEMSize == "" {
// If need to seal OEM partition and the oem-size is not set,
// assume the OEM fs size is 16M as it is in a COS image,
// and the OEM partition size is doubled to 32M.
// It will use space reclaimed from sda3.
provConfig.BootDisk.OEMSize = "32M"
provConfig.BootDisk.OEMFSSize4K = 4096
return nil
}
// need extra space to seal the OEM partition.
// The OEM partition size should be doubled to store the
// hash tree of dm-verity. The following must be true:
// disk-size >= imgSize + oem-size x 2 - reclaimed-size.
sizeError = fmt.Errorf("'disk-size-gb' must be at least 'oem-size' x 2 - reclaimed space "+
"(%dMB) + image size (%dGB)", reclaimedMB, imgSize)
oemSizeBytes, err = partutil.ConvertSizeToBytes(provConfig.BootDisk.OEMSize)
if err != nil {
return fmt.Errorf("invalid format of oem-size: %q, error msg:(%v)", provConfig.BootDisk.OEMSize, err)
}
provConfig.BootDisk.OEMFSSize4K = oemSizeBytes >> 12
// double the oem size.
oemSizeBytes <<= 1
}
// Since we allow user input like "500M", and the "resize-disk" API can only take GB as input,
// the oem-size is rounded up to GB to make sure there is enough space.
// If the auto-update is disabled, space in sda3 will be reclaimed and used.
// Extra space will be taken by the stateful partition.
oemSizeReclaimBytes := oemSizeBytes
if provConfig.BootDisk.ReclaimSDA3 {
if oemSizeReclaimBytes <= reclaimedBytes {
oemSizeReclaimBytes = 0
} else {
oemSizeReclaimBytes -= reclaimedBytes
}
}
oemSizeGB, err := partutil.ConvertSizeToGBRoundUp(strconv.FormatUint(oemSizeReclaimBytes, 10) + "B")
if err != nil {
return fmt.Errorf("invalid format of oem-size: %q, error msg:(%v)", provConfig.BootDisk.OEMSize, err)
}
// if no disk-size-gb input, assume the default image size to be 10GB.
var diskSize uint64 = imgSize
if buildConfig.DiskSize != 0 {
diskSize = (uint64)(buildConfig.DiskSize)
}
if diskSize < imgSize+oemSizeGB {
return sizeError
}
// Shrink OEM size input (rounded down) by 1MB to deal with cases
// where disk size is 1MB smaller than needed.
// This will take 1MB from the hash tree part (the second half)
// of the OEM partition if seal-oem is set. Otherwise, it will
// take 1MB from user data space of the OEM partition.
// For example oem-size=1G, disk-size-gb=11, seal-oem not set.
// Or oem-size=1G, disk-size-gb=11, seal-oem set.
// In those cases the disk size is not large enough without shrinking
// the OEM partition size by 1MB.
provConfig.BootDisk.OEMSize = strconv.FormatUint((oemSizeBytes>>20)-1, 10) + "M"
return nil
}
func update(dst, src map[string]string) {
for k, v := range src {
if _, ok := dst[k]; !ok {
dst[k] = v
}
}
}
// Execute implements subcommands.Command.Execute. It gathers image configuration parameters
// and creates a GCE image.
func (f *FinishImageBuild) Execute(ctx context.Context, flags *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
if flags.NArg() != 0 {
flags.Usage()
return subcommands.ExitUsageError
}
files := args[0].(*fs.Files)
defer files.CleanupAllPersistent()
svc, gcsClient, err := args[1].(ServiceClients)(ctx, false)
if err != nil {
log.Println(err)
return subcommands.ExitFailure
}
defer gcsClient.Close()
if err := f.validate(); err != nil {
log.Println(err)
return subcommands.ExitFailure
}
sourceImage, buildConfig, outputImage, provConfig, err := f.loadConfigs(files)
if err != nil {
log.Println(err)
return subcommands.ExitFailure
}
// Set non-default GCE endpoints in the buildConfig.
if !strings.Contains(svc.BasePath, "compute.googleapis.com/compute/v1") {
buildConfig.GCEEndpoint = svc.BasePath
}
if err := validateOEM(buildConfig, provConfig); err != nil {
log.Println(err)
return subcommands.ExitFailure
}
exists, err := gce.ImageExists(svc, outputImage.Project, outputImage.Name)
if err != nil {
log.Println(err)
return subcommands.ExitFailure
}
if exists {
log.Printf("Result image %s already exists in project %s. Exiting.\n", outputImage.Name, outputImage.Project)
return subcommands.ExitSuccess
}
if f.inheritLabels {
image, err := svc.Images.Get(sourceImage.Project, sourceImage.Name).Do()
if err != nil {
log.Println(err)
return subcommands.ExitFailure
}
update(outputImage.Labels, image.Labels)
}
if err := preloader.BuildImage(ctx, gcsClient, files, sourceImage, outputImage, buildConfig, provConfig); err != nil {
if _, ok := err.(*exec.ExitError); ok {
log.Printf("command failed: %s. See stdout logs for details", err)
return subcommands.ExitFailure
}
log.Println(err)
return subcommands.ExitFailure
}
if f.deprecateOld {
if err := gce.DeprecateInFamily(ctx, svc, outputImage, f.oldImageTTLSec); err != nil {
log.Printf("deprecating images failed: %s", err)
return subcommands.ExitFailure
}
}
return subcommands.ExitSuccess
}