image/tarexport/save.go

package tarexport // import "github.com/docker/docker/image/tarexport"

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/tracing"
	"github.com/containerd/log"
	"github.com/distribution/reference"
	"github.com/docker/distribution"
	"github.com/docker/docker/api/types/events"
	"github.com/docker/docker/image"
	v1 "github.com/docker/docker/image/v1"
	"github.com/docker/docker/internal/ioutils"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/system"
	"github.com/moby/sys/sequential"
	"github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/specs-go"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type imageDescriptor struct {
	refs     []reference.NamedTagged
	layers   []layer.DiffID
	image    *image.Image
	layerRef layer.Layer
}

type saveSession struct {
	*tarexporter
	outDir       string
	images       map[image.ID]*imageDescriptor
	savedLayers  map[layer.DiffID]distribution.Descriptor
	savedConfigs map[string]struct{}
}

func (l *tarexporter) Save(ctx context.Context, names []string, outStream io.Writer) error {
	images, err := l.parseNames(ctx, names)
	if err != nil {
		return err
	}

	// Release all the image top layer references
	defer l.releaseLayerReferences(images)
	return (&saveSession{tarexporter: l, images: images}).save(ctx, outStream)
}

// parseNames will parse the image names to a map which contains image.ID to *imageDescriptor.
// Each imageDescriptor holds an image top layer reference named 'layerRef'. It is taken here, should be released later.
func (l *tarexporter) parseNames(ctx context.Context, names []string) (desc map[image.ID]*imageDescriptor, rErr error) {
	imgDescr := make(map[image.ID]*imageDescriptor)
	defer func() {
		if rErr != nil {
			l.releaseLayerReferences(imgDescr)
		}
	}()

	addAssoc := func(id image.ID, ref reference.Named) error {
		if _, ok := imgDescr[id]; !ok {
			descr := &imageDescriptor{}
			if err := l.takeLayerReference(id, descr); err != nil {
				return err
			}
			imgDescr[id] = descr
		}

		if ref != nil {
			if _, ok := ref.(reference.Canonical); ok {
				return nil
			}
			tagged, ok := reference.TagNameOnly(ref).(reference.NamedTagged)
			if !ok {
				return nil
			}

			for _, t := range imgDescr[id].refs {
				if tagged.String() == t.String() {
					return nil
				}
			}
			imgDescr[id].refs = append(imgDescr[id].refs, tagged)
		}
		return nil
	}

	for _, name := range names {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}

		ref, err := reference.ParseAnyReference(name)
		if err != nil {
			return nil, err
		}
		namedRef, ok := ref.(reference.Named)
		if !ok {
			// Check if digest ID reference
			if digested, ok := ref.(reference.Digested); ok {
				if err := addAssoc(image.ID(digested.Digest()), nil); err != nil {
					return nil, err
				}
				continue
			}
			return nil, errors.Errorf("invalid reference: %v", name)
		}

		if reference.FamiliarName(namedRef) == string(digest.Canonical) {
			imgID, err := l.is.Search(name)
			if err != nil {
				return nil, err
			}
			if err := addAssoc(imgID, nil); err != nil {
				return nil, err
			}
			continue
		}
		if reference.IsNameOnly(namedRef) {
			assocs := l.rs.ReferencesByName(namedRef)
			for _, assoc := range assocs {
				if err := addAssoc(image.ID(assoc.ID), assoc.Ref); err != nil {
					return nil, err
				}
			}
			if len(assocs) == 0 {
				imgID, err := l.is.Search(name)
				if err != nil {
					return nil, err
				}
				if err := addAssoc(imgID, nil); err != nil {
					return nil, err
				}
			}
			continue
		}
		id, err := l.rs.Get(namedRef)
		if err != nil {
			return nil, err
		}
		if err := addAssoc(image.ID(id), namedRef); err != nil {
			return nil, err
		}
	}
	return imgDescr, nil
}

// takeLayerReference will take/Get the image top layer reference
func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor) error {
	img, err := l.is.Get(id)
	if err != nil {
		return err
	}
	if err := image.CheckOS(img.OperatingSystem()); err != nil {
		return fmt.Errorf("os %q is not supported", img.OperatingSystem())
	}
	imgDescr.image = img
	topLayerID := img.RootFS.ChainID()
	if topLayerID == "" {
		return nil
	}
	layer, err := l.lss.Get(topLayerID)
	if err != nil {
		return err
	}
	imgDescr.layerRef = layer
	return nil
}

// releaseLayerReferences will release all the image top layer references
func (l *tarexporter) releaseLayerReferences(imgDescr map[image.ID]*imageDescriptor) error {
	for _, descr := range imgDescr {
		if descr.layerRef != nil {
			l.lss.Release(descr.layerRef)
		}
	}
	return nil
}

func (s *saveSession) save(ctx context.Context, outStream io.Writer) error {
	s.savedConfigs = make(map[string]struct{})
	s.savedLayers = make(map[layer.DiffID]distribution.Descriptor)

	// get image json
	tempDir, err := os.MkdirTemp("", "docker-export-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempDir)

	s.outDir = tempDir
	reposLegacy := make(map[string]map[string]string)

	var manifest []manifestItem
	var parentLinks []parentLink

	var manifestDescriptors []ocispec.Descriptor

	for id, imageDescr := range s.images {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		foreignSrcs, err := s.saveImage(ctx, id)
		if err != nil {
			return err
		}

		var (
			repoTags []string
			layers   []string
			foreign  = make([]ocispec.Descriptor, 0, len(foreignSrcs))
		)

		// Layers in manifest must follow the actual layer order from config.
		for _, l := range imageDescr.layers {
			desc := foreignSrcs[l]
			foreign = append(foreign, ocispec.Descriptor{
				MediaType:   desc.MediaType,
				Digest:      desc.Digest,
				Size:        desc.Size,
				URLs:        desc.URLs,
				Annotations: desc.Annotations,
				Platform:    desc.Platform,
			})
		}

		m := ocispec.Manifest{
			Versioned: specs.Versioned{
				SchemaVersion: 2,
			},
			MediaType: ocispec.MediaTypeImageManifest,
			Config: ocispec.Descriptor{
				MediaType: ocispec.MediaTypeImageConfig,
				Digest:    digest.Digest(imageDescr.image.ID()),
				Size:      int64(len(imageDescr.image.RawJSON())),
			},
			Layers: foreign,
		}

		data, err := json.Marshal(m)
		if err != nil {
			return errors.Wrap(err, "error marshaling manifest")
		}
		dgst := digest.FromBytes(data)

		mFile := filepath.Join(s.outDir, ocispec.ImageBlobsDir, dgst.Algorithm().String(), dgst.Encoded())
		if err := os.MkdirAll(filepath.Dir(mFile), 0o755); err != nil {
			return errors.Wrap(err, "error creating blob directory")
		}
		if err := system.Chtimes(filepath.Dir(mFile), time.Unix(0, 0), time.Unix(0, 0)); err != nil {
			return errors.Wrap(err, "error setting blob directory timestamps")
		}
		if err := os.WriteFile(mFile, data, 0o644); err != nil {
			return errors.Wrap(err, "error writing oci manifest file")
		}
		if err := system.Chtimes(mFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
			return errors.Wrap(err, "error setting blob directory timestamps")
		}
		size := int64(len(data))

		untaggedMfstDesc := ocispec.Descriptor{
			MediaType: ocispec.MediaTypeImageManifest,
			Digest:    dgst,
			Size:      size,
			Platform:  m.Config.Platform,
		}
		for _, ref := range imageDescr.refs {
			familiarName := reference.FamiliarName(ref)
			if _, ok := reposLegacy[familiarName]; !ok {
				reposLegacy[familiarName] = make(map[string]string)
			}
			reposLegacy[familiarName][ref.Tag()] = digest.Digest(imageDescr.layers[len(imageDescr.layers)-1]).Encoded()
			repoTags = append(repoTags, reference.FamiliarString(ref))

			taggedManifest := untaggedMfstDesc
			taggedManifest.Annotations = map[string]string{
				images.AnnotationImageName: ref.String(),
				ocispec.AnnotationRefName:  ref.Tag(),
			}
			manifestDescriptors = append(manifestDescriptors, taggedManifest)
		}

		// If no ref was assigned, make sure still add the image is still included in index.json.
		if len(manifestDescriptors) == 0 {
			manifestDescriptors = append(manifestDescriptors, untaggedMfstDesc)
		}

		for _, l := range imageDescr.layers {
			// IMPORTANT: We use path, not filepath here to ensure the layers
			// in the manifest use Unix-style forward-slashes.
			lDgst := digest.Digest(l)
			layers = append(layers, path.Join(ocispec.ImageBlobsDir, lDgst.Algorithm().String(), lDgst.Encoded()))
		}

		manifest = append(manifest, manifestItem{
			Config:       path.Join(ocispec.ImageBlobsDir, id.Digest().Algorithm().String(), id.Digest().Encoded()),
			RepoTags:     repoTags,
			Layers:       layers,
			LayerSources: foreignSrcs,
		})

		parentID, _ := s.is.GetParent(id)
		parentLinks = append(parentLinks, parentLink{id, parentID})
		s.tarexporter.loggerImgEvent.LogImageEvent(id.String(), id.String(), events.ActionSave)
	}

	for i, p := range validatedParentLinks(parentLinks) {
		if p.parentID != "" {
			manifest[i].Parent = p.parentID
		}
	}

	if len(reposLegacy) > 0 {
		reposFile := filepath.Join(tempDir, legacyRepositoriesFileName)
		rf, err := os.OpenFile(reposFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
		if err != nil {
			return err
		}

		if err := json.NewEncoder(rf).Encode(reposLegacy); err != nil {
			rf.Close()
			return err
		}

		rf.Close()

		if err := system.Chtimes(reposFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
			return err
		}
	}

	manifestPath := filepath.Join(tempDir, manifestFileName)
	f, err := os.OpenFile(manifestPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
	if err != nil {
		return err
	}

	if err := json.NewEncoder(f).Encode(manifest); err != nil {
		f.Close()
		return err
	}

	f.Close()

	if err := system.Chtimes(manifestPath, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
		return err
	}

	const ociLayoutContent = `{"imageLayoutVersion": "` + ocispec.ImageLayoutVersion + `"}`
	layoutPath := filepath.Join(tempDir, ocispec.ImageLayoutFile)
	if err := os.WriteFile(layoutPath, []byte(ociLayoutContent), 0o644); err != nil {
		return errors.Wrap(err, "error writing oci layout file")
	}
	if err := system.Chtimes(layoutPath, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
		return errors.Wrap(err, "error setting oci layout file timestamps")
	}

	data, err := json.Marshal(ocispec.Index{
		Versioned: specs.Versioned{
			SchemaVersion: 2,
		},
		MediaType: ocispec.MediaTypeImageIndex,
		Manifests: manifestDescriptors,
	})
	if err != nil {
		return errors.Wrap(err, "error marshaling oci index")
	}

	idxFile := filepath.Join(s.outDir, ocispec.ImageIndexFile)
	if err := os.WriteFile(idxFile, data, 0o644); err != nil {
		return errors.Wrap(err, "error writing oci index file")
	}

	return s.writeTar(ctx, tempDir, outStream)
}

func (s *saveSession) writeTar(ctx context.Context, tempDir string, outStream io.Writer) error {
	ctx, span := tracing.StartSpan(ctx, "writeTar")
	defer span.End()

	fs, err := archive.Tar(tempDir, archive.Uncompressed)
	if err != nil {
		span.SetStatus(err)
		return err
	}
	defer fs.Close()

	_, err = ioutils.CopyCtx(ctx, outStream, fs)

	span.SetStatus(err)
	return err
}

func (s *saveSession) saveImage(ctx context.Context, id image.ID) (_ map[layer.DiffID]distribution.Descriptor, outErr error) {
	ctx, span := tracing.StartSpan(ctx, "saveImage")
	span.SetAttributes(tracing.Attribute("image.id", id.String()))
	defer span.End()
	defer func() {
		span.SetStatus(outErr)
	}()

	img := s.images[id].image
	if len(img.RootFS.DiffIDs) == 0 {
		return nil, fmt.Errorf("empty export - not implemented")
	}

	var parent digest.Digest
	var layers []layer.DiffID
	var foreignSrcs map[layer.DiffID]distribution.Descriptor
	for i, diffID := range img.RootFS.DiffIDs {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		v1ImgCreated := time.Unix(0, 0)
		v1Img := image.V1Image{
			// This is for backward compatibility used for
			// pre v1.9 docker.
			Created: &v1ImgCreated,
		}
		if i == len(img.RootFS.DiffIDs)-1 {
			v1Img = img.V1Image
		}
		rootFS := *img.RootFS
		rootFS.DiffIDs = rootFS.DiffIDs[:i+1]
		v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent)
		if err != nil {
			return nil, err
		}

		v1Img.ID = v1ID.Encoded()
		if parent != "" {
			v1Img.Parent = parent.Encoded()
		}

		v1Img.OS = img.OS
		src, err := s.saveConfigAndLayer(ctx, rootFS.ChainID(), v1Img, img.Created)
		if err != nil {
			return nil, err
		}

		layers = append(layers, diffID)
		parent = v1ID
		if src.Digest != "" {
			if foreignSrcs == nil {
				foreignSrcs = make(map[layer.DiffID]distribution.Descriptor)
			}
			foreignSrcs[img.RootFS.DiffIDs[i]] = src
		}
	}

	data := img.RawJSON()
	dgst := digest.FromBytes(data)

	blobDir := filepath.Join(s.outDir, ocispec.ImageBlobsDir, dgst.Algorithm().String())
	if err := os.MkdirAll(blobDir, 0o755); err != nil {
		return nil, err
	}
	if img.Created != nil {
		if err := system.Chtimes(blobDir, *img.Created, *img.Created); err != nil {
			return nil, err
		}
		if err := system.Chtimes(filepath.Dir(blobDir), *img.Created, *img.Created); err != nil {
			return nil, err
		}
	}

	configFile := filepath.Join(blobDir, dgst.Encoded())
	if err := os.WriteFile(configFile, img.RawJSON(), 0o644); err != nil {
		return nil, err
	}
	if img.Created != nil {
		if err := system.Chtimes(configFile, *img.Created, *img.Created); err != nil {
			return nil, err
		}
	}

	s.images[id].layers = layers
	return foreignSrcs, nil
}

func (s *saveSession) saveConfigAndLayer(ctx context.Context, id layer.ChainID, legacyImg image.V1Image, createdTime *time.Time) (_ distribution.Descriptor, outErr error) {
	ctx, span := tracing.StartSpan(ctx, "saveConfigAndLayer")
	span.SetAttributes(
		tracing.Attribute("layer.id", id.String()),
		tracing.Attribute("image.id", legacyImg.ID),
	)
	defer span.End()
	defer func() {
		span.SetStatus(outErr)
	}()

	outDir := filepath.Join(s.outDir, ocispec.ImageBlobsDir)

	if _, ok := s.savedConfigs[legacyImg.ID]; !ok {
		if err := s.saveConfig(legacyImg, outDir, createdTime); err != nil {
			return distribution.Descriptor{}, err
		}
	}

	// serialize filesystem
	l, err := s.lss.Get(id)
	if err != nil {
		return distribution.Descriptor{}, err
	}

	lDiffID := l.DiffID()
	lDgst := digest.Digest(lDiffID)
	if _, ok := s.savedLayers[lDiffID]; ok {
		return s.savedLayers[lDiffID], nil
	}
	layerPath := filepath.Join(outDir, lDgst.Algorithm().String(), lDgst.Encoded())
	defer layer.ReleaseAndLog(s.lss, l)

	if _, err = os.Stat(layerPath); err == nil {
		// This is should not happen. If the layer path was already created, we should have returned early.
		// Log a warning an proceed to recreate the archive.
		log.G(context.TODO()).WithFields(log.Fields{
			"layerPath": layerPath,
			"id":        id,
			"lDgst":     lDgst,
		}).Warn("LayerPath already exists but the descriptor is not cached")
	} else if !os.IsNotExist(err) {
		return distribution.Descriptor{}, err
	}

	// We use sequential file access to avoid depleting the standby list on
	// Windows. On Linux, this equates to a regular os.Create.
	if err := os.MkdirAll(filepath.Dir(layerPath), 0o755); err != nil {
		return distribution.Descriptor{}, errors.Wrap(err, "could not create layer dir parent")
	}
	tarFile, err := sequential.Create(layerPath)
	if err != nil {
		return distribution.Descriptor{}, errors.Wrap(err, "error creating layer file")
	}
	defer tarFile.Close()

	arch, err := l.TarStream()
	if err != nil {
		return distribution.Descriptor{}, err
	}
	defer arch.Close()

	digester := digest.Canonical.Digester()
	digestedArch := io.TeeReader(arch, digester.Hash())

	tarSize, err := ioutils.CopyCtx(ctx, tarFile, digestedArch)
	if err != nil {
		return distribution.Descriptor{}, err
	}

	tarDigest := digester.Digest()
	if lDgst != tarDigest {
		log.G(context.TODO()).WithFields(log.Fields{
			"layerDigest":  lDgst,
			"actualDigest": tarDigest,
		}).Warn("layer digest doesn't match its tar archive digest")

		lDgst = digester.Digest()
		layerPath = filepath.Join(outDir, lDgst.Algorithm().String(), lDgst.Encoded())
	}

	if createdTime != nil {
		for _, fname := range []string{outDir, layerPath} {
			// todo: maybe save layer created timestamp?
			if err := system.Chtimes(fname, *createdTime, *createdTime); err != nil {
				return distribution.Descriptor{}, errors.Wrap(err, "could not set layer timestamp")
			}
		}
	}

	var desc distribution.Descriptor
	if fs, ok := l.(distribution.Describable); ok {
		desc = fs.Descriptor()
	}

	if desc.Digest == "" {
		desc.Digest = tarDigest
		desc.Size = tarSize
	}
	if desc.MediaType == "" {
		desc.MediaType = ocispec.MediaTypeImageLayer
	}
	s.savedLayers[lDiffID] = desc

	return desc, nil
}

func (s *saveSession) saveConfig(legacyImg image.V1Image, outDir string, createdTime *time.Time) error {
	imageConfig, err := json.Marshal(legacyImg)
	if err != nil {
		return err
	}

	cfgDgst := digest.FromBytes(imageConfig)
	configPath := filepath.Join(outDir, cfgDgst.Algorithm().String(), cfgDgst.Encoded())
	if err := os.MkdirAll(filepath.Dir(configPath), 0o755); err != nil {
		return errors.Wrap(err, "could not create layer dir parent")
	}

	if err := os.WriteFile(configPath, imageConfig, 0o644); err != nil {
		return err
	}

	if createdTime != nil {
		if err := system.Chtimes(configPath, *createdTime, *createdTime); err != nil {
			return errors.Wrap(err, "could not set config timestamp")
		}
	}

	s.savedConfigs[legacyImg.ID] = struct{}{}
	return nil
}