integration/plugin/graphdriver/external_test.go

package graphdriver

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"runtime"
	"testing"

	"github.com/docker/docker/api/types"
	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/image"
	"github.com/docker/docker/client"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/daemon/graphdriver/vfs"
	"github.com/docker/docker/integration/internal/container"
	"github.com/docker/docker/integration/internal/requirement"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/plugins"
	"github.com/docker/docker/testutil"
	"github.com/docker/docker/testutil/daemon"
	"gotest.tools/v3/assert"
	is "gotest.tools/v3/assert/cmp"
	"gotest.tools/v3/skip"
)

type graphEventsCounter struct {
	activations int
	creations   int
	removals    int
	gets        int
	puts        int
	stats       int
	cleanups    int
	exists      int
	init        int
	metadata    int
	diff        int
	applydiff   int
	changes     int
	diffsize    int
}

func TestExternalGraphDriver(t *testing.T) {
	skip.If(t, testEnv.UsingSnapshotter())
	skip.If(t, runtime.GOOS == "windows")
	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
	skip.If(t, !requirement.HasHubConnectivity(t))
	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support external graph driver")

	ctx := testutil.StartSpan(baseContext, t)

	// Setup plugin(s)
	ec := make(map[string]*graphEventsCounter)
	sserver := setupPluginViaSpecFile(t, ec)
	jserver := setupPluginViaJSONFile(t, ec)
	// Create daemon
	d := daemon.New(t, daemon.WithExperimental(), daemon.WithEnvVars("DOCKERD_DEPRECATED_GRAPHDRIVER_PLUGINS=1"))
	c := d.NewClientT(t)

	for _, tc := range []struct {
		name string
		test func(context.Context, client.APIClient, *daemon.Daemon) func(*testing.T)
	}{
		{
			name: "json",
			test: testExternalGraphDriver("json", ec),
		},
		{
			name: "spec",
			test: testExternalGraphDriver("spec", ec),
		},
		{
			name: "pull",
			test: testGraphDriverPull,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			ctx := testutil.StartSpan(ctx, t)
			tc.test(ctx, c, d)
		})
	}

	sserver.Close()
	jserver.Close()
	err := os.RemoveAll("/etc/docker/plugins")
	assert.NilError(t, err)
}

func setupPluginViaSpecFile(t *testing.T, ec map[string]*graphEventsCounter) *httptest.Server {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)

	setupPlugin(t, ec, "spec", mux, []byte(server.URL))

	return server
}

func setupPluginViaJSONFile(t *testing.T, ec map[string]*graphEventsCounter) *httptest.Server {
	mux := http.NewServeMux()
	server := httptest.NewServer(mux)

	p := plugins.NewLocalPlugin("json-external-graph-driver", server.URL)
	b, err := json.Marshal(p)
	assert.NilError(t, err)

	setupPlugin(t, ec, "json", mux, b)

	return server
}

func setupPlugin(t *testing.T, ec map[string]*graphEventsCounter, ext string, mux *http.ServeMux, b []byte) {
	name := fmt.Sprintf("%s-external-graph-driver", ext)
	type graphDriverRequest struct {
		ID         string `json:",omitempty"`
		Parent     string `json:",omitempty"`
		MountLabel string `json:",omitempty"`
		ReadOnly   bool   `json:",omitempty"`
	}

	type graphDriverResponse struct {
		Err      error             `json:",omitempty"`
		Dir      string            `json:",omitempty"`
		Exists   bool              `json:",omitempty"`
		Status   [][2]string       `json:",omitempty"`
		Metadata map[string]string `json:",omitempty"`
		Changes  []archive.Change  `json:",omitempty"`
		Size     int64             `json:",omitempty"`
	}

	respond := func(w http.ResponseWriter, data interface{}) {
		w.Header().Set("Content-Type", plugins.VersionMimetype)
		switch t := data.(type) {
		case error:
			fmt.Fprintf(w, "{\"Err\": %q}\n", t.Error())
		case string:
			fmt.Fprintln(w, t)
		default:
			json.NewEncoder(w).Encode(&data)
		}
	}

	decReq := func(b io.ReadCloser, out interface{}, w http.ResponseWriter) error {
		defer b.Close()
		if err := json.NewDecoder(b).Decode(&out); err != nil {
			http.Error(w, fmt.Sprintf("error decoding json: %s", err.Error()), 500)
		}
		return nil
	}

	base, err := os.MkdirTemp("", name)
	assert.NilError(t, err)
	vfsProto, err := vfs.Init(base, []string{}, idtools.IdentityMapping{})
	assert.NilError(t, err, "error initializing graph driver")
	driver := graphdriver.NewNaiveDiffDriver(vfsProto, idtools.IdentityMapping{})

	ec[ext] = &graphEventsCounter{}
	mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].activations++
		respond(w, `{"Implements": ["GraphDriver"]}`)
	})

	mux.HandleFunc("/GraphDriver.Init", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].init++
		respond(w, "{}")
	})

	mux.HandleFunc("/GraphDriver.CreateReadWrite", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].creations++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}
		if err := driver.CreateReadWrite(req.ID, req.Parent, nil); err != nil {
			respond(w, err)
			return
		}
		respond(w, "{}")
	})

	mux.HandleFunc("/GraphDriver.Create", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].creations++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}
		if err := driver.Create(req.ID, req.Parent, nil); err != nil {
			respond(w, err)
			return
		}
		respond(w, "{}")
	})

	mux.HandleFunc("/GraphDriver.Remove", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].removals++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		if err := driver.Remove(req.ID); err != nil {
			respond(w, err)
			return
		}
		respond(w, "{}")
	})

	mux.HandleFunc("/GraphDriver.Get", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].gets++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		dir, err := driver.Get(req.ID, req.MountLabel)
		if err != nil {
			respond(w, err)
			return
		}
		respond(w, &graphDriverResponse{Dir: dir})
	})

	mux.HandleFunc("/GraphDriver.Put", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].puts++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		if err := driver.Put(req.ID); err != nil {
			respond(w, err)
			return
		}
		respond(w, "{}")
	})

	mux.HandleFunc("/GraphDriver.Exists", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].exists++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}
		respond(w, &graphDriverResponse{Exists: driver.Exists(req.ID)})
	})

	mux.HandleFunc("/GraphDriver.Status", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].stats++
		respond(w, &graphDriverResponse{Status: driver.Status()})
	})

	mux.HandleFunc("/GraphDriver.Cleanup", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].cleanups++
		err := driver.Cleanup()
		if err != nil {
			respond(w, err)
			return
		}
		respond(w, `{}`)
	})

	mux.HandleFunc("/GraphDriver.GetMetadata", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].metadata++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		data, err := driver.GetMetadata(req.ID)
		if err != nil {
			respond(w, err)
			return
		}
		respond(w, &graphDriverResponse{Metadata: data})
	})

	mux.HandleFunc("/GraphDriver.Diff", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].diff++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		diff, err := driver.Diff(req.ID, req.Parent)
		if err != nil {
			respond(w, err)
			return
		}
		io.Copy(w, diff)
	})

	mux.HandleFunc("/GraphDriver.Changes", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].changes++
		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		changes, err := driver.Changes(req.ID, req.Parent)
		if err != nil {
			respond(w, err)
			return
		}
		respond(w, &graphDriverResponse{Changes: changes})
	})

	mux.HandleFunc("/GraphDriver.ApplyDiff", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].applydiff++
		diff := r.Body
		defer r.Body.Close()

		id := r.URL.Query().Get("id")
		parent := r.URL.Query().Get("parent")

		if id == "" {
			http.Error(w, "missing id", 409)
		}

		size, err := driver.ApplyDiff(id, parent, diff)
		if err != nil {
			respond(w, err)
			return
		}
		respond(w, &graphDriverResponse{Size: size})
	})

	mux.HandleFunc("/GraphDriver.DiffSize", func(w http.ResponseWriter, r *http.Request) {
		ec[ext].diffsize++

		var req graphDriverRequest
		if err := decReq(r.Body, &req, w); err != nil {
			return
		}

		size, err := driver.DiffSize(req.ID, req.Parent)
		if err != nil {
			respond(w, err)
			return
		}
		respond(w, &graphDriverResponse{Size: size})
	})

	err = os.MkdirAll("/etc/docker/plugins", 0o755)
	assert.NilError(t, err)

	specFile := "/etc/docker/plugins/" + name + "." + ext
	err = os.WriteFile(specFile, b, 0o644)
	assert.NilError(t, err)
}

func testExternalGraphDriver(ext string, ec map[string]*graphEventsCounter) func(context.Context, client.APIClient, *daemon.Daemon) func(*testing.T) {
	return func(ctx context.Context, c client.APIClient, d *daemon.Daemon) func(*testing.T) {
		return func(t *testing.T) {
			driverName := fmt.Sprintf("%s-external-graph-driver", ext)
			d.StartWithBusybox(ctx, t, "-s", driverName)

			testGraphDriver(ctx, t, c, driverName, func(t *testing.T) {
				d.Restart(t, "-s", driverName)
			})

			_, err := c.Info(ctx)
			assert.NilError(t, err)

			d.Stop(t)

			// Don't check ec.exists, because the daemon no longer calls the
			// Exists function.
			assert.Check(t, is.Equal(ec[ext].activations, 2))
			assert.Check(t, is.Equal(ec[ext].init, 2))
			assert.Check(t, ec[ext].creations >= 1)
			assert.Check(t, ec[ext].removals >= 1)
			assert.Check(t, ec[ext].gets >= 1)
			assert.Check(t, ec[ext].puts >= 1)
			assert.Check(t, is.Equal(ec[ext].stats, 5))
			assert.Check(t, is.Equal(ec[ext].cleanups, 2))
			assert.Check(t, ec[ext].applydiff >= 1)
			assert.Check(t, is.Equal(ec[ext].changes, 1))
			assert.Check(t, is.Equal(ec[ext].diffsize, 0))
			assert.Check(t, is.Equal(ec[ext].diff, 0))
			assert.Check(t, is.Equal(ec[ext].metadata, 1))
		}
	}
}

func testGraphDriverPull(ctx context.Context, c client.APIClient, d *daemon.Daemon) func(*testing.T) {
	return func(t *testing.T) {
		d.Start(t)
		defer d.Stop(t)

		r, err := c.ImagePull(ctx, "busybox:latest@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", image.PullOptions{})
		assert.NilError(t, err)
		_, err = io.Copy(io.Discard, r)
		assert.NilError(t, err)

		container.Run(ctx, t, c, container.WithImage("busybox:latest@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209"))
	}
}

func TestGraphdriverPluginV2(t *testing.T) {
	skip.If(t, testEnv.UsingSnapshotter())
	skip.If(t, runtime.GOOS == "windows")
	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
	skip.If(t, !requirement.HasHubConnectivity(t))
	skip.If(t, testEnv.NotAmd64)
	skip.If(t, !requirement.Overlay2Supported(testEnv.DaemonInfo.KernelVersion))

	ctx := testutil.StartSpan(baseContext, t)

	d := daemon.New(t, daemon.WithExperimental(), daemon.WithEnvVars("DOCKERD_DEPRECATED_GRAPHDRIVER_PLUGINS=1"))
	d.Start(t)
	defer d.Stop(t)

	apiClient := d.NewClientT(t)
	defer apiClient.Close()

	// install the plugin
	plugin := "cpuguy83/docker-overlay2-graphdriver-plugin"
	responseReader, err := apiClient.PluginInstall(ctx, plugin, types.PluginInstallOptions{
		RemoteRef:            plugin,
		AcceptAllPermissions: true,
	})
	assert.NilError(t, err)
	defer responseReader.Close()
	// ensure it's done by waiting for EOF on the response
	_, err = io.Copy(io.Discard, responseReader)
	assert.NilError(t, err)

	// restart the daemon with the plugin set as the storage driver
	d.Stop(t)
	d.StartWithBusybox(ctx, t, "-s", plugin)

	testGraphDriver(ctx, t, apiClient, plugin, nil)
}

func testGraphDriver(ctx context.Context, t *testing.T, c client.APIClient, driverName string, afterContainerRunFn func(*testing.T)) {
	id := container.Run(ctx, t, c, container.WithCmd("sh", "-c", "echo hello > /hello"))

	if afterContainerRunFn != nil {
		afterContainerRunFn(t)
	}

	i, err := c.ContainerInspect(ctx, id)
	assert.NilError(t, err)
	assert.Check(t, is.Equal(i.GraphDriver.Name, driverName))

	diffs, err := c.ContainerDiff(ctx, id)
	assert.NilError(t, err)
	assert.Check(t, is.Contains(diffs, containertypes.FilesystemChange{
		Kind: containertypes.ChangeAdd,
		Path: "/hello",
	}), "diffs: %v", diffs)

	err = c.ContainerRemove(ctx, id, containertypes.RemoveOptions{
		Force: true,
	})
	assert.NilError(t, err)
}