integration/container/restart_test.go

package container

import (
	"context"
	"fmt"
	"runtime"
	"strconv"
	"testing"
	"time"

	"github.com/moby/moby/api/types/container"
	"github.com/moby/moby/api/types/events"
	"github.com/moby/moby/client"
	testContainer "github.com/moby/moby/v2/integration/internal/container"
	"github.com/moby/moby/v2/internal/testutil"
	"github.com/moby/moby/v2/internal/testutil/daemon"
	"gotest.tools/v3/assert"
	is "gotest.tools/v3/assert/cmp"
	"gotest.tools/v3/poll"
	"gotest.tools/v3/skip"
)

func TestDaemonRestartKillContainers(t *testing.T) {
	skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support live-restore")

	ctx := testutil.StartSpan(baseContext, t)

	type testCase struct {
		desc          string
		restartPolicy container.RestartPolicy

		xRunning            bool
		xRunningLiveRestore bool
		xStart              bool
		xHealthCheck        bool
	}

	for _, tc := range []testCase{
		{
			desc:                "container without restart policy",
			xRunningLiveRestore: true,
			xStart:              true,
		},
		{
			desc:                "container with restart=always",
			restartPolicy:       container.RestartPolicy{Name: "always"},
			xRunning:            true,
			xRunningLiveRestore: true,
			xStart:              true,
		},
		{
			desc:                "container with restart=always and with healthcheck",
			restartPolicy:       container.RestartPolicy{Name: "always"},
			xRunning:            true,
			xRunningLiveRestore: true,
			xStart:              true,
			xHealthCheck:        true,
		},
		{
			desc:          "container created should not be restarted",
			restartPolicy: container.RestartPolicy{Name: "always"},
		},
	} {
		for _, liveRestoreEnabled := range []bool{false, true} {
			for fnName, stopDaemon := range map[string]func(*testing.T, *daemon.Daemon){
				"kill-daemon": func(t *testing.T, d *daemon.Daemon) {
					err := d.Kill()
					assert.NilError(t, err)
				},
				"stop-daemon": func(t *testing.T, d *daemon.Daemon) {
					d.Stop(t)
				},
			} {
				tc := tc
				liveRestoreEnabled := liveRestoreEnabled
				t.Run(fmt.Sprintf("live-restore=%v/%s/%s", liveRestoreEnabled, tc.desc, fnName), func(t *testing.T) {
					t.Parallel()

					ctx := testutil.StartSpan(ctx, t)

					d := daemon.New(t)
					apiClient := d.NewClientT(t)

					args := []string{"--iptables=false", "--ip6tables=false"}
					if liveRestoreEnabled {
						args = append(args, "--live-restore")
					}

					d.StartWithBusybox(ctx, t, args...)
					defer d.Stop(t)

					config := container.Config{Image: "busybox", Cmd: []string{"top"}}
					hostConfig := container.HostConfig{RestartPolicy: tc.restartPolicy}
					if tc.xHealthCheck {
						config.Healthcheck = &container.HealthConfig{
							Test:          []string{"CMD-SHELL", "! test -f /tmp/unhealthy"},
							StartPeriod:   60 * time.Second,
							StartInterval: 1 * time.Second,
							Interval:      60 * time.Second,
						}
					}
					resp, err := apiClient.ContainerCreate(ctx, client.ContainerCreateOptions{
						Config:     &config,
						HostConfig: &hostConfig,
					})
					assert.NilError(t, err)
					defer apiClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})

					if tc.xStart {
						_, err = apiClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
						assert.NilError(t, err)
						if tc.xHealthCheck {
							poll.WaitOn(t, pollForHealthStatus(ctx, apiClient, resp.ID, container.Healthy), poll.WithTimeout(30*time.Second))
							testContainer.ExecT(ctx, t, apiClient, resp.ID, []string{"touch", "/tmp/unhealthy"}).AssertSuccess(t)
						}
					}

					stopDaemon(t, d)
					startTime := time.Now()
					d.Start(t, args...)

					expected := tc.xRunning
					if liveRestoreEnabled {
						expected = tc.xRunningLiveRestore
					}

					poll.WaitOn(t, testContainer.RunningStateFlagIs(ctx, apiClient, resp.ID, expected), poll.WithTimeout(30*time.Second))

					if tc.xHealthCheck {
						// We have arranged to have the container's health probes fail until we tell it
						// to become healthy, which gives us the entire StartPeriod (60s) to assert that
						// the container's health state is Starting before we have to worry about racing
						// the health monitor.
						assert.Equal(t, testContainer.Inspect(ctx, t, apiClient, resp.ID).State.Health.Status, container.Starting)
						poll.WaitOn(t, pollForNewHealthCheck(ctx, apiClient, startTime, resp.ID), poll.WithTimeout(30*time.Second))

						testContainer.ExecT(ctx, t, apiClient, resp.ID, []string{"rm", "/tmp/unhealthy"}).AssertSuccess(t)
						poll.WaitOn(t, pollForHealthStatus(ctx, apiClient, resp.ID, container.Healthy), poll.WithTimeout(30*time.Second))
					}
					// TODO(cpuguy83): test pause states... this seems to be rather undefined currently
				})
			}
		}
	}
}

func pollForNewHealthCheck(ctx context.Context, apiClient *client.Client, startTime time.Time, containerID string) func(log poll.LogT) poll.Result {
	return func(log poll.LogT) poll.Result {
		inspect, err := apiClient.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
		if err != nil {
			return poll.Error(err)
		}
		healthChecksTotal := len(inspect.Container.State.Health.Log)
		if healthChecksTotal > 0 {
			if inspect.Container.State.Health.Log[healthChecksTotal-1].Start.After(startTime) {
				return poll.Success()
			}
		}
		return poll.Continue("waiting for a new container healthcheck")
	}
}

// Container started with --rm should be able to be restarted.
// It should be removed only if killed or stopped
func TestContainerWithAutoRemoveCanBeRestarted(t *testing.T) {
	ctx := setupTest(t)
	apiClient := testEnv.APIClient()

	noWaitTimeout := 0

	for _, tc := range []struct {
		desc  string
		doSth func(ctx context.Context, containerID string) error
	}{
		{
			desc: "kill",
			doSth: func(ctx context.Context, containerID string) error {
				_, err := apiClient.ContainerKill(ctx, containerID, client.ContainerKillOptions{})
				return err
			},
		},
		{
			desc: "stop",
			doSth: func(ctx context.Context, containerID string) error {
				_, err := apiClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: &noWaitTimeout})
				return err
			},
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			testutil.StartSpan(ctx, t)
			cID := testContainer.Run(ctx, t, apiClient,
				testContainer.WithName("autoremove-restart-and-"+tc.desc),
				testContainer.WithAutoRemove,
			)
			defer func() {
				_, err := apiClient.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
				if t.Failed() && err != nil {
					t.Logf("Cleaning up test container failed with error: %v", err)
				}
			}()

			_, err := apiClient.ContainerRestart(ctx, cID, client.ContainerRestartOptions{
				Timeout: &noWaitTimeout,
			})
			assert.NilError(t, err)

			inspect, err := apiClient.ContainerInspect(ctx, cID, client.ContainerInspectOptions{})
			assert.NilError(t, err)
			assert.Assert(t, inspect.Container.State.Status != container.StateRemoving, "Container should not be removing yet")

			poll.WaitOn(t, testContainer.IsInState(ctx, apiClient, cID, container.StateRunning))

			err = tc.doSth(ctx, cID)
			assert.NilError(t, err)

			poll.WaitOn(t, testContainer.IsRemoved(ctx, apiClient, cID))
		})
	}
}

// TestContainerRestartWithCancelledRequest verifies that cancelling a restart
// request does not cancel the restart operation, and still starts the container
// after it was stopped.
//
// Regression test for https://github.com/moby/moby/discussions/46682
func TestContainerRestartWithCancelledRequest(t *testing.T) {
	ctx := setupTest(t)

	testutil.StartSpan(ctx, t)

	// The test relies on "trap" to ignore SIGTERM so that the stop takes
	// the full stopTimeout, giving the client time to cancel the request.
	// On Windows, busybox-w32 doesn't support signal trapping (see
	// https://github.com/rmyorston/busybox-w32/issues/303) so the
	// container may exit immediately on SIGTERM, making the test
	// scenario impossible to set up reliably.
	// Allow multiple attempts on Windows so the test can pass when the timing
	// happens to work out.
	if runtime.GOOS == "windows" {
		for retry := range 10 {
			success := true
			fail := func(t *testing.T) {
				success = false
			}
			t.Run(strconv.Itoa(retry), func(t *testing.T) {
				testContainerRestartWithCancelledRequest(ctx, t, fail)
			})
			if success {
				return
			}
		}
		return
	}

	testContainerRestartWithCancelledRequest(ctx, t, func(t *testing.T) {
		t.Fatal("timeout waiting for restart event")
	})
}

func testContainerRestartWithCancelledRequest(ctx context.Context, t *testing.T, fail func(t *testing.T)) {
	apiClient := testEnv.APIClient()
	// Create a container that ignores SIGTERM and doesn't stop immediately,
	// giving us time to cancel the request.
	//
	// Restarting a container is "stop" (and, if needed, "kill"), then "start"
	// the container. We're trying to create the scenario where the "stop" is
	// handled, but the request was cancelled and therefore the "start" not
	// taking place.
	cID := testContainer.Run(ctx, t, apiClient, testContainer.WithCmd("sh", "-c", "trap 'echo received TERM' TERM; while true; do usleep 10; done"))
	defer func() {
		_, err := apiClient.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
		if t.Failed() && err != nil {
			t.Logf("Cleaning up test container failed with error: %v", err)
		}
	}()

	// Start listening for events.
	result := apiClient.Events(ctx, client.EventsListOptions{
		Filters: make(client.Filters).Add("container", cID).Add("event", string(events.ActionRestart)),
	})
	messages := result.Messages
	errs := result.Err

	// Make restart request, but cancel the request before the container
	// is (forcibly) killed.
	ctx2, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
	stopTimeout := 1
	_, err := apiClient.ContainerRestart(ctx2, cID, client.ContainerRestartOptions{
		Timeout: &stopTimeout,
	})
	assert.Check(t, is.ErrorIs(err, context.DeadlineExceeded))
	cancel()

	// Validate that the restart event occurred, which is emitted
	// after the restart (stop (kill) start) finished.
	//
	// Note that we cannot use RestartCount for this, as that's only
	// used for restart-policies.
	restartTimeout := 10 * time.Second
	select {
	case m := <-messages:
		assert.Check(t, is.Equal(m.Actor.ID, cID))
		assert.Check(t, is.Equal(m.Action, events.ActionRestart))
	case err := <-errs:
		assert.NilError(t, err)
	case <-time.After(restartTimeout):
		fail(t)
		return
	}

	// Container should be restarted (running).
	inspect, err := apiClient.ContainerInspect(ctx, cID, client.ContainerInspectOptions{})
	assert.NilError(t, err)
	assert.Check(t, is.Equal(inspect.Container.State.Status, container.StateRunning))
}