integration-cli/docker_api_swarm_test.go

//go:build !windows

package main

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/cloudflare/cfssl/csr"
	"github.com/cloudflare/cfssl/helpers"
	"github.com/cloudflare/cfssl/initca"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/integration-cli/checker"
	"github.com/docker/docker/integration-cli/daemon"
	"github.com/docker/docker/testutil"
	testdaemon "github.com/docker/docker/testutil/daemon"
	"github.com/docker/docker/testutil/request"
	"github.com/moby/swarmkit/v2/ca"
	"gotest.tools/v3/assert"
	is "gotest.tools/v3/assert/cmp"
	"gotest.tools/v3/poll"
)

var defaultReconciliationTimeout = 30 * time.Second

func (s *DockerSwarmSuite) TestAPISwarmInit(c *testing.T) {
	ctx := testutil.GetContext(c)
	// todo: should find a better way to verify that components are running than /info
	d1 := s.AddDaemon(ctx, c, true, true)
	info := d1.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, true)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
	assert.Equal(c, info.Cluster.RootRotationInProgress, false)

	d2 := s.AddDaemon(ctx, c, true, false)
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, false)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)

	// Leaving cluster
	assert.NilError(c, d2.SwarmLeave(ctx, c, false))

	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, false)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	d2.SwarmJoin(ctx, c, swarm.JoinRequest{
		ListenAddr:  d1.SwarmListenAddr(),
		JoinToken:   d1.JoinTokens(c).Worker,
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})

	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, false)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)

	// Current state restoring after restarts
	d1.Stop(c)
	d2.Stop(c)

	d1.StartNode(c)
	d2.StartNode(c)

	info = d1.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, true)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)

	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, false)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
}

func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *testing.T) {
	ctx := testutil.GetContext(c)
	d1 := s.AddDaemon(ctx, c, false, false)
	d1.SwarmInit(ctx, c, swarm.InitRequest{})

	// todo: error message differs depending if some components of token are valid

	d2 := s.AddDaemon(ctx, c, false, false)
	c2 := d2.NewClientT(c)
	err := c2.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})
	assert.ErrorContains(c, err, "join token is necessary")
	info := d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	err = c2.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		JoinToken:   "foobaz",
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})
	assert.ErrorContains(c, err, "invalid join token")
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	workerToken := d1.JoinTokens(c).Worker

	d2.SwarmJoin(ctx, c, swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		JoinToken:   workerToken,
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
	assert.NilError(c, d2.SwarmLeave(ctx, c, false))
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	// change tokens
	d1.RotateTokens(c)

	err = c2.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		JoinToken:   workerToken,
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})
	assert.ErrorContains(c, err, "join token is necessary")
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	workerToken = d1.JoinTokens(c).Worker

	d2.SwarmJoin(ctx, c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}})
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
	assert.NilError(c, d2.SwarmLeave(ctx, c, false))
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	// change spec, don't change tokens
	d1.UpdateSwarm(c, func(s *swarm.Spec) {})

	err = c2.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})
	assert.ErrorContains(c, err, "join token is necessary")
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)

	d2.SwarmJoin(ctx, c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}})
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
	assert.NilError(c, d2.SwarmLeave(ctx, c, false))
	info = d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)
}

func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *testing.T) {
	ctx := testutil.GetContext(c)
	d1 := s.AddDaemon(ctx, c, false, false)
	d1.SwarmInit(ctx, c, swarm.InitRequest{})
	d1.UpdateSwarm(c, func(s *swarm.Spec) {
		s.CAConfig.ExternalCAs = []*swarm.ExternalCA{
			{
				Protocol: swarm.ExternalCAProtocolCFSSL,
				URL:      "https://thishasnoca.org",
			},
			{
				Protocol: swarm.ExternalCAProtocolCFSSL,
				URL:      "https://thishasacacert.org",
				CACert:   "cacert",
			},
		}
	})
	info := d1.SwarmInfo(ctx, c)
	assert.Equal(c, len(info.Cluster.Spec.CAConfig.ExternalCAs), 2)
	assert.Equal(c, info.Cluster.Spec.CAConfig.ExternalCAs[0].CACert, "")
	assert.Equal(c, info.Cluster.Spec.CAConfig.ExternalCAs[1].CACert, "cacert")
}

func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *testing.T) {
	ctx := testutil.GetContext(c)
	d1 := s.AddDaemon(ctx, c, true, true)
	d2 := s.AddDaemon(ctx, c, false, false)
	splitToken := strings.Split(d1.JoinTokens(c).Worker, "-")
	splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e"
	replacementToken := strings.Join(splitToken, "-")
	c2 := d2.NewClientT(c)
	err := c2.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		JoinToken:   replacementToken,
		RemoteAddrs: []string{d1.SwarmListenAddr()},
	})
	assert.ErrorContains(c, err, "remote CA does not match fingerprint")
}

func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *testing.T) {
	ctx := testutil.GetContext(c)
	d1 := s.AddDaemon(ctx, c, false, false)
	d1.SwarmInit(ctx, c, swarm.InitRequest{})
	d2 := s.AddDaemon(ctx, c, true, false)

	info := d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, false)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)

	d1.UpdateNode(ctx, c, d2.NodeID(), func(n *swarm.Node) {
		n.Spec.Role = swarm.NodeRoleManager
	})

	poll.WaitOn(c, pollCheck(c, d2.CheckControlAvailable(ctx), checker.True()), poll.WithTimeout(defaultReconciliationTimeout))

	d1.UpdateNode(ctx, c, d2.NodeID(), func(n *swarm.Node) {
		n.Spec.Role = swarm.NodeRoleWorker
	})

	poll.WaitOn(c, pollCheck(c, d2.CheckControlAvailable(ctx), checker.False()), poll.WithTimeout(defaultReconciliationTimeout))

	// Wait for the role to change to worker in the cert. This is partially
	// done because it's something worth testing in its own right, and
	// partially because changing the role from manager to worker and then
	// back to manager quickly might cause the node to pause for awhile
	// while waiting for the role to change to worker, and the test can
	// time out during this interval.
	poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) {
		certBytes, err := os.ReadFile(filepath.Join(d2.Folder, "root", "swarm", "certificates", "swarm-node.crt"))
		if err != nil {
			return "", fmt.Sprintf("error: %v", err)
		}
		certs, err := helpers.ParseCertificatesPEM(certBytes)
		if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 {
			return certs[0].Subject.OrganizationalUnit[0], ""
		}
		return "", "could not get organizational unit from certificate"
	}, checker.Equals("swarm-worker")), poll.WithTimeout(defaultReconciliationTimeout))

	// Demoting last node should fail
	node := d1.GetNode(ctx, c, d1.NodeID())
	node.Spec.Role = swarm.NodeRoleWorker
	url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index)
	res, body, err := request.Post(testutil.GetContext(c), url, request.Host(d1.Sock()), request.JSONBody(node.Spec))
	assert.NilError(c, err)
	b, err := request.ReadBody(body)
	assert.NilError(c, err)
	assert.Equal(c, res.StatusCode, http.StatusBadRequest, "output: %q", string(b))

	// The warning specific to demoting the last manager is best-effort and
	// won't appear until the Role field of the demoted manager has been
	// updated.
	// Yes, I know this looks silly, but checker.Matches is broken, since
	// it anchors the regexp contrary to the documentation, and this makes
	// it impossible to match something that includes a line break.
	if !strings.Contains(string(b), "last manager of the swarm") {
		assert.Assert(c, strings.Contains(string(b), "this would result in a loss of quorum"))
	}
	info = d1.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
	assert.Equal(c, info.ControlAvailable, true)

	// Promote already demoted node
	d1.UpdateNode(ctx, c, d2.NodeID(), func(n *swarm.Node) {
		n.Spec.Role = swarm.NodeRoleManager
	})

	poll.WaitOn(c, pollCheck(c, d2.CheckControlAvailable(ctx), checker.True()), poll.WithTimeout(defaultReconciliationTimeout))
}

func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *testing.T) {
	ctx := testutil.GetContext(c)
	// add three managers, one of these is leader
	d1 := s.AddDaemon(ctx, c, true, true)
	d2 := s.AddDaemon(ctx, c, true, true)
	d3 := s.AddDaemon(ctx, c, true, true)

	// start a service by hitting each of the 3 managers
	d1.CreateService(ctx, c, simpleTestService, func(s *swarm.Service) {
		s.Spec.Name = "test1"
	})
	d2.CreateService(ctx, c, simpleTestService, func(s *swarm.Service) {
		s.Spec.Name = "test2"
	})
	d3.CreateService(ctx, c, simpleTestService, func(s *swarm.Service) {
		s.Spec.Name = "test3"
	})

	// 3 services should be started now, because the requests were proxied to leader
	// query each node and make sure it returns 3 services
	for _, d := range []*daemon.Daemon{d1, d2, d3} {
		services := d.ListServices(ctx, c)
		assert.Equal(c, len(services), 3)
	}
}

func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *testing.T) {
	ctx := testutil.GetContext(c)
	if runtime.GOARCH == "s390x" {
		c.Skip("Disabled on s390x")
	}
	if runtime.GOARCH == "ppc64le" {
		c.Skip("Disabled on  ppc64le")
	}

	// Create 3 nodes
	d1 := s.AddDaemon(ctx, c, true, true)
	d2 := s.AddDaemon(ctx, c, true, true)
	d3 := s.AddDaemon(ctx, c, true, true)

	// assert that the first node we made is the leader, and the other two are followers
	assert.Equal(c, d1.GetNode(ctx, c, d1.NodeID()).ManagerStatus.Leader, true)
	assert.Equal(c, d1.GetNode(ctx, c, d2.NodeID()).ManagerStatus.Leader, false)
	assert.Equal(c, d1.GetNode(ctx, c, d3.NodeID()).ManagerStatus.Leader, false)

	d1.Stop(c)

	var (
		leader    *daemon.Daemon   // keep track of leader
		followers []*daemon.Daemon // keep track of followers
	)
	var lastErr error
	checkLeader := func(nodes ...*daemon.Daemon) checkF {
		return func(c *testing.T) (interface{}, string) {
			// clear these out before each run
			leader = nil
			followers = nil
			for _, d := range nodes {
				n := d.GetNode(ctx, c, d.NodeID(), func(err error) bool {
					if strings.Contains(err.Error(), context.DeadlineExceeded.Error()) || strings.Contains(err.Error(), "swarm does not have a leader") {
						lastErr = err
						return true
					}
					return false
				})
				if n == nil {
					return false, fmt.Sprintf("failed to get node: %v", lastErr)
				}
				if n.ManagerStatus.Leader {
					leader = d
				} else {
					followers = append(followers, d)
				}
			}

			if leader == nil {
				return false, "no leader elected"
			}

			return true, fmt.Sprintf("elected %v", leader.ID())
		}
	}

	// wait for an election to occur
	c.Logf("Waiting for election to occur...")
	poll.WaitOn(c, pollCheck(c, checkLeader(d2, d3), checker.True()), poll.WithTimeout(defaultReconciliationTimeout))

	// assert that we have a new leader
	assert.Assert(c, leader != nil)

	// Keep track of the current leader, since we want that to be chosen.
	stableleader := leader

	// add the d1, the initial leader, back
	d1.StartNode(c)

	// wait for possible election
	c.Logf("Waiting for possible election...")
	poll.WaitOn(c, pollCheck(c, checkLeader(d1, d2, d3), checker.True()), poll.WithTimeout(defaultReconciliationTimeout))
	// pick out the leader and the followers again

	// verify that we still only have 1 leader and 2 followers
	assert.Assert(c, leader != nil)
	assert.Equal(c, len(followers), 2)
	// and that after we added d1 back, the leader hasn't changed
	assert.Equal(c, leader.NodeID(), stableleader.NodeID())
}

func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *testing.T) {
	ctx := testutil.GetContext(c)
	if runtime.GOARCH == "s390x" {
		c.Skip("Disabled on s390x")
	}
	if runtime.GOARCH == "ppc64le" {
		c.Skip("Disabled on  ppc64le")
	}

	d1 := s.AddDaemon(ctx, c, true, true)
	d2 := s.AddDaemon(ctx, c, true, true)
	d3 := s.AddDaemon(ctx, c, true, true)

	d1.CreateService(ctx, c, simpleTestService)

	d2.Stop(c)

	// make sure there is a leader
	poll.WaitOn(c, pollCheck(c, d1.CheckLeader(ctx), checker.IsNil()), poll.WithTimeout(defaultReconciliationTimeout))

	d1.CreateService(ctx, c, simpleTestService, func(s *swarm.Service) {
		s.Spec.Name = "top1"
	})

	d3.Stop(c)

	var service swarm.Service
	simpleTestService(&service)
	service.Spec.Name = "top2"
	cli := d1.NewClientT(c)
	defer cli.Close()

	// d1 will eventually step down from leader because there is no longer an active quorum, wait for that to happen
	poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) {
		_, err := cli.ServiceCreate(testutil.GetContext(c), service.Spec, types.ServiceCreateOptions{})
		return err.Error(), ""
	}, checker.Contains("Make sure more than half of the managers are online.")), poll.WithTimeout(defaultReconciliationTimeout*2))

	d2.StartNode(c)

	// make sure there is a leader
	poll.WaitOn(c, pollCheck(c, d1.CheckLeader(ctx), checker.IsNil()), poll.WithTimeout(defaultReconciliationTimeout))

	d1.CreateService(ctx, c, simpleTestService, func(s *swarm.Service) {
		s.Spec.Name = "top3"
	})
}

func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *testing.T) {
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, true, true)

	instances := 2
	d.CreateService(ctx, c, simpleTestService, setInstances(instances))

	id, err := d.Cmd("run", "-d", "busybox", "top")
	assert.NilError(c, err, id)
	id = strings.TrimSpace(id)

	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances+1)), poll.WithTimeout(defaultReconciliationTimeout))

	assert.ErrorContains(c, d.SwarmLeave(ctx, c, false), "")
	assert.NilError(c, d.SwarmLeave(ctx, c, true))

	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))

	id2, err := d.Cmd("ps", "-q")
	assert.NilError(c, err, id2)
	assert.Assert(c, strings.HasPrefix(id, strings.TrimSpace(id2)))
}

// #23629
func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *testing.T) {
	testRequires(c, Network)

	ctx := testutil.GetContext(c)
	s.AddDaemon(ctx, c, true, true)
	d2 := s.AddDaemon(ctx, c, false, false)

	id, err := d2.Cmd("run", "-d", "busybox", "top")
	assert.NilError(c, err, id)
	id = strings.TrimSpace(id)

	c2 := d2.NewClientT(c)
	err = c2.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d2.SwarmListenAddr(),
		RemoteAddrs: []string{"123.123.123.123:1234"},
	})
	assert.ErrorContains(c, err, "Timeout was reached")

	info := d2.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStatePending)

	assert.NilError(c, d2.SwarmLeave(ctx, c, true))

	poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout))

	id2, err := d2.Cmd("ps", "-q")
	assert.NilError(c, err, id2)
	assert.Assert(c, strings.HasPrefix(id, strings.TrimSpace(id2)))
}

// #23705
func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *testing.T) {
	testRequires(c, Network)

	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, false, false)
	client := d.NewClientT(c)
	err := client.SwarmJoin(testutil.GetContext(c), swarm.JoinRequest{
		ListenAddr:  d.SwarmListenAddr(),
		RemoteAddrs: []string{"123.123.123.123:1234"},
	})
	assert.ErrorContains(c, err, "Timeout was reached")

	poll.WaitOn(c, pollCheck(c, d.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStatePending)), poll.WithTimeout(defaultReconciliationTimeout))

	d.RestartNode(c)

	info := d.SwarmInfo(ctx, c)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive)
}

func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *testing.T) {
	ctx := testutil.GetContext(c)
	d1 := s.AddDaemon(ctx, c, true, true)

	instances := 2
	id := d1.CreateService(ctx, c, simpleTestService, setInstances(instances))

	d1.GetService(ctx, c, id)
	d1.RestartNode(c)
	d1.GetService(ctx, c, id)

	d2 := s.AddDaemon(ctx, c, true, true)
	d2.GetService(ctx, c, id)
	d2.RestartNode(c)
	d2.GetService(ctx, c, id)

	d3 := s.AddDaemon(ctx, c, true, true)
	d3.GetService(ctx, c, id)
	d3.RestartNode(c)
	d3.GetService(ctx, c, id)

	err := d3.Kill()
	assert.NilError(c, err)
	d3.StartNode(c)
	d3.GetService(ctx, c, id)
}

func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *testing.T) {
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, true, true)

	instances := 2
	id := d.CreateService(ctx, c, simpleTestService, setInstances(instances))

	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
	containers := d.ActiveContainers(ctx, c)
	instances = 4
	d.UpdateService(ctx, c, d.GetService(ctx, c, id), setInstances(instances))
	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
	containers2 := d.ActiveContainers(ctx, c)

loop0:
	for _, c1 := range containers {
		for _, c2 := range containers2 {
			if c1 == c2 {
				continue loop0
			}
		}
		c.Errorf("container %v not found in new set %#v", c1, containers2)
	}
}

func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *testing.T) {
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, false, false)
	req := swarm.InitRequest{
		ListenAddr: "",
	}
	res, _, err := request.Post(testutil.GetContext(c), "/swarm/init", request.Host(d.Sock()), request.JSONBody(req))
	assert.NilError(c, err)
	assert.Equal(c, res.StatusCode, http.StatusBadRequest)

	req2 := swarm.JoinRequest{
		ListenAddr:  "0.0.0.0:2377",
		RemoteAddrs: []string{""},
	}
	res, _, err = request.Post(testutil.GetContext(c), "/swarm/join", request.Host(d.Sock()), request.JSONBody(req2))
	assert.NilError(c, err)
	assert.Equal(c, res.StatusCode, http.StatusBadRequest)
}

func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *testing.T) {
	ctx := testutil.GetContext(c)
	d1 := s.AddDaemon(ctx, c, true, true)
	d2 := s.AddDaemon(ctx, c, true, true)

	instances := 2
	id := d1.CreateService(ctx, c, simpleTestService, setInstances(instances))
	poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))

	// drain d2, all containers should move to d1
	d1.UpdateNode(ctx, c, d2.NodeID(), func(n *swarm.Node) {
		n.Spec.Availability = swarm.NodeAvailabilityDrain
	})
	poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
	poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount(ctx), checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout))

	d2.Stop(c)

	d1.SwarmInit(ctx, c, swarm.InitRequest{
		ForceNewCluster: true,
		Spec:            swarm.Spec{},
	})

	poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))

	d3 := s.AddDaemon(ctx, c, true, true)
	info := d3.SwarmInfo(ctx, c)
	assert.Equal(c, info.ControlAvailable, true)
	assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)

	instances = 4
	d3.UpdateService(ctx, c, d3.GetService(ctx, c, id), setInstances(instances))

	poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
}

func simpleTestService(s *swarm.Service) {
	ureplicas := uint64(1)
	restartDelay := 100 * time.Millisecond

	s.Spec = swarm.ServiceSpec{
		TaskTemplate: swarm.TaskSpec{
			ContainerSpec: &swarm.ContainerSpec{
				Image:   "busybox:latest",
				Command: []string{"/bin/top"},
			},
			RestartPolicy: &swarm.RestartPolicy{
				Delay: &restartDelay,
			},
		},
		Mode: swarm.ServiceMode{
			Replicated: &swarm.ReplicatedService{
				Replicas: &ureplicas,
			},
		},
	}
	s.Spec.Name = "top"
}

func serviceForUpdate(s *swarm.Service) {
	ureplicas := uint64(1)
	restartDelay := 100 * time.Millisecond

	s.Spec = swarm.ServiceSpec{
		TaskTemplate: swarm.TaskSpec{
			ContainerSpec: &swarm.ContainerSpec{
				Image:   "busybox:latest",
				Command: []string{"/bin/top"},
			},
			RestartPolicy: &swarm.RestartPolicy{
				Delay: &restartDelay,
			},
		},
		Mode: swarm.ServiceMode{
			Replicated: &swarm.ReplicatedService{
				Replicas: &ureplicas,
			},
		},
		UpdateConfig: &swarm.UpdateConfig{
			Parallelism:   2,
			Delay:         4 * time.Second,
			FailureAction: swarm.UpdateFailureActionContinue,
		},
		RollbackConfig: &swarm.UpdateConfig{
			Parallelism:   3,
			Delay:         4 * time.Second,
			FailureAction: swarm.UpdateFailureActionContinue,
		},
	}
	s.Spec.Name = "updatetest"
}

func setInstances(replicas int) testdaemon.ServiceConstructor {
	ureplicas := uint64(replicas)
	return func(s *swarm.Service) {
		s.Spec.Mode = swarm.ServiceMode{
			Replicated: &swarm.ReplicatedService{
				Replicas: &ureplicas,
			},
		}
	}
}

func setUpdateOrder(order string) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		if s.Spec.UpdateConfig == nil {
			s.Spec.UpdateConfig = &swarm.UpdateConfig{}
		}
		s.Spec.UpdateConfig.Order = order
	}
}

func setRollbackOrder(order string) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		if s.Spec.RollbackConfig == nil {
			s.Spec.RollbackConfig = &swarm.UpdateConfig{}
		}
		s.Spec.RollbackConfig.Order = order
	}
}

func setImage(image string) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		if s.Spec.TaskTemplate.ContainerSpec == nil {
			s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{}
		}
		s.Spec.TaskTemplate.ContainerSpec.Image = image
	}
}

func setFailureAction(failureAction string) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		s.Spec.UpdateConfig.FailureAction = failureAction
	}
}

func setMaxFailureRatio(maxFailureRatio float32) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio
	}
}

func setParallelism(parallelism uint64) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		s.Spec.UpdateConfig.Parallelism = parallelism
	}
}

func setConstraints(constraints []string) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		if s.Spec.TaskTemplate.Placement == nil {
			s.Spec.TaskTemplate.Placement = &swarm.Placement{}
		}
		s.Spec.TaskTemplate.Placement.Constraints = constraints
	}
}

func setPlacementPrefs(prefs []swarm.PlacementPreference) testdaemon.ServiceConstructor {
	return func(s *swarm.Service) {
		if s.Spec.TaskTemplate.Placement == nil {
			s.Spec.TaskTemplate.Placement = &swarm.Placement{}
		}
		s.Spec.TaskTemplate.Placement.Preferences = prefs
	}
}

func setGlobalMode(s *swarm.Service) {
	s.Spec.Mode = swarm.ServiceMode{
		Global: &swarm.GlobalService{},
	}
}

func checkClusterHealth(c *testing.T, cl []*daemon.Daemon, managerCount, workerCount int) {
	var totalMCount, totalWCount int

	ctx := testutil.GetContext(c)
	for _, d := range cl {
		var info swarm.Info

		// check info in a poll.WaitOn(), because if the cluster doesn't have a leader, `info` will return an error
		checkInfo := func(c *testing.T) (interface{}, string) {
			client := d.NewClientT(c)
			daemonInfo, err := client.Info(ctx)
			info = daemonInfo.Swarm
			return err, "cluster not ready in time"
		}
		poll.WaitOn(c, pollCheck(c, checkInfo, checker.IsNil()), poll.WithTimeout(defaultReconciliationTimeout))
		if !info.ControlAvailable {
			totalWCount++
			continue
		}

		var leaderFound bool
		totalMCount++
		var mCount, wCount int

		for _, n := range d.ListNodes(ctx, c) {
			waitReady := func(c *testing.T) (interface{}, string) {
				if n.Status.State == swarm.NodeStateReady {
					return true, ""
				}
				nn := d.GetNode(ctx, c, n.ID)
				n = *nn
				return n.Status.State == swarm.NodeStateReady, fmt.Sprintf("state of node %s, reported by %s", n.ID, d.NodeID())
			}
			poll.WaitOn(c, pollCheck(c, waitReady, checker.True()), poll.WithTimeout(defaultReconciliationTimeout))

			waitActive := func(c *testing.T) (interface{}, string) {
				if n.Spec.Availability == swarm.NodeAvailabilityActive {
					return true, ""
				}
				nn := d.GetNode(ctx, c, n.ID)
				n = *nn
				return n.Spec.Availability == swarm.NodeAvailabilityActive, fmt.Sprintf("availability of node %s, reported by %s", n.ID, d.NodeID())
			}
			poll.WaitOn(c, pollCheck(c, waitActive, checker.True()), poll.WithTimeout(defaultReconciliationTimeout))

			if n.Spec.Role == swarm.NodeRoleManager {
				assert.Assert(c, n.ManagerStatus != nil, "manager status of node %s (manager), reported by %s", n.ID, d.NodeID())
				if n.ManagerStatus.Leader {
					leaderFound = true
				}
				mCount++
			} else {
				assert.Assert(c, n.ManagerStatus == nil, "manager status of node %s (worker), reported by %s", n.ID, d.NodeID())
				wCount++
			}
		}
		assert.Equal(c, leaderFound, true, "lack of leader reported by node %s", info.NodeID)
		assert.Equal(c, mCount, managerCount, "managers count reported by node %s", info.NodeID)
		assert.Equal(c, wCount, workerCount, "workers count reported by node %s", info.NodeID)
	}
	assert.Equal(c, totalMCount, managerCount)
	assert.Equal(c, totalWCount, workerCount)
}

func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *testing.T) {
	ctx := testutil.GetContext(c)
	mCount, wCount := 5, 1

	var nodes []*daemon.Daemon
	for i := 0; i < mCount; i++ {
		manager := s.AddDaemon(ctx, c, true, true)
		info := manager.SwarmInfo(ctx, c)
		assert.Equal(c, info.ControlAvailable, true)
		assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
		nodes = append(nodes, manager)
	}

	for i := 0; i < wCount; i++ {
		worker := s.AddDaemon(ctx, c, true, false)
		info := worker.SwarmInfo(ctx, c)
		assert.Equal(c, info.ControlAvailable, false)
		assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive)
		nodes = append(nodes, worker)
	}

	// stop whole cluster
	{
		var wg sync.WaitGroup
		wg.Add(len(nodes))
		errs := make(chan error, len(nodes))

		for _, d := range nodes {
			go func(daemon *daemon.Daemon) {
				defer wg.Done()
				if err := daemon.StopWithError(); err != nil {
					errs <- err
				}
			}(d)
		}
		wg.Wait()
		close(errs)
		for err := range errs {
			assert.NilError(c, err)
		}
	}

	// start whole cluster
	{
		var wg sync.WaitGroup
		wg.Add(len(nodes))
		errs := make(chan error, len(nodes))

		for _, d := range nodes {
			go func(daemon *daemon.Daemon) {
				defer wg.Done()
				if err := daemon.StartWithError("--iptables=false", "--ip6tables=false"); err != nil {
					errs <- err
				}
			}(d)
		}
		wg.Wait()
		close(errs)
		for err := range errs {
			assert.NilError(c, err)
		}
	}

	checkClusterHealth(c, nodes, mCount, wCount)
}

func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *testing.T) {
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, true, true)

	instances := 2
	id := d.CreateService(ctx, c, simpleTestService, setInstances(instances))
	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))

	service := d.GetService(ctx, c, id)
	instances = 5

	setInstances(instances)(service)
	cli := d.NewClientT(c)
	defer cli.Close()
	_, err := cli.ServiceUpdate(ctx, service.Spec.Name, service.Version, service.Spec, types.ServiceUpdateOptions{})
	assert.NilError(c, err)
	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))
}

// Unlocking an unlocked swarm results in an error
func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *testing.T) {
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, true, true)
	err := d.SwarmUnlock(c, swarm.UnlockRequest{UnlockKey: "wrong-key"})
	assert.ErrorContains(c, err, "swarm is not locked")
}

// #29885
func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *testing.T) {
	ctx := testutil.GetContext(c)
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", defaultSwarmPort))
	assert.NilError(c, err)
	defer ln.Close()
	d := s.AddDaemon(ctx, c, false, false)
	client := d.NewClientT(c)
	_, err = client.SwarmInit(testutil.GetContext(c), swarm.InitRequest{
		ListenAddr: d.SwarmListenAddr(),
	})
	assert.ErrorContains(c, err, "address already in use")
}

// Test case for 30178
func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *testing.T) {
	// Issue #36386 can be a independent one, which is worth further investigation.
	c.Skip("Root cause of Issue #36386 is needed")
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, true, true)

	out, err := d.Cmd("network", "create", "-d", "overlay", "lb")
	assert.NilError(c, err, out)

	instances := 1
	d.CreateService(ctx, c, simpleTestService, setInstances(instances), func(s *swarm.Service) {
		if s.Spec.TaskTemplate.ContainerSpec == nil {
			s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{}
		}
		s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{}
		s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{
			{Target: "lb"},
		}
	})

	poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout))

	containers := d.ActiveContainers(testutil.GetContext(c), c)

	out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top")
	assert.NilError(c, err, out)
}

func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *testing.T) {
	ctx := testutil.GetContext(c)
	m := s.AddDaemon(ctx, c, true, true)
	w := s.AddDaemon(ctx, c, true, false)

	info := m.SwarmInfo(ctx, c)

	currentTrustRoot := info.Cluster.TLSInfo.TrustRoot

	// rotate multiple times
	for i := 0; i < 4; i++ {
		var err error
		var cert, key []byte
		if i%2 != 0 {
			cert, _, key, err = initca.New(&csr.CertificateRequest{
				CN:         "newRoot",
				KeyRequest: csr.NewKeyRequest(),
				CA:         &csr.CAConfig{Expiry: ca.RootCAExpiration},
			})
			assert.NilError(c, err)
		}
		expectedCert := string(cert)
		m.UpdateSwarm(c, func(s *swarm.Spec) {
			s.CAConfig.SigningCACert = expectedCert
			s.CAConfig.SigningCAKey = string(key)
			s.CAConfig.ForceRotate++
		})

		// poll to make sure update succeeds
		var clusterTLSInfo swarm.TLSInfo
		for j := 0; j < 18; j++ {
			info := m.SwarmInfo(ctx, c)

			// the desired CA cert and key is always redacted
			assert.Equal(c, info.Cluster.Spec.CAConfig.SigningCAKey, "")
			assert.Equal(c, info.Cluster.Spec.CAConfig.SigningCACert, "")

			clusterTLSInfo = info.Cluster.TLSInfo

			// if root rotation is done and the trust root has changed, we don't have to poll anymore
			if !info.Cluster.RootRotationInProgress && clusterTLSInfo.TrustRoot != currentTrustRoot {
				break
			}

			// root rotation not done
			time.Sleep(250 * time.Millisecond)
		}
		if cert != nil {
			assert.Equal(c, clusterTLSInfo.TrustRoot, expectedCert)
		}
		// could take another second or two for the nodes to trust the new roots after they've all gotten
		// new TLS certificates
		for j := 0; j < 18; j++ {
			mInfo := m.GetNode(ctx, c, m.NodeID()).Description.TLSInfo
			wInfo := m.GetNode(ctx, c, w.NodeID()).Description.TLSInfo

			if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot {
				break
			}

			// nodes don't trust root certs yet
			time.Sleep(250 * time.Millisecond)
		}

		assert.DeepEqual(c, m.GetNode(ctx, c, m.NodeID()).Description.TLSInfo, clusterTLSInfo)
		assert.DeepEqual(c, m.GetNode(ctx, c, w.NodeID()).Description.TLSInfo, clusterTLSInfo)
		currentTrustRoot = clusterTLSInfo.TrustRoot
	}
}

func (s *DockerSwarmSuite) TestAPINetworkInspectWithScope(c *testing.T) {
	ctx := testutil.GetContext(c)
	d := s.AddDaemon(ctx, c, true, true)

	name := "test-scoped-network"
	apiclient := d.NewClientT(c)

	resp, err := apiclient.NetworkCreate(ctx, name, network.CreateOptions{Driver: "overlay"})
	assert.NilError(c, err)

	nw, err := apiclient.NetworkInspect(ctx, name, network.InspectOptions{})
	assert.NilError(c, err)
	assert.Check(c, is.Equal("swarm", nw.Scope))
	assert.Check(c, is.Equal(resp.ID, nw.ID))

	_, err = apiclient.NetworkInspect(ctx, name, network.InspectOptions{Scope: "local"})
	assert.Check(c, is.ErrorType(err, errdefs.IsNotFound))
}