libcontainer/dmz/overlayfs_linux.go

package dmz

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"golang.org/x/sys/unix"

	"github.com/opencontainers/runc/libcontainer/utils"
)

func fsopen(fsName string, flags int) (*os.File, error) {
	// Make sure we always set O_CLOEXEC.
	flags |= unix.FSOPEN_CLOEXEC
	fd, err := unix.Fsopen(fsName, flags)
	if err != nil {
		return nil, os.NewSyscallError("fsopen "+fsName, err)
	}
	return os.NewFile(uintptr(fd), "fscontext:"+fsName), nil
}

func fsmount(ctx *os.File, flags, mountAttrs int) (*os.File, error) {
	// Make sure we always set O_CLOEXEC.
	flags |= unix.FSMOUNT_CLOEXEC
	fd, err := unix.Fsmount(int(ctx.Fd()), flags, mountAttrs)
	if err != nil {
		return nil, os.NewSyscallError("fsmount "+ctx.Name(), err)
	}
	runtime.KeepAlive(ctx) // make sure fd is kept alive while it's used
	return os.NewFile(uintptr(fd), "fsmount:"+ctx.Name()), nil
}

func escapeOverlayLowerDir(path string) string {
	// If the lowerdir path contains ":" we need to escape them, and if there
	// were any escape characters already (\) we need to escape those first.
	return strings.ReplaceAll(strings.ReplaceAll(path, `\`, `\\`), `:`, `\:`)
}

// sealedOverlayfs will create an internal overlayfs mount using fsopen() that
// uses the directory containing the binary as a lowerdir and a temporary tmpfs
// as an upperdir. There is no way to "unwrap" this (unlike MS_BIND+MS_RDONLY)
// and so we can create a safe zero-copy sealed version of /proc/self/exe.
// This only works for privileged users and on kernels with overlayfs and
// fsopen() enabled.
//
// TODO: Since Linux 5.11, overlayfs can be created inside user namespaces so
// it is technically possible to create an overlayfs even for rootless
// containers. Unfortunately, this would require some ugly manual CGo+fork
// magic so we can do this later if we feel it's really needed.
func sealedOverlayfs(binPath, tmpDir string) (_ *os.File, Err error) {
	// Try to do the superblock creation first to bail out early if we can't
	// use this method.
	overlayCtx, err := fsopen("overlay", unix.FSOPEN_CLOEXEC)
	if err != nil {
		return nil, err
	}
	defer overlayCtx.Close()

	// binPath is going to be /proc/self/exe, so do a readlink to get the real
	// path. overlayfs needs the real underlying directory for this protection
	// mode to work properly.
	if realPath, err := os.Readlink(binPath); err == nil {
		binPath = realPath
	}
	binLowerDirPath, binName := filepath.Split(binPath)
	// Escape any ":"s or "\"s in the path.
	binLowerDirPath = escapeOverlayLowerDir(binLowerDirPath)

	// Overlayfs requires two lowerdirs in order to run in "lower-only" mode,
	// where writes are completely blocked. Ideally we would create a dummy
	// tmpfs for this, but it turns out that overlayfs doesn't allow for
	// anonymous mountns paths.
	// NOTE: I'm working on a patch to fix this but it won't be backported.
	dummyLowerDirPath := escapeOverlayLowerDir(tmpDir)

	// Configure the lowerdirs. The binary lowerdir needs to be on the top to
	// ensure that a file called "runc" (binName) in the dummy lowerdir doesn't
	// mask the binary.
	lowerDirStr := binLowerDirPath + ":" + dummyLowerDirPath
	if err := unix.FsconfigSetString(int(overlayCtx.Fd()), "lowerdir", lowerDirStr); err != nil {
		return nil, fmt.Errorf("fsconfig set overlayfs lowerdir=%s: %w", lowerDirStr, err)
	}

	// We don't care about xino (Linux 4.17) but it will be auto-enabled on
	// some systems (if /run/runc and /usr/bin are on different filesystems)
	// and this produces spurious dmesg log entries. We can safely ignore
	// errors when disabling this because we don't actually care about the
	// setting and we're just opportunistically disabling it.
	_ = unix.FsconfigSetString(int(overlayCtx.Fd()), "xino", "off")

	// Get an actual handle to the overlayfs.
	if err := unix.FsconfigCreate(int(overlayCtx.Fd())); err != nil {
		return nil, os.NewSyscallError("fsconfig create overlayfs", err)
	}
	overlayFd, err := fsmount(overlayCtx, unix.FSMOUNT_CLOEXEC, unix.MS_RDONLY|unix.MS_NODEV|unix.MS_NOSUID)
	if err != nil {
		return nil, err
	}
	defer overlayFd.Close()

	// Grab a handle to the binary through overlayfs.
	exeFile, err := utils.Openat(overlayFd, binName, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
	if err != nil {
		return nil, fmt.Errorf("open %s from overlayfs (lowerdir=%s): %w", binName, lowerDirStr, err)
	}
	// NOTE: We would like to check that exeFile is the same as /proc/self/exe,
	// except this is a little difficult. Depending on what filesystems the
	// layers are on, overlayfs can remap the inode numbers (and it always
	// creates its own device numbers -- see ovl_map_dev_ino) so we can't do a
	// basic stat-based check. The only reasonable option would be to hash both
	// files and compare them, but this would require fully reading both files
	// which would produce a similar performance overhead to memfd cloning.
	//
	// Ultimately, there isn't a real attack to be worried about here. An
	// attacker would need to be able to modify files in /usr/sbin (or wherever
	// runc lives), at which point they could just replace the runc binary with
	// something malicious anyway.
	return exeFile, nil
}