| # Copyright 2007-2010 Google Inc. Released under the GPL v2 |
| __author__ = "duanes (Duane Sand), pdahl (Peter Dahl)" |
| |
| # A basic cpuset/cgroup container manager for limiting memory use during tests |
| # for use on kernels not running some site-specific container manager |
| |
| import os, sys, re, glob, fcntl, logging |
| from autotest_lib.client.bin import utils |
| from autotest_lib.client.common_lib import error |
| |
| SUPER_ROOT = '' # root of all containers or cgroups |
| NO_LIMIT = (1 << 63) - 1 # containername/memory.limit_in_bytes if no limit |
| |
| # propio service classes: |
| PROPIO_PRIO = 1 |
| PROPIO_NORMAL = 2 |
| PROPIO_IDLE = 3 |
| |
| super_root_path = '' # usually '/dev/cgroup'; '/dev/cpuset' on 2.6.18 |
| cpuset_prefix = None # usually 'cpuset.'; '' on 2.6.18 |
| fake_numa_containers = False # container mem via numa=fake mem nodes, else pages |
| mem_isolation_on = False |
| node_mbytes = 0 # mbytes in one typical mem node |
| root_container_bytes = 0 # squishy limit on effective size of root container |
| |
| |
| def discover_container_style(): |
| global super_root_path, cpuset_prefix |
| global mem_isolation_on, fake_numa_containers |
| global node_mbytes, root_container_bytes |
| if super_root_path != '': |
| return # already looked up |
| if os.path.exists('/dev/cgroup/tasks'): |
| # running on 2.6.26 or later kernel with containers on: |
| super_root_path = '/dev/cgroup' |
| cpuset_prefix = 'cpuset.' |
| if get_boot_numa(): |
| mem_isolation_on = fake_numa_containers = True |
| else: # memcg containers IFF compiled-in & mounted & non-fakenuma boot |
| fake_numa_containers = False |
| mem_isolation_on = os.path.exists( |
| '/dev/cgroup/memory.limit_in_bytes') |
| # TODO: handle possibility of where memcg is mounted as its own |
| # cgroup hierarchy, separate from cpuset?? |
| elif os.path.exists('/dev/cpuset/tasks'): |
| # running on 2.6.18 kernel with containers on: |
| super_root_path = '/dev/cpuset' |
| cpuset_prefix = '' |
| mem_isolation_on = fake_numa_containers = get_boot_numa() != '' |
| else: |
| # neither cpuset nor cgroup filesystem active: |
| super_root_path = None |
| cpuset_prefix = 'no_cpusets_or_cgroups_exist' |
| mem_isolation_on = fake_numa_containers = False |
| |
| logging.debug('mem_isolation: %s', mem_isolation_on) |
| logging.debug('fake_numa_containers: %s', fake_numa_containers) |
| if fake_numa_containers: |
| node_mbytes = int(mbytes_per_mem_node()) |
| elif mem_isolation_on: # memcg-style containers |
| # For now, limit total of all containers to using just 98% of system's |
| # visible total ram, to avoid oom events at system level, and avoid |
| # page reclaim overhead from going above kswapd highwater mark. |
| system_visible_pages = utils.memtotal() >> 2 |
| usable_pages = int(system_visible_pages * 0.98) |
| root_container_bytes = usable_pages << 12 |
| logging.debug('root_container_bytes: %s', |
| utils.human_format(root_container_bytes)) |
| |
| |
| def need_mem_containers(): |
| discover_container_style() |
| if not mem_isolation_on: |
| raise error.AutotestError('Mem-isolation containers not enabled ' |
| 'by latest reboot') |
| |
| def need_fake_numa(): |
| discover_container_style() |
| if not fake_numa_containers: |
| raise error.AutotestError('fake=numa not enabled by latest reboot') |
| |
| |
| def full_path(container_name): |
| discover_container_style() |
| return os.path.join(super_root_path, container_name) |
| |
| |
| def unpath(container_path): |
| return container_path[len(super_root_path)+1:] |
| |
| |
| def cpuset_attr(container_name, attr): |
| discover_container_style() |
| return os.path.join(super_root_path, container_name, cpuset_prefix+attr) |
| |
| |
| def io_attr(container_name, attr): |
| discover_container_style() |
| # current version assumes shared cgroup hierarchy |
| return os.path.join(super_root_path, container_name, 'io.'+attr) |
| |
| |
| def tasks_path(container_name): |
| return os.path.join(full_path(container_name), 'tasks') |
| |
| |
| def mems_path(container_name): |
| return cpuset_attr(container_name, 'mems') |
| |
| |
| def memory_path(container_name): |
| return os.path.join(super_root_path, container_name, 'memory') |
| |
| |
| def cpus_path(container_name): |
| return cpuset_attr(container_name, 'cpus') |
| |
| |
| def container_exists(name): |
| return name is not None and os.path.exists(tasks_path(name)) |
| |
| |
| def move_tasks_into_container(name, tasks): |
| task_file = tasks_path(name) |
| for task in tasks: |
| try: |
| logging.debug('moving task %s into container "%s"', task, name) |
| utils.write_one_line(task_file, task) |
| except Exception: |
| if utils.pid_is_alive(task): |
| raise # task exists but couldn't move it |
| # task is gone or zombie so ignore this exception |
| |
| |
| def move_self_into_container(name): |
| me = str(os.getpid()) |
| move_tasks_into_container(name, [me]) |
| logging.debug('running self (pid %s) in container "%s"', me, name) |
| |
| |
| def _avail_mbytes_via_nodes(parent): |
| # total mbytes of mem nodes available for new containers in parent |
| free_nodes = available_exclusive_mem_nodes(parent) |
| mbytes = nodes_avail_mbytes(free_nodes) |
| # don't have exact model for how container mgr measures mem space |
| # better here to underestimate than overestimate |
| mbytes = max(mbytes - node_mbytes//2, 0) |
| return mbytes |
| |
| |
| def _avail_bytes_via_pages(parent): |
| # Get memory bytes available to parent container which could |
| # be allocated exclusively to new child containers. |
| # This excludes mem previously allocated to existing children. |
| available = container_bytes(parent) |
| mem_files_pattern = os.path.join(full_path(parent), |
| '*', 'memory.limit_in_bytes') |
| for mem_file in glob.glob(mem_files_pattern): |
| child_container = unpath(os.path.dirname(mem_file)) |
| available -= container_bytes(child_container) |
| return available |
| |
| |
| def avail_mbytes(parent=SUPER_ROOT): |
| # total mbytes available in parent, for exclusive use in new containers |
| if fake_numa_containers: |
| return _avail_mbytes_via_nodes(parent) |
| else: |
| return _avail_bytes_via_pages(parent) >> 20 |
| |
| |
| def delete_leftover_test_containers(): |
| # recover mems and cores tied up by containers of prior failed tests: |
| for child in inner_containers_of(SUPER_ROOT): |
| _release_container_nest(child) |
| |
| |
| def my_lock(lockname): |
| # lockname is 'inner' |
| lockdir = os.environ['AUTODIR'] |
| lockname = os.path.join(lockdir, '.cpuset.lock.'+lockname) |
| lockfile = open(lockname, 'w') |
| fcntl.flock(lockfile, fcntl.LOCK_EX) |
| return lockfile |
| |
| |
| def my_unlock(lockfile): |
| fcntl.flock(lockfile, fcntl.LOCK_UN) |
| lockfile.close() |
| |
| |
| # Convert '1-3,7,9-12' to set(1,2,3,7,9,10,11,12) |
| def rangelist_to_set(rangelist): |
| result = set() |
| if not rangelist: |
| return result |
| for x in rangelist.split(','): |
| if re.match(r'^(\d+)$', x): |
| result.add(int(x)) |
| continue |
| m = re.match(r'^(\d+)-(\d+)$', x) |
| if m: |
| start = int(m.group(1)) |
| end = int(m.group(2)) |
| result.update(set(range(start, end+1))) |
| continue |
| msg = 'Cannot understand data input: %s %s' % (x, rangelist) |
| raise ValueError(msg) |
| return result |
| |
| |
| def my_container_name(): |
| # Get current process's inherited or self-built container name |
| # within /dev/cpuset or /dev/cgroup. Is '' for root container. |
| name = utils.read_one_line('/proc/%i/cpuset' % os.getpid()) |
| return name[1:] # strip leading / |
| |
| |
| def get_mem_nodes(container_name): |
| # all mem nodes now available to a container, both exclusive & shared |
| file_name = mems_path(container_name) |
| if os.path.exists(file_name): |
| return rangelist_to_set(utils.read_one_line(file_name)) |
| else: |
| return set() |
| |
| |
| def _busy_mem_nodes(parent_container): |
| # Get set of numa memory nodes now used (exclusively or shared) |
| # by existing children of parent container |
| busy = set() |
| mem_files_pattern = os.path.join(full_path(parent_container), |
| '*', cpuset_prefix+'mems') |
| for mem_file in glob.glob(mem_files_pattern): |
| child_container = os.path.dirname(mem_file) |
| busy |= get_mem_nodes(child_container) |
| return busy |
| |
| |
| def available_exclusive_mem_nodes(parent_container): |
| # Get subset of numa memory nodes of parent container which could |
| # be allocated exclusively to new child containers. |
| # This excludes nodes now allocated to existing children. |
| need_fake_numa() |
| available = get_mem_nodes(parent_container) |
| available -= _busy_mem_nodes(parent_container) |
| return available |
| |
| |
| def my_mem_nodes(): |
| # Get set of numa memory nodes owned by current process's container. |
| discover_container_style() |
| if not mem_isolation_on: |
| return set() # as expected by vmstress |
| return get_mem_nodes(my_container_name()) |
| |
| |
| def my_available_exclusive_mem_nodes(): |
| # Get subset of numa memory nodes owned by current process's |
| # container, which could be allocated exclusively to new child |
| # containers. This excludes any nodes now allocated |
| # to existing children. |
| return available_exclusive_mem_nodes(my_container_name()) |
| |
| |
| def node_avail_kbytes(node): |
| return node_mbytes << 10 # crude; fixed numa node size |
| |
| |
| def nodes_avail_mbytes(nodes): |
| # nodes' combined user+avail size, in Mbytes |
| return sum(node_avail_kbytes(n) for n in nodes) // 1024 |
| |
| |
| def container_bytes(name): |
| if fake_numa_containers: |
| return nodes_avail_mbytes(get_mem_nodes(name)) << 20 |
| else: |
| while True: |
| file = memory_path(name) + '.limit_in_bytes' |
| limit = int(utils.read_one_line(file)) |
| if limit < NO_LIMIT: |
| return limit |
| if name == SUPER_ROOT: |
| return root_container_bytes |
| name = os.path.dirname(name) |
| |
| |
| def container_mbytes(name): |
| return container_bytes(name) >> 20 |
| |
| |
| def mbytes_per_mem_node(): |
| # Get mbyte size of standard numa mem node, as float |
| # (some nodes are bigger than this) |
| # Replaces utils.node_size(). |
| numa = get_boot_numa() |
| if numa.endswith('M'): |
| return float(numa[:-1]) # mbyte size of fake nodes |
| elif numa: |
| nodecnt = int(numa) # fake numa mem nodes for container isolation |
| else: |
| nodecnt = len(utils.numa_nodes()) # phys mem-controller nodes |
| # Use guessed total physical mem size, not kernel's |
| # lesser 'available memory' after various system tables. |
| return utils.rounded_memtotal() / (nodecnt * 1024.0) |
| |
| |
| def get_cpus(container_name): |
| file_name = cpus_path(container_name) |
| if os.path.exists(file_name): |
| return rangelist_to_set(utils.read_one_line(file_name)) |
| else: |
| return set() |
| |
| |
| def get_tasks(container_name): |
| file_name = tasks_path(container_name) |
| try: |
| tasks = [x.rstrip() for x in open(file_name).readlines()] |
| except IOError: |
| if os.path.exists(file_name): |
| raise |
| tasks = [] # container doesn't exist anymore |
| return tasks |
| |
| |
| def inner_containers_of(parent): |
| pattern = os.path.join(full_path(parent), '*/tasks') |
| return [unpath(os.path.dirname(task_file)) |
| for task_file in glob.glob(pattern)] |
| |
| |
| def _release_container_nest(nest): |
| # Destroy a container, and any nested sub-containers |
| nest_path = full_path(nest) |
| if os.path.exists(nest_path): |
| |
| # bottom-up walk of tree, releasing all nested sub-containers |
| for child in inner_containers_of(nest): |
| _release_container_nest(child) |
| |
| logging.debug("releasing container %s", nest) |
| |
| # Transfer any survivor tasks (e.g. self) to parent container |
| parent = os.path.dirname(nest) |
| move_tasks_into_container(parent, get_tasks(nest)) |
| |
| # remove the now-empty outermost container of this nest |
| if os.path.exists(nest_path): |
| os.rmdir(nest_path) # nested, or dead manager |
| |
| |
| def release_container(container_name=None): |
| # Destroy a container |
| my_container = my_container_name() |
| if container_name is None: |
| container_name = my_container |
| _release_container_nest(container_name) |
| displaced = my_container_name() |
| if displaced != my_container: |
| logging.debug('now running self (pid %d) in container "%s"', |
| os.getpid(), displaced) |
| |
| |
| def remove_empty_prio_classes(prios): |
| # remove prio classes whose set of allowed priorities is empty |
| # e.g 'no:3;rt:;be:3;id:' --> 'no:3;be:3' |
| return ';'.join(p for p in prios.split(';') if p.split(':')[1]) |
| |
| |
| def all_drive_names(): |
| # list of all disk drives sda,sdb,... |
| paths = glob.glob('/sys/block/sd*') |
| if not paths: |
| paths = glob.glob('/sys/block/hd*') |
| return [os.path.basename(path) for path in paths] |
| |
| |
| def set_io_controls(container_name, disks=[], ioprio_classes=[PROPIO_NORMAL], |
| io_shares=[95], io_limits=[0]): |
| # set the propio controls for one container, for selected disks |
| # writing directly to /dev/cgroup/container_name/io.io_service_level |
| # without using containerd or container.py |
| # See wiki ProportionalIOScheduler for definitions |
| # ioprio_classes: list of service classes, one per disk |
| # using numeric propio service classes as used by kernel API, namely |
| # 1: RT, Real Time, aka PROPIO_PRIO |
| # 2: BE, Best Effort, aka PROPIO_NORMAL |
| # 3: PROPIO_IDLE |
| # io_shares: list of disk-time-fractions, one per disk, |
| # as percentage integer 0..100 |
| # io_limits: list of limit on/off, one per disk |
| # 0: no limit, shares use of other containers' unused disk time |
| # 1: limited, container's use of disk time is capped to given DTF |
| # ioprio_classes defaults to best-effort |
| # io_limit defaults to no limit, use slack time |
| if not disks: # defaults to all drives |
| disks = all_drive_names() |
| io_shares = [io_shares [0]] * len(disks) |
| ioprio_classes = [ioprio_classes[0]] * len(disks) |
| io_limits = [io_limits [0]] * len(disks) |
| if not (len(disks) == len(ioprio_classes) and len(disks) == len(io_shares) |
| and len(disks) == len(io_limits)): |
| raise error.AutotestError('Unequal number of values for io controls') |
| service_level = io_attr(container_name, 'io_service_level') |
| if not os.path.exists(service_level): |
| return # kernel predates propio features |
| # or io cgroup is mounted separately from cpusets |
| disk_infos = [] |
| for disk,ioclass,limit,share in zip(disks, ioprio_classes, |
| io_limits, io_shares): |
| parts = (disk, str(ioclass), str(limit), str(share)) |
| disk_info = ' '.join(parts) |
| utils.write_one_line(service_level, disk_info) |
| disk_infos.append(disk_info) |
| logging.debug('set_io_controls of %s to %s', |
| container_name, ', '.join(disk_infos)) |
| |
| |
| def abbrev_list(vals): |
| """Condense unsigned (0,4,5,6,7,10) to '0,4-7,10'.""" |
| ranges = [] |
| lower = 0 |
| upper = -2 |
| for val in sorted(vals)+[-1]: |
| if val != upper+1: |
| if lower == upper: |
| ranges.append(str(lower)) |
| elif lower <= upper: |
| ranges.append('%d-%d' % (lower, upper)) |
| lower = val |
| upper = val |
| return ','.join(ranges) |
| |
| |
| def create_container_with_specific_mems_cpus(name, mems, cpus): |
| need_fake_numa() |
| os.mkdir(full_path(name)) |
| utils.write_one_line(cpuset_attr(name, 'mem_hardwall'), '1') |
| utils.write_one_line(mems_path(name), ','.join(map(str, mems))) |
| utils.write_one_line(cpus_path(name), ','.join(map(str, cpus))) |
| logging.debug('container %s has %d cpus and %d nodes totalling %s bytes', |
| name, len(cpus), len(get_mem_nodes(name)), |
| utils.human_format(container_bytes(name)) ) |
| |
| |
| def create_container_via_memcg(name, parent, bytes, cpus): |
| # create container via direct memcg cgroup writes |
| os.mkdir(full_path(name)) |
| nodes = utils.read_one_line(mems_path(parent)) |
| utils.write_one_line(mems_path(name), nodes) # inherit parent's nodes |
| utils.write_one_line(memory_path(name)+'.limit_in_bytes', str(bytes)) |
| utils.write_one_line(cpus_path(name), ','.join(map(str, cpus))) |
| logging.debug('Created container %s directly via memcg,' |
| ' has %d cpus and %s bytes', |
| name, len(cpus), utils.human_format(container_bytes(name))) |
| |
| |
| def _create_fake_numa_container_directly(name, parent, mbytes, cpus): |
| need_fake_numa() |
| lockfile = my_lock('inner') # serialize race between parallel tests |
| try: |
| # Pick specific mem nodes for new cpuset's exclusive use |
| # For now, arbitrarily pick highest available node numbers |
| needed_kbytes = mbytes * 1024 |
| nodes = sorted(list(available_exclusive_mem_nodes(parent))) |
| kbytes = 0 |
| nodecnt = 0 |
| while kbytes < needed_kbytes and nodecnt < len(nodes): |
| nodecnt += 1 |
| kbytes += node_avail_kbytes(nodes[-nodecnt]) |
| if kbytes < needed_kbytes: |
| parent_mbytes = container_mbytes(parent) |
| if mbytes > parent_mbytes: |
| raise error.AutotestError( |
| "New container's %d Mbytes exceeds " |
| "parent container's %d Mbyte size" |
| % (mbytes, parent_mbytes) ) |
| else: |
| raise error.AutotestError( |
| "Existing sibling containers hold " |
| "%d Mbytes needed by new container" |
| % ((needed_kbytes - kbytes)//1024) ) |
| mems = nodes[-nodecnt:] |
| |
| create_container_with_specific_mems_cpus(name, mems, cpus) |
| finally: |
| my_unlock(lockfile) |
| |
| |
| def create_container_directly(name, mbytes, cpus): |
| parent = os.path.dirname(name) |
| if fake_numa_containers: |
| _create_fake_numa_container_directly(name, parent, mbytes, cpus) |
| else: |
| create_container_via_memcg(name, parent, mbytes<<20, cpus) |
| |
| |
| def create_container_with_mbytes_and_specific_cpus(name, mbytes, |
| cpus=None, root=SUPER_ROOT, io={}, move_in=True, timeout=0): |
| """\ |
| Create a cpuset container and move job's current pid into it |
| Allocate the list "cpus" of cpus to that container |
| |
| name = arbitrary string tag |
| mbytes = reqested memory for job in megabytes |
| cpus = list of cpu indicies to associate with the cpuset |
| defaults to all cpus avail with given root |
| root = the parent cpuset to nest this new set within |
| '': unnested top-level container |
| io = arguments for proportional IO containers |
| move_in = True: Move current process into the new container now. |
| timeout = must be 0: persist until explicitly deleted. |
| """ |
| need_mem_containers() |
| if not container_exists(root): |
| raise error.AutotestError('Parent container "%s" does not exist' |
| % root) |
| if cpus is None: |
| # default to biggest container we can make under root |
| cpus = get_cpus(root) |
| else: |
| cpus = set(cpus) # interface uses list |
| if not cpus: |
| raise error.AutotestError('Creating container with no cpus') |
| name = os.path.join(root, name) # path relative to super_root |
| if os.path.exists(full_path(name)): |
| raise error.AutotestError('Container %s already exists' % name) |
| create_container_directly(name, mbytes, cpus) |
| set_io_controls(name, **io) |
| if move_in: |
| move_self_into_container(name) |
| return name |
| |
| |
| def get_boot_numa(): |
| # get boot-time numa=fake=xyz option for current boot |
| # eg numa=fake=nnn, numa=fake=nnnM, or nothing |
| label = 'numa=fake=' |
| for arg in utils.read_one_line('/proc/cmdline').split(): |
| if arg.startswith(label): |
| return arg[len(label):] |
| return '' |
