| # -*- coding: utf-8 -*- |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Common python commands used by various internal build scripts.""" |
| |
| from __future__ import print_function |
| |
| import os |
| import time |
| import multiprocessing |
| |
| from collections import namedtuple |
| |
| |
| AcquireResult = namedtuple('AcquireResult', ['result', 'reason']) |
| |
| def ListdirFullpath(directory): |
| """Return all files in a directory with full pathnames. |
| |
| Args: |
| directory: directory to find files for. |
| |
| Returns: |
| Full paths to every file in that directory. |
| """ |
| return [os.path.join(directory, f) for f in os.listdir(directory)] |
| |
| |
| class RestrictedAttrDict(dict): |
| """Define a dictionary which is also a struct. |
| |
| The keys will belong to a restricted list of values. |
| """ |
| |
| _slots = () |
| |
| def __init__(self, *args, **kwargs): |
| """Ensure that only the expected keys are added during initialization.""" |
| dict.__init__(self, *args, **kwargs) |
| |
| # Ensure all slots are at least populated with None. |
| for key in self._slots: |
| self.setdefault(key) |
| |
| for key in self.keys(): |
| assert key in self._slots, 'Unexpected key %s in %s' % (key, self._slots) |
| |
| def __setattr__(self, name, val): |
| """Setting an attribute, actually sets a dictionary value.""" |
| if name not in self._slots: |
| raise AttributeError("'%s' may not have attribute '%s'" % |
| (self.__class__.__name__, name)) |
| self[name] = val |
| |
| def __getattr__(self, name): |
| """Fetching an attribute, actually fetches a dictionary value.""" |
| if name not in self: |
| raise AttributeError("'%s' has no attribute '%s'" % |
| (self.__class__.__name__, name)) |
| return self[name] |
| |
| def __setitem__(self, name, val): |
| """Restrict which keys can be stored in this dictionary.""" |
| if name not in self._slots: |
| raise KeyError(name) |
| dict.__setitem__(self, name, val) |
| |
| def __str__(self): |
| """Default stringification behavior.""" |
| name = self._name if hasattr(self, '_name') else self.__class__.__name__ |
| return '%s (%s)' % (name, self._GetAttrString()) |
| |
| def _GetAttrString(self, delim=', ', equal='='): |
| """Return string showing all non-None values of self._slots. |
| |
| The ordering of attributes in self._slots is honored in string. |
| |
| Args: |
| delim: String for separating key/value elements in result. |
| equal: String to put between key and associated value in result. |
| |
| Returns: |
| A string like "a='foo', b=12". |
| """ |
| slots = [s for s in self._slots if self[s] is not None] |
| elems = ['%s%s%r' % (s, equal, self[s]) for s in slots] |
| return delim.join(elems) |
| |
| def _clear_if_default(self, key, default): |
| """Helper for constructors. |
| |
| If they key value is set to the default value, set it to None. |
| |
| Args: |
| key: Key value to check and possibly clear. |
| default: Default value to compare the key value against. |
| """ |
| if self[key] == default: |
| self[key] = None |
| |
| |
| def ReadLsbRelease(sysroot): |
| """Reads the /etc/lsb-release file out of the given sysroot. |
| |
| Args: |
| sysroot: The path to sysroot of an image to read sysroot/etc/lsb-release. |
| |
| Returns: |
| The lsb-release file content in a dictionary of key/values. |
| """ |
| lsb_release_file = os.path.join(sysroot, 'etc', 'lsb-release') |
| lsb_release = {} |
| with open(lsb_release_file, 'r') as f: |
| for line in f: |
| tokens = line.strip().split('=') |
| lsb_release[tokens[0]] = tokens[1] |
| |
| return lsb_release |
| |
| |
| class MemoryConsumptionSemaphore(object): |
| """A semaphore that tries to acquire only if there is enough memory available. |
| |
| Watch the free memory of the host in order to not oversubscribe. Also, |
| rate limit so that memory consumption of previously launched |
| fledgling process can swell to peak(ish) level. Also assumes this semaphore |
| controls the vast majority of the memory utilization on the host when |
| active. |
| |
| It will also measure the available total memory when there are no |
| acquires (and when it was initialized) and use that to baseline a guess |
| based on the configured max memory per acquire to limit the total of |
| acquires. |
| """ |
| SYSTEM_POLLING_INTERVAL_SECONDS = 0.5 |
| |
| def __init__(self, system_available_buffer_bytes=None, |
| single_proc_max_bytes=None, |
| quiescence_time_seconds=None, |
| unchecked_acquires=0, |
| total_max=10, |
| clock=time.time): |
| """Create a new MemoryConsumptionSemaphore. |
| |
| Args: |
| system_available_buffer_bytes (int): The number of bytes to reserve |
| on the system as a buffer against moving into swap (or OOM). |
| single_proc_max_bytes (int): The number of bytes we expect a process |
| to consume on the system. |
| quiescence_time_seconds (float): The number of seconds to wait at a |
| minimum between acquires. The purpose is to ensure the subprocess |
| begins to consume a stable amount of memory. |
| unchecked_acquires (int): The number acquires to allow without checking |
| available memory. This is to allow users to supply a mandatory minimum |
| even if the semaphore would otherwise not allow it (because of the |
| current available memory being to low). |
| total_max (int): The upper bound of maximum concurrent runs (default 10). |
| clock (fn): Function that gets float time. |
| |
| Returns: |
| A new MemoryConsumptionSemaphore. |
| """ |
| self.quiescence_time_seconds = quiescence_time_seconds |
| self.unchecked_acquires = unchecked_acquires |
| self._lock = multiprocessing.RLock() # single proc may acquire lock twice. |
| self._total_max = multiprocessing.RawValue('I', total_max) |
| self._n_within = multiprocessing.RawValue('I', 0) |
| self._timer_future = multiprocessing.RawValue('d', 0) |
| self._clock = clock # injected, primarily useful for testing. |
| self._system_available_buffer_bytes = system_available_buffer_bytes |
| self._single_proc_max_bytes = single_proc_max_bytes |
| self._base_available = self._get_system_available() |
| |
| def _get_system_available(self): |
| """Get the system's available memory (memory free before swapping).""" |
| with open('/proc/meminfo') as fp: |
| for line in fp: |
| fields = line.split() |
| if fields[0] == 'MemAvailable:': |
| size = int(fields[1]) |
| if len(fields) > 2: |
| assert fields[2] == 'kB', line |
| size *= 1024 |
| return size |
| return 0 |
| |
| def _timer_blocked(self): |
| """Check the timer, if we're past it return true, otherwise false.""" |
| if self._clock() >= self._timer_future.value: |
| return False |
| else: |
| return True |
| |
| def _inc_within(self): |
| """Inc the lock.""" |
| with self._lock: |
| self._n_within.value += 1 |
| |
| def _dec_within(self): |
| """Dec the lock.""" |
| with self._lock: |
| self._n_within.value -= 1 |
| |
| def _set_timer(self): |
| """Set a time in the future to unblock after.""" |
| with self._lock: |
| self._timer_future.value = \ |
| max(self._clock() + self.quiescence_time_seconds, |
| self._timer_future.value) |
| |
| def _allow_consumption(self): |
| """Calculate max utilization to determine if another should be allowed. |
| |
| Returns: |
| Boolean if you're allowed to consume (acquire). |
| """ |
| with self._lock: |
| one_more_total = (self._n_within.value + 1) * self._single_proc_max_bytes |
| |
| total_avail = self._base_available - self._system_available_buffer_bytes |
| # If the guessed max plus yourself is above what's available including |
| # the buffer then refuse to admit. |
| if total_avail < one_more_total: |
| return False |
| else: |
| return True |
| |
| def acquire(self, timeout): |
| """Block until enough available memory, or timeout. |
| |
| Polls the system every SYSTEM_POLLING_INTERVAL_SECONDS and determines |
| if there is enough available memory to proceed, or potentially timeout. |
| |
| Args: |
| timeout (float): Time to block for available memory before return. |
| |
| Returns: |
| A tuple (bool, string) bool flags if you should go, and string is a text |
| representation of the reason for the acquire result. |
| """ |
| |
| # Remeasure the base. |
| if self._n_within.value == 0: |
| self._base_available = self._get_system_available() |
| |
| # If you're under the unchecked_acquires go for it, but lock |
| # so that we can't race for it. |
| with self._lock: |
| if self._n_within.value < self.unchecked_acquires: |
| self._set_timer() |
| self._inc_within() |
| return AcquireResult(True, 'Succeeded as unchecked') |
| init_time = self._clock() |
| |
| # If not enough memory or timer is running then block. |
| while init_time + timeout > self._clock(): |
| with self._lock: |
| if not self._timer_blocked(): |
| # Extrapolate system state and perhaps allow. |
| if (self._allow_consumption() and |
| self._n_within.value < self._total_max.value): |
| self._set_timer() |
| self._inc_within() |
| return AcquireResult(True, 'Allowed due to available memory') |
| time.sleep(MemoryConsumptionSemaphore.SYSTEM_POLLING_INTERVAL_SECONDS) |
| |
| # There was no moment before timeout where we could have ran the task. |
| return AcquireResult(False, |
| 'Timed out (due to quiescence, ' |
| 'total max, or avail memory)') |
| |
| def release(self): |
| """Releases a single acquire.""" |
| self._dec_within() |
