| #!/bin/sh |
| # Copyright 2013 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # Some startup functions are split into a separate library which may be |
| # different for different targets (e.g., regular Chrome OS vs. embedded). |
| # shellcheck source=encrypted_stateful/startup_utils.sh |
| . /usr/share/cros/startup_utils.sh |
| |
| # Shutdown is best-effort. We don't want to die on errors. |
| set +e |
| |
| bootstat shutdown-start |
| |
| # Let's send output of this script through the kernel. This means that |
| # we can see any relevant output after bootup in console-ramoops. |
| # We'll also explicitly set `printk_devkmsg` to on which will prevent |
| # rate-limiting. |
| # |
| # NOTE: this trick generally works more reliably with stderr than stdout. |
| # Specifically writes to stdout may fail because programs may detect the |
| # "/dev/kmsg" is a file and try to buffer output to stdout. If any individual |
| # write is over the 1024 byte limit in the kernel then it will fail. We |
| # expect most things we care about to be on stderr, but if an individual |
| # program is getting a failure it could likely be handled by piping its |
| # stdout to `sed -u -e ''`. |
| sysctl -w kernel.printk_devkmsg=on |
| exec 1> /dev/kmsg 2> /dev/kmsg |
| |
| # Set nologin so services (like usb_bouncer) can check that a shutdown is |
| # happening. |
| touch /run/nologin |
| |
| # Remount root in case a developer has remounted it rw for some reason. |
| mount -n -o remount,ro / |
| |
| # TODO: swapoff as necessary. |
| |
| PROCESS_KILLER_LOG=/run/process_killer.log |
| |
| # Instruct upstart to not respawn jobs beyond this point in the shutdown process. |
| initctl disable-respawn |
| |
| # Kill any that may prevent us from unmounting the stateful partition |
| # or the crypto-home and then unmount. These should be all that we need |
| # to unmount for a clean shutdown. |
| # |
| # We send the output of this script in two places: stdout (which is |
| # redirected to /dev/kmsg) and "${PROCESS_KILLER_LOG}". |
| # |
| # Having ouptut in kmsg is useful because it captures the kills that |
| # process_killer does even in the case that we don't have a catastrophic |
| # unmount failure. We still care about these non-catastrophic cases because |
| # processes _should_ have been killed before this script was even called (this |
| # script is a last resort). Thus, having these logs show up in console-ramoops |
| # is a good way to call attention to these kills. |
| # |
| # Another benefit of having process_killer logs in /dev/kmsg is that the |
| # system is more likely than normal to crash at reboot/shutdown time as it |
| # executes code that isn't exercised as much. Writing to /dev/kmsg shows up in |
| # console-ramoops even if the kernel crashes partway through. |
| # |
| # Having process_killer logs in "${PROCESS_KILLER_LOG}" is also useful because, |
| # if we do have an unmount failure, the output of process_killer is very |
| # relevant and unmount failure crash reports don't include the kernel messages. |
| process_killer --shutdown --log_to_stderr --file_holders \ |
| --mount_holders 2>&1 | tee "${PROCESS_KILLER_LOG}" | sed -u -e "" |
| |
| # CROS_DEBUG equals one if we've booted in developer mode or we've booted a |
| # developer image. |
| crossystem "cros_debug?1" |
| CROS_DEBUG="$((! $?))" |
| |
| dev_unmount_packages() { true; } |
| dev_push_paths_to_preserve() { true; } |
| |
| collect_shutdown_umount_failure_logs() { |
| ( |
| # Get mount table as seen by this process. This list may not be |
| # comprehensive: private mounts inside mount namespaces may not |
| # appear in this list. |
| echo "Active mounts:" |
| cat "/proc/self/mountinfo" |
| # Log information about the dm-crypt device. |
| echo "Device-mapper target state:" |
| dmsetup info -c -o name,open,attr |
| # Log information about loop devices. |
| # Helpful in debugging "bad" loopback devices that shouldn't exist. (e.g. |
| # legacy DLC image associated loopback devices) |
| echo "Loop devices:" |
| losetup -a |
| # Log upstart jobs that are either still running or tracking active |
| # processes: jobs here can stop the stateful partition from getting |
| # unmounted. |
| echo "Active Upstart jobs:" |
| initctl list | grep -e process -e running |
| # Log dbus services still connected to the bus, with a 1 second timeout. |
| echo "Active D-Bus services:" |
| dbus-send --system --dest=org.freedesktop.DBus --type=method_call \ |
| --print-reply --reply-timeout=1000 /org/freedesktop/DBus \ |
| org.freedesktop.DBus.ListNames |
| |
| # Print process tree. |
| echo "Process tree:" |
| ps --deselect --ppid 2 --forest -f -o pid,ppid,state,comm |
| # Parse pids of processes with active mount namespaces. PID is the fourth |
| # field and the first line refers to the root mount namespace. |
| for pid in $(lsns --raw -n -t mnt | cut -f4 -d' ' | tail -n+2); do |
| echo "====== PID: ${pid} Comm: $(cat "/proc/${pid}/comm") ======" |
| # Use findmnt to look inside the mount namespace for any mounts that |
| # refer to the (enc)stateful partition. These mounts may keep the |
| # stateful partition from unmounting cleanly. |
| echo "Open mounts into (enc)stateful:" |
| findmnt --raw -N "${pid}" | grep -e stateful -e mmcblk0p1 -e nvme0n1p1 \ |
| -e sda1 |
| done |
| |
| # Get a list of processes with files open. lsof is verbose: keep it at the |
| # end of the log. |
| echo "Processes with files open:" |
| lsof -n /mnt/stateful_partition /var /home/chronos |
| cat "${PROCESS_KILLER_LOG}" |
| ) >/run/shutdown_umount_failure.log 2>&1 |
| } |
| |
| # Attempt multiple retries for each mount point. This prevents transient |
| # failures from stopping the unmount process. |
| umount_mountpoint() { |
| local mnt="$1" |
| # Check if the mount point exists. |
| if ! mountpoint -q "${mnt}"; then |
| return 0 |
| fi |
| |
| local rc=0 |
| for _ in 1 2 3 4 5 6 7 8 9 10; do |
| umount -n "${mnt}" |
| rc="$?" |
| if [ "${rc}" -eq "0" ]; then |
| break |
| fi |
| sleep 0.1 |
| done |
| return "${rc}" |
| } |
| |
| if [ "${CROS_DEBUG}" -eq 1 ]; then |
| . /usr/share/cros/dev_utils.sh |
| fi |
| |
| STATEFUL_PARTITION="/mnt/stateful_partition" |
| STATEFUL_UPDATE="${STATEFUL_PARTITION}/.update_available" |
| |
| # target_version should only be created for test lab DUTs. |
| TARGET_VERSION="/run/update_target_version" |
| UPDATE_TARGET="" |
| STATE_DEV="" |
| |
| if [ "${CROS_DEBUG}" = "1" ] && [ -f "${STATEFUL_UPDATE}" ]; then |
| STATEFUL_UPDATE_ARGS="$(cat "${STATEFUL_UPDATE}")" |
| |
| if [ -r "${TARGET_VERSION}" ] && [ ! -L "${TARGET_VERSION}" ]; then |
| # Used later to clear Quota parameters from stateful. |
| UPDATE_TARGET="$(cut -d '.' -f 1 "${TARGET_VERSION}")" |
| STATE_DEV="$(findmnt -n -o SOURCE -M "${STATEFUL_PARTITION}")" |
| rm -f "${TARGET_VERSION}" |
| fi |
| |
| if [ "${STATEFUL_UPDATE_ARGS}" = "clobber" ]; then |
| PRESERVE_DIR="${STATEFUL_PARTITION}/unencrypted/preserve" |
| |
| # Measure shutdown time up to this point. |
| bootstat before_preserve |
| |
| # We preserve a few files. Make sure preservation directory starts empty. |
| rm -rf "${PRESERVE_DIR}/log" |
| # shellcheck disable=SC2174 |
| mkdir -p -m 0755 "${PRESERVE_DIR}" |
| cp -a "/var/log" "${PRESERVE_DIR}" |
| dev_push_paths_to_preserve |
| |
| # We are about to put this into a directory that will shortly be wiped |
| # out. Keep a timestamp where it will be preserved as well. |
| PRESERVE_METRICS="${PRESERVE_DIR}/log/metrics" |
| bootstat_archive \ |
| "${PRESERVE_METRICS}/shutdown.$(date --utc '+%Y%m%d%H%M%S')" |
| fi |
| fi |
| |
| # Signal that the clean shutdown point was reached (or at least as |
| # close to that point as we can be before stateful is unmounted). |
| # Log to stderr since syslog may not be available at this stage. |
| crash_reporter --log_to_stderr --clean_shutdown |
| |
| # Flush buffers to disk to reflect this part of shutdown in the metrics. |
| sync |
| |
| if [ -d /var/log ]; then |
| # Remove old metrics since we only care about the last one. |
| rm -rf /var/log/metrics/shutdown.* |
| # Measure shutdown time up to this point. |
| bootstat_archive "/var/log/metrics/shutdown.$(date --utc '+%Y%m%d%H%M%S')" |
| fi |
| |
| # Preserves the rollback-prefixed content of pstore across one warm reboot. |
| preserve_rollback_key() { |
| local pmsg="/dev/pmsg0" |
| local key f |
| for f in /sys/fs/pstore/pmsg-ramoops-*; do |
| # The glob may not match any file. |
| [ -e "${f}" ] || continue |
| |
| # Ignore failures. |
| key="$(grep "^rollback " "${f}")" || continue |
| echo "${key}" >>"${pmsg}" || true |
| |
| # We've found a key. We're done. |
| break |
| done |
| } |
| |
| # Preserve rollback key across firmware update reboot. |
| # shellcheck disable=SC2154 |
| if [ "${SHUTDOWN_REASON}" = "firmware-update" ]; then |
| preserve_rollback_key |
| fi |
| |
| # To be safe, flush buffers to disk again before unmounting. (From |
| # https://crbug.com/760007 it seems that a failed umount can get the filesystem |
| # into a state that renders a subsequent sync ineffective.) |
| sync |
| |
| # Log all the unmount logic to a temp file and move it over to stateful if any |
| # of the steps failed. |
| ( |
| set -x |
| |
| # Unmount stateful partition for dev packages. Will be a NOP unless we're in |
| # dev mode. |
| dev_unmount_packages |
| |
| # Unmount /var/run and /var/lock, which were bind mounted to /run and /run/lock |
| # respectively to enable backwards compatibility for accessing /run (tmpfs for |
| # runtime data) through /var. |
| umount_mountpoint "/var/run" |
| umount_mountpoint "/var/lock" |
| |
| # Unmount /var, /home and encrypted mountpoints, then try to |
| # unmount /mnt/stateful_partition. Log to /mnt/stateful_partition if any of |
| # them failed to unmount. |
| # Note that the other mounts are submounts of /mnt/stateful_partition on |
| # regular images, but not always true on factory images. To handle both, we |
| # should unmount /mnt/stateful_partition only if the others successfully |
| # unmounted, otherwise system may fail to log. See crbug.com/835557. |
| umount_var_and_home_chronos |
| rc="$?" |
| |
| # Check if /home is mounted before attempting to umount(). |
| umount_mountpoint "/home" |
| : "$(( rc |= $? ))" |
| |
| # Unmount /mnt/stateful_partition only if the previous unmounts succeeded. |
| if [ "${rc}" -eq 0 ]; then |
| umount_mountpoint "${STATEFUL_PARTITION}" |
| fi |
| exit "$(( rc | $? ))" |
| ) >/run/mount_encrypted/umount-encrypted.log 2>&1 |
| |
| # shellcheck disable=SC2181 |
| if [ "$?" -ne 0 ]; then |
| # Collect information about active mount namespaces and if there are bind |
| # mounts open inside these namespaces into the (enc)stateful partition. |
| collect_shutdown_umount_failure_logs |
| crash_reporter --early --log_to_stderr --umount_failure \ |
| --mount_device="stateful" |
| crash_reporter --early --log_to_stderr --preserve_across_clobber \ |
| --ephemeral_collect |
| mv /run/mount_encrypted/umount-encrypted.log "${STATEFUL_PARTITION}/" |
| mv /run/shutdown_umount_failure.log "${STATEFUL_PARTITION}/" |
| else |
| if [ -n "${UPDATE_TARGET}" ] && [ -n "${STATE_DEV}" ]; then |
| # 10756.0.0 is the first build to turn on ext4 quota. |
| # See https://crrev.com/c/1016226 |
| # Older builds will fail to mount stateful if quota is enabled. |
| # This code can be removed when we stop testing pre-R69 FSI updates. |
| if [ "${UPDATE_TARGET}" -lt 10756 ]; then |
| if dumpe2fs -h "${STATE_DEV}" 2>/dev/null | \ |
| grep -qe "^Filesystem features:.* quota.*"; then |
| tune2fs -O^quota -Q^usrquota,^grpquota,^prjquota "${STATE_DEV}" |
| fi |
| fi |
| fi |
| rm -f /run/mount_encrypted/umount-encrypted.log |
| fi |
| |
| # Just in case something didn't unmount properly above. |
| sync |
| |
| # Display low battery icon if shutting down due to low battery. |
| # SHUTDOWN_REASON is passed in with the runlevel event from power manager. |
| # shellcheck disable=SC2154 |
| if [ "${SHUTDOWN_REASON}" = "low-battery" ]; then |
| display_low_battery_alert |
| fi |
| |
| # Ensure that we always claim success. |
| exit 0 |