| // SPDX-License-Identifier: GPL-2.0-only |
| /* |
| * x86_pkg_temp_thermal driver |
| * Copyright (c) 2013, Intel Corporation. |
| */ |
| #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| |
| #include <linux/module.h> |
| #include <linux/init.h> |
| #include <linux/err.h> |
| #include <linux/param.h> |
| #include <linux/device.h> |
| #include <linux/platform_device.h> |
| #include <linux/cpu.h> |
| #include <linux/smp.h> |
| #include <linux/slab.h> |
| #include <linux/pm.h> |
| #include <linux/thermal.h> |
| #include <linux/debugfs.h> |
| #include <asm/cpu_device_id.h> |
| #include <asm/mce.h> |
| |
| /* |
| * Rate control delay: Idea is to introduce denounce effect |
| * This should be long enough to avoid reduce events, when |
| * threshold is set to a temperature, which is constantly |
| * violated, but at the short enough to take any action. |
| * The action can be remove threshold or change it to next |
| * interesting setting. Based on experiments, in around |
| * every 5 seconds under load will give us a significant |
| * temperature change. |
| */ |
| #define PKG_TEMP_THERMAL_NOTIFY_DELAY 5000 |
| static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY; |
| module_param(notify_delay_ms, int, 0644); |
| MODULE_PARM_DESC(notify_delay_ms, |
| "User space notification delay in milli seconds."); |
| |
| /* Number of trip points in thermal zone. Currently it can't |
| * be more than 2. MSR can allow setting and getting notifications |
| * for only 2 thresholds. This define enforces this, if there |
| * is some wrong values returned by cpuid for number of thresholds. |
| */ |
| #define MAX_NUMBER_OF_TRIPS 2 |
| |
| struct zone_device { |
| int cpu; |
| bool work_scheduled; |
| u32 tj_max; |
| u32 msr_pkg_therm_low; |
| u32 msr_pkg_therm_high; |
| struct delayed_work work; |
| struct thermal_zone_device *tzone; |
| struct cpumask cpumask; |
| }; |
| |
| static struct thermal_zone_params pkg_temp_tz_params = { |
| .no_hwmon = true, |
| }; |
| |
| /* Keep track of how many zone pointers we allocated in init() */ |
| static int max_id __read_mostly; |
| /* Array of zone pointers */ |
| static struct zone_device **zones; |
| /* Serializes interrupt notification, work and hotplug */ |
| static DEFINE_SPINLOCK(pkg_temp_lock); |
| /* Protects zone operation in the work function against hotplug removal */ |
| static DEFINE_MUTEX(thermal_zone_mutex); |
| |
| /* The dynamically assigned cpu hotplug state for module_exit() */ |
| static enum cpuhp_state pkg_thermal_hp_state __read_mostly; |
| |
| /* Debug counters to show using debugfs */ |
| static struct dentry *debugfs; |
| static unsigned int pkg_interrupt_cnt; |
| static unsigned int pkg_work_cnt; |
| |
| static void pkg_temp_debugfs_init(void) |
| { |
| debugfs = debugfs_create_dir("pkg_temp_thermal", NULL); |
| |
| debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs, |
| &pkg_interrupt_cnt); |
| debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs, |
| &pkg_work_cnt); |
| } |
| |
| /* |
| * Protection: |
| * |
| * - cpu hotplug: Read serialized by cpu hotplug lock |
| * Write must hold pkg_temp_lock |
| * |
| * - Other callsites: Must hold pkg_temp_lock |
| */ |
| static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu) |
| { |
| int id = topology_logical_die_id(cpu); |
| |
| if (id >= 0 && id < max_id) |
| return zones[id]; |
| return NULL; |
| } |
| |
| /* |
| * tj-max is is interesting because threshold is set relative to this |
| * temperature. |
| */ |
| static int get_tj_max(int cpu, u32 *tj_max) |
| { |
| u32 eax, edx, val; |
| int err; |
| |
| err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx); |
| if (err) |
| return err; |
| |
| val = (eax >> 16) & 0xff; |
| *tj_max = val * 1000; |
| |
| return val ? 0 : -EINVAL; |
| } |
| |
| static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp) |
| { |
| struct zone_device *zonedev = tzd->devdata; |
| u32 eax, edx; |
| |
| rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_STATUS, |
| &eax, &edx); |
| if (eax & 0x80000000) { |
| *temp = zonedev->tj_max - ((eax >> 16) & 0x7f) * 1000; |
| pr_debug("sys_get_curr_temp %d\n", *temp); |
| return 0; |
| } |
| return -EINVAL; |
| } |
| |
| static int sys_get_trip_temp(struct thermal_zone_device *tzd, |
| int trip, int *temp) |
| { |
| struct zone_device *zonedev = tzd->devdata; |
| unsigned long thres_reg_value; |
| u32 mask, shift, eax, edx; |
| int ret; |
| |
| if (trip >= MAX_NUMBER_OF_TRIPS) |
| return -EINVAL; |
| |
| if (trip) { |
| mask = THERM_MASK_THRESHOLD1; |
| shift = THERM_SHIFT_THRESHOLD1; |
| } else { |
| mask = THERM_MASK_THRESHOLD0; |
| shift = THERM_SHIFT_THRESHOLD0; |
| } |
| |
| ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, |
| &eax, &edx); |
| if (ret < 0) |
| return ret; |
| |
| thres_reg_value = (eax & mask) >> shift; |
| if (thres_reg_value) |
| *temp = zonedev->tj_max - thres_reg_value * 1000; |
| else |
| *temp = 0; |
| pr_debug("sys_get_trip_temp %d\n", *temp); |
| |
| return 0; |
| } |
| |
| static int |
| sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp) |
| { |
| struct zone_device *zonedev = tzd->devdata; |
| u32 l, h, mask, shift, intr; |
| int ret; |
| |
| if (trip >= MAX_NUMBER_OF_TRIPS || temp >= zonedev->tj_max) |
| return -EINVAL; |
| |
| ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, |
| &l, &h); |
| if (ret < 0) |
| return ret; |
| |
| if (trip) { |
| mask = THERM_MASK_THRESHOLD1; |
| shift = THERM_SHIFT_THRESHOLD1; |
| intr = THERM_INT_THRESHOLD1_ENABLE; |
| } else { |
| mask = THERM_MASK_THRESHOLD0; |
| shift = THERM_SHIFT_THRESHOLD0; |
| intr = THERM_INT_THRESHOLD0_ENABLE; |
| } |
| l &= ~mask; |
| /* |
| * When users space sets a trip temperature == 0, which is indication |
| * that, it is no longer interested in receiving notifications. |
| */ |
| if (!temp) { |
| l &= ~intr; |
| } else { |
| l |= (zonedev->tj_max - temp)/1000 << shift; |
| l |= intr; |
| } |
| |
| return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, |
| l, h); |
| } |
| |
| static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip, |
| enum thermal_trip_type *type) |
| { |
| *type = THERMAL_TRIP_PASSIVE; |
| return 0; |
| } |
| |
| /* Thermal zone callback registry */ |
| static struct thermal_zone_device_ops tzone_ops = { |
| .get_temp = sys_get_curr_temp, |
| .get_trip_temp = sys_get_trip_temp, |
| .get_trip_type = sys_get_trip_type, |
| .set_trip_temp = sys_set_trip_temp, |
| }; |
| |
| static bool pkg_thermal_rate_control(void) |
| { |
| return true; |
| } |
| |
| /* Enable threshold interrupt on local package/cpu */ |
| static inline void enable_pkg_thres_interrupt(void) |
| { |
| u8 thres_0, thres_1; |
| u32 l, h; |
| |
| rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); |
| /* only enable/disable if it had valid threshold value */ |
| thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0; |
| thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1; |
| if (thres_0) |
| l |= THERM_INT_THRESHOLD0_ENABLE; |
| if (thres_1) |
| l |= THERM_INT_THRESHOLD1_ENABLE; |
| wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); |
| } |
| |
| /* Disable threshold interrupt on local package/cpu */ |
| static inline void disable_pkg_thres_interrupt(void) |
| { |
| u32 l, h; |
| |
| rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); |
| |
| l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE); |
| wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); |
| } |
| |
| static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work) |
| { |
| struct thermal_zone_device *tzone = NULL; |
| int cpu = smp_processor_id(); |
| struct zone_device *zonedev; |
| u64 msr_val, wr_val; |
| |
| mutex_lock(&thermal_zone_mutex); |
| spin_lock_irq(&pkg_temp_lock); |
| ++pkg_work_cnt; |
| |
| zonedev = pkg_temp_thermal_get_dev(cpu); |
| if (!zonedev) { |
| spin_unlock_irq(&pkg_temp_lock); |
| mutex_unlock(&thermal_zone_mutex); |
| return; |
| } |
| zonedev->work_scheduled = false; |
| |
| rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); |
| wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1); |
| if (wr_val != msr_val) { |
| wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val); |
| tzone = zonedev->tzone; |
| } |
| |
| enable_pkg_thres_interrupt(); |
| spin_unlock_irq(&pkg_temp_lock); |
| |
| /* |
| * If tzone is not NULL, then thermal_zone_mutex will prevent the |
| * concurrent removal in the cpu offline callback. |
| */ |
| if (tzone) |
| thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED); |
| |
| mutex_unlock(&thermal_zone_mutex); |
| } |
| |
| static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work) |
| { |
| unsigned long ms = msecs_to_jiffies(notify_delay_ms); |
| |
| schedule_delayed_work_on(cpu, work, ms); |
| } |
| |
| static int pkg_thermal_notify(u64 msr_val) |
| { |
| int cpu = smp_processor_id(); |
| struct zone_device *zonedev; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&pkg_temp_lock, flags); |
| ++pkg_interrupt_cnt; |
| |
| disable_pkg_thres_interrupt(); |
| |
| /* Work is per package, so scheduling it once is enough. */ |
| zonedev = pkg_temp_thermal_get_dev(cpu); |
| if (zonedev && !zonedev->work_scheduled) { |
| zonedev->work_scheduled = true; |
| pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work); |
| } |
| |
| spin_unlock_irqrestore(&pkg_temp_lock, flags); |
| return 0; |
| } |
| |
| static int pkg_temp_thermal_device_add(unsigned int cpu) |
| { |
| int id = topology_logical_die_id(cpu); |
| u32 tj_max, eax, ebx, ecx, edx; |
| struct zone_device *zonedev; |
| int thres_count, err; |
| |
| if (id >= max_id) |
| return -ENOMEM; |
| |
| cpuid(6, &eax, &ebx, &ecx, &edx); |
| thres_count = ebx & 0x07; |
| if (!thres_count) |
| return -ENODEV; |
| |
| thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS); |
| |
| err = get_tj_max(cpu, &tj_max); |
| if (err) |
| return err; |
| |
| zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL); |
| if (!zonedev) |
| return -ENOMEM; |
| |
| INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn); |
| zonedev->cpu = cpu; |
| zonedev->tj_max = tj_max; |
| zonedev->tzone = thermal_zone_device_register("x86_pkg_temp", |
| thres_count, |
| (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01, |
| zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0); |
| if (IS_ERR(zonedev->tzone)) { |
| err = PTR_ERR(zonedev->tzone); |
| kfree(zonedev); |
| return err; |
| } |
| /* Store MSR value for package thermal interrupt, to restore at exit */ |
| rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low, |
| zonedev->msr_pkg_therm_high); |
| |
| cpumask_set_cpu(cpu, &zonedev->cpumask); |
| spin_lock_irq(&pkg_temp_lock); |
| zones[id] = zonedev; |
| spin_unlock_irq(&pkg_temp_lock); |
| return 0; |
| } |
| |
| static int pkg_thermal_cpu_offline(unsigned int cpu) |
| { |
| struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu); |
| bool lastcpu, was_target; |
| int target; |
| |
| if (!zonedev) |
| return 0; |
| |
| target = cpumask_any_but(&zonedev->cpumask, cpu); |
| cpumask_clear_cpu(cpu, &zonedev->cpumask); |
| lastcpu = target >= nr_cpu_ids; |
| /* |
| * Remove the sysfs files, if this is the last cpu in the package |
| * before doing further cleanups. |
| */ |
| if (lastcpu) { |
| struct thermal_zone_device *tzone = zonedev->tzone; |
| |
| /* |
| * We must protect against a work function calling |
| * thermal_zone_update, after/while unregister. We null out |
| * the pointer under the zone mutex, so the worker function |
| * won't try to call. |
| */ |
| mutex_lock(&thermal_zone_mutex); |
| zonedev->tzone = NULL; |
| mutex_unlock(&thermal_zone_mutex); |
| |
| thermal_zone_device_unregister(tzone); |
| } |
| |
| /* Protect against work and interrupts */ |
| spin_lock_irq(&pkg_temp_lock); |
| |
| /* |
| * Check whether this cpu was the current target and store the new |
| * one. When we drop the lock, then the interrupt notify function |
| * will see the new target. |
| */ |
| was_target = zonedev->cpu == cpu; |
| zonedev->cpu = target; |
| |
| /* |
| * If this is the last CPU in the package remove the package |
| * reference from the array and restore the interrupt MSR. When we |
| * drop the lock neither the interrupt notify function nor the |
| * worker will see the package anymore. |
| */ |
| if (lastcpu) { |
| zones[topology_logical_die_id(cpu)] = NULL; |
| /* After this point nothing touches the MSR anymore. */ |
| wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, |
| zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high); |
| } |
| |
| /* |
| * Check whether there is work scheduled and whether the work is |
| * targeted at the outgoing CPU. |
| */ |
| if (zonedev->work_scheduled && was_target) { |
| /* |
| * To cancel the work we need to drop the lock, otherwise |
| * we might deadlock if the work needs to be flushed. |
| */ |
| spin_unlock_irq(&pkg_temp_lock); |
| cancel_delayed_work_sync(&zonedev->work); |
| spin_lock_irq(&pkg_temp_lock); |
| /* |
| * If this is not the last cpu in the package and the work |
| * did not run after we dropped the lock above, then we |
| * need to reschedule the work, otherwise the interrupt |
| * stays disabled forever. |
| */ |
| if (!lastcpu && zonedev->work_scheduled) |
| pkg_thermal_schedule_work(target, &zonedev->work); |
| } |
| |
| spin_unlock_irq(&pkg_temp_lock); |
| |
| /* Final cleanup if this is the last cpu */ |
| if (lastcpu) |
| kfree(zonedev); |
| return 0; |
| } |
| |
| static int pkg_thermal_cpu_online(unsigned int cpu) |
| { |
| struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu); |
| struct cpuinfo_x86 *c = &cpu_data(cpu); |
| |
| /* Paranoia check */ |
| if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS)) |
| return -ENODEV; |
| |
| /* If the package exists, nothing to do */ |
| if (zonedev) { |
| cpumask_set_cpu(cpu, &zonedev->cpumask); |
| return 0; |
| } |
| return pkg_temp_thermal_device_add(cpu); |
| } |
| |
| static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = { |
| { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_PTS }, |
| {} |
| }; |
| MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids); |
| |
| static int __init pkg_temp_thermal_init(void) |
| { |
| int ret; |
| |
| if (!x86_match_cpu(pkg_temp_thermal_ids)) |
| return -ENODEV; |
| |
| max_id = topology_max_packages() * topology_max_die_per_package(); |
| zones = kcalloc(max_id, sizeof(struct zone_device *), |
| GFP_KERNEL); |
| if (!zones) |
| return -ENOMEM; |
| |
| ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online", |
| pkg_thermal_cpu_online, pkg_thermal_cpu_offline); |
| if (ret < 0) |
| goto err; |
| |
| /* Store the state for module exit */ |
| pkg_thermal_hp_state = ret; |
| |
| platform_thermal_package_notify = pkg_thermal_notify; |
| platform_thermal_package_rate_control = pkg_thermal_rate_control; |
| |
| /* Don't care if it fails */ |
| pkg_temp_debugfs_init(); |
| return 0; |
| |
| err: |
| kfree(zones); |
| return ret; |
| } |
| module_init(pkg_temp_thermal_init) |
| |
| static void __exit pkg_temp_thermal_exit(void) |
| { |
| platform_thermal_package_notify = NULL; |
| platform_thermal_package_rate_control = NULL; |
| |
| cpuhp_remove_state(pkg_thermal_hp_state); |
| debugfs_remove_recursive(debugfs); |
| kfree(zones); |
| } |
| module_exit(pkg_temp_thermal_exit) |
| |
| MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver"); |
| MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>"); |
| MODULE_LICENSE("GPL v2"); |