intel_pstate.c

/*
 * intel_pstate.c: Native P state management for Intel processors
 *
 * (C) Copyright 2012 Intel Corporation
 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/ktime.h>
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#include <linux/sysfs.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
#include <linux/acpi.h>
#include <linux/vmalloc.h>
#include <trace/events/power.h>

#include <asm/div64.h>
#include <asm/msr.h>
#include <asm/cpu_device_id.h>
#include <asm/cpufeature.h>

#if IS_ENABLED(CONFIG_ACPI)
#include <acpi/processor.h>
#endif

#define BYT_RATIOS		0x66a
#define BYT_VIDS		0x66b
#define BYT_TURBO_RATIOS	0x66c
#define BYT_TURBO_VIDS		0x66d

#define FRAC_BITS 8
#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
#define fp_toint(X) ((X) >> FRAC_BITS)

static inline int32_t mul_fp(int32_t x, int32_t y)
{
	return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
}

static inline int32_t div_fp(s64 x, s64 y)
{
	return div64_s64((int64_t)x << FRAC_BITS, y);
}

static inline int ceiling_fp(int32_t x)
{
	int mask, ret;

	ret = fp_toint(x);
	mask = (1 << FRAC_BITS) - 1;
	if (x & mask)
		ret += 1;
	return ret;
}

struct sample {
	int32_t core_pct_busy;
	u64 aperf;
	u64 mperf;
	u64 tsc;
	int freq;
	ktime_t time;
};

struct pstate_data {
	int	current_pstate;
	int	min_pstate;
	int	max_pstate;
	int	max_pstate_physical;
	int	scaling;
	int	turbo_pstate;
};

struct vid_data {
	int min;
	int max;
	int turbo;
	int32_t ratio;
};

struct _pid {
	int setpoint;
	int32_t integral;
	int32_t p_gain;
	int32_t i_gain;
	int32_t d_gain;
	int deadband;
	int32_t last_err;
};

struct cpudata {
	int cpu;

	struct timer_list timer;

	struct pstate_data pstate;
	struct vid_data vid;
	struct _pid pid;

	ktime_t last_sample_time;
	u64	prev_aperf;
	u64	prev_mperf;
	u64	prev_tsc;
	struct sample sample;
#if IS_ENABLED(CONFIG_ACPI)
	struct acpi_processor_performance acpi_perf_data;
#endif
};

static struct cpudata **all_cpu_data;
struct pstate_adjust_policy {
	int sample_rate_ms;
	int deadband;
	int setpoint;
	int p_gain_pct;
	int d_gain_pct;
	int i_gain_pct;
};

struct pstate_funcs {
	int (*get_max)(void);
	int (*get_max_physical)(void);
	int (*get_min)(void);
	int (*get_turbo)(void);
	int (*get_scaling)(void);
	void (*set)(struct cpudata*, int pstate);
	void (*get_vid)(struct cpudata *);
};

struct cpu_defaults {
	struct pstate_adjust_policy pid_policy;
	struct pstate_funcs funcs;
};

static struct pstate_adjust_policy pid_params;
static struct pstate_funcs pstate_funcs;
static int hwp_active;
static int no_acpi_perf;

struct perf_limits {
	int no_turbo;
	int turbo_disabled;
	int max_perf_pct;
	int min_perf_pct;
	int32_t max_perf;
	int32_t min_perf;
	int max_policy_pct;
	int max_sysfs_pct;
	int min_policy_pct;
	int min_sysfs_pct;
	int max_perf_ctl;
	int min_perf_ctl;
};

static struct perf_limits limits = {
	.no_turbo = 0,
	.turbo_disabled = 0,
	.max_perf_pct = 100,
	.max_perf = int_tofp(1),
	.min_perf_pct = 0,
	.min_perf = 0,
	.max_policy_pct = 100,
	.max_sysfs_pct = 100,
	.min_policy_pct = 0,
	.min_sysfs_pct = 0,
	.max_perf_ctl = 0,
	.min_perf_ctl = 0,
};

#if IS_ENABLED(CONFIG_ACPI)
/*
 * The max target pstate ratio is a 8 bit value in both PLATFORM_INFO MSR and
 * in TURBO_RATIO_LIMIT MSR, which pstate driver stores in max_pstate and
 * max_turbo_pstate fields. The PERF_CTL MSR contains 16 bit value for P state
 * ratio, out of it only high 8 bits are used. For example 0x1700 is setting
 * target ratio 0x17. The _PSS control value stores in a format which can be
 * directly written to PERF_CTL MSR. But in intel_pstate driver this shift
 * occurs during write to PERF_CTL (E.g. for cores core_set_pstate()).
 * This function converts the _PSS control value to intel pstate driver format
 * for comparison and assignment.
 */
static int convert_to_native_pstate_format(struct cpudata *cpu, int index)
{
	return cpu->acpi_perf_data.states[index].control >> 8;
}