~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,13 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* intel_pstate.c: Native P state management for Intel processors
3	4	*
4	5	* (C) Copyright 2012 Intel Corporation
5	6	* Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
6		- *
7		- * This program is free software; you can redistribute it and/or
8		- * modify it under the terms of the GNU General Public License
9		- * as published by the Free Software Foundation; version 2
10		- * of the License.
11	7	*/
12	8
13	9	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
..	..	@@ -28,6 +24,7 @@
28	24	#include <linux/fs.h>
29	25	#include <linux/acpi.h>
30	26	#include <linux/vmalloc.h>
	27	+#include <linux/pm_qos.h>
31	28	#include <trace/events/power.h>
32	29
33	30	#include <asm/div64.h>
..	..	@@ -39,6 +36,7 @@
39	36	#define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
40	37
41	38	#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
	39	+#define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
42	40	#define INTEL_CPUFREQ_TRANSITION_DELAY 500
43	41
44	42	#ifdef CONFIG_ACPI
..	..	@@ -49,6 +47,8 @@
49	47	#define FRAC_BITS 8
50	48	#define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
51	49	#define fp_toint(X) ((X) >> FRAC_BITS)
	50	+
	51	+#define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
52	52
53	53	#define EXT_BITS 6
54	54	#define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
..	..	@@ -173,10 +173,11 @@
173	173	/**
174	174	* struct global_params - Global parameters, mostly tunable via sysfs.
175	175	* @no_turbo: Whether or not to use turbo P-states.
176		- * @turbo_disabled: Whethet or not turbo P-states are available at all,
	176	+ * @turbo_disabled: Whether or not turbo P-states are available at all,
177	177	* based on the MSR_IA32_MISC_ENABLE value and whether or
178	178	* not the maximum reported turbo P-state is different from
179	179	* the maximum reported non-turbo one.
	180	+ * @turbo_disabled_mf: The @turbo_disabled value reflected by cpuinfo.max_freq.
180	181	* @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
181	182	* P-state capacity.
182	183	* @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
..	..	@@ -185,6 +186,7 @@
185	186	struct global_params {
186	187	bool no_turbo;
187	188	bool turbo_disabled;
	189	+ bool turbo_disabled_mf;
188	190	int max_perf_pct;
189	191	int min_perf_pct;
190	192	};
..	..	@@ -200,9 +202,7 @@
200	202	* @pstate: Stores P state limits for this CPU
201	203	* @vid: Stores VID limits for this CPU
202	204	* @last_sample_time: Last Sample time
203		- * @aperf_mperf_shift: Number of clock cycles after aperf, merf is incremented
204		- * This shift is a multiplier to mperf delta to
205		- * calculate CPU busy.
	205	+ * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
206	206	* @prev_aperf: Last APERF value read from APERF MSR
207	207	* @prev_mperf: Last MPERF value read from MPERF MSR
208	208	* @prev_tsc: Last timestamp counter (TSC) value
..	..	@@ -219,13 +219,13 @@
219	219	* @epp_policy: Last saved policy used to set EPP/EPB
220	220	* @epp_default: Power on default HWP energy performance
221	221	* preference/bias
222		- * @epp_saved: Saved EPP/EPB during system suspend or CPU offline
223		- * operation
	222	+ * @epp_cached Cached HWP energy-performance preference value
224	223	* @hwp_req_cached: Cached value of the last HWP Request MSR
225	224	* @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
226	225	* @last_io_update: Last time when IO wake flag was set
227	226	* @sched_flags: Store scheduler flags for possible cross CPU update
228	227	* @hwp_boost_min: Last HWP boosted min performance
	228	+ * @suspended: Whether or not the driver has been suspended.
229	229	*
230	230	* This structure stores per CPU instance data for all CPUs.
231	231	*/
..	..	@@ -257,12 +257,13 @@
257	257	s16 epp_powersave;
258	258	s16 epp_policy;
259	259	s16 epp_default;
260		- s16 epp_saved;
	260	+ s16 epp_cached;
261	261	u64 hwp_req_cached;
262	262	u64 hwp_cap_cached;
263	263	u64 last_io_update;
264	264	unsigned int sched_flags;
265	265	u32 hwp_boost_min;
	266	+ bool suspended;
266	267	};
267	268
268	269	static struct cpudata **all_cpu_data;
..	..	@@ -274,6 +275,7 @@
274	275	* @get_min: Callback to get minimum P state
275	276	* @get_turbo: Callback to get turbo P state
276	277	* @get_scaling: Callback to get frequency scaling factor
	278	+ * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
277	279	* @get_val: Callback to convert P state to actual MSR write value
278	280	* @get_vid: Callback to get VID data for Atom platforms
279	281	*
..	..	@@ -373,11 +375,27 @@
373	375	}
374	376	}
375	377	}
376		-#else
	378	+
	379	+static int intel_pstate_get_cppc_guranteed(int cpu)
	380	+{
	381	+ struct cppc_perf_caps cppc_perf;
	382	+ int ret;
	383	+
	384	+ ret = cppc_get_perf_caps(cpu, &cppc_perf);
	385	+ if (ret)
	386	+ return ret;
	387	+
	388	+ if (cppc_perf.guaranteed_perf)
	389	+ return cppc_perf.guaranteed_perf;
	390	+
	391	+ return cppc_perf.nominal_perf;
	392	+}
	393	+
	394	+#else /* CONFIG_ACPI_CPPC_LIB */
377	395	static void intel_pstate_set_itmt_prio(int cpu)
378	396	{
379	397	}
380		-#endif
	398	+#endif /* CONFIG_ACPI_CPPC_LIB */
381	399
382	400	static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
383	401	{
..	..	@@ -425,20 +443,6 @@
425	443	(u32) cpu->acpi_perf_data.states[i].control);
426	444	}
427	445
428		- /*
429		- * The _PSS table doesn't contain whole turbo frequency range.
430		- * This just contains +1 MHZ above the max non turbo frequency,
431		- * with control value corresponding to max turbo ratio. But
432		- * when cpufreq set policy is called, it will call with this
433		- * max frequency, which will cause a reduced performance as
434		- * this driver uses real max turbo frequency as the max
435		- * frequency. So correct this frequency in _PSS table to
436		- * correct max turbo frequency based on the turbo state.
437		- * Also need to convert to MHz as _PSS freq is in MHz.
438		- */
439		- if (!global.turbo_disabled)
440		- cpu->acpi_perf_data.states[0].core_frequency =
441		- policy->cpuinfo.max_freq / 1000;
442	446	cpu->valid_pss_table = true;
443	447	pr_debug("_PPC limits will be enforced\n");
444	448
..	..	@@ -459,7 +463,7 @@
459	463
460	464	acpi_processor_unregister_performance(policy->cpu);
461	465	}
462		-#else
	466	+#else /* CONFIG_ACPI */
463	467	static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
464	468	{
465	469	}
..	..	@@ -472,7 +476,14 @@
472	476	{
473	477	return false;
474	478	}
475		-#endif
	479	+#endif /* CONFIG_ACPI */
	480	+
	481	+#ifndef CONFIG_ACPI_CPPC_LIB
	482	+static int intel_pstate_get_cppc_guranteed(int cpu)
	483	+{
	484	+ return -ENOTSUPP;
	485	+}
	486	+#endif /* CONFIG_ACPI_CPPC_LIB */
476	487
477	488	static inline void update_turbo_state(void)
478	489	{
..	..	@@ -500,7 +511,7 @@
500	511	u64 epb;
501	512	int ret;
502	513
503		- if (!static_cpu_has(X86_FEATURE_EPB))
	514	+ if (!boot_cpu_has(X86_FEATURE_EPB))
504	515	return -ENXIO;
505	516
506	517	ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
..	..	@@ -514,7 +525,7 @@
514	525	{
515	526	s16 epp;
516	527
517		- if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
	528	+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
518	529	/*
519	530	* When hwp_req_data is 0, means that caller didn't read
520	531	* MSR_HWP_REQUEST, so need to read and get EPP.
..	..	@@ -539,7 +550,7 @@
539	550	u64 epb;
540	551	int ret;
541	552
542		- if (!static_cpu_has(X86_FEATURE_EPB))
	553	+ if (!boot_cpu_has(X86_FEATURE_EPB))
543	554	return -ENXIO;
544	555
545	556	ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
..	..	@@ -578,25 +589,28 @@
578	589	HWP_EPP_POWERSAVE
579	590	};
580	591
581		-static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data)
	592	+static int intel_pstate_get_energy_pref_index(struct cpudata cpu_data, int raw_epp)
582	593	{
583	594	s16 epp;
584	595	int index = -EINVAL;
585	596
	597	+ *raw_epp = 0;
586	598	epp = intel_pstate_get_epp(cpu_data, 0);
587	599	if (epp < 0)
588	600	return epp;
589	601
590		- if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
	602	+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
591	603	if (epp == HWP_EPP_PERFORMANCE)
592	604	return 1;
593		- if (epp <= HWP_EPP_BALANCE_PERFORMANCE)
	605	+ if (epp == HWP_EPP_BALANCE_PERFORMANCE)
594	606	return 2;
595		- if (epp <= HWP_EPP_BALANCE_POWERSAVE)
	607	+ if (epp == HWP_EPP_BALANCE_POWERSAVE)
596	608	return 3;
597		- else
	609	+ if (epp == HWP_EPP_POWERSAVE)
598	610	return 4;
599		- } else if (static_cpu_has(X86_FEATURE_EPB)) {
	611	+ *raw_epp = epp;
	612	+ return 0;
	613	+ } else if (boot_cpu_has(X86_FEATURE_EPB)) {
600	614	/*
601	615	* Range:
602	616	* 0x00-0x03 : Performance
..	..	@@ -613,8 +627,35 @@
613	627	return index;
614	628	}
615	629
	630	+static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)
	631	+{
	632	+ int ret;
	633	+
	634	+ /*
	635	+ * Use the cached HWP Request MSR value, because in the active mode the
	636	+ * register itself may be updated by intel_pstate_hwp_boost_up() or
	637	+ * intel_pstate_hwp_boost_down() at any time.
	638	+ */
	639	+ u64 value = READ_ONCE(cpu->hwp_req_cached);
	640	+
	641	+ value &= ~GENMASK_ULL(31, 24);
	642	+ value \|= (u64)epp << 24;
	643	+ /*
	644	+ * The only other updater of hwp_req_cached in the active mode,
	645	+ * intel_pstate_hwp_set(), is called under the same lock as this
	646	+ * function, so it cannot run in parallel with the update below.
	647	+ */
	648	+ WRITE_ONCE(cpu->hwp_req_cached, value);
	649	+ ret = wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
	650	+ if (!ret)
	651	+ cpu->epp_cached = epp;
	652	+
	653	+ return ret;
	654	+}
	655	+
616	656	static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
617		- int pref_index)
	657	+ int pref_index, bool use_raw,
	658	+ u32 raw_epp)
618	659	{
619	660	int epp = -EINVAL;
620	661	int ret;
..	..	@@ -622,29 +663,26 @@
622	663	if (!pref_index)
623	664	epp = cpu_data->epp_default;
624	665
625		- mutex_lock(&intel_pstate_limits_lock);
626		-
627		- if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
628		- u64 value;
629		-
630		- ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &value);
631		- if (ret)
632		- goto return_pref;
633		-
634		- value &= ~GENMASK_ULL(31, 24);
635		-
636		- if (epp == -EINVAL)
	666	+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
	667	+ if (use_raw)
	668	+ epp = raw_epp;
	669	+ else if (epp == -EINVAL)
637	670	epp = epp_values[pref_index - 1];
638	671
639		- value \|= (u64)epp << 24;
640		- ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value);
	672	+ /*
	673	+ * To avoid confusion, refuse to set EPP to any values different
	674	+ * from 0 (performance) if the current policy is "performance",
	675	+ * because those values would be overridden.
	676	+ */
	677	+ if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
	678	+ return -EBUSY;
	679	+
	680	+ ret = intel_pstate_set_epp(cpu_data, epp);
641	681	} else {
642	682	if (epp == -EINVAL)
643	683	epp = (pref_index - 1) << 2;
644	684	ret = intel_pstate_set_epb(cpu_data->cpu, epp);
645	685	}
646		-return_pref:
647		- mutex_unlock(&intel_pstate_limits_lock);
648	686
649	687	return ret;
650	688	}
..	..	@@ -665,53 +703,127 @@
665	703
666	704	cpufreq_freq_attr_ro(energy_performance_available_preferences);
667	705
	706	+static struct cpufreq_driver intel_pstate;
	707	+
668	708	static ssize_t store_energy_performance_preference(
669	709	struct cpufreq_policy policy, const char buf, size_t count)
670	710	{
671		- struct cpudata *cpu_data = all_cpu_data[policy->cpu];
	711	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
672	712	char str_preference[21];
673		- int ret;
	713	+ bool raw = false;
	714	+ ssize_t ret;
	715	+ u32 epp = 0;
674	716
675	717	ret = sscanf(buf, "%20s", str_preference);
676	718	if (ret != 1)
677	719	return -EINVAL;
678	720
679	721	ret = match_string(energy_perf_strings, -1, str_preference);
680		- if (ret < 0)
681		- return ret;
	722	+ if (ret < 0) {
	723	+ if (!boot_cpu_has(X86_FEATURE_HWP_EPP))
	724	+ return ret;
682	725
683		- intel_pstate_set_energy_pref_index(cpu_data, ret);
684		- return count;
	726	+ ret = kstrtouint(buf, 10, &epp);
	727	+ if (ret)
	728	+ return ret;
	729	+
	730	+ if (epp > 255)
	731	+ return -EINVAL;
	732	+
	733	+ raw = true;
	734	+ }
	735	+
	736	+ /*
	737	+ * This function runs with the policy R/W semaphore held, which
	738	+ * guarantees that the driver pointer will not change while it is
	739	+ * running.
	740	+ */
	741	+ if (!intel_pstate_driver)
	742	+ return -EAGAIN;
	743	+
	744	+ mutex_lock(&intel_pstate_limits_lock);
	745	+
	746	+ if (intel_pstate_driver == &intel_pstate) {
	747	+ ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp);
	748	+ } else {
	749	+ /*
	750	+ * In the passive mode the governor needs to be stopped on the
	751	+ * target CPU before the EPP update and restarted after it,
	752	+ * which is super-heavy-weight, so make sure it is worth doing
	753	+ * upfront.
	754	+ */
	755	+ if (!raw)
	756	+ epp = ret ? epp_values[ret - 1] : cpu->epp_default;
	757	+
	758	+ if (cpu->epp_cached != epp) {
	759	+ int err;
	760	+
	761	+ cpufreq_stop_governor(policy);
	762	+ ret = intel_pstate_set_epp(cpu, epp);
	763	+ err = cpufreq_start_governor(policy);
	764	+ if (!ret)
	765	+ ret = err;
	766	+ } else {
	767	+ ret = 0;
	768	+ }
	769	+ }
	770	+
	771	+ mutex_unlock(&intel_pstate_limits_lock);
	772	+
	773	+ return ret ?: count;
685	774	}
686	775
687	776	static ssize_t show_energy_performance_preference(
688	777	struct cpufreq_policy policy, char buf)
689	778	{
690	779	struct cpudata *cpu_data = all_cpu_data[policy->cpu];
691		- int preference;
	780	+ int preference, raw_epp;
692	781
693		- preference = intel_pstate_get_energy_pref_index(cpu_data);
	782	+ preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp);
694	783	if (preference < 0)
695	784	return preference;
696	785
697		- return sprintf(buf, "%s\n", energy_perf_strings[preference]);
	786	+ if (raw_epp)
	787	+ return sprintf(buf, "%d\n", raw_epp);
	788	+ else
	789	+ return sprintf(buf, "%s\n", energy_perf_strings[preference]);
698	790	}
699	791
700	792	cpufreq_freq_attr_rw(energy_performance_preference);
701	793
	794	+static ssize_t show_base_frequency(struct cpufreq_policy policy, char buf)
	795	+{
	796	+ struct cpudata *cpu;
	797	+ u64 cap;
	798	+ int ratio;
	799	+
	800	+ ratio = intel_pstate_get_cppc_guranteed(policy->cpu);
	801	+ if (ratio <= 0) {
	802	+ rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
	803	+ ratio = HWP_GUARANTEED_PERF(cap);
	804	+ }
	805	+
	806	+ cpu = all_cpu_data[policy->cpu];
	807	+
	808	+ return sprintf(buf, "%d\n", ratio * cpu->pstate.scaling);
	809	+}
	810	+
	811	+cpufreq_freq_attr_ro(base_frequency);
	812	+
702	813	static struct freq_attr *hwp_cpufreq_attrs[] = {
703	814	&energy_performance_preference,
704	815	&energy_performance_available_preferences,
	816	+ &base_frequency,
705	817	NULL,
706	818	};
707	819
708		-static void intel_pstate_get_hwp_max(unsigned int cpu, int *phy_max,
	820	+static void intel_pstate_get_hwp_max(struct cpudata cpu, int phy_max,
709	821	int *current_max)
710	822	{
711	823	u64 cap;
712	824
713		- rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
714		- WRITE_ONCE(all_cpu_data[cpu]->hwp_cap_cached, cap);
	825	+ rdmsrl_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap);
	826	+ WRITE_ONCE(cpu->hwp_cap_cached, cap);
715	827	if (global.no_turbo \|\| global.turbo_disabled)
716	828	*current_max = HWP_GUARANTEED_PERF(cap);
717	829	else
..	..	@@ -746,12 +858,6 @@
746	858
747	859	cpu_data->epp_policy = cpu_data->policy;
748	860
749		- if (cpu_data->epp_saved >= 0) {
750		- epp = cpu_data->epp_saved;
751		- cpu_data->epp_saved = -EINVAL;
752		- goto update_epp;
753		- }
754		-
755	861	if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
756	862	epp = intel_pstate_get_epp(cpu_data, value);
757	863	cpu_data->epp_powersave = epp;
..	..	@@ -778,8 +884,7 @@
778	884
779	885	epp = cpu_data->epp_powersave;
780	886	}
781		-update_epp:
782		- if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
	887	+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
783	888	value &= ~GENMASK_ULL(31, 24);
784	889	value \|= (u64)epp << 24;
785	890	} else {
..	..	@@ -790,34 +895,99 @@
790	895	wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
791	896	}
792	897
793		-static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
	898	+static void intel_pstate_hwp_offline(struct cpudata *cpu)
794	899	{
795		- struct cpudata *cpu_data = all_cpu_data[policy->cpu];
	900	+ u64 value = READ_ONCE(cpu->hwp_req_cached);
	901	+ int min_perf;
796	902
797		- if (!hwp_active)
798		- return 0;
	903	+ if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
	904	+ /*
	905	+ * In case the EPP has been set to "performance" by the
	906	+ * active mode "performance" scaling algorithm, replace that
	907	+ * temporary value with the cached EPP one.
	908	+ */
	909	+ value &= ~GENMASK_ULL(31, 24);
	910	+ value \|= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
	911	+ WRITE_ONCE(cpu->hwp_req_cached, value);
	912	+ }
799	913
800		- cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0);
	914	+ value &= ~GENMASK_ULL(31, 0);
	915	+ min_perf = HWP_LOWEST_PERF(cpu->hwp_cap_cached);
801	916
802		- return 0;
	917	+ /* Set hwp_max = hwp_min */
	918	+ value \|= HWP_MAX_PERF(min_perf);
	919	+ value \|= HWP_MIN_PERF(min_perf);
	920	+
	921	+ /* Set EPP to min */
	922	+ if (boot_cpu_has(X86_FEATURE_HWP_EPP))
	923	+ value \|= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE);
	924	+
	925	+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
	926	+}
	927	+
	928	+#define POWER_CTL_EE_ENABLE 1
	929	+#define POWER_CTL_EE_DISABLE 2
	930	+
	931	+static int power_ctl_ee_state;
	932	+
	933	+static void set_power_ctl_ee_state(bool input)
	934	+{
	935	+ u64 power_ctl;
	936	+
	937	+ mutex_lock(&intel_pstate_driver_lock);
	938	+ rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
	939	+ if (input) {
	940	+ power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE);
	941	+ power_ctl_ee_state = POWER_CTL_EE_ENABLE;
	942	+ } else {
	943	+ power_ctl \|= BIT(MSR_IA32_POWER_CTL_BIT_EE);
	944	+ power_ctl_ee_state = POWER_CTL_EE_DISABLE;
	945	+ }
	946	+ wrmsrl(MSR_IA32_POWER_CTL, power_ctl);
	947	+ mutex_unlock(&intel_pstate_driver_lock);
803	948	}
804	949
805	950	static void intel_pstate_hwp_enable(struct cpudata *cpudata);
806	951
	952	+static void intel_pstate_hwp_reenable(struct cpudata *cpu)
	953	+{
	954	+ intel_pstate_hwp_enable(cpu);
	955	+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached));
	956	+}
	957	+
	958	+static int intel_pstate_suspend(struct cpufreq_policy *policy)
	959	+{
	960	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
	961	+
	962	+ pr_debug("CPU %d suspending\n", cpu->cpu);
	963	+
	964	+ cpu->suspended = true;
	965	+
	966	+ return 0;
	967	+}
	968	+
807	969	static int intel_pstate_resume(struct cpufreq_policy *policy)
808	970	{
809		- if (!hwp_active)
810		- return 0;
	971	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
811	972
812		- mutex_lock(&intel_pstate_limits_lock);
	973	+ pr_debug("CPU %d resuming\n", cpu->cpu);
813	974
814		- if (policy->cpu == 0)
815		- intel_pstate_hwp_enable(all_cpu_data[policy->cpu]);
	975	+ /* Only restore if the system default is changed */
	976	+ if (power_ctl_ee_state == POWER_CTL_EE_ENABLE)
	977	+ set_power_ctl_ee_state(true);
	978	+ else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE)
	979	+ set_power_ctl_ee_state(false);
816	980
817		- all_cpu_data[policy->cpu]->epp_policy = 0;
818		- intel_pstate_hwp_set(policy->cpu);
	981	+ if (cpu->suspended && hwp_active) {
	982	+ mutex_lock(&intel_pstate_limits_lock);
819	983
820		- mutex_unlock(&intel_pstate_limits_lock);
	984	+ /* Re-enable HWP, because "online" has not done that. */
	985	+ intel_pstate_hwp_reenable(cpu);
	986	+
	987	+ mutex_unlock(&intel_pstate_limits_lock);
	988	+ }
	989	+
	990	+ cpu->suspended = false;
821	991
822	992	return 0;
823	993	}
..	..	@@ -828,6 +998,44 @@
828	998
829	999	for_each_possible_cpu(cpu)
830	1000	cpufreq_update_policy(cpu);
	1001	+}
	1002	+
	1003	+static void intel_pstate_update_max_freq(unsigned int cpu)
	1004	+{
	1005	+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
	1006	+ struct cpudata *cpudata;
	1007	+
	1008	+ if (!policy)
	1009	+ return;
	1010	+
	1011	+ cpudata = all_cpu_data[cpu];
	1012	+ policy->cpuinfo.max_freq = global.turbo_disabled_mf ?
	1013	+ cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;
	1014	+
	1015	+ refresh_frequency_limits(policy);
	1016	+
	1017	+ cpufreq_cpu_release(policy);
	1018	+}
	1019	+
	1020	+static void intel_pstate_update_limits(unsigned int cpu)
	1021	+{
	1022	+ mutex_lock(&intel_pstate_driver_lock);
	1023	+
	1024	+ update_turbo_state();
	1025	+ /*
	1026	+ * If turbo has been turned on or off globally, policy limits for
	1027	+ * all CPUs need to be updated to reflect that.
	1028	+ */
	1029	+ if (global.turbo_disabled_mf != global.turbo_disabled) {
	1030	+ global.turbo_disabled_mf = global.turbo_disabled;
	1031	+ arch_set_max_freq_ratio(global.turbo_disabled);
	1032	+ for_each_possible_cpu(cpu)
	1033	+ intel_pstate_update_max_freq(cpu);
	1034	+ } else {
	1035	+ cpufreq_update_policy(cpu);
	1036	+ }
	1037	+
	1038	+ mutex_unlock(&intel_pstate_driver_lock);
831	1039	}
832	1040
833	1041	/************************ sysfs begin **********************/
..	..	@@ -983,6 +1191,45 @@
983	1191	return count;
984	1192	}
985	1193
	1194	+static void update_qos_request(enum freq_qos_req_type type)
	1195	+{
	1196	+ int max_state, turbo_max, freq, i, perf_pct;
	1197	+ struct freq_qos_request *req;
	1198	+ struct cpufreq_policy *policy;
	1199	+
	1200	+ for_each_possible_cpu(i) {
	1201	+ struct cpudata *cpu = all_cpu_data[i];
	1202	+
	1203	+ policy = cpufreq_cpu_get(i);
	1204	+ if (!policy)
	1205	+ continue;
	1206	+
	1207	+ req = policy->driver_data;
	1208	+ cpufreq_cpu_put(policy);
	1209	+
	1210	+ if (!req)
	1211	+ continue;
	1212	+
	1213	+ if (hwp_active)
	1214	+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
	1215	+ else
	1216	+ turbo_max = cpu->pstate.turbo_pstate;
	1217	+
	1218	+ if (type == FREQ_QOS_MIN) {
	1219	+ perf_pct = global.min_perf_pct;
	1220	+ } else {
	1221	+ req++;
	1222	+ perf_pct = global.max_perf_pct;
	1223	+ }
	1224	+
	1225	+ freq = DIV_ROUND_UP(turbo_max * perf_pct, 100);
	1226	+ freq *= cpu->pstate.scaling;
	1227	+
	1228	+ if (freq_qos_update_request(req, freq) < 0)
	1229	+ pr_warn("Failed to update freq constraint: CPU%d\n", i);
	1230	+ }
	1231	+}
	1232	+
986	1233	static ssize_t store_max_perf_pct(struct kobject a, struct kobj_attribute b,
987	1234	const char *buf, size_t count)
988	1235	{
..	..	@@ -1006,7 +1253,10 @@
1006	1253
1007	1254	mutex_unlock(&intel_pstate_limits_lock);
1008	1255
1009		- intel_pstate_update_policies();
	1256	+ if (intel_pstate_driver == &intel_pstate)
	1257	+ intel_pstate_update_policies();
	1258	+ else
	1259	+ update_qos_request(FREQ_QOS_MAX);
1010	1260
1011	1261	mutex_unlock(&intel_pstate_driver_lock);
1012	1262
..	..	@@ -1037,7 +1287,10 @@
1037	1287
1038	1288	mutex_unlock(&intel_pstate_limits_lock);
1039	1289
1040		- intel_pstate_update_policies();
	1290	+ if (intel_pstate_driver == &intel_pstate)
	1291	+ intel_pstate_update_policies();
	1292	+ else
	1293	+ update_qos_request(FREQ_QOS_MIN);
1041	1294
1042	1295	mutex_unlock(&intel_pstate_driver_lock);
1043	1296
..	..	@@ -1069,6 +1322,32 @@
1069	1322	return count;
1070	1323	}
1071	1324
	1325	+static ssize_t show_energy_efficiency(struct kobject kobj, struct kobj_attribute attr,
	1326	+ char *buf)
	1327	+{
	1328	+ u64 power_ctl;
	1329	+ int enable;
	1330	+
	1331	+ rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
	1332	+ enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE));
	1333	+ return sprintf(buf, "%d\n", !enable);
	1334	+}
	1335	+
	1336	+static ssize_t store_energy_efficiency(struct kobject a, struct kobj_attribute b,
	1337	+ const char *buf, size_t count)
	1338	+{
	1339	+ bool input;
	1340	+ int ret;
	1341	+
	1342	+ ret = kstrtobool(buf, &input);
	1343	+ if (ret)
	1344	+ return ret;
	1345	+
	1346	+ set_power_ctl_ee_state(input);
	1347	+
	1348	+ return count;
	1349	+}
	1350	+
1072	1351	show_one(max_perf_pct, max_perf_pct);
1073	1352	show_one(min_perf_pct, min_perf_pct);
1074	1353
..	..	@@ -1079,6 +1358,7 @@
1079	1358	define_one_global_ro(turbo_pct);
1080	1359	define_one_global_ro(num_pstates);
1081	1360	define_one_global_rw(hwp_dynamic_boost);
	1361	+define_one_global_rw(energy_efficiency);
1082	1362
1083	1363	static struct attribute *intel_pstate_attributes[] = {
1084	1364	&status.attr,
..	..	@@ -1092,9 +1372,12 @@
1092	1372	.attrs = intel_pstate_attributes,
1093	1373	};
1094	1374
	1375	+static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[];
	1376	+
	1377	+static struct kobject *intel_pstate_kobject;
	1378	+
1095	1379	static void __init intel_pstate_sysfs_expose_params(void)
1096	1380	{
1097		- struct kobject *intel_pstate_kobject;
1098	1381	int rc;
1099	1382
1100	1383	intel_pstate_kobject = kobject_create_and_add("intel_pstate",
..	..	@@ -1119,43 +1402,60 @@
1119	1402	rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
1120	1403	WARN_ON(rc);
1121	1404
1122		- if (hwp_active) {
1123		- rc = sysfs_create_file(intel_pstate_kobject,
1124		- &hwp_dynamic_boost.attr);
	1405	+ if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) {
	1406	+ rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr);
1125	1407	WARN_ON(rc);
1126	1408	}
1127	1409	}
	1410	+
	1411	+static void __init intel_pstate_sysfs_remove(void)
	1412	+{
	1413	+ if (!intel_pstate_kobject)
	1414	+ return;
	1415	+
	1416	+ sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
	1417	+
	1418	+ if (!per_cpu_limits) {
	1419	+ sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
	1420	+ sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
	1421	+
	1422	+ if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids))
	1423	+ sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr);
	1424	+ }
	1425	+
	1426	+ kobject_put(intel_pstate_kobject);
	1427	+}
	1428	+
	1429	+static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
	1430	+{
	1431	+ int rc;
	1432	+
	1433	+ if (!hwp_active)
	1434	+ return;
	1435	+
	1436	+ rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
	1437	+ WARN_ON_ONCE(rc);
	1438	+}
	1439	+
	1440	+static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
	1441	+{
	1442	+ if (!hwp_active)
	1443	+ return;
	1444	+
	1445	+ sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
	1446	+}
	1447	+
1128	1448	/************************ sysfs end **********************/
1129	1449
1130	1450	static void intel_pstate_hwp_enable(struct cpudata *cpudata)
1131	1451	{
1132	1452	/* First disable HWP notification interrupt as we don't process them */
1133		- if (static_cpu_has(X86_FEATURE_HWP_NOTIFY))
	1453	+ if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
1134	1454	wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
1135	1455
1136	1456	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
1137		- cpudata->epp_policy = 0;
1138	1457	if (cpudata->epp_default == -EINVAL)
1139	1458	cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
1140		-}
1141		-
1142		-#define MSR_IA32_POWER_CTL_BIT_EE 19
1143		-
1144		-/* Disable energy efficiency optimization */
1145		-static void intel_pstate_disable_ee(int cpu)
1146		-{
1147		- u64 power_ctl;
1148		- int ret;
1149		-
1150		- ret = rdmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, &power_ctl);
1151		- if (ret)
1152		- return;
1153		-
1154		- if (!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE))) {
1155		- pr_info("Disabling energy efficiency optimization\n");
1156		- power_ctl \|= BIT(MSR_IA32_POWER_CTL_BIT_EE);
1157		- wrmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, power_ctl);
1158		- }
1159	1459	}
1160	1460
1161	1461	static int atom_get_min_pstate(void)
..	..	@@ -1383,12 +1683,6 @@
1383	1683	return ret;
1384	1684	}
1385	1685
1386		-static int intel_pstate_get_base_pstate(struct cpudata *cpu)
1387		-{
1388		- return global.no_turbo \|\| global.turbo_disabled ?
1389		- cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1390		-}
1391		-
1392	1686	static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
1393	1687	{
1394	1688	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
..	..	@@ -1409,11 +1703,9 @@
1409	1703
1410	1704	static void intel_pstate_max_within_limits(struct cpudata *cpu)
1411	1705	{
1412		- int pstate;
	1706	+ int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
1413	1707
1414	1708	update_turbo_state();
1415		- pstate = intel_pstate_get_base_pstate(cpu);
1416		- pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
1417	1709	intel_pstate_set_pstate(cpu, pstate);
1418	1710	}
1419	1711
..	..	@@ -1427,7 +1719,7 @@
1427	1719	if (hwp_active && !hwp_mode_bdw) {
1428	1720	unsigned int phy_max, current_max;
1429	1721
1430		- intel_pstate_get_hwp_max(cpu->cpu, &phy_max, &current_max);
	1722	+ intel_pstate_get_hwp_max(cpu, &phy_max, &current_max);
1431	1723	cpu->pstate.turbo_freq = phy_max * cpu->pstate.scaling;
1432	1724	cpu->pstate.turbo_pstate = phy_max;
1433	1725	cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(READ_ONCE(cpu->hwp_cap_cached));
..	..	@@ -1619,17 +1911,14 @@
1619	1911	static inline int32_t get_target_pstate(struct cpudata *cpu)
1620	1912	{
1621	1913	struct sample *sample = &cpu->sample;
1622		- int32_t busy_frac, boost;
	1914	+ int32_t busy_frac;
1623	1915	int target, avg_pstate;
1624	1916
1625	1917	busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift,
1626	1918	sample->tsc);
1627	1919
1628		- boost = cpu->iowait_boost;
1629		- cpu->iowait_boost >>= 1;
1630		-
1631		- if (busy_frac < boost)
1632		- busy_frac = boost;
	1920	+ if (busy_frac < cpu->iowait_boost)
	1921	+ busy_frac = cpu->iowait_boost;
1633	1922
1634	1923	sample->busy_scaled = busy_frac * 100;
1635	1924
..	..	@@ -1656,11 +1945,9 @@
1656	1945
1657	1946	static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
1658	1947	{
1659		- int max_pstate = intel_pstate_get_base_pstate(cpu);
1660		- int min_pstate;
	1948	+ int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
	1949	+ int max_pstate = max(min_pstate, cpu->max_perf_ratio);
1661	1950
1662		- min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
1663		- max_pstate = max(min_pstate, cpu->max_perf_ratio);
1664	1951	return clamp_t(int, pstate, min_pstate, max_pstate);
1665	1952	}
1666	1953
..	..	@@ -1708,29 +1995,30 @@
1708	1995	if (smp_processor_id() != cpu->cpu)
1709	1996	return;
1710	1997
	1998	+ delta_ns = time - cpu->last_update;
1711	1999	if (flags & SCHED_CPUFREQ_IOWAIT) {
1712		- cpu->iowait_boost = int_tofp(1);
1713		- cpu->last_update = time;
1714		- /*
1715		- * The last time the busy was 100% so P-state was max anyway
1716		- * so avoid overhead of computation.
1717		- */
1718		- if (fp_toint(cpu->sample.busy_scaled) == 100)
1719		- return;
1720		-
1721		- goto set_pstate;
	2000	+ /* Start over if the CPU may have been idle. */
	2001	+ if (delta_ns > TICK_NSEC) {
	2002	+ cpu->iowait_boost = ONE_EIGHTH_FP;
	2003	+ } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) {
	2004	+ cpu->iowait_boost <<= 1;
	2005	+ if (cpu->iowait_boost > int_tofp(1))
	2006	+ cpu->iowait_boost = int_tofp(1);
	2007	+ } else {
	2008	+ cpu->iowait_boost = ONE_EIGHTH_FP;
	2009	+ }
1722	2010	} else if (cpu->iowait_boost) {
1723	2011	/* Clear iowait_boost if the CPU may have been idle. */
1724		- delta_ns = time - cpu->last_update;
1725	2012	if (delta_ns > TICK_NSEC)
1726	2013	cpu->iowait_boost = 0;
	2014	+ else
	2015	+ cpu->iowait_boost >>= 1;
1727	2016	}
1728	2017	cpu->last_update = time;
1729	2018	delta_ns = time - cpu->sample.time;
1730	2019	if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
1731	2020	return;
1732	2021
1733		-set_pstate:
1734	2022	if (intel_pstate_sample(cpu, time))
1735	2023	intel_pstate_adjust_pstate(cpu);
1736	2024	}
..	..	@@ -1774,51 +2062,51 @@
1774	2062	.get_val = core_get_val,
1775	2063	};
1776	2064
1777		-#define ICPU(model, policy) \
1778		- { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1779		- (unsigned long)&policy }
	2065	+#define X86_MATCH(model, policy) \
	2066	+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
	2067	+ X86_FEATURE_APERFMPERF, &policy)
1780	2068
1781	2069	static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1782		- ICPU(INTEL_FAM6_SANDYBRIDGE, core_funcs),
1783		- ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_funcs),
1784		- ICPU(INTEL_FAM6_ATOM_SILVERMONT, silvermont_funcs),
1785		- ICPU(INTEL_FAM6_IVYBRIDGE, core_funcs),
1786		- ICPU(INTEL_FAM6_HASWELL_CORE, core_funcs),
1787		- ICPU(INTEL_FAM6_BROADWELL_CORE, core_funcs),
1788		- ICPU(INTEL_FAM6_IVYBRIDGE_X, core_funcs),
1789		- ICPU(INTEL_FAM6_HASWELL_X, core_funcs),
1790		- ICPU(INTEL_FAM6_HASWELL_ULT, core_funcs),
1791		- ICPU(INTEL_FAM6_HASWELL_GT3E, core_funcs),
1792		- ICPU(INTEL_FAM6_BROADWELL_GT3E, core_funcs),
1793		- ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_funcs),
1794		- ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_funcs),
1795		- ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1796		- ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
1797		- ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1798		- ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_funcs),
1799		- ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_funcs),
1800		- ICPU(INTEL_FAM6_ATOM_GOLDMONT, core_funcs),
1801		- ICPU(INTEL_FAM6_ATOM_GOLDMONT_PLUS, core_funcs),
1802		- ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
	2070	+ X86_MATCH(SANDYBRIDGE, core_funcs),
	2071	+ X86_MATCH(SANDYBRIDGE_X, core_funcs),
	2072	+ X86_MATCH(ATOM_SILVERMONT, silvermont_funcs),
	2073	+ X86_MATCH(IVYBRIDGE, core_funcs),
	2074	+ X86_MATCH(HASWELL, core_funcs),
	2075	+ X86_MATCH(BROADWELL, core_funcs),
	2076	+ X86_MATCH(IVYBRIDGE_X, core_funcs),
	2077	+ X86_MATCH(HASWELL_X, core_funcs),
	2078	+ X86_MATCH(HASWELL_L, core_funcs),
	2079	+ X86_MATCH(HASWELL_G, core_funcs),
	2080	+ X86_MATCH(BROADWELL_G, core_funcs),
	2081	+ X86_MATCH(ATOM_AIRMONT, airmont_funcs),
	2082	+ X86_MATCH(SKYLAKE_L, core_funcs),
	2083	+ X86_MATCH(BROADWELL_X, core_funcs),
	2084	+ X86_MATCH(SKYLAKE, core_funcs),
	2085	+ X86_MATCH(BROADWELL_D, core_funcs),
	2086	+ X86_MATCH(XEON_PHI_KNL, knl_funcs),
	2087	+ X86_MATCH(XEON_PHI_KNM, knl_funcs),
	2088	+ X86_MATCH(ATOM_GOLDMONT, core_funcs),
	2089	+ X86_MATCH(ATOM_GOLDMONT_PLUS, core_funcs),
	2090	+ X86_MATCH(SKYLAKE_X, core_funcs),
1803	2091	{}
1804	2092	};
1805	2093	MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1806	2094
1807	2095	static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1808		- ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_funcs),
1809		- ICPU(INTEL_FAM6_BROADWELL_X, core_funcs),
1810		- ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
	2096	+ X86_MATCH(BROADWELL_D, core_funcs),
	2097	+ X86_MATCH(BROADWELL_X, core_funcs),
	2098	+ X86_MATCH(SKYLAKE_X, core_funcs),
1811	2099	{}
1812	2100	};
1813	2101
1814	2102	static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
1815		- ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_funcs),
	2103	+ X86_MATCH(KABYLAKE, core_funcs),
1816	2104	{}
1817	2105	};
1818	2106
1819	2107	static const struct x86_cpu_id intel_pstate_hwp_boost_ids[] = {
1820		- ICPU(INTEL_FAM6_SKYLAKE_X, core_funcs),
1821		- ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_funcs),
	2108	+ X86_MATCH(SKYLAKE_X, core_funcs),
	2109	+ X86_MATCH(SKYLAKE, core_funcs),
1822	2110	{}
1823	2111	};
1824	2112
..	..	@@ -1835,28 +2123,30 @@
1835	2123
1836	2124	all_cpu_data[cpunum] = cpu;
1837	2125
	2126	+ cpu->cpu = cpunum;
	2127	+
1838	2128	cpu->epp_default = -EINVAL;
1839		- cpu->epp_powersave = -EINVAL;
1840		- cpu->epp_saved = -EINVAL;
	2129	+
	2130	+ if (hwp_active) {
	2131	+ const struct x86_cpu_id *id;
	2132	+
	2133	+ intel_pstate_hwp_enable(cpu);
	2134	+
	2135	+ id = x86_match_cpu(intel_pstate_hwp_boost_ids);
	2136	+ if (id && intel_pstate_acpi_pm_profile_server())
	2137	+ hwp_boost = true;
	2138	+ }
	2139	+ } else if (hwp_active) {
	2140	+ /*
	2141	+ * Re-enable HWP in case this happens after a resume from ACPI
	2142	+ * S3 if the CPU was offline during the whole system/resume
	2143	+ * cycle.
	2144	+ */
	2145	+ intel_pstate_hwp_reenable(cpu);
1841	2146	}
1842	2147
1843		- cpu = all_cpu_data[cpunum];
1844		-
1845		- cpu->cpu = cpunum;
1846		-
1847		- if (hwp_active) {
1848		- const struct x86_cpu_id *id;
1849		-
1850		- id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
1851		- if (id)
1852		- intel_pstate_disable_ee(cpunum);
1853		-
1854		- intel_pstate_hwp_enable(cpu);
1855		-
1856		- id = x86_match_cpu(intel_pstate_hwp_boost_ids);
1857		- if (id && intel_pstate_acpi_pm_profile_server())
1858		- hwp_boost = true;
1859		- }
	2148	+ cpu->epp_powersave = -EINVAL;
	2149	+ cpu->epp_policy = 0;
1860	2150
1861	2151	intel_pstate_get_cpu_pstates(cpu);
1862	2152
..	..	@@ -1893,7 +2183,7 @@
1893	2183
1894	2184	cpufreq_remove_update_util_hook(cpu);
1895	2185	cpu_data->update_util_set = false;
1896		- synchronize_sched();
	2186	+ synchronize_rcu();
1897	2187	}
1898	2188
1899	2189	static int intel_pstate_get_max_freq(struct cpudata *cpu)
..	..	@@ -1902,12 +2192,13 @@
1902	2192	cpu->pstate.max_freq : cpu->pstate.turbo_freq;
1903	2193	}
1904	2194
1905		-static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
1906		- struct cpudata *cpu)
	2195	+static void intel_pstate_update_perf_limits(struct cpudata *cpu,
	2196	+ unsigned int policy_min,
	2197	+ unsigned int policy_max)
1907	2198	{
1908		- int max_freq = intel_pstate_get_max_freq(cpu);
1909	2199	int32_t max_policy_perf, min_policy_perf;
1910	2200	int max_state, turbo_max;
	2201	+ int max_freq;
1911	2202
1912	2203	/*
1913	2204	* HWP needs some special consideration, because on BDX the
..	..	@@ -1915,24 +2206,25 @@
1915	2206	* rather than pure ratios.
1916	2207	*/
1917	2208	if (hwp_active) {
1918		- intel_pstate_get_hwp_max(cpu->cpu, &turbo_max, &max_state);
	2209	+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
1919	2210	} else {
1920		- max_state = intel_pstate_get_base_pstate(cpu);
	2211	+ max_state = global.no_turbo \|\| global.turbo_disabled ?
	2212	+ cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1921	2213	turbo_max = cpu->pstate.turbo_pstate;
1922	2214	}
	2215	+ max_freq = max_state * cpu->pstate.scaling;
1923	2216
1924		- max_policy_perf = max_state * policy->max / max_freq;
1925		- if (policy->max == policy->min) {
	2217	+ max_policy_perf = max_state * policy_max / max_freq;
	2218	+ if (policy_max == policy_min) {
1926	2219	min_policy_perf = max_policy_perf;
1927	2220	} else {
1928		- min_policy_perf = max_state * policy->min / max_freq;
	2221	+ min_policy_perf = max_state * policy_min / max_freq;
1929	2222	min_policy_perf = clamp_t(int32_t, min_policy_perf,
1930	2223	0, max_policy_perf);
1931	2224	}
1932	2225
1933	2226	pr_debug("cpu:%d max_state %d min_policy_perf:%d max_policy_perf:%d\n",
1934		- policy->cpu, max_state,
1935		- min_policy_perf, max_policy_perf);
	2227	+ cpu->cpu, max_state, min_policy_perf, max_policy_perf);
1936	2228
1937	2229	/* Normalize user input to [min_perf, max_perf] */
1938	2230	if (per_cpu_limits) {
..	..	@@ -1946,7 +2238,7 @@
1946	2238	global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
1947	2239	global_min = clamp_t(int32_t, global_min, 0, global_max);
1948	2240
1949		- pr_debug("cpu:%d global_min:%d global_max:%d\n", policy->cpu,
	2241	+ pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu,
1950	2242	global_min, global_max);
1951	2243
1952	2244	cpu->min_perf_ratio = max(min_policy_perf, global_min);
..	..	@@ -1959,7 +2251,7 @@
1959	2251	cpu->max_perf_ratio);
1960	2252
1961	2253	}
1962		- pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", policy->cpu,
	2254	+ pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu,
1963	2255	cpu->max_perf_ratio,
1964	2256	cpu->min_perf_ratio);
1965	2257	}
..	..	@@ -1979,7 +2271,7 @@
1979	2271
1980	2272	mutex_lock(&intel_pstate_limits_lock);
1981	2273
1982		- intel_pstate_update_perf_limits(policy, cpu);
	2274	+ intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
1983	2275
1984	2276	if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
1985	2277	/*
..	..	@@ -2008,8 +2300,8 @@
2008	2300	return 0;
2009	2301	}
2010	2302
2011		-static void intel_pstate_adjust_policy_max(struct cpufreq_policy *policy,
2012		- struct cpudata *cpu)
	2303	+static void intel_pstate_adjust_policy_max(struct cpudata *cpu,
	2304	+ struct cpufreq_policy_data *policy)
2013	2305	{
2014	2306	if (!hwp_active &&
2015	2307	cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
..	..	@@ -2020,42 +2312,87 @@
2020	2312	}
2021	2313	}
2022	2314
2023		-static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
	2315	+static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,
	2316	+ struct cpufreq_policy_data *policy)
2024	2317	{
2025		- struct cpudata *cpu = all_cpu_data[policy->cpu];
	2318	+ int max_freq;
2026	2319
2027	2320	update_turbo_state();
2028		- cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2029		- intel_pstate_get_max_freq(cpu));
	2321	+ if (hwp_active) {
	2322	+ int max_state, turbo_max;
2030	2323
2031		- if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
2032		- policy->policy != CPUFREQ_POLICY_PERFORMANCE)
2033		- return -EINVAL;
	2324	+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
	2325	+ max_freq = max_state * cpu->pstate.scaling;
	2326	+ } else {
	2327	+ max_freq = intel_pstate_get_max_freq(cpu);
	2328	+ }
	2329	+ cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq);
2034	2330
2035		- intel_pstate_adjust_policy_max(policy, cpu);
	2331	+ intel_pstate_adjust_policy_max(cpu, policy);
	2332	+}
	2333	+
	2334	+static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
	2335	+{
	2336	+ intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy);
2036	2337
2037	2338	return 0;
2038	2339	}
2039	2340
2040		-static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy)
	2341	+static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
2041	2342	{
2042		- intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]);
	2343	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
	2344	+
	2345	+ pr_debug("CPU %d going offline\n", cpu->cpu);
	2346	+
	2347	+ if (cpu->suspended)
	2348	+ return 0;
	2349	+
	2350	+ /*
	2351	+ * If the CPU is an SMT thread and it goes offline with the performance
	2352	+ * settings different from the minimum, it will prevent its sibling
	2353	+ * from getting to lower performance levels, so force the minimum
	2354	+ * performance on CPU offline to prevent that from happening.
	2355	+ */
	2356	+ if (hwp_active)
	2357	+ intel_pstate_hwp_offline(cpu);
	2358	+ else
	2359	+ intel_pstate_set_min_pstate(cpu);
	2360	+
	2361	+ intel_pstate_exit_perf_limits(policy);
	2362	+
	2363	+ return 0;
	2364	+}
	2365	+
	2366	+static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
	2367	+{
	2368	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
	2369	+
	2370	+ pr_debug("CPU %d going online\n", cpu->cpu);
	2371	+
	2372	+ intel_pstate_init_acpi_perf_limits(policy);
	2373	+
	2374	+ if (hwp_active) {
	2375	+ /*
	2376	+ * Re-enable HWP and clear the "suspended" flag to let "resume"
	2377	+ * know that it need not do that.
	2378	+ */
	2379	+ intel_pstate_hwp_reenable(cpu);
	2380	+ cpu->suspended = false;
	2381	+ }
	2382	+
	2383	+ return 0;
2043	2384	}
2044	2385
2045	2386	static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
2046	2387	{
2047		- pr_debug("CPU %d exiting\n", policy->cpu);
	2388	+ pr_debug("CPU %d stopping\n", policy->cpu);
2048	2389
2049	2390	intel_pstate_clear_update_util_hook(policy->cpu);
2050		- if (hwp_active)
2051		- intel_pstate_hwp_save_state(policy);
2052		- else
2053		- intel_cpufreq_stop_cpu(policy);
2054	2391	}
2055	2392
2056	2393	static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
2057	2394	{
2058		- intel_pstate_exit_perf_limits(policy);
	2395	+ pr_debug("CPU %d exiting\n", policy->cpu);
2059	2396
2060	2397	policy->fast_switch_possible = false;
2061	2398
..	..	@@ -2082,6 +2419,7 @@
2082	2419	/* cpuinfo and default policy values */
2083	2420	policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
2084	2421	update_turbo_state();
	2422	+ global.turbo_disabled_mf = global.turbo_disabled;
2085	2423	policy->cpuinfo.max_freq = global.turbo_disabled ?
2086	2424	cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
2087	2425	policy->cpuinfo.max_freq *= cpu->pstate.scaling;
..	..	@@ -2109,10 +2447,17 @@
2109	2447	if (ret)
2110	2448	return ret;
2111	2449
2112		- if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE))
2113		- policy->policy = CPUFREQ_POLICY_PERFORMANCE;
2114		- else
2115		- policy->policy = CPUFREQ_POLICY_POWERSAVE;
	2450	+ /*
	2451	+ * Set the policy to powersave to provide a valid fallback value in case
	2452	+ * the default cpufreq governor is neither powersave nor performance.
	2453	+ */
	2454	+ policy->policy = CPUFREQ_POLICY_POWERSAVE;
	2455	+
	2456	+ if (hwp_active) {
	2457	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
	2458	+
	2459	+ cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
	2460	+ }
2116	2461
2117	2462	return 0;
2118	2463	}
..	..	@@ -2121,25 +2466,23 @@
2121	2466	.flags = CPUFREQ_CONST_LOOPS,
2122	2467	.verify = intel_pstate_verify_policy,
2123	2468	.setpolicy = intel_pstate_set_policy,
2124		- .suspend = intel_pstate_hwp_save_state,
	2469	+ .suspend = intel_pstate_suspend,
2125	2470	.resume = intel_pstate_resume,
2126	2471	.init = intel_pstate_cpu_init,
2127	2472	.exit = intel_pstate_cpu_exit,
2128	2473	.stop_cpu = intel_pstate_stop_cpu,
	2474	+ .offline = intel_pstate_cpu_offline,
	2475	+ .online = intel_pstate_cpu_online,
	2476	+ .update_limits = intel_pstate_update_limits,
2129	2477	.name = "intel_pstate",
2130	2478	};
2131	2479
2132		-static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
	2480	+static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy)
2133	2481	{
2134	2482	struct cpudata *cpu = all_cpu_data[policy->cpu];
2135	2483
2136		- update_turbo_state();
2137		- cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq,
2138		- intel_pstate_get_max_freq(cpu));
2139		-
2140		- intel_pstate_adjust_policy_max(policy, cpu);
2141		-
2142		- intel_pstate_update_perf_limits(policy, cpu);
	2484	+ intel_pstate_verify_cpu_policy(cpu, policy);
	2485	+ intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
2143	2486
2144	2487	return 0;
2145	2488	}
..	..	@@ -2182,13 +2525,71 @@
2182	2525	fp_toint(cpu->iowait_boost * 100));
2183	2526	}
2184	2527
	2528	+static void intel_cpufreq_adjust_hwp(struct cpudata *cpu, u32 target_pstate,
	2529	+ bool strict, bool fast_switch)
	2530	+{
	2531	+ u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev;
	2532	+
	2533	+ value &= ~HWP_MIN_PERF(~0L);
	2534	+ value \|= HWP_MIN_PERF(target_pstate);
	2535	+
	2536	+ /*
	2537	+ * The entire MSR needs to be updated in order to update the HWP min
	2538	+ * field in it, so opportunistically update the max too if needed.
	2539	+ */
	2540	+ value &= ~HWP_MAX_PERF(~0L);
	2541	+ value \|= HWP_MAX_PERF(strict ? target_pstate : cpu->max_perf_ratio);
	2542	+
	2543	+ if (value == prev)
	2544	+ return;
	2545	+
	2546	+ WRITE_ONCE(cpu->hwp_req_cached, value);
	2547	+ if (fast_switch)
	2548	+ wrmsrl(MSR_HWP_REQUEST, value);
	2549	+ else
	2550	+ wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
	2551	+}
	2552	+
	2553	+static void intel_cpufreq_adjust_perf_ctl(struct cpudata *cpu,
	2554	+ u32 target_pstate, bool fast_switch)
	2555	+{
	2556	+ if (fast_switch)
	2557	+ wrmsrl(MSR_IA32_PERF_CTL,
	2558	+ pstate_funcs.get_val(cpu, target_pstate));
	2559	+ else
	2560	+ wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
	2561	+ pstate_funcs.get_val(cpu, target_pstate));
	2562	+}
	2563	+
	2564	+static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy,
	2565	+ int target_pstate, bool fast_switch)
	2566	+{
	2567	+ struct cpudata *cpu = all_cpu_data[policy->cpu];
	2568	+ int old_pstate = cpu->pstate.current_pstate;
	2569	+
	2570	+ target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
	2571	+ if (hwp_active) {
	2572	+ intel_cpufreq_adjust_hwp(cpu, target_pstate,
	2573	+ policy->strict_target, fast_switch);
	2574	+ cpu->pstate.current_pstate = target_pstate;
	2575	+ } else if (target_pstate != old_pstate) {
	2576	+ intel_cpufreq_adjust_perf_ctl(cpu, target_pstate, fast_switch);
	2577	+ cpu->pstate.current_pstate = target_pstate;
	2578	+ }
	2579	+
	2580	+ intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH :
	2581	+ INTEL_PSTATE_TRACE_TARGET, old_pstate);
	2582	+
	2583	+ return target_pstate;
	2584	+}
	2585	+
2185	2586	static int intel_cpufreq_target(struct cpufreq_policy *policy,
2186	2587	unsigned int target_freq,
2187	2588	unsigned int relation)
2188	2589	{
2189	2590	struct cpudata *cpu = all_cpu_data[policy->cpu];
2190	2591	struct cpufreq_freqs freqs;
2191		- int target_pstate, old_pstate;
	2592	+ int target_pstate;
2192	2593
2193	2594	update_turbo_state();
2194	2595
..	..	@@ -2196,6 +2597,7 @@
2196	2597	freqs.new = target_freq;
2197	2598
2198	2599	cpufreq_freq_transition_begin(policy, &freqs);
	2600	+
2199	2601	switch (relation) {
2200	2602	case CPUFREQ_RELATION_L:
2201	2603	target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
..	..	@@ -2207,15 +2609,11 @@
2207	2609	target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
2208	2610	break;
2209	2611	}
2210		- target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2211		- old_pstate = cpu->pstate.current_pstate;
2212		- if (target_pstate != cpu->pstate.current_pstate) {
2213		- cpu->pstate.current_pstate = target_pstate;
2214		- wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
2215		- pstate_funcs.get_val(cpu, target_pstate));
2216		- }
	2612	+
	2613	+ target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false);
	2614	+
2217	2615	freqs.new = target_pstate * cpu->pstate.scaling;
2218		- intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_TARGET, old_pstate);
	2616	+
2219	2617	cpufreq_freq_transition_end(policy, &freqs, false);
2220	2618
2221	2619	return 0;
..	..	@@ -2225,31 +2623,101 @@
2225	2623	unsigned int target_freq)
2226	2624	{
2227	2625	struct cpudata *cpu = all_cpu_data[policy->cpu];
2228		- int target_pstate, old_pstate;
	2626	+ int target_pstate;
2229	2627
2230	2628	update_turbo_state();
2231	2629
2232	2630	target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
2233		- target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2234		- old_pstate = cpu->pstate.current_pstate;
2235		- intel_pstate_update_pstate(cpu, target_pstate);
2236		- intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
	2631	+
	2632	+ target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true);
	2633	+
2237	2634	return target_pstate * cpu->pstate.scaling;
2238	2635	}
2239	2636
2240	2637	static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
2241	2638	{
2242		- int ret = __intel_pstate_cpu_init(policy);
	2639	+ int max_state, turbo_max, min_freq, max_freq, ret;
	2640	+ struct freq_qos_request *req;
	2641	+ struct cpudata *cpu;
	2642	+ struct device *dev;
2243	2643
	2644	+ dev = get_cpu_device(policy->cpu);
	2645	+ if (!dev)
	2646	+ return -ENODEV;
	2647	+
	2648	+ ret = __intel_pstate_cpu_init(policy);
2244	2649	if (ret)
2245	2650	return ret;
2246	2651
2247	2652	policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
2248		- policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
2249	2653	/* This reflects the intel_pstate_get_cpu_pstates() setting. */
2250	2654	policy->cur = policy->cpuinfo.min_freq;
2251	2655
	2656	+ req = kcalloc(2, sizeof(*req), GFP_KERNEL);
	2657	+ if (!req) {
	2658	+ ret = -ENOMEM;
	2659	+ goto pstate_exit;
	2660	+ }
	2661	+
	2662	+ cpu = all_cpu_data[policy->cpu];
	2663	+
	2664	+ if (hwp_active) {
	2665	+ u64 value;
	2666	+
	2667	+ intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
	2668	+ policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP;
	2669	+ rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value);
	2670	+ WRITE_ONCE(cpu->hwp_req_cached, value);
	2671	+ cpu->epp_cached = intel_pstate_get_epp(cpu, value);
	2672	+ } else {
	2673	+ turbo_max = cpu->pstate.turbo_pstate;
	2674	+ policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
	2675	+ }
	2676	+
	2677	+ min_freq = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
	2678	+ min_freq *= cpu->pstate.scaling;
	2679	+ max_freq = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
	2680	+ max_freq *= cpu->pstate.scaling;
	2681	+
	2682	+ ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN,
	2683	+ min_freq);
	2684	+ if (ret < 0) {
	2685	+ dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
	2686	+ goto free_req;
	2687	+ }
	2688	+
	2689	+ ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX,
	2690	+ max_freq);
	2691	+ if (ret < 0) {
	2692	+ dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
	2693	+ goto remove_min_req;
	2694	+ }
	2695	+
	2696	+ policy->driver_data = req;
	2697	+
2252	2698	return 0;
	2699	+
	2700	+remove_min_req:
	2701	+ freq_qos_remove_request(req);
	2702	+free_req:
	2703	+ kfree(req);
	2704	+pstate_exit:
	2705	+ intel_pstate_exit_perf_limits(policy);
	2706	+
	2707	+ return ret;
	2708	+}
	2709	+
	2710	+static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
	2711	+{
	2712	+ struct freq_qos_request *req;
	2713	+
	2714	+ req = policy->driver_data;
	2715	+
	2716	+ freq_qos_remove_request(req + 1);
	2717	+ freq_qos_remove_request(req);
	2718	+ kfree(req);
	2719	+
	2720	+ return intel_pstate_cpu_exit(policy);
2253	2721	}
2254	2722
2255	2723	static struct cpufreq_driver intel_cpufreq = {
..	..	@@ -2258,12 +2726,16 @@
2258	2726	.target = intel_cpufreq_target,
2259	2727	.fast_switch = intel_cpufreq_fast_switch,
2260	2728	.init = intel_cpufreq_cpu_init,
2261		- .exit = intel_pstate_cpu_exit,
2262		- .stop_cpu = intel_cpufreq_stop_cpu,
	2729	+ .exit = intel_cpufreq_cpu_exit,
	2730	+ .offline = intel_pstate_cpu_offline,
	2731	+ .online = intel_pstate_cpu_online,
	2732	+ .suspend = intel_pstate_suspend,
	2733	+ .resume = intel_pstate_resume,
	2734	+ .update_limits = intel_pstate_update_limits,
2263	2735	.name = "intel_cpufreq",
2264	2736	};
2265	2737
2266		-static struct cpufreq_driver *default_driver = &intel_pstate;
	2738	+static struct cpufreq_driver *default_driver;
2267	2739
2268	2740	static void intel_pstate_driver_cleanup(void)
2269	2741	{
..	..	@@ -2280,12 +2752,16 @@
2280	2752	}
2281	2753	}
2282	2754	put_online_cpus();
	2755	+
2283	2756	intel_pstate_driver = NULL;
2284	2757	}
2285	2758
2286	2759	static int intel_pstate_register_driver(struct cpufreq_driver *driver)
2287	2760	{
2288	2761	int ret;
	2762	+
	2763	+ if (driver == &intel_pstate)
	2764	+ intel_pstate_sysfs_expose_hwp_dynamic_boost();
2289	2765
2290	2766	memset(&global, 0, sizeof(global));
2291	2767	global.max_perf_pct = 100;
..	..	@@ -2302,17 +2778,6 @@
2302	2778	return 0;
2303	2779	}
2304	2780
2305		-static int intel_pstate_unregister_driver(void)
2306		-{
2307		- if (hwp_active)
2308		- return -EBUSY;
2309		-
2310		- cpufreq_unregister_driver(intel_pstate_driver);
2311		- intel_pstate_driver_cleanup();
2312		-
2313		- return 0;
2314		-}
2315		-
2316	2781	static ssize_t intel_pstate_show_status(char *buf)
2317	2782	{
2318	2783	if (!intel_pstate_driver)
..	..	@@ -2324,8 +2789,6 @@
2324	2789
2325	2790	static int intel_pstate_update_status(const char *buf, size_t size)
2326	2791	{
2327		- int ret;
2328		-
2329	2792	if (size == 3 && !strncmp(buf, "off", size)) {
2330	2793	if (!intel_pstate_driver)
2331	2794	return -EINVAL;
..	..	@@ -2333,7 +2796,9 @@
2333	2796	if (hwp_active)
2334	2797	return -EBUSY;
2335	2798
2336		- return intel_pstate_unregister_driver();
	2799	+ cpufreq_unregister_driver(intel_pstate_driver);
	2800	+ intel_pstate_driver_cleanup();
	2801	+ return 0;
2337	2802	}
2338	2803
2339	2804	if (size == 6 && !strncmp(buf, "active", size)) {
..	..	@@ -2341,9 +2806,7 @@
2341	2806	if (intel_pstate_driver == &intel_pstate)
2342	2807	return 0;
2343	2808
2344		- ret = intel_pstate_unregister_driver();
2345		- if (ret)
2346		- return ret;
	2809	+ cpufreq_unregister_driver(intel_pstate_driver);
2347	2810	}
2348	2811
2349	2812	return intel_pstate_register_driver(&intel_pstate);
..	..	@@ -2354,9 +2817,8 @@
2354	2817	if (intel_pstate_driver == &intel_cpufreq)
2355	2818	return 0;
2356	2819
2357		- ret = intel_pstate_unregister_driver();
2358		- if (ret)
2359		- return ret;
	2820	+ cpufreq_unregister_driver(intel_pstate_driver);
	2821	+ intel_pstate_sysfs_hide_hwp_dynamic_boost();
2360	2822	}
2361	2823
2362	2824	return intel_pstate_register_driver(&intel_cpufreq);
..	..	@@ -2420,6 +2882,7 @@
2420	2882	kfree(pss);
2421	2883	}
2422	2884
	2885	+ pr_debug("ACPI _PSS not found\n");
2423	2886	return true;
2424	2887	}
2425	2888
..	..	@@ -2430,9 +2893,14 @@
2430	2893
2431	2894	status = acpi_get_handle(NULL, "\\_SB", &handle);
2432	2895	if (ACPI_FAILURE(status))
2433		- return true;
	2896	+ goto not_found;
2434	2897
2435		- return !acpi_has_method(handle, "PCCH");
	2898	+ if (acpi_has_method(handle, "PCCH"))
	2899	+ return false;
	2900	+
	2901	+not_found:
	2902	+ pr_debug("ACPI PCCH not found\n");
	2903	+ return true;
2436	2904	}
2437	2905
2438	2906	static bool __init intel_pstate_has_acpi_ppc(void)
..	..	@@ -2447,6 +2915,7 @@
2447	2915	if (acpi_has_method(pr->handle, "_PPC"))
2448	2916	return true;
2449	2917	}
	2918	+ pr_debug("ACPI _PPC not found\n");
2450	2919	return false;
2451	2920	}
2452	2921
..	..	@@ -2457,23 +2926,25 @@
2457	2926
2458	2927	/* Hardware vendor-specific info that has its own power management modes */
2459	2928	static struct acpi_platform_list plat_info[] __initdata = {
2460		- {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, 0, PSS},
2461		- {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2462		- {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2463		- {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2464		- {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2465		- {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2466		- {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2467		- {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2468		- {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2469		- {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2470		- {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2471		- {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2472		- {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2473		- {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
2474		- {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
	2929	+ {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS},
	2930	+ {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2931	+ {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2932	+ {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2933	+ {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2934	+ {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2935	+ {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2936	+ {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2937	+ {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2938	+ {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2939	+ {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2940	+ {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2941	+ {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2942	+ {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
	2943	+ {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
2475	2944	{ } /* End */
2476	2945	};
	2946	+
	2947	+#define BITMASK_OOB (BIT(8) \| BIT(18))
2477	2948
2478	2949	static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
2479	2950	{
..	..	@@ -2484,8 +2955,11 @@
2484	2955	id = x86_match_cpu(intel_pstate_cpu_oob_ids);
2485	2956	if (id) {
2486	2957	rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
2487		- if ( misc_pwr & (1 << 8))
	2958	+ if (misc_pwr & BITMASK_OOB) {
	2959	+ pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
	2960	+ pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
2488	2961	return true;
	2962	+ }
2489	2963	}
2490	2964
2491	2965	idx = acpi_match_platform_list(plat_info);
..	..	@@ -2522,51 +2996,93 @@
2522	2996
2523	2997	#define INTEL_PSTATE_HWP_BROADWELL 0x01
2524	2998
2525		-#define ICPU_HWP(model, hwp_mode) \
2526		- { X86_VENDOR_INTEL, 6, model, X86_FEATURE_HWP, hwp_mode }
	2999	+#define X86_MATCH_HWP(model, hwp_mode) \
	3000	+ X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
	3001	+ X86_FEATURE_HWP, hwp_mode)
2527	3002
2528	3003	static const struct x86_cpu_id hwp_support_ids[] __initconst = {
2529		- ICPU_HWP(INTEL_FAM6_BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
2530		- ICPU_HWP(INTEL_FAM6_BROADWELL_XEON_D, INTEL_PSTATE_HWP_BROADWELL),
2531		- ICPU_HWP(X86_MODEL_ANY, 0),
	3004	+ X86_MATCH_HWP(BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
	3005	+ X86_MATCH_HWP(BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
	3006	+ X86_MATCH_HWP(ANY, 0),
2532	3007	{}
2533	3008	};
	3009	+
	3010	+static bool intel_pstate_hwp_is_enabled(void)
	3011	+{
	3012	+ u64 value;
	3013	+
	3014	+ rdmsrl(MSR_PM_ENABLE, value);
	3015	+ return !!(value & 0x1);
	3016	+}
2534	3017
2535	3018	static int __init intel_pstate_init(void)
2536	3019	{
2537	3020	const struct x86_cpu_id *id;
2538	3021	int rc;
2539	3022
2540		- if (no_load)
	3023	+ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2541	3024	return -ENODEV;
2542	3025
2543	3026	id = x86_match_cpu(hwp_support_ids);
2544	3027	if (id) {
	3028	+ bool hwp_forced = intel_pstate_hwp_is_enabled();
	3029	+
	3030	+ if (hwp_forced)
	3031	+ pr_info("HWP enabled by BIOS\n");
	3032	+ else if (no_load)
	3033	+ return -ENODEV;
	3034	+
2545	3035	copy_cpu_funcs(&core_funcs);
2546		- if (!no_hwp) {
	3036	+ /*
	3037	+ * Avoid enabling HWP for processors without EPP support,
	3038	+ * because that means incomplete HWP implementation which is a
	3039	+ * corner case and supporting it is generally problematic.
	3040	+ *
	3041	+ * If HWP is enabled already, though, there is no choice but to
	3042	+ * deal with it.
	3043	+ */
	3044	+ if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) \|\| hwp_forced) {
2547	3045	hwp_active++;
2548	3046	hwp_mode_bdw = id->driver_data;
2549	3047	intel_pstate.attr = hwp_cpufreq_attrs;
	3048	+ intel_cpufreq.attr = hwp_cpufreq_attrs;
	3049	+ intel_cpufreq.flags \|= CPUFREQ_NEED_UPDATE_LIMITS;
	3050	+ if (!default_driver)
	3051	+ default_driver = &intel_pstate;
	3052	+
2550	3053	goto hwp_cpu_matched;
2551	3054	}
	3055	+ pr_info("HWP not enabled\n");
2552	3056	} else {
2553		- id = x86_match_cpu(intel_pstate_cpu_ids);
2554		- if (!id)
	3057	+ if (no_load)
2555	3058	return -ENODEV;
	3059	+
	3060	+ id = x86_match_cpu(intel_pstate_cpu_ids);
	3061	+ if (!id) {
	3062	+ pr_info("CPU model not supported\n");
	3063	+ return -ENODEV;
	3064	+ }
2556	3065
2557	3066	copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
2558	3067	}
2559	3068
2560		- if (intel_pstate_msrs_not_valid())
	3069	+ if (intel_pstate_msrs_not_valid()) {
	3070	+ pr_info("Invalid MSRs\n");
2561	3071	return -ENODEV;
	3072	+ }
	3073	+ /* Without HWP start in the passive mode. */
	3074	+ if (!default_driver)
	3075	+ default_driver = &intel_cpufreq;
2562	3076
2563	3077	hwp_cpu_matched:
2564	3078	/*
2565	3079	* The Intel pstate driver will be ignored if the platform
2566	3080	* firmware has its own power management modes.
2567	3081	*/
2568		- if (intel_pstate_platform_pwr_mgmt_exists())
	3082	+ if (intel_pstate_platform_pwr_mgmt_exists()) {
	3083	+ pr_info("P-states controlled by the platform\n");
2569	3084	return -ENODEV;
	3085	+ }
2570	3086
2571	3087	if (!hwp_active && hwp_only)
2572	3088	return -ENOTSUPP;
..	..	@@ -2584,11 +3100,22 @@
2584	3100	mutex_lock(&intel_pstate_driver_lock);
2585	3101	rc = intel_pstate_register_driver(default_driver);
2586	3102	mutex_unlock(&intel_pstate_driver_lock);
2587		- if (rc)
	3103	+ if (rc) {
	3104	+ intel_pstate_sysfs_remove();
2588	3105	return rc;
	3106	+ }
2589	3107
2590		- if (hwp_active)
	3108	+ if (hwp_active) {
	3109	+ const struct x86_cpu_id *id;
	3110	+
	3111	+ id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
	3112	+ if (id) {
	3113	+ set_power_ctl_ee_state(false);
	3114	+ pr_info("Disabling energy efficiency optimization\n");
	3115	+ }
	3116	+
2591	3117	pr_info("HWP enabled\n");
	3118	+ }
2592	3119
2593	3120	return 0;
2594	3121	}
..	..	@@ -2599,17 +3126,16 @@
2599	3126	if (!str)
2600	3127	return -EINVAL;
2601	3128
2602		- if (!strcmp(str, "disable")) {
	3129	+ if (!strcmp(str, "disable"))
2603	3130	no_load = 1;
2604		- } else if (!strcmp(str, "passive")) {
2605		- pr_info("Passive mode enabled\n");
	3131	+ else if (!strcmp(str, "active"))
	3132	+ default_driver = &intel_pstate;
	3133	+ else if (!strcmp(str, "passive"))
2606	3134	default_driver = &intel_cpufreq;
	3135	+
	3136	+ if (!strcmp(str, "no_hwp"))
2607	3137	no_hwp = 1;
2608		- }
2609		- if (!strcmp(str, "no_hwp")) {
2610		- pr_info("HWP disabled\n");
2611		- no_hwp = 1;
2612		- }
	3138	+
2613	3139	if (!strcmp(str, "force"))
2614	3140	force_load = 1;
2615	3141	if (!strcmp(str, "hwp_only"))