.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * Copyright (C) 2011 Google, Inc. |
---|
3 | 4 | * |
---|
4 | 5 | * Author: |
---|
5 | 6 | * Colin Cross <ccross@android.com> |
---|
6 | | - * |
---|
7 | | - * This software is licensed under the terms of the GNU General Public |
---|
8 | | - * License version 2, as published by the Free Software Foundation, and |
---|
9 | | - * may be copied, distributed, and modified under those terms. |
---|
10 | | - * |
---|
11 | | - * This program is distributed in the hope that it will be useful, |
---|
12 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
13 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
14 | | - * GNU General Public License for more details. |
---|
15 | | - * |
---|
16 | 7 | */ |
---|
17 | 8 | |
---|
18 | 9 | #include <linux/kernel.h> |
---|
.. | .. |
---|
22 | 13 | #include <linux/spinlock.h> |
---|
23 | 14 | #include <linux/syscore_ops.h> |
---|
24 | 15 | |
---|
25 | | -static ATOMIC_NOTIFIER_HEAD(cpu_pm_notifier_chain); |
---|
| 16 | +/* |
---|
| 17 | + * atomic_notifiers use a spinlock_t, which can block under PREEMPT_RT. |
---|
| 18 | + * Notifications for cpu_pm will be issued by the idle task itself, which can |
---|
| 19 | + * never block, IOW it requires using a raw_spinlock_t. |
---|
| 20 | + */ |
---|
| 21 | +static struct { |
---|
| 22 | + struct raw_notifier_head chain; |
---|
| 23 | + raw_spinlock_t lock; |
---|
| 24 | +} cpu_pm_notifier = { |
---|
| 25 | + .chain = RAW_NOTIFIER_INIT(cpu_pm_notifier.chain), |
---|
| 26 | + .lock = __RAW_SPIN_LOCK_UNLOCKED(cpu_pm_notifier.lock), |
---|
| 27 | +}; |
---|
26 | 28 | |
---|
27 | | -static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls) |
---|
| 29 | +static int cpu_pm_notify(enum cpu_pm_event event) |
---|
28 | 30 | { |
---|
29 | 31 | int ret; |
---|
30 | 32 | |
---|
31 | 33 | /* |
---|
32 | | - * __atomic_notifier_call_chain has a RCU read critical section, which |
---|
33 | | - * could be disfunctional in cpu idle. Copy RCU_NONIDLE code to let |
---|
34 | | - * RCU know this. |
---|
| 34 | + * This introduces a RCU read critical section, which could be |
---|
| 35 | + * disfunctional in cpu idle. Copy RCU_NONIDLE code to let RCU know |
---|
| 36 | + * this. |
---|
35 | 37 | */ |
---|
36 | 38 | rcu_irq_enter_irqson(); |
---|
37 | | - ret = __atomic_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL, |
---|
38 | | - nr_to_call, nr_calls); |
---|
| 39 | + rcu_read_lock(); |
---|
| 40 | + ret = raw_notifier_call_chain(&cpu_pm_notifier.chain, event, NULL); |
---|
| 41 | + rcu_read_unlock(); |
---|
| 42 | + rcu_irq_exit_irqson(); |
---|
| 43 | + |
---|
| 44 | + return notifier_to_errno(ret); |
---|
| 45 | +} |
---|
| 46 | + |
---|
| 47 | +static int cpu_pm_notify_robust(enum cpu_pm_event event_up, enum cpu_pm_event event_down) |
---|
| 48 | +{ |
---|
| 49 | + unsigned long flags; |
---|
| 50 | + int ret; |
---|
| 51 | + |
---|
| 52 | + rcu_irq_enter_irqson(); |
---|
| 53 | + raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags); |
---|
| 54 | + ret = raw_notifier_call_chain_robust(&cpu_pm_notifier.chain, event_up, event_down, NULL); |
---|
| 55 | + raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags); |
---|
39 | 56 | rcu_irq_exit_irqson(); |
---|
40 | 57 | |
---|
41 | 58 | return notifier_to_errno(ret); |
---|
.. | .. |
---|
48 | 65 | * Add a driver to a list of drivers that are notified about |
---|
49 | 66 | * CPU and CPU cluster low power entry and exit. |
---|
50 | 67 | * |
---|
51 | | - * This function may sleep, and has the same return conditions as |
---|
52 | | - * raw_notifier_chain_register. |
---|
| 68 | + * This function has the same return conditions as raw_notifier_chain_register. |
---|
53 | 69 | */ |
---|
54 | 70 | int cpu_pm_register_notifier(struct notifier_block *nb) |
---|
55 | 71 | { |
---|
56 | | - return atomic_notifier_chain_register(&cpu_pm_notifier_chain, nb); |
---|
| 72 | + unsigned long flags; |
---|
| 73 | + int ret; |
---|
| 74 | + |
---|
| 75 | + raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags); |
---|
| 76 | + ret = raw_notifier_chain_register(&cpu_pm_notifier.chain, nb); |
---|
| 77 | + raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags); |
---|
| 78 | + return ret; |
---|
57 | 79 | } |
---|
58 | 80 | EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); |
---|
59 | 81 | |
---|
.. | .. |
---|
63 | 85 | * |
---|
64 | 86 | * Remove a driver from the CPU PM notifier list. |
---|
65 | 87 | * |
---|
66 | | - * This function may sleep, and has the same return conditions as |
---|
67 | | - * raw_notifier_chain_unregister. |
---|
| 88 | + * This function has the same return conditions as raw_notifier_chain_unregister. |
---|
68 | 89 | */ |
---|
69 | 90 | int cpu_pm_unregister_notifier(struct notifier_block *nb) |
---|
70 | 91 | { |
---|
71 | | - return atomic_notifier_chain_unregister(&cpu_pm_notifier_chain, nb); |
---|
| 92 | + unsigned long flags; |
---|
| 93 | + int ret; |
---|
| 94 | + |
---|
| 95 | + raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags); |
---|
| 96 | + ret = raw_notifier_chain_unregister(&cpu_pm_notifier.chain, nb); |
---|
| 97 | + raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags); |
---|
| 98 | + return ret; |
---|
72 | 99 | } |
---|
73 | 100 | EXPORT_SYMBOL_GPL(cpu_pm_unregister_notifier); |
---|
74 | 101 | |
---|
.. | .. |
---|
89 | 116 | */ |
---|
90 | 117 | int cpu_pm_enter(void) |
---|
91 | 118 | { |
---|
92 | | - int nr_calls = 0; |
---|
93 | | - int ret = 0; |
---|
94 | | - |
---|
95 | | - ret = cpu_pm_notify(CPU_PM_ENTER, -1, &nr_calls); |
---|
96 | | - if (ret) |
---|
97 | | - /* |
---|
98 | | - * Inform listeners (nr_calls - 1) about failure of CPU PM |
---|
99 | | - * PM entry who are notified earlier to prepare for it. |
---|
100 | | - */ |
---|
101 | | - cpu_pm_notify(CPU_PM_ENTER_FAILED, nr_calls - 1, NULL); |
---|
102 | | - |
---|
103 | | - return ret; |
---|
| 119 | + return cpu_pm_notify_robust(CPU_PM_ENTER, CPU_PM_ENTER_FAILED); |
---|
104 | 120 | } |
---|
105 | 121 | EXPORT_SYMBOL_GPL(cpu_pm_enter); |
---|
106 | 122 | |
---|
.. | .. |
---|
118 | 134 | */ |
---|
119 | 135 | int cpu_pm_exit(void) |
---|
120 | 136 | { |
---|
121 | | - return cpu_pm_notify(CPU_PM_EXIT, -1, NULL); |
---|
| 137 | + return cpu_pm_notify(CPU_PM_EXIT); |
---|
122 | 138 | } |
---|
123 | 139 | EXPORT_SYMBOL_GPL(cpu_pm_exit); |
---|
124 | 140 | |
---|
.. | .. |
---|
140 | 156 | */ |
---|
141 | 157 | int cpu_cluster_pm_enter(void) |
---|
142 | 158 | { |
---|
143 | | - int nr_calls = 0; |
---|
144 | | - int ret = 0; |
---|
145 | | - |
---|
146 | | - ret = cpu_pm_notify(CPU_CLUSTER_PM_ENTER, -1, &nr_calls); |
---|
147 | | - if (ret) |
---|
148 | | - /* |
---|
149 | | - * Inform listeners (nr_calls - 1) about failure of CPU cluster |
---|
150 | | - * PM entry who are notified earlier to prepare for it. |
---|
151 | | - */ |
---|
152 | | - cpu_pm_notify(CPU_CLUSTER_PM_ENTER_FAILED, nr_calls - 1, NULL); |
---|
153 | | - |
---|
154 | | - return ret; |
---|
| 159 | + return cpu_pm_notify_robust(CPU_CLUSTER_PM_ENTER, CPU_CLUSTER_PM_ENTER_FAILED); |
---|
155 | 160 | } |
---|
156 | 161 | EXPORT_SYMBOL_GPL(cpu_cluster_pm_enter); |
---|
157 | 162 | |
---|
.. | .. |
---|
172 | 177 | */ |
---|
173 | 178 | int cpu_cluster_pm_exit(void) |
---|
174 | 179 | { |
---|
175 | | - return cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL); |
---|
| 180 | + return cpu_pm_notify(CPU_CLUSTER_PM_EXIT); |
---|
176 | 181 | } |
---|
177 | 182 | EXPORT_SYMBOL_GPL(cpu_cluster_pm_exit); |
---|
178 | 183 | |
---|