hc
2024-05-10 61598093bbdd283a7edc367d900f223070ead8d2
kernel/kernel/sched/stats.h
....@@ -70,7 +70,7 @@
7070 return;
7171
7272 if (!wakeup || p->sched_psi_wake_requeue) {
73
- if (p->flags & PF_MEMSTALL)
73
+ if (p->in_memstall)
7474 set |= TSK_MEMSTALL;
7575 if (p->sched_psi_wake_requeue)
7676 p->sched_psi_wake_requeue = 0;
....@@ -90,9 +90,17 @@
9090 return;
9191
9292 if (!sleep) {
93
- if (p->flags & PF_MEMSTALL)
93
+ if (p->in_memstall)
9494 clear |= TSK_MEMSTALL;
9595 } else {
96
+ /*
97
+ * When a task sleeps, schedule() dequeues it before
98
+ * switching to the next one. Merge the clearing of
99
+ * TSK_RUNNING and TSK_ONCPU to save an unnecessary
100
+ * psi_task_change() call in psi_sched_switch().
101
+ */
102
+ clear |= TSK_ONCPU;
103
+
96104 if (p->in_iowait)
97105 set |= TSK_IOWAIT;
98106 }
....@@ -109,14 +117,14 @@
109117 * deregister its sleep-persistent psi states from the old
110118 * queue, and let psi_enqueue() know it has to requeue.
111119 */
112
- if (unlikely(p->in_iowait || (p->flags & PF_MEMSTALL))) {
120
+ if (unlikely(p->in_iowait || p->in_memstall)) {
113121 struct rq_flags rf;
114122 struct rq *rq;
115123 int clear = 0;
116124
117125 if (p->in_iowait)
118126 clear |= TSK_IOWAIT;
119
- if (p->flags & PF_MEMSTALL)
127
+ if (p->in_memstall)
120128 clear |= TSK_MEMSTALL;
121129
122130 rq = __task_rq_lock(p, &rf);
....@@ -126,18 +134,31 @@
126134 }
127135 }
128136
137
+static inline void psi_sched_switch(struct task_struct *prev,
138
+ struct task_struct *next,
139
+ bool sleep)
140
+{
141
+ if (static_branch_likely(&psi_disabled))
142
+ return;
143
+
144
+ psi_task_switch(prev, next, sleep);
145
+}
146
+
129147 static inline void psi_task_tick(struct rq *rq)
130148 {
131149 if (static_branch_likely(&psi_disabled))
132150 return;
133151
134
- if (unlikely(rq->curr->flags & PF_MEMSTALL))
152
+ if (unlikely(rq->curr->in_memstall))
135153 psi_memstall_tick(rq->curr, cpu_of(rq));
136154 }
137155 #else /* CONFIG_PSI */
138156 static inline void psi_enqueue(struct task_struct *p, bool wakeup) {}
139157 static inline void psi_dequeue(struct task_struct *p, bool sleep) {}
140158 static inline void psi_ttwu_dequeue(struct task_struct *p) {}
159
+static inline void psi_sched_switch(struct task_struct *prev,
160
+ struct task_struct *next,
161
+ bool sleep) {}
141162 static inline void psi_task_tick(struct rq *rq) {}
142163 #endif /* CONFIG_PSI */
143164
....@@ -157,9 +178,10 @@
157178 {
158179 unsigned long long now = rq_clock(rq), delta = 0;
159180
160
- if (unlikely(sched_info_on()))
181
+ if (sched_info_on()) {
161182 if (t->sched_info.last_queued)
162183 delta = now - t->sched_info.last_queued;
184
+ }
163185 sched_info_reset_dequeued(t);
164186 t->sched_info.run_delay += delta;
165187
....@@ -192,7 +214,7 @@
192214 */
193215 static inline void sched_info_queued(struct rq *rq, struct task_struct *t)
194216 {
195
- if (unlikely(sched_info_on())) {
217
+ if (sched_info_on()) {
196218 if (!t->sched_info.last_queued)
197219 t->sched_info.last_queued = rq_clock(rq);
198220 }
....@@ -239,7 +261,7 @@
239261 static inline void
240262 sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next)
241263 {
242
- if (unlikely(sched_info_on()))
264
+ if (sched_info_on())
243265 __sched_info_switch(rq, prev, next);
244266 }
245267