forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/include/linux/sched.h
....@@ -18,17 +18,23 @@
1818 #include <linux/mutex.h>
1919 #include <linux/plist.h>
2020 #include <linux/hrtimer.h>
21
+#include <linux/irqflags.h>
2122 #include <linux/seccomp.h>
2223 #include <linux/nodemask.h>
2324 #include <linux/rcupdate.h>
25
+#include <linux/refcount.h>
2426 #include <linux/resource.h>
2527 #include <linux/latencytop.h>
2628 #include <linux/sched/prio.h>
29
+#include <linux/sched/types.h>
2730 #include <linux/signal_types.h>
2831 #include <linux/mm_types_task.h>
29
-#include <linux/mm_event.h>
3032 #include <linux/task_io_accounting.h>
33
+#include <linux/posix-timers.h>
3134 #include <linux/rseq.h>
35
+#include <linux/seqlock.h>
36
+#include <linux/kcsan.h>
37
+#include <linux/android_vendor.h>
3238 #include <linux/android_kabi.h>
3339 
3440 /* task_struct member predeclarations (sorted alphabetically): */
....@@ -36,6 +42,7 @@
3642 struct backing_dev_info;
3743 struct bio_list;
3844 struct blk_plug;
45
+struct capture_control;
3946 struct cfs_rq;
4047 struct fs_struct;
4148 struct futex_pi_state;
....@@ -49,6 +56,8 @@
4956 struct rcu_node;
5057 struct reclaim_state;
5158 struct robust_list_head;
59
+struct root_domain;
60
+struct rq;
5261 struct sched_attr;
5362 struct sched_param;
5463 struct seq_file;
....@@ -56,6 +65,7 @@
5665 struct signal_struct;
5766 struct task_delay_info;
5867 struct task_group;
68
+struct io_uring_task;
5969 
6070 /*
6171  * Task state bitmask. NOTE! These bits are also
....@@ -109,10 +119,6 @@
109119 
110120 #define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
111121 
112
-#define task_contributes_to_load(task)	((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
113
-					 (task->flags & PF_FROZEN) == 0 && \
114
-					 (task->state & TASK_NOLOAD) == 0)
115
-
116122 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
117123 
118124 /*
....@@ -153,31 +159,31 @@
153159  *
154160  *   for (;;) {
155161  *	set_current_state(TASK_UNINTERRUPTIBLE);
156
- *	if (!need_sleep)
157
- *		break;
162
+ *	if (CONDITION)
163
+ *	   break;
158164  *
159165  *	schedule();
160166  *   }
161167  *   __set_current_state(TASK_RUNNING);
162168  *
163169  * If the caller does not need such serialisation (because, for instance, the
164
- * condition test and condition change and wakeup are under the same lock) then
170
+ * CONDITION test and condition change and wakeup are under the same lock) then
165171  * use __set_current_state().
166172  *
167173  * The above is typically ordered against the wakeup, which does:
168174  *
169
- *   need_sleep = false;
175
+ *   CONDITION = 1;
170176  *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
171177  *
172
- * where wake_up_state() executes a full memory barrier before accessing the
173
- * task state.
178
+ * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
179
+ * accessing p->state.
174180  *
175181  * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
176182  * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
177183  * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
178184  *
179185  * However, with slightly different timing the wakeup TASK_RUNNING store can
180
- * also collide with the TASK_UNINTERRUPTIBLE store. Loosing that store is not
186
+ * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
181187  * a problem either because that will result in one extra go around the loop
182188  * and our @cond test will save the day.
183189  *
....@@ -219,6 +225,7 @@
219225 extern long schedule_timeout_idle(long timeout);
220226 asmlinkage void schedule(void);
221227 extern void schedule_preempt_disabled(void);
228
+asmlinkage void preempt_schedule_irq(void);
222229 
223230 extern int __must_check io_schedule_prepare(void);
224231 extern void io_schedule_finish(int token);
....@@ -242,40 +249,24 @@
242249 #endif
243250 };
244251 
245
-/**
246
- * struct task_cputime - collected CPU time counts
247
- * @utime:		time spent in user mode, in nanoseconds
248
- * @stime:		time spent in kernel mode, in nanoseconds
249
- * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
250
- *
251
- * This structure groups together three kinds of CPU time that are tracked for
252
- * threads and thread groups.  Most things considering CPU time want to group
253
- * these counts together and treat all three of them in parallel.
254
- */
255
-struct task_cputime {
256
-	u64				utime;
257
-	u64				stime;
258
-	unsigned long long		sum_exec_runtime;
259
-};
260
-
261
-/* Alternate field names when used on cache expirations: */
262
-#define virt_exp			utime
263
-#define prof_exp			stime
264
-#define sched_exp			sum_exec_runtime
265
-
266252 enum vtime_state {
267253 	/* Task is sleeping or running in a CPU with VTIME inactive: */
268254 	VTIME_INACTIVE = 0,
269
-	/* Task runs in userspace in a CPU with VTIME active: */
270
-	VTIME_USER,
255
+	/* Task is idle */
256
+	VTIME_IDLE,
271257 	/* Task runs in kernelspace in a CPU with VTIME active: */
272258 	VTIME_SYS,
259
+	/* Task runs in userspace in a CPU with VTIME active: */
260
+	VTIME_USER,
261
+	/* Task runs as guests in a CPU with VTIME active: */
262
+	VTIME_GUEST,
273263 };
274264 
275265 struct vtime {
276266 	seqcount_t		seqcount;
277267 	unsigned long long	starttime;
278268 	enum vtime_state	state;
269
+	unsigned int		cpu;
279270 	u64			utime;
280271 	u64			stime;
281272 	u64			gtime;
....@@ -292,6 +283,11 @@
292283 	UCLAMP_MAX,
293284 	UCLAMP_CNT
294285 };
286
+
287
+#ifdef CONFIG_SMP
288
+extern struct root_domain def_root_domain;
289
+extern struct mutex sched_domains_mutex;
290
+#endif
295291 
296292 struct sched_info {
297293 #ifdef CONFIG_SCHED_INFO
....@@ -354,36 +350,46 @@
354350  * Only for tasks we track a moving average of the past instantaneous
355351  * estimated utilization. This allows to absorb sporadic drops in utilization
356352  * of an otherwise almost periodic task.
353
+ *
354
+ * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
355
+ * updates. When a task is dequeued, its util_est should not be updated if its
356
+ * util_avg has not been updated in the meantime.
357
+ * This information is mapped into the MSB bit of util_est.enqueued at dequeue
358
+ * time. Since max value of util_est.enqueued for a task is 1024 (PELT util_avg
359
+ * for a task) it is safe to use MSB.
357360  */
358361 struct util_est {
359362 	unsigned int			enqueued;
360363 	unsigned int			ewma;
361364 #define UTIL_EST_WEIGHT_SHIFT		2
365
+#define UTIL_AVG_UNCHANGED		0x80000000
362366 } __attribute__((__aligned__(sizeof(u64))));
363367 
364368 /*
365
- * The load_avg/util_avg accumulates an infinite geometric series
366
- * (see __update_load_avg() in kernel/sched/fair.c).
369
+ * The load/runnable/util_avg accumulates an infinite geometric series
370
+ * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
367371  *
368372  * [load_avg definition]
369373  *
370374  *   load_avg = runnable% * scale_load_down(load)
371375  *
372
- * where runnable% is the time ratio that a sched_entity is runnable.
373
- * For cfs_rq, it is the aggregated load_avg of all runnable and
374
- * blocked sched_entities.
376
+ * [runnable_avg definition]
377
+ *
378
+ *   runnable_avg = runnable% * SCHED_CAPACITY_SCALE
375379  *
376380  * [util_avg definition]
377381  *
378382  *   util_avg = running% * SCHED_CAPACITY_SCALE
379383  *
380
- * where running% is the time ratio that a sched_entity is running on
381
- * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
382
- * and blocked sched_entities.
384
+ * where runnable% is the time ratio that a sched_entity is runnable and
385
+ * running% the time ratio that a sched_entity is running.
383386  *
384
- * load_avg and util_avg don't direcly factor frequency scaling and CPU
385
- * capacity scaling. The scaling is done through the rq_clock_pelt that
386
- * is used for computing those signals (see update_rq_clock_pelt())
387
+ * For cfs_rq, they are the aggregated values of all runnable and blocked
388
+ * sched_entities.
389
+ *
390
+ * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
391
+ * capacity scaling. The scaling is done through the rq_clock_pelt that is used
392
+ * for computing those signals (see update_rq_clock_pelt())
387393  *
388394  * N.B., the above ratios (runnable% and running%) themselves are in the
389395  * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
....@@ -407,11 +413,11 @@
407413 struct sched_avg {
408414 	u64				last_update_time;
409415 	u64				load_sum;
410
-	u64				runnable_load_sum;
416
+	u64				runnable_sum;
411417 	u32				util_sum;
412418 	u32				period_contrib;
413419 	unsigned long			load_avg;
414
-	unsigned long			runnable_load_avg;
420
+	unsigned long			runnable_avg;
415421 	unsigned long			util_avg;
416422 	struct util_est			util_est;
417423 } ____cacheline_aligned;
....@@ -455,7 +461,6 @@
455461 struct sched_entity {
456462 	/* For load-balancing: */
457463 	struct load_weight		load;
458
-	unsigned long			runnable_weight;
459464 	struct rb_node			run_node;
460465 	struct list_head		group_node;
461466 	unsigned int			on_rq;
....@@ -476,6 +481,8 @@
476481 	struct cfs_rq			*cfs_rq;
477482 	/* rq "owned" by this entity/group: */
478483 	struct cfs_rq			*my_q;
484
+	/* cached value of my_q->h_nr_running */
485
+	unsigned long			runnable_weight;
479486 #endif
480487 
481488 #ifdef CONFIG_SMP
....@@ -533,7 +540,7 @@
533540 
534541 	/*
535542 	 * Actual scheduling parameters. Initialized with the values above,
536
-	 * they are continously updated during task execution. Note that
543
+	 * they are continuously updated during task execution. Note that
537544 	 * the remaining runtime could be < 0 in case we are in overrun.
538545 	 */
539546 	s64				runtime;	/* Remaining runtime for this instance	*/
....@@ -546,10 +553,6 @@
546553 	 * @dl_throttled tells if we exhausted the runtime. If so, the
547554 	 * task has to wait for a replenishment to be performed at the
548555 	 * next firing of dl_timer.
549
-	 *
550
-	 * @dl_boosted tells if we are boosted due to DI. If so we are
551
-	 * outside bandwidth enforcement mechanism (but only until we
552
-	 * exit the critical section);
553556 	 *
554557 	 * @dl_yielded tells if task gave up the CPU before consuming
555558 	 * all its available runtime during the last job.
....@@ -565,7 +568,6 @@
565568 	 * overruns.
566569 	 */
567570 	unsigned int			dl_throttled      : 1;
568
-	unsigned int			dl_boosted        : 1;
569571 	unsigned int			dl_yielded        : 1;
570572 	unsigned int			dl_non_contending : 1;
571573 	unsigned int			dl_overrun	  : 1;
....@@ -584,6 +586,15 @@
584586 	 * time.
585587 	 */
586588 	struct hrtimer inactive_timer;
589
+
590
+#ifdef CONFIG_RT_MUTEXES
591
+	/*
592
+	 * Priority Inheritance. When a DEADLINE scheduling entity is boosted
593
+	 * pi_se points to the donor, otherwise points to the dl_se it belongs
594
+	 * to (the original one/itself).
595
+	 */
596
+	struct sched_dl_entity *pi_se;
597
+#endif
587598 };
588599 
589600 #ifdef CONFIG_UCLAMP_TASK
....@@ -625,10 +636,8 @@
625636 	struct {
626637 		u8			blocked;
627638 		u8			need_qs;
628
-		u8			exp_need_qs;
629
-
630
-		/* Otherwise the compiler can store garbage here: */
631
-		u8			pad;
639
+		u8			exp_hint; /* Hint for performance. */
640
+		u8			need_mb; /* Readers need smp_mb(). */
632641 	} b; /* Bits. */
633642 	u32 s; /* Set of bits. */
634643 };
....@@ -662,14 +671,14 @@
662671 	randomized_struct_fields_start
663672 
664673 	void				*stack;
665
-	atomic_t			usage;
674
+	refcount_t			usage;
666675 	/* Per task flags (PF_*), defined further below: */
667676 	unsigned int			flags;
668677 	unsigned int			ptrace;
669678 
670679 #ifdef CONFIG_SMP
671
-	struct llist_node		wake_entry;
672680 	int				on_cpu;
681
+	struct __call_single_node	wake_entry;
673682 #ifdef CONFIG_THREAD_INFO_IN_TASK
674683 	/* Current CPU: */
675684 	unsigned int			cpu;
....@@ -698,23 +707,26 @@
698707 	const struct sched_class	*sched_class;
699708 	struct sched_entity		se;
700709 	struct sched_rt_entity		rt;
701
-
702
-	/* task boost vendor fields */
703
-	u64				last_sleep_ts;
704
-	int				boost;
705
-	u64				boost_period;
706
-	u64				boost_expires;
707
-
708710 #ifdef CONFIG_CGROUP_SCHED
709711 	struct task_group		*sched_task_group;
710712 #endif
711713 	struct sched_dl_entity		dl;
712714 
713715 #ifdef CONFIG_UCLAMP_TASK
714
-	/* Clamp values requested for a scheduling entity */
716
+	/*
717
+	 * Clamp values requested for a scheduling entity.
718
+	 * Must be updated with task_rq_lock() held.
719
+	 */
715720 	struct uclamp_se		uclamp_req[UCLAMP_CNT];
716
-	/* Effective clamp values used for a scheduling entity */
721
+	/*
722
+	 * Effective clamp values used for a scheduling entity.
723
+	 * Must be updated with task_rq_lock() held.
724
+	 */
717725 	struct uclamp_se		uclamp[UCLAMP_CNT];
726
+#endif
727
+
728
+#ifdef CONFIG_HOTPLUG_CPU
729
+	struct list_head		percpu_kthread_node;
718730 #endif
719731 
720732 #ifdef CONFIG_PREEMPT_NOTIFIERS
....@@ -728,8 +740,8 @@
728740 
729741 	unsigned int			policy;
730742 	int				nr_cpus_allowed;
731
-	cpumask_t			cpus_allowed;
732
-	cpumask_t			cpus_requested;
743
+	const cpumask_t			*cpus_ptr;
744
+	cpumask_t			cpus_mask;
733745 
734746 #ifdef CONFIG_PREEMPT_RCU
735747 	int				rcu_read_lock_nesting;
....@@ -745,6 +757,14 @@
745757 	int				rcu_tasks_idle_cpu;
746758 	struct list_head		rcu_tasks_holdout_list;
747759 #endif /* #ifdef CONFIG_TASKS_RCU */
760
+
761
+#ifdef CONFIG_TASKS_TRACE_RCU
762
+	int				trc_reader_nesting;
763
+	int				trc_ipi_to_cpu;
764
+	union rcu_special		trc_reader_special;
765
+	bool				trc_reader_checked;
766
+	struct list_head		trc_holdout_list;
767
+#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
748768 
749769 	struct sched_info		sched_info;
750770 
....@@ -778,7 +798,6 @@
778798 	unsigned			sched_reset_on_fork:1;
779799 	unsigned			sched_contributes_to_load:1;
780800 	unsigned			sched_migrated:1;
781
-	unsigned			sched_remote_wakeup:1;
782801 #ifdef CONFIG_PSI
783802 	unsigned			sched_psi_wake_requeue:1;
784803 #endif
....@@ -788,6 +807,21 @@
788807 
789808 	/* Unserialized, strictly 'current' */
790809 
810
+	/*
811
+	 * This field must not be in the scheduler word above due to wakelist
812
+	 * queueing no longer being serialized by p->on_cpu. However:
813
+	 *
814
+	 * p->XXX = X;			ttwu()
815
+	 * schedule()			  if (p->on_rq && ..) // false
816
+	 *   smp_mb__after_spinlock();	  if (smp_load_acquire(&p->on_cpu) && //true
817
+	 *   deactivate_task()		      ttwu_queue_wakelist())
818
+	 *     p->on_rq = 0;			p->sched_remote_wakeup = Y;
819
+	 *
820
+	 * guarantees all stores of 'current' are visible before
821
+	 * ->sched_remote_wakeup gets used, so it can be in this word.
822
+	 */
823
+	unsigned			sched_remote_wakeup:1;
824
+
791825 	/* Bit to tell LSMs we're in execve(): */
792826 	unsigned			in_execve:1;
793827 	unsigned			in_iowait:1;
....@@ -796,9 +830,6 @@
796830 #endif
797831 #ifdef CONFIG_MEMCG
798832 	unsigned			in_user_fault:1;
799
-#ifdef CONFIG_MEMCG_KMEM
800
-	unsigned			memcg_kmem_skip_account:1;
801
-#endif
802833 #endif
803834 #ifdef CONFIG_COMPAT_BRK
804835 	unsigned			brk_randomized:1;
....@@ -806,10 +837,15 @@
806837 #ifdef CONFIG_CGROUPS
807838 	/* disallow userland-initiated cgroup migration */
808839 	unsigned			no_cgroup_migration:1;
840
+	/* task is frozen/stopped (used by the cgroup freezer) */
841
+	unsigned			frozen:1;
809842 #endif
810843 #ifdef CONFIG_BLK_CGROUP
811
-	/* to be used once the psi infrastructure lands upstream. */
812844 	unsigned			use_memdelay:1;
845
+#endif
846
+#ifdef CONFIG_PSI
847
+	/* Stalled due to lack of memory */
848
+	unsigned			in_memstall:1;
813849 #endif
814850 
815851 	unsigned long			atomic_flags; /* Flags requiring atomic access. */
....@@ -892,15 +928,17 @@
892928 	u64				start_time;
893929 
894930 	/* Boot based time in nsecs: */
895
-	u64				real_start_time;
931
+	u64				start_boottime;
896932 
897933 	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
898934 	unsigned long			min_flt;
899935 	unsigned long			maj_flt;
900936 
901
-#ifdef CONFIG_POSIX_TIMERS
902
-	struct task_cputime		cputime_expires;
903
-	struct list_head		cpu_timers[3];
937
+	/* Empty if CONFIG_POSIX_CPUTIMERS=n */
938
+	struct posix_cputimers		posix_cputimers;
939
+
940
+#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
941
+	struct posix_cputimers_work	posix_cputimers_work;
904942 #endif
905943 
906944 	/* Process credentials: */
....@@ -913,6 +951,11 @@
913951 
914952 	/* Effective (overridable) subjective task credentials (COW): */
915953 	const struct cred __rcu		*cred;
954
+
955
+#ifdef CONFIG_KEYS
956
+	/* Cached requested key. */
957
+	struct key			*cached_requested_key;
958
+#endif
916959 
917960 	/*
918961 	 * executable name, excluding path.
....@@ -939,12 +982,16 @@
939982 	/* Open file information: */
940983 	struct files_struct		*files;
941984 
985
+#ifdef CONFIG_IO_URING
986
+	struct io_uring_task		*io_uring;
987
+#endif
988
+
942989 	/* Namespaces: */
943990 	struct nsproxy			*nsproxy;
944991 
945992 	/* Signal handlers: */
946993 	struct signal_struct		*signal;
947
-	struct sighand_struct		*sighand;
994
+	struct sighand_struct __rcu		*sighand;
948995 	sigset_t			blocked;
949996 	sigset_t			real_blocked;
950997 	/* Restored if set_restore_sigmask() was used: */
....@@ -956,8 +1003,10 @@
9561003 
9571004 	struct callback_head		*task_works;
9581005 
959
-	struct audit_context		*audit_context;
1006
+#ifdef CONFIG_AUDIT
9601007 #ifdef CONFIG_AUDITSYSCALL
1008
+	struct audit_context		*audit_context;
1009
+#endif
9611010 	kuid_t				loginuid;
9621011 	unsigned int			sessionid;
9631012 #endif
....@@ -974,6 +1023,7 @@
9741023 	raw_spinlock_t			pi_lock;
9751024 
9761025 	struct wake_q_node		wake_q;
1026
+	int				wake_q_count;
9771027 
9781028 #ifdef CONFIG_RT_MUTEXES
9791029 	/* PI waiters blocked on a rt_mutex held by this task: */
....@@ -983,29 +1033,23 @@
9831033 	/* Deadlock detection and priority inheritance handling: */
9841034 	struct rt_mutex_waiter		*pi_blocked_on;
9851035 #endif
986
-#ifdef CONFIG_MM_EVENT_STAT
987
-	struct mm_event_task	mm_event[MM_TYPE_NUM];
988
-	unsigned long		next_period;
989
-#endif
1036
+
9901037 #ifdef CONFIG_DEBUG_MUTEXES
9911038 	/* Mutex deadlock detection: */
9921039 	struct mutex_waiter		*blocked_on;
9931040 #endif
9941041 
1042
+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
1043
+	int				non_block_count;
1044
+#endif
1045
+
9951046 #ifdef CONFIG_TRACE_IRQFLAGS
996
-	unsigned int			irq_events;
997
-	unsigned long			hardirq_enable_ip;
998
-	unsigned long			hardirq_disable_ip;
999
-	unsigned int			hardirq_enable_event;
1000
-	unsigned int			hardirq_disable_event;
1001
-	int				hardirqs_enabled;
1002
-	int				hardirq_context;
1003
-	unsigned long			softirq_disable_ip;
1004
-	unsigned long			softirq_enable_ip;
1005
-	unsigned int			softirq_disable_event;
1006
-	unsigned int			softirq_enable_event;
1047
+	struct irqtrace_events		irqtrace;
1048
+	unsigned int			hardirq_threaded;
1049
+	u64				hardirq_chain_key;
10071050 	int				softirqs_enabled;
10081051 	int				softirq_context;
1052
+	int				irq_config;
10091053 #endif
10101054 
10111055 #ifdef CONFIG_LOCKDEP
....@@ -1016,7 +1060,7 @@
10161060 	struct held_lock		held_locks[MAX_LOCK_DEPTH];
10171061 #endif
10181062 
1019
-#ifdef CONFIG_UBSAN
1063
+#if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
10201064 	unsigned int			in_ubsan;
10211065 #endif
10221066 
....@@ -1038,9 +1082,12 @@
10381082 
10391083 	struct io_context		*io_context;
10401084 
1085
+#ifdef CONFIG_COMPACTION
1086
+	struct capture_control		*capture_control;
1087
+#endif
10411088 	/* Ptrace state: */
10421089 	unsigned long			ptrace_message;
1043
-	siginfo_t			*last_siginfo;
1090
+	kernel_siginfo_t		*last_siginfo;
10441091 
10451092 	struct task_io_accounting	ioac;
10461093 #ifdef CONFIG_PSI
....@@ -1059,7 +1106,7 @@
10591106 	/* Protected by ->alloc_lock: */
10601107 	nodemask_t			mems_allowed;
10611108 	/* Seqence number to catch updates: */
1062
-	seqcount_t			mems_allowed_seq;
1109
+	seqcount_spinlock_t		mems_allowed_seq;
10631110 	int				cpuset_mem_spread_rotor;
10641111 	int				cpuset_slab_spread_rotor;
10651112 #endif
....@@ -1069,7 +1116,7 @@
10691116 	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
10701117 	struct list_head		cg_list;
10711118 #endif
1072
-#ifdef CONFIG_INTEL_RDT
1119
+#ifdef CONFIG_X86_CPU_RESCTRL
10731120 	u32				closid;
10741121 	u32				rmid;
10751122 #endif
....@@ -1080,6 +1127,8 @@
10801127 #endif
10811128 	struct list_head		pi_state_list;
10821129 	struct futex_pi_state		*pi_state_cache;
1130
+	struct mutex			futex_exit_mutex;
1131
+	unsigned int			futex_state;
10831132 #endif
10841133 #ifdef CONFIG_PERF_EVENTS
10851134 	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
....@@ -1147,7 +1196,6 @@
11471196 
11481197 #ifdef CONFIG_RSEQ
11491198 	struct rseq __user *rseq;
1150
-	u32 rseq_len;
11511199 	u32 rseq_sig;
11521200 	/*
11531201 	 * RmW on rseq_event_mask must be performed atomically
....@@ -1158,7 +1206,10 @@
11581206 
11591207 	struct tlbflush_unmap_batch	tlb_ubc;
11601208 
1161
-	struct rcu_head			rcu;
1209
+	union {
1210
+		refcount_t		rcu_users;
1211
+		struct rcu_head		rcu;
1212
+	};
11621213 
11631214 	/* Cache last used pipe for splice(): */
11641215 	struct pipe_inode_info		*splice_pipe;
....@@ -1193,8 +1244,19 @@
11931244 	u64				timer_slack_ns;
11941245 	u64				default_timer_slack_ns;
11951246 
1196
-#ifdef CONFIG_KASAN
1247
+#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
11971248 	unsigned int			kasan_depth;
1249
+#endif
1250
+
1251
+#ifdef CONFIG_KCSAN
1252
+	struct kcsan_ctx		kcsan_ctx;
1253
+#ifdef CONFIG_TRACE_IRQFLAGS
1254
+	struct irqtrace_events		kcsan_save_irqtrace;
1255
+#endif
1256
+#endif
1257
+
1258
+#if IS_ENABLED(CONFIG_KUNIT)
1259
+	struct kunit			*kunit_test;
11981260 #endif
11991261 
12001262 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
....@@ -1246,6 +1308,9 @@
12461308 
12471309 	/* KCOV sequence number: */
12481310 	int				kcov_sequence;
1311
+
1312
+	/* Collect coverage from softirq context: */
1313
+	unsigned int			kcov_softirq;
12491314 #endif
12501315 
12511316 #ifdef CONFIG_MEMCG
....@@ -1283,7 +1348,7 @@
12831348 #endif
12841349 #ifdef CONFIG_THREAD_INFO_IN_TASK
12851350 	/* A live task holds one reference: */
1286
-	atomic_t			stack_refcount;
1351
+	refcount_t			stack_refcount;
12871352 #endif
12881353 #ifdef CONFIG_LIVEPATCH
12891354 	int patch_state;
....@@ -1292,27 +1357,33 @@
12921357 	/* Used by LSM modules for access restriction: */
12931358 	void				*security;
12941359 #endif
1295
-	/* task is frozen/stopped (used by the cgroup freezer) */
1296
-	ANDROID_KABI_USE(1, unsigned frozen:1);
12971360 
1298
-	/* 095444fad7e3 ("futex: Replace PF_EXITPIDONE with a state") */
1299
-	ANDROID_KABI_USE(2, unsigned int futex_state);
1361
+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1362
+	unsigned long			lowest_stack;
1363
+	unsigned long			prev_lowest_stack;
1364
+#endif
13001365 
1301
-	/*
1302
-	 * f9b0c6c556db ("futex: Add mutex around futex exit")
1303
-	 * A struct mutex takes 32 bytes, or 4 64bit entries, so pick off
1304
-	 * 4 of the reserved members, and replace them with a struct mutex.
1305
-	 * Do the GENKSYMS hack to work around the CRC issues
1306
-	 */
1307
-#ifdef __GENKSYMS__
1366
+#ifdef CONFIG_X86_MCE
1367
+	void __user			*mce_vaddr;
1368
+	__u64				mce_kflags;
1369
+	u64				mce_addr;
1370
+	__u64				mce_ripv : 1,
1371
+					mce_whole_page : 1,
1372
+					__mce_reserved : 62;
1373
+	struct callback_head		mce_kill_me;
1374
+	int				mce_count;
1375
+#endif
1376
+	ANDROID_VENDOR_DATA_ARRAY(1, 64);
1377
+	ANDROID_OEM_DATA_ARRAY(1, 32);
1378
+
1379
+	/* PF_IO_WORKER */
1380
+	ANDROID_KABI_USE(1, void *pf_io_worker);
1381
+
1382
+	ANDROID_KABI_RESERVE(2);
13081383 	ANDROID_KABI_RESERVE(3);
13091384 	ANDROID_KABI_RESERVE(4);
13101385 	ANDROID_KABI_RESERVE(5);
13111386 	ANDROID_KABI_RESERVE(6);
1312
-#else
1313
-	struct mutex			futex_exit_mutex;
1314
-#endif
1315
-
13161387 	ANDROID_KABI_RESERVE(7);
13171388 	ANDROID_KABI_RESERVE(8);
13181389 
....@@ -1490,9 +1561,10 @@
14901561 /*
14911562  * Per process flags
14921563  */
1564
+#define PF_VCPU			0x00000001	/* I'm a virtual CPU */
14931565 #define PF_IDLE			0x00000002	/* I am an IDLE thread */
14941566 #define PF_EXITING		0x00000004	/* Getting shut down */
1495
-#define PF_VCPU			0x00000010	/* I'm a virtual CPU */
1567
+#define PF_IO_WORKER		0x00000010	/* Task is an IO worker */
14961568 #define PF_WQ_WORKER		0x00000020	/* I'm a workqueue worker */
14971569 #define PF_FORKNOEXEC		0x00000040	/* Forked but didn't exec */
14981570 #define PF_MCE_PROCESS		0x00000080      /* Process policy on mce errors */
....@@ -1507,14 +1579,14 @@
15071579 #define PF_KSWAPD		0x00020000	/* I am kswapd */
15081580 #define PF_MEMALLOC_NOFS	0x00040000	/* All allocation requests will inherit GFP_NOFS */
15091581 #define PF_MEMALLOC_NOIO	0x00080000	/* All allocation requests will inherit GFP_NOIO */
1510
-#define PF_LESS_THROTTLE	0x00100000	/* Throttle me less: I clean memory */
1582
+#define PF_LOCAL_THROTTLE	0x00100000	/* Throttle writes only against the bdi I write to,
1583
+						 * I am cleaning dirty pages from some other bdi. */
15111584 #define PF_KTHREAD		0x00200000	/* I am a kernel thread */
15121585 #define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
15131586 #define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */
1514
-#define PF_MEMSTALL		0x01000000	/* Stalled due to lack of memory */
1515
-#define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_allowed */
1587
+#define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
15161588 #define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */
1517
-#define PF_MUTEX_TESTER		0x20000000	/* Thread belongs to the rt mutex tester */
1589
+#define PF_MEMALLOC_NOCMA	0x10000000	/* All allocation request will have _GFP_MOVABLE cleared */
15181590 #define PF_FREEZER_SKIP		0x40000000	/* Freezer should not count it as freezable */
15191591 #define PF_SUSPEND_TASK		0x80000000      /* This thread called freeze_processes() and should not be frozen */
15201592 
....@@ -1564,6 +1636,7 @@
15641636 #define PFA_SPEC_SSB_FORCE_DISABLE	4	/* Speculative Store Bypass force disabled*/
15651637 #define PFA_SPEC_IB_DISABLE		5	/* Indirect branch speculation restricted */
15661638 #define PFA_SPEC_IB_FORCE_DISABLE	6	/* Indirect branch speculation permanently restricted */
1639
+#define PFA_SPEC_SSB_NOEXEC		7	/* Speculative Store Bypass clear on execve() */
15671640 
15681641 #define TASK_PFA_TEST(name, func)					\
15691642 	static inline bool task_##func(struct task_struct *p)		\
....@@ -1592,6 +1665,10 @@
15921665 TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
15931666 TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
15941667 
1668
+TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1669
+TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1670
+TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
1671
+
15951672 TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
15961673 TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
15971674 
....@@ -1610,10 +1687,21 @@
16101687 }
16111688 
16121689 extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
1613
-extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed);
1690
+extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_effective_cpus);
1691
+
1692
+#ifdef CONFIG_RT_SOFTINT_OPTIMIZATION
1693
+extern bool cpupri_check_rt(void);
1694
+#else
1695
+static inline bool cpupri_check_rt(void)
1696
+{
1697
+	return false;
1698
+}
1699
+#endif
1700
+
16141701 #ifdef CONFIG_SMP
16151702 extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
16161703 extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
1704
+extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
16171705 #else
16181706 static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
16191707 {
....@@ -1624,10 +1712,6 @@
16241712 		return -EINVAL;
16251713 	return 0;
16261714 }
1627
-#endif
1628
-
1629
-#ifndef cpu_relax_yield
1630
-#define cpu_relax_yield() cpu_relax()
16311715 #endif
16321716 
16331717 extern int yield_to(struct task_struct *p, bool preempt);
....@@ -1651,6 +1735,9 @@
16511735 extern int available_idle_cpu(int cpu);
16521736 extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
16531737 extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
1738
+extern void sched_set_fifo(struct task_struct *p);
1739
+extern void sched_set_fifo_low(struct task_struct *p);
1740
+extern void sched_set_normal(struct task_struct *p, int nice);
16541741 extern int sched_setattr(struct task_struct *, const struct sched_attr *);
16551742 extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
16561743 extern struct task_struct *idle_task(int cpu);
....@@ -1661,7 +1748,7 @@
16611748  *
16621749  * Return: 1 if @p is an idle task. 0 otherwise.
16631750  */
1664
-static inline bool is_idle_task(const struct task_struct *p)
1751
+static __always_inline bool is_idle_task(const struct task_struct *p)
16651752 {
16661753 	return !!(p->flags & PF_IDLE);
16671754 }
....@@ -1739,7 +1826,15 @@
17391826 })
17401827 
17411828 #ifdef CONFIG_SMP
1742
-void scheduler_ipi(void);
1829
+static __always_inline void scheduler_ipi(void)
1830
+{
1831
+	/*
1832
+	 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1833
+	 * TIF_NEED_RESCHED remotely (for the first time) will also send
1834
+	 * this IPI.
1835
+	 */
1836
+	preempt_fold_need_resched();
1837
+}
17431838 extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
17441839 #else
17451840 static inline void scheduler_ipi(void) { }
....@@ -1805,7 +1900,7 @@
18051900  * value indicates whether a reschedule was done in fact.
18061901  * cond_resched_lock() will drop the spinlock before scheduling,
18071902  */
1808
-#ifndef CONFIG_PREEMPT
1903
+#ifndef CONFIG_PREEMPTION
18091904 extern int _cond_resched(void);
18101905 #else
18111906 static inline int _cond_resched(void) { return 0; }
....@@ -1834,12 +1929,12 @@
18341929 
18351930 /*
18361931  * Does a critical section need to be broken due to another
1837
- * task waiting?: (technically does not depend on CONFIG_PREEMPT,
1932
+ * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
18381933  * but a general need for low latency)
18391934  */
18401935 static inline int spin_needbreak(spinlock_t *lock)
18411936 {
1842
-#ifdef CONFIG_PREEMPT
1937
+#ifdef CONFIG_PREEMPTION
18431938 	return spin_is_contended(lock);
18441939 #else
18451940 	return 0;
....@@ -1889,7 +1984,10 @@
18891984  * running or not.
18901985  */
18911986 #ifndef vcpu_is_preempted
1892
-# define vcpu_is_preempted(cpu)	false
1987
+static inline bool vcpu_is_preempted(int cpu)
1988
+{
1989
+	return false;
1990
+}
18931991 #endif
18941992 
18951993 extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
....@@ -1963,12 +2061,10 @@
19632061 {
19642062 	if (clone_flags & CLONE_VM) {
19652063 		t->rseq = NULL;
1966
-		t->rseq_len = 0;
19672064 		t->rseq_sig = 0;
19682065 		t->rseq_event_mask = 0;
19692066 	} else {
19702067 		t->rseq = current->rseq;
1971
-		t->rseq_len = current->rseq_len;
19722068 		t->rseq_sig = current->rseq_sig;
19732069 		t->rseq_event_mask = current->rseq_event_mask;
19742070 	}
....@@ -1977,7 +2073,6 @@
19772073 static inline void rseq_execve(struct task_struct *t)
19782074 {
19792075 	t->rseq = NULL;
1980
-	t->rseq_len = 0;
19812076 	t->rseq_sig = 0;
19822077 	t->rseq_event_mask = 0;
19832078 }
....@@ -2022,4 +2117,18 @@
20222117 
20232118 #endif
20242119 
2120
+const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq);
2121
+char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len);
2122
+int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq);
2123
+
2124
+const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq);
2125
+const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq);
2126
+const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
2127
+
2128
+int sched_trace_rq_cpu(struct rq *rq);
2129
+int sched_trace_rq_cpu_capacity(struct rq *rq);
2130
+int sched_trace_rq_nr_running(struct rq *rq);
2131
+
2132
+const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
2133
+
20252134 #endif