change system file

hc

2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771

 kernel/mm/percpu-internal.h
..
..
@@ -6,11 +6,39 @@
6
6
 #include <linux/percpu.h>
7
7
 
8
8
 /*
9
+ * There are two chunk types: root and memcg-aware.
10
+ * Chunks of each type have separate slots list.
11
+ *
12
+ * Memcg-aware chunks have an attached vector of obj_cgroup pointers, which is
13
+ * used to store memcg membership data of a percpu object.  Obj_cgroups are
14
+ * ref-counted pointers to a memory cgroup with an ability to switch dynamically
15
+ * to the parent memory cgroup.  This allows to reclaim a deleted memory cgroup
16
+ * without reclaiming of all outstanding objects, which hold a reference at it.
17
+ */
18
+enum pcpu_chunk_type {
19
+	PCPU_CHUNK_ROOT,
20
+#ifdef CONFIG_MEMCG_KMEM
21
+	PCPU_CHUNK_MEMCG,
22
+#endif
23
+	PCPU_NR_CHUNK_TYPES,
24
+	PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES
25
+};
26
+
27
+/*
9
28
  * pcpu_block_md is the metadata block struct.
10
29
  * Each chunk's bitmap is split into a number of full blocks.
11
30
  * All units are in terms of bits.
31
+ *
32
+ * The scan hint is the largest known contiguous area before the contig hint.
33
+ * It is not necessarily the actual largest contig hint though.  There is an
34
+ * invariant that the scan_hint_start > contig_hint_start iff
35
+ * scan_hint == contig_hint.  This is necessary because when scanning forward,
36
+ * we don't know if a new contig hint would be better than the current one.
12
37
  */
13
38
 struct pcpu_block_md {
39
+	int			scan_hint;	/* scan hint for block */
40
+	int			scan_hint_start; /* block relative starting
41
+						    position of the scan hint */
14
42
 	int                     contig_hint;    /* contig hint for block */
15
43
 	int                     contig_hint_start; /* block relative starting
16
44
 						      position of the contig hint */
..
..
@@ -19,6 +47,7 @@
19
47
 	int                     right_free;     /* size of free space along
20
48
 						   the right side of the block */
21
49
 	int                     first_free;     /* block position of first free */
50
+	int			nr_bits;	/* total bits responsible for */
22
51
 };
23
52
 
24
53
 struct pcpu_chunk {
..
..
@@ -29,9 +58,7 @@
29
58
 
30
59
 	struct list_head	list;		/* linked to pcpu_slot lists */
31
60
 	int			free_bytes;	/* free bytes in the chunk */
32
-	int			contig_bits;	/* max contiguous size hint */
33
-	int			contig_bits_start; /* contig_bits starting
34
-						      offset */
61
+	struct pcpu_block_md	chunk_md;
35
62
 	void			*base_addr;	/* base address of this chunk */
36
63
 
37
64
 	unsigned long		*alloc_map;	/* allocation map */
..
..
@@ -39,7 +66,6 @@
39
66
 	struct pcpu_block_md	*md_blocks;	/* metadata blocks */
40
67
 
41
68
 	void			*data;		/* chunk data */
42
-	int			first_bit;	/* no free below this */
43
69
 	bool			immutable;	/* no [de]population allowed */
44
70
 	int			start_offset;	/* the overlap with the previous
45
71
 						   region to have a page aligned
..
..
@@ -47,6 +73,9 @@
47
73
 	int			end_offset;	/* additional area required to
48
74
 						   have the region end page
49
75
 						   aligned */
76
+#ifdef CONFIG_MEMCG_KMEM
77
+	struct obj_cgroup	**obj_cgroups;	/* vector of object cgroups */
78
+#endif
50
79
 
51
80
 	int			nr_pages;	/* # of pages served by this chunk */
52
81
 	int			nr_populated;	/* # of populated pages */
..
..
@@ -56,9 +85,9 @@
56
85
 
57
86
 extern spinlock_t pcpu_lock;
58
87
 
59
-extern struct list_head *pcpu_slot;
88
+extern struct list_head *pcpu_chunk_lists;
60
89
 extern int pcpu_nr_slots;
61
-extern int pcpu_nr_empty_pop_pages;
90
+extern int pcpu_nr_empty_pop_pages[];
62
91
 
63
92
 extern struct pcpu_chunk *pcpu_first_chunk;
64
93
 extern struct pcpu_chunk *pcpu_reserved_chunk;
..
..
@@ -99,6 +128,37 @@
99
128
 	return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
100
129
 }
101
130
 
131
+#ifdef CONFIG_MEMCG_KMEM
132
+static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
133
+{
134
+	if (chunk->obj_cgroups)
135
+		return PCPU_CHUNK_MEMCG;
136
+	return PCPU_CHUNK_ROOT;
137
+}
138
+
139
+static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
140
+{
141
+	return chunk_type == PCPU_CHUNK_MEMCG;
142
+}
143
+
144
+#else
145
+static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
146
+{
147
+	return PCPU_CHUNK_ROOT;
148
+}
149
+
150
+static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
151
+{
152
+	return false;
153
+}
154
+#endif
155
+
156
+static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type)
157
+{
158
+	return &pcpu_chunk_lists[pcpu_nr_slots *
159
+				 pcpu_is_memcg_chunk(chunk_type)];
160
+}
161
+
102
162
 #ifdef CONFIG_PERCPU_STATS
103
163
 
104
164
 #include <linux/spinlock.h>