hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb
kernel/include/linux/hugetlb_cgroup.h
....@@ -18,32 +18,94 @@
1818 #include <linux/mmdebug.h>
1919
2020 struct hugetlb_cgroup;
21
+struct resv_map;
22
+struct file_region;
23
+
2124 /*
2225 * Minimum page order trackable by hugetlb cgroup.
23
- * At least 3 pages are necessary for all the tracking information.
26
+ * At least 4 pages are necessary for all the tracking information.
27
+ * The second tail page (hpage[2]) is the fault usage cgroup.
28
+ * The third tail page (hpage[3]) is the reservation usage cgroup.
2429 */
2530 #define HUGETLB_CGROUP_MIN_ORDER 2
2631
2732 #ifdef CONFIG_CGROUP_HUGETLB
33
+enum hugetlb_memory_event {
34
+ HUGETLB_MAX,
35
+ HUGETLB_NR_MEMORY_EVENTS,
36
+};
2837
29
-static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page)
38
+struct hugetlb_cgroup {
39
+ struct cgroup_subsys_state css;
40
+
41
+ /*
42
+ * the counter to account for hugepages from hugetlb.
43
+ */
44
+ struct page_counter hugepage[HUGE_MAX_HSTATE];
45
+
46
+ /*
47
+ * the counter to account for hugepage reservations from hugetlb.
48
+ */
49
+ struct page_counter rsvd_hugepage[HUGE_MAX_HSTATE];
50
+
51
+ atomic_long_t events[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS];
52
+ atomic_long_t events_local[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS];
53
+
54
+ /* Handle for "hugetlb.events" */
55
+ struct cgroup_file events_file[HUGE_MAX_HSTATE];
56
+
57
+ /* Handle for "hugetlb.events.local" */
58
+ struct cgroup_file events_local_file[HUGE_MAX_HSTATE];
59
+};
60
+
61
+static inline struct hugetlb_cgroup *
62
+__hugetlb_cgroup_from_page(struct page *page, bool rsvd)
3063 {
3164 VM_BUG_ON_PAGE(!PageHuge(page), page);
3265
3366 if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER)
3467 return NULL;
35
- return (struct hugetlb_cgroup *)page[2].private;
68
+ if (rsvd)
69
+ return (struct hugetlb_cgroup *)page[3].private;
70
+ else
71
+ return (struct hugetlb_cgroup *)page[2].private;
3672 }
3773
38
-static inline
39
-int set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg)
74
+static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page)
75
+{
76
+ return __hugetlb_cgroup_from_page(page, false);
77
+}
78
+
79
+static inline struct hugetlb_cgroup *
80
+hugetlb_cgroup_from_page_rsvd(struct page *page)
81
+{
82
+ return __hugetlb_cgroup_from_page(page, true);
83
+}
84
+
85
+static inline int __set_hugetlb_cgroup(struct page *page,
86
+ struct hugetlb_cgroup *h_cg, bool rsvd)
4087 {
4188 VM_BUG_ON_PAGE(!PageHuge(page), page);
4289
4390 if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER)
4491 return -1;
45
- page[2].private = (unsigned long)h_cg;
92
+ if (rsvd)
93
+ page[3].private = (unsigned long)h_cg;
94
+ else
95
+ page[2].private = (unsigned long)h_cg;
4696 return 0;
97
+}
98
+
99
+static inline int set_hugetlb_cgroup(struct page *page,
100
+ struct hugetlb_cgroup *h_cg)
101
+{
102
+ return __set_hugetlb_cgroup(page, h_cg, false);
103
+}
104
+
105
+static inline int set_hugetlb_cgroup_rsvd(struct page *page,
106
+ struct hugetlb_cgroup *h_cg)
107
+{
108
+ return __set_hugetlb_cgroup(page, h_cg, true);
47109 }
48110
49111 static inline bool hugetlb_cgroup_disabled(void)
....@@ -51,27 +113,83 @@
51113 return !cgroup_subsys_enabled(hugetlb_cgrp_subsys);
52114 }
53115
116
+static inline void hugetlb_cgroup_put_rsvd_cgroup(struct hugetlb_cgroup *h_cg)
117
+{
118
+ css_put(&h_cg->css);
119
+}
120
+
121
+static inline void resv_map_dup_hugetlb_cgroup_uncharge_info(
122
+ struct resv_map *resv_map)
123
+{
124
+ if (resv_map->css)
125
+ css_get(resv_map->css);
126
+}
127
+
54128 extern int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
55129 struct hugetlb_cgroup **ptr);
130
+extern int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages,
131
+ struct hugetlb_cgroup **ptr);
56132 extern void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
57133 struct hugetlb_cgroup *h_cg,
58134 struct page *page);
135
+extern void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
136
+ struct hugetlb_cgroup *h_cg,
137
+ struct page *page);
59138 extern void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
60139 struct page *page);
140
+extern void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages,
141
+ struct page *page);
142
+
61143 extern void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
62144 struct hugetlb_cgroup *h_cg);
145
+extern void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages,
146
+ struct hugetlb_cgroup *h_cg);
147
+extern void hugetlb_cgroup_uncharge_counter(struct resv_map *resv,
148
+ unsigned long start,
149
+ unsigned long end);
150
+
151
+extern void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
152
+ struct file_region *rg,
153
+ unsigned long nr_pages,
154
+ bool region_del);
155
+
63156 extern void hugetlb_cgroup_file_init(void) __init;
64157 extern void hugetlb_cgroup_migrate(struct page *oldhpage,
65158 struct page *newhpage);
66159
67160 #else
161
+static inline void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
162
+ struct file_region *rg,
163
+ unsigned long nr_pages,
164
+ bool region_del)
165
+{
166
+}
167
+
68168 static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page)
69169 {
70170 return NULL;
71171 }
72172
73
-static inline
74
-int set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg)
173
+static inline struct hugetlb_cgroup *
174
+hugetlb_cgroup_from_page_resv(struct page *page)
175
+{
176
+ return NULL;
177
+}
178
+
179
+static inline struct hugetlb_cgroup *
180
+hugetlb_cgroup_from_page_rsvd(struct page *page)
181
+{
182
+ return NULL;
183
+}
184
+
185
+static inline int set_hugetlb_cgroup(struct page *page,
186
+ struct hugetlb_cgroup *h_cg)
187
+{
188
+ return 0;
189
+}
190
+
191
+static inline int set_hugetlb_cgroup_rsvd(struct page *page,
192
+ struct hugetlb_cgroup *h_cg)
75193 {
76194 return 0;
77195 }
....@@ -81,28 +199,66 @@
81199 return true;
82200 }
83201
84
-static inline int
85
-hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
86
- struct hugetlb_cgroup **ptr)
202
+static inline void hugetlb_cgroup_put_rsvd_cgroup(struct hugetlb_cgroup *h_cg)
203
+{
204
+}
205
+
206
+static inline void resv_map_dup_hugetlb_cgroup_uncharge_info(
207
+ struct resv_map *resv_map)
208
+{
209
+}
210
+
211
+static inline int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
212
+ struct hugetlb_cgroup **ptr)
87213 {
88214 return 0;
89215 }
90216
91
-static inline void
92
-hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
93
- struct hugetlb_cgroup *h_cg,
94
- struct page *page)
217
+static inline int hugetlb_cgroup_charge_cgroup_rsvd(int idx,
218
+ unsigned long nr_pages,
219
+ struct hugetlb_cgroup **ptr)
220
+{
221
+ return 0;
222
+}
223
+
224
+static inline void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
225
+ struct hugetlb_cgroup *h_cg,
226
+ struct page *page)
95227 {
96228 }
97229
98230 static inline void
99
-hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page)
231
+hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
232
+ struct hugetlb_cgroup *h_cg,
233
+ struct page *page)
234
+{
235
+}
236
+
237
+static inline void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
238
+ struct page *page)
239
+{
240
+}
241
+
242
+static inline void hugetlb_cgroup_uncharge_page_rsvd(int idx,
243
+ unsigned long nr_pages,
244
+ struct page *page)
245
+{
246
+}
247
+static inline void hugetlb_cgroup_uncharge_cgroup(int idx,
248
+ unsigned long nr_pages,
249
+ struct hugetlb_cgroup *h_cg)
100250 {
101251 }
102252
103253 static inline void
104
-hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
105
- struct hugetlb_cgroup *h_cg)
254
+hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages,
255
+ struct hugetlb_cgroup *h_cg)
256
+{
257
+}
258
+
259
+static inline void hugetlb_cgroup_uncharge_counter(struct resv_map *resv,
260
+ unsigned long start,
261
+ unsigned long end)
106262 {
107263 }
108264