.. | .. |
---|
18 | 18 | #include <linux/mmdebug.h> |
---|
19 | 19 | |
---|
20 | 20 | struct hugetlb_cgroup; |
---|
| 21 | +struct resv_map; |
---|
| 22 | +struct file_region; |
---|
| 23 | + |
---|
21 | 24 | /* |
---|
22 | 25 | * Minimum page order trackable by hugetlb cgroup. |
---|
23 | | - * At least 3 pages are necessary for all the tracking information. |
---|
| 26 | + * At least 4 pages are necessary for all the tracking information. |
---|
| 27 | + * The second tail page (hpage[2]) is the fault usage cgroup. |
---|
| 28 | + * The third tail page (hpage[3]) is the reservation usage cgroup. |
---|
24 | 29 | */ |
---|
25 | 30 | #define HUGETLB_CGROUP_MIN_ORDER 2 |
---|
26 | 31 | |
---|
27 | 32 | #ifdef CONFIG_CGROUP_HUGETLB |
---|
| 33 | +enum hugetlb_memory_event { |
---|
| 34 | + HUGETLB_MAX, |
---|
| 35 | + HUGETLB_NR_MEMORY_EVENTS, |
---|
| 36 | +}; |
---|
28 | 37 | |
---|
29 | | -static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) |
---|
| 38 | +struct hugetlb_cgroup { |
---|
| 39 | + struct cgroup_subsys_state css; |
---|
| 40 | + |
---|
| 41 | + /* |
---|
| 42 | + * the counter to account for hugepages from hugetlb. |
---|
| 43 | + */ |
---|
| 44 | + struct page_counter hugepage[HUGE_MAX_HSTATE]; |
---|
| 45 | + |
---|
| 46 | + /* |
---|
| 47 | + * the counter to account for hugepage reservations from hugetlb. |
---|
| 48 | + */ |
---|
| 49 | + struct page_counter rsvd_hugepage[HUGE_MAX_HSTATE]; |
---|
| 50 | + |
---|
| 51 | + atomic_long_t events[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; |
---|
| 52 | + atomic_long_t events_local[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; |
---|
| 53 | + |
---|
| 54 | + /* Handle for "hugetlb.events" */ |
---|
| 55 | + struct cgroup_file events_file[HUGE_MAX_HSTATE]; |
---|
| 56 | + |
---|
| 57 | + /* Handle for "hugetlb.events.local" */ |
---|
| 58 | + struct cgroup_file events_local_file[HUGE_MAX_HSTATE]; |
---|
| 59 | +}; |
---|
| 60 | + |
---|
| 61 | +static inline struct hugetlb_cgroup * |
---|
| 62 | +__hugetlb_cgroup_from_page(struct page *page, bool rsvd) |
---|
30 | 63 | { |
---|
31 | 64 | VM_BUG_ON_PAGE(!PageHuge(page), page); |
---|
32 | 65 | |
---|
33 | 66 | if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) |
---|
34 | 67 | return NULL; |
---|
35 | | - return (struct hugetlb_cgroup *)page[2].private; |
---|
| 68 | + if (rsvd) |
---|
| 69 | + return (struct hugetlb_cgroup *)page[3].private; |
---|
| 70 | + else |
---|
| 71 | + return (struct hugetlb_cgroup *)page[2].private; |
---|
36 | 72 | } |
---|
37 | 73 | |
---|
38 | | -static inline |
---|
39 | | -int set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg) |
---|
| 74 | +static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) |
---|
| 75 | +{ |
---|
| 76 | + return __hugetlb_cgroup_from_page(page, false); |
---|
| 77 | +} |
---|
| 78 | + |
---|
| 79 | +static inline struct hugetlb_cgroup * |
---|
| 80 | +hugetlb_cgroup_from_page_rsvd(struct page *page) |
---|
| 81 | +{ |
---|
| 82 | + return __hugetlb_cgroup_from_page(page, true); |
---|
| 83 | +} |
---|
| 84 | + |
---|
| 85 | +static inline int __set_hugetlb_cgroup(struct page *page, |
---|
| 86 | + struct hugetlb_cgroup *h_cg, bool rsvd) |
---|
40 | 87 | { |
---|
41 | 88 | VM_BUG_ON_PAGE(!PageHuge(page), page); |
---|
42 | 89 | |
---|
43 | 90 | if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) |
---|
44 | 91 | return -1; |
---|
45 | | - page[2].private = (unsigned long)h_cg; |
---|
| 92 | + if (rsvd) |
---|
| 93 | + page[3].private = (unsigned long)h_cg; |
---|
| 94 | + else |
---|
| 95 | + page[2].private = (unsigned long)h_cg; |
---|
46 | 96 | return 0; |
---|
| 97 | +} |
---|
| 98 | + |
---|
| 99 | +static inline int set_hugetlb_cgroup(struct page *page, |
---|
| 100 | + struct hugetlb_cgroup *h_cg) |
---|
| 101 | +{ |
---|
| 102 | + return __set_hugetlb_cgroup(page, h_cg, false); |
---|
| 103 | +} |
---|
| 104 | + |
---|
| 105 | +static inline int set_hugetlb_cgroup_rsvd(struct page *page, |
---|
| 106 | + struct hugetlb_cgroup *h_cg) |
---|
| 107 | +{ |
---|
| 108 | + return __set_hugetlb_cgroup(page, h_cg, true); |
---|
47 | 109 | } |
---|
48 | 110 | |
---|
49 | 111 | static inline bool hugetlb_cgroup_disabled(void) |
---|
.. | .. |
---|
51 | 113 | return !cgroup_subsys_enabled(hugetlb_cgrp_subsys); |
---|
52 | 114 | } |
---|
53 | 115 | |
---|
| 116 | +static inline void hugetlb_cgroup_put_rsvd_cgroup(struct hugetlb_cgroup *h_cg) |
---|
| 117 | +{ |
---|
| 118 | + css_put(&h_cg->css); |
---|
| 119 | +} |
---|
| 120 | + |
---|
| 121 | +static inline void resv_map_dup_hugetlb_cgroup_uncharge_info( |
---|
| 122 | + struct resv_map *resv_map) |
---|
| 123 | +{ |
---|
| 124 | + if (resv_map->css) |
---|
| 125 | + css_get(resv_map->css); |
---|
| 126 | +} |
---|
| 127 | + |
---|
54 | 128 | extern int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, |
---|
55 | 129 | struct hugetlb_cgroup **ptr); |
---|
| 130 | +extern int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages, |
---|
| 131 | + struct hugetlb_cgroup **ptr); |
---|
56 | 132 | extern void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, |
---|
57 | 133 | struct hugetlb_cgroup *h_cg, |
---|
58 | 134 | struct page *page); |
---|
| 135 | +extern void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, |
---|
| 136 | + struct hugetlb_cgroup *h_cg, |
---|
| 137 | + struct page *page); |
---|
59 | 138 | extern void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, |
---|
60 | 139 | struct page *page); |
---|
| 140 | +extern void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages, |
---|
| 141 | + struct page *page); |
---|
| 142 | + |
---|
61 | 143 | extern void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, |
---|
62 | 144 | struct hugetlb_cgroup *h_cg); |
---|
| 145 | +extern void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, |
---|
| 146 | + struct hugetlb_cgroup *h_cg); |
---|
| 147 | +extern void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, |
---|
| 148 | + unsigned long start, |
---|
| 149 | + unsigned long end); |
---|
| 150 | + |
---|
| 151 | +extern void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, |
---|
| 152 | + struct file_region *rg, |
---|
| 153 | + unsigned long nr_pages, |
---|
| 154 | + bool region_del); |
---|
| 155 | + |
---|
63 | 156 | extern void hugetlb_cgroup_file_init(void) __init; |
---|
64 | 157 | extern void hugetlb_cgroup_migrate(struct page *oldhpage, |
---|
65 | 158 | struct page *newhpage); |
---|
66 | 159 | |
---|
67 | 160 | #else |
---|
| 161 | +static inline void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, |
---|
| 162 | + struct file_region *rg, |
---|
| 163 | + unsigned long nr_pages, |
---|
| 164 | + bool region_del) |
---|
| 165 | +{ |
---|
| 166 | +} |
---|
| 167 | + |
---|
68 | 168 | static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) |
---|
69 | 169 | { |
---|
70 | 170 | return NULL; |
---|
71 | 171 | } |
---|
72 | 172 | |
---|
73 | | -static inline |
---|
74 | | -int set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg) |
---|
| 173 | +static inline struct hugetlb_cgroup * |
---|
| 174 | +hugetlb_cgroup_from_page_resv(struct page *page) |
---|
| 175 | +{ |
---|
| 176 | + return NULL; |
---|
| 177 | +} |
---|
| 178 | + |
---|
| 179 | +static inline struct hugetlb_cgroup * |
---|
| 180 | +hugetlb_cgroup_from_page_rsvd(struct page *page) |
---|
| 181 | +{ |
---|
| 182 | + return NULL; |
---|
| 183 | +} |
---|
| 184 | + |
---|
| 185 | +static inline int set_hugetlb_cgroup(struct page *page, |
---|
| 186 | + struct hugetlb_cgroup *h_cg) |
---|
| 187 | +{ |
---|
| 188 | + return 0; |
---|
| 189 | +} |
---|
| 190 | + |
---|
| 191 | +static inline int set_hugetlb_cgroup_rsvd(struct page *page, |
---|
| 192 | + struct hugetlb_cgroup *h_cg) |
---|
75 | 193 | { |
---|
76 | 194 | return 0; |
---|
77 | 195 | } |
---|
.. | .. |
---|
81 | 199 | return true; |
---|
82 | 200 | } |
---|
83 | 201 | |
---|
84 | | -static inline int |
---|
85 | | -hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, |
---|
86 | | - struct hugetlb_cgroup **ptr) |
---|
| 202 | +static inline void hugetlb_cgroup_put_rsvd_cgroup(struct hugetlb_cgroup *h_cg) |
---|
| 203 | +{ |
---|
| 204 | +} |
---|
| 205 | + |
---|
| 206 | +static inline void resv_map_dup_hugetlb_cgroup_uncharge_info( |
---|
| 207 | + struct resv_map *resv_map) |
---|
| 208 | +{ |
---|
| 209 | +} |
---|
| 210 | + |
---|
| 211 | +static inline int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, |
---|
| 212 | + struct hugetlb_cgroup **ptr) |
---|
87 | 213 | { |
---|
88 | 214 | return 0; |
---|
89 | 215 | } |
---|
90 | 216 | |
---|
91 | | -static inline void |
---|
92 | | -hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, |
---|
93 | | - struct hugetlb_cgroup *h_cg, |
---|
94 | | - struct page *page) |
---|
| 217 | +static inline int hugetlb_cgroup_charge_cgroup_rsvd(int idx, |
---|
| 218 | + unsigned long nr_pages, |
---|
| 219 | + struct hugetlb_cgroup **ptr) |
---|
| 220 | +{ |
---|
| 221 | + return 0; |
---|
| 222 | +} |
---|
| 223 | + |
---|
| 224 | +static inline void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, |
---|
| 225 | + struct hugetlb_cgroup *h_cg, |
---|
| 226 | + struct page *page) |
---|
95 | 227 | { |
---|
96 | 228 | } |
---|
97 | 229 | |
---|
98 | 230 | static inline void |
---|
99 | | -hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page) |
---|
| 231 | +hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, |
---|
| 232 | + struct hugetlb_cgroup *h_cg, |
---|
| 233 | + struct page *page) |
---|
| 234 | +{ |
---|
| 235 | +} |
---|
| 236 | + |
---|
| 237 | +static inline void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, |
---|
| 238 | + struct page *page) |
---|
| 239 | +{ |
---|
| 240 | +} |
---|
| 241 | + |
---|
| 242 | +static inline void hugetlb_cgroup_uncharge_page_rsvd(int idx, |
---|
| 243 | + unsigned long nr_pages, |
---|
| 244 | + struct page *page) |
---|
| 245 | +{ |
---|
| 246 | +} |
---|
| 247 | +static inline void hugetlb_cgroup_uncharge_cgroup(int idx, |
---|
| 248 | + unsigned long nr_pages, |
---|
| 249 | + struct hugetlb_cgroup *h_cg) |
---|
100 | 250 | { |
---|
101 | 251 | } |
---|
102 | 252 | |
---|
103 | 253 | static inline void |
---|
104 | | -hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, |
---|
105 | | - struct hugetlb_cgroup *h_cg) |
---|
| 254 | +hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, |
---|
| 255 | + struct hugetlb_cgroup *h_cg) |
---|
| 256 | +{ |
---|
| 257 | +} |
---|
| 258 | + |
---|
| 259 | +static inline void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, |
---|
| 260 | + unsigned long start, |
---|
| 261 | + unsigned long end) |
---|
106 | 262 | { |
---|
107 | 263 | } |
---|
108 | 264 | |
---|