.. | .. |
---|
17 | 17 | unsigned long usage) |
---|
18 | 18 | { |
---|
19 | 19 | unsigned long protected, old_protected; |
---|
| 20 | + unsigned long low, min; |
---|
20 | 21 | long delta; |
---|
21 | 22 | |
---|
22 | 23 | if (!c->parent) |
---|
23 | 24 | return; |
---|
24 | 25 | |
---|
25 | | - if (c->min || atomic_long_read(&c->min_usage)) { |
---|
26 | | - if (usage <= c->min) |
---|
27 | | - protected = usage; |
---|
28 | | - else |
---|
29 | | - protected = 0; |
---|
30 | | - |
---|
| 26 | + min = READ_ONCE(c->min); |
---|
| 27 | + if (min || atomic_long_read(&c->min_usage)) { |
---|
| 28 | + protected = min(usage, min); |
---|
31 | 29 | old_protected = atomic_long_xchg(&c->min_usage, protected); |
---|
32 | 30 | delta = protected - old_protected; |
---|
33 | 31 | if (delta) |
---|
34 | 32 | atomic_long_add(delta, &c->parent->children_min_usage); |
---|
35 | 33 | } |
---|
36 | 34 | |
---|
37 | | - if (c->low || atomic_long_read(&c->low_usage)) { |
---|
38 | | - if (usage <= c->low) |
---|
39 | | - protected = usage; |
---|
40 | | - else |
---|
41 | | - protected = 0; |
---|
42 | | - |
---|
| 35 | + low = READ_ONCE(c->low); |
---|
| 36 | + if (low || atomic_long_read(&c->low_usage)) { |
---|
| 37 | + protected = min(usage, low); |
---|
43 | 38 | old_protected = atomic_long_xchg(&c->low_usage, protected); |
---|
44 | 39 | delta = protected - old_protected; |
---|
45 | 40 | if (delta) |
---|
.. | .. |
---|
82 | 77 | * This is indeed racy, but we can live with some |
---|
83 | 78 | * inaccuracy in the watermark. |
---|
84 | 79 | */ |
---|
85 | | - if (new > c->watermark) |
---|
86 | | - c->watermark = new; |
---|
| 80 | + if (new > READ_ONCE(c->watermark)) |
---|
| 81 | + WRITE_ONCE(c->watermark, new); |
---|
87 | 82 | } |
---|
88 | 83 | } |
---|
89 | 84 | |
---|
.. | .. |
---|
114 | 109 | * |
---|
115 | 110 | * The atomic_long_add_return() implies a full memory |
---|
116 | 111 | * barrier between incrementing the count and reading |
---|
117 | | - * the limit. When racing with page_counter_limit(), |
---|
| 112 | + * the limit. When racing with page_counter_set_max(), |
---|
118 | 113 | * we either see the new limit or the setter sees the |
---|
119 | 114 | * counter has changed and retries. |
---|
120 | 115 | */ |
---|
.. | .. |
---|
124 | 119 | propagate_protected_usage(c, new); |
---|
125 | 120 | /* |
---|
126 | 121 | * This is racy, but we can live with some |
---|
127 | | - * inaccuracy in the failcnt. |
---|
| 122 | + * inaccuracy in the failcnt which is only used |
---|
| 123 | + * to report stats. |
---|
128 | 124 | */ |
---|
129 | | - c->failcnt++; |
---|
| 125 | + data_race(c->failcnt++); |
---|
130 | 126 | *fail = c; |
---|
131 | 127 | goto failed; |
---|
132 | 128 | } |
---|
.. | .. |
---|
135 | 131 | * Just like with failcnt, we can live with some |
---|
136 | 132 | * inaccuracy in the watermark. |
---|
137 | 133 | */ |
---|
138 | | - if (new > c->watermark) |
---|
139 | | - c->watermark = new; |
---|
| 134 | + if (new > READ_ONCE(c->watermark)) |
---|
| 135 | + WRITE_ONCE(c->watermark, new); |
---|
140 | 136 | } |
---|
141 | 137 | return true; |
---|
142 | 138 | |
---|
.. | .. |
---|
213 | 209 | { |
---|
214 | 210 | struct page_counter *c; |
---|
215 | 211 | |
---|
216 | | - counter->min = nr_pages; |
---|
| 212 | + WRITE_ONCE(counter->min, nr_pages); |
---|
217 | 213 | |
---|
218 | 214 | for (c = counter; c; c = c->parent) |
---|
219 | 215 | propagate_protected_usage(c, atomic_long_read(&c->usage)); |
---|
.. | .. |
---|
230 | 226 | { |
---|
231 | 227 | struct page_counter *c; |
---|
232 | 228 | |
---|
233 | | - counter->low = nr_pages; |
---|
| 229 | + WRITE_ONCE(counter->low, nr_pages); |
---|
234 | 230 | |
---|
235 | 231 | for (c = counter; c; c = c->parent) |
---|
236 | 232 | propagate_protected_usage(c, atomic_long_read(&c->usage)); |
---|