.. | .. |
---|
4 | 4 | #endif |
---|
5 | 5 | |
---|
6 | 6 | #include <linux/hash.h> |
---|
7 | | -#include <linux/bootmem.h> |
---|
| 7 | +#include <linux/memblock.h> |
---|
8 | 8 | #include <linux/debug_locks.h> |
---|
9 | 9 | |
---|
10 | 10 | /* |
---|
.. | .. |
---|
49 | 49 | |
---|
50 | 50 | struct pv_node { |
---|
51 | 51 | struct mcs_spinlock mcs; |
---|
52 | | - struct mcs_spinlock __res[3]; |
---|
53 | | - |
---|
54 | 52 | int cpu; |
---|
55 | 53 | u8 state; |
---|
56 | 54 | }; |
---|
.. | .. |
---|
91 | 89 | |
---|
92 | 90 | if (!(val & _Q_LOCKED_PENDING_MASK) && |
---|
93 | 91 | (cmpxchg_acquire(&lock->locked, 0, _Q_LOCKED_VAL) == 0)) { |
---|
94 | | - qstat_inc(qstat_pv_lock_stealing, true); |
---|
| 92 | + lockevent_inc(pv_lock_stealing); |
---|
95 | 93 | return true; |
---|
96 | 94 | } |
---|
97 | 95 | if (!(val & _Q_TAIL_MASK) || (val & _Q_PENDING_MASK)) |
---|
.. | .. |
---|
221 | 219 | hopcnt++; |
---|
222 | 220 | if (!cmpxchg(&he->lock, NULL, lock)) { |
---|
223 | 221 | WRITE_ONCE(he->node, node); |
---|
224 | | - qstat_hop(hopcnt); |
---|
| 222 | + lockevent_pv_hop(hopcnt); |
---|
225 | 223 | return &he->lock; |
---|
226 | 224 | } |
---|
227 | 225 | } |
---|
.. | .. |
---|
281 | 279 | { |
---|
282 | 280 | struct pv_node *pn = (struct pv_node *)node; |
---|
283 | 281 | |
---|
284 | | - BUILD_BUG_ON(sizeof(struct pv_node) > 5*sizeof(struct mcs_spinlock)); |
---|
| 282 | + BUILD_BUG_ON(sizeof(struct pv_node) > sizeof(struct qnode)); |
---|
285 | 283 | |
---|
286 | 284 | pn->cpu = smp_processor_id(); |
---|
287 | 285 | pn->state = vcpu_running; |
---|
.. | .. |
---|
322 | 320 | smp_store_mb(pn->state, vcpu_halted); |
---|
323 | 321 | |
---|
324 | 322 | if (!READ_ONCE(node->locked)) { |
---|
325 | | - qstat_inc(qstat_pv_wait_node, true); |
---|
326 | | - qstat_inc(qstat_pv_wait_early, wait_early); |
---|
| 323 | + lockevent_inc(pv_wait_node); |
---|
| 324 | + lockevent_cond_inc(pv_wait_early, wait_early); |
---|
327 | 325 | pv_wait(&pn->state, vcpu_halted); |
---|
328 | 326 | } |
---|
329 | 327 | |
---|
.. | .. |
---|
341 | 339 | * So it is better to spin for a while in the hope that the |
---|
342 | 340 | * MCS lock will be released soon. |
---|
343 | 341 | */ |
---|
344 | | - qstat_inc(qstat_pv_spurious_wakeup, !READ_ONCE(node->locked)); |
---|
| 342 | + lockevent_cond_inc(pv_spurious_wakeup, |
---|
| 343 | + !READ_ONCE(node->locked)); |
---|
345 | 344 | } |
---|
346 | 345 | |
---|
347 | 346 | /* |
---|
.. | .. |
---|
418 | 417 | /* |
---|
419 | 418 | * Tracking # of slowpath locking operations |
---|
420 | 419 | */ |
---|
421 | | - qstat_inc(qstat_lock_slowpath, true); |
---|
| 420 | + lockevent_inc(lock_slowpath); |
---|
422 | 421 | |
---|
423 | 422 | for (;; waitcnt++) { |
---|
424 | 423 | /* |
---|
.. | .. |
---|
466 | 465 | } |
---|
467 | 466 | } |
---|
468 | 467 | WRITE_ONCE(pn->state, vcpu_hashed); |
---|
469 | | - qstat_inc(qstat_pv_wait_head, true); |
---|
470 | | - qstat_inc(qstat_pv_wait_again, waitcnt); |
---|
| 468 | + lockevent_inc(pv_wait_head); |
---|
| 469 | + lockevent_cond_inc(pv_wait_again, waitcnt); |
---|
471 | 470 | pv_wait(&lock->locked, _Q_SLOW_VAL); |
---|
472 | 471 | |
---|
473 | 472 | /* |
---|
.. | .. |
---|
530 | 529 | * vCPU is harmless other than the additional latency in completing |
---|
531 | 530 | * the unlock. |
---|
532 | 531 | */ |
---|
533 | | - qstat_inc(qstat_pv_kick_unlock, true); |
---|
| 532 | + lockevent_inc(pv_kick_unlock); |
---|
534 | 533 | pv_kick(node->cpu); |
---|
535 | 534 | } |
---|
536 | 535 | |
---|