.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * Copyright (C) 2013 Advanced Micro Devices, Inc. |
---|
3 | 4 | * |
---|
.. | .. |
---|
5 | 6 | * Author: Suravee Suthikulpanit <Suraveee.Suthikulpanit@amd.com> |
---|
6 | 7 | * |
---|
7 | 8 | * Perf: amd_iommu - AMD IOMMU Performance Counter PMU implementation |
---|
8 | | - * |
---|
9 | | - * This program is free software; you can redistribute it and/or modify |
---|
10 | | - * it under the terms of the GNU General Public License version 2 as |
---|
11 | | - * published by the Free Software Foundation. |
---|
12 | 9 | */ |
---|
13 | 10 | |
---|
14 | 11 | #define pr_fmt(fmt) "perf/amd_iommu: " fmt |
---|
.. | .. |
---|
20 | 17 | |
---|
21 | 18 | #include "../perf_event.h" |
---|
22 | 19 | #include "iommu.h" |
---|
23 | | - |
---|
24 | | -#define COUNTER_SHIFT 16 |
---|
25 | 20 | |
---|
26 | 21 | /* iommu pmu conf masks */ |
---|
27 | 22 | #define GET_CSOURCE(x) ((x)->conf & 0xFFULL) |
---|
.. | .. |
---|
223 | 218 | if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) |
---|
224 | 219 | return -EINVAL; |
---|
225 | 220 | |
---|
226 | | - /* IOMMU counters do not have usr/os/guest/host bits */ |
---|
227 | | - if (event->attr.exclude_user || event->attr.exclude_kernel || |
---|
228 | | - event->attr.exclude_host || event->attr.exclude_guest) |
---|
229 | | - return -EINVAL; |
---|
230 | | - |
---|
231 | 221 | if (event->cpu < 0) |
---|
232 | 222 | return -EINVAL; |
---|
233 | 223 | |
---|
.. | .. |
---|
293 | 283 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
---|
294 | 284 | hwc->state = 0; |
---|
295 | 285 | |
---|
| 286 | + /* |
---|
| 287 | + * To account for power-gating, which prevents write to |
---|
| 288 | + * the counter, we need to enable the counter |
---|
| 289 | + * before setting up counter register. |
---|
| 290 | + */ |
---|
| 291 | + perf_iommu_enable_event(event); |
---|
| 292 | + |
---|
296 | 293 | if (flags & PERF_EF_RELOAD) { |
---|
297 | | - u64 prev_raw_count = local64_read(&hwc->prev_count); |
---|
| 294 | + u64 count = 0; |
---|
298 | 295 | struct amd_iommu *iommu = perf_event_2_iommu(event); |
---|
299 | 296 | |
---|
| 297 | + /* |
---|
| 298 | + * Since the IOMMU PMU only support counting mode, |
---|
| 299 | + * the counter always start with value zero. |
---|
| 300 | + */ |
---|
300 | 301 | amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr, |
---|
301 | | - IOMMU_PC_COUNTER_REG, &prev_raw_count); |
---|
| 302 | + IOMMU_PC_COUNTER_REG, &count); |
---|
302 | 303 | } |
---|
303 | 304 | |
---|
304 | | - perf_iommu_enable_event(event); |
---|
305 | 305 | perf_event_update_userpage(event); |
---|
306 | | - |
---|
307 | 306 | } |
---|
308 | 307 | |
---|
309 | 308 | static void perf_iommu_read(struct perf_event *event) |
---|
310 | 309 | { |
---|
311 | | - u64 count, prev, delta; |
---|
| 310 | + u64 count; |
---|
312 | 311 | struct hw_perf_event *hwc = &event->hw; |
---|
313 | 312 | struct amd_iommu *iommu = perf_event_2_iommu(event); |
---|
314 | 313 | |
---|
.. | .. |
---|
319 | 318 | /* IOMMU pc counter register is only 48 bits */ |
---|
320 | 319 | count &= GENMASK_ULL(47, 0); |
---|
321 | 320 | |
---|
322 | | - prev = local64_read(&hwc->prev_count); |
---|
323 | | - if (local64_cmpxchg(&hwc->prev_count, prev, count) != prev) |
---|
324 | | - return; |
---|
325 | | - |
---|
326 | | - /* Handle 48-bit counter overflow */ |
---|
327 | | - delta = (count << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); |
---|
328 | | - delta >>= COUNTER_SHIFT; |
---|
329 | | - local64_add(delta, &event->count); |
---|
| 321 | + /* |
---|
| 322 | + * Since the counter always start with value zero, |
---|
| 323 | + * simply just accumulate the count for the event. |
---|
| 324 | + */ |
---|
| 325 | + local64_add(count, &event->count); |
---|
330 | 326 | } |
---|
331 | 327 | |
---|
332 | 328 | static void perf_iommu_stop(struct perf_event *event, int flags) |
---|
.. | .. |
---|
336 | 332 | if (hwc->state & PERF_HES_UPTODATE) |
---|
337 | 333 | return; |
---|
338 | 334 | |
---|
| 335 | + /* |
---|
| 336 | + * To account for power-gating, in which reading the counter would |
---|
| 337 | + * return zero, we need to read the register before disabling. |
---|
| 338 | + */ |
---|
| 339 | + perf_iommu_read(event); |
---|
| 340 | + hwc->state |= PERF_HES_UPTODATE; |
---|
| 341 | + |
---|
339 | 342 | perf_iommu_disable_event(event); |
---|
340 | 343 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
---|
341 | 344 | hwc->state |= PERF_HES_STOPPED; |
---|
342 | | - |
---|
343 | | - if (hwc->state & PERF_HES_UPTODATE) |
---|
344 | | - return; |
---|
345 | | - |
---|
346 | | - perf_iommu_read(event); |
---|
347 | | - hwc->state |= PERF_HES_UPTODATE; |
---|
348 | 345 | } |
---|
349 | 346 | |
---|
350 | 347 | static int perf_iommu_add(struct perf_event *event, int flags) |
---|
.. | .. |
---|
398 | 395 | return 0; |
---|
399 | 396 | } |
---|
400 | 397 | |
---|
401 | | -const struct attribute_group *amd_iommu_attr_groups[] = { |
---|
| 398 | +static const struct attribute_group *amd_iommu_attr_groups[] = { |
---|
402 | 399 | &amd_iommu_format_group, |
---|
403 | 400 | &amd_iommu_cpumask_group, |
---|
404 | 401 | &amd_iommu_events_group, |
---|
.. | .. |
---|
414 | 411 | .read = perf_iommu_read, |
---|
415 | 412 | .task_ctx_nr = perf_invalid_context, |
---|
416 | 413 | .attr_groups = amd_iommu_attr_groups, |
---|
| 414 | + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
---|
417 | 415 | }; |
---|
418 | 416 | |
---|
419 | 417 | static __init int init_one_iommu(unsigned int idx) |
---|