| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Copyright (C) 2013 Advanced Micro Devices, Inc. |
|---|
| 3 | 4 | * |
|---|
| .. | .. |
|---|
| 5 | 6 | * Author: Suravee Suthikulpanit <Suraveee.Suthikulpanit@amd.com> |
|---|
| 6 | 7 | * |
|---|
| 7 | 8 | * Perf: amd_iommu - AMD IOMMU Performance Counter PMU implementation |
|---|
| 8 | | - * |
|---|
| 9 | | - * This program is free software; you can redistribute it and/or modify |
|---|
| 10 | | - * it under the terms of the GNU General Public License version 2 as |
|---|
| 11 | | - * published by the Free Software Foundation. |
|---|
| 12 | 9 | */ |
|---|
| 13 | 10 | |
|---|
| 14 | 11 | #define pr_fmt(fmt) "perf/amd_iommu: " fmt |
|---|
| .. | .. |
|---|
| 20 | 17 | |
|---|
| 21 | 18 | #include "../perf_event.h" |
|---|
| 22 | 19 | #include "iommu.h" |
|---|
| 23 | | - |
|---|
| 24 | | -#define COUNTER_SHIFT 16 |
|---|
| 25 | 20 | |
|---|
| 26 | 21 | /* iommu pmu conf masks */ |
|---|
| 27 | 22 | #define GET_CSOURCE(x) ((x)->conf & 0xFFULL) |
|---|
| .. | .. |
|---|
| 223 | 218 | if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) |
|---|
| 224 | 219 | return -EINVAL; |
|---|
| 225 | 220 | |
|---|
| 226 | | - /* IOMMU counters do not have usr/os/guest/host bits */ |
|---|
| 227 | | - if (event->attr.exclude_user || event->attr.exclude_kernel || |
|---|
| 228 | | - event->attr.exclude_host || event->attr.exclude_guest) |
|---|
| 229 | | - return -EINVAL; |
|---|
| 230 | | - |
|---|
| 231 | 221 | if (event->cpu < 0) |
|---|
| 232 | 222 | return -EINVAL; |
|---|
| 233 | 223 | |
|---|
| .. | .. |
|---|
| 293 | 283 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
|---|
| 294 | 284 | hwc->state = 0; |
|---|
| 295 | 285 | |
|---|
| 286 | + /* |
|---|
| 287 | + * To account for power-gating, which prevents write to |
|---|
| 288 | + * the counter, we need to enable the counter |
|---|
| 289 | + * before setting up counter register. |
|---|
| 290 | + */ |
|---|
| 291 | + perf_iommu_enable_event(event); |
|---|
| 292 | + |
|---|
| 296 | 293 | if (flags & PERF_EF_RELOAD) { |
|---|
| 297 | | - u64 prev_raw_count = local64_read(&hwc->prev_count); |
|---|
| 294 | + u64 count = 0; |
|---|
| 298 | 295 | struct amd_iommu *iommu = perf_event_2_iommu(event); |
|---|
| 299 | 296 | |
|---|
| 297 | + /* |
|---|
| 298 | + * Since the IOMMU PMU only support counting mode, |
|---|
| 299 | + * the counter always start with value zero. |
|---|
| 300 | + */ |
|---|
| 300 | 301 | amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr, |
|---|
| 301 | | - IOMMU_PC_COUNTER_REG, &prev_raw_count); |
|---|
| 302 | + IOMMU_PC_COUNTER_REG, &count); |
|---|
| 302 | 303 | } |
|---|
| 303 | 304 | |
|---|
| 304 | | - perf_iommu_enable_event(event); |
|---|
| 305 | 305 | perf_event_update_userpage(event); |
|---|
| 306 | | - |
|---|
| 307 | 306 | } |
|---|
| 308 | 307 | |
|---|
| 309 | 308 | static void perf_iommu_read(struct perf_event *event) |
|---|
| 310 | 309 | { |
|---|
| 311 | | - u64 count, prev, delta; |
|---|
| 310 | + u64 count; |
|---|
| 312 | 311 | struct hw_perf_event *hwc = &event->hw; |
|---|
| 313 | 312 | struct amd_iommu *iommu = perf_event_2_iommu(event); |
|---|
| 314 | 313 | |
|---|
| .. | .. |
|---|
| 319 | 318 | /* IOMMU pc counter register is only 48 bits */ |
|---|
| 320 | 319 | count &= GENMASK_ULL(47, 0); |
|---|
| 321 | 320 | |
|---|
| 322 | | - prev = local64_read(&hwc->prev_count); |
|---|
| 323 | | - if (local64_cmpxchg(&hwc->prev_count, prev, count) != prev) |
|---|
| 324 | | - return; |
|---|
| 325 | | - |
|---|
| 326 | | - /* Handle 48-bit counter overflow */ |
|---|
| 327 | | - delta = (count << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); |
|---|
| 328 | | - delta >>= COUNTER_SHIFT; |
|---|
| 329 | | - local64_add(delta, &event->count); |
|---|
| 321 | + /* |
|---|
| 322 | + * Since the counter always start with value zero, |
|---|
| 323 | + * simply just accumulate the count for the event. |
|---|
| 324 | + */ |
|---|
| 325 | + local64_add(count, &event->count); |
|---|
| 330 | 326 | } |
|---|
| 331 | 327 | |
|---|
| 332 | 328 | static void perf_iommu_stop(struct perf_event *event, int flags) |
|---|
| .. | .. |
|---|
| 336 | 332 | if (hwc->state & PERF_HES_UPTODATE) |
|---|
| 337 | 333 | return; |
|---|
| 338 | 334 | |
|---|
| 335 | + /* |
|---|
| 336 | + * To account for power-gating, in which reading the counter would |
|---|
| 337 | + * return zero, we need to read the register before disabling. |
|---|
| 338 | + */ |
|---|
| 339 | + perf_iommu_read(event); |
|---|
| 340 | + hwc->state |= PERF_HES_UPTODATE; |
|---|
| 341 | + |
|---|
| 339 | 342 | perf_iommu_disable_event(event); |
|---|
| 340 | 343 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
|---|
| 341 | 344 | hwc->state |= PERF_HES_STOPPED; |
|---|
| 342 | | - |
|---|
| 343 | | - if (hwc->state & PERF_HES_UPTODATE) |
|---|
| 344 | | - return; |
|---|
| 345 | | - |
|---|
| 346 | | - perf_iommu_read(event); |
|---|
| 347 | | - hwc->state |= PERF_HES_UPTODATE; |
|---|
| 348 | 345 | } |
|---|
| 349 | 346 | |
|---|
| 350 | 347 | static int perf_iommu_add(struct perf_event *event, int flags) |
|---|
| .. | .. |
|---|
| 398 | 395 | return 0; |
|---|
| 399 | 396 | } |
|---|
| 400 | 397 | |
|---|
| 401 | | -const struct attribute_group *amd_iommu_attr_groups[] = { |
|---|
| 398 | +static const struct attribute_group *amd_iommu_attr_groups[] = { |
|---|
| 402 | 399 | &amd_iommu_format_group, |
|---|
| 403 | 400 | &amd_iommu_cpumask_group, |
|---|
| 404 | 401 | &amd_iommu_events_group, |
|---|
| .. | .. |
|---|
| 414 | 411 | .read = perf_iommu_read, |
|---|
| 415 | 412 | .task_ctx_nr = perf_invalid_context, |
|---|
| 416 | 413 | .attr_groups = amd_iommu_attr_groups, |
|---|
| 414 | + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
|---|
| 417 | 415 | }; |
|---|
| 418 | 416 | |
|---|
| 419 | 417 | static __init int init_one_iommu(unsigned int idx) |
|---|