| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * SN Platform GRU Driver |
|---|
| 3 | 4 | * |
|---|
| .. | .. |
|---|
| 8 | 9 | * the user CB. |
|---|
| 9 | 10 | * |
|---|
| 10 | 11 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. |
|---|
| 11 | | - * |
|---|
| 12 | | - * This program is free software; you can redistribute it and/or modify |
|---|
| 13 | | - * it under the terms of the GNU General Public License as published by |
|---|
| 14 | | - * the Free Software Foundation; either version 2 of the License, or |
|---|
| 15 | | - * (at your option) any later version. |
|---|
| 16 | | - * |
|---|
| 17 | | - * This program is distributed in the hope that it will be useful, |
|---|
| 18 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 19 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 20 | | - * GNU General Public License for more details. |
|---|
| 21 | | - * |
|---|
| 22 | | - * You should have received a copy of the GNU General Public License |
|---|
| 23 | | - * along with this program; if not, write to the Free Software |
|---|
| 24 | | - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
|---|
| 25 | 12 | */ |
|---|
| 26 | 13 | |
|---|
| 27 | 14 | #include <linux/kernel.h> |
|---|
| .. | .. |
|---|
| 33 | 20 | #include <linux/io.h> |
|---|
| 34 | 21 | #include <linux/uaccess.h> |
|---|
| 35 | 22 | #include <linux/security.h> |
|---|
| 23 | +#include <linux/sync_core.h> |
|---|
| 36 | 24 | #include <linux/prefetch.h> |
|---|
| 37 | | -#include <asm/pgtable.h> |
|---|
| 38 | 25 | #include "gru.h" |
|---|
| 39 | 26 | #include "grutables.h" |
|---|
| 40 | 27 | #include "grulib.h" |
|---|
| .. | .. |
|---|
| 56 | 43 | } |
|---|
| 57 | 44 | |
|---|
| 58 | 45 | /* |
|---|
| 59 | | - * Find the vma of a GRU segment. Caller must hold mmap_sem. |
|---|
| 46 | + * Find the vma of a GRU segment. Caller must hold mmap_lock. |
|---|
| 60 | 47 | */ |
|---|
| 61 | 48 | struct vm_area_struct *gru_find_vma(unsigned long vaddr) |
|---|
| 62 | 49 | { |
|---|
| .. | .. |
|---|
| 72 | 59 | * Find and lock the gts that contains the specified user vaddr. |
|---|
| 73 | 60 | * |
|---|
| 74 | 61 | * Returns: |
|---|
| 75 | | - * - *gts with the mmap_sem locked for read and the GTS locked. |
|---|
| 62 | + * - *gts with the mmap_lock locked for read and the GTS locked. |
|---|
| 76 | 63 | * - NULL if vaddr invalid OR is not a valid GSEG vaddr. |
|---|
| 77 | 64 | */ |
|---|
| 78 | 65 | |
|---|
| .. | .. |
|---|
| 82 | 69 | struct vm_area_struct *vma; |
|---|
| 83 | 70 | struct gru_thread_state *gts = NULL; |
|---|
| 84 | 71 | |
|---|
| 85 | | - down_read(&mm->mmap_sem); |
|---|
| 72 | + mmap_read_lock(mm); |
|---|
| 86 | 73 | vma = gru_find_vma(vaddr); |
|---|
| 87 | 74 | if (vma) |
|---|
| 88 | 75 | gts = gru_find_thread_state(vma, TSID(vaddr, vma)); |
|---|
| 89 | 76 | if (gts) |
|---|
| 90 | 77 | mutex_lock(>s->ts_ctxlock); |
|---|
| 91 | 78 | else |
|---|
| 92 | | - up_read(&mm->mmap_sem); |
|---|
| 79 | + mmap_read_unlock(mm); |
|---|
| 93 | 80 | return gts; |
|---|
| 94 | 81 | } |
|---|
| 95 | 82 | |
|---|
| .. | .. |
|---|
| 99 | 86 | struct vm_area_struct *vma; |
|---|
| 100 | 87 | struct gru_thread_state *gts = ERR_PTR(-EINVAL); |
|---|
| 101 | 88 | |
|---|
| 102 | | - down_write(&mm->mmap_sem); |
|---|
| 89 | + mmap_write_lock(mm); |
|---|
| 103 | 90 | vma = gru_find_vma(vaddr); |
|---|
| 104 | 91 | if (!vma) |
|---|
| 105 | 92 | goto err; |
|---|
| .. | .. |
|---|
| 108 | 95 | if (IS_ERR(gts)) |
|---|
| 109 | 96 | goto err; |
|---|
| 110 | 97 | mutex_lock(>s->ts_ctxlock); |
|---|
| 111 | | - downgrade_write(&mm->mmap_sem); |
|---|
| 98 | + mmap_write_downgrade(mm); |
|---|
| 112 | 99 | return gts; |
|---|
| 113 | 100 | |
|---|
| 114 | 101 | err: |
|---|
| 115 | | - up_write(&mm->mmap_sem); |
|---|
| 102 | + mmap_write_unlock(mm); |
|---|
| 116 | 103 | return gts; |
|---|
| 117 | 104 | } |
|---|
| 118 | 105 | |
|---|
| .. | .. |
|---|
| 122 | 109 | static void gru_unlock_gts(struct gru_thread_state *gts) |
|---|
| 123 | 110 | { |
|---|
| 124 | 111 | mutex_unlock(>s->ts_ctxlock); |
|---|
| 125 | | - up_read(¤t->mm->mmap_sem); |
|---|
| 112 | + mmap_read_unlock(current->mm); |
|---|
| 126 | 113 | } |
|---|
| 127 | 114 | |
|---|
| 128 | 115 | /* |
|---|
| .. | .. |
|---|
| 212 | 199 | * Only supports Intel large pages (2MB only) on x86_64. |
|---|
| 213 | 200 | * ZZZ - hugepage support is incomplete |
|---|
| 214 | 201 | * |
|---|
| 215 | | - * NOTE: mmap_sem is already held on entry to this function. This |
|---|
| 202 | + * NOTE: mmap_lock is already held on entry to this function. This |
|---|
| 216 | 203 | * guarantees existence of the page tables. |
|---|
| 217 | 204 | */ |
|---|
| 218 | 205 | static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, |
|---|
| .. | .. |
|---|
| 583 | 570 | } |
|---|
| 584 | 571 | |
|---|
| 585 | 572 | /* |
|---|
| 586 | | - * This is running in interrupt context. Trylock the mmap_sem. |
|---|
| 573 | + * This is running in interrupt context. Trylock the mmap_lock. |
|---|
| 587 | 574 | * If it fails, retry the fault in user context. |
|---|
| 588 | 575 | */ |
|---|
| 589 | 576 | gts->ustats.fmm_tlbmiss++; |
|---|
| 590 | 577 | if (!gts->ts_force_cch_reload && |
|---|
| 591 | | - down_read_trylock(>s->ts_mm->mmap_sem)) { |
|---|
| 578 | + mmap_read_trylock(gts->ts_mm)) { |
|---|
| 592 | 579 | gru_try_dropin(gru, gts, tfh, NULL); |
|---|
| 593 | | - up_read(>s->ts_mm->mmap_sem); |
|---|
| 580 | + mmap_read_unlock(gts->ts_mm); |
|---|
| 594 | 581 | } else { |
|---|
| 595 | 582 | tfh_user_polling_mode(tfh); |
|---|
| 596 | 583 | STAT(intr_mm_lock_failed); |
|---|
| .. | .. |
|---|
| 616 | 603 | for_each_possible_blade(blade) { |
|---|
| 617 | 604 | if (uv_blade_nr_possible_cpus(blade)) |
|---|
| 618 | 605 | continue; |
|---|
| 619 | | - gru_intr(0, blade); |
|---|
| 620 | | - gru_intr(1, blade); |
|---|
| 606 | + gru_intr(0, blade); |
|---|
| 607 | + gru_intr(1, blade); |
|---|
| 621 | 608 | } |
|---|
| 622 | 609 | return IRQ_HANDLED; |
|---|
| 623 | 610 | } |
|---|