.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
---|
1 | 2 | /* |
---|
2 | 3 | * Copyright (C) 2012 ARM Ltd. |
---|
3 | | - * |
---|
4 | | - * This program is free software; you can redistribute it and/or modify |
---|
5 | | - * it under the terms of the GNU General Public License version 2 as |
---|
6 | | - * published by the Free Software Foundation. |
---|
7 | | - * |
---|
8 | | - * This program is distributed in the hope that it will be useful, |
---|
9 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
10 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
11 | | - * GNU General Public License for more details. |
---|
12 | | - * |
---|
13 | | - * You should have received a copy of the GNU General Public License |
---|
14 | | - * along with this program. If not, see <http://www.gnu.org/licenses/>. |
---|
15 | 4 | */ |
---|
16 | 5 | #ifndef __ASM_FP_H |
---|
17 | 6 | #define __ASM_FP_H |
---|
.. | .. |
---|
24 | 13 | |
---|
25 | 14 | #ifndef __ASSEMBLY__ |
---|
26 | 15 | |
---|
| 16 | +#include <linux/bitmap.h> |
---|
27 | 17 | #include <linux/build_bug.h> |
---|
| 18 | +#include <linux/bug.h> |
---|
28 | 19 | #include <linux/cache.h> |
---|
29 | 20 | #include <linux/init.h> |
---|
30 | 21 | #include <linux/stddef.h> |
---|
| 22 | +#include <linux/types.h> |
---|
31 | 23 | |
---|
32 | | -#if defined(__KERNEL__) && defined(CONFIG_COMPAT) |
---|
| 24 | +#ifdef CONFIG_COMPAT |
---|
33 | 25 | /* Masks for extracting the FPSR and FPCR from the FPSCR */ |
---|
34 | 26 | #define VFP_FPSCR_STAT_MASK 0xf800009f |
---|
35 | 27 | #define VFP_FPSCR_CTRL_MASK 0x07f79f00 |
---|
.. | .. |
---|
45 | 37 | extern void fpsimd_save_state(struct user_fpsimd_state *state); |
---|
46 | 38 | extern void fpsimd_load_state(struct user_fpsimd_state *state); |
---|
47 | 39 | |
---|
48 | | -extern void fpsimd_save(void); |
---|
49 | | - |
---|
50 | 40 | extern void fpsimd_thread_switch(struct task_struct *next); |
---|
51 | 41 | extern void fpsimd_flush_thread(void); |
---|
52 | 42 | |
---|
.. | .. |
---|
56 | 46 | extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); |
---|
57 | 47 | |
---|
58 | 48 | extern void fpsimd_bind_task_to_cpu(void); |
---|
59 | | -extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state); |
---|
| 49 | +extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state, |
---|
| 50 | + void *sve_state, unsigned int sve_vl); |
---|
60 | 51 | |
---|
61 | 52 | extern void fpsimd_flush_task_state(struct task_struct *target); |
---|
62 | | -extern void fpsimd_flush_cpu_state(void); |
---|
63 | | -extern void sve_flush_cpu_state(void); |
---|
| 53 | +extern void fpsimd_save_and_flush_cpu_state(void); |
---|
64 | 54 | |
---|
65 | 55 | /* Maximum VL that SVE VL-agnostic software can transparently support */ |
---|
66 | 56 | #define SVE_VL_ARCH_MAX 0x100 |
---|
.. | .. |
---|
79 | 69 | extern void sve_save_state(void *state, u32 *pfpsr); |
---|
80 | 70 | extern void sve_load_state(void const *state, u32 const *pfpsr, |
---|
81 | 71 | unsigned long vq_minus_1); |
---|
| 72 | +extern void sve_flush_live(void); |
---|
| 73 | +extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state, |
---|
| 74 | + unsigned long vq_minus_1); |
---|
82 | 75 | extern unsigned int sve_get_vl(void); |
---|
83 | 76 | |
---|
84 | 77 | struct arm64_cpu_capabilities; |
---|
.. | .. |
---|
87 | 80 | extern u64 read_zcr_features(void); |
---|
88 | 81 | |
---|
89 | 82 | extern int __ro_after_init sve_max_vl; |
---|
| 83 | +extern int __ro_after_init sve_max_virtualisable_vl; |
---|
| 84 | +extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); |
---|
| 85 | + |
---|
| 86 | +/* |
---|
| 87 | + * Helpers to translate bit indices in sve_vq_map to VQ values (and |
---|
| 88 | + * vice versa). This allows find_next_bit() to be used to find the |
---|
| 89 | + * _maximum_ VQ not exceeding a certain value. |
---|
| 90 | + */ |
---|
| 91 | +static inline unsigned int __vq_to_bit(unsigned int vq) |
---|
| 92 | +{ |
---|
| 93 | + return SVE_VQ_MAX - vq; |
---|
| 94 | +} |
---|
| 95 | + |
---|
| 96 | +static inline unsigned int __bit_to_vq(unsigned int bit) |
---|
| 97 | +{ |
---|
| 98 | + return SVE_VQ_MAX - bit; |
---|
| 99 | +} |
---|
| 100 | + |
---|
| 101 | +/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */ |
---|
| 102 | +static inline bool sve_vq_available(unsigned int vq) |
---|
| 103 | +{ |
---|
| 104 | + return test_bit(__vq_to_bit(vq), sve_vq_map); |
---|
| 105 | +} |
---|
90 | 106 | |
---|
91 | 107 | #ifdef CONFIG_ARM64_SVE |
---|
92 | 108 | |
---|
.. | .. |
---|
114 | 130 | sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN); |
---|
115 | 131 | } |
---|
116 | 132 | |
---|
| 133 | +#define sve_cond_update_zcr_vq(val, reg) \ |
---|
| 134 | + do { \ |
---|
| 135 | + u64 __zcr = read_sysreg_s((reg)); \ |
---|
| 136 | + u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \ |
---|
| 137 | + __new |= (val) & ZCR_ELx_LEN_MASK; \ |
---|
| 138 | + if (__zcr != __new) \ |
---|
| 139 | + write_sysreg_s(__new, (reg)); \ |
---|
| 140 | + } while (0) |
---|
| 141 | + |
---|
117 | 142 | /* |
---|
118 | 143 | * Probing and setup functions. |
---|
119 | 144 | * Calls to these functions must be serialised with one another. |
---|