| .. | .. |
|---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Copyright (C) 2012 ARM Ltd. |
|---|
| 3 | | - * |
|---|
| 4 | | - * This program is free software; you can redistribute it and/or modify |
|---|
| 5 | | - * it under the terms of the GNU General Public License version 2 as |
|---|
| 6 | | - * published by the Free Software Foundation. |
|---|
| 7 | | - * |
|---|
| 8 | | - * This program is distributed in the hope that it will be useful, |
|---|
| 9 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 10 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 11 | | - * GNU General Public License for more details. |
|---|
| 12 | | - * |
|---|
| 13 | | - * You should have received a copy of the GNU General Public License |
|---|
| 14 | | - * along with this program. If not, see <http://www.gnu.org/licenses/>. |
|---|
| 15 | 4 | */ |
|---|
| 16 | 5 | #ifndef __ASM_FP_H |
|---|
| 17 | 6 | #define __ASM_FP_H |
|---|
| .. | .. |
|---|
| 24 | 13 | |
|---|
| 25 | 14 | #ifndef __ASSEMBLY__ |
|---|
| 26 | 15 | |
|---|
| 16 | +#include <linux/bitmap.h> |
|---|
| 27 | 17 | #include <linux/build_bug.h> |
|---|
| 18 | +#include <linux/bug.h> |
|---|
| 28 | 19 | #include <linux/cache.h> |
|---|
| 29 | 20 | #include <linux/init.h> |
|---|
| 30 | 21 | #include <linux/stddef.h> |
|---|
| 22 | +#include <linux/types.h> |
|---|
| 31 | 23 | |
|---|
| 32 | | -#if defined(__KERNEL__) && defined(CONFIG_COMPAT) |
|---|
| 24 | +#ifdef CONFIG_COMPAT |
|---|
| 33 | 25 | /* Masks for extracting the FPSR and FPCR from the FPSCR */ |
|---|
| 34 | 26 | #define VFP_FPSCR_STAT_MASK 0xf800009f |
|---|
| 35 | 27 | #define VFP_FPSCR_CTRL_MASK 0x07f79f00 |
|---|
| .. | .. |
|---|
| 45 | 37 | extern void fpsimd_save_state(struct user_fpsimd_state *state); |
|---|
| 46 | 38 | extern void fpsimd_load_state(struct user_fpsimd_state *state); |
|---|
| 47 | 39 | |
|---|
| 48 | | -extern void fpsimd_save(void); |
|---|
| 49 | | - |
|---|
| 50 | 40 | extern void fpsimd_thread_switch(struct task_struct *next); |
|---|
| 51 | 41 | extern void fpsimd_flush_thread(void); |
|---|
| 52 | 42 | |
|---|
| .. | .. |
|---|
| 56 | 46 | extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); |
|---|
| 57 | 47 | |
|---|
| 58 | 48 | extern void fpsimd_bind_task_to_cpu(void); |
|---|
| 59 | | -extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state); |
|---|
| 49 | +extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state, |
|---|
| 50 | + void *sve_state, unsigned int sve_vl); |
|---|
| 60 | 51 | |
|---|
| 61 | 52 | extern void fpsimd_flush_task_state(struct task_struct *target); |
|---|
| 62 | | -extern void fpsimd_flush_cpu_state(void); |
|---|
| 63 | | -extern void sve_flush_cpu_state(void); |
|---|
| 53 | +extern void fpsimd_save_and_flush_cpu_state(void); |
|---|
| 64 | 54 | |
|---|
| 65 | 55 | /* Maximum VL that SVE VL-agnostic software can transparently support */ |
|---|
| 66 | 56 | #define SVE_VL_ARCH_MAX 0x100 |
|---|
| .. | .. |
|---|
| 79 | 69 | extern void sve_save_state(void *state, u32 *pfpsr); |
|---|
| 80 | 70 | extern void sve_load_state(void const *state, u32 const *pfpsr, |
|---|
| 81 | 71 | unsigned long vq_minus_1); |
|---|
| 72 | +extern void sve_flush_live(void); |
|---|
| 73 | +extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state, |
|---|
| 74 | + unsigned long vq_minus_1); |
|---|
| 82 | 75 | extern unsigned int sve_get_vl(void); |
|---|
| 83 | 76 | |
|---|
| 84 | 77 | struct arm64_cpu_capabilities; |
|---|
| .. | .. |
|---|
| 87 | 80 | extern u64 read_zcr_features(void); |
|---|
| 88 | 81 | |
|---|
| 89 | 82 | extern int __ro_after_init sve_max_vl; |
|---|
| 83 | +extern int __ro_after_init sve_max_virtualisable_vl; |
|---|
| 84 | +extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX); |
|---|
| 85 | + |
|---|
| 86 | +/* |
|---|
| 87 | + * Helpers to translate bit indices in sve_vq_map to VQ values (and |
|---|
| 88 | + * vice versa). This allows find_next_bit() to be used to find the |
|---|
| 89 | + * _maximum_ VQ not exceeding a certain value. |
|---|
| 90 | + */ |
|---|
| 91 | +static inline unsigned int __vq_to_bit(unsigned int vq) |
|---|
| 92 | +{ |
|---|
| 93 | + return SVE_VQ_MAX - vq; |
|---|
| 94 | +} |
|---|
| 95 | + |
|---|
| 96 | +static inline unsigned int __bit_to_vq(unsigned int bit) |
|---|
| 97 | +{ |
|---|
| 98 | + return SVE_VQ_MAX - bit; |
|---|
| 99 | +} |
|---|
| 100 | + |
|---|
| 101 | +/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */ |
|---|
| 102 | +static inline bool sve_vq_available(unsigned int vq) |
|---|
| 103 | +{ |
|---|
| 104 | + return test_bit(__vq_to_bit(vq), sve_vq_map); |
|---|
| 105 | +} |
|---|
| 90 | 106 | |
|---|
| 91 | 107 | #ifdef CONFIG_ARM64_SVE |
|---|
| 92 | 108 | |
|---|
| .. | .. |
|---|
| 114 | 130 | sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN); |
|---|
| 115 | 131 | } |
|---|
| 116 | 132 | |
|---|
| 133 | +#define sve_cond_update_zcr_vq(val, reg) \ |
|---|
| 134 | + do { \ |
|---|
| 135 | + u64 __zcr = read_sysreg_s((reg)); \ |
|---|
| 136 | + u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \ |
|---|
| 137 | + __new |= (val) & ZCR_ELx_LEN_MASK; \ |
|---|
| 138 | + if (__zcr != __new) \ |
|---|
| 139 | + write_sysreg_s(__new, (reg)); \ |
|---|
| 140 | + } while (0) |
|---|
| 141 | + |
|---|
| 117 | 142 | /* |
|---|
| 118 | 143 | * Probing and setup functions. |
|---|
| 119 | 144 | * Calls to these functions must be serialised with one another. |
|---|