| .. | .. |
|---|
| 16 | 16 | #include <linux/cpu.h> |
|---|
| 17 | 17 | #include <asm/ctl_reg.h> |
|---|
| 18 | 18 | #include <asm/io.h> |
|---|
| 19 | +#include <asm/stacktrace.h> |
|---|
| 19 | 20 | |
|---|
| 20 | 21 | static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) |
|---|
| 21 | 22 | { |
|---|
| .. | .. |
|---|
| 51 | 52 | * Therefore we have a read-modify-write sequence: the function reads eight |
|---|
| 52 | 53 | * bytes from destination at an eight byte boundary, modifies the bytes |
|---|
| 53 | 54 | * requested and writes the result back in a loop. |
|---|
| 54 | | - * |
|---|
| 55 | | - * Note: this means that this function may not be called concurrently on |
|---|
| 56 | | - * several cpus with overlapping words, since this may potentially |
|---|
| 57 | | - * cause data corruption. |
|---|
| 58 | 55 | */ |
|---|
| 59 | | -void notrace s390_kernel_write(void *dst, const void *src, size_t size) |
|---|
| 56 | +static DEFINE_SPINLOCK(s390_kernel_write_lock); |
|---|
| 57 | + |
|---|
| 58 | +notrace void *s390_kernel_write(void *dst, const void *src, size_t size) |
|---|
| 60 | 59 | { |
|---|
| 60 | + void *tmp = dst; |
|---|
| 61 | + unsigned long flags; |
|---|
| 61 | 62 | long copied; |
|---|
| 62 | 63 | |
|---|
| 63 | | - while (size) { |
|---|
| 64 | | - copied = s390_kernel_write_odd(dst, src, size); |
|---|
| 65 | | - dst += copied; |
|---|
| 66 | | - src += copied; |
|---|
| 67 | | - size -= copied; |
|---|
| 64 | + spin_lock_irqsave(&s390_kernel_write_lock, flags); |
|---|
| 65 | + if (!(flags & PSW_MASK_DAT)) { |
|---|
| 66 | + memcpy(dst, src, size); |
|---|
| 67 | + } else { |
|---|
| 68 | + while (size) { |
|---|
| 69 | + copied = s390_kernel_write_odd(tmp, src, size); |
|---|
| 70 | + tmp += copied; |
|---|
| 71 | + src += copied; |
|---|
| 72 | + size -= copied; |
|---|
| 73 | + } |
|---|
| 68 | 74 | } |
|---|
| 75 | + spin_unlock_irqrestore(&s390_kernel_write_lock, flags); |
|---|
| 76 | + |
|---|
| 77 | + return dst; |
|---|
| 69 | 78 | } |
|---|
| 70 | 79 | |
|---|
| 71 | | -static int __memcpy_real(void *dest, void *src, size_t count) |
|---|
| 80 | +static int __no_sanitize_address __memcpy_real(void *dest, void *src, size_t count) |
|---|
| 72 | 81 | { |
|---|
| 73 | 82 | register unsigned long _dest asm("2") = (unsigned long) dest; |
|---|
| 74 | 83 | register unsigned long _len1 asm("3") = (unsigned long) count; |
|---|
| .. | .. |
|---|
| 89 | 98 | return rc; |
|---|
| 90 | 99 | } |
|---|
| 91 | 100 | |
|---|
| 92 | | -/* |
|---|
| 93 | | - * Copy memory in real mode (kernel to kernel) |
|---|
| 94 | | - */ |
|---|
| 95 | | -int memcpy_real(void *dest, void *src, size_t count) |
|---|
| 101 | +static unsigned long __no_sanitize_address _memcpy_real(unsigned long dest, |
|---|
| 102 | + unsigned long src, |
|---|
| 103 | + unsigned long count) |
|---|
| 96 | 104 | { |
|---|
| 97 | 105 | int irqs_disabled, rc; |
|---|
| 98 | 106 | unsigned long flags; |
|---|
| 99 | 107 | |
|---|
| 100 | 108 | if (!count) |
|---|
| 101 | 109 | return 0; |
|---|
| 102 | | - flags = __arch_local_irq_stnsm(0xf8UL); |
|---|
| 110 | + flags = arch_local_irq_save(); |
|---|
| 103 | 111 | irqs_disabled = arch_irqs_disabled_flags(flags); |
|---|
| 104 | 112 | if (!irqs_disabled) |
|---|
| 105 | 113 | trace_hardirqs_off(); |
|---|
| 106 | | - rc = __memcpy_real(dest, src, count); |
|---|
| 114 | + __arch_local_irq_stnsm(0xf8); // disable DAT |
|---|
| 115 | + rc = __memcpy_real((void *) dest, (void *) src, (size_t) count); |
|---|
| 116 | + if (flags & PSW_MASK_DAT) |
|---|
| 117 | + __arch_local_irq_stosm(0x04); // enable DAT |
|---|
| 107 | 118 | if (!irqs_disabled) |
|---|
| 108 | 119 | trace_hardirqs_on(); |
|---|
| 109 | 120 | __arch_local_irq_ssm(flags); |
|---|
| .. | .. |
|---|
| 111 | 122 | } |
|---|
| 112 | 123 | |
|---|
| 113 | 124 | /* |
|---|
| 125 | + * Copy memory in real mode (kernel to kernel) |
|---|
| 126 | + */ |
|---|
| 127 | +int memcpy_real(void *dest, void *src, size_t count) |
|---|
| 128 | +{ |
|---|
| 129 | + int rc; |
|---|
| 130 | + |
|---|
| 131 | + if (S390_lowcore.nodat_stack != 0) { |
|---|
| 132 | + preempt_disable(); |
|---|
| 133 | + rc = CALL_ON_STACK(_memcpy_real, S390_lowcore.nodat_stack, 3, |
|---|
| 134 | + dest, src, count); |
|---|
| 135 | + preempt_enable(); |
|---|
| 136 | + return rc; |
|---|
| 137 | + } |
|---|
| 138 | + /* |
|---|
| 139 | + * This is a really early memcpy_real call, the stacks are |
|---|
| 140 | + * not set up yet. Just call _memcpy_real on the early boot |
|---|
| 141 | + * stack |
|---|
| 142 | + */ |
|---|
| 143 | + return _memcpy_real((unsigned long) dest,(unsigned long) src, |
|---|
| 144 | + (unsigned long) count); |
|---|
| 145 | +} |
|---|
| 146 | + |
|---|
| 147 | +/* |
|---|
| 114 | 148 | * Copy memory in absolute mode (kernel to kernel) |
|---|
| 115 | 149 | */ |
|---|
| 116 | 150 | void memcpy_absolute(void *dest, void *src, size_t count) |
|---|