| .. | .. |
|---|
| 39 | 39 | #define insqh(x,y,z) \ |
|---|
| 40 | 40 | __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) |
|---|
| 41 | 41 | |
|---|
| 42 | | - |
|---|
| 43 | | -#define __get_user_u(x,ptr) \ |
|---|
| 42 | +#define __get_word(insn,x,ptr) \ |
|---|
| 44 | 43 | ({ \ |
|---|
| 45 | 44 | long __guu_err; \ |
|---|
| 46 | 45 | __asm__ __volatile__( \ |
|---|
| 47 | | - "1: ldq_u %0,%2\n" \ |
|---|
| 46 | + "1: "#insn" %0,%2\n" \ |
|---|
| 48 | 47 | "2:\n" \ |
|---|
| 49 | 48 | EXC(1b,2b,%0,%1) \ |
|---|
| 50 | 49 | : "=r"(x), "=r"(__guu_err) \ |
|---|
| 51 | 50 | : "m"(__m(ptr)), "1"(0)); \ |
|---|
| 52 | 51 | __guu_err; \ |
|---|
| 53 | 52 | }) |
|---|
| 54 | | - |
|---|
| 55 | | -#define __put_user_u(x,ptr) \ |
|---|
| 56 | | -({ \ |
|---|
| 57 | | - long __puu_err; \ |
|---|
| 58 | | - __asm__ __volatile__( \ |
|---|
| 59 | | - "1: stq_u %2,%1\n" \ |
|---|
| 60 | | - "2:\n" \ |
|---|
| 61 | | - EXC(1b,2b,$31,%0) \ |
|---|
| 62 | | - : "=r"(__puu_err) \ |
|---|
| 63 | | - : "m"(__m(addr)), "rJ"(x), "0"(0)); \ |
|---|
| 64 | | - __puu_err; \ |
|---|
| 65 | | -}) |
|---|
| 66 | | - |
|---|
| 67 | 53 | |
|---|
| 68 | 54 | static inline unsigned short from64to16(unsigned long x) |
|---|
| 69 | 55 | { |
|---|
| .. | .. |
|---|
| 95 | 81 | */ |
|---|
| 96 | 82 | static inline unsigned long |
|---|
| 97 | 83 | csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst, |
|---|
| 98 | | - long len, unsigned long checksum, |
|---|
| 99 | | - int *errp) |
|---|
| 84 | + long len) |
|---|
| 100 | 85 | { |
|---|
| 86 | + unsigned long checksum = ~0U; |
|---|
| 101 | 87 | unsigned long carry = 0; |
|---|
| 102 | | - int err = 0; |
|---|
| 103 | 88 | |
|---|
| 104 | 89 | while (len >= 0) { |
|---|
| 105 | 90 | unsigned long word; |
|---|
| 106 | | - err |= __get_user(word, src); |
|---|
| 91 | + if (__get_word(ldq, word, src)) |
|---|
| 92 | + return 0; |
|---|
| 107 | 93 | checksum += carry; |
|---|
| 108 | 94 | src++; |
|---|
| 109 | 95 | checksum += word; |
|---|
| .. | .. |
|---|
| 116 | 102 | checksum += carry; |
|---|
| 117 | 103 | if (len) { |
|---|
| 118 | 104 | unsigned long word, tmp; |
|---|
| 119 | | - err |= __get_user(word, src); |
|---|
| 105 | + if (__get_word(ldq, word, src)) |
|---|
| 106 | + return 0; |
|---|
| 120 | 107 | tmp = *dst; |
|---|
| 121 | 108 | mskql(word, len, word); |
|---|
| 122 | 109 | checksum += word; |
|---|
| .. | .. |
|---|
| 125 | 112 | *dst = word | tmp; |
|---|
| 126 | 113 | checksum += carry; |
|---|
| 127 | 114 | } |
|---|
| 128 | | - if (err && errp) *errp = err; |
|---|
| 129 | 115 | return checksum; |
|---|
| 130 | 116 | } |
|---|
| 131 | 117 | |
|---|
| .. | .. |
|---|
| 137 | 123 | csum_partial_cfu_dest_aligned(const unsigned long __user *src, |
|---|
| 138 | 124 | unsigned long *dst, |
|---|
| 139 | 125 | unsigned long soff, |
|---|
| 140 | | - long len, unsigned long checksum, |
|---|
| 141 | | - int *errp) |
|---|
| 126 | + long len) |
|---|
| 142 | 127 | { |
|---|
| 143 | 128 | unsigned long first; |
|---|
| 144 | 129 | unsigned long word, carry; |
|---|
| 145 | 130 | unsigned long lastsrc = 7+len+(unsigned long)src; |
|---|
| 146 | | - int err = 0; |
|---|
| 131 | + unsigned long checksum = ~0U; |
|---|
| 147 | 132 | |
|---|
| 148 | | - err |= __get_user_u(first,src); |
|---|
| 133 | + if (__get_word(ldq_u, first,src)) |
|---|
| 134 | + return 0; |
|---|
| 149 | 135 | carry = 0; |
|---|
| 150 | 136 | while (len >= 0) { |
|---|
| 151 | 137 | unsigned long second; |
|---|
| 152 | 138 | |
|---|
| 153 | | - err |= __get_user_u(second, src+1); |
|---|
| 139 | + if (__get_word(ldq_u, second, src+1)) |
|---|
| 140 | + return 0; |
|---|
| 154 | 141 | extql(first, soff, word); |
|---|
| 155 | 142 | len -= 8; |
|---|
| 156 | 143 | src++; |
|---|
| .. | .. |
|---|
| 168 | 155 | if (len) { |
|---|
| 169 | 156 | unsigned long tmp; |
|---|
| 170 | 157 | unsigned long second; |
|---|
| 171 | | - err |= __get_user_u(second, lastsrc); |
|---|
| 158 | + if (__get_word(ldq_u, second, lastsrc)) |
|---|
| 159 | + return 0; |
|---|
| 172 | 160 | tmp = *dst; |
|---|
| 173 | 161 | extql(first, soff, word); |
|---|
| 174 | 162 | extqh(second, soff, first); |
|---|
| .. | .. |
|---|
| 180 | 168 | *dst = word | tmp; |
|---|
| 181 | 169 | checksum += carry; |
|---|
| 182 | 170 | } |
|---|
| 183 | | - if (err && errp) *errp = err; |
|---|
| 184 | 171 | return checksum; |
|---|
| 185 | 172 | } |
|---|
| 186 | 173 | |
|---|
| .. | .. |
|---|
| 191 | 178 | csum_partial_cfu_src_aligned(const unsigned long __user *src, |
|---|
| 192 | 179 | unsigned long *dst, |
|---|
| 193 | 180 | unsigned long doff, |
|---|
| 194 | | - long len, unsigned long checksum, |
|---|
| 195 | | - unsigned long partial_dest, |
|---|
| 196 | | - int *errp) |
|---|
| 181 | + long len, |
|---|
| 182 | + unsigned long partial_dest) |
|---|
| 197 | 183 | { |
|---|
| 198 | 184 | unsigned long carry = 0; |
|---|
| 199 | 185 | unsigned long word; |
|---|
| 200 | 186 | unsigned long second_dest; |
|---|
| 201 | | - int err = 0; |
|---|
| 187 | + unsigned long checksum = ~0U; |
|---|
| 202 | 188 | |
|---|
| 203 | 189 | mskql(partial_dest, doff, partial_dest); |
|---|
| 204 | 190 | while (len >= 0) { |
|---|
| 205 | | - err |= __get_user(word, src); |
|---|
| 191 | + if (__get_word(ldq, word, src)) |
|---|
| 192 | + return 0; |
|---|
| 206 | 193 | len -= 8; |
|---|
| 207 | 194 | insql(word, doff, second_dest); |
|---|
| 208 | 195 | checksum += carry; |
|---|
| .. | .. |
|---|
| 216 | 203 | len += 8; |
|---|
| 217 | 204 | if (len) { |
|---|
| 218 | 205 | checksum += carry; |
|---|
| 219 | | - err |= __get_user(word, src); |
|---|
| 206 | + if (__get_word(ldq, word, src)) |
|---|
| 207 | + return 0; |
|---|
| 220 | 208 | mskql(word, len, word); |
|---|
| 221 | 209 | len -= 8; |
|---|
| 222 | 210 | checksum += word; |
|---|
| .. | .. |
|---|
| 237 | 225 | stq_u(partial_dest | second_dest, dst); |
|---|
| 238 | 226 | out: |
|---|
| 239 | 227 | checksum += carry; |
|---|
| 240 | | - if (err && errp) *errp = err; |
|---|
| 241 | 228 | return checksum; |
|---|
| 242 | 229 | } |
|---|
| 243 | 230 | |
|---|
| .. | .. |
|---|
| 249 | 236 | csum_partial_cfu_unaligned(const unsigned long __user * src, |
|---|
| 250 | 237 | unsigned long * dst, |
|---|
| 251 | 238 | unsigned long soff, unsigned long doff, |
|---|
| 252 | | - long len, unsigned long checksum, |
|---|
| 253 | | - unsigned long partial_dest, |
|---|
| 254 | | - int *errp) |
|---|
| 239 | + long len, unsigned long partial_dest) |
|---|
| 255 | 240 | { |
|---|
| 256 | 241 | unsigned long carry = 0; |
|---|
| 257 | 242 | unsigned long first; |
|---|
| 258 | 243 | unsigned long lastsrc; |
|---|
| 259 | | - int err = 0; |
|---|
| 244 | + unsigned long checksum = ~0U; |
|---|
| 260 | 245 | |
|---|
| 261 | | - err |= __get_user_u(first, src); |
|---|
| 246 | + if (__get_word(ldq_u, first, src)) |
|---|
| 247 | + return 0; |
|---|
| 262 | 248 | lastsrc = 7+len+(unsigned long)src; |
|---|
| 263 | 249 | mskql(partial_dest, doff, partial_dest); |
|---|
| 264 | 250 | while (len >= 0) { |
|---|
| 265 | 251 | unsigned long second, word; |
|---|
| 266 | 252 | unsigned long second_dest; |
|---|
| 267 | 253 | |
|---|
| 268 | | - err |= __get_user_u(second, src+1); |
|---|
| 254 | + if (__get_word(ldq_u, second, src+1)) |
|---|
| 255 | + return 0; |
|---|
| 269 | 256 | extql(first, soff, word); |
|---|
| 270 | 257 | checksum += carry; |
|---|
| 271 | 258 | len -= 8; |
|---|
| .. | .. |
|---|
| 286 | 273 | unsigned long second, word; |
|---|
| 287 | 274 | unsigned long second_dest; |
|---|
| 288 | 275 | |
|---|
| 289 | | - err |= __get_user_u(second, lastsrc); |
|---|
| 276 | + if (__get_word(ldq_u, second, lastsrc)) |
|---|
| 277 | + return 0; |
|---|
| 290 | 278 | extql(first, soff, word); |
|---|
| 291 | 279 | extqh(second, soff, first); |
|---|
| 292 | 280 | word |= first; |
|---|
| .. | .. |
|---|
| 307 | 295 | unsigned long second, word; |
|---|
| 308 | 296 | unsigned long second_dest; |
|---|
| 309 | 297 | |
|---|
| 310 | | - err |= __get_user_u(second, lastsrc); |
|---|
| 298 | + if (__get_word(ldq_u, second, lastsrc)) |
|---|
| 299 | + return 0; |
|---|
| 311 | 300 | extql(first, soff, word); |
|---|
| 312 | 301 | extqh(second, soff, first); |
|---|
| 313 | 302 | word |= first; |
|---|
| .. | .. |
|---|
| 320 | 309 | stq_u(partial_dest | word | second_dest, dst); |
|---|
| 321 | 310 | checksum += carry; |
|---|
| 322 | 311 | } |
|---|
| 323 | | - if (err && errp) *errp = err; |
|---|
| 324 | 312 | return checksum; |
|---|
| 325 | 313 | } |
|---|
| 326 | 314 | |
|---|
| 327 | | -__wsum |
|---|
| 328 | | -csum_partial_copy_from_user(const void __user *src, void *dst, int len, |
|---|
| 329 | | - __wsum sum, int *errp) |
|---|
| 315 | +static __wsum __csum_and_copy(const void __user *src, void *dst, int len) |
|---|
| 330 | 316 | { |
|---|
| 331 | | - unsigned long checksum = (__force u32) sum; |
|---|
| 332 | 317 | unsigned long soff = 7 & (unsigned long) src; |
|---|
| 333 | 318 | unsigned long doff = 7 & (unsigned long) dst; |
|---|
| 319 | + unsigned long checksum; |
|---|
| 334 | 320 | |
|---|
| 335 | | - if (len) { |
|---|
| 336 | | - if (!access_ok(VERIFY_READ, src, len)) { |
|---|
| 337 | | - if (errp) *errp = -EFAULT; |
|---|
| 338 | | - memset(dst, 0, len); |
|---|
| 339 | | - return sum; |
|---|
| 340 | | - } |
|---|
| 341 | | - if (!doff) { |
|---|
| 342 | | - if (!soff) |
|---|
| 343 | | - checksum = csum_partial_cfu_aligned( |
|---|
| 344 | | - (const unsigned long __user *) src, |
|---|
| 345 | | - (unsigned long *) dst, |
|---|
| 346 | | - len-8, checksum, errp); |
|---|
| 347 | | - else |
|---|
| 348 | | - checksum = csum_partial_cfu_dest_aligned( |
|---|
| 349 | | - (const unsigned long __user *) src, |
|---|
| 350 | | - (unsigned long *) dst, |
|---|
| 351 | | - soff, len-8, checksum, errp); |
|---|
| 352 | | - } else { |
|---|
| 353 | | - unsigned long partial_dest; |
|---|
| 354 | | - ldq_u(partial_dest, dst); |
|---|
| 355 | | - if (!soff) |
|---|
| 356 | | - checksum = csum_partial_cfu_src_aligned( |
|---|
| 357 | | - (const unsigned long __user *) src, |
|---|
| 358 | | - (unsigned long *) dst, |
|---|
| 359 | | - doff, len-8, checksum, |
|---|
| 360 | | - partial_dest, errp); |
|---|
| 361 | | - else |
|---|
| 362 | | - checksum = csum_partial_cfu_unaligned( |
|---|
| 363 | | - (const unsigned long __user *) src, |
|---|
| 364 | | - (unsigned long *) dst, |
|---|
| 365 | | - soff, doff, len-8, checksum, |
|---|
| 366 | | - partial_dest, errp); |
|---|
| 367 | | - } |
|---|
| 368 | | - checksum = from64to16 (checksum); |
|---|
| 321 | + if (!doff) { |
|---|
| 322 | + if (!soff) |
|---|
| 323 | + checksum = csum_partial_cfu_aligned( |
|---|
| 324 | + (const unsigned long __user *) src, |
|---|
| 325 | + (unsigned long *) dst, len-8); |
|---|
| 326 | + else |
|---|
| 327 | + checksum = csum_partial_cfu_dest_aligned( |
|---|
| 328 | + (const unsigned long __user *) src, |
|---|
| 329 | + (unsigned long *) dst, |
|---|
| 330 | + soff, len-8); |
|---|
| 331 | + } else { |
|---|
| 332 | + unsigned long partial_dest; |
|---|
| 333 | + ldq_u(partial_dest, dst); |
|---|
| 334 | + if (!soff) |
|---|
| 335 | + checksum = csum_partial_cfu_src_aligned( |
|---|
| 336 | + (const unsigned long __user *) src, |
|---|
| 337 | + (unsigned long *) dst, |
|---|
| 338 | + doff, len-8, partial_dest); |
|---|
| 339 | + else |
|---|
| 340 | + checksum = csum_partial_cfu_unaligned( |
|---|
| 341 | + (const unsigned long __user *) src, |
|---|
| 342 | + (unsigned long *) dst, |
|---|
| 343 | + soff, doff, len-8, partial_dest); |
|---|
| 369 | 344 | } |
|---|
| 370 | | - return (__force __wsum)checksum; |
|---|
| 345 | + return (__force __wsum)from64to16 (checksum); |
|---|
| 371 | 346 | } |
|---|
| 372 | | -EXPORT_SYMBOL(csum_partial_copy_from_user); |
|---|
| 373 | 347 | |
|---|
| 374 | 348 | __wsum |
|---|
| 375 | | -csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) |
|---|
| 349 | +csum_and_copy_from_user(const void __user *src, void *dst, int len) |
|---|
| 376 | 350 | { |
|---|
| 377 | | - __wsum checksum; |
|---|
| 378 | | - mm_segment_t oldfs = get_fs(); |
|---|
| 379 | | - set_fs(KERNEL_DS); |
|---|
| 380 | | - checksum = csum_partial_copy_from_user((__force const void __user *)src, |
|---|
| 381 | | - dst, len, sum, NULL); |
|---|
| 382 | | - set_fs(oldfs); |
|---|
| 383 | | - return checksum; |
|---|
| 351 | + if (!access_ok(src, len)) |
|---|
| 352 | + return 0; |
|---|
| 353 | + return __csum_and_copy(src, dst, len); |
|---|
| 354 | +} |
|---|
| 355 | +EXPORT_SYMBOL(csum_and_copy_from_user); |
|---|
| 356 | + |
|---|
| 357 | +__wsum |
|---|
| 358 | +csum_partial_copy_nocheck(const void *src, void *dst, int len) |
|---|
| 359 | +{ |
|---|
| 360 | + return __csum_and_copy((__force const void __user *)src, |
|---|
| 361 | + dst, len); |
|---|
| 384 | 362 | } |
|---|
| 385 | 363 | EXPORT_SYMBOL(csum_partial_copy_nocheck); |
|---|