| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights |
|---|
| 3 | 4 | * reserved. |
|---|
| 4 | | - * |
|---|
| 5 | | - * This program is free software; you can redistribute it and/or modify it |
|---|
| 6 | | - * under the terms of version 2 of the GNU General Public License |
|---|
| 7 | | - * as published by the Free Software Foundation. |
|---|
| 8 | 5 | */ |
|---|
| 9 | 6 | |
|---|
| 10 | 7 | /* |
|---|
| .. | .. |
|---|
| 12 | 9 | * |
|---|
| 13 | 10 | * This driver exports the SN special memory (mspec) facility to user |
|---|
| 14 | 11 | * processes. |
|---|
| 15 | | - * There are three types of memory made available thru this driver: |
|---|
| 16 | | - * fetchops, uncached and cached. |
|---|
| 17 | | - * |
|---|
| 18 | | - * Fetchops are atomic memory operations that are implemented in the |
|---|
| 19 | | - * memory controller on SGI SN hardware. |
|---|
| 12 | + * There are two types of memory made available thru this driver: |
|---|
| 13 | + * uncached and cached. |
|---|
| 20 | 14 | * |
|---|
| 21 | 15 | * Uncached are used for memory write combining feature of the ia64 |
|---|
| 22 | 16 | * cpu. |
|---|
| .. | .. |
|---|
| 45 | 39 | #include <linux/numa.h> |
|---|
| 46 | 40 | #include <linux/refcount.h> |
|---|
| 47 | 41 | #include <asm/page.h> |
|---|
| 48 | | -#include <asm/pgtable.h> |
|---|
| 49 | 42 | #include <linux/atomic.h> |
|---|
| 50 | 43 | #include <asm/tlbflush.h> |
|---|
| 51 | 44 | #include <asm/uncached.h> |
|---|
| 52 | | -#include <asm/sn/addrs.h> |
|---|
| 53 | | -#include <asm/sn/arch.h> |
|---|
| 54 | | -#include <asm/sn/mspec.h> |
|---|
| 55 | | -#include <asm/sn/sn_cpuid.h> |
|---|
| 56 | | -#include <asm/sn/io.h> |
|---|
| 57 | | -#include <asm/sn/bte.h> |
|---|
| 58 | | -#include <asm/sn/shubio.h> |
|---|
| 59 | 45 | |
|---|
| 60 | 46 | |
|---|
| 61 | | -#define FETCHOP_ID "SGI Fetchop," |
|---|
| 62 | 47 | #define CACHED_ID "Cached," |
|---|
| 63 | 48 | #define UNCACHED_ID "Uncached" |
|---|
| 64 | 49 | #define REVISION "4.0" |
|---|
| .. | .. |
|---|
| 68 | 53 | * Page types allocated by the device. |
|---|
| 69 | 54 | */ |
|---|
| 70 | 55 | enum mspec_page_type { |
|---|
| 71 | | - MSPEC_FETCHOP = 1, |
|---|
| 72 | | - MSPEC_CACHED, |
|---|
| 56 | + MSPEC_CACHED = 2, |
|---|
| 73 | 57 | MSPEC_UNCACHED |
|---|
| 74 | 58 | }; |
|---|
| 75 | | - |
|---|
| 76 | | -#ifdef CONFIG_SGI_SN |
|---|
| 77 | | -static int is_sn2; |
|---|
| 78 | | -#else |
|---|
| 79 | | -#define is_sn2 0 |
|---|
| 80 | | -#endif |
|---|
| 81 | 59 | |
|---|
| 82 | 60 | /* |
|---|
| 83 | 61 | * One of these structures is allocated when an mspec region is mmaped. The |
|---|
| .. | .. |
|---|
| 86 | 64 | * This structure is shared by all vma's that are split off from the |
|---|
| 87 | 65 | * original vma when split_vma()'s are done. |
|---|
| 88 | 66 | * |
|---|
| 89 | | - * The refcnt is incremented atomically because mm->mmap_sem does not |
|---|
| 67 | + * The refcnt is incremented atomically because mm->mmap_lock does not |
|---|
| 90 | 68 | * protect in fork case where multiple tasks share the vma_data. |
|---|
| 91 | 69 | */ |
|---|
| 92 | 70 | struct vma_data { |
|---|
| .. | .. |
|---|
| 96 | 74 | enum mspec_page_type type; /* Type of pages allocated. */ |
|---|
| 97 | 75 | unsigned long vm_start; /* Original (unsplit) base. */ |
|---|
| 98 | 76 | unsigned long vm_end; /* Original (unsplit) end. */ |
|---|
| 99 | | - unsigned long maddr[0]; /* Array of MSPEC addresses. */ |
|---|
| 77 | + unsigned long maddr[]; /* Array of MSPEC addresses. */ |
|---|
| 100 | 78 | }; |
|---|
| 101 | | - |
|---|
| 102 | | -/* used on shub2 to clear FOP cache in the HUB */ |
|---|
| 103 | | -static unsigned long scratch_page[MAX_NUMNODES]; |
|---|
| 104 | | -#define SH2_AMO_CACHE_ENTRIES 4 |
|---|
| 105 | | - |
|---|
| 106 | | -static inline int |
|---|
| 107 | | -mspec_zero_block(unsigned long addr, int len) |
|---|
| 108 | | -{ |
|---|
| 109 | | - int status; |
|---|
| 110 | | - |
|---|
| 111 | | - if (is_sn2) { |
|---|
| 112 | | - if (is_shub2()) { |
|---|
| 113 | | - int nid; |
|---|
| 114 | | - void *p; |
|---|
| 115 | | - int i; |
|---|
| 116 | | - |
|---|
| 117 | | - nid = nasid_to_cnodeid(get_node_number(__pa(addr))); |
|---|
| 118 | | - p = (void *)TO_AMO(scratch_page[nid]); |
|---|
| 119 | | - |
|---|
| 120 | | - for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) { |
|---|
| 121 | | - FETCHOP_LOAD_OP(p, FETCHOP_LOAD); |
|---|
| 122 | | - p += FETCHOP_VAR_SIZE; |
|---|
| 123 | | - } |
|---|
| 124 | | - } |
|---|
| 125 | | - |
|---|
| 126 | | - status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len, |
|---|
| 127 | | - BTE_WACQUIRE | BTE_ZERO_FILL, NULL); |
|---|
| 128 | | - } else { |
|---|
| 129 | | - memset((char *) addr, 0, len); |
|---|
| 130 | | - status = 0; |
|---|
| 131 | | - } |
|---|
| 132 | | - return status; |
|---|
| 133 | | -} |
|---|
| 134 | 79 | |
|---|
| 135 | 80 | /* |
|---|
| 136 | 81 | * mspec_open |
|---|
| .. | .. |
|---|
| 176 | 121 | */ |
|---|
| 177 | 122 | my_page = vdata->maddr[index]; |
|---|
| 178 | 123 | vdata->maddr[index] = 0; |
|---|
| 179 | | - if (!mspec_zero_block(my_page, PAGE_SIZE)) |
|---|
| 180 | | - uncached_free_page(my_page, 1); |
|---|
| 181 | | - else |
|---|
| 182 | | - printk(KERN_WARNING "mspec_close(): " |
|---|
| 183 | | - "failed to zero page %ld\n", my_page); |
|---|
| 124 | + memset((char *)my_page, 0, PAGE_SIZE); |
|---|
| 125 | + uncached_free_page(my_page, 1); |
|---|
| 184 | 126 | } |
|---|
| 185 | 127 | |
|---|
| 186 | 128 | kvfree(vdata); |
|---|
| .. | .. |
|---|
| 216 | 158 | spin_unlock(&vdata->lock); |
|---|
| 217 | 159 | } |
|---|
| 218 | 160 | |
|---|
| 219 | | - if (vdata->type == MSPEC_FETCHOP) |
|---|
| 220 | | - paddr = TO_AMO(maddr); |
|---|
| 221 | | - else |
|---|
| 222 | | - paddr = maddr & ~__IA64_UNCACHED_OFFSET; |
|---|
| 223 | | - |
|---|
| 161 | + paddr = maddr & ~__IA64_UNCACHED_OFFSET; |
|---|
| 224 | 162 | pfn = paddr >> PAGE_SHIFT; |
|---|
| 225 | 163 | |
|---|
| 226 | 164 | return vmf_insert_pfn(vmf->vma, vmf->address, pfn); |
|---|
| .. | .. |
|---|
| 257 | 195 | |
|---|
| 258 | 196 | pages = vma_pages(vma); |
|---|
| 259 | 197 | vdata_size = sizeof(struct vma_data) + pages * sizeof(long); |
|---|
| 260 | | - if (vdata_size <= PAGE_SIZE) |
|---|
| 261 | | - vdata = kzalloc(vdata_size, GFP_KERNEL); |
|---|
| 262 | | - else |
|---|
| 263 | | - vdata = vzalloc(vdata_size); |
|---|
| 198 | + vdata = kvzalloc(vdata_size, GFP_KERNEL); |
|---|
| 264 | 199 | if (!vdata) |
|---|
| 265 | 200 | return -ENOMEM; |
|---|
| 266 | 201 | |
|---|
| .. | .. |
|---|
| 272 | 207 | vma->vm_private_data = vdata; |
|---|
| 273 | 208 | |
|---|
| 274 | 209 | vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; |
|---|
| 275 | | - if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED) |
|---|
| 210 | + if (vdata->type == MSPEC_UNCACHED) |
|---|
| 276 | 211 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
|---|
| 277 | 212 | vma->vm_ops = &mspec_vm_ops; |
|---|
| 278 | 213 | |
|---|
| 279 | 214 | return 0; |
|---|
| 280 | | -} |
|---|
| 281 | | - |
|---|
| 282 | | -static int |
|---|
| 283 | | -fetchop_mmap(struct file *file, struct vm_area_struct *vma) |
|---|
| 284 | | -{ |
|---|
| 285 | | - return mspec_mmap(file, vma, MSPEC_FETCHOP); |
|---|
| 286 | 215 | } |
|---|
| 287 | 216 | |
|---|
| 288 | 217 | static int |
|---|
| .. | .. |
|---|
| 296 | 225 | { |
|---|
| 297 | 226 | return mspec_mmap(file, vma, MSPEC_UNCACHED); |
|---|
| 298 | 227 | } |
|---|
| 299 | | - |
|---|
| 300 | | -static const struct file_operations fetchop_fops = { |
|---|
| 301 | | - .owner = THIS_MODULE, |
|---|
| 302 | | - .mmap = fetchop_mmap, |
|---|
| 303 | | - .llseek = noop_llseek, |
|---|
| 304 | | -}; |
|---|
| 305 | | - |
|---|
| 306 | | -static struct miscdevice fetchop_miscdev = { |
|---|
| 307 | | - .minor = MISC_DYNAMIC_MINOR, |
|---|
| 308 | | - .name = "sgi_fetchop", |
|---|
| 309 | | - .fops = &fetchop_fops |
|---|
| 310 | | -}; |
|---|
| 311 | 228 | |
|---|
| 312 | 229 | static const struct file_operations cached_fops = { |
|---|
| 313 | 230 | .owner = THIS_MODULE, |
|---|
| .. | .. |
|---|
| 342 | 259 | mspec_init(void) |
|---|
| 343 | 260 | { |
|---|
| 344 | 261 | int ret; |
|---|
| 345 | | - int nid; |
|---|
| 346 | 262 | |
|---|
| 347 | | - /* |
|---|
| 348 | | - * The fetchop device only works on SN2 hardware, uncached and cached |
|---|
| 349 | | - * memory drivers should both be valid on all ia64 hardware |
|---|
| 350 | | - */ |
|---|
| 351 | | -#ifdef CONFIG_SGI_SN |
|---|
| 352 | | - if (ia64_platform_is("sn2")) { |
|---|
| 353 | | - is_sn2 = 1; |
|---|
| 354 | | - if (is_shub2()) { |
|---|
| 355 | | - ret = -ENOMEM; |
|---|
| 356 | | - for_each_node_state(nid, N_ONLINE) { |
|---|
| 357 | | - int actual_nid; |
|---|
| 358 | | - int nasid; |
|---|
| 359 | | - unsigned long phys; |
|---|
| 360 | | - |
|---|
| 361 | | - scratch_page[nid] = uncached_alloc_page(nid, 1); |
|---|
| 362 | | - if (scratch_page[nid] == 0) |
|---|
| 363 | | - goto free_scratch_pages; |
|---|
| 364 | | - phys = __pa(scratch_page[nid]); |
|---|
| 365 | | - nasid = get_node_number(phys); |
|---|
| 366 | | - actual_nid = nasid_to_cnodeid(nasid); |
|---|
| 367 | | - if (actual_nid != nid) |
|---|
| 368 | | - goto free_scratch_pages; |
|---|
| 369 | | - } |
|---|
| 370 | | - } |
|---|
| 371 | | - |
|---|
| 372 | | - ret = misc_register(&fetchop_miscdev); |
|---|
| 373 | | - if (ret) { |
|---|
| 374 | | - printk(KERN_ERR |
|---|
| 375 | | - "%s: failed to register device %i\n", |
|---|
| 376 | | - FETCHOP_ID, ret); |
|---|
| 377 | | - goto free_scratch_pages; |
|---|
| 378 | | - } |
|---|
| 379 | | - } |
|---|
| 380 | | -#endif |
|---|
| 381 | 263 | ret = misc_register(&cached_miscdev); |
|---|
| 382 | 264 | if (ret) { |
|---|
| 383 | 265 | printk(KERN_ERR "%s: failed to register device %i\n", |
|---|
| 384 | 266 | CACHED_ID, ret); |
|---|
| 385 | | - if (is_sn2) |
|---|
| 386 | | - misc_deregister(&fetchop_miscdev); |
|---|
| 387 | | - goto free_scratch_pages; |
|---|
| 267 | + return ret; |
|---|
| 388 | 268 | } |
|---|
| 389 | 269 | ret = misc_register(&uncached_miscdev); |
|---|
| 390 | 270 | if (ret) { |
|---|
| 391 | 271 | printk(KERN_ERR "%s: failed to register device %i\n", |
|---|
| 392 | 272 | UNCACHED_ID, ret); |
|---|
| 393 | 273 | misc_deregister(&cached_miscdev); |
|---|
| 394 | | - if (is_sn2) |
|---|
| 395 | | - misc_deregister(&fetchop_miscdev); |
|---|
| 396 | | - goto free_scratch_pages; |
|---|
| 274 | + return ret; |
|---|
| 397 | 275 | } |
|---|
| 398 | 276 | |
|---|
| 399 | | - printk(KERN_INFO "%s %s initialized devices: %s %s %s\n", |
|---|
| 400 | | - MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "", |
|---|
| 401 | | - CACHED_ID, UNCACHED_ID); |
|---|
| 277 | + printk(KERN_INFO "%s %s initialized devices: %s %s\n", |
|---|
| 278 | + MSPEC_BASENAME, REVISION, CACHED_ID, UNCACHED_ID); |
|---|
| 402 | 279 | |
|---|
| 403 | 280 | return 0; |
|---|
| 404 | | - |
|---|
| 405 | | - free_scratch_pages: |
|---|
| 406 | | - for_each_node(nid) { |
|---|
| 407 | | - if (scratch_page[nid] != 0) |
|---|
| 408 | | - uncached_free_page(scratch_page[nid], 1); |
|---|
| 409 | | - } |
|---|
| 410 | | - return ret; |
|---|
| 411 | 281 | } |
|---|
| 412 | 282 | |
|---|
| 413 | 283 | static void __exit |
|---|
| 414 | 284 | mspec_exit(void) |
|---|
| 415 | 285 | { |
|---|
| 416 | | - int nid; |
|---|
| 417 | | - |
|---|
| 418 | 286 | misc_deregister(&uncached_miscdev); |
|---|
| 419 | 287 | misc_deregister(&cached_miscdev); |
|---|
| 420 | | - if (is_sn2) { |
|---|
| 421 | | - misc_deregister(&fetchop_miscdev); |
|---|
| 422 | | - |
|---|
| 423 | | - for_each_node(nid) { |
|---|
| 424 | | - if (scratch_page[nid] != 0) |
|---|
| 425 | | - uncached_free_page(scratch_page[nid], 1); |
|---|
| 426 | | - } |
|---|
| 427 | | - } |
|---|
| 428 | 288 | } |
|---|
| 429 | 289 | |
|---|
| 430 | 290 | module_init(mspec_init); |
|---|