| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Copyright (C) 2012 Google, Inc. |
|---|
| 3 | | - * |
|---|
| 4 | | - * This software is licensed under the terms of the GNU General Public |
|---|
| 5 | | - * License version 2, as published by the Free Software Foundation, and |
|---|
| 6 | | - * may be copied, distributed, and modified under those terms. |
|---|
| 7 | | - * |
|---|
| 8 | | - * This program is distributed in the hope that it will be useful, |
|---|
| 9 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 10 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 11 | | - * GNU General Public License for more details. |
|---|
| 12 | | - * |
|---|
| 13 | 4 | */ |
|---|
| 14 | 5 | |
|---|
| 15 | | -#define pr_fmt(fmt) "persistent_ram: " fmt |
|---|
| 6 | +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
|---|
| 16 | 7 | |
|---|
| 17 | 8 | #include <linux/device.h> |
|---|
| 18 | 9 | #include <linux/err.h> |
|---|
| .. | .. |
|---|
| 28 | 19 | #include <linux/uaccess.h> |
|---|
| 29 | 20 | #include <linux/vmalloc.h> |
|---|
| 30 | 21 | #include <asm/page.h> |
|---|
| 22 | + |
|---|
| 23 | +/** |
|---|
| 24 | + * struct persistent_ram_buffer - persistent circular RAM buffer |
|---|
| 25 | + * |
|---|
| 26 | + * @sig: |
|---|
| 27 | + * signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value) |
|---|
| 28 | + * @start: |
|---|
| 29 | + * offset into @data where the beginning of the stored bytes begin |
|---|
| 30 | + * @size: |
|---|
| 31 | + * number of valid bytes stored in @data |
|---|
| 32 | + */ |
|---|
| 33 | +struct persistent_ram_buffer { |
|---|
| 34 | + uint32_t sig; |
|---|
| 35 | + atomic_t start; |
|---|
| 36 | + atomic_t size; |
|---|
| 37 | + uint8_t data[]; |
|---|
| 38 | +}; |
|---|
| 31 | 39 | |
|---|
| 32 | 40 | #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ |
|---|
| 33 | 41 | |
|---|
| .. | .. |
|---|
| 275 | 283 | const void __user *s, unsigned int start, unsigned int count) |
|---|
| 276 | 284 | { |
|---|
| 277 | 285 | struct persistent_ram_buffer *buffer = prz->buffer; |
|---|
| 278 | | - int ret = unlikely(__copy_from_user(buffer->data + start, s, count)) ? |
|---|
| 286 | + int ret = unlikely(copy_from_user(buffer->data + start, s, count)) ? |
|---|
| 279 | 287 | -EFAULT : 0; |
|---|
| 280 | 288 | persistent_ram_update_ecc(prz, start, count); |
|---|
| 281 | 289 | return ret; |
|---|
| .. | .. |
|---|
| 340 | 348 | int rem, ret = 0, c = count; |
|---|
| 341 | 349 | size_t start; |
|---|
| 342 | 350 | |
|---|
| 343 | | - if (unlikely(!access_ok(VERIFY_READ, s, count))) |
|---|
| 344 | | - return -EFAULT; |
|---|
| 345 | 351 | if (unlikely(c > prz->buffer_size)) { |
|---|
| 346 | 352 | s += c - prz->buffer_size; |
|---|
| 347 | 353 | c = prz->buffer_size; |
|---|
| .. | .. |
|---|
| 390 | 396 | persistent_ram_update_header_ecc(prz); |
|---|
| 391 | 397 | } |
|---|
| 392 | 398 | |
|---|
| 399 | +#define MEM_TYPE_WCOMBINE 0 |
|---|
| 400 | +#define MEM_TYPE_NONCACHED 1 |
|---|
| 401 | +#define MEM_TYPE_NORMAL 2 |
|---|
| 402 | + |
|---|
| 393 | 403 | static void *persistent_ram_vmap(phys_addr_t start, size_t size, |
|---|
| 394 | 404 | unsigned int memtype) |
|---|
| 395 | 405 | { |
|---|
| .. | .. |
|---|
| 403 | 413 | page_start = start - offset_in_page(start); |
|---|
| 404 | 414 | page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); |
|---|
| 405 | 415 | |
|---|
| 406 | | - if (memtype) |
|---|
| 416 | + switch (memtype) { |
|---|
| 417 | + case MEM_TYPE_NORMAL: |
|---|
| 418 | + prot = PAGE_KERNEL; |
|---|
| 419 | + break; |
|---|
| 420 | + case MEM_TYPE_NONCACHED: |
|---|
| 407 | 421 | prot = pgprot_noncached(PAGE_KERNEL); |
|---|
| 408 | | - else |
|---|
| 422 | + break; |
|---|
| 423 | + case MEM_TYPE_WCOMBINE: |
|---|
| 409 | 424 | prot = pgprot_writecombine(PAGE_KERNEL); |
|---|
| 425 | + break; |
|---|
| 426 | + default: |
|---|
| 427 | + pr_err("invalid mem_type=%d\n", memtype); |
|---|
| 428 | + return NULL; |
|---|
| 429 | + } |
|---|
| 410 | 430 | |
|---|
| 411 | 431 | pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); |
|---|
| 412 | 432 | if (!pages) { |
|---|
| .. | .. |
|---|
| 419 | 439 | phys_addr_t addr = page_start + i * PAGE_SIZE; |
|---|
| 420 | 440 | pages[i] = pfn_to_page(addr >> PAGE_SHIFT); |
|---|
| 421 | 441 | } |
|---|
| 422 | | - vaddr = vmap(pages, page_count, VM_MAP, prot); |
|---|
| 442 | + /* |
|---|
| 443 | + * VM_IOREMAP used here to bypass this region during vread() |
|---|
| 444 | + * and kmap_atomic() (i.e. kcore) to avoid __va() failures. |
|---|
| 445 | + */ |
|---|
| 446 | + vaddr = vmap(pages, page_count, VM_MAP | VM_IOREMAP, prot); |
|---|
| 423 | 447 | kfree(pages); |
|---|
| 424 | 448 | |
|---|
| 425 | 449 | /* |
|---|
| .. | .. |
|---|
| 431 | 455 | } |
|---|
| 432 | 456 | |
|---|
| 433 | 457 | static void *persistent_ram_iomap(phys_addr_t start, size_t size, |
|---|
| 434 | | - unsigned int memtype) |
|---|
| 458 | + unsigned int memtype, char *label) |
|---|
| 435 | 459 | { |
|---|
| 436 | 460 | void *va; |
|---|
| 437 | 461 | |
|---|
| 438 | | - if (!request_mem_region(start, size, "persistent_ram")) { |
|---|
| 439 | | - pr_err("request mem region (0x%llx@0x%llx) failed\n", |
|---|
| 462 | + if (!request_mem_region(start, size, label ?: "ramoops")) { |
|---|
| 463 | + pr_err("request mem region (%s 0x%llx@0x%llx) failed\n", |
|---|
| 464 | + label ?: "ramoops", |
|---|
| 440 | 465 | (unsigned long long)size, (unsigned long long)start); |
|---|
| 441 | 466 | return NULL; |
|---|
| 442 | 467 | } |
|---|
| .. | .. |
|---|
| 463 | 488 | if (pfn_valid(start >> PAGE_SHIFT)) |
|---|
| 464 | 489 | prz->vaddr = persistent_ram_vmap(start, size, memtype); |
|---|
| 465 | 490 | else |
|---|
| 466 | | - prz->vaddr = persistent_ram_iomap(start, size, memtype); |
|---|
| 491 | + prz->vaddr = persistent_ram_iomap(start, size, memtype, |
|---|
| 492 | + prz->label); |
|---|
| 467 | 493 | |
|---|
| 468 | 494 | if (!prz->vaddr) { |
|---|
| 469 | 495 | pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, |
|---|
| .. | .. |
|---|
| 481 | 507 | struct persistent_ram_ecc_info *ecc_info) |
|---|
| 482 | 508 | { |
|---|
| 483 | 509 | int ret; |
|---|
| 510 | + bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD); |
|---|
| 484 | 511 | |
|---|
| 485 | 512 | ret = persistent_ram_init_ecc(prz, ecc_info); |
|---|
| 486 | | - if (ret) |
|---|
| 513 | + if (ret) { |
|---|
| 514 | + pr_warn("ECC failed %s\n", prz->label); |
|---|
| 487 | 515 | return ret; |
|---|
| 516 | + } |
|---|
| 488 | 517 | |
|---|
| 489 | 518 | sig ^= PERSISTENT_RAM_SIG; |
|---|
| 490 | 519 | |
|---|
| 491 | 520 | if (prz->buffer->sig == sig) { |
|---|
| 492 | | - if (buffer_size(prz) == 0) { |
|---|
| 521 | + if (buffer_size(prz) == 0 && buffer_start(prz) == 0) { |
|---|
| 493 | 522 | pr_debug("found existing empty buffer\n"); |
|---|
| 494 | 523 | return 0; |
|---|
| 495 | 524 | } |
|---|
| 496 | 525 | |
|---|
| 497 | 526 | if (buffer_size(prz) > prz->buffer_size || |
|---|
| 498 | | - buffer_start(prz) > buffer_size(prz)) |
|---|
| 527 | + buffer_start(prz) > buffer_size(prz)) { |
|---|
| 499 | 528 | pr_info("found existing invalid buffer, size %zu, start %zu\n", |
|---|
| 500 | 529 | buffer_size(prz), buffer_start(prz)); |
|---|
| 501 | | - else { |
|---|
| 530 | + zap = true; |
|---|
| 531 | + } else { |
|---|
| 502 | 532 | pr_debug("found existing buffer, size %zu, start %zu\n", |
|---|
| 503 | 533 | buffer_size(prz), buffer_start(prz)); |
|---|
| 504 | 534 | persistent_ram_save_old(prz); |
|---|
| 505 | | - return 0; |
|---|
| 506 | 535 | } |
|---|
| 507 | 536 | } else { |
|---|
| 508 | 537 | pr_debug("no valid data in buffer (sig = 0x%08x)\n", |
|---|
| 509 | 538 | prz->buffer->sig); |
|---|
| 539 | + prz->buffer->sig = sig; |
|---|
| 540 | + zap = true; |
|---|
| 510 | 541 | } |
|---|
| 511 | 542 | |
|---|
| 512 | | - /* Rewind missing or invalid memory area. */ |
|---|
| 513 | | - prz->buffer->sig = sig; |
|---|
| 514 | | - persistent_ram_zap(prz); |
|---|
| 543 | + /* Reset missing, invalid, or single-use memory area. */ |
|---|
| 544 | + if (zap) |
|---|
| 545 | + persistent_ram_zap(prz); |
|---|
| 515 | 546 | |
|---|
| 516 | 547 | return 0; |
|---|
| 517 | 548 | } |
|---|
| .. | .. |
|---|
| 539 | 570 | prz->ecc_info.par = NULL; |
|---|
| 540 | 571 | |
|---|
| 541 | 572 | persistent_ram_free_old(prz); |
|---|
| 573 | + kfree(prz->label); |
|---|
| 542 | 574 | kfree(prz); |
|---|
| 543 | 575 | } |
|---|
| 544 | 576 | |
|---|
| 545 | 577 | struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, |
|---|
| 546 | 578 | u32 sig, struct persistent_ram_ecc_info *ecc_info, |
|---|
| 547 | | - unsigned int memtype, u32 flags) |
|---|
| 579 | + unsigned int memtype, u32 flags, char *label) |
|---|
| 548 | 580 | { |
|---|
| 549 | 581 | struct persistent_ram_zone *prz; |
|---|
| 550 | 582 | int ret = -ENOMEM; |
|---|
| .. | .. |
|---|
| 558 | 590 | /* Initialize general buffer state. */ |
|---|
| 559 | 591 | raw_spin_lock_init(&prz->buffer_lock); |
|---|
| 560 | 592 | prz->flags = flags; |
|---|
| 593 | + prz->label = kstrdup(label, GFP_KERNEL); |
|---|
| 594 | + if (!prz->label) |
|---|
| 595 | + goto err; |
|---|
| 561 | 596 | |
|---|
| 562 | 597 | ret = persistent_ram_buffer_map(start, size, prz, memtype); |
|---|
| 563 | 598 | if (ret) |
|---|
| .. | .. |
|---|
| 567 | 602 | if (ret) |
|---|
| 568 | 603 | goto err; |
|---|
| 569 | 604 | |
|---|
| 605 | + pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n", |
|---|
| 606 | + prz->label, prz->size, (unsigned long long)prz->paddr, |
|---|
| 607 | + sizeof(*prz->buffer), prz->buffer_size, |
|---|
| 608 | + prz->size - sizeof(*prz->buffer) - prz->buffer_size, |
|---|
| 609 | + prz->ecc_info.ecc_size, prz->ecc_info.block_size); |
|---|
| 610 | + |
|---|
| 570 | 611 | return prz; |
|---|
| 571 | 612 | err: |
|---|
| 572 | 613 | persistent_ram_free(prz); |
|---|