.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /****************************************************************************** |
---|
2 | 3 | * privcmd.c |
---|
3 | 4 | * |
---|
.. | .. |
---|
24 | 25 | #include <linux/miscdevice.h> |
---|
25 | 26 | #include <linux/moduleparam.h> |
---|
26 | 27 | |
---|
27 | | -#include <asm/pgalloc.h> |
---|
28 | | -#include <asm/pgtable.h> |
---|
29 | | -#include <asm/tlb.h> |
---|
30 | 28 | #include <asm/xen/hypervisor.h> |
---|
31 | 29 | #include <asm/xen/hypercall.h> |
---|
32 | 30 | |
---|
.. | .. |
---|
277 | 275 | if (rc || list_empty(&pagelist)) |
---|
278 | 276 | goto out; |
---|
279 | 277 | |
---|
280 | | - down_write(&mm->mmap_sem); |
---|
| 278 | + mmap_write_lock(mm); |
---|
281 | 279 | |
---|
282 | 280 | { |
---|
283 | 281 | struct page *page = list_first_entry(&pagelist, |
---|
.. | .. |
---|
302 | 300 | |
---|
303 | 301 | |
---|
304 | 302 | out_up: |
---|
305 | | - up_write(&mm->mmap_sem); |
---|
| 303 | + mmap_write_unlock(mm); |
---|
306 | 304 | |
---|
307 | 305 | out: |
---|
308 | 306 | free_page_list(&pagelist); |
---|
.. | .. |
---|
426 | 424 | if (pages == NULL) |
---|
427 | 425 | return -ENOMEM; |
---|
428 | 426 | |
---|
429 | | - rc = alloc_xenballooned_pages(numpgs, pages); |
---|
| 427 | + rc = xen_alloc_unpopulated_pages(numpgs, pages); |
---|
430 | 428 | if (rc != 0) { |
---|
431 | 429 | pr_warn("%s Could not alloc %d pfns rc:%d\n", __func__, |
---|
432 | 430 | numpgs, rc); |
---|
.. | .. |
---|
459 | 457 | return -EFAULT; |
---|
460 | 458 | /* Returns per-frame error in m.arr. */ |
---|
461 | 459 | m.err = NULL; |
---|
462 | | - if (!access_ok(VERIFY_WRITE, m.arr, m.num * sizeof(*m.arr))) |
---|
| 460 | + if (!access_ok(m.arr, m.num * sizeof(*m.arr))) |
---|
463 | 461 | return -EFAULT; |
---|
464 | 462 | break; |
---|
465 | 463 | case 2: |
---|
466 | 464 | if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch_v2))) |
---|
467 | 465 | return -EFAULT; |
---|
468 | 466 | /* Returns per-frame error code in m.err. */ |
---|
469 | | - if (!access_ok(VERIFY_WRITE, m.err, m.num * (sizeof(*m.err)))) |
---|
| 467 | + if (!access_ok(m.err, m.num * (sizeof(*m.err)))) |
---|
470 | 468 | return -EFAULT; |
---|
471 | 469 | break; |
---|
472 | 470 | default: |
---|
.. | .. |
---|
498 | 496 | } |
---|
499 | 497 | } |
---|
500 | 498 | |
---|
501 | | - down_write(&mm->mmap_sem); |
---|
| 499 | + mmap_write_lock(mm); |
---|
502 | 500 | |
---|
503 | 501 | vma = find_vma(mm, m.addr); |
---|
504 | 502 | if (!vma || |
---|
.. | .. |
---|
554 | 552 | BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t), |
---|
555 | 553 | &pagelist, mmap_batch_fn, &state)); |
---|
556 | 554 | |
---|
557 | | - up_write(&mm->mmap_sem); |
---|
| 555 | + mmap_write_unlock(mm); |
---|
558 | 556 | |
---|
559 | 557 | if (state.global_error) { |
---|
560 | 558 | /* Write back errors in second pass. */ |
---|
.. | .. |
---|
575 | 573 | return ret; |
---|
576 | 574 | |
---|
577 | 575 | out_unlock: |
---|
578 | | - up_write(&mm->mmap_sem); |
---|
| 576 | + mmap_write_unlock(mm); |
---|
579 | 577 | goto out; |
---|
580 | 578 | } |
---|
581 | 579 | |
---|
582 | 580 | static int lock_pages( |
---|
583 | 581 | struct privcmd_dm_op_buf kbufs[], unsigned int num, |
---|
584 | | - struct page *pages[], unsigned int nr_pages) |
---|
| 582 | + struct page *pages[], unsigned int nr_pages, unsigned int *pinned) |
---|
585 | 583 | { |
---|
586 | | - unsigned int i; |
---|
| 584 | + unsigned int i, off = 0; |
---|
587 | 585 | |
---|
588 | | - for (i = 0; i < num; i++) { |
---|
| 586 | + for (i = 0; i < num; ) { |
---|
589 | 587 | unsigned int requested; |
---|
590 | | - int pinned; |
---|
| 588 | + int page_count; |
---|
591 | 589 | |
---|
592 | 590 | requested = DIV_ROUND_UP( |
---|
593 | 591 | offset_in_page(kbufs[i].uptr) + kbufs[i].size, |
---|
594 | | - PAGE_SIZE); |
---|
| 592 | + PAGE_SIZE) - off; |
---|
595 | 593 | if (requested > nr_pages) |
---|
596 | 594 | return -ENOSPC; |
---|
597 | 595 | |
---|
598 | | - pinned = get_user_pages_fast( |
---|
599 | | - (unsigned long) kbufs[i].uptr, |
---|
| 596 | + page_count = pin_user_pages_fast( |
---|
| 597 | + (unsigned long)kbufs[i].uptr + off * PAGE_SIZE, |
---|
600 | 598 | requested, FOLL_WRITE, pages); |
---|
601 | | - if (pinned < 0) |
---|
602 | | - return pinned; |
---|
| 599 | + if (page_count <= 0) |
---|
| 600 | + return page_count ? : -EFAULT; |
---|
603 | 601 | |
---|
604 | | - nr_pages -= pinned; |
---|
605 | | - pages += pinned; |
---|
| 602 | + *pinned += page_count; |
---|
| 603 | + nr_pages -= page_count; |
---|
| 604 | + pages += page_count; |
---|
| 605 | + |
---|
| 606 | + off = (requested == page_count) ? 0 : off + page_count; |
---|
| 607 | + i += !off; |
---|
606 | 608 | } |
---|
607 | 609 | |
---|
608 | 610 | return 0; |
---|
.. | .. |
---|
610 | 612 | |
---|
611 | 613 | static void unlock_pages(struct page *pages[], unsigned int nr_pages) |
---|
612 | 614 | { |
---|
613 | | - unsigned int i; |
---|
614 | | - |
---|
615 | | - if (!pages) |
---|
616 | | - return; |
---|
617 | | - |
---|
618 | | - for (i = 0; i < nr_pages; i++) { |
---|
619 | | - if (pages[i]) |
---|
620 | | - put_page(pages[i]); |
---|
621 | | - } |
---|
| 615 | + unpin_user_pages_dirty_lock(pages, nr_pages, true); |
---|
622 | 616 | } |
---|
623 | 617 | |
---|
624 | 618 | static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) |
---|
.. | .. |
---|
631 | 625 | struct xen_dm_op_buf *xbufs = NULL; |
---|
632 | 626 | unsigned int i; |
---|
633 | 627 | long rc; |
---|
| 628 | + unsigned int pinned = 0; |
---|
634 | 629 | |
---|
635 | 630 | if (copy_from_user(&kdata, udata, sizeof(kdata))) |
---|
636 | 631 | return -EFAULT; |
---|
.. | .. |
---|
661 | 656 | goto out; |
---|
662 | 657 | } |
---|
663 | 658 | |
---|
664 | | - if (!access_ok(VERIFY_WRITE, kbufs[i].uptr, |
---|
| 659 | + if (!access_ok(kbufs[i].uptr, |
---|
665 | 660 | kbufs[i].size)) { |
---|
666 | 661 | rc = -EFAULT; |
---|
667 | 662 | goto out; |
---|
.. | .. |
---|
684 | 679 | goto out; |
---|
685 | 680 | } |
---|
686 | 681 | |
---|
687 | | - rc = lock_pages(kbufs, kdata.num, pages, nr_pages); |
---|
688 | | - if (rc) |
---|
| 682 | + rc = lock_pages(kbufs, kdata.num, pages, nr_pages, &pinned); |
---|
| 683 | + if (rc < 0) |
---|
689 | 684 | goto out; |
---|
690 | 685 | |
---|
691 | 686 | for (i = 0; i < kdata.num; i++) { |
---|
.. | .. |
---|
698 | 693 | xen_preemptible_hcall_end(); |
---|
699 | 694 | |
---|
700 | 695 | out: |
---|
701 | | - unlock_pages(pages, nr_pages); |
---|
| 696 | + unlock_pages(pages, pinned); |
---|
702 | 697 | kfree(xbufs); |
---|
703 | 698 | kfree(pages); |
---|
704 | 699 | kfree(kbufs); |
---|
.. | .. |
---|
719 | 714 | data->domid = dom; |
---|
720 | 715 | else if (data->domid != dom) |
---|
721 | 716 | return -EINVAL; |
---|
722 | | - |
---|
723 | | - return 0; |
---|
724 | | -} |
---|
725 | | - |
---|
726 | | -struct remap_pfn { |
---|
727 | | - struct mm_struct *mm; |
---|
728 | | - struct page **pages; |
---|
729 | | - pgprot_t prot; |
---|
730 | | - unsigned long i; |
---|
731 | | -}; |
---|
732 | | - |
---|
733 | | -static int remap_pfn_fn(pte_t *ptep, pgtable_t token, unsigned long addr, |
---|
734 | | - void *data) |
---|
735 | | -{ |
---|
736 | | - struct remap_pfn *r = data; |
---|
737 | | - struct page *page = r->pages[r->i]; |
---|
738 | | - pte_t pte = pte_mkspecial(pfn_pte(page_to_pfn(page), r->prot)); |
---|
739 | | - |
---|
740 | | - set_pte_at(r->mm, addr, ptep, pte); |
---|
741 | | - r->i++; |
---|
742 | 717 | |
---|
743 | 718 | return 0; |
---|
744 | 719 | } |
---|
.. | .. |
---|
777 | 752 | return __put_user(xdata.nr_frames, &udata->num); |
---|
778 | 753 | } |
---|
779 | 754 | |
---|
780 | | - down_write(&mm->mmap_sem); |
---|
| 755 | + mmap_write_lock(mm); |
---|
781 | 756 | |
---|
782 | 757 | vma = find_vma(mm, kdata.addr); |
---|
783 | 758 | if (!vma || vma->vm_ops != &privcmd_vm_ops) { |
---|
.. | .. |
---|
785 | 760 | goto out; |
---|
786 | 761 | } |
---|
787 | 762 | |
---|
788 | | - pfns = kcalloc(kdata.num, sizeof(*pfns), GFP_KERNEL); |
---|
| 763 | + pfns = kcalloc(kdata.num, sizeof(*pfns), GFP_KERNEL | __GFP_NOWARN); |
---|
789 | 764 | if (!pfns) { |
---|
790 | 765 | rc = -ENOMEM; |
---|
791 | 766 | goto out; |
---|
792 | 767 | } |
---|
793 | 768 | |
---|
794 | | - if (xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
| 769 | + if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && |
---|
| 770 | + xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
795 | 771 | unsigned int nr = DIV_ROUND_UP(kdata.num, XEN_PFN_PER_PAGE); |
---|
796 | 772 | struct page **pages; |
---|
797 | 773 | unsigned int i; |
---|
.. | .. |
---|
821 | 797 | if (rc) |
---|
822 | 798 | goto out; |
---|
823 | 799 | |
---|
824 | | - if (xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
825 | | - struct remap_pfn r = { |
---|
826 | | - .mm = vma->vm_mm, |
---|
827 | | - .pages = vma->vm_private_data, |
---|
828 | | - .prot = vma->vm_page_prot, |
---|
829 | | - }; |
---|
830 | | - |
---|
831 | | - rc = apply_to_page_range(r.mm, kdata.addr, |
---|
832 | | - kdata.num << PAGE_SHIFT, |
---|
833 | | - remap_pfn_fn, &r); |
---|
| 800 | + if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && |
---|
| 801 | + xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
| 802 | + rc = xen_remap_vma_range(vma, kdata.addr, kdata.num << PAGE_SHIFT); |
---|
834 | 803 | } else { |
---|
835 | 804 | unsigned int domid = |
---|
836 | 805 | (xdata.flags & XENMEM_rsrc_acq_caller_owned) ? |
---|
.. | .. |
---|
859 | 828 | } |
---|
860 | 829 | |
---|
861 | 830 | out: |
---|
862 | | - up_write(&mm->mmap_sem); |
---|
| 831 | + mmap_write_unlock(mm); |
---|
863 | 832 | kfree(pfns); |
---|
864 | 833 | |
---|
865 | 834 | return rc; |
---|
.. | .. |
---|
941 | 910 | |
---|
942 | 911 | rc = xen_unmap_domain_gfn_range(vma, numgfns, pages); |
---|
943 | 912 | if (rc == 0) |
---|
944 | | - free_xenballooned_pages(numpgs, pages); |
---|
| 913 | + xen_free_unpopulated_pages(numpgs, pages); |
---|
945 | 914 | else |
---|
946 | 915 | pr_crit("unable to unmap MFN range: leaking %d pages. rc=%d\n", |
---|
947 | 916 | numpgs, rc); |
---|
.. | .. |
---|
979 | 948 | * on a per pfn/pte basis. Mapping calls that fail with ENOENT |
---|
980 | 949 | * can be then retried until success. |
---|
981 | 950 | */ |
---|
982 | | -static int is_mapped_fn(pte_t *pte, struct page *pmd_page, |
---|
983 | | - unsigned long addr, void *data) |
---|
| 951 | +static int is_mapped_fn(pte_t *pte, unsigned long addr, void *data) |
---|
984 | 952 | { |
---|
985 | 953 | return pte_none(*pte) ? 0 : -EBUSY; |
---|
986 | 954 | } |
---|