forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/kernel/dma/debug.c
....@@ -1,28 +1,17 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Copyright (C) 2008 Advanced Micro Devices, Inc.
34  *
45  * Author: Joerg Roedel <joerg.roedel@amd.com>
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms of the GNU General Public License version 2 as published
8
- * by the Free Software Foundation.
9
- *
10
- * This program is distributed in the hope that it will be useful,
11
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
12
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
- * GNU General Public License for more details.
14
- *
15
- * You should have received a copy of the GNU General Public License
16
- * along with this program; if not, write to the Free Software
17
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
186  */
7
+
8
+#define pr_fmt(fmt)	"DMA-API: " fmt
199 
2010 #include <linux/sched/task_stack.h>
2111 #include <linux/scatterlist.h>
22
-#include <linux/dma-mapping.h>
12
+#include <linux/dma-map-ops.h>
2313 #include <linux/sched/task.h>
2414 #include <linux/stacktrace.h>
25
-#include <linux/dma-debug.h>
2615 #include <linux/spinlock.h>
2716 #include <linux/vmalloc.h>
2817 #include <linux/debugfs.h>
....@@ -34,21 +23,19 @@
3423 #include <linux/ctype.h>
3524 #include <linux/list.h>
3625 #include <linux/slab.h>
37
-
3826 #include <asm/sections.h>
27
+#include "debug.h"
3928 
40
-#define HASH_SIZE       1024ULL
29
+#define HASH_SIZE       16384ULL
4130 #define HASH_FN_SHIFT   13
4231 #define HASH_FN_MASK    (HASH_SIZE - 1)
4332 
44
-/* allow architectures to override this if absolutely required */
45
-#ifndef PREALLOC_DMA_DEBUG_ENTRIES
4633 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
47
-#endif
34
+/* If the pool runs out, add this many new entries at once */
35
+#define DMA_DEBUG_DYNAMIC_ENTRIES (PAGE_SIZE / sizeof(struct dma_debug_entry))
4836 
4937 enum {
5038 	dma_debug_single,
51
-	dma_debug_page,
5239 	dma_debug_sg,
5340 	dma_debug_coherent,
5441 	dma_debug_resource,
....@@ -66,40 +53,40 @@
6653  * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
6754  * @list: node on pre-allocated free_entries list
6855  * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
69
- * @type: single, page, sg, coherent
70
- * @pfn: page frame of the start address
71
- * @offset: offset of mapping relative to pfn
7256  * @size: length of the mapping
57
+ * @type: single, page, sg, coherent
7358  * @direction: enum dma_data_direction
7459  * @sg_call_ents: 'nents' from dma_map_sg
7560  * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
61
+ * @pfn: page frame of the start address
62
+ * @offset: offset of mapping relative to pfn
7663  * @map_err_type: track whether dma_mapping_error() was checked
7764  * @stacktrace: support backtraces when a violation is detected
7865  */
7966 struct dma_debug_entry {
8067 	struct list_head list;
8168 	struct device    *dev;
82
-	int              type;
83
-	unsigned long	 pfn;
84
-	size_t		 offset;
8569 	u64              dev_addr;
8670 	u64              size;
71
+	int              type;
8772 	int              direction;
8873 	int		 sg_call_ents;
8974 	int		 sg_mapped_ents;
75
+	unsigned long	 pfn;
76
+	size_t		 offset;
9077 	enum map_err_types  map_err_type;
9178 #ifdef CONFIG_STACKTRACE
92
-	struct		 stack_trace stacktrace;
93
-	unsigned long	 st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
79
+	unsigned int	stack_len;
80
+	unsigned long	stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
9481 #endif
95
-};
82
+} ____cacheline_aligned_in_smp;
9683 
9784 typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
9885 
9986 struct hash_bucket {
10087 	struct list_head list;
10188 	spinlock_t lock;
102
-} ____cacheline_aligned_in_smp;
89
+};
10390 
10491 /* Hash list to save the allocated dma addresses */
10592 static struct hash_bucket dma_entry_hash[HASH_SIZE];
....@@ -134,16 +121,6 @@
134121 /* number of preallocated entries requested by kernel cmdline */
135122 static u32 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
136123 
137
-/* debugfs dentry's for the stuff above */
138
-static struct dentry *dma_debug_dent        __read_mostly;
139
-static struct dentry *global_disable_dent   __read_mostly;
140
-static struct dentry *error_count_dent      __read_mostly;
141
-static struct dentry *show_all_errors_dent  __read_mostly;
142
-static struct dentry *show_num_errors_dent  __read_mostly;
143
-static struct dentry *num_free_entries_dent __read_mostly;
144
-static struct dentry *min_free_entries_dent __read_mostly;
145
-static struct dentry *filter_dent           __read_mostly;
146
-
147124 /* per-driver filter related state */
148125 
149126 #define NAME_MAX_LEN	64
....@@ -159,12 +136,19 @@
159136 	[MAP_ERR_CHECKED] = "dma map error checked",
160137 };
161138 
162
-static const char *type2name[5] = { "single", "page",
163
-				    "scather-gather", "coherent",
164
-				    "resource" };
139
+static const char *type2name[] = {
140
+	[dma_debug_single] = "single",
141
+	[dma_debug_sg] = "scather-gather",
142
+	[dma_debug_coherent] = "coherent",
143
+	[dma_debug_resource] = "resource",
144
+};
165145 
166
-static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
167
-				   "DMA_FROM_DEVICE", "DMA_NONE" };
146
+static const char *dir2name[] = {
147
+	[DMA_BIDIRECTIONAL]	= "DMA_BIDIRECTIONAL",
148
+	[DMA_TO_DEVICE]		= "DMA_TO_DEVICE",
149
+	[DMA_FROM_DEVICE]	= "DMA_FROM_DEVICE",
150
+	[DMA_NONE]		= "DMA_NONE",
151
+};
168152 
169153 /*
170154  * The access to some variables in this macro is racy. We can't use atomic_t
....@@ -183,8 +167,8 @@
183167 {
184168 #ifdef CONFIG_STACKTRACE
185169 	if (entry) {
186
-		pr_warning("Mapped at:\n");
187
-		print_stack_trace(&entry->stacktrace, 0);
170
+		pr_warn("Mapped at:\n");
171
+		stack_trace_print(entry->stack_entries, entry->stack_len, 0);
188172 	}
189173 #endif
190174 }
....@@ -234,7 +218,7 @@
234218 		error_count += 1;					\
235219 		if (driver_filter(dev) &&				\
236220 		    (show_all_errors || show_num_errors > 0)) {		\
237
-			WARN(1, "%s %s: " format,			\
221
+			WARN(1, pr_fmt("%s %s: ") format,		\
238222 			     dev ? dev_driver_string(dev) : "NULL",	\
239223 			     dev ? dev_name(dev) : "NULL", ## arg);	\
240224 			dump_entry_trace(entry);			\
....@@ -277,12 +261,10 @@
277261  * Give up exclusive access to the hash bucket
278262  */
279263 static void put_hash_bucket(struct hash_bucket *bucket,
280
-			    unsigned long *flags)
264
+			    unsigned long flags)
281265 	__releases(&bucket->lock)
282266 {
283
-	unsigned long __flags = *flags;
284
-
285
-	spin_unlock_irqrestore(&bucket->lock, __flags);
267
+	spin_unlock_irqrestore(&bucket->lock, flags);
286268 }
287269 
288270 static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
....@@ -381,7 +363,7 @@
381363 		/*
382364 		 * Nothing found, go back a hash bucket
383365 		 */
384
-		put_hash_bucket(*bucket, flags);
366
+		put_hash_bucket(*bucket, *flags);
385367 		range          += (1 << HASH_FN_SHIFT);
386368 		index.dev_addr -= (1 << HASH_FN_SHIFT);
387369 		*bucket = get_hash_bucket(&index, flags);
....@@ -465,11 +447,8 @@
465447  * dma_active_cacheline entry to track per event.  dma_map_sg(), on the
466448  * other hand, consumes a single dma_debug_entry, but inserts 'nents'
467449  * entries into the tree.
468
- *
469
- * At any time debug_dma_assert_idle() can be called to trigger a
470
- * warning if any cachelines in the given page are in the active set.
471450  */
472
-static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
451
+static RADIX_TREE(dma_active_cacheline, GFP_ATOMIC);
473452 static DEFINE_SPINLOCK(radix_lock);
474453 #define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
475454 #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
....@@ -514,13 +493,10 @@
514493 	overlap = active_cacheline_set_overlap(cln, ++overlap);
515494 
516495 	/* If we overflowed the overlap counter then we're potentially
517
-	 * leaking dma-mappings.  Otherwise, if maps and unmaps are
518
-	 * balanced then this overflow may cause false negatives in
519
-	 * debug_dma_assert_idle() as the cacheline may be marked idle
520
-	 * prematurely.
496
+	 * leaking dma-mappings.
521497 	 */
522498 	WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
523
-		  "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
499
+		  pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
524500 		  ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
525501 }
526502 
....@@ -572,53 +548,6 @@
572548 	spin_unlock_irqrestore(&radix_lock, flags);
573549 }
574550 
575
-/**
576
- * debug_dma_assert_idle() - assert that a page is not undergoing dma
577
- * @page: page to lookup in the dma_active_cacheline tree
578
- *
579
- * Place a call to this routine in cases where the cpu touching the page
580
- * before the dma completes (page is dma_unmapped) will lead to data
581
- * corruption.
582
- */
583
-void debug_dma_assert_idle(struct page *page)
584
-{
585
-	static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
586
-	struct dma_debug_entry *entry = NULL;
587
-	void **results = (void **) &ents;
588
-	unsigned int nents, i;
589
-	unsigned long flags;
590
-	phys_addr_t cln;
591
-
592
-	if (dma_debug_disabled())
593
-		return;
594
-
595
-	if (!page)
596
-		return;
597
-
598
-	cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
599
-	spin_lock_irqsave(&radix_lock, flags);
600
-	nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
601
-				       CACHELINES_PER_PAGE);
602
-	for (i = 0; i < nents; i++) {
603
-		phys_addr_t ent_cln = to_cacheline_number(ents[i]);
604
-
605
-		if (ent_cln == cln) {
606
-			entry = ents[i];
607
-			break;
608
-		} else if (ent_cln >= cln + CACHELINES_PER_PAGE)
609
-			break;
610
-	}
611
-	spin_unlock_irqrestore(&radix_lock, flags);
612
-
613
-	if (!entry)
614
-		return;
615
-
616
-	cln = to_cacheline_number(entry);
617
-	err_printk(entry->dev, entry,
618
-		   "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
619
-		   &cln);
620
-}
621
-
622551 /*
623552  * Wrapper function for adding an entry to the hash.
624553  * This function takes care of locking itself.
....@@ -631,17 +560,35 @@
631560 
632561 	bucket = get_hash_bucket(entry, &flags);
633562 	hash_bucket_add(bucket, entry);
634
-	put_hash_bucket(bucket, &flags);
563
+	put_hash_bucket(bucket, flags);
635564 
636565 	rc = active_cacheline_insert(entry);
637566 	if (rc == -ENOMEM) {
638
-		pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
567
+		pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n");
639568 		global_disable = true;
640569 	}
641570 
642571 	/* TODO: report -EEXIST errors here as overlapping mappings are
643572 	 * not supported by the DMA API
644573 	 */
574
+}
575
+
576
+static int dma_debug_create_entries(gfp_t gfp)
577
+{
578
+	struct dma_debug_entry *entry;
579
+	int i;
580
+
581
+	entry = (void *)get_zeroed_page(gfp);
582
+	if (!entry)
583
+		return -ENOMEM;
584
+
585
+	for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++)
586
+		list_add_tail(&entry[i].list, &free_entries);
587
+
588
+	num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
589
+	nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
590
+
591
+	return 0;
645592 }
646593 
647594 static struct dma_debug_entry *__dma_entry_alloc(void)
....@@ -659,6 +606,18 @@
659606 	return entry;
660607 }
661608 
609
+static void __dma_entry_alloc_check_leak(void)
610
+{
611
+	u32 tmp = nr_total_entries % nr_prealloc_entries;
612
+
613
+	/* Shout each time we tick over some multiple of the initial pool */
614
+	if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) {
615
+		pr_info("dma_debug_entry pool grown to %u (%u00%%)\n",
616
+			nr_total_entries,
617
+			(nr_total_entries / nr_prealloc_entries));
618
+	}
619
+}
620
+
662621 /* struct dma_entry allocator
663622  *
664623  * The next two functions implement the allocator for
....@@ -670,12 +629,14 @@
670629 	unsigned long flags;
671630 
672631 	spin_lock_irqsave(&free_entries_lock, flags);
673
-
674
-	if (list_empty(&free_entries)) {
675
-		global_disable = true;
676
-		spin_unlock_irqrestore(&free_entries_lock, flags);
677
-		pr_err("DMA-API: debugging out of memory - disabling\n");
678
-		return NULL;
632
+	if (num_free_entries == 0) {
633
+		if (dma_debug_create_entries(GFP_ATOMIC)) {
634
+			global_disable = true;
635
+			spin_unlock_irqrestore(&free_entries_lock, flags);
636
+			pr_err("debugging out of memory - disabling\n");
637
+			return NULL;
638
+		}
639
+		__dma_entry_alloc_check_leak();
679640 	}
680641 
681642 	entry = __dma_entry_alloc();
....@@ -683,12 +644,10 @@
683644 	spin_unlock_irqrestore(&free_entries_lock, flags);
684645 
685646 #ifdef CONFIG_STACKTRACE
686
-	entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
687
-	entry->stacktrace.entries = entry->st_entries;
688
-	entry->stacktrace.skip = 2;
689
-	save_stack_trace(&entry->stacktrace);
647
+	entry->stack_len = stack_trace_save(entry->stack_entries,
648
+					    ARRAY_SIZE(entry->stack_entries),
649
+					    1);
690650 #endif
691
-
692651 	return entry;
693652 }
694653 
....@@ -708,52 +667,6 @@
708667 	spin_unlock_irqrestore(&free_entries_lock, flags);
709668 }
710669 
711
-int dma_debug_resize_entries(u32 num_entries)
712
-{
713
-	int i, delta, ret = 0;
714
-	unsigned long flags;
715
-	struct dma_debug_entry *entry;
716
-	LIST_HEAD(tmp);
717
-
718
-	spin_lock_irqsave(&free_entries_lock, flags);
719
-
720
-	if (nr_total_entries < num_entries) {
721
-		delta = num_entries - nr_total_entries;
722
-
723
-		spin_unlock_irqrestore(&free_entries_lock, flags);
724
-
725
-		for (i = 0; i < delta; i++) {
726
-			entry = kzalloc(sizeof(*entry), GFP_KERNEL);
727
-			if (!entry)
728
-				break;
729
-
730
-			list_add_tail(&entry->list, &tmp);
731
-		}
732
-
733
-		spin_lock_irqsave(&free_entries_lock, flags);
734
-
735
-		list_splice(&tmp, &free_entries);
736
-		nr_total_entries += i;
737
-		num_free_entries += i;
738
-	} else {
739
-		delta = nr_total_entries - num_entries;
740
-
741
-		for (i = 0; i < delta && !list_empty(&free_entries); i++) {
742
-			entry = __dma_entry_alloc();
743
-			kfree(entry);
744
-		}
745
-
746
-		nr_total_entries -= i;
747
-	}
748
-
749
-	if (nr_total_entries != num_entries)
750
-		ret = 1;
751
-
752
-	spin_unlock_irqrestore(&free_entries_lock, flags);
753
-
754
-	return ret;
755
-}
756
-
757670 /*
758671  * DMA-API debugging init code
759672  *
....@@ -761,36 +674,6 @@
761674  *   1. Initialize core data structures
762675  *   2. Preallocate a given number of dma_debug_entry structs
763676  */
764
-
765
-static int prealloc_memory(u32 num_entries)
766
-{
767
-	struct dma_debug_entry *entry, *next_entry;
768
-	int i;
769
-
770
-	for (i = 0; i < num_entries; ++i) {
771
-		entry = kzalloc(sizeof(*entry), GFP_KERNEL);
772
-		if (!entry)
773
-			goto out_err;
774
-
775
-		list_add_tail(&entry->list, &free_entries);
776
-	}
777
-
778
-	num_free_entries = num_entries;
779
-	min_free_entries = num_entries;
780
-
781
-	pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
782
-
783
-	return 0;
784
-
785
-out_err:
786
-
787
-	list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
788
-		list_del(&entry->list);
789
-		kfree(entry);
790
-	}
791
-
792
-	return -ENOMEM;
793
-}
794677 
795678 static ssize_t filter_read(struct file *file, char __user *user_buf,
796679 			   size_t count, loff_t *ppos)
....@@ -851,7 +734,7 @@
851734 		 * switched off.
852735 		 */
853736 		if (current_driver_name[0])
854
-			pr_info("DMA-API: switching off dma-debug driver filter\n");
737
+			pr_info("switching off dma-debug driver filter\n");
855738 		current_driver_name[0] = 0;
856739 		current_driver = NULL;
857740 		goto out_unlock;
....@@ -869,7 +752,7 @@
869752 	current_driver_name[i] = 0;
870753 	current_driver = NULL;
871754 
872
-	pr_info("DMA-API: enable driver filter for driver [%s]\n",
755
+	pr_info("enable driver filter for driver [%s]\n",
873756 		current_driver_name);
874757 
875758 out_unlock:
....@@ -884,61 +767,50 @@
884767 	.llseek = default_llseek,
885768 };
886769 
887
-static int dma_debug_fs_init(void)
770
+static int dump_show(struct seq_file *seq, void *v)
888771 {
889
-	dma_debug_dent = debugfs_create_dir("dma-api", NULL);
890
-	if (!dma_debug_dent) {
891
-		pr_err("DMA-API: can not create debugfs directory\n");
892
-		return -ENOMEM;
772
+	int idx;
773
+
774
+	for (idx = 0; idx < HASH_SIZE; idx++) {
775
+		struct hash_bucket *bucket = &dma_entry_hash[idx];
776
+		struct dma_debug_entry *entry;
777
+		unsigned long flags;
778
+
779
+		spin_lock_irqsave(&bucket->lock, flags);
780
+		list_for_each_entry(entry, &bucket->list, list) {
781
+			seq_printf(seq,
782
+				   "%s %s %s idx %d P=%llx N=%lx D=%llx L=%llx %s %s\n",
783
+				   dev_name(entry->dev),
784
+				   dev_driver_string(entry->dev),
785
+				   type2name[entry->type], idx,
786
+				   phys_addr(entry), entry->pfn,
787
+				   entry->dev_addr, entry->size,
788
+				   dir2name[entry->direction],
789
+				   maperr2str[entry->map_err_type]);
790
+		}
791
+		spin_unlock_irqrestore(&bucket->lock, flags);
893792 	}
793
+	return 0;
794
+}
795
+DEFINE_SHOW_ATTRIBUTE(dump);
894796 
895
-	global_disable_dent = debugfs_create_bool("disabled", 0444,
896
-			dma_debug_dent,
897
-			&global_disable);
898
-	if (!global_disable_dent)
899
-		goto out_err;
797
+static int __init dma_debug_fs_init(void)
798
+{
799
+	struct dentry *dentry = debugfs_create_dir("dma-api", NULL);
900800 
901
-	error_count_dent = debugfs_create_u32("error_count", 0444,
902
-			dma_debug_dent, &error_count);
903
-	if (!error_count_dent)
904
-		goto out_err;
905
-
906
-	show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
907
-			dma_debug_dent,
908
-			&show_all_errors);
909
-	if (!show_all_errors_dent)
910
-		goto out_err;
911
-
912
-	show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
913
-			dma_debug_dent,
914
-			&show_num_errors);
915
-	if (!show_num_errors_dent)
916
-		goto out_err;
917
-
918
-	num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
919
-			dma_debug_dent,
920
-			&num_free_entries);
921
-	if (!num_free_entries_dent)
922
-		goto out_err;
923
-
924
-	min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
925
-			dma_debug_dent,
926
-			&min_free_entries);
927
-	if (!min_free_entries_dent)
928
-		goto out_err;
929
-
930
-	filter_dent = debugfs_create_file("driver_filter", 0644,
931
-					  dma_debug_dent, NULL, &filter_fops);
932
-	if (!filter_dent)
933
-		goto out_err;
801
+	debugfs_create_bool("disabled", 0444, dentry, &global_disable);
802
+	debugfs_create_u32("error_count", 0444, dentry, &error_count);
803
+	debugfs_create_u32("all_errors", 0644, dentry, &show_all_errors);
804
+	debugfs_create_u32("num_errors", 0644, dentry, &show_num_errors);
805
+	debugfs_create_u32("num_free_entries", 0444, dentry, &num_free_entries);
806
+	debugfs_create_u32("min_free_entries", 0444, dentry, &min_free_entries);
807
+	debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries);
808
+	debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops);
809
+	debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops);
934810 
935811 	return 0;
936
-
937
-out_err:
938
-	debugfs_remove_recursive(dma_debug_dent);
939
-
940
-	return -ENOMEM;
941812 }
813
+core_initcall_sync(dma_debug_fs_init);
942814 
943815 static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
944816 {
....@@ -963,7 +835,7 @@
963835 static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
964836 {
965837 	struct device *dev = data;
966
-	struct dma_debug_entry *uninitialized_var(entry);
838
+	struct dma_debug_entry *entry;
967839 	int count;
968840 
969841 	if (dma_debug_disabled())
....@@ -974,7 +846,7 @@
974846 		count = device_dma_allocations(dev, &entry);
975847 		if (count == 0)
976848 			break;
977
-		err_printk(dev, entry, "DMA-API: device driver has pending "
849
+		err_printk(dev, entry, "device driver has pending "
978850 				"DMA allocations while released from device "
979851 				"[count=%d]\n"
980852 				"One of leaked entries details: "
....@@ -1010,7 +882,7 @@
1010882 
1011883 static int dma_debug_init(void)
1012884 {
1013
-	int i;
885
+	int i, nr_pages;
1014886 
1015887 	/* Do not use dma_debug_initialized here, since we really want to be
1016888 	 * called to set dma_debug_initialized
....@@ -1023,25 +895,25 @@
1023895 		spin_lock_init(&dma_entry_hash[i].lock);
1024896 	}
1025897 
1026
-	if (dma_debug_fs_init() != 0) {
1027
-		pr_err("DMA-API: error creating debugfs entries - disabling\n");
898
+	nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);
899
+	for (i = 0; i < nr_pages; ++i)
900
+		dma_debug_create_entries(GFP_KERNEL);
901
+	if (num_free_entries >= nr_prealloc_entries) {
902
+		pr_info("preallocated %d debug entries\n", nr_total_entries);
903
+	} else if (num_free_entries > 0) {
904
+		pr_warn("%d debug entries requested but only %d allocated\n",
905
+			nr_prealloc_entries, nr_total_entries);
906
+	} else {
907
+		pr_err("debugging out of memory error - disabled\n");
1028908 		global_disable = true;
1029909 
1030910 		return 0;
1031911 	}
1032
-
1033
-	if (prealloc_memory(nr_prealloc_entries) != 0) {
1034
-		pr_err("DMA-API: debugging out of memory error - disabled\n");
1035
-		global_disable = true;
1036
-
1037
-		return 0;
1038
-	}
1039
-
1040
-	nr_total_entries = num_free_entries;
912
+	min_free_entries = num_free_entries;
1041913 
1042914 	dma_debug_initialized = true;
1043915 
1044
-	pr_info("DMA-API: debugging enabled by kernel config\n");
916
+	pr_info("debugging enabled by kernel config\n");
1045917 	return 0;
1046918 }
1047919 core_initcall(dma_debug_init);
....@@ -1052,11 +924,11 @@
1052924 		return -EINVAL;
1053925 
1054926 	if (strncmp(str, "off", 3) == 0) {
1055
-		pr_info("DMA-API: debugging disabled on kernel command line\n");
927
+		pr_info("debugging disabled on kernel command line\n");
1056928 		global_disable = true;
1057929 	}
1058930 
1059
-	return 0;
931
+	return 1;
1060932 }
1061933 
1062934 static __init int dma_debug_entries_cmdline(char *str)
....@@ -1065,7 +937,7 @@
1065937 		return -EINVAL;
1066938 	if (!get_option(&str, &nr_prealloc_entries))
1067939 		nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
1068
-	return 0;
940
+	return 1;
1069941 }
1070942 
1071943 __setup("dma_debug=", dma_debug_cmdline);
....@@ -1082,15 +954,15 @@
1082954 
1083955 	if (!entry) {
1084956 		/* must drop lock before calling dma_mapping_error */
1085
-		put_hash_bucket(bucket, &flags);
957
+		put_hash_bucket(bucket, flags);
1086958 
1087959 		if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1088960 			err_printk(ref->dev, NULL,
1089
-				   "DMA-API: device driver tries to free an "
961
+				   "device driver tries to free an "
1090962 				   "invalid DMA memory address\n");
1091963 		} else {
1092964 			err_printk(ref->dev, NULL,
1093
-				   "DMA-API: device driver tries to free DMA "
965
+				   "device driver tries to free DMA "
1094966 				   "memory it has not allocated [device "
1095967 				   "address=0x%016llx] [size=%llu bytes]\n",
1096968 				   ref->dev_addr, ref->size);
....@@ -1099,7 +971,7 @@
1099971 	}
1100972 
1101973 	if (ref->size != entry->size) {
1102
-		err_printk(ref->dev, entry, "DMA-API: device driver frees "
974
+		err_printk(ref->dev, entry, "device driver frees "
1103975 			   "DMA memory with different size "
1104976 			   "[device address=0x%016llx] [map size=%llu bytes] "
1105977 			   "[unmap size=%llu bytes]\n",
....@@ -1107,7 +979,7 @@
1107979 	}
1108980 
1109981 	if (ref->type != entry->type) {
1110
-		err_printk(ref->dev, entry, "DMA-API: device driver frees "
982
+		err_printk(ref->dev, entry, "device driver frees "
1111983 			   "DMA memory with wrong function "
1112984 			   "[device address=0x%016llx] [size=%llu bytes] "
1113985 			   "[mapped as %s] [unmapped as %s]\n",
....@@ -1115,7 +987,7 @@
1115987 			   type2name[entry->type], type2name[ref->type]);
1116988 	} else if ((entry->type == dma_debug_coherent) &&
1117989 		   (phys_addr(ref) != phys_addr(entry))) {
1118
-		err_printk(ref->dev, entry, "DMA-API: device driver frees "
990
+		err_printk(ref->dev, entry, "device driver frees "
1119991 			   "DMA memory with different CPU address "
1120992 			   "[device address=0x%016llx] [size=%llu bytes] "
1121993 			   "[cpu alloc address=0x%016llx] "
....@@ -1127,7 +999,7 @@
1127999 
11281000 	if (ref->sg_call_ents && ref->type == dma_debug_sg &&
11291001 	    ref->sg_call_ents != entry->sg_call_ents) {
1130
-		err_printk(ref->dev, entry, "DMA-API: device driver frees "
1002
+		err_printk(ref->dev, entry, "device driver frees "
11311003 			   "DMA sg list with different entry count "
11321004 			   "[map count=%d] [unmap count=%d]\n",
11331005 			   entry->sg_call_ents, ref->sg_call_ents);
....@@ -1138,7 +1010,7 @@
11381010 	 * DMA API don't handle this properly, so check for it here
11391011 	 */
11401012 	if (ref->direction != entry->direction) {
1141
-		err_printk(ref->dev, entry, "DMA-API: device driver frees "
1013
+		err_printk(ref->dev, entry, "device driver frees "
11421014 			   "DMA memory with different direction "
11431015 			   "[device address=0x%016llx] [size=%llu bytes] "
11441016 			   "[mapped with %s] [unmapped with %s]\n",
....@@ -1150,11 +1022,11 @@
11501022 	/*
11511023 	 * Drivers should use dma_mapping_error() to check the returned
11521024 	 * addresses of dma_map_single() and dma_map_page().
1153
-	 * If not, print this warning message. See Documentation/DMA-API.txt.
1025
+	 * If not, print this warning message. See Documentation/core-api/dma-api.rst.
11541026 	 */
11551027 	if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
11561028 		err_printk(ref->dev, entry,
1157
-			   "DMA-API: device driver failed to check map error"
1029
+			   "device driver failed to check map error"
11581030 			   "[device address=0x%016llx] [size=%llu bytes] "
11591031 			   "[mapped as %s]",
11601032 			   ref->dev_addr, ref->size,
....@@ -1164,7 +1036,7 @@
11641036 	hash_bucket_del(entry);
11651037 	dma_entry_free(entry);
11661038 
1167
-	put_hash_bucket(bucket, &flags);
1039
+	put_hash_bucket(bucket, flags);
11681040 }
11691041 
11701042 static void check_for_stack(struct device *dev,
....@@ -1179,7 +1051,7 @@
11791051 			return;
11801052 		addr = page_address(page) + offset;
11811053 		if (object_is_on_stack(addr))
1182
-			err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [addr=%p]\n", addr);
1054
+			err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n", addr);
11831055 	} else {
11841056 		/* Stack is vmalloced. */
11851057 		int i;
....@@ -1189,7 +1061,7 @@
11891061 				continue;
11901062 
11911063 			addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1192
-			err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [probable addr=%p]\n", addr);
1064
+			err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n", addr);
11931065 			break;
11941066 		}
11951067 	}
....@@ -1209,7 +1081,7 @@
12091081 {
12101082 	if (overlap(addr, len, _stext, _etext) ||
12111083 	    overlap(addr, len, __start_rodata, __end_rodata))
1212
-		err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1084
+		err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
12131085 }
12141086 
12151087 static void check_sync(struct device *dev,
....@@ -1225,7 +1097,7 @@
12251097 	entry = bucket_find_contain(&bucket, ref, &flags);
12261098 
12271099 	if (!entry) {
1228
-		err_printk(dev, NULL, "DMA-API: device driver tries "
1100
+		err_printk(dev, NULL, "device driver tries "
12291101 				"to sync DMA memory it has not allocated "
12301102 				"[device address=0x%016llx] [size=%llu bytes]\n",
12311103 				(unsigned long long)ref->dev_addr, ref->size);
....@@ -1233,7 +1105,7 @@
12331105 	}
12341106 
12351107 	if (ref->size > entry->size) {
1236
-		err_printk(dev, entry, "DMA-API: device driver syncs"
1108
+		err_printk(dev, entry, "device driver syncs"
12371109 				" DMA memory outside allocated range "
12381110 				"[device address=0x%016llx] "
12391111 				"[allocation size=%llu bytes] "
....@@ -1246,7 +1118,7 @@
12461118 		goto out;
12471119 
12481120 	if (ref->direction != entry->direction) {
1249
-		err_printk(dev, entry, "DMA-API: device driver syncs "
1121
+		err_printk(dev, entry, "device driver syncs "
12501122 				"DMA memory with different direction "
12511123 				"[device address=0x%016llx] [size=%llu bytes] "
12521124 				"[mapped with %s] [synced with %s]\n",
....@@ -1257,7 +1129,7 @@
12571129 
12581130 	if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
12591131 		      !(ref->direction == DMA_TO_DEVICE))
1260
-		err_printk(dev, entry, "DMA-API: device driver syncs "
1132
+		err_printk(dev, entry, "device driver syncs "
12611133 				"device read-only DMA memory for cpu "
12621134 				"[device address=0x%016llx] [size=%llu bytes] "
12631135 				"[mapped with %s] [synced with %s]\n",
....@@ -1267,7 +1139,7 @@
12671139 
12681140 	if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
12691141 		       !(ref->direction == DMA_FROM_DEVICE))
1270
-		err_printk(dev, entry, "DMA-API: device driver syncs "
1142
+		err_printk(dev, entry, "device driver syncs "
12711143 				"device write-only DMA memory to device "
12721144 				"[device address=0x%016llx] [size=%llu bytes] "
12731145 				"[mapped with %s] [synced with %s]\n",
....@@ -1275,16 +1147,17 @@
12751147 				dir2name[entry->direction],
12761148 				dir2name[ref->direction]);
12771149 
1150
+	/* sg list count can be less than map count when partial cache sync */
12781151 	if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1279
-	    ref->sg_call_ents != entry->sg_call_ents) {
1280
-		err_printk(ref->dev, entry, "DMA-API: device driver syncs "
1281
-			   "DMA sg list with different entry count "
1152
+	    ref->sg_call_ents > entry->sg_call_ents) {
1153
+		err_printk(ref->dev, entry, "device driver syncs "
1154
+			   "DMA sg list count larger than map count "
12821155 			   "[map count=%d] [sync count=%d]\n",
12831156 			   entry->sg_call_ents, ref->sg_call_ents);
12841157 	}
12851158 
12861159 out:
1287
-	put_hash_bucket(bucket, &flags);
1160
+	put_hash_bucket(bucket, flags);
12881161 }
12891162 
12901163 static void check_sg_segment(struct device *dev, struct scatterlist *sg)
....@@ -1298,7 +1171,7 @@
12981171 	 * whoever generated the list forgot to check them.
12991172 	 */
13001173 	if (sg->length > max_seg)
1301
-		err_printk(dev, NULL, "DMA-API: mapping sg segment longer than device claims to support [len=%u] [max=%u]\n",
1174
+		err_printk(dev, NULL, "mapping sg segment longer than device claims to support [len=%u] [max=%u]\n",
13021175 			   sg->length, max_seg);
13031176 	/*
13041177 	 * In some cases this could potentially be the DMA API
....@@ -1308,14 +1181,29 @@
13081181 	start = sg_dma_address(sg);
13091182 	end = start + sg_dma_len(sg) - 1;
13101183 	if ((start ^ end) & ~boundary)
1311
-		err_printk(dev, NULL, "DMA-API: mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n",
1184
+		err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n",
13121185 			   start, end, boundary);
13131186 #endif
13141187 }
13151188 
1189
+void debug_dma_map_single(struct device *dev, const void *addr,
1190
+			    unsigned long len)
1191
+{
1192
+	if (unlikely(dma_debug_disabled()))
1193
+		return;
1194
+
1195
+	if (!virt_addr_valid(addr))
1196
+		err_printk(dev, NULL, "device driver maps memory from invalid area [addr=%p] [len=%lu]\n",
1197
+			   addr, len);
1198
+
1199
+	if (is_vmalloc_addr(addr))
1200
+		err_printk(dev, NULL, "device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n",
1201
+			   addr, len);
1202
+}
1203
+EXPORT_SYMBOL(debug_dma_map_single);
1204
+
13161205 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1317
-			size_t size, int direction, dma_addr_t dma_addr,
1318
-			bool map_single)
1206
+			size_t size, int direction, dma_addr_t dma_addr)
13191207 {
13201208 	struct dma_debug_entry *entry;
13211209 
....@@ -1330,16 +1218,13 @@
13301218 		return;
13311219 
13321220 	entry->dev       = dev;
1333
-	entry->type      = dma_debug_page;
1221
+	entry->type      = dma_debug_single;
13341222 	entry->pfn	 = page_to_pfn(page);
1335
-	entry->offset	 = offset,
1223
+	entry->offset	 = offset;
13361224 	entry->dev_addr  = dma_addr;
13371225 	entry->size      = size;
13381226 	entry->direction = direction;
13391227 	entry->map_err_type = MAP_ERR_NOT_CHECKED;
1340
-
1341
-	if (map_single)
1342
-		entry->type = dma_debug_single;
13431228 
13441229 	check_for_stack(dev, page, offset);
13451230 
....@@ -1351,7 +1236,6 @@
13511236 
13521237 	add_dma_entry(entry);
13531238 }
1354
-EXPORT_SYMBOL(debug_dma_map_page);
13551239 
13561240 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
13571241 {
....@@ -1387,15 +1271,15 @@
13871271 		}
13881272 	}
13891273 
1390
-	put_hash_bucket(bucket, &flags);
1274
+	put_hash_bucket(bucket, flags);
13911275 }
13921276 EXPORT_SYMBOL(debug_dma_mapping_error);
13931277 
13941278 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1395
-			  size_t size, int direction, bool map_single)
1279
+			  size_t size, int direction)
13961280 {
13971281 	struct dma_debug_entry ref = {
1398
-		.type           = dma_debug_page,
1282
+		.type           = dma_debug_single,
13991283 		.dev            = dev,
14001284 		.dev_addr       = addr,
14011285 		.size           = size,
....@@ -1404,13 +1288,8 @@
14041288 
14051289 	if (unlikely(dma_debug_disabled()))
14061290 		return;
1407
-
1408
-	if (map_single)
1409
-		ref.type = dma_debug_single;
1410
-
14111291 	check_unmap(&ref);
14121292 }
1413
-EXPORT_SYMBOL(debug_dma_unmap_page);
14141293 
14151294 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
14161295 		      int nents, int mapped_ents, int direction)
....@@ -1436,7 +1315,7 @@
14361315 		entry->type           = dma_debug_sg;
14371316 		entry->dev            = dev;
14381317 		entry->pfn	      = page_to_pfn(sg_page(s));
1439
-		entry->offset	      = s->offset,
1318
+		entry->offset	      = s->offset;
14401319 		entry->size           = sg_dma_len(s);
14411320 		entry->dev_addr       = sg_dma_address(s);
14421321 		entry->direction      = direction;
....@@ -1448,7 +1327,6 @@
14481327 		add_dma_entry(entry);
14491328 	}
14501329 }
1451
-EXPORT_SYMBOL(debug_dma_map_sg);
14521330 
14531331 static int get_nr_mapped_entries(struct device *dev,
14541332 				 struct dma_debug_entry *ref)
....@@ -1464,7 +1342,7 @@
14641342 
14651343 	if (entry)
14661344 		mapped_ents = entry->sg_mapped_ents;
1467
-	put_hash_bucket(bucket, &flags);
1345
+	put_hash_bucket(bucket, flags);
14681346 
14691347 	return mapped_ents;
14701348 }
....@@ -1500,7 +1378,6 @@
15001378 		check_unmap(&ref);
15011379 	}
15021380 }
1503
-EXPORT_SYMBOL(debug_dma_unmap_sg);
15041381 
15051382 void debug_dma_alloc_coherent(struct device *dev, size_t size,
15061383 			      dma_addr_t dma_addr, void *virt)
....@@ -1535,7 +1412,6 @@
15351412 
15361413 	add_dma_entry(entry);
15371414 }
1538
-EXPORT_SYMBOL(debug_dma_alloc_coherent);
15391415 
15401416 void debug_dma_free_coherent(struct device *dev, size_t size,
15411417 			 void *virt, dma_addr_t addr)
....@@ -1563,7 +1439,6 @@
15631439 
15641440 	check_unmap(&ref);
15651441 }
1566
-EXPORT_SYMBOL(debug_dma_free_coherent);
15671442 
15681443 void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
15691444 			    int direction, dma_addr_t dma_addr)
....@@ -1588,7 +1463,6 @@
15881463 
15891464 	add_dma_entry(entry);
15901465 }
1591
-EXPORT_SYMBOL(debug_dma_map_resource);
15921466 
15931467 void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr,
15941468 			      size_t size, int direction)
....@@ -1606,7 +1480,6 @@
16061480 
16071481 	check_unmap(&ref);
16081482 }
1609
-EXPORT_SYMBOL(debug_dma_unmap_resource);
16101483 
16111484 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
16121485 				   size_t size, int direction)
....@@ -1625,7 +1498,6 @@
16251498 
16261499 	check_sync(dev, &ref, true);
16271500 }
1628
-EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
16291501 
16301502 void debug_dma_sync_single_for_device(struct device *dev,
16311503 				      dma_addr_t dma_handle, size_t size,
....@@ -1645,49 +1517,6 @@
16451517 
16461518 	check_sync(dev, &ref, false);
16471519 }
1648
-EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1649
-
1650
-void debug_dma_sync_single_range_for_cpu(struct device *dev,
1651
-					 dma_addr_t dma_handle,
1652
-					 unsigned long offset, size_t size,
1653
-					 int direction)
1654
-{
1655
-	struct dma_debug_entry ref;
1656
-
1657
-	if (unlikely(dma_debug_disabled()))
1658
-		return;
1659
-
1660
-	ref.type         = dma_debug_single;
1661
-	ref.dev          = dev;
1662
-	ref.dev_addr     = dma_handle;
1663
-	ref.size         = offset + size;
1664
-	ref.direction    = direction;
1665
-	ref.sg_call_ents = 0;
1666
-
1667
-	check_sync(dev, &ref, true);
1668
-}
1669
-EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1670
-
1671
-void debug_dma_sync_single_range_for_device(struct device *dev,
1672
-					    dma_addr_t dma_handle,
1673
-					    unsigned long offset,
1674
-					    size_t size, int direction)
1675
-{
1676
-	struct dma_debug_entry ref;
1677
-
1678
-	if (unlikely(dma_debug_disabled()))
1679
-		return;
1680
-
1681
-	ref.type         = dma_debug_single;
1682
-	ref.dev          = dev;
1683
-	ref.dev_addr     = dma_handle;
1684
-	ref.size         = offset + size;
1685
-	ref.direction    = direction;
1686
-	ref.sg_call_ents = 0;
1687
-
1688
-	check_sync(dev, &ref, false);
1689
-}
1690
-EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
16911520 
16921521 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
16931522 			       int nelems, int direction)
....@@ -1720,7 +1549,6 @@
17201549 		check_sync(dev, &ref, true);
17211550 	}
17221551 }
1723
-EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
17241552 
17251553 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
17261554 				  int nelems, int direction)
....@@ -1752,7 +1580,6 @@
17521580 		check_sync(dev, &ref, false);
17531581 	}
17541582 }
1755
-EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
17561583 
17571584 static int __init dma_debug_driver_setup(char *str)
17581585 {
....@@ -1765,7 +1592,7 @@
17651592 	}
17661593 
17671594 	if (current_driver_name[0])
1768
-		pr_info("DMA-API: enable driver filter for driver [%s]\n",
1595
+		pr_info("enable driver filter for driver [%s]\n",
17691596 			current_driver_name);
17701597 
17711598