forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc 
 kernel/drivers/rknpu/rknpu_gem.c
....@@ -13,7 +13,6 @@
1313 #include <linux/shmem_fs.h>
1414 #include <linux/dma-buf.h>
1515 #include <linux/iommu.h>
16
-#include <linux/dma-iommu.h>
1716 #include <linux/pfn_t.h>
1817 #include <linux/version.h>
1918 #include <asm/cacheflush.h>
....@@ -25,6 +24,7 @@
2524 #include "rknpu_drv.h"
2625 #include "rknpu_ioctl.h"
2726 #include "rknpu_gem.h"
27
+#include "rknpu_iommu.h"
2828 
2929 #define RKNPU_GEM_ALLOC_FROM_PAGES 1
3030 
....@@ -67,6 +67,7 @@
6767 			      rknpu_obj->size);
6868 		goto free_sgt;
6969 	}
70
+	iommu_flush_iotlb_all(iommu_get_domain_for_dev(drm->dev));
7071 
7172 	if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
7273 		rknpu_obj->cookie = vmap(rknpu_obj->pages, rknpu_obj->num_pages,
....@@ -115,15 +116,14 @@
115116 		rknpu_obj->kv_addr = NULL;
116117 	}
117118 
118
-	dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
119
-		     DMA_BIDIRECTIONAL);
120
-
121
-	drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true);
122
-
123119 	if (rknpu_obj->sgt != NULL) {
120
+		dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl,
121
+			     rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL);
124122 		sg_free_table(rknpu_obj->sgt);
125123 		kfree(rknpu_obj->sgt);
126124 	}
125
+
126
+	drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true);
127127 }
128128 #endif
129129 
....@@ -182,7 +182,9 @@
182182 	if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
183183 		gfp_mask |= __GFP_ZERO;
184184 
185
-	if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) {
185
+	if (!rknpu_dev->iommu_en ||
186
+	    rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) ||
187
+	    (rknpu_obj->flags & RKNPU_MEM_DMA32)) {
186188 		gfp_mask &= ~__GFP_HIGHMEM;
187189 		gfp_mask |= __GFP_DMA32;
188190 	}
....@@ -253,10 +255,15 @@
253255 			  i, &s->dma_address, s->length);
254256 	}
255257 
256
-	if (drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
257
-					     nr_pages)) {
258
-		LOG_DEV_ERROR(drm->dev, "invalid sgtable.\n");
259
-		ret = -EINVAL;
258
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
259
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
260
+					       nr_pages);
261
+#else
262
+	ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, nr_pages);
263
+#endif
264
+
265
+	if (ret < 0) {
266
+		LOG_DEV_ERROR(drm->dev, "invalid sgtable, ret: %d\n", ret);
260267 		goto err_free_sg_table;
261268 	}
262269 
....@@ -335,9 +342,28 @@
335342 	return drm_gem_handle_delete(file_priv, handle);
336343 }
337344 
345
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
346
+static const struct vm_operations_struct vm_ops = {
347
+	.fault = rknpu_gem_fault,
348
+	.open = drm_gem_vm_open,
349
+	.close = drm_gem_vm_close,
350
+};
351
+
352
+static const struct drm_gem_object_funcs rknpu_gem_object_funcs = {
353
+	.free = rknpu_gem_free_object,
354
+	.export = drm_gem_prime_export,
355
+	.get_sg_table = rknpu_gem_prime_get_sg_table,
356
+	.vmap = rknpu_gem_prime_vmap,
357
+	.vunmap = rknpu_gem_prime_vunmap,
358
+	.mmap = rknpu_gem_mmap_obj,
359
+	.vm_ops = &vm_ops,
360
+};
361
+#endif
362
+
338363 static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm,
339364 					       unsigned long size)
340365 {
366
+	struct rknpu_device *rknpu_dev = drm->dev_private;
341367 	struct rknpu_gem_object *rknpu_obj = NULL;
342368 	struct drm_gem_object *obj = NULL;
343369 	gfp_t gfp_mask;
....@@ -348,6 +374,9 @@
348374 		return ERR_PTR(-ENOMEM);
349375 
350376 	obj = &rknpu_obj->base;
377
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
378
+	obj->funcs = &rknpu_gem_object_funcs;
379
+#endif
351380 
352381 	ret = drm_gem_object_init(drm, obj, size);
353382 	if (ret < 0) {
....@@ -363,7 +392,9 @@
363392 	if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
364393 		gfp_mask |= __GFP_ZERO;
365394 
366
-	if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) {
395
+	if (!rknpu_dev->iommu_en ||
396
+	    rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) ||
397
+	    (rknpu_obj->flags & RKNPU_MEM_DMA32)) {
367398 		gfp_mask &= ~__GFP_HIGHMEM;
368399 		gfp_mask |= __GFP_DMA32;
369400 	}
....@@ -380,7 +411,8 @@
380411 	kfree(rknpu_obj);
381412 }
382413 
383
-static int rknpu_gem_alloc_buf_with_sram(struct rknpu_gem_object *rknpu_obj)
414
+static int rknpu_gem_alloc_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
415
+					  enum rknpu_cache_type cache_type)
384416 {
385417 	struct drm_device *drm = rknpu_obj->base.dev;
386418 	struct rknpu_device *rknpu_dev = drm->dev_private;
....@@ -393,18 +425,37 @@
393425 	unsigned long offset = 0;
394426 	int i = 0;
395427 	int ret = -EINVAL;
428
+	phys_addr_t cache_start = 0;
429
+	unsigned long cache_offset = 0;
430
+	unsigned long cache_size = 0;
396431 
397
-	/* iova map to sram */
432
+	switch (cache_type) {
433
+	case RKNPU_CACHE_SRAM:
434
+		cache_start = rknpu_dev->sram_start;
435
+		cache_offset = rknpu_obj->sram_obj->range_start *
436
+			       rknpu_dev->sram_mm->chunk_size;
437
+		cache_size = rknpu_obj->sram_size;
438
+		break;
439
+	case RKNPU_CACHE_NBUF:
440
+		cache_start = rknpu_dev->nbuf_start;
441
+		cache_offset = 0;
442
+		cache_size = rknpu_obj->nbuf_size;
443
+		break;
444
+	default:
445
+		LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
446
+		return -EINVAL;
447
+	}
448
+
449
+	/* iova map to cache */
398450 	domain = iommu_get_domain_for_dev(rknpu_dev->dev);
399451 	if (!domain) {
400452 		LOG_ERROR("failed to get iommu domain!");
401453 		return -EINVAL;
402454 	}
403455 
404
-	cookie = domain->iova_cookie;
456
+	cookie = (void *)domain->iova_cookie;
405457 	iovad = &cookie->iovad;
406
-	rknpu_obj->iova_size =
407
-		iova_align(iovad, rknpu_obj->sram_size + rknpu_obj->size);
458
+	rknpu_obj->iova_size = iova_align(iovad, cache_size + rknpu_obj->size);
408459 	rknpu_obj->iova_start = rknpu_iommu_dma_alloc_iova(
409460 		domain, rknpu_obj->iova_size, dma_get_mask(drm->dev), drm->dev);
410461 	if (!rknpu_obj->iova_start) {
....@@ -416,20 +467,20 @@
416467 		 &rknpu_obj->iova_start, rknpu_obj->iova_size);
417468 
418469 	/*
419
-	 * Overview SRAM + DDR map to IOVA
470
+	 * Overview cache + DDR map to IOVA
420471 	 * --------
421
-	 * sram_size: rknpu_obj->sram_size
422
-	 *   - allocate from SRAM, this size value has been page-aligned
472
+	 * cache_size:
473
+	 *   - allocate from CACHE, this size value has been page-aligned
423474 	 * size: rknpu_obj->size
424475 	 *   - allocate from DDR pages, this size value has been page-aligned
425476 	 * iova_size: rknpu_obj->iova_size
426
-	 *   - from iova_align(sram_size + size)
427
-	 *   - it may be larger than the (sram_size + size), and the larger part is not mapped
477
+	 *   - from iova_align(cache_size + size)
478
+	 *   - it may be larger than the (cache_size + size), and the larger part is not mapped
428479 	 * --------
429480 	 *
430
-	 * |<- sram_size ->|      |<- - - - size - - - ->|
481
+	 * |<- cache_size ->|      |<- - - - size - - - ->|
431482 	 * +---------------+      +----------------------+
432
-	 * |     SRAM      |      |         DDR          |
483
+	 * |     CACHE      |      |         DDR          |
433484 	 * +---------------+      +----------------------+
434485 	 *         |                    |
435486 	 * |       V       |            V          |
....@@ -439,20 +490,18 @@
439490 	 * |<- - - - - - - iova_size - - - - - - ->|
440491 	 *
441492 	 */
442
-	offset = rknpu_obj->sram_obj->range_start *
443
-		 rknpu_dev->sram_mm->chunk_size;
444493 	ret = iommu_map(domain, rknpu_obj->iova_start,
445
-			rknpu_dev->sram_start + offset, rknpu_obj->sram_size,
494
+			cache_start + cache_offset, cache_size,
446495 			IOMMU_READ | IOMMU_WRITE);
447496 	if (ret) {
448
-		LOG_ERROR("sram iommu_map error: %d\n", ret);
497
+		LOG_ERROR("cache iommu_map error: %d\n", ret);
449498 		goto free_iova;
450499 	}
451500 
452501 	rknpu_obj->dma_addr = rknpu_obj->iova_start;
453502 
454503 	if (rknpu_obj->size == 0) {
455
-		LOG_INFO("allocate sram size: %lu\n", rknpu_obj->sram_size);
504
+		LOG_INFO("allocate cache size: %lu\n", cache_size);
456505 		return 0;
457506 	}
458507 
....@@ -460,7 +509,7 @@
460509 	if (IS_ERR(rknpu_obj->pages)) {
461510 		ret = PTR_ERR(rknpu_obj->pages);
462511 		LOG_ERROR("failed to get pages: %d\n", ret);
463
-		goto sram_unmap;
512
+		goto cache_unmap;
464513 	}
465514 
466515 	rknpu_obj->num_pages = rknpu_obj->size >> PAGE_SHIFT;
....@@ -479,7 +528,7 @@
479528 	}
480529 
481530 	length = rknpu_obj->size;
482
-	offset = rknpu_obj->iova_start + rknpu_obj->sram_size;
531
+	offset = rknpu_obj->iova_start + cache_size;
483532 
484533 	for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
485534 		size = (length < s->length) ? length : s->length;
....@@ -498,13 +547,13 @@
498547 			break;
499548 	}
500549 
501
-	LOG_INFO("allocate size: %lu with sram size: %lu\n", rknpu_obj->size,
502
-		 rknpu_obj->sram_size);
550
+	LOG_INFO("allocate size: %lu with cache size: %lu\n", rknpu_obj->size,
551
+		 cache_size);
503552 
504553 	return 0;
505554 
506555 sgl_unmap:
507
-	iommu_unmap(domain, rknpu_obj->iova_start + rknpu_obj->sram_size,
556
+	iommu_unmap(domain, rknpu_obj->iova_start + cache_size,
508557 		    rknpu_obj->size - length);
509558 	sg_free_table(rknpu_obj->sgt);
510559 	kfree(rknpu_obj->sgt);
....@@ -512,32 +561,43 @@
512561 put_pages:
513562 	drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
514563 
515
-sram_unmap:
516
-	iommu_unmap(domain, rknpu_obj->iova_start, rknpu_obj->sram_size);
564
+cache_unmap:
565
+	iommu_unmap(domain, rknpu_obj->iova_start, cache_size);
517566 
518567 free_iova:
519
-	rknpu_iommu_dma_free_iova(domain->iova_cookie, rknpu_obj->iova_start,
520
-				  rknpu_obj->iova_size);
568
+	rknpu_iommu_dma_free_iova((void *)domain->iova_cookie,
569
+				  rknpu_obj->iova_start, rknpu_obj->iova_size);
521570 
522571 	return ret;
523572 }
524573 
525
-static void rknpu_gem_free_buf_with_sram(struct rknpu_gem_object *rknpu_obj)
574
+static void rknpu_gem_free_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
575
+					  enum rknpu_cache_type cache_type)
526576 {
527577 	struct drm_device *drm = rknpu_obj->base.dev;
528578 	struct rknpu_device *rknpu_dev = drm->dev_private;
529579 	struct iommu_domain *domain = NULL;
580
+	unsigned long cache_size = 0;
581
+
582
+	switch (cache_type) {
583
+	case RKNPU_CACHE_SRAM:
584
+		cache_size = rknpu_obj->sram_size;
585
+		break;
586
+	case RKNPU_CACHE_NBUF:
587
+		cache_size = rknpu_obj->nbuf_size;
588
+		break;
589
+	default:
590
+		LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
591
+		return;
592
+	}
530593 
531594 	domain = iommu_get_domain_for_dev(rknpu_dev->dev);
532595 	if (domain) {
533
-		iommu_unmap(domain, rknpu_obj->iova_start,
534
-			    rknpu_obj->sram_size);
596
+		iommu_unmap(domain, rknpu_obj->iova_start, cache_size);
535597 		if (rknpu_obj->size > 0)
536
-			iommu_unmap(domain,
537
-				    rknpu_obj->iova_start +
538
-					    rknpu_obj->sram_size,
598
+			iommu_unmap(domain, rknpu_obj->iova_start + cache_size,
539599 				    rknpu_obj->size);
540
-		rknpu_iommu_dma_free_iova(domain->iova_cookie,
600
+		rknpu_iommu_dma_free_iova((void *)domain->iova_cookie,
541601 					  rknpu_obj->iova_start,
542602 					  rknpu_obj->iova_size);
543603 	}
....@@ -618,9 +678,35 @@
618678 		if (real_sram_size > 0) {
619679 			rknpu_obj->sram_size = real_sram_size;
620680 
621
-			ret = rknpu_gem_alloc_buf_with_sram(rknpu_obj);
681
+			ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj,
682
+							     RKNPU_CACHE_SRAM);
622683 			if (ret < 0)
623684 				goto mm_free;
685
+			remain_ddr_size = 0;
686
+		}
687
+	} else if (IS_ENABLED(CONFIG_NO_GKI) &&
688
+		   (flags & RKNPU_MEM_TRY_ALLOC_NBUF) &&
689
+		   rknpu_dev->nbuf_size > 0) {
690
+		size_t nbuf_size = 0;
691
+
692
+		rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
693
+		if (IS_ERR(rknpu_obj))
694
+			return rknpu_obj;
695
+
696
+		nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ?
697
+				    remain_ddr_size :
698
+				    rknpu_dev->nbuf_size;
699
+
700
+		/* set memory type and cache attribute from user side. */
701
+		rknpu_obj->flags = flags;
702
+
703
+		if (nbuf_size > 0) {
704
+			rknpu_obj->nbuf_size = nbuf_size;
705
+
706
+			ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj,
707
+							     RKNPU_CACHE_NBUF);
708
+			if (ret < 0)
709
+				goto gem_release;
624710 			remain_ddr_size = 0;
625711 		}
626712 	}
....@@ -640,9 +726,11 @@
640726 
641727 	if (rknpu_obj)
642728 		LOG_DEBUG(
643
-			"created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, attrs: %#lx, flags: %#x\n",
644
-			&rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
645
-			rknpu_obj->sram_size, rknpu_obj->dma_attrs, rknpu_obj->flags);
729
+			"created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x\n",
730
+			&rknpu_obj->dma_addr, rknpu_obj->cookie,
731
+			rknpu_obj->size, rknpu_obj->sram_size,
732
+			rknpu_obj->nbuf_size, rknpu_obj->dma_attrs,
733
+			rknpu_obj->flags);
646734 
647735 	return rknpu_obj;
648736 
....@@ -683,7 +771,12 @@
683771 			if (rknpu_obj->sram_obj != NULL)
684772 				rknpu_mm_free(rknpu_dev->sram_mm,
685773 					      rknpu_obj->sram_obj);
686
-			rknpu_gem_free_buf_with_sram(rknpu_obj);
774
+			rknpu_gem_free_buf_with_cache(rknpu_obj,
775
+						      RKNPU_CACHE_SRAM);
776
+		} else if (IS_ENABLED(CONFIG_NO_GKI) &&
777
+			   rknpu_obj->nbuf_size > 0) {
778
+			rknpu_gem_free_buf_with_cache(rknpu_obj,
779
+						      RKNPU_CACHE_NBUF);
687780 		} else {
688781 			rknpu_gem_free_buf(rknpu_obj);
689782 		}
....@@ -808,6 +901,75 @@
808901 }
809902 #endif
810903 
904
+static int rknpu_gem_mmap_cache(struct rknpu_gem_object *rknpu_obj,
905
+				struct vm_area_struct *vma,
906
+				enum rknpu_cache_type cache_type)
907
+{
908
+	struct drm_device *drm = rknpu_obj->base.dev;
909
+#if RKNPU_GEM_ALLOC_FROM_PAGES
910
+	struct rknpu_device *rknpu_dev = drm->dev_private;
911
+#endif
912
+	unsigned long vm_size = 0;
913
+	int ret = -EINVAL;
914
+	unsigned long offset = 0;
915
+	unsigned long num_pages = 0;
916
+	int i = 0;
917
+	phys_addr_t cache_start = 0;
918
+	unsigned long cache_offset = 0;
919
+	unsigned long cache_size = 0;
920
+
921
+	switch (cache_type) {
922
+	case RKNPU_CACHE_SRAM:
923
+		cache_start = rknpu_dev->sram_start;
924
+		cache_offset = rknpu_obj->sram_obj->range_start *
925
+			       rknpu_dev->sram_mm->chunk_size;
926
+		cache_size = rknpu_obj->sram_size;
927
+		break;
928
+	case RKNPU_CACHE_NBUF:
929
+		cache_start = rknpu_dev->nbuf_start;
930
+		cache_offset = 0;
931
+		cache_size = rknpu_obj->nbuf_size;
932
+		break;
933
+	default:
934
+		LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
935
+		return -EINVAL;
936
+	}
937
+
938
+	vma->vm_flags |= VM_MIXEDMAP;
939
+
940
+	vm_size = vma->vm_end - vma->vm_start;
941
+
942
+	/*
943
+	 * Convert a physical address in a cache area to a page frame number (PFN),
944
+	 * and store the resulting PFN in the vm_pgoff field of the given VMA.
945
+	 *
946
+	 * NOTE: This conversion carries a risk because the resulting PFN is not a true
947
+	 * page frame number and may not be valid or usable in all contexts.
948
+	 */
949
+	vma->vm_pgoff = __phys_to_pfn(cache_start + cache_offset);
950
+
951
+	ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, cache_size,
952
+			      vma->vm_page_prot);
953
+	if (ret)
954
+		return -EAGAIN;
955
+
956
+	if (rknpu_obj->size == 0)
957
+		return 0;
958
+
959
+	offset = cache_size;
960
+
961
+	num_pages = (vm_size - cache_size) / PAGE_SIZE;
962
+	for (i = 0; i < num_pages; ++i) {
963
+		ret = vm_insert_page(vma, vma->vm_start + offset,
964
+				     rknpu_obj->pages[i]);
965
+		if (ret < 0)
966
+			return ret;
967
+		offset += PAGE_SIZE;
968
+	}
969
+
970
+	return 0;
971
+}
972
+
811973 static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj,
812974 				 struct vm_area_struct *vma)
813975 {
....@@ -823,6 +985,7 @@
823985 	 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
824986 	 * the whole buffer.
825987 	 */
988
+	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
826989 	vma->vm_flags &= ~VM_PFNMAP;
827990 	vma->vm_pgoff = 0;
828991 
....@@ -832,38 +995,10 @@
832995 	if (vm_size > rknpu_obj->size)
833996 		return -EINVAL;
834997 
835
-	if (rknpu_obj->sram_size > 0) {
836
-		unsigned long offset = 0;
837
-		unsigned long num_pages = 0;
838
-		int i = 0;
839
-
840
-		vma->vm_flags |= VM_MIXEDMAP;
841
-
842
-		offset = rknpu_obj->sram_obj->range_start *
843
-			 rknpu_dev->sram_mm->chunk_size;
844
-		vma->vm_pgoff = __phys_to_pfn(rknpu_dev->sram_start + offset);
845
-
846
-		ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
847
-				      rknpu_obj->sram_size, vma->vm_page_prot);
848
-		if (ret)
849
-			return -EAGAIN;
850
-
851
-		if (rknpu_obj->size == 0)
852
-			return 0;
853
-
854
-		offset = rknpu_obj->sram_size;
855
-
856
-		num_pages = (vm_size - rknpu_obj->sram_size) / PAGE_SIZE;
857
-		for (i = 0; i < num_pages; ++i) {
858
-			ret = vm_insert_page(vma, vma->vm_start + offset,
859
-					     rknpu_obj->pages[i]);
860
-			if (ret < 0)
861
-				return ret;
862
-			offset += PAGE_SIZE;
863
-		}
864
-
865
-		return 0;
866
-	}
998
+	if (rknpu_obj->sram_size > 0)
999
+		return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_SRAM);
1000
+	else if (rknpu_obj->nbuf_size > 0)
1001
+		return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_NBUF);
8671002 
8681003 #if RKNPU_GEM_ALLOC_FROM_PAGES
8691004 	if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
....@@ -1045,8 +1180,7 @@
10451180 }
10461181 #endif
10471182 
1048
-static int rknpu_gem_mmap_obj(struct drm_gem_object *obj,
1049
-			      struct vm_area_struct *vma)
1183
+int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma)
10501184 {
10511185 	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
10521186 	int ret = -EINVAL;
....@@ -1143,8 +1277,12 @@
11431277 		goto err;
11441278 	}
11451279 
1280
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
11461281 	ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
11471282 					       npages);
1283
+#else
1284
+	ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, npages);
1285
+#endif
11481286 	if (ret < 0)
11491287 		goto err_free_large;
11501288 
....@@ -1172,6 +1310,7 @@
11721310 	return ERR_PTR(ret);
11731311 }
11741312 
1313
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
11751314 void *rknpu_gem_prime_vmap(struct drm_gem_object *obj)
11761315 {
11771316 	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
....@@ -1187,6 +1326,35 @@
11871326 {
11881327 	vunmap(vaddr);
11891328 }
1329
+#else
1330
+int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1331
+{
1332
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1333
+	void *vaddr = NULL;
1334
+
1335
+	if (!rknpu_obj->pages)
1336
+		return -EINVAL;
1337
+
1338
+	vaddr = vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
1339
+			  PAGE_KERNEL);
1340
+	if (!vaddr)
1341
+		return -ENOMEM;
1342
+
1343
+	iosys_map_set_vaddr(map, vaddr);
1344
+
1345
+	return 0;
1346
+}
1347
+
1348
+void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1349
+{
1350
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
1351
+
1352
+	if (rknpu_obj->pages) {
1353
+		vunmap(map->vaddr);
1354
+		map->vaddr = NULL;
1355
+	}
1356
+}
1357
+#endif
11901358 
11911359 int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
11921360 {
....@@ -1199,14 +1367,68 @@
11991367 	return rknpu_gem_mmap_obj(obj, vma);
12001368 }
12011369 
1370
+static int rknpu_cache_sync(struct rknpu_gem_object *rknpu_obj,
1371
+			    unsigned long *length, unsigned long *offset,
1372
+			    enum rknpu_cache_type cache_type)
1373
+{
1374
+#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1375
+	struct drm_gem_object *obj = &rknpu_obj->base;
1376
+	struct rknpu_device *rknpu_dev = obj->dev->dev_private;
1377
+	void __iomem *cache_base_io = NULL;
1378
+	unsigned long cache_offset = 0;
1379
+	unsigned long cache_size = 0;
1380
+
1381
+	switch (cache_type) {
1382
+	case RKNPU_CACHE_SRAM:
1383
+		cache_base_io = rknpu_dev->sram_base_io;
1384
+		cache_offset = rknpu_obj->sram_obj->range_start *
1385
+			       rknpu_dev->sram_mm->chunk_size;
1386
+		cache_size = rknpu_obj->sram_size;
1387
+		break;
1388
+	case RKNPU_CACHE_NBUF:
1389
+		cache_base_io = rknpu_dev->nbuf_base_io;
1390
+		cache_offset = 0;
1391
+		cache_size = rknpu_obj->nbuf_size;
1392
+		break;
1393
+	default:
1394
+		LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
1395
+		return -EINVAL;
1396
+	}
1397
+
1398
+	if ((*offset + *length) <= cache_size) {
1399
+		__dma_map_area(cache_base_io + *offset + cache_offset, *length,
1400
+			       DMA_TO_DEVICE);
1401
+		__dma_unmap_area(cache_base_io + *offset + cache_offset,
1402
+				 *length, DMA_FROM_DEVICE);
1403
+		*length = 0;
1404
+		*offset = 0;
1405
+	} else if (*offset >= cache_size) {
1406
+		*offset -= cache_size;
1407
+	} else {
1408
+		unsigned long cache_length = cache_size - *offset;
1409
+
1410
+		__dma_map_area(cache_base_io + *offset + cache_offset,
1411
+			       cache_length, DMA_TO_DEVICE);
1412
+		__dma_unmap_area(cache_base_io + *offset + cache_offset,
1413
+				 cache_length, DMA_FROM_DEVICE);
1414
+		*length -= cache_length;
1415
+		*offset = 0;
1416
+	}
1417
+#endif
1418
+
1419
+	return 0;
1420
+}
1421
+
12021422 int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
12031423 			 struct drm_file *file_priv)
12041424 {
12051425 	struct rknpu_gem_object *rknpu_obj = NULL;
1426
+	struct rknpu_device *rknpu_dev = dev->dev_private;
12061427 	struct rknpu_mem_sync *args = data;
12071428 	struct scatterlist *sg;
1429
+	dma_addr_t sg_phys_addr;
12081430 	unsigned long length, offset = 0;
1209
-	unsigned long sg_left, size = 0;
1431
+	unsigned long sg_offset, sg_left, size = 0;
12101432 	unsigned long len = 0;
12111433 	int i;
12121434 
....@@ -1230,38 +1452,20 @@
12301452 						      DMA_FROM_DEVICE);
12311453 		}
12321454 	} else {
1455
+		WARN_ON(!rknpu_dev->fake_dev);
1456
+
12331457 		length = args->size;
12341458 		offset = args->offset;
12351459 
1236
-		if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_obj->sram_size > 0) {
1237
-			struct drm_gem_object *obj = &rknpu_obj->base;
1238
-			struct rknpu_device *rknpu_dev = obj->dev->dev_private;
1239
-			unsigned long sram_offset =
1240
-				rknpu_obj->sram_obj->range_start *
1241
-				rknpu_dev->sram_mm->chunk_size;
1242
-			if ((offset + length) <= rknpu_obj->sram_size) {
1243
-				__dma_map_area(rknpu_dev->sram_base_io +
1244
-						       offset + sram_offset,
1245
-					       length, DMA_TO_DEVICE);
1246
-				__dma_unmap_area(rknpu_dev->sram_base_io +
1247
-							 offset + sram_offset,
1248
-						 length, DMA_FROM_DEVICE);
1249
-				length = 0;
1250
-				offset = 0;
1251
-			} else if (offset >= rknpu_obj->sram_size) {
1252
-				offset -= rknpu_obj->sram_size;
1253
-			} else {
1254
-				unsigned long sram_length =
1255
-					rknpu_obj->sram_size - offset;
1256
-				__dma_map_area(rknpu_dev->sram_base_io +
1257
-						       offset + sram_offset,
1258
-					       sram_length, DMA_TO_DEVICE);
1259
-				__dma_unmap_area(rknpu_dev->sram_base_io +
1260
-							 offset + sram_offset,
1261
-						 sram_length, DMA_FROM_DEVICE);
1262
-				length -= sram_length;
1263
-				offset = 0;
1264
-			}
1460
+		if (IS_ENABLED(CONFIG_NO_GKI) &&
1461
+		    IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
1462
+		    rknpu_obj->sram_size > 0) {
1463
+			rknpu_cache_sync(rknpu_obj, &length, &offset,
1464
+					 RKNPU_CACHE_SRAM);
1465
+		} else if (IS_ENABLED(CONFIG_NO_GKI) &&
1466
+			   rknpu_obj->nbuf_size > 0) {
1467
+			rknpu_cache_sync(rknpu_obj, &length, &offset,
1468
+					 RKNPU_CACHE_NBUF);
12651469 		}
12661470 
12671471 		for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents,
....@@ -1273,17 +1477,23 @@
12731477 			if (len <= offset)
12741478 				continue;
12751479 
1480
+			sg_phys_addr = sg_phys(sg);
1481
+
12761482 			sg_left = len - offset;
1483
+			sg_offset = sg->length - sg_left;
1484
+
12771485 			size = (length < sg_left) ? length : sg_left;
12781486 
12791487 			if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
1280
-				dma_sync_sg_for_device(dev->dev, sg, 1,
1281
-						       DMA_TO_DEVICE);
1488
+				dma_sync_single_range_for_device(
1489
+					rknpu_dev->fake_dev, sg_phys_addr,
1490
+					sg_offset, size, DMA_TO_DEVICE);
12821491 			}
12831492 
12841493 			if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
1285
-				dma_sync_sg_for_cpu(dev->dev, sg, 1,
1286
-						    DMA_FROM_DEVICE);
1494
+				dma_sync_single_range_for_cpu(
1495
+					rknpu_dev->fake_dev, sg_phys_addr,
1496
+					sg_offset, size, DMA_FROM_DEVICE);
12871497 			}
12881498 
12891499 			offset += size;