// SPDX-License-Identifier: GPL-2.0
/*
 * ION Memory Allocator system heap exporter
 *
 * Copyright (C) 2011 Google, Inc.
 */

#include <asm/page.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/highmem.h>
#include <linux/ion.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

#include "ion_page_pool.h"

#define NUM_ORDERS ARRAY_SIZE(orders)

static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
				     __GFP_NORETRY) & ~__GFP_RECLAIM;
static gfp_t low_order_gfp_flags  = GFP_HIGHUSER | __GFP_ZERO;
static const unsigned int orders[] = {8, 4, 0};

static int order_to_index(unsigned int order)
{
	int i;

	for (i = 0; i < NUM_ORDERS; i++)
		if (order == orders[i])
			return i;
	BUG();
	return -1;
}

static inline unsigned int order_to_size(int order)
{
	return PAGE_SIZE << order;
}

struct ion_system_heap {
	struct ion_heap heap;
	struct ion_page_pool *pools[NUM_ORDERS];
};

static struct page *alloc_buffer_page(struct ion_system_heap *heap,
				      struct ion_buffer *buffer,
				      unsigned long order)
{
	struct ion_page_pool *pool = heap->pools[order_to_index(order)];

	return ion_page_pool_alloc(pool);
}

static void free_buffer_page(struct ion_system_heap *heap,
			     struct ion_buffer *buffer, struct page *page)
{
	struct ion_page_pool *pool;
	unsigned int order = compound_order(page);

	/* go to system */
	if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) {
		__free_pages(page, order);
		return;
	}

	pool = heap->pools[order_to_index(order)];

	ion_page_pool_free(pool, page);
}

static struct page *alloc_largest_available(struct ion_system_heap *heap,
					    struct ion_buffer *buffer,
					    unsigned long size,
					    unsigned int max_order)
{
	struct page *page;
	int i;

	for (i = 0; i < NUM_ORDERS; i++) {
		if (size < order_to_size(orders[i]))
			continue;
		if (max_order < orders[i])
			continue;

		page = alloc_buffer_page(heap, buffer, orders[i]);
		if (!page)
			continue;

		return page;
	}

	return NULL;
}

static int ion_system_heap_allocate(struct ion_heap *heap,
				    struct ion_buffer *buffer,
				    unsigned long size,
				    unsigned long flags)
{
	struct ion_system_heap *sys_heap = container_of(heap,
							struct ion_system_heap,
							heap);
	struct sg_table *table;
	struct scatterlist *sg;
	struct list_head pages;
	struct page *page, *tmp_page;
	int i = 0;
	unsigned long size_remaining = PAGE_ALIGN(size);
	unsigned int max_order = orders[0];
	struct list_head lists[8];
	unsigned int block_index[8] = {0};
	unsigned int block_1M = 0;
	unsigned int block_64K = 0;
	unsigned int maximum;
	int j;

	if (size / PAGE_SIZE > totalram_pages() / 2)
		return -ENOMEM;

	INIT_LIST_HEAD(&pages);
	for (i = 0; i < 8; i++)
		INIT_LIST_HEAD(&lists[i]);

	i = 0;
	while (size_remaining > 0) {
		page = alloc_largest_available(sys_heap, buffer, size_remaining,
					       max_order);
		if (!page)
			goto free_pages;

		size_remaining -= page_size(page);
		max_order = compound_order(page);
		if (max_order) {
			if (max_order == 8)
				block_1M++;
			if (max_order == 4)
				block_64K++;
			list_add_tail(&page->lru, &pages);
		} else {
			dma_addr_t phys = page_to_phys(page);
			unsigned int bit12_14 = (phys >> 12) & 0x7;

			list_add_tail(&page->lru, &lists[bit12_14]);
			block_index[bit12_14]++;
		}

		i++;
	}

	pr_debug("%s, %d, i = %d, size = %ld\n", __func__, __LINE__, i, size);

	table = kmalloc(sizeof(*table), GFP_KERNEL);
	if (!table)
		goto free_pages;

	if (sg_alloc_table(table, i, GFP_KERNEL))
		goto free_table;

	maximum = block_index[0];
	for (i = 1; i < 8; i++)
		maximum = max(maximum, block_index[i]);

	pr_debug("%s, %d, maximum = %d, block_1M = %d, block_64K = %d\n",
		 __func__, __LINE__, maximum, block_1M, block_64K);

	for (i = 0; i < 8; i++)
		pr_debug("block_index[%d] = %d\n", i, block_index[i]);

	sg = table->sgl;
	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
		sg_set_page(sg, page, page_size(page), 0);
		sg = sg_next(sg);
		list_del(&page->lru);
	}

	for (i = 0; i < maximum; i++) {
		for (j = 0; j < 8; j++) {
			if (!list_empty(&lists[j])) {
				page = list_first_entry(&lists[j], struct page,
							lru);
				sg_set_page(sg, page, PAGE_SIZE, 0);
				sg = sg_next(sg);
				list_del(&page->lru);
			}
		}
	}

	buffer->sg_table = table;

	ion_buffer_prep_noncached(buffer);

	return 0;

free_table:
	kfree(table);
free_pages:
	list_for_each_entry_safe(page, tmp_page, &pages, lru)
		free_buffer_page(sys_heap, buffer, page);

	for (i = 0; i < 8; i++) {
		list_for_each_entry_safe(page, tmp_page, &lists[i], lru)
			free_buffer_page(sys_heap, buffer, page);
	}
	return -ENOMEM;
}

static void ion_system_heap_free(struct ion_buffer *buffer)
{
	struct ion_system_heap *sys_heap = container_of(buffer->heap,
							struct ion_system_heap,
							heap);
	struct sg_table *table = buffer->sg_table;
	struct scatterlist *sg;
	int i;

	/* zero the buffer before goto page pool */
	if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))
		ion_buffer_zero(buffer);

	for_each_sgtable_sg(table, sg, i)
		free_buffer_page(sys_heap, buffer, sg_page(sg));
	sg_free_table(table);
	kfree(table);
}

static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
				  int nr_to_scan)
{
	struct ion_page_pool *pool;
	struct ion_system_heap *sys_heap;
	int nr_total = 0;
	int i, nr_freed;
	int only_scan = 0;

	sys_heap = container_of(heap, struct ion_system_heap, heap);

	if (!nr_to_scan)
		only_scan = 1;

	for (i = 0; i < NUM_ORDERS; i++) {
		pool = sys_heap->pools[i];

		if (only_scan) {
			nr_total += ion_page_pool_shrink(pool,
							 gfp_mask,
							 nr_to_scan);

		} else {
			nr_freed = ion_page_pool_shrink(pool,
							gfp_mask,
							nr_to_scan);
			nr_to_scan -= nr_freed;
			nr_total += nr_freed;
			if (nr_to_scan <= 0)
				break;
		}
	}
	return nr_total;
}

static long ion_system_get_pool_size(struct ion_heap *heap)
{
	struct ion_system_heap *sys_heap;
	long total_pages = 0;
	int i;

	sys_heap = container_of(heap, struct ion_system_heap, heap);
	for (i = 0; i < NUM_ORDERS; i++)
		total_pages += ion_page_pool_nr_pages(sys_heap->pools[i]);

	return total_pages;
}

static void ion_system_heap_destroy_pools(struct ion_page_pool **pools)
{
	int i;

	for (i = 0; i < NUM_ORDERS; i++)
		if (pools[i])
			ion_page_pool_destroy(pools[i]);
}

static int ion_system_heap_create_pools(struct ion_page_pool **pools)
{
	int i;

	for (i = 0; i < NUM_ORDERS; i++) {
		struct ion_page_pool *pool;
		gfp_t gfp_flags = low_order_gfp_flags;

		if (orders[i] > 4)
			gfp_flags = high_order_gfp_flags;

		pool = ion_page_pool_create(gfp_flags, orders[i]);
		if (!pool)
			goto err_create_pool;
		pools[i] = pool;
	}

	return 0;

err_create_pool:
	ion_system_heap_destroy_pools(pools);
	return -ENOMEM;
}

static struct ion_heap_ops system_heap_ops = {
	.allocate = ion_system_heap_allocate,
	.free = ion_system_heap_free,
	.shrink = ion_system_heap_shrink,
	.get_pool_size = ion_system_get_pool_size,
};

static struct ion_system_heap system_heap = {
	.heap = {
		.ops = &system_heap_ops,
		.type = ION_HEAP_TYPE_SYSTEM,
		.flags = ION_HEAP_FLAG_DEFER_FREE,
		.name = "ion_system_heap",
	}
};

static int __init ion_system_heap_init(void)
{
	int ret = ion_system_heap_create_pools(system_heap.pools);
	if (ret)
		return ret;

	return ion_device_add_heap(&system_heap.heap);
}

static void __exit ion_system_heap_exit(void)
{
	ion_device_remove_heap(&system_heap.heap);
	ion_system_heap_destroy_pools(system_heap.pools);
}

module_init(ion_system_heap_init);
module_exit(ion_system_heap_exit);
MODULE_LICENSE("GPL v2");