~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* z3fold.c
3	4	*
..	..	@@ -24,60 +25,25 @@
24	25
25	26	#include <linux/atomic.h>
26	27	#include <linux/sched.h>
	28	+#include <linux/cpumask.h>
27	29	#include <linux/list.h>
28	30	#include <linux/mm.h>
29	31	#include <linux/module.h>
	32	+#include <linux/page-flags.h>
	33	+#include <linux/migrate.h>
	34	+#include <linux/node.h>
	35	+#include <linux/compaction.h>
30	36	#include <linux/percpu.h>
	37	+#include <linux/mount.h>
	38	+#include <linux/pseudo_fs.h>
	39	+#include <linux/fs.h>
31	40	#include <linux/preempt.h>
32	41	#include <linux/workqueue.h>
33	42	#include <linux/slab.h>
34	43	#include <linux/spinlock.h>
35	44	#include <linux/zpool.h>
36		-
37		-/*****************
38		- * Structures
39		-*****************/
40		-struct z3fold_pool;
41		-struct z3fold_ops {
42		- int (evict)(struct z3fold_pool pool, unsigned long handle);
43		-};
44		-
45		-enum buddy {
46		- HEADLESS = 0,
47		- FIRST,
48		- MIDDLE,
49		- LAST,
50		- BUDDIES_MAX
51		-};
52		-
53		-/*
54		- * struct z3fold_header - z3fold page metadata occupying first chunks of each
55		- * z3fold page, except for HEADLESS pages
56		- * @buddy: links the z3fold page into the relevant list in the
57		- * pool
58		- * @page_lock: per-page lock
59		- * @refcount: reference count for the z3fold page
60		- * @work: work_struct for page layout optimization
61		- * @pool: pointer to the pool which this page belongs to
62		- * @cpu: CPU which this page "belongs" to
63		- * @first_chunks: the size of the first buddy in chunks, 0 if free
64		- * @middle_chunks: the size of the middle buddy in chunks, 0 if free
65		- * @last_chunks: the size of the last buddy in chunks, 0 if free
66		- * @first_num: the starting number (for the first handle)
67		- */
68		-struct z3fold_header {
69		- struct list_head buddy;
70		- spinlock_t page_lock;
71		- struct kref refcount;
72		- struct work_struct work;
73		- struct z3fold_pool *pool;
74		- short cpu;
75		- unsigned short first_chunks;
76		- unsigned short middle_chunks;
77		- unsigned short last_chunks;
78		- unsigned short start_middle;
79		- unsigned short first_num:2;
80		-};
	45	+#include <linux/magic.h>
	46	+#include <linux/kmemleak.h>
81	47
82	48	/*
83	49	* NCHUNKS_ORDER determines the internal allocation granularity, effectively
..	..	@@ -100,6 +66,68 @@
100	66
101	67	#define BUDDY_MASK (0x3)
102	68	#define BUDDY_SHIFT 2
	69	+#define SLOTS_ALIGN (0x40)
	70	+
	71	+/*****************
	72	+ * Structures
	73	+*****************/
	74	+struct z3fold_pool;
	75	+struct z3fold_ops {
	76	+ int (evict)(struct z3fold_pool pool, unsigned long handle);
	77	+};
	78	+
	79	+enum buddy {
	80	+ HEADLESS = 0,
	81	+ FIRST,
	82	+ MIDDLE,
	83	+ LAST,
	84	+ BUDDIES_MAX = LAST
	85	+};
	86	+
	87	+struct z3fold_buddy_slots {
	88	+ /*
	89	+ * we are using BUDDY_MASK in handle_to_buddy etc. so there should
	90	+ * be enough slots to hold all possible variants
	91	+ */
	92	+ unsigned long slot[BUDDY_MASK + 1];
	93	+ unsigned long pool; /* back link */
	94	+ rwlock_t lock;
	95	+};
	96	+#define HANDLE_FLAG_MASK (0x03)
	97	+
	98	+/*
	99	+ * struct z3fold_header - z3fold page metadata occupying first chunks of each
	100	+ * z3fold page, except for HEADLESS pages
	101	+ * @buddy: links the z3fold page into the relevant list in the
	102	+ * pool
	103	+ * @page_lock: per-page lock
	104	+ * @refcount: reference count for the z3fold page
	105	+ * @work: work_struct for page layout optimization
	106	+ * @slots: pointer to the structure holding buddy slots
	107	+ * @pool: pointer to the containing pool
	108	+ * @cpu: CPU which this page "belongs" to
	109	+ * @first_chunks: the size of the first buddy in chunks, 0 if free
	110	+ * @middle_chunks: the size of the middle buddy in chunks, 0 if free
	111	+ * @last_chunks: the size of the last buddy in chunks, 0 if free
	112	+ * @first_num: the starting number (for the first handle)
	113	+ * @mapped_count: the number of objects currently mapped
	114	+ */
	115	+struct z3fold_header {
	116	+ struct list_head buddy;
	117	+ spinlock_t page_lock;
	118	+ struct kref refcount;
	119	+ struct work_struct work;
	120	+ struct z3fold_buddy_slots *slots;
	121	+ struct z3fold_pool *pool;
	122	+ short cpu;
	123	+ unsigned short first_chunks;
	124	+ unsigned short middle_chunks;
	125	+ unsigned short last_chunks;
	126	+ unsigned short start_middle;
	127	+ unsigned short first_num:2;
	128	+ unsigned short mapped_count:2;
	129	+ unsigned short foreign_handles:2;
	130	+};
103	131
104	132	/**
105	133	* struct z3fold_pool - stores metadata for each z3fold pool
..	..	@@ -113,11 +141,13 @@
113	141	* added buddy.
114	142	* @stale: list of pages marked for freeing
115	143	* @pages_nr: number of z3fold pages in the pool.
	144	+ * @c_handle: cache for z3fold_buddy_slots allocation
116	145	* @ops: pointer to a structure of user defined operations specified at
117	146	* pool creation time.
118	147	* @compact_wq: workqueue for page layout background optimization
119	148	* @release_wq: workqueue for safe page release
120	149	* @work: work_struct for safe page release
	150	+ * @inode: inode for z3fold pseudo filesystem
121	151	*
122	152	* This structure is allocated at pool creation time and maintains metadata
123	153	* pertaining to a particular z3fold pool.
..	..	@@ -130,12 +160,14 @@
130	160	struct list_head lru;
131	161	struct list_head stale;
132	162	atomic64_t pages_nr;
	163	+ struct kmem_cache *c_handle;
133	164	const struct z3fold_ops *ops;
134	165	struct zpool *zpool;
135	166	const struct zpool_ops *zpool_ops;
136	167	struct workqueue_struct *compact_wq;
137	168	struct workqueue_struct *release_wq;
138	169	struct work_struct work;
	170	+ struct inode *inode;
139	171	};
140	172
141	173	/*
..	..	@@ -148,6 +180,19 @@
148	180	PAGE_STALE,
149	181	PAGE_CLAIMED, /* by either reclaim or free */
150	182	};
	183	+
	184	+/*
	185	+ * handle flags, go under HANDLE_FLAG_MASK
	186	+ */
	187	+enum z3fold_handle_flags {
	188	+ HANDLES_NOFREE = 0,
	189	+};
	190	+
	191	+/*
	192	+ * Forward declarations
	193	+ */
	194	+static struct z3fold_header __z3fold_alloc(struct z3fold_pool , size_t, bool);
	195	+static void compact_page_work(struct work_struct *w);
151	196
152	197	/*****************
153	198	* Helpers
..	..	@@ -162,39 +207,32 @@
162	207	#define for_each_unbuddied_list(_iter, _begin) \
163	208	for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
164	209
165		-static void compact_page_work(struct work_struct *w);
166		-
167		-/* Initializes the z3fold header of a newly allocated z3fold page */
168		-static struct z3fold_header init_z3fold_page(struct page page,
169		- struct z3fold_pool *pool)
	210	+static inline struct z3fold_buddy_slots alloc_slots(struct z3fold_pool pool,
	211	+ gfp_t gfp)
170	212	{
171		- struct z3fold_header *zhdr = page_address(page);
	213	+ struct z3fold_buddy_slots *slots;
172	214
173		- INIT_LIST_HEAD(&page->lru);
174		- clear_bit(PAGE_HEADLESS, &page->private);
175		- clear_bit(MIDDLE_CHUNK_MAPPED, &page->private);
176		- clear_bit(NEEDS_COMPACTING, &page->private);
177		- clear_bit(PAGE_STALE, &page->private);
178		- clear_bit(PAGE_CLAIMED, &page->private);
	215	+ slots = kmem_cache_zalloc(pool->c_handle,
	216	+ (gfp & ~(__GFP_HIGHMEM \| __GFP_MOVABLE)));
179	217
180		- spin_lock_init(&zhdr->page_lock);
181		- kref_init(&zhdr->refcount);
182		- zhdr->first_chunks = 0;
183		- zhdr->middle_chunks = 0;
184		- zhdr->last_chunks = 0;
185		- zhdr->first_num = 0;
186		- zhdr->start_middle = 0;
187		- zhdr->cpu = -1;
188		- zhdr->pool = pool;
189		- INIT_LIST_HEAD(&zhdr->buddy);
190		- INIT_WORK(&zhdr->work, compact_page_work);
191		- return zhdr;
	218	+ if (slots) {
	219	+ /* It will be freed separately in free_handle(). */
	220	+ kmemleak_not_leak(slots);
	221	+ slots->pool = (unsigned long)pool;
	222	+ rwlock_init(&slots->lock);
	223	+ }
	224	+
	225	+ return slots;
192	226	}
193	227
194		-/* Resets the struct page fields and frees the page */
195		-static void free_z3fold_page(struct page *page)
	228	+static inline struct z3fold_pool slots_to_pool(struct z3fold_buddy_slots s)
196	229	{
197		- __free_page(page);
	230	+ return (struct z3fold_pool *)(s->pool & ~HANDLE_FLAG_MASK);
	231	+}
	232	+
	233	+static inline struct z3fold_buddy_slots *handle_to_slots(unsigned long handle)
	234	+{
	235	+ return (struct z3fold_buddy_slots *)(handle & ~(SLOTS_ALIGN - 1));
198	236	}
199	237
200	238	/* Lock a z3fold page */
..	..	@@ -215,33 +253,246 @@
215	253	spin_unlock(&zhdr->page_lock);
216	254	}
217	255
	256	+
	257	+static inline struct z3fold_header *__get_z3fold_header(unsigned long handle,
	258	+ bool lock)
	259	+{
	260	+ struct z3fold_buddy_slots *slots;
	261	+ struct z3fold_header *zhdr;
	262	+ int locked = 0;
	263	+
	264	+ if (!(handle & (1 << PAGE_HEADLESS))) {
	265	+ slots = handle_to_slots(handle);
	266	+ do {
	267	+ unsigned long addr;
	268	+
	269	+ read_lock(&slots->lock);
	270	+ addr = (unsigned long )handle;
	271	+ zhdr = (struct z3fold_header *)(addr & PAGE_MASK);
	272	+ if (lock)
	273	+ locked = z3fold_page_trylock(zhdr);
	274	+ read_unlock(&slots->lock);
	275	+ if (locked)
	276	+ break;
	277	+ cpu_relax();
	278	+ } while (lock);
	279	+ } else {
	280	+ zhdr = (struct z3fold_header *)(handle & PAGE_MASK);
	281	+ }
	282	+
	283	+ return zhdr;
	284	+}
	285	+
	286	+/* Returns the z3fold page where a given handle is stored */
	287	+static inline struct z3fold_header *handle_to_z3fold_header(unsigned long h)
	288	+{
	289	+ return __get_z3fold_header(h, false);
	290	+}
	291	+
	292	+/* return locked z3fold page if it's not headless */
	293	+static inline struct z3fold_header *get_z3fold_header(unsigned long h)
	294	+{
	295	+ return __get_z3fold_header(h, true);
	296	+}
	297	+
	298	+static inline void put_z3fold_header(struct z3fold_header *zhdr)
	299	+{
	300	+ struct page *page = virt_to_page(zhdr);
	301	+
	302	+ if (!test_bit(PAGE_HEADLESS, &page->private))
	303	+ z3fold_page_unlock(zhdr);
	304	+}
	305	+
	306	+static inline void free_handle(unsigned long handle, struct z3fold_header *zhdr)
	307	+{
	308	+ struct z3fold_buddy_slots *slots;
	309	+ int i;
	310	+ bool is_free;
	311	+
	312	+ if (handle & (1 << PAGE_HEADLESS))
	313	+ return;
	314	+
	315	+ if (WARN_ON((unsigned long )handle == 0))
	316	+ return;
	317	+
	318	+ slots = handle_to_slots(handle);
	319	+ write_lock(&slots->lock);
	320	+ (unsigned long )handle = 0;
	321	+
	322	+ if (test_bit(HANDLES_NOFREE, &slots->pool)) {
	323	+ write_unlock(&slots->lock);
	324	+ return; /* simple case, nothing else to do */
	325	+ }
	326	+
	327	+ if (zhdr->slots != slots)
	328	+ zhdr->foreign_handles--;
	329	+
	330	+ is_free = true;
	331	+ for (i = 0; i <= BUDDY_MASK; i++) {
	332	+ if (slots->slot[i]) {
	333	+ is_free = false;
	334	+ break;
	335	+ }
	336	+ }
	337	+ write_unlock(&slots->lock);
	338	+
	339	+ if (is_free) {
	340	+ struct z3fold_pool *pool = slots_to_pool(slots);
	341	+
	342	+ if (zhdr->slots == slots)
	343	+ zhdr->slots = NULL;
	344	+ kmem_cache_free(pool->c_handle, slots);
	345	+ }
	346	+}
	347	+
	348	+static int z3fold_init_fs_context(struct fs_context *fc)
	349	+{
	350	+ return init_pseudo(fc, Z3FOLD_MAGIC) ? 0 : -ENOMEM;
	351	+}
	352	+
	353	+static struct file_system_type z3fold_fs = {
	354	+ .name = "z3fold",
	355	+ .init_fs_context = z3fold_init_fs_context,
	356	+ .kill_sb = kill_anon_super,
	357	+};
	358	+
	359	+static struct vfsmount *z3fold_mnt;
	360	+static int z3fold_mount(void)
	361	+{
	362	+ int ret = 0;
	363	+
	364	+ z3fold_mnt = kern_mount(&z3fold_fs);
	365	+ if (IS_ERR(z3fold_mnt))
	366	+ ret = PTR_ERR(z3fold_mnt);
	367	+
	368	+ return ret;
	369	+}
	370	+
	371	+static void z3fold_unmount(void)
	372	+{
	373	+ kern_unmount(z3fold_mnt);
	374	+}
	375	+
	376	+static const struct address_space_operations z3fold_aops;
	377	+static int z3fold_register_migration(struct z3fold_pool *pool)
	378	+{
	379	+ pool->inode = alloc_anon_inode(z3fold_mnt->mnt_sb);
	380	+ if (IS_ERR(pool->inode)) {
	381	+ pool->inode = NULL;
	382	+ return 1;
	383	+ }
	384	+
	385	+ pool->inode->i_mapping->private_data = pool;
	386	+ pool->inode->i_mapping->a_ops = &z3fold_aops;
	387	+ return 0;
	388	+}
	389	+
	390	+static void z3fold_unregister_migration(struct z3fold_pool *pool)
	391	+{
	392	+ if (pool->inode)
	393	+ iput(pool->inode);
	394	+ }
	395	+
	396	+/* Initializes the z3fold header of a newly allocated z3fold page */
	397	+static struct z3fold_header init_z3fold_page(struct page page, bool headless,
	398	+ struct z3fold_pool *pool, gfp_t gfp)
	399	+{
	400	+ struct z3fold_header *zhdr = page_address(page);
	401	+ struct z3fold_buddy_slots *slots;
	402	+
	403	+ INIT_LIST_HEAD(&page->lru);
	404	+ clear_bit(PAGE_HEADLESS, &page->private);
	405	+ clear_bit(MIDDLE_CHUNK_MAPPED, &page->private);
	406	+ clear_bit(NEEDS_COMPACTING, &page->private);
	407	+ clear_bit(PAGE_STALE, &page->private);
	408	+ clear_bit(PAGE_CLAIMED, &page->private);
	409	+ if (headless)
	410	+ return zhdr;
	411	+
	412	+ slots = alloc_slots(pool, gfp);
	413	+ if (!slots)
	414	+ return NULL;
	415	+
	416	+ spin_lock_init(&zhdr->page_lock);
	417	+ kref_init(&zhdr->refcount);
	418	+ zhdr->first_chunks = 0;
	419	+ zhdr->middle_chunks = 0;
	420	+ zhdr->last_chunks = 0;
	421	+ zhdr->first_num = 0;
	422	+ zhdr->start_middle = 0;
	423	+ zhdr->cpu = -1;
	424	+ zhdr->foreign_handles = 0;
	425	+ zhdr->mapped_count = 0;
	426	+ zhdr->slots = slots;
	427	+ zhdr->pool = pool;
	428	+ INIT_LIST_HEAD(&zhdr->buddy);
	429	+ INIT_WORK(&zhdr->work, compact_page_work);
	430	+ return zhdr;
	431	+}
	432	+
	433	+/* Resets the struct page fields and frees the page */
	434	+static void free_z3fold_page(struct page *page, bool headless)
	435	+{
	436	+ if (!headless) {
	437	+ lock_page(page);
	438	+ __ClearPageMovable(page);
	439	+ unlock_page(page);
	440	+ }
	441	+ ClearPagePrivate(page);
	442	+ __free_page(page);
	443	+}
	444	+
	445	+/* Helper function to build the index */
	446	+static inline int __idx(struct z3fold_header *zhdr, enum buddy bud)
	447	+{
	448	+ return (bud + zhdr->first_num) & BUDDY_MASK;
	449	+}
	450	+
218	451	/*
219	452	* Encodes the handle of a particular buddy within a z3fold page
220	453	* Pool lock should be held as this function accesses first_num
221	454	*/
222		-static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud)
	455	+static unsigned long __encode_handle(struct z3fold_header *zhdr,
	456	+ struct z3fold_buddy_slots *slots,
	457	+ enum buddy bud)
223	458	{
224		- unsigned long handle;
	459	+ unsigned long h = (unsigned long)zhdr;
	460	+ int idx = 0;
225	461
226		- handle = (unsigned long)zhdr;
227		- if (bud != HEADLESS) {
228		- handle \|= (bud + zhdr->first_num) & BUDDY_MASK;
229		- if (bud == LAST)
230		- handle \|= (zhdr->last_chunks << BUDDY_SHIFT);
231		- }
232		- return handle;
	462	+ /*
	463	+ * For a headless page, its handle is its pointer with the extra
	464	+ * PAGE_HEADLESS bit set
	465	+ */
	466	+ if (bud == HEADLESS)
	467	+ return h \| (1 << PAGE_HEADLESS);
	468	+
	469	+ /* otherwise, return pointer to encoded handle */
	470	+ idx = __idx(zhdr, bud);
	471	+ h += idx;
	472	+ if (bud == LAST)
	473	+ h \|= (zhdr->last_chunks << BUDDY_SHIFT);
	474	+
	475	+ write_lock(&slots->lock);
	476	+ slots->slot[idx] = h;
	477	+ write_unlock(&slots->lock);
	478	+ return (unsigned long)&slots->slot[idx];
233	479	}
234	480
235		-/* Returns the z3fold page where a given handle is stored */
236		-static struct z3fold_header *handle_to_z3fold_header(unsigned long handle)
	481	+static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud)
237	482	{
238		- return (struct z3fold_header *)(handle & PAGE_MASK);
	483	+ return __encode_handle(zhdr, zhdr->slots, bud);
239	484	}
240	485
241	486	/* only for LAST bud, returns zero otherwise */
242	487	static unsigned short handle_to_chunks(unsigned long handle)
243	488	{
244		- return (handle & ~PAGE_MASK) >> BUDDY_SHIFT;
	489	+ struct z3fold_buddy_slots *slots = handle_to_slots(handle);
	490	+ unsigned long addr;
	491	+
	492	+ read_lock(&slots->lock);
	493	+ addr = (unsigned long )handle;
	494	+ read_unlock(&slots->lock);
	495	+ return (addr & ~PAGE_MASK) >> BUDDY_SHIFT;
245	496	}
246	497
247	498	/*
..	..	@@ -251,24 +502,39 @@
251	502	*/
252	503	static enum buddy handle_to_buddy(unsigned long handle)
253	504	{
254		- struct z3fold_header *zhdr = handle_to_z3fold_header(handle);
255		- return (handle - zhdr->first_num) & BUDDY_MASK;
	505	+ struct z3fold_header *zhdr;
	506	+ struct z3fold_buddy_slots *slots = handle_to_slots(handle);
	507	+ unsigned long addr;
	508	+
	509	+ read_lock(&slots->lock);
	510	+ WARN_ON(handle & (1 << PAGE_HEADLESS));
	511	+ addr = (unsigned long )handle;
	512	+ read_unlock(&slots->lock);
	513	+ zhdr = (struct z3fold_header *)(addr & PAGE_MASK);
	514	+ return (addr - zhdr->first_num) & BUDDY_MASK;
	515	+}
	516	+
	517	+static inline struct z3fold_pool zhdr_to_pool(struct z3fold_header zhdr)
	518	+{
	519	+ return zhdr->pool;
256	520	}
257	521
258	522	static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked)
259	523	{
260	524	struct page *page = virt_to_page(zhdr);
261		- struct z3fold_pool *pool = zhdr->pool;
	525	+ struct z3fold_pool *pool = zhdr_to_pool(zhdr);
262	526
263	527	WARN_ON(!list_empty(&zhdr->buddy));
264	528	set_bit(PAGE_STALE, &page->private);
265	529	clear_bit(NEEDS_COMPACTING, &page->private);
266	530	spin_lock(&pool->lock);
267	531	if (!list_empty(&page->lru))
268		- list_del(&page->lru);
	532	+ list_del_init(&page->lru);
269	533	spin_unlock(&pool->lock);
	534	+
270	535	if (locked)
271	536	z3fold_page_unlock(zhdr);
	537	+
272	538	spin_lock(&pool->stale_lock);
273	539	list_add(&zhdr->buddy, &pool->stale);
274	540	queue_work(pool->release_wq, &pool->work);
..	..	@@ -295,9 +561,11 @@
295	561	{
296	562	struct z3fold_header *zhdr = container_of(ref, struct z3fold_header,
297	563	refcount);
298		- spin_lock(&zhdr->pool->lock);
	564	+ struct z3fold_pool *pool = zhdr_to_pool(zhdr);
	565	+
	566	+ spin_lock(&pool->lock);
299	567	list_del_init(&zhdr->buddy);
300		- spin_unlock(&zhdr->pool->lock);
	568	+ spin_unlock(&pool->lock);
301	569
302	570	WARN_ON(z3fold_page_trylock(zhdr));
303	571	__release_z3fold_page(zhdr, true);
..	..	@@ -318,7 +586,7 @@
318	586	continue;
319	587	spin_unlock(&pool->stale_lock);
320	588	cancel_work_sync(&zhdr->work);
321		- free_z3fold_page(page);
	589	+ free_z3fold_page(page, false);
322	590	cond_resched();
323	591	spin_lock(&pool->stale_lock);
324	592	}
..	..	@@ -349,6 +617,45 @@
349	617	return nfree;
350	618	}
351	619
	620	+/* Add to the appropriate unbuddied list */
	621	+static inline void add_to_unbuddied(struct z3fold_pool *pool,
	622	+ struct z3fold_header *zhdr)
	623	+{
	624	+ if (zhdr->first_chunks == 0 \|\| zhdr->last_chunks == 0 \|\|
	625	+ zhdr->middle_chunks == 0) {
	626	+ struct list_head *unbuddied = get_cpu_ptr(pool->unbuddied);
	627	+
	628	+ int freechunks = num_free_chunks(zhdr);
	629	+ spin_lock(&pool->lock);
	630	+ list_add(&zhdr->buddy, &unbuddied[freechunks]);
	631	+ spin_unlock(&pool->lock);
	632	+ zhdr->cpu = smp_processor_id();
	633	+ put_cpu_ptr(pool->unbuddied);
	634	+ }
	635	+}
	636	+
	637	+static inline enum buddy get_free_buddy(struct z3fold_header *zhdr, int chunks)
	638	+{
	639	+ enum buddy bud = HEADLESS;
	640	+
	641	+ if (zhdr->middle_chunks) {
	642	+ if (!zhdr->first_chunks &&
	643	+ chunks <= zhdr->start_middle - ZHDR_CHUNKS)
	644	+ bud = FIRST;
	645	+ else if (!zhdr->last_chunks)
	646	+ bud = LAST;
	647	+ } else {
	648	+ if (!zhdr->first_chunks)
	649	+ bud = FIRST;
	650	+ else if (!zhdr->last_chunks)
	651	+ bud = LAST;
	652	+ else
	653	+ bud = MIDDLE;
	654	+ }
	655	+
	656	+ return bud;
	657	+}
	658	+
352	659	static inline void mchunk_memmove(struct z3fold_header zhdr,
353	660	unsigned short dst_chunk)
354	661	{
..	..	@@ -356,6 +663,108 @@
356	663	return memmove(beg + (dst_chunk << CHUNK_SHIFT),
357	664	beg + (zhdr->start_middle << CHUNK_SHIFT),
358	665	zhdr->middle_chunks << CHUNK_SHIFT);
	666	+}
	667	+
	668	+static inline bool buddy_single(struct z3fold_header *zhdr)
	669	+{
	670	+ return !((zhdr->first_chunks && zhdr->middle_chunks) \|\|
	671	+ (zhdr->first_chunks && zhdr->last_chunks) \|\|
	672	+ (zhdr->middle_chunks && zhdr->last_chunks));
	673	+}
	674	+
	675	+static struct z3fold_header compact_single_buddy(struct z3fold_header zhdr)
	676	+{
	677	+ struct z3fold_pool *pool = zhdr_to_pool(zhdr);
	678	+ void *p = zhdr;
	679	+ unsigned long old_handle = 0;
	680	+ size_t sz = 0;
	681	+ struct z3fold_header *new_zhdr = NULL;
	682	+ int first_idx = __idx(zhdr, FIRST);
	683	+ int middle_idx = __idx(zhdr, MIDDLE);
	684	+ int last_idx = __idx(zhdr, LAST);
	685	+ unsigned short *moved_chunks = NULL;
	686	+
	687	+ /*
	688	+ * No need to protect slots here -- all the slots are "local" and
	689	+ * the page lock is already taken
	690	+ */
	691	+ if (zhdr->first_chunks && zhdr->slots->slot[first_idx]) {
	692	+ p += ZHDR_SIZE_ALIGNED;
	693	+ sz = zhdr->first_chunks << CHUNK_SHIFT;
	694	+ old_handle = (unsigned long)&zhdr->slots->slot[first_idx];
	695	+ moved_chunks = &zhdr->first_chunks;
	696	+ } else if (zhdr->middle_chunks && zhdr->slots->slot[middle_idx]) {
	697	+ p += zhdr->start_middle << CHUNK_SHIFT;
	698	+ sz = zhdr->middle_chunks << CHUNK_SHIFT;
	699	+ old_handle = (unsigned long)&zhdr->slots->slot[middle_idx];
	700	+ moved_chunks = &zhdr->middle_chunks;
	701	+ } else if (zhdr->last_chunks && zhdr->slots->slot[last_idx]) {
	702	+ p += PAGE_SIZE - (zhdr->last_chunks << CHUNK_SHIFT);
	703	+ sz = zhdr->last_chunks << CHUNK_SHIFT;
	704	+ old_handle = (unsigned long)&zhdr->slots->slot[last_idx];
	705	+ moved_chunks = &zhdr->last_chunks;
	706	+ }
	707	+
	708	+ if (sz > 0) {
	709	+ enum buddy new_bud = HEADLESS;
	710	+ short chunks = size_to_chunks(sz);
	711	+ void *q;
	712	+
	713	+ new_zhdr = __z3fold_alloc(pool, sz, false);
	714	+ if (!new_zhdr)
	715	+ return NULL;
	716	+
	717	+ if (WARN_ON(new_zhdr == zhdr))
	718	+ goto out_fail;
	719	+
	720	+ new_bud = get_free_buddy(new_zhdr, chunks);
	721	+ q = new_zhdr;
	722	+ switch (new_bud) {
	723	+ case FIRST:
	724	+ new_zhdr->first_chunks = chunks;
	725	+ q += ZHDR_SIZE_ALIGNED;
	726	+ break;
	727	+ case MIDDLE:
	728	+ new_zhdr->middle_chunks = chunks;
	729	+ new_zhdr->start_middle =
	730	+ new_zhdr->first_chunks + ZHDR_CHUNKS;
	731	+ q += new_zhdr->start_middle << CHUNK_SHIFT;
	732	+ break;
	733	+ case LAST:
	734	+ new_zhdr->last_chunks = chunks;
	735	+ q += PAGE_SIZE - (new_zhdr->last_chunks << CHUNK_SHIFT);
	736	+ break;
	737	+ default:
	738	+ goto out_fail;
	739	+ }
	740	+ new_zhdr->foreign_handles++;
	741	+ memcpy(q, p, sz);
	742	+ write_lock(&zhdr->slots->lock);
	743	+ (unsigned long )old_handle = (unsigned long)new_zhdr +
	744	+ __idx(new_zhdr, new_bud);
	745	+ if (new_bud == LAST)
	746	+ (unsigned long )old_handle \|=
	747	+ (new_zhdr->last_chunks << BUDDY_SHIFT);
	748	+ write_unlock(&zhdr->slots->lock);
	749	+ add_to_unbuddied(pool, new_zhdr);
	750	+ z3fold_page_unlock(new_zhdr);
	751	+
	752	+ *moved_chunks = 0;
	753	+ }
	754	+
	755	+ return new_zhdr;
	756	+
	757	+out_fail:
	758	+ if (new_zhdr) {
	759	+ if (kref_put(&new_zhdr->refcount, release_z3fold_page_locked))
	760	+ atomic64_dec(&pool->pages_nr);
	761	+ else {
	762	+ add_to_unbuddied(pool, new_zhdr);
	763	+ z3fold_page_unlock(new_zhdr);
	764	+ }
	765	+ }
	766	+ return NULL;
	767	+
359	768	}
360	769
361	770	#define BIG_CHUNK_GAP 3
..	..	@@ -366,6 +775,9 @@
366	775
367	776	if (test_bit(MIDDLE_CHUNK_MAPPED, &page->private))
368	777	return 0; /* can't move middle chunk, it's used */
	778	+
	779	+ if (unlikely(PageIsolated(page)))
	780	+ return 0;
369	781
370	782	if (zhdr->middle_chunks == 0)
371	783	return 0; /* nothing to compact */
..	..	@@ -406,10 +818,8 @@
406	818
407	819	static void do_compact_page(struct z3fold_header *zhdr, bool locked)
408	820	{
409		- struct z3fold_pool *pool = zhdr->pool;
	821	+ struct z3fold_pool *pool = zhdr_to_pool(zhdr);
410	822	struct page *page;
411		- struct list_head *unbuddied;
412		- int fchunks;
413	823
414	824	page = virt_to_page(zhdr);
415	825	if (locked)
..	..	@@ -429,19 +839,26 @@
429	839	return;
430	840	}
431	841
432		- z3fold_compact_page(zhdr);
433		- unbuddied = get_cpu_ptr(pool->unbuddied);
434		- fchunks = num_free_chunks(zhdr);
435		- if (fchunks < NCHUNKS &&
436		- (!zhdr->first_chunks \|\| !zhdr->middle_chunks \|\|
437		- !zhdr->last_chunks)) {
438		- /* the page's not completely free and it's unbuddied */
439		- spin_lock(&pool->lock);
440		- list_add(&zhdr->buddy, &unbuddied[fchunks]);
441		- spin_unlock(&pool->lock);
442		- zhdr->cpu = smp_processor_id();
	842	+ if (test_bit(PAGE_STALE, &page->private) \|\|
	843	+ test_and_set_bit(PAGE_CLAIMED, &page->private)) {
	844	+ z3fold_page_unlock(zhdr);
	845	+ return;
443	846	}
444		- put_cpu_ptr(pool->unbuddied);
	847	+
	848	+ if (!zhdr->foreign_handles && buddy_single(zhdr) &&
	849	+ zhdr->mapped_count == 0 && compact_single_buddy(zhdr)) {
	850	+ if (kref_put(&zhdr->refcount, release_z3fold_page_locked))
	851	+ atomic64_dec(&pool->pages_nr);
	852	+ else {
	853	+ clear_bit(PAGE_CLAIMED, &page->private);
	854	+ z3fold_page_unlock(zhdr);
	855	+ }
	856	+ return;
	857	+ }
	858	+
	859	+ z3fold_compact_page(zhdr);
	860	+ add_to_unbuddied(pool, zhdr);
	861	+ clear_bit(PAGE_CLAIMED, &page->private);
445	862	z3fold_page_unlock(zhdr);
446	863	}
447	864
..	..	@@ -453,6 +870,108 @@
453	870	do_compact_page(zhdr, false);
454	871	}
455	872
	873	+/* returns _locked_ z3fold page header or NULL */
	874	+static inline struct z3fold_header __z3fold_alloc(struct z3fold_pool pool,
	875	+ size_t size, bool can_sleep)
	876	+{
	877	+ struct z3fold_header *zhdr = NULL;
	878	+ struct page *page;
	879	+ struct list_head *unbuddied;
	880	+ int chunks = size_to_chunks(size), i;
	881	+
	882	+lookup:
	883	+ /* First, try to find an unbuddied z3fold page. */
	884	+ unbuddied = get_cpu_ptr(pool->unbuddied);
	885	+ for_each_unbuddied_list(i, chunks) {
	886	+ struct list_head *l = &unbuddied[i];
	887	+
	888	+ zhdr = list_first_entry_or_null(READ_ONCE(l),
	889	+ struct z3fold_header, buddy);
	890	+
	891	+ if (!zhdr)
	892	+ continue;
	893	+
	894	+ /* Re-check under lock. */
	895	+ spin_lock(&pool->lock);
	896	+ l = &unbuddied[i];
	897	+ if (unlikely(zhdr != list_first_entry(READ_ONCE(l),
	898	+ struct z3fold_header, buddy)) \|\|
	899	+ !z3fold_page_trylock(zhdr)) {
	900	+ spin_unlock(&pool->lock);
	901	+ zhdr = NULL;
	902	+ put_cpu_ptr(pool->unbuddied);
	903	+ if (can_sleep)
	904	+ cond_resched();
	905	+ goto lookup;
	906	+ }
	907	+ list_del_init(&zhdr->buddy);
	908	+ zhdr->cpu = -1;
	909	+ spin_unlock(&pool->lock);
	910	+
	911	+ page = virt_to_page(zhdr);
	912	+ if (test_bit(NEEDS_COMPACTING, &page->private) \|\|
	913	+ test_bit(PAGE_CLAIMED, &page->private)) {
	914	+ z3fold_page_unlock(zhdr);
	915	+ zhdr = NULL;
	916	+ put_cpu_ptr(pool->unbuddied);
	917	+ if (can_sleep)
	918	+ cond_resched();
	919	+ goto lookup;
	920	+ }
	921	+
	922	+ /*
	923	+ * this page could not be removed from its unbuddied
	924	+ * list while pool lock was held, and then we've taken
	925	+ * page lock so kref_put could not be called before
	926	+ * we got here, so it's safe to just call kref_get()
	927	+ */
	928	+ kref_get(&zhdr->refcount);
	929	+ break;
	930	+ }
	931	+ put_cpu_ptr(pool->unbuddied);
	932	+
	933	+ if (!zhdr) {
	934	+ int cpu;
	935	+
	936	+ /* look for _exact_ match on other cpus' lists */
	937	+ for_each_online_cpu(cpu) {
	938	+ struct list_head *l;
	939	+
	940	+ unbuddied = per_cpu_ptr(pool->unbuddied, cpu);
	941	+ spin_lock(&pool->lock);
	942	+ l = &unbuddied[chunks];
	943	+
	944	+ zhdr = list_first_entry_or_null(READ_ONCE(l),
	945	+ struct z3fold_header, buddy);
	946	+
	947	+ if (!zhdr \|\| !z3fold_page_trylock(zhdr)) {
	948	+ spin_unlock(&pool->lock);
	949	+ zhdr = NULL;
	950	+ continue;
	951	+ }
	952	+ list_del_init(&zhdr->buddy);
	953	+ zhdr->cpu = -1;
	954	+ spin_unlock(&pool->lock);
	955	+
	956	+ page = virt_to_page(zhdr);
	957	+ if (test_bit(NEEDS_COMPACTING, &page->private) \|\|
	958	+ test_bit(PAGE_CLAIMED, &page->private)) {
	959	+ z3fold_page_unlock(zhdr);
	960	+ zhdr = NULL;
	961	+ if (can_sleep)
	962	+ cond_resched();
	963	+ continue;
	964	+ }
	965	+ kref_get(&zhdr->refcount);
	966	+ break;
	967	+ }
	968	+ }
	969	+
	970	+ if (zhdr && !zhdr->slots)
	971	+ zhdr->slots = alloc_slots(pool,
	972	+ can_sleep ? GFP_NOIO : GFP_ATOMIC);
	973	+ return zhdr;
	974	+}
456	975
457	976	/*
458	977	* API Functions
..	..	@@ -476,6 +995,11 @@
476	995	pool = kzalloc(sizeof(struct z3fold_pool), gfp);
477	996	if (!pool)
478	997	goto out;
	998	+ pool->c_handle = kmem_cache_create("z3fold_handle",
	999	+ sizeof(struct z3fold_buddy_slots),
	1000	+ SLOTS_ALIGN, 0, NULL);
	1001	+ if (!pool->c_handle)
	1002	+ goto out_c;
479	1003	spin_lock_init(&pool->lock);
480	1004	spin_lock_init(&pool->stale_lock);
481	1005	pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2);
..	..	@@ -497,15 +1021,21 @@
497	1021	pool->release_wq = create_singlethread_workqueue(pool->name);
498	1022	if (!pool->release_wq)
499	1023	goto out_wq;
	1024	+ if (z3fold_register_migration(pool))
	1025	+ goto out_rwq;
500	1026	INIT_WORK(&pool->work, free_pages_work);
501	1027	pool->ops = ops;
502	1028	return pool;
503	1029
	1030	+out_rwq:
	1031	+ destroy_workqueue(pool->release_wq);
504	1032	out_wq:
505	1033	destroy_workqueue(pool->compact_wq);
506	1034	out_unbuddied:
507	1035	free_percpu(pool->unbuddied);
508	1036	out_pool:
	1037	+ kmem_cache_destroy(pool->c_handle);
	1038	+out_c:
509	1039	kfree(pool);
510	1040	out:
511	1041	return NULL;
..	..	@@ -519,8 +1049,21 @@
519	1049	*/
520	1050	static void z3fold_destroy_pool(struct z3fold_pool *pool)
521	1051	{
522		- destroy_workqueue(pool->release_wq);
	1052	+ kmem_cache_destroy(pool->c_handle);
	1053	+
	1054	+ /*
	1055	+ * We need to destroy pool->compact_wq before pool->release_wq,
	1056	+ * as any pending work on pool->compact_wq will call
	1057	+ * queue_work(pool->release_wq, &pool->work).
	1058	+ *
	1059	+ * There are still outstanding pages until both workqueues are drained,
	1060	+ * so we cannot unregister migration until then.
	1061	+ */
	1062	+
523	1063	destroy_workqueue(pool->compact_wq);
	1064	+ destroy_workqueue(pool->release_wq);
	1065	+ z3fold_unregister_migration(pool);
	1066	+ free_percpu(pool->unbuddied);
524	1067	kfree(pool);
525	1068	}
526	1069
..	..	@@ -546,13 +1089,13 @@
546	1089	static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp,
547	1090	unsigned long *handle)
548	1091	{
549		- int chunks = 0, i, freechunks;
	1092	+ int chunks = size_to_chunks(size);
550	1093	struct z3fold_header *zhdr = NULL;
551	1094	struct page *page = NULL;
552	1095	enum buddy bud;
553	1096	bool can_sleep = gfpflags_allow_blocking(gfp);
554	1097
555		- if (!size \|\| (gfp & __GFP_HIGHMEM))
	1098	+ if (!size)
556	1099	return -EINVAL;
557	1100
558	1101	if (size > PAGE_SIZE)
..	..	@@ -561,68 +1104,11 @@
561	1104	if (size > PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE)
562	1105	bud = HEADLESS;
563	1106	else {
564		- struct list_head *unbuddied;
565		- chunks = size_to_chunks(size);
566		-
567		-lookup:
568		- /* First, try to find an unbuddied z3fold page. */
569		- unbuddied = get_cpu_ptr(pool->unbuddied);
570		- for_each_unbuddied_list(i, chunks) {
571		- struct list_head *l = &unbuddied[i];
572		-
573		- zhdr = list_first_entry_or_null(READ_ONCE(l),
574		- struct z3fold_header, buddy);
575		-
576		- if (!zhdr)
577		- continue;
578		-
579		- /* Re-check under lock. */
580		- spin_lock(&pool->lock);
581		- l = &unbuddied[i];
582		- if (unlikely(zhdr != list_first_entry(READ_ONCE(l),
583		- struct z3fold_header, buddy)) \|\|
584		- !z3fold_page_trylock(zhdr)) {
585		- spin_unlock(&pool->lock);
586		- put_cpu_ptr(pool->unbuddied);
587		- goto lookup;
588		- }
589		- list_del_init(&zhdr->buddy);
590		- zhdr->cpu = -1;
591		- spin_unlock(&pool->lock);
592		-
593		- page = virt_to_page(zhdr);
594		- if (test_bit(NEEDS_COMPACTING, &page->private)) {
595		- z3fold_page_unlock(zhdr);
596		- zhdr = NULL;
597		- put_cpu_ptr(pool->unbuddied);
598		- if (can_sleep)
599		- cond_resched();
600		- goto lookup;
601		- }
602		-
603		- /*
604		- * this page could not be removed from its unbuddied
605		- * list while pool lock was held, and then we've taken
606		- * page lock so kref_put could not be called before
607		- * we got here, so it's safe to just call kref_get()
608		- */
609		- kref_get(&zhdr->refcount);
610		- break;
611		- }
612		- put_cpu_ptr(pool->unbuddied);
613		-
	1107	+retry:
	1108	+ zhdr = __z3fold_alloc(pool, size, can_sleep);
614	1109	if (zhdr) {
615		- if (zhdr->first_chunks == 0) {
616		- if (zhdr->middle_chunks != 0 &&
617		- chunks >= zhdr->start_middle)
618		- bud = LAST;
619		- else
620		- bud = FIRST;
621		- } else if (zhdr->last_chunks == 0)
622		- bud = LAST;
623		- else if (zhdr->middle_chunks == 0)
624		- bud = MIDDLE;
625		- else {
	1110	+ bud = get_free_buddy(zhdr, chunks);
	1111	+ if (bud == HEADLESS) {
626	1112	if (kref_put(&zhdr->refcount,
627	1113	release_z3fold_page_locked))
628	1114	atomic64_dec(&pool->pages_nr);
..	..	@@ -630,8 +1116,9 @@
630	1116	z3fold_page_unlock(zhdr);
631	1117	pr_err("No free chunks in unbuddied\n");
632	1118	WARN_ON(1);
633		- goto lookup;
	1119	+ goto retry;
634	1120	}
	1121	+ page = virt_to_page(zhdr);
635	1122	goto found;
636	1123	}
637	1124	bud = FIRST;
..	..	@@ -662,12 +1149,26 @@
662	1149	if (!page)
663	1150	return -ENOMEM;
664	1151
	1152	+ zhdr = init_z3fold_page(page, bud == HEADLESS, pool, gfp);
	1153	+ if (!zhdr) {
	1154	+ __free_page(page);
	1155	+ return -ENOMEM;
	1156	+ }
665	1157	atomic64_inc(&pool->pages_nr);
666		- zhdr = init_z3fold_page(page, pool);
667	1158
668	1159	if (bud == HEADLESS) {
669	1160	set_bit(PAGE_HEADLESS, &page->private);
670	1161	goto headless;
	1162	+ }
	1163	+ if (can_sleep) {
	1164	+ lock_page(page);
	1165	+ __SetPageMovable(page, pool->inode->i_mapping);
	1166	+ unlock_page(page);
	1167	+ } else {
	1168	+ if (trylock_page(page)) {
	1169	+ __SetPageMovable(page, pool->inode->i_mapping);
	1170	+ unlock_page(page);
	1171	+ }
671	1172	}
672	1173	z3fold_page_lock(zhdr);
673	1174
..	..	@@ -680,19 +1181,7 @@
680	1181	zhdr->middle_chunks = chunks;
681	1182	zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS;
682	1183	}
683		-
684		- if (zhdr->first_chunks == 0 \|\| zhdr->last_chunks == 0 \|\|
685		- zhdr->middle_chunks == 0) {
686		- struct list_head *unbuddied = get_cpu_ptr(pool->unbuddied);
687		-
688		- /* Add to unbuddied list */
689		- freechunks = num_free_chunks(zhdr);
690		- spin_lock(&pool->lock);
691		- list_add(&zhdr->buddy, &unbuddied[freechunks]);
692		- spin_unlock(&pool->lock);
693		- zhdr->cpu = smp_processor_id();
694		- put_cpu_ptr(pool->unbuddied);
695		- }
	1184	+ add_to_unbuddied(pool, zhdr);
696	1185
697	1186	headless:
698	1187	spin_lock(&pool->lock);
..	..	@@ -725,9 +1214,11 @@
725	1214	struct z3fold_header *zhdr;
726	1215	struct page *page;
727	1216	enum buddy bud;
	1217	+ bool page_claimed;
728	1218
729		- zhdr = handle_to_z3fold_header(handle);
	1219	+ zhdr = get_z3fold_header(handle);
730	1220	page = virt_to_page(zhdr);
	1221	+ page_claimed = test_and_set_bit(PAGE_CLAIMED, &page->private);
731	1222
732	1223	if (test_bit(PAGE_HEADLESS, &page->private)) {
733	1224	/* if a headless page is under reclaim, just leave.
..	..	@@ -735,18 +1226,18 @@
735	1226	* has not been set before, we release this page
736	1227	* immediately so we don't care about its value any more.
737	1228	*/
738		- if (!test_and_set_bit(PAGE_CLAIMED, &page->private)) {
	1229	+ if (!page_claimed) {
739	1230	spin_lock(&pool->lock);
740	1231	list_del(&page->lru);
741	1232	spin_unlock(&pool->lock);
742		- free_z3fold_page(page);
	1233	+ put_z3fold_header(zhdr);
	1234	+ free_z3fold_page(page, true);
743	1235	atomic64_dec(&pool->pages_nr);
744	1236	}
745	1237	return;
746	1238	}
747	1239
748	1240	/* Non-headless case */
749		- z3fold_page_lock(zhdr);
750	1241	bud = handle_to_buddy(handle);
751	1242
752	1243	switch (bud) {
..	..	@@ -762,20 +1253,24 @@
762	1253	default:
763	1254	pr_err("%s: unknown bud %d\n", __func__, bud);
764	1255	WARN_ON(1);
765		- z3fold_page_unlock(zhdr);
	1256	+ put_z3fold_header(zhdr);
766	1257	return;
767	1258	}
768	1259
	1260	+ if (!page_claimed)
	1261	+ free_handle(handle, zhdr);
769	1262	if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) {
770	1263	atomic64_dec(&pool->pages_nr);
771	1264	return;
772	1265	}
773		- if (test_bit(PAGE_CLAIMED, &page->private)) {
	1266	+ if (page_claimed) {
	1267	+ /* the page has not been claimed by us */
774	1268	z3fold_page_unlock(zhdr);
775	1269	return;
776	1270	}
777	1271	if (test_and_set_bit(NEEDS_COMPACTING, &page->private)) {
778		- z3fold_page_unlock(zhdr);
	1272	+ put_z3fold_header(zhdr);
	1273	+ clear_bit(PAGE_CLAIMED, &page->private);
779	1274	return;
780	1275	}
781	1276	if (zhdr->cpu < 0 \|\| !cpu_online(zhdr->cpu)) {
..	..	@@ -784,12 +1279,14 @@
784	1279	spin_unlock(&pool->lock);
785	1280	zhdr->cpu = -1;
786	1281	kref_get(&zhdr->refcount);
	1282	+ clear_bit(PAGE_CLAIMED, &page->private);
787	1283	do_compact_page(zhdr, true);
788	1284	return;
789	1285	}
790	1286	kref_get(&zhdr->refcount);
	1287	+ clear_bit(PAGE_CLAIMED, &page->private);
791	1288	queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);
792		- z3fold_page_unlock(zhdr);
	1289	+ put_z3fold_header(zhdr);
793	1290	}
794	1291
795	1292	/**
..	..	@@ -830,11 +1327,15 @@
830	1327	*/
831	1328	static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
832	1329	{
833		- int i, ret = 0;
	1330	+ int i, ret = -1;
834	1331	struct z3fold_header *zhdr = NULL;
835	1332	struct page *page = NULL;
836	1333	struct list_head *pos;
837	1334	unsigned long first_handle = 0, middle_handle = 0, last_handle = 0;
	1335	+ struct z3fold_buddy_slots slots __attribute__((aligned(SLOTS_ALIGN)));
	1336	+
	1337	+ rwlock_init(&slots.lock);
	1338	+ slots.pool = (unsigned long)pool \| (1 << HANDLES_NOFREE);
838	1339
839	1340	spin_lock(&pool->lock);
840	1341	if (!pool->ops \|\| !pool->ops->evict \|\| retries == 0) {
..	..	@@ -849,21 +1350,50 @@
849	1350	list_for_each_prev(pos, &pool->lru) {
850	1351	page = list_entry(pos, struct page, lru);
851	1352
852		- /* this bit could have been set by free, in which case
853		- * we pass over to the next page in the pool.
854		- */
855		- if (test_and_set_bit(PAGE_CLAIMED, &page->private))
856		- continue;
857		-
858	1353	zhdr = page_address(page);
859		- if (test_bit(PAGE_HEADLESS, &page->private))
860		- break;
	1354	+ if (test_bit(PAGE_HEADLESS, &page->private)) {
	1355	+ /*
	1356	+ * For non-headless pages, we wait to do this
	1357	+ * until we have the page lock to avoid racing
	1358	+ * with __z3fold_alloc(). Headless pages don't
	1359	+ * have a lock (and __z3fold_alloc() will never
	1360	+ * see them), but we still need to test and set
	1361	+ * PAGE_CLAIMED to avoid racing with
	1362	+ * z3fold_free(), so just do it now before
	1363	+ * leaving the loop.
	1364	+ */
	1365	+ if (test_and_set_bit(PAGE_CLAIMED, &page->private))
	1366	+ continue;
861	1367
	1368	+ break;
	1369	+ }
	1370	+
	1371	+ if (kref_get_unless_zero(&zhdr->refcount) == 0) {
	1372	+ zhdr = NULL;
	1373	+ break;
	1374	+ }
862	1375	if (!z3fold_page_trylock(zhdr)) {
	1376	+ if (kref_put(&zhdr->refcount,
	1377	+ release_z3fold_page))
	1378	+ atomic64_dec(&pool->pages_nr);
863	1379	zhdr = NULL;
864	1380	continue; /* can't evict at this point */
865	1381	}
866		- kref_get(&zhdr->refcount);
	1382	+
	1383	+ /* test_and_set_bit is of course atomic, but we still
	1384	+ * need to do it under page lock, otherwise checking
	1385	+ * that bit in __z3fold_alloc wouldn't make sense
	1386	+ */
	1387	+ if (zhdr->foreign_handles \|\|
	1388	+ test_and_set_bit(PAGE_CLAIMED, &page->private)) {
	1389	+ if (kref_put(&zhdr->refcount,
	1390	+ release_z3fold_page_locked))
	1391	+ atomic64_dec(&pool->pages_nr);
	1392	+ else
	1393	+ z3fold_page_unlock(zhdr);
	1394	+ zhdr = NULL;
	1395	+ continue; /* can't evict such page */
	1396	+ }
867	1397	list_del_init(&zhdr->buddy);
868	1398	zhdr->cpu = -1;
869	1399	break;
..	..	@@ -877,19 +1407,24 @@
877	1407
878	1408	if (!test_bit(PAGE_HEADLESS, &page->private)) {
879	1409	/*
880		- * We need encode the handles before unlocking, since
881		- * we can race with free that will set
882		- * (first\|last)_chunks to 0
	1410	+ * We need encode the handles before unlocking, and
	1411	+ * use our local slots structure because z3fold_free
	1412	+ * can zero out zhdr->slots and we can't do much
	1413	+ * about that
883	1414	*/
884	1415	first_handle = 0;
885	1416	last_handle = 0;
886	1417	middle_handle = 0;
	1418	+ memset(slots.slot, 0, sizeof(slots.slot));
887	1419	if (zhdr->first_chunks)
888		- first_handle = encode_handle(zhdr, FIRST);
	1420	+ first_handle = __encode_handle(zhdr, &slots,
	1421	+ FIRST);
889	1422	if (zhdr->middle_chunks)
890		- middle_handle = encode_handle(zhdr, MIDDLE);
	1423	+ middle_handle = __encode_handle(zhdr, &slots,
	1424	+ MIDDLE);
891	1425	if (zhdr->last_chunks)
892		- last_handle = encode_handle(zhdr, LAST);
	1426	+ last_handle = __encode_handle(zhdr, &slots,
	1427	+ LAST);
893	1428	/*
894	1429	* it's safe to unlock here because we hold a
895	1430	* reference to this page
..	..	@@ -899,7 +1434,6 @@
899	1434	first_handle = encode_handle(zhdr, HEADLESS);
900	1435	last_handle = middle_handle = 0;
901	1436	}
902		-
903	1437	/* Issue the eviction callback(s) */
904	1438	if (middle_handle) {
905	1439	ret = pool->ops->evict(pool, middle_handle);
..	..	@@ -919,18 +1453,20 @@
919	1453	next:
920	1454	if (test_bit(PAGE_HEADLESS, &page->private)) {
921	1455	if (ret == 0) {
922		- free_z3fold_page(page);
	1456	+ free_z3fold_page(page, true);
923	1457	atomic64_dec(&pool->pages_nr);
924	1458	return 0;
925	1459	}
926	1460	spin_lock(&pool->lock);
927	1461	list_add(&page->lru, &pool->lru);
928	1462	spin_unlock(&pool->lock);
929		- } else {
930		- z3fold_page_lock(zhdr);
931	1463	clear_bit(PAGE_CLAIMED, &page->private);
	1464	+ } else {
	1465	+ struct z3fold_buddy_slots *slots = zhdr->slots;
	1466	+ z3fold_page_lock(zhdr);
932	1467	if (kref_put(&zhdr->refcount,
933	1468	release_z3fold_page_locked)) {
	1469	+ kmem_cache_free(pool->c_handle, slots);
934	1470	atomic64_dec(&pool->pages_nr);
935	1471	return 0;
936	1472	}
..	..	@@ -943,6 +1479,7 @@
943	1479	list_add(&page->lru, &pool->lru);
944	1480	spin_unlock(&pool->lock);
945	1481	z3fold_page_unlock(zhdr);
	1482	+ clear_bit(PAGE_CLAIMED, &page->private);
946	1483	}
947	1484
948	1485	/* We started off locked to we need to lock the pool back */
..	..	@@ -969,14 +1506,13 @@
969	1506	void *addr;
970	1507	enum buddy buddy;
971	1508
972		- zhdr = handle_to_z3fold_header(handle);
	1509	+ zhdr = get_z3fold_header(handle);
973	1510	addr = zhdr;
974	1511	page = virt_to_page(zhdr);
975	1512
976	1513	if (test_bit(PAGE_HEADLESS, &page->private))
977	1514	goto out;
978	1515
979		- z3fold_page_lock(zhdr);
980	1516	buddy = handle_to_buddy(handle);
981	1517	switch (buddy) {
982	1518	case FIRST:
..	..	@@ -996,8 +1532,10 @@
996	1532	break;
997	1533	}
998	1534
999		- z3fold_page_unlock(zhdr);
	1535	+ if (addr)
	1536	+ zhdr->mapped_count++;
1000	1537	out:
	1538	+ put_z3fold_header(zhdr);
1001	1539	return addr;
1002	1540	}
1003	1541
..	..	@@ -1012,17 +1550,17 @@
1012	1550	struct page *page;
1013	1551	enum buddy buddy;
1014	1552
1015		- zhdr = handle_to_z3fold_header(handle);
	1553	+ zhdr = get_z3fold_header(handle);
1016	1554	page = virt_to_page(zhdr);
1017	1555
1018	1556	if (test_bit(PAGE_HEADLESS, &page->private))
1019	1557	return;
1020	1558
1021		- z3fold_page_lock(zhdr);
1022	1559	buddy = handle_to_buddy(handle);
1023	1560	if (buddy == MIDDLE)
1024	1561	clear_bit(MIDDLE_CHUNK_MAPPED, &page->private);
1025		- z3fold_page_unlock(zhdr);
	1562	+ zhdr->mapped_count--;
	1563	+ put_z3fold_header(zhdr);
1026	1564	}
1027	1565
1028	1566	/**
..	..	@@ -1035,6 +1573,140 @@
1035	1573	{
1036	1574	return atomic64_read(&pool->pages_nr);
1037	1575	}
	1576	+
	1577	+static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode)
	1578	+{
	1579	+ struct z3fold_header *zhdr;
	1580	+ struct z3fold_pool *pool;
	1581	+
	1582	+ VM_BUG_ON_PAGE(!PageMovable(page), page);
	1583	+ VM_BUG_ON_PAGE(PageIsolated(page), page);
	1584	+
	1585	+ if (test_bit(PAGE_HEADLESS, &page->private))
	1586	+ return false;
	1587	+
	1588	+ zhdr = page_address(page);
	1589	+ z3fold_page_lock(zhdr);
	1590	+ if (test_bit(NEEDS_COMPACTING, &page->private) \|\|
	1591	+ test_bit(PAGE_STALE, &page->private))
	1592	+ goto out;
	1593	+
	1594	+ if (zhdr->mapped_count != 0 \|\| zhdr->foreign_handles != 0)
	1595	+ goto out;
	1596	+
	1597	+ if (test_and_set_bit(PAGE_CLAIMED, &page->private))
	1598	+ goto out;
	1599	+ pool = zhdr_to_pool(zhdr);
	1600	+ spin_lock(&pool->lock);
	1601	+ if (!list_empty(&zhdr->buddy))
	1602	+ list_del_init(&zhdr->buddy);
	1603	+ if (!list_empty(&page->lru))
	1604	+ list_del_init(&page->lru);
	1605	+ spin_unlock(&pool->lock);
	1606	+
	1607	+ kref_get(&zhdr->refcount);
	1608	+ z3fold_page_unlock(zhdr);
	1609	+ return true;
	1610	+
	1611	+out:
	1612	+ z3fold_page_unlock(zhdr);
	1613	+ return false;
	1614	+}
	1615	+
	1616	+static int z3fold_page_migrate(struct address_space mapping, struct page newpage,
	1617	+ struct page *page, enum migrate_mode mode)
	1618	+{
	1619	+ struct z3fold_header zhdr, new_zhdr;
	1620	+ struct z3fold_pool *pool;
	1621	+ struct address_space *new_mapping;
	1622	+
	1623	+ VM_BUG_ON_PAGE(!PageMovable(page), page);
	1624	+ VM_BUG_ON_PAGE(!PageIsolated(page), page);
	1625	+ VM_BUG_ON_PAGE(!test_bit(PAGE_CLAIMED, &page->private), page);
	1626	+ VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
	1627	+
	1628	+ zhdr = page_address(page);
	1629	+ pool = zhdr_to_pool(zhdr);
	1630	+
	1631	+ if (!z3fold_page_trylock(zhdr))
	1632	+ return -EAGAIN;
	1633	+ if (zhdr->mapped_count != 0 \|\| zhdr->foreign_handles != 0) {
	1634	+ z3fold_page_unlock(zhdr);
	1635	+ clear_bit(PAGE_CLAIMED, &page->private);
	1636	+ return -EBUSY;
	1637	+ }
	1638	+ if (work_pending(&zhdr->work)) {
	1639	+ z3fold_page_unlock(zhdr);
	1640	+ return -EAGAIN;
	1641	+ }
	1642	+ new_zhdr = page_address(newpage);
	1643	+ memcpy(new_zhdr, zhdr, PAGE_SIZE);
	1644	+ newpage->private = page->private;
	1645	+ page->private = 0;
	1646	+ z3fold_page_unlock(zhdr);
	1647	+ spin_lock_init(&new_zhdr->page_lock);
	1648	+ INIT_WORK(&new_zhdr->work, compact_page_work);
	1649	+ /*
	1650	+ * z3fold_page_isolate() ensures that new_zhdr->buddy is empty,
	1651	+ * so we only have to reinitialize it.
	1652	+ */
	1653	+ INIT_LIST_HEAD(&new_zhdr->buddy);
	1654	+ new_mapping = page_mapping(page);
	1655	+ __ClearPageMovable(page);
	1656	+ ClearPagePrivate(page);
	1657	+
	1658	+ get_page(newpage);
	1659	+ z3fold_page_lock(new_zhdr);
	1660	+ if (new_zhdr->first_chunks)
	1661	+ encode_handle(new_zhdr, FIRST);
	1662	+ if (new_zhdr->last_chunks)
	1663	+ encode_handle(new_zhdr, LAST);
	1664	+ if (new_zhdr->middle_chunks)
	1665	+ encode_handle(new_zhdr, MIDDLE);
	1666	+ set_bit(NEEDS_COMPACTING, &newpage->private);
	1667	+ new_zhdr->cpu = smp_processor_id();
	1668	+ spin_lock(&pool->lock);
	1669	+ list_add(&newpage->lru, &pool->lru);
	1670	+ spin_unlock(&pool->lock);
	1671	+ __SetPageMovable(newpage, new_mapping);
	1672	+ z3fold_page_unlock(new_zhdr);
	1673	+
	1674	+ queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work);
	1675	+
	1676	+ page_mapcount_reset(page);
	1677	+ clear_bit(PAGE_CLAIMED, &page->private);
	1678	+ put_page(page);
	1679	+ return 0;
	1680	+}
	1681	+
	1682	+static void z3fold_page_putback(struct page *page)
	1683	+{
	1684	+ struct z3fold_header *zhdr;
	1685	+ struct z3fold_pool *pool;
	1686	+
	1687	+ zhdr = page_address(page);
	1688	+ pool = zhdr_to_pool(zhdr);
	1689	+
	1690	+ z3fold_page_lock(zhdr);
	1691	+ if (!list_empty(&zhdr->buddy))
	1692	+ list_del_init(&zhdr->buddy);
	1693	+ INIT_LIST_HEAD(&page->lru);
	1694	+ if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) {
	1695	+ atomic64_dec(&pool->pages_nr);
	1696	+ return;
	1697	+ }
	1698	+ spin_lock(&pool->lock);
	1699	+ list_add(&page->lru, &pool->lru);
	1700	+ spin_unlock(&pool->lock);
	1701	+ clear_bit(PAGE_CLAIMED, &page->private);
	1702	+ z3fold_page_unlock(zhdr);
	1703	+}
	1704	+
	1705	+static const struct address_space_operations z3fold_aops = {
	1706	+ .isolate_page = z3fold_page_isolate,
	1707	+ .migratepage = z3fold_page_migrate,
	1708	+ .putback_page = z3fold_page_putback,
	1709	+};
1038	1710
1039	1711	/*****************
1040	1712	* zpool
..	..	@@ -1133,8 +1805,14 @@
1133	1805
1134	1806	static int __init init_z3fold(void)
1135	1807	{
	1808	+ int ret;
	1809	+
1136	1810	/* Make sure the z3fold header is not larger than the page size */
1137	1811	BUILD_BUG_ON(ZHDR_SIZE_ALIGNED > PAGE_SIZE);
	1812	+ ret = z3fold_mount();
	1813	+ if (ret)
	1814	+ return ret;
	1815	+
1138	1816	zpool_register_driver(&z3fold_zpool_driver);
1139	1817
1140	1818	return 0;
..	..	@@ -1142,6 +1820,7 @@
1142	1820
1143	1821	static void __exit exit_z3fold(void)
1144	1822	{
	1823	+ z3fold_unmount();
1145	1824	zpool_unregister_driver(&z3fold_zpool_driver);
1146	1825	}
1147	1826