~hc/RK356X_SDK_RELEASE.git

..	..	@@ -14,16 +14,17 @@
14	14	#include <linux/pagevec.h>
15	15	#include <linux/blkdev.h>
16	16	#include <linux/bio.h>
	17	+#include <linux/blk-crypto.h>
17	18	#include <linux/swap.h>
18	19	#include <linux/prefetch.h>
19	20	#include <linux/uio.h>
20	21	#include <linux/cleancache.h>
21	22	#include <linux/sched/signal.h>
	23	+#include <linux/fiemap.h>
22	24
23	25	#include "f2fs.h"
24	26	#include "node.h"
25	27	#include "segment.h"
26		-#include "trace.h"
27	28	#include <trace/events/f2fs.h>
28	29	#include <trace/events/android_fs.h>
29	30
..	..	@@ -49,27 +50,6 @@
49	50	bioset_exit(&f2fs_bioset);
50	51	}
51	52
52		-static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask,
53		- unsigned int nr_iovecs)
54		-{
55		- return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset);
56		-}
57		-
58		-struct bio f2fs_bio_alloc(struct f2fs_sb_info sbi, int npages, bool noio)
59		-{
60		- if (noio) {
61		- /* No failure on bio allocation */
62		- return __f2fs_bio_alloc(GFP_NOIO, npages);
63		- }
64		-
65		- if (time_to_inject(sbi, FAULT_ALLOC_BIO)) {
66		- f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO);
67		- return NULL;
68		- }
69		-
70		- return __f2fs_bio_alloc(GFP_KERNEL, npages);
71		-}
72		-
73	53	static bool __is_cp_guaranteed(struct page *page)
74	54	{
75	55	struct address_space *mapping = page->mapping;
..	..	@@ -79,18 +59,19 @@
79	59	if (!mapping)
80	60	return false;
81	61
82		- if (f2fs_is_compressed_page(page))
83		- return false;
84		-
85	62	inode = mapping->host;
86	63	sbi = F2FS_I_SB(inode);
87	64
88	65	if (inode->i_ino == F2FS_META_INO(sbi) \|\|
89		- inode->i_ino == F2FS_NODE_INO(sbi) \|\|
90		- S_ISDIR(inode->i_mode) \|\|
91		- (S_ISREG(inode->i_mode) &&
	66	+ inode->i_ino == F2FS_NODE_INO(sbi) \|\|
	67	+ S_ISDIR(inode->i_mode))
	68	+ return true;
	69	+
	70	+ if (f2fs_is_compressed_page(page))
	71	+ return false;
	72	+ if ((S_ISREG(inode->i_mode) &&
92	73	(f2fs_is_atomic_file(inode) \|\| IS_NOQUOTA(inode))) \|\|
93		- is_cold_data(page))
	74	+ page_private_gcing(page))
94	75	return true;
95	76	return false;
96	77	}
..	..	@@ -114,10 +95,21 @@
114	95
115	96	/* postprocessing steps for read bios */
116	97	enum bio_post_read_step {
117		- STEP_DECRYPT,
118		- STEP_DECOMPRESS_NOWQ, /* handle normal cluster data inplace */
119		- STEP_DECOMPRESS, /* handle compressed cluster data in workqueue */
120		- STEP_VERITY,
	98	+#ifdef CONFIG_FS_ENCRYPTION
	99	+ STEP_DECRYPT = 1 << 0,
	100	+#else
	101	+ STEP_DECRYPT = 0, /* compile out the decryption-related code */
	102	+#endif
	103	+#ifdef CONFIG_F2FS_FS_COMPRESSION
	104	+ STEP_DECOMPRESS = 1 << 1,
	105	+#else
	106	+ STEP_DECOMPRESS = 0, /* compile out the decompression-related code */
	107	+#endif
	108	+#ifdef CONFIG_FS_VERITY
	109	+ STEP_VERITY = 1 << 2,
	110	+#else
	111	+ STEP_VERITY = 0, /* compile out the verity-related code */
	112	+#endif
121	113	};
122	114
123	115	struct bio_post_read_ctx {
..	..	@@ -127,25 +119,27 @@
127	119	unsigned int enabled_steps;
128	120	};
129	121
130		-static void __read_end_io(struct bio *bio, bool compr, bool verity)
	122	+static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
131	123	{
132		- struct page *page;
133	124	struct bio_vec *bv;
134		- int i;
	125	+ struct bvec_iter_all iter_all;
135	126
136		- bio_for_each_segment_all(bv, bio, i) {
137		- page = bv->bv_page;
	127	+ /*
	128	+ * Update and unlock the bio's pagecache pages, and put the
	129	+ * decompression context for any compressed pages.
	130	+ */
	131	+ bio_for_each_segment_all(bv, bio, iter_all) {
	132	+ struct page *page = bv->bv_page;
138	133
139		-#ifdef CONFIG_F2FS_FS_COMPRESSION
140		- if (compr && f2fs_is_compressed_page(page)) {
141		- f2fs_decompress_pages(bio, page, verity);
	134	+ if (f2fs_is_compressed_page(page)) {
	135	+ if (bio->bi_status)
	136	+ f2fs_end_read_compressed_page(page, true, 0,
	137	+ in_task);
	138	+ f2fs_put_page_dic(page, in_task);
142	139	continue;
143	140	}
144		- if (verity)
145		- continue;
146		-#endif
147	141
148		- /* PG_error was set if any post_read step failed */
	142	+ /* PG_error was set if decryption or verity failed. */
149	143	if (bio->bi_status \|\| PageError(page)) {
150	144	ClearPageUptodate(page);
151	145	/* will re-read again later */
..	..	@@ -156,106 +150,109 @@
156	150	dec_page_count(F2FS_P_SB(page), __read_io_type(page));
157	151	unlock_page(page);
158	152	}
	153	+
	154	+ if (bio->bi_private)
	155	+ mempool_free(bio->bi_private, bio_post_read_ctx_pool);
	156	+ bio_put(bio);
159	157	}
160	158
161		-static void f2fs_release_read_bio(struct bio *bio);
162		-static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity)
163		-{
164		- if (!compr)
165		- __read_end_io(bio, false, verity);
166		- f2fs_release_read_bio(bio);
167		-}
168		-
169		-static void f2fs_decompress_bio(struct bio *bio, bool verity)
170		-{
171		- __read_end_io(bio, true, verity);
172		-}
173		-
174		-static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
175		-
176		-static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx)
177		-{
178		- fscrypt_decrypt_bio(ctx->bio);
179		-}
180		-
181		-static void f2fs_decompress_work(struct bio_post_read_ctx *ctx)
182		-{
183		- f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY));
184		-}
185		-
186		-#ifdef CONFIG_F2FS_FS_COMPRESSION
187		-static void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size)
188		-{
189		- f2fs_decompress_end_io(rpages, cluster_size, false, true);
190		-}
191		-
192		-static void f2fs_verify_bio(struct bio *bio)
193		-{
194		- struct bio_vec *bv;
195		- int i;
196		-
197		- bio_for_each_segment_all(bv, bio, i) {
198		- struct page *page = bv->bv_page;
199		- struct decompress_io_ctx *dic;
200		-
201		- dic = (struct decompress_io_ctx *)page_private(page);
202		-
203		- if (dic) {
204		- if (refcount_dec_not_one(&dic->ref))
205		- continue;
206		- f2fs_verify_pages(dic->rpages,
207		- dic->cluster_size);
208		- f2fs_free_dic(dic);
209		- continue;
210		- }
211		-
212		- if (bio->bi_status \|\| PageError(page))
213		- goto clear_uptodate;
214		-
215		- if (fsverity_verify_page(page)) {
216		- SetPageUptodate(page);
217		- goto unlock;
218		- }
219		-clear_uptodate:
220		- ClearPageUptodate(page);
221		- ClearPageError(page);
222		-unlock:
223		- dec_page_count(F2FS_P_SB(page), __read_io_type(page));
224		- unlock_page(page);
225		- }
226		-}
227		-#endif
228		-
229		-static void f2fs_verity_work(struct work_struct *work)
	159	+static void f2fs_verify_bio(struct work_struct *work)
230	160	{
231	161	struct bio_post_read_ctx *ctx =
232	162	container_of(work, struct bio_post_read_ctx, work);
233	163	struct bio *bio = ctx->bio;
234		-#ifdef CONFIG_F2FS_FS_COMPRESSION
235		- unsigned int enabled_steps = ctx->enabled_steps;
236		-#endif
	164	+ bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS);
237	165
238	166	/*
239	167	* fsverity_verify_bio() may call readpages() again, and while verity
240		- * will be disabled for this, decryption may still be needed, resulting
241		- * in another bio_post_read_ctx being allocated. So to prevent
242		- * deadlocks we need to release the current ctx to the mempool first.
243		- * This assumes that verity is the last post-read step.
	168	+ * will be disabled for this, decryption and/or decompression may still
	169	+ * be needed, resulting in another bio_post_read_ctx being allocated.
	170	+ * So to prevent deadlocks we need to release the current ctx to the
	171	+ * mempool first. This assumes that verity is the last post-read step.
244	172	*/
245	173	mempool_free(ctx, bio_post_read_ctx_pool);
246	174	bio->bi_private = NULL;
247	175
248		-#ifdef CONFIG_F2FS_FS_COMPRESSION
249		- /* previous step is decompression */
250		- if (enabled_steps & (1 << STEP_DECOMPRESS)) {
251		- f2fs_verify_bio(bio);
252		- f2fs_release_read_bio(bio);
253		- return;
254		- }
255		-#endif
	176	+ /*
	177	+ * Verify the bio's pages with fs-verity. Exclude compressed pages,
	178	+ * as those were handled separately by f2fs_end_read_compressed_page().
	179	+ */
	180	+ if (may_have_compressed_pages) {
	181	+ struct bio_vec *bv;
	182	+ struct bvec_iter_all iter_all;
256	183
257		- fsverity_verify_bio(bio);
258		- __f2fs_read_end_io(bio, false, false);
	184	+ bio_for_each_segment_all(bv, bio, iter_all) {
	185	+ struct page *page = bv->bv_page;
	186	+
	187	+ if (!f2fs_is_compressed_page(page) &&
	188	+ !PageError(page) && !fsverity_verify_page(page))
	189	+ SetPageError(page);
	190	+ }
	191	+ } else {
	192	+ fsverity_verify_bio(bio);
	193	+ }
	194	+
	195	+ f2fs_finish_read_bio(bio, true);
	196	+}
	197	+
	198	+/*
	199	+ * If the bio's data needs to be verified with fs-verity, then enqueue the
	200	+ * verity work for the bio. Otherwise finish the bio now.
	201	+ *
	202	+ * Note that to avoid deadlocks, the verity work can't be done on the
	203	+ * decryption/decompression workqueue. This is because verifying the data pages
	204	+ * can involve reading verity metadata pages from the file, and these verity
	205	+ * metadata pages may be encrypted and/or compressed.
	206	+ */
	207	+static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)
	208	+{
	209	+ struct bio_post_read_ctx *ctx = bio->bi_private;
	210	+
	211	+ if (ctx && (ctx->enabled_steps & STEP_VERITY)) {
	212	+ INIT_WORK(&ctx->work, f2fs_verify_bio);
	213	+ fsverity_enqueue_verify_work(&ctx->work);
	214	+ } else {
	215	+ f2fs_finish_read_bio(bio, in_task);
	216	+ }
	217	+}
	218	+
	219	+/*
	220	+ * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last
	221	+ * remaining page was read by @ctx->bio.
	222	+ *
	223	+ * Note that a bio may span clusters (even a mix of compressed and uncompressed
	224	+ * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates
	225	+ * that the bio includes at least one compressed page. The actual decompression
	226	+ * is done on a per-cluster basis, not a per-bio basis.
	227	+ */
	228	+static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx,
	229	+ bool in_task)
	230	+{
	231	+ struct bio_vec *bv;
	232	+ struct bvec_iter_all iter_all;
	233	+ bool all_compressed = true;
	234	+ block_t blkaddr = SECTOR_TO_BLOCK(ctx->bio->bi_iter.bi_sector);
	235	+
	236	+ bio_for_each_segment_all(bv, ctx->bio, iter_all) {
	237	+ struct page *page = bv->bv_page;
	238	+
	239	+ /* PG_error was set if decryption failed. */
	240	+ if (f2fs_is_compressed_page(page))
	241	+ f2fs_end_read_compressed_page(page, PageError(page),
	242	+ blkaddr, in_task);
	243	+ else
	244	+ all_compressed = false;
	245	+
	246	+ blkaddr++;
	247	+ }
	248	+
	249	+ /*
	250	+ * Optimization: if all the bio's pages are compressed, then scheduling
	251	+ * the per-bio verity work is unnecessary, as verity will be fully
	252	+ * handled at the compression cluster level.
	253	+ */
	254	+ if (all_compressed)
	255	+ ctx->enabled_steps &= ~STEP_VERITY;
259	256	}
260	257
261	258	static void f2fs_post_read_work(struct work_struct *work)
..	..	@@ -263,107 +260,75 @@
263	260	struct bio_post_read_ctx *ctx =
264	261	container_of(work, struct bio_post_read_ctx, work);
265	262
266		- if (ctx->enabled_steps & (1 << STEP_DECRYPT))
267		- f2fs_decrypt_work(ctx);
	263	+ if (ctx->enabled_steps & STEP_DECRYPT)
	264	+ fscrypt_decrypt_bio(ctx->bio);
268	265
269		- if (ctx->enabled_steps & (1 << STEP_DECOMPRESS))
270		- f2fs_decompress_work(ctx);
	266	+ if (ctx->enabled_steps & STEP_DECOMPRESS)
	267	+ f2fs_handle_step_decompress(ctx, true);
271	268
272		- if (ctx->enabled_steps & (1 << STEP_VERITY)) {
273		- INIT_WORK(&ctx->work, f2fs_verity_work);
274		- fsverity_enqueue_verify_work(&ctx->work);
275		- return;
276		- }
277		-
278		- __f2fs_read_end_io(ctx->bio,
279		- ctx->enabled_steps & (1 << STEP_DECOMPRESS), false);
280		-}
281		-
282		-static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi,
283		- struct work_struct *work)
284		-{
285		- queue_work(sbi->post_read_wq, work);
286		-}
287		-
288		-static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
289		-{
290		- /*
291		- * We use different work queues for decryption and for verity because
292		- * verity may require reading metadata pages that need decryption, and
293		- * we shouldn't recurse to the same workqueue.
294		- */
295		-
296		- if (ctx->enabled_steps & (1 << STEP_DECRYPT) \|\|
297		- ctx->enabled_steps & (1 << STEP_DECOMPRESS)) {
298		- INIT_WORK(&ctx->work, f2fs_post_read_work);
299		- f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work);
300		- return;
301		- }
302		-
303		- if (ctx->enabled_steps & (1 << STEP_VERITY)) {
304		- INIT_WORK(&ctx->work, f2fs_verity_work);
305		- fsverity_enqueue_verify_work(&ctx->work);
306		- return;
307		- }
308		-
309		- __f2fs_read_end_io(ctx->bio, false, false);
310		-}
311		-
312		-static bool f2fs_bio_post_read_required(struct bio *bio)
313		-{
314		- return bio->bi_private;
	269	+ f2fs_verify_and_finish_bio(ctx->bio, true);
315	270	}
316	271
317	272	static void f2fs_read_end_io(struct bio *bio)
318	273	{
319		- struct page *first_page = bio->bi_io_vec[0].bv_page;
320		- struct f2fs_sb_info *sbi = F2FS_P_SB(first_page);
	274	+ struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio));
	275	+ struct bio_post_read_ctx *ctx = bio->bi_private;
	276	+ bool intask = in_task();
321	277
322	278	if (time_to_inject(sbi, FAULT_READ_IO)) {
323	279	f2fs_show_injection_info(sbi, FAULT_READ_IO);
324	280	bio->bi_status = BLK_STS_IOERR;
325	281	}
326	282
327		- if (f2fs_bio_post_read_required(bio)) {
328		- struct bio_post_read_ctx *ctx = bio->bi_private;
329		-
330		- bio_post_read_processing(ctx);
	283	+ if (bio->bi_status) {
	284	+ f2fs_finish_read_bio(bio, intask);
331	285	return;
332	286	}
333	287
334		- if (first_page != NULL &&
335		- __read_io_type(first_page) == F2FS_RD_DATA) {
336		- trace_android_fs_dataread_end(first_page->mapping->host,
337		- page_offset(first_page),
338		- bio->bi_iter.bi_size);
	288	+ if (ctx) {
	289	+ unsigned int enabled_steps = ctx->enabled_steps &
	290	+ (STEP_DECRYPT \| STEP_DECOMPRESS);
	291	+
	292	+ /*
	293	+ * If we have only decompression step between decompression and
	294	+ * decrypt, we don't need post processing for this.
	295	+ */
	296	+ if (enabled_steps == STEP_DECOMPRESS &&
	297	+ !f2fs_low_mem_mode(sbi)) {
	298	+ f2fs_handle_step_decompress(ctx, intask);
	299	+ } else if (enabled_steps) {
	300	+ INIT_WORK(&ctx->work, f2fs_post_read_work);
	301	+ queue_work(ctx->sbi->post_read_wq, &ctx->work);
	302	+ return;
	303	+ }
339	304	}
340	305
341		- __f2fs_read_end_io(bio, false, false);
	306	+ f2fs_verify_and_finish_bio(bio, intask);
342	307	}
343	308
344	309	static void f2fs_write_end_io(struct bio *bio)
345	310	{
346	311	struct f2fs_sb_info *sbi = bio->bi_private;
347	312	struct bio_vec *bvec;
348		- int i;
	313	+ struct bvec_iter_all iter_all;
349	314
350	315	if (time_to_inject(sbi, FAULT_WRITE_IO)) {
351	316	f2fs_show_injection_info(sbi, FAULT_WRITE_IO);
352	317	bio->bi_status = BLK_STS_IOERR;
353	318	}
354	319
355		- bio_for_each_segment_all(bvec, bio, i) {
	320	+ bio_for_each_segment_all(bvec, bio, iter_all) {
356	321	struct page *page = bvec->bv_page;
357	322	enum count_type type = WB_DATA_TYPE(page);
358	323
359		- if (IS_DUMMY_WRITTEN_PAGE(page)) {
360		- set_page_private(page, (unsigned long)NULL);
361		- ClearPagePrivate(page);
	324	+ if (page_private_dummy(page)) {
	325	+ clear_page_private_dummy(page);
362	326	unlock_page(page);
363	327	mempool_free(page, sbi->write_io_dummy);
364	328
365	329	if (unlikely(bio->bi_status))
366		- f2fs_stop_checkpoint(sbi, true);
	330	+ f2fs_stop_checkpoint(sbi, true,
	331	+ STOP_CP_REASON_WRITE_FAIL);
367	332	continue;
368	333	}
369	334
..	..	@@ -379,7 +344,8 @@
379	344	if (unlikely(bio->bi_status)) {
380	345	mapping_set_error(page->mapping, -EIO);
381	346	if (type == F2FS_WB_CP_DATA)
382		- f2fs_stop_checkpoint(sbi, true);
	347	+ f2fs_stop_checkpoint(sbi, true,
	348	+ STOP_CP_REASON_WRITE_FAIL);
383	349	}
384	350
385	351	f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
..	..	@@ -388,7 +354,7 @@
388	354	dec_page_count(sbi, type);
389	355	if (f2fs_in_warm_node_list(sbi, page))
390	356	f2fs_del_fsync_node_entry(sbi, page);
391		- clear_cold_data(page);
	357	+ clear_page_private_gcing(page);
392	358	end_page_writeback(page);
393	359	}
394	360	if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
..	..	@@ -449,7 +415,7 @@
449	415	struct f2fs_sb_info *sbi = fio->sbi;
450	416	struct bio *bio;
451	417
452		- bio = f2fs_bio_alloc(sbi, npages, true);
	418	+ bio = bio_alloc_bioset(GFP_NOIO, npages, &f2fs_bioset);
453	419
454	420	f2fs_target_device(sbi, fio->new_blkaddr, bio);
455	421	if (is_read_io(fio->op)) {
..	..	@@ -510,7 +476,7 @@
510	476	if (f2fs_lfs_mode(sbi) && current->plug)
511	477	blk_finish_plug(current->plug);
512	478
513		- if (F2FS_IO_ALIGNED(sbi))
	479	+ if (!F2FS_IO_ALIGNED(sbi))
514	480	goto submit_io;
515	481
516	482	start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
..	..	@@ -526,10 +492,11 @@
526	492	GFP_NOIO \| __GFP_NOFAIL);
527	493	f2fs_bug_on(sbi, !page);
528	494
529		- zero_user_segment(page, 0, PAGE_SIZE);
530		- SetPagePrivate(page);
531		- set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
532	495	lock_page(page);
	496	+
	497	+ zero_user_segment(page, 0, PAGE_SIZE);
	498	+ set_page_private_dummy(page);
	499	+
533	500	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
534	501	f2fs_bug_on(sbi, 1);
535	502	}
..	..	@@ -546,32 +513,6 @@
546	513	else
547	514	trace_f2fs_submit_write_bio(sbi->sb, type, bio);
548	515	submit_bio(bio);
549		-}
550		-
551		-static void __f2fs_submit_read_bio(struct f2fs_sb_info *sbi,
552		- struct bio *bio, enum page_type type)
553		-{
554		- if (trace_android_fs_dataread_start_enabled() && (type == DATA)) {
555		- struct page *first_page = bio->bi_io_vec[0].bv_page;
556		-
557		- if (first_page != NULL &&
558		- __read_io_type(first_page) == F2FS_RD_DATA) {
559		- char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
560		-
561		- path = android_fstrace_get_pathname(pathbuf,
562		- MAX_TRACE_PATHBUF_LEN,
563		- first_page->mapping->host);
564		-
565		- trace_android_fs_dataread_start(
566		- first_page->mapping->host,
567		- page_offset(first_page),
568		- bio->bi_iter.bi_size,
569		- current->pid,
570		- path,
571		- current->comm);
572		- }
573		- }
574		- __submit_bio(sbi, bio, type);
575	516	}
576	517
577	518	void f2fs_submit_bio(struct f2fs_sb_info *sbi,
..	..	@@ -631,7 +572,7 @@
631	572	struct page *page, nid_t ino)
632	573	{
633	574	struct bio_vec *bvec;
634		- int i;
	575	+ struct bvec_iter_all iter_all;
635	576
636	577	if (!bio)
637	578	return false;
..	..	@@ -639,7 +580,7 @@
639	580	if (!inode && !page && !ino)
640	581	return true;
641	582
642		- bio_for_each_segment_all(bvec, bio, i) {
	583	+ bio_for_each_segment_all(bvec, bio, iter_all) {
643	584	struct page *target = bvec->bv_page;
644	585
645	586	if (fscrypt_is_bounce_page(target)) {
..	..	@@ -670,7 +611,7 @@
670	611	enum page_type btype = PAGE_TYPE_OF_BIO(type);
671	612	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
672	613
673		- down_write(&io->io_rwsem);
	614	+ f2fs_down_write(&io->io_rwsem);
674	615
675	616	/* change META to META_FLUSH in the checkpoint procedure */
676	617	if (type >= META_FLUSH) {
..	..	@@ -681,7 +622,7 @@
681	622	io->fio.op_flags \|= REQ_PREFLUSH \| REQ_FUA;
682	623	}
683	624	__submit_merged_bio(io);
684		- up_write(&io->io_rwsem);
	625	+ f2fs_up_write(&io->io_rwsem);
685	626	}
686	627
687	628	static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
..	..	@@ -696,9 +637,9 @@
696	637	enum page_type btype = PAGE_TYPE_OF_BIO(type);
697	638	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
698	639
699		- down_read(&io->io_rwsem);
	640	+ f2fs_down_read(&io->io_rwsem);
700	641	ret = __has_merged_page(io->bio, inode, page, ino);
701		- up_read(&io->io_rwsem);
	642	+ f2fs_up_read(&io->io_rwsem);
702	643	}
703	644	if (ret)
704	645	__f2fs_submit_merged_write(sbi, type, temp);
..	..	@@ -744,7 +685,6 @@
744	685	return -EFSCORRUPTED;
745	686
746	687	trace_f2fs_submit_page_bio(page, fio);
747		- f2fs_trace_ios(fio, 0);
748	688
749	689	/* Allocate a new bio */
750	690	bio = __bio_alloc(fio, 1);
..	..	@@ -758,7 +698,7 @@
758	698	}
759	699
760	700	if (fio->io_wbc && !is_read_io(fio->op))
761		- wbc_account_io(fio->io_wbc, page, PAGE_SIZE);
	701	+ wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
762	702
763	703	__attach_io_flag(fio);
764	704	bio_set_op_attrs(bio, fio->op, fio->op_flags);
..	..	@@ -766,16 +706,16 @@
766	706	inc_page_count(fio->sbi, is_read_io(fio->op) ?
767	707	__read_io_type(page): WB_DATA_TYPE(fio->page));
768	708
769		- if (is_read_io(fio->op))
770		- __f2fs_submit_read_bio(fio->sbi, bio, fio->type);
771		- else
772		- __submit_bio(fio->sbi, bio, fio->type);
	709	+ __submit_bio(fio->sbi, bio, fio->type);
773	710	return 0;
774	711	}
775	712
776	713	static bool page_is_mergeable(struct f2fs_sb_info sbi, struct bio bio,
777	714	block_t last_blkaddr, block_t cur_blkaddr)
778	715	{
	716	+ if (unlikely(sbi->max_io_bytes &&
	717	+ bio->bi_iter.bi_size >= sbi->max_io_bytes))
	718	+ return false;
779	719	if (last_blkaddr + 1 != cur_blkaddr)
780	720	return false;
781	721	return __same_bdev(sbi, cur_blkaddr, bio);
..	..	@@ -823,9 +763,9 @@
823	763	if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
824	764	f2fs_bug_on(sbi, 1);
825	765
826		- down_write(&io->bio_list_lock);
	766	+ f2fs_down_write(&io->bio_list_lock);
827	767	list_add_tail(&be->list, &io->bio_list);
828		- up_write(&io->bio_list_lock);
	768	+ f2fs_up_write(&io->bio_list_lock);
829	769	}
830	770
831	771	static void del_bio_entry(struct bio_entry *be)
..	..	@@ -847,16 +787,17 @@
847	787	struct list_head *head = &io->bio_list;
848	788	struct bio_entry *be;
849	789
850		- down_write(&io->bio_list_lock);
	790	+ f2fs_down_write(&io->bio_list_lock);
851	791	list_for_each_entry(be, head, list) {
852	792	if (be->bio != *bio)
853	793	continue;
854	794
855	795	found = true;
856	796
857		- if (page_is_mergeable(sbi, bio, fio->last_block,
858		- fio->new_blkaddr) &&
859		- f2fs_crypt_mergeable_bio(*bio,
	797	+ f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
	798	+ *fio->last_block,
	799	+ fio->new_blkaddr));
	800	+ if (f2fs_crypt_mergeable_bio(*bio,
860	801	fio->page->mapping->host,
861	802	fio->page->index, fio) &&
862	803	bio_add_page(*bio, page, PAGE_SIZE, 0) ==
..	..	@@ -870,7 +811,7 @@
870	811	__submit_bio(sbi, *bio, DATA);
871	812	break;
872	813	}
873		- up_write(&io->bio_list_lock);
	814	+ f2fs_up_write(&io->bio_list_lock);
874	815	}
875	816
876	817	if (ret) {
..	..	@@ -896,7 +837,7 @@
896	837	if (list_empty(head))
897	838	continue;
898	839
899		- down_read(&io->bio_list_lock);
	840	+ f2fs_down_read(&io->bio_list_lock);
900	841	list_for_each_entry(be, head, list) {
901	842	if (target)
902	843	found = (target == be->bio);
..	..	@@ -906,14 +847,14 @@
906	847	if (found)
907	848	break;
908	849	}
909		- up_read(&io->bio_list_lock);
	850	+ f2fs_up_read(&io->bio_list_lock);
910	851
911	852	if (!found)
912	853	continue;
913	854
914	855	found = false;
915	856
916		- down_write(&io->bio_list_lock);
	857	+ f2fs_down_write(&io->bio_list_lock);
917	858	list_for_each_entry(be, head, list) {
918	859	if (target)
919	860	found = (target == be->bio);
..	..	@@ -926,7 +867,7 @@
926	867	break;
927	868	}
928	869	}
929		- up_write(&io->bio_list_lock);
	870	+ f2fs_up_write(&io->bio_list_lock);
930	871	}
931	872
932	873	if (found)
..	..	@@ -948,15 +889,16 @@
948	889	return -EFSCORRUPTED;
949	890
950	891	trace_f2fs_submit_page_bio(page, fio);
951		- f2fs_trace_ios(fio, 0);
952	892
	893	+ if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
	894	+ fio->new_blkaddr))
	895	+ f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
953	896	alloc_new:
954	897	if (!bio) {
955	898	bio = __bio_alloc(fio, BIO_MAX_PAGES);
956		- f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,
957		- fio->page->index, fio,
958		- GFP_NOIO);
959	899	__attach_io_flag(fio);
	900	+ f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,
	901	+ fio->page->index, fio, GFP_NOIO);
960	902	bio_set_op_attrs(bio, fio->op, fio->op_flags);
961	903
962	904	add_bio_entry(fio->sbi, bio, page, fio->temp);
..	..	@@ -966,7 +908,7 @@
966	908	}
967	909
968	910	if (fio->io_wbc)
969		- wbc_account_io(fio->io_wbc, page, PAGE_SIZE);
	911	+ wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
970	912
971	913	inc_page_count(fio->sbi, WB_DATA_TYPE(page));
972	914
..	..	@@ -985,7 +927,7 @@
985	927
986	928	f2fs_bug_on(sbi, is_read_io(fio->op));
987	929
988		- down_write(&io->io_rwsem);
	930	+ f2fs_down_write(&io->io_rwsem);
989	931	next:
990	932	if (fio->in_list) {
991	933	spin_lock(&io->io_lock);
..	..	@@ -1017,7 +959,7 @@
1017	959	(!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,
1018	960	fio->new_blkaddr) \|\|
1019	961	!f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host,
1020		- fio->page->index, fio)))
	962	+ bio_page->index, fio)))
1021	963	__submit_merged_bio(io);
1022	964	alloc_new:
1023	965	if (io->bio == NULL) {
..	..	@@ -1030,8 +972,7 @@
1030	972	}
1031	973	io->bio = __bio_alloc(fio, BIO_MAX_PAGES);
1032	974	f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host,
1033		- fio->page->index, fio,
1034		- GFP_NOIO);
	975	+ bio_page->index, fio, GFP_NOIO);
1035	976	io->fio = *fio;
1036	977	}
1037	978
..	..	@@ -1041,10 +982,9 @@
1041	982	}
1042	983
1043	984	if (fio->io_wbc)
1044		- wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
	985	+ wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE);
1045	986
1046	987	io->last_block_in_bio = fio->new_blkaddr;
1047		- f2fs_trace_ios(fio, 0);
1048	988
1049	989	trace_f2fs_submit_page_write(fio->page, fio);
1050	990	skip:
..	..	@@ -1054,13 +994,7 @@
1054	994	if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) \|\|
1055	995	!f2fs_is_checkpoint_ready(sbi))
1056	996	__submit_merged_bio(io);
1057		- up_write(&io->io_rwsem);
1058		-}
1059		-
1060		-static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
1061		-{
1062		- return fsverity_active(inode) &&
1063		- idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
	997	+ f2fs_up_write(&io->io_rwsem);
1064	998	}
1065	999
1066	1000	static struct bio f2fs_grab_read_bio(struct inode inode, block_t blkaddr,
..	..	@@ -1072,8 +1006,9 @@
1072	1006	struct bio_post_read_ctx *ctx;
1073	1007	unsigned int post_read_steps = 0;
1074	1008
1075		- bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES),
1076		- for_write);
	1009	+ bio = bio_alloc_bioset(for_write ? GFP_NOIO : GFP_KERNEL,
	1010	+ min_t(int, nr_pages, BIO_MAX_PAGES),
	1011	+ &f2fs_bioset);
1077	1012	if (!bio)
1078	1013	return ERR_PTR(-ENOMEM);
1079	1014
..	..	@@ -1084,13 +1019,19 @@
1084	1019	bio_set_op_attrs(bio, REQ_OP_READ, op_flag);
1085	1020
1086	1021	if (fscrypt_inode_uses_fs_layer_crypto(inode))
1087		- post_read_steps \|= 1 << STEP_DECRYPT;
1088		- if (f2fs_compressed_file(inode))
1089		- post_read_steps \|= 1 << STEP_DECOMPRESS_NOWQ;
1090		- if (f2fs_need_verity(inode, first_idx))
1091		- post_read_steps \|= 1 << STEP_VERITY;
	1022	+ post_read_steps \|= STEP_DECRYPT;
1092	1023
1093		- if (post_read_steps) {
	1024	+ if (f2fs_need_verity(inode, first_idx))
	1025	+ post_read_steps \|= STEP_VERITY;
	1026	+
	1027	+ /*
	1028	+ * STEP_DECOMPRESS is handled specially, since a compressed file might
	1029	+ * contain both compressed and uncompressed clusters. We'll allocate a
	1030	+ * bio_post_read_ctx if the file is compressed, but the caller is
	1031	+ * responsible for enabling STEP_DECOMPRESS if it's actually needed.
	1032	+ */
	1033	+
	1034	+ if (post_read_steps \|\| f2fs_compressed_file(inode)) {
1094	1035	/* Due to the mempool, this never fails. */
1095	1036	ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
1096	1037	ctx->bio = bio;
..	..	@@ -1102,21 +1043,15 @@
1102	1043	return bio;
1103	1044	}
1104	1045
1105		-static void f2fs_release_read_bio(struct bio *bio)
1106		-{
1107		- if (bio->bi_private)
1108		- mempool_free(bio->bi_private, bio_post_read_ctx_pool);
1109		- bio_put(bio);
1110		-}
1111		-
1112	1046	/* This can handle encryption stuffs */
1113	1047	static int f2fs_submit_page_read(struct inode inode, struct page page,
1114		- block_t blkaddr, bool for_write)
	1048	+ block_t blkaddr, int op_flags, bool for_write)
1115	1049	{
1116	1050	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1117	1051	struct bio *bio;
1118	1052
1119		- bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0, page->index, for_write);
	1053	+ bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags,
	1054	+ page->index, for_write);
1120	1055	if (IS_ERR(bio))
1121	1056	return PTR_ERR(bio);
1122	1057
..	..	@@ -1130,7 +1065,7 @@
1130	1065	ClearPageError(page);
1131	1066	inc_page_count(sbi, F2FS_RD_DATA);
1132	1067	f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
1133		- __f2fs_submit_read_bio(sbi, bio, DATA);
	1068	+ __submit_bio(sbi, bio, DATA);
1134	1069	return 0;
1135	1070	}
1136	1071
..	..	@@ -1166,7 +1101,7 @@
1166	1101	{
1167	1102	dn->data_blkaddr = blkaddr;
1168	1103	f2fs_set_data_blkaddr(dn);
1169		- f2fs_update_extent_cache(dn);
	1104	+ f2fs_update_read_extent_cache(dn);
1170	1105	}
1171	1106
1172	1107	/* dn->ofs_in_node will be returned with up-to-date last block pointer */
..	..	@@ -1190,6 +1125,7 @@
1190	1125
1191	1126	for (; count > 0; dn->ofs_in_node++) {
1192	1127	block_t blkaddr = f2fs_data_blkaddr(dn);
	1128	+
1193	1129	if (blkaddr == NULL_ADDR) {
1194	1130	dn->data_blkaddr = NEW_ADDR;
1195	1131	__set_data_blkaddr(dn);
..	..	@@ -1231,10 +1167,10 @@
1231	1167
1232	1168	int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
1233	1169	{
1234		- struct extent_info ei = {0,0,0};
	1170	+ struct extent_info ei = {0, };
1235	1171	struct inode *inode = dn->inode;
1236	1172
1237		- if (f2fs_lookup_extent_cache(inode, index, &ei)) {
	1173	+ if (f2fs_lookup_read_extent_cache(inode, index, &ei)) {
1238	1174	dn->data_blkaddr = ei.blk + index - ei.fofs;
1239	1175	return 0;
1240	1176	}
..	..	@@ -1248,14 +1184,14 @@
1248	1184	struct address_space *mapping = inode->i_mapping;
1249	1185	struct dnode_of_data dn;
1250	1186	struct page *page;
1251		- struct extent_info ei = {0,0,0};
	1187	+ struct extent_info ei = {0, };
1252	1188	int err;
1253	1189
1254	1190	page = f2fs_grab_cache_page(mapping, index, for_write);
1255	1191	if (!page)
1256	1192	return ERR_PTR(-ENOMEM);
1257	1193
1258		- if (f2fs_lookup_extent_cache(inode, index, &ei)) {
	1194	+ if (f2fs_lookup_read_extent_cache(inode, index, &ei)) {
1259	1195	dn.data_blkaddr = ei.blk + index - ei.fofs;
1260	1196	if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
1261	1197	DATA_GENERIC_ENHANCE_READ)) {
..	..	@@ -1303,7 +1239,8 @@
1303	1239	return page;
1304	1240	}
1305	1241
1306		- err = f2fs_submit_page_read(inode, page, dn.data_blkaddr, for_write);
	1242	+ err = f2fs_submit_page_read(inode, page, dn.data_blkaddr,
	1243	+ op_flags, for_write);
1307	1244	if (err)
1308	1245	goto put_err;
1309	1246	return page;
..	..	@@ -1437,7 +1374,7 @@
1437	1374	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1438	1375	return -EPERM;
1439	1376
1440		- err = f2fs_get_node_info(sbi, dn->nid, &ni);
	1377	+ err = f2fs_get_node_info(sbi, dn->nid, &ni, false);
1441	1378	if (err)
1442	1379	return err;
1443	1380
..	..	@@ -1452,10 +1389,12 @@
1452	1389	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1453	1390	old_blkaddr = dn->data_blkaddr;
1454	1391	f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
1455		- &sum, seg_type, NULL, false);
1456		- if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
	1392	+ &sum, seg_type, NULL);
	1393	+ if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
1457	1394	invalidate_mapping_pages(META_MAPPING(sbi),
1458	1395	old_blkaddr, old_blkaddr);
	1396	+ f2fs_invalidate_compress_page(sbi, old_blkaddr);
	1397	+ }
1459	1398	f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
1460	1399
1461	1400	/*
..	..	@@ -1512,13 +1451,13 @@
1512	1451	return err;
1513	1452	}
1514	1453
1515		-void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
	1454	+void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1516	1455	{
1517	1456	if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1518	1457	if (lock)
1519		- down_read(&sbi->node_change);
	1458	+ f2fs_down_read(&sbi->node_change);
1520	1459	else
1521		- up_read(&sbi->node_change);
	1460	+ f2fs_up_read(&sbi->node_change);
1522	1461	} else {
1523	1462	if (lock)
1524	1463	f2fs_lock_op(sbi);
..	..	@@ -1543,7 +1482,7 @@
1543	1482	int err = 0, ofs = 1;
1544	1483	unsigned int ofs_in_node, last_ofs_in_node;
1545	1484	blkcnt_t prealloc;
1546		- struct extent_info ei = {0,0,0};
	1485	+ struct extent_info ei = {0, };
1547	1486	block_t blkaddr;
1548	1487	unsigned int start_pgofs;
1549	1488
..	..	@@ -1557,7 +1496,7 @@
1557	1496	pgofs = (pgoff_t)map->m_lblk;
1558	1497	end = pgofs + maxblocks;
1559	1498
1560		- if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
	1499	+ if (!create && f2fs_lookup_read_extent_cache(inode, pgofs, &ei)) {
1561	1500	if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
1562	1501	map->m_may_create)
1563	1502	goto next_dnode;
..	..	@@ -1577,7 +1516,7 @@
1577	1516
1578	1517	next_dnode:
1579	1518	if (map->m_may_create)
1580		- __do_map_lock(sbi, flag, true);
	1519	+ f2fs_do_map_lock(sbi, flag, true);
1581	1520
1582	1521	/* When reading holes, we need its node page */
1583	1522	set_new_dnode(&dn, inode, NULL, NULL, 0);
..	..	@@ -1585,7 +1524,21 @@
1585	1524	if (err) {
1586	1525	if (flag == F2FS_GET_BLOCK_BMAP)
1587	1526	map->m_pblk = 0;
	1527	+
1588	1528	if (err == -ENOENT) {
	1529	+ /*
	1530	+ * There is one exceptional case that read_node_page()
	1531	+ * may return -ENOENT due to filesystem has been
	1532	+ * shutdown or cp_error, so force to convert error
	1533	+ * number to EIO for such case.
	1534	+ */
	1535	+ if (map->m_may_create &&
	1536	+ (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) \|\|
	1537	+ f2fs_cp_error(sbi))) {
	1538	+ err = -EIO;
	1539	+ goto unlock_out;
	1540	+ }
	1541	+
1589	1542	err = 0;
1590	1543	if (map->m_next_pgofs)
1591	1544	*map->m_next_pgofs =
..	..	@@ -1717,7 +1670,7 @@
1717	1670	if (map->m_flags & F2FS_MAP_MAPPED) {
1718	1671	unsigned int ofs = start_pgofs - map->m_lblk;
1719	1672
1720		- f2fs_update_extent_cache_range(&dn,
	1673	+ f2fs_update_read_extent_cache_range(&dn,
1721	1674	start_pgofs, map->m_pblk + ofs,
1722	1675	map->m_len - ofs);
1723	1676	}
..	..	@@ -1726,7 +1679,7 @@
1726	1679	f2fs_put_dnode(&dn);
1727	1680
1728	1681	if (map->m_may_create) {
1729		- __do_map_lock(sbi, flag, false);
	1682	+ f2fs_do_map_lock(sbi, flag, false);
1730	1683	f2fs_balance_fs(sbi, dn.node_changed);
1731	1684	}
1732	1685	goto next_dnode;
..	..	@@ -1742,7 +1695,7 @@
1742	1695	if (map->m_flags & F2FS_MAP_MAPPED) {
1743	1696	unsigned int ofs = start_pgofs - map->m_lblk;
1744	1697
1745		- f2fs_update_extent_cache_range(&dn,
	1698	+ f2fs_update_read_extent_cache_range(&dn,
1746	1699	start_pgofs, map->m_pblk + ofs,
1747	1700	map->m_len - ofs);
1748	1701	}
..	..	@@ -1752,7 +1705,7 @@
1752	1705	f2fs_put_dnode(&dn);
1753	1706	unlock_out:
1754	1707	if (map->m_may_create) {
1755		- __do_map_lock(sbi, flag, false);
	1708	+ f2fs_do_map_lock(sbi, flag, false);
1756	1709	f2fs_balance_fs(sbi, dn.node_changed);
1757	1710	}
1758	1711	out:
..	..	@@ -1786,6 +1739,16 @@
1786	1739	return true;
1787	1740	}
1788	1741
	1742	+static inline u64 bytes_to_blks(struct inode *inode, u64 bytes)
	1743	+{
	1744	+ return (bytes >> inode->i_blkbits);
	1745	+}
	1746	+
	1747	+static inline u64 blks_to_bytes(struct inode *inode, u64 blks)
	1748	+{
	1749	+ return (blks << inode->i_blkbits);
	1750	+}
	1751	+
1789	1752	static int __get_data_block(struct inode *inode, sector_t iblock,
1790	1753	struct buffer_head *bh, int create, int flag,
1791	1754	pgoff_t *next_pgofs, int seg_type, bool may_write)
..	..	@@ -1794,7 +1757,7 @@
1794	1757	int err;
1795	1758
1796	1759	map.m_lblk = iblock;
1797		- map.m_len = bh->b_size >> inode->i_blkbits;
	1760	+ map.m_len = bytes_to_blks(inode, bh->b_size);
1798	1761	map.m_next_pgofs = next_pgofs;
1799	1762	map.m_next_extent = NULL;
1800	1763	map.m_seg_type = seg_type;
..	..	@@ -1804,18 +1767,9 @@
1804	1767	if (!err) {
1805	1768	map_bh(bh, inode->i_sb, map.m_pblk);
1806	1769	bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) \| map.m_flags;
1807		- bh->b_size = (u64)map.m_len << inode->i_blkbits;
	1770	+ bh->b_size = blks_to_bytes(inode, map.m_len);
1808	1771	}
1809	1772	return err;
1810		-}
1811		-
1812		-static int get_data_block(struct inode *inode, sector_t iblock,
1813		- struct buffer_head *bh_result, int create, int flag,
1814		- pgoff_t *next_pgofs)
1815		-{
1816		- return __get_data_block(inode, iblock, bh_result, create,
1817		- flag, next_pgofs,
1818		- NO_CHECK_TYPE, create);
1819	1773	}
1820	1774
1821	1775	static int get_data_block_dio_write(struct inode *inode, sector_t iblock,
..	..	@@ -1824,7 +1778,7 @@
1824	1778	return __get_data_block(inode, iblock, bh_result, create,
1825	1779	F2FS_GET_BLOCK_DIO, NULL,
1826	1780	f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1827		- IS_SWAPFILE(inode) ? false : true);
	1781	+ true);
1828	1782	}
1829	1783
1830	1784	static int get_data_block_dio(struct inode *inode, sector_t iblock,
..	..	@@ -1834,28 +1788,6 @@
1834	1788	F2FS_GET_BLOCK_DIO, NULL,
1835	1789	f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1836	1790	false);
1837		-}
1838		-
1839		-static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1840		- struct buffer_head *bh_result, int create)
1841		-{
1842		- /* Block number less than F2FS MAX BLOCKS */
1843		- if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1844		- return -EFBIG;
1845		-
1846		- return __get_data_block(inode, iblock, bh_result, create,
1847		- F2FS_GET_BLOCK_BMAP, NULL,
1848		- NO_CHECK_TYPE, create);
1849		-}
1850		-
1851		-static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1852		-{
1853		- return (offset >> inode->i_blkbits);
1854		-}
1855		-
1856		-static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1857		-{
1858		- return (blk << inode->i_blkbits);
1859	1791	}
1860	1792
1861	1793	static int f2fs_xattr_fiemap(struct inode *inode,
..	..	@@ -1877,13 +1809,13 @@
1877	1809	if (!page)
1878	1810	return -ENOMEM;
1879	1811
1880		- err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
	1812	+ err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false);
1881	1813	if (err) {
1882	1814	f2fs_put_page(page, 1);
1883	1815	return err;
1884	1816	}
1885	1817
1886		- phys = (__u64)blk_to_logical(inode, ni.blk_addr);
	1818	+ phys = blks_to_bytes(inode, ni.blk_addr);
1887	1819	offset = offsetof(struct f2fs_inode, i_addr) +
1888	1820	sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1889	1821	get_inline_xattr_addrs(inode));
..	..	@@ -1899,6 +1831,7 @@
1899	1831	flags \|= FIEMAP_EXTENT_LAST;
1900	1832
1901	1833	err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
	1834	+ trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
1902	1835	if (err \|\| err == 1)
1903	1836	return err;
1904	1837	}
..	..	@@ -1908,13 +1841,13 @@
1908	1841	if (!page)
1909	1842	return -ENOMEM;
1910	1843
1911		- err = f2fs_get_node_info(sbi, xnid, &ni);
	1844	+ err = f2fs_get_node_info(sbi, xnid, &ni, false);
1912	1845	if (err) {
1913	1846	f2fs_put_page(page, 1);
1914	1847	return err;
1915	1848	}
1916	1849
1917		- phys = (__u64)blk_to_logical(inode, ni.blk_addr);
	1850	+ phys = blks_to_bytes(inode, ni.blk_addr);
1918	1851	len = inode->i_sb->s_blocksize;
1919	1852
1920	1853	f2fs_put_page(page, 1);
..	..	@@ -1922,8 +1855,10 @@
1922	1855	flags = FIEMAP_EXTENT_LAST;
1923	1856	}
1924	1857
1925		- if (phys)
	1858	+ if (phys) {
1926	1859	err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
	1860	+ trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
	1861	+ }
1927	1862
1928	1863	return (err < 0 ? err : 0);
1929	1864	}
..	..	@@ -1950,14 +1885,16 @@
1950	1885	int f2fs_fiemap(struct inode inode, struct fiemap_extent_info fieinfo,
1951	1886	u64 start, u64 len)
1952	1887	{
1953		- struct buffer_head map_bh;
	1888	+ struct f2fs_map_blocks map;
1954	1889	sector_t start_blk, last_blk;
1955	1890	pgoff_t next_pgofs;
1956	1891	u64 logical = 0, phys = 0, size = 0;
1957	1892	u32 flags = 0;
1958	1893	int ret = 0;
1959		- bool compr_cluster = false;
	1894	+ bool compr_cluster = false, compr_appended;
1960	1895	unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
	1896	+ unsigned int count_in_cluster = 0;
	1897	+ loff_t maxbytes;
1961	1898
1962	1899	if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1963	1900	ret = f2fs_precache_extents(inode);
..	..	@@ -1965,11 +1902,20 @@
1965	1902	return ret;
1966	1903	}
1967	1904
1968		- ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC \| FIEMAP_FLAG_XATTR);
	1905	+ ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR);
1969	1906	if (ret)
1970	1907	return ret;
1971	1908
1972	1909	inode_lock(inode);
	1910	+
	1911	+ maxbytes = max_file_blocks(inode) << F2FS_BLKSIZE_BITS;
	1912	+ if (start > maxbytes) {
	1913	+ ret = -EFBIG;
	1914	+ goto out;
	1915	+ }
	1916	+
	1917	+ if (len > maxbytes \|\| (maxbytes - len) < start)
	1918	+ len = maxbytes - start;
1973	1919
1974	1920	if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1975	1921	ret = f2fs_xattr_fiemap(inode, fieinfo);
..	..	@@ -1982,41 +1928,55 @@
1982	1928	goto out;
1983	1929	}
1984	1930
1985		- if (logical_to_blk(inode, len) == 0)
1986		- len = blk_to_logical(inode, 1);
	1931	+ if (bytes_to_blks(inode, len) == 0)
	1932	+ len = blks_to_bytes(inode, 1);
1987	1933
1988		- start_blk = logical_to_blk(inode, start);
1989		- last_blk = logical_to_blk(inode, start + len - 1);
	1934	+ start_blk = bytes_to_blks(inode, start);
	1935	+ last_blk = bytes_to_blks(inode, start + len - 1);
1990	1936
1991	1937	next:
1992		- memset(&map_bh, 0, sizeof(struct buffer_head));
1993		- map_bh.b_size = len;
	1938	+ memset(&map, 0, sizeof(map));
	1939	+ map.m_lblk = start_blk;
	1940	+ map.m_len = bytes_to_blks(inode, len);
	1941	+ map.m_next_pgofs = &next_pgofs;
	1942	+ map.m_seg_type = NO_CHECK_TYPE;
1994	1943
1995		- if (compr_cluster)
1996		- map_bh.b_size = blk_to_logical(inode, cluster_size - 1);
	1944	+ if (compr_cluster) {
	1945	+ map.m_lblk += 1;
	1946	+ map.m_len = cluster_size - count_in_cluster;
	1947	+ }
1997	1948
1998		- ret = get_data_block(inode, start_blk, &map_bh, 0,
1999		- F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
	1949	+ ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP);
2000	1950	if (ret)
2001	1951	goto out;
2002	1952
2003	1953	/* HOLE */
2004		- if (!buffer_mapped(&map_bh)) {
	1954	+ if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) {
2005	1955	start_blk = next_pgofs;
2006	1956
2007		- if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
	1957	+ if (blks_to_bytes(inode, start_blk) < blks_to_bytes(inode,
2008	1958	max_inode_blocks(inode)))
2009	1959	goto prep_next;
2010	1960
2011	1961	flags \|= FIEMAP_EXTENT_LAST;
2012	1962	}
2013	1963
	1964	+ compr_appended = false;
	1965	+ /* In a case of compressed cluster, append this to the last extent */
	1966	+ if (compr_cluster && ((map.m_flags & F2FS_MAP_UNWRITTEN) \|\|
	1967	+ !(map.m_flags & F2FS_MAP_FLAGS))) {
	1968	+ compr_appended = true;
	1969	+ goto skip_fill;
	1970	+ }
	1971	+
2014	1972	if (size) {
	1973	+ flags \|= FIEMAP_EXTENT_MERGED;
2015	1974	if (IS_ENCRYPTED(inode))
2016	1975	flags \|= FIEMAP_EXTENT_DATA_ENCRYPTED;
2017	1976
2018	1977	ret = fiemap_fill_next_extent(fieinfo, logical,
2019	1978	phys, size, flags);
	1979	+ trace_f2fs_fiemap(inode, logical, phys, size, flags, ret);
2020	1980	if (ret)
2021	1981	goto out;
2022	1982	size = 0;
..	..	@@ -2025,38 +1985,36 @@
2025	1985	if (start_blk > last_blk)
2026	1986	goto out;
2027	1987
2028		- if (compr_cluster) {
2029		- compr_cluster = false;
2030		-
2031		-
2032		- logical = blk_to_logical(inode, start_blk - 1);
2033		- phys = blk_to_logical(inode, map_bh.b_blocknr);
2034		- size = blk_to_logical(inode, cluster_size);
2035		-
2036		- flags \|= FIEMAP_EXTENT_ENCODED;
2037		-
2038		- start_blk += cluster_size - 1;
2039		-
2040		- if (start_blk > last_blk)
2041		- goto out;
2042		-
2043		- goto prep_next;
2044		- }
2045		-
2046		- if (map_bh.b_blocknr == COMPRESS_ADDR) {
	1988	+skip_fill:
	1989	+ if (map.m_pblk == COMPRESS_ADDR) {
2047	1990	compr_cluster = true;
2048		- start_blk++;
2049		- goto prep_next;
	1991	+ count_in_cluster = 1;
	1992	+ } else if (compr_appended) {
	1993	+ unsigned int appended_blks = cluster_size -
	1994	+ count_in_cluster + 1;
	1995	+ size += blks_to_bytes(inode, appended_blks);
	1996	+ start_blk += appended_blks;
	1997	+ compr_cluster = false;
	1998	+ } else {
	1999	+ logical = blks_to_bytes(inode, start_blk);
	2000	+ phys = __is_valid_data_blkaddr(map.m_pblk) ?
	2001	+ blks_to_bytes(inode, map.m_pblk) : 0;
	2002	+ size = blks_to_bytes(inode, map.m_len);
	2003	+ flags = 0;
	2004	+
	2005	+ if (compr_cluster) {
	2006	+ flags = FIEMAP_EXTENT_ENCODED;
	2007	+ count_in_cluster += map.m_len;
	2008	+ if (count_in_cluster == cluster_size) {
	2009	+ compr_cluster = false;
	2010	+ size += blks_to_bytes(inode, 1);
	2011	+ }
	2012	+ } else if (map.m_flags & F2FS_MAP_UNWRITTEN) {
	2013	+ flags = FIEMAP_EXTENT_UNWRITTEN;
	2014	+ }
	2015	+
	2016	+ start_blk += bytes_to_blks(inode, size);
2050	2017	}
2051		-
2052		- logical = blk_to_logical(inode, start_blk);
2053		- phys = blk_to_logical(inode, map_bh.b_blocknr);
2054		- size = map_bh.b_size;
2055		- flags = 0;
2056		- if (buffer_unwritten(&map_bh))
2057		- flags = FIEMAP_EXTENT_UNWRITTEN;
2058		-
2059		- start_blk += logical_to_blk(inode, size);
2060	2018
2061	2019	prep_next:
2062	2020	cond_resched();
..	..	@@ -2089,8 +2047,7 @@
2089	2047	bool is_readahead)
2090	2048	{
2091	2049	struct bio bio = bio_ret;
2092		- const unsigned blkbits = inode->i_blkbits;
2093		- const unsigned blocksize = 1 << blkbits;
	2050	+ const unsigned blocksize = blks_to_bytes(inode, 1);
2094	2051	sector_t block_in_file;
2095	2052	sector_t last_block;
2096	2053	sector_t last_block_in_file;
..	..	@@ -2099,8 +2056,8 @@
2099	2056
2100	2057	block_in_file = (sector_t)page_index(page);
2101	2058	last_block = block_in_file + nr_pages;
2102		- last_block_in_file = (f2fs_readpage_limit(inode) + blocksize - 1) >>
2103		- blkbits;
	2059	+ last_block_in_file = bytes_to_blks(inode,
	2060	+ f2fs_readpage_limit(inode) + blocksize - 1);
2104	2061	if (last_block > last_block_in_file)
2105	2062	last_block = last_block_in_file;
2106	2063
..	..	@@ -2163,7 +2120,7 @@
2163	2120	*last_block_in_bio, block_nr) \|\|
2164	2121	!f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2165	2122	submit_and_realloc:
2166		- __f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
	2123	+ __submit_bio(F2FS_I_SB(inode), bio, DATA);
2167	2124	bio = NULL;
2168	2125	}
2169	2126	if (bio == NULL) {
..	..	@@ -2193,7 +2150,7 @@
2193	2150	goto out;
2194	2151	confused:
2195	2152	if (bio) {
2196		- __f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
	2153	+ __submit_bio(F2FS_I_SB(inode), bio, DATA);
2197	2154	bio = NULL;
2198	2155	}
2199	2156	unlock_page(page);
..	..	@@ -2213,16 +2170,17 @@
2213	2170	struct bio bio = bio_ret;
2214	2171	unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
2215	2172	sector_t last_block_in_file;
2216		- const unsigned blkbits = inode->i_blkbits;
2217		- const unsigned blocksize = 1 << blkbits;
	2173	+ const unsigned blocksize = blks_to_bytes(inode, 1);
2218	2174	struct decompress_io_ctx *dic = NULL;
	2175	+ struct extent_info ei = {};
	2176	+ bool from_dnode = true;
2219	2177	int i;
2220	2178	int ret = 0;
2221	2179
2222	2180	f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));
2223	2181
2224		- last_block_in_file = (f2fs_readpage_limit(inode) +
2225		- blocksize - 1) >> blkbits;
	2182	+ last_block_in_file = bytes_to_blks(inode,
	2183	+ f2fs_readpage_limit(inode) + blocksize - 1);
2226	2184
2227	2185	/* get rid of pages beyond EOF */
2228	2186	for (i = 0; i < cc->cluster_size; i++) {
..	..	@@ -2238,6 +2196,8 @@
2238	2196	continue;
2239	2197	}
2240	2198	unlock_page(page);
	2199	+ if (for_write)
	2200	+ put_page(page);
2241	2201	cc->rpages[i] = NULL;
2242	2202	cc->nr_rpages--;
2243	2203	}
..	..	@@ -2246,20 +2206,26 @@
2246	2206	if (f2fs_cluster_is_empty(cc))
2247	2207	goto out;
2248	2208
	2209	+ if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei))
	2210	+ from_dnode = false;
	2211	+
	2212	+ if (!from_dnode)
	2213	+ goto skip_reading_dnode;
	2214	+
2249	2215	set_new_dnode(&dn, inode, NULL, NULL, 0);
2250	2216	ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
2251	2217	if (ret)
2252	2218	goto out;
2253	2219
2254		- /* cluster was overwritten as normal cluster */
2255		- if (dn.data_blkaddr != COMPRESS_ADDR)
2256		- goto out;
	2220	+ f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
2257	2221
	2222	+skip_reading_dnode:
2258	2223	for (i = 1; i < cc->cluster_size; i++) {
2259	2224	block_t blkaddr;
2260	2225
2261		- blkaddr = data_blkaddr(dn.inode, dn.node_page,
2262		- dn.ofs_in_node + i);
	2226	+ blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page,
	2227	+ dn.ofs_in_node + i) :
	2228	+ ei.blk + i - 1;
2263	2229
2264	2230	if (!__is_valid_data_blkaddr(blkaddr))
2265	2231	break;
..	..	@@ -2269,6 +2235,9 @@
2269	2235	goto out_put_dnode;
2270	2236	}
2271	2237	cc->nr_cpages++;
	2238	+
	2239	+ if (!from_dnode && i >= ei.c_len)
	2240	+ break;
2272	2241	}
2273	2242
2274	2243	/* nothing to decompress */
..	..	@@ -2283,13 +2252,22 @@
2283	2252	goto out_put_dnode;
2284	2253	}
2285	2254
2286		- for (i = 0; i < dic->nr_cpages; i++) {
	2255	+ for (i = 0; i < cc->nr_cpages; i++) {
2287	2256	struct page *page = dic->cpages[i];
2288	2257	block_t blkaddr;
2289	2258	struct bio_post_read_ctx *ctx;
2290	2259
2291		- blkaddr = data_blkaddr(dn.inode, dn.node_page,
2292		- dn.ofs_in_node + i + 1);
	2260	+ blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page,
	2261	+ dn.ofs_in_node + i + 1) :
	2262	+ ei.blk + i;
	2263	+
	2264	+ f2fs_wait_on_block_writeback(inode, blkaddr);
	2265	+
	2266	+ if (f2fs_load_compressed_page(sbi, page, blkaddr)) {
	2267	+ if (atomic_dec_and_test(&dic->remaining_pages))
	2268	+ f2fs_decompress_cluster(dic, true);
	2269	+ continue;
	2270	+ }
2293	2271
2294	2272	if (bio && (!page_is_mergeable(sbi, bio,
2295	2273	*last_block_in_bio, blkaddr) \|\|
..	..	@@ -2305,29 +2283,19 @@
2305	2283	page->index, for_write);
2306	2284	if (IS_ERR(bio)) {
2307	2285	ret = PTR_ERR(bio);
2308		- dic->failed = true;
2309		- if (refcount_sub_and_test(dic->nr_cpages - i,
2310		- &dic->ref)) {
2311		- f2fs_decompress_end_io(dic->rpages,
2312		- cc->cluster_size, true,
2313		- false);
2314		- f2fs_free_dic(dic);
2315		- }
	2286	+ f2fs_decompress_end_io(dic, ret, true);
2316	2287	f2fs_put_dnode(&dn);
2317	2288	*bio_ret = NULL;
2318	2289	return ret;
2319	2290	}
2320	2291	}
2321	2292
2322		- f2fs_wait_on_block_writeback(inode, blkaddr);
2323		-
2324	2293	if (bio_add_page(bio, page, blocksize, 0) < blocksize)
2325	2294	goto submit_and_realloc;
2326	2295
2327		- /* tag STEP_DECOMPRESS to handle IO in wq */
2328	2296	ctx = bio->bi_private;
2329		- if (!(ctx->enabled_steps & (1 << STEP_DECOMPRESS)))
2330		- ctx->enabled_steps \|= 1 << STEP_DECOMPRESS;
	2297	+ ctx->enabled_steps \|= STEP_DECOMPRESS;
	2298	+ refcount_inc(&dic->refcnt);
2331	2299
2332	2300	inc_page_count(sbi, F2FS_RD_DATA);
2333	2301	f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
..	..	@@ -2336,15 +2304,23 @@
2336	2304	*last_block_in_bio = blkaddr;
2337	2305	}
2338	2306
2339		- f2fs_put_dnode(&dn);
	2307	+ if (from_dnode)
	2308	+ f2fs_put_dnode(&dn);
2340	2309
2341	2310	*bio_ret = bio;
2342	2311	return 0;
2343	2312
2344	2313	out_put_dnode:
2345		- f2fs_put_dnode(&dn);
	2314	+ if (from_dnode)
	2315	+ f2fs_put_dnode(&dn);
2346	2316	out:
2347		- f2fs_decompress_end_io(cc->rpages, cc->cluster_size, true, false);
	2317	+ for (i = 0; i < cc->cluster_size; i++) {
	2318	+ if (cc->rpages[i]) {
	2319	+ ClearPageUptodate(cc->rpages[i]);
	2320	+ ClearPageError(cc->rpages[i]);
	2321	+ unlock_page(cc->rpages[i]);
	2322	+ }
	2323	+ }
2348	2324	*bio_ret = bio;
2349	2325	return ret;
2350	2326	}
..	..	@@ -2353,19 +2329,12 @@
2353	2329	/*
2354	2330	* This function was originally taken from fs/mpage.c, and customized for f2fs.
2355	2331	* Major change was from block_size == page_size in f2fs by default.
2356		- *
2357		- * Note that the aops->readpages() function is ONLY used for read-ahead. If
2358		- * this function ever deviates from doing just read-ahead, it should either
2359		- * use ->readpage() or do the necessary surgery to decouple ->readpages()
2360		- * from read-ahead.
2361	2332	*/
2362		-int f2fs_mpage_readpages(struct address_space *mapping,
2363		- struct list_head pages, struct page page,
2364		- unsigned nr_pages, bool is_readahead)
	2333	+static int f2fs_mpage_readpages(struct inode *inode,
	2334	+ struct readahead_control rac, struct page page)
2365	2335	{
2366	2336	struct bio *bio = NULL;
2367	2337	sector_t last_block_in_bio = 0;
2368		- struct inode *inode = mapping->host;
2369	2338	struct f2fs_map_blocks map;
2370	2339	#ifdef CONFIG_F2FS_FS_COMPRESSION
2371	2340	struct compress_ctx cc = {
..	..	@@ -2379,6 +2348,7 @@
2379	2348	.nr_cpages = 0,
2380	2349	};
2381	2350	#endif
	2351	+ unsigned nr_pages = rac ? readahead_count(rac) : 1;
2382	2352	unsigned max_nr_pages = nr_pages;
2383	2353	int ret = 0;
2384	2354
..	..	@@ -2392,15 +2362,9 @@
2392	2362	map.m_may_create = false;
2393	2363
2394	2364	for (; nr_pages; nr_pages--) {
2395		- if (pages) {
2396		- page = list_last_entry(pages, struct page, lru);
2397		-
	2365	+ if (rac) {
	2366	+ page = readahead_page(rac);
2398	2367	prefetchw(&page->flags);
2399		- list_del(&page->lru);
2400		- if (add_to_page_cache_lru(page, mapping,
2401		- page_index(page),
2402		- readahead_gfp_mask(mapping)))
2403		- goto next_page;
2404	2368	}
2405	2369
2406	2370	#ifdef CONFIG_F2FS_FS_COMPRESSION
..	..	@@ -2410,8 +2374,8 @@
2410	2374	ret = f2fs_read_multi_pages(&cc, &bio,
2411	2375	max_nr_pages,
2412	2376	&last_block_in_bio,
2413		- is_readahead, false);
2414		- f2fs_destroy_compress_ctx(&cc);
	2377	+ rac != NULL, false);
	2378	+ f2fs_destroy_compress_ctx(&cc, false);
2415	2379	if (ret)
2416	2380	goto set_error_page;
2417	2381	}
..	..	@@ -2433,7 +2397,7 @@
2433	2397	#endif
2434	2398
2435	2399	ret = f2fs_read_single_page(inode, page, max_nr_pages, &map,
2436		- &bio, &last_block_in_bio, is_readahead);
	2400	+ &bio, &last_block_in_bio, rac);
2437	2401	if (ret) {
2438	2402	#ifdef CONFIG_F2FS_FS_COMPRESSION
2439	2403	set_error_page:
..	..	@@ -2442,8 +2406,10 @@
2442	2406	zero_user_segment(page, 0, PAGE_SIZE);
2443	2407	unlock_page(page);
2444	2408	}
	2409	+#ifdef CONFIG_F2FS_FS_COMPRESSION
2445	2410	next_page:
2446		- if (pages)
	2411	+#endif
	2412	+ if (rac)
2447	2413	put_page(page);
2448	2414
2449	2415	#ifdef CONFIG_F2FS_FS_COMPRESSION
..	..	@@ -2453,16 +2419,15 @@
2453	2419	ret = f2fs_read_multi_pages(&cc, &bio,
2454	2420	max_nr_pages,
2455	2421	&last_block_in_bio,
2456		- is_readahead, false);
2457		- f2fs_destroy_compress_ctx(&cc);
	2422	+ rac != NULL, false);
	2423	+ f2fs_destroy_compress_ctx(&cc, false);
2458	2424	}
2459	2425	}
2460	2426	#endif
2461	2427	}
2462		- BUG_ON(pages && !list_empty(pages));
2463	2428	if (bio)
2464		- __f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
2465		- return pages ? 0 : ret;
	2429	+ __submit_bio(F2FS_I_SB(inode), bio, DATA);
	2430	+ return ret;
2466	2431	}
2467	2432
2468	2433	static int f2fs_read_data_page(struct file file, struct page page)
..	..	@@ -2481,28 +2446,24 @@
2481	2446	if (f2fs_has_inline_data(inode))
2482	2447	ret = f2fs_read_inline_data(inode, page);
2483	2448	if (ret == -EAGAIN)
2484		- ret = f2fs_mpage_readpages(page_file_mapping(page),
2485		- NULL, page, 1, false);
	2449	+ ret = f2fs_mpage_readpages(inode, NULL, page);
2486	2450	return ret;
2487	2451	}
2488	2452
2489		-static int f2fs_read_data_pages(struct file *file,
2490		- struct address_space *mapping,
2491		- struct list_head *pages, unsigned nr_pages)
	2453	+static void f2fs_readahead(struct readahead_control *rac)
2492	2454	{
2493		- struct inode *inode = mapping->host;
2494		- struct page *page = list_last_entry(pages, struct page, lru);
	2455	+ struct inode *inode = rac->mapping->host;
2495	2456
2496		- trace_f2fs_readpages(inode, page, nr_pages);
	2457	+ trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac));
2497	2458
2498	2459	if (!f2fs_is_compress_backend_ready(inode))
2499		- return 0;
	2460	+ return;
2500	2461
2501	2462	/* If the file has inline data, skip readpages */
2502	2463	if (f2fs_has_inline_data(inode))
2503		- return 0;
	2464	+ return;
2504	2465
2505		- return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages, true);
	2466	+ f2fs_mpage_readpages(inode, rac, NULL);
2506	2467	}
2507	2468
2508	2469	int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
..	..	@@ -2552,6 +2513,9 @@
2552	2513	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2553	2514	unsigned int policy = SM_I(sbi)->ipu_policy;
2554	2515
	2516	+ if (policy & (0x1 << F2FS_IPU_HONOR_OPU_WRITE) &&
	2517	+ is_inode_flag_set(inode, FI_OPU_WRITE))
	2518	+ return false;
2555	2519	if (policy & (0x1 << F2FS_IPU_FORCE))
2556	2520	return true;
2557	2521	if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
..	..	@@ -2586,11 +2550,15 @@
2586	2550
2587	2551	bool f2fs_should_update_inplace(struct inode inode, struct f2fs_io_info fio)
2588	2552	{
	2553	+ /* swap file is migrating in aligned write mode */
	2554	+ if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
	2555	+ return false;
	2556	+
2589	2557	if (f2fs_is_pinned_file(inode))
2590	2558	return true;
2591	2559
2592	2560	/* if this is cold file, we should overwrite to avoid fragmentation */
2593		- if (file_is_cold(inode))
	2561	+ if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE))
2594	2562	return true;
2595	2563
2596	2564	return check_inplace_update_policy(inode, fio);
..	..	@@ -2613,10 +2581,18 @@
2613	2581	return true;
2614	2582	if (f2fs_is_atomic_file(inode))
2615	2583	return true;
	2584	+
	2585	+ /* swap file is migrating in aligned write mode */
	2586	+ if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
	2587	+ return true;
	2588	+
	2589	+ if (is_inode_flag_set(inode, FI_OPU_WRITE))
	2590	+ return true;
	2591	+
2616	2592	if (fio) {
2617		- if (is_cold_data(fio->page))
	2593	+ if (page_private_gcing(fio->page))
2618	2594	return true;
2619		- if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
	2595	+ if (page_private_dummy(fio->page))
2620	2596	return true;
2621	2597	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2622	2598	f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
..	..	@@ -2640,14 +2616,14 @@
2640	2616	struct page *page = fio->page;
2641	2617	struct inode *inode = page->mapping->host;
2642	2618	struct dnode_of_data dn;
2643		- struct extent_info ei = {0,0,0};
	2619	+ struct extent_info ei = {0, };
2644	2620	struct node_info ni;
2645	2621	bool ipu_force = false;
2646	2622	int err = 0;
2647	2623
2648	2624	set_new_dnode(&dn, inode, NULL, NULL, 0);
2649	2625	if (need_inplace_update(fio) &&
2650		- f2fs_lookup_extent_cache(inode, page->index, &ei)) {
	2626	+ f2fs_lookup_read_extent_cache(inode, page->index, &ei)) {
2651	2627	fio->old_blkaddr = ei.blk + page->index - ei.fofs;
2652	2628
2653	2629	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
..	..	@@ -2672,7 +2648,7 @@
2672	2648	/* This page is already truncated */
2673	2649	if (fio->old_blkaddr == NULL_ADDR) {
2674	2650	ClearPageUptodate(page);
2675		- clear_cold_data(page);
	2651	+ clear_page_private_gcing(page);
2676	2652	goto out_writepage;
2677	2653	}
2678	2654	got_it:
..	..	@@ -2719,7 +2695,7 @@
2719	2695	fio->need_lock = LOCK_REQ;
2720	2696	}
2721	2697
2722		- err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);
	2698	+ err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false);
2723	2699	if (err)
2724	2700	goto out_writepage;
2725	2701
..	..	@@ -2754,7 +2730,8 @@
2754	2730	sector_t *last_block,
2755	2731	struct writeback_control *wbc,
2756	2732	enum iostat_type io_type,
2757		- int compr_blocks)
	2733	+ int compr_blocks,
	2734	+ bool allow_balance)
2758	2735	{
2759	2736	struct inode *inode = page->mapping->host;
2760	2737	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
..	..	@@ -2777,6 +2754,7 @@
2777	2754	.submitted = false,
2778	2755	.compr_blocks = compr_blocks,
2779	2756	.need_lock = LOCK_RETRY,
	2757	+ .post_read = f2fs_post_read_required(inode),
2780	2758	.io_type = io_type,
2781	2759	.io_wbc = wbc,
2782	2760	.bio = bio,
..	..	@@ -2825,8 +2803,20 @@
2825	2803
2826	2804	/* Dentry/quota blocks are controlled by checkpoint */
2827	2805	if (S_ISDIR(inode->i_mode) \|\| IS_NOQUOTA(inode)) {
	2806	+ /*
	2807	+ * We need to wait for node_write to avoid block allocation during
	2808	+ * checkpoint. This can only happen to quota writes which can cause
	2809	+ * the below discard race condition.
	2810	+ */
	2811	+ if (IS_NOQUOTA(inode))
	2812	+ f2fs_down_read(&sbi->node_write);
	2813	+
2828	2814	fio.need_lock = LOCK_DONE;
2829	2815	err = f2fs_do_write_data_page(&fio);
	2816	+
	2817	+ if (IS_NOQUOTA(inode))
	2818	+ f2fs_up_read(&sbi->node_write);
	2819	+
2830	2820	goto done;
2831	2821	}
2832	2822
..	..	@@ -2869,7 +2859,7 @@
2869	2859	inode_dec_dirty_pages(inode);
2870	2860	if (err) {
2871	2861	ClearPageUptodate(page);
2872		- clear_cold_data(page);
	2862	+ clear_page_private_gcing(page);
2873	2863	}
2874	2864
2875	2865	if (wbc->for_reclaim) {
..	..	@@ -2880,7 +2870,7 @@
2880	2870	}
2881	2871	unlock_page(page);
2882	2872	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2883		- !F2FS_I(inode)->cp_task)
	2873	+ !F2FS_I(inode)->wb_task && allow_balance)
2884	2874	f2fs_balance_fs(sbi, need_balance_fs);
2885	2875
2886	2876	if (unlikely(f2fs_cp_error(sbi))) {
..	..	@@ -2927,7 +2917,7 @@
2927	2917	#endif
2928	2918
2929	2919	return f2fs_write_single_data_page(page, NULL, NULL, NULL,
2930		- wbc, FS_DATA_IO, 0);
	2920	+ wbc, FS_DATA_IO, 0, true);
2931	2921	}
2932	2922
2933	2923	/*
..	..	@@ -2962,12 +2952,11 @@
2962	2952	};
2963	2953	#endif
2964	2954	int nr_pages;
2965		- pgoff_t uninitialized_var(writeback_index);
2966	2955	pgoff_t index;
2967	2956	pgoff_t end; /* Inclusive */
2968	2957	pgoff_t done_index;
2969	2958	int range_whole = 0;
2970		- int tag;
	2959	+ xa_mark_t tag;
2971	2960	int nwritten = 0;
2972	2961	int submitted = 0;
2973	2962	int i;
..	..	@@ -2981,8 +2970,7 @@
2981	2970	clear_inode_flag(mapping->host, FI_HOT_DATA);
2982	2971
2983	2972	if (wbc->range_cyclic) {
2984		- writeback_index = mapping->writeback_index; /* prev offset */
2985		- index = writeback_index;
	2973	+ index = mapping->writeback_index; /* prev offset */
2986	2974	end = -1;
2987	2975	} else {
2988	2976	index = wbc->range_start >> PAGE_SHIFT;
..	..	@@ -3097,7 +3085,8 @@
3097	3085	}
3098	3086	#endif
3099	3087	ret = f2fs_write_single_data_page(page, &submitted,
3100		- &bio, &last_block, wbc, io_type, 0);
	3088	+ &bio, &last_block, wbc, io_type,
	3089	+ 0, true);
3101	3090	if (ret == AOP_WRITEPAGE_ACTIVATE)
3102	3091	unlock_page(page);
3103	3092	#ifdef CONFIG_F2FS_FS_COMPRESSION
..	..	@@ -3152,6 +3141,8 @@
3152	3141	retry = 0;
3153	3142	}
3154	3143	}
	3144	+ if (f2fs_compressed_file(inode))
	3145	+ f2fs_destroy_compress_ctx(&cc, false);
3155	3146	#endif
3156	3147	if (retry) {
3157	3148	index = 0;
..	..	@@ -3177,7 +3168,7 @@
3177	3168	struct writeback_control *wbc)
3178	3169	{
3179	3170	/* to avoid deadlock in path of data flush */
3180		- if (F2FS_I(inode)->cp_task)
	3171	+ if (F2FS_I(inode)->wb_task)
3181	3172	return false;
3182	3173
3183	3174	if (!S_ISREG(inode->i_mode))
..	..	@@ -3185,7 +3176,7 @@
3185	3176	if (IS_NOQUOTA(inode))
3186	3177	return false;
3187	3178
3188		- if (f2fs_compressed_file(inode))
	3179	+ if (f2fs_need_compress_data(inode))
3189	3180	return true;
3190	3181	if (wbc->sync_mode != WB_SYNC_ALL)
3191	3182	return true;
..	..	@@ -3222,8 +3213,8 @@
3222	3213	f2fs_available_free_memory(sbi, DIRTY_DENTS))
3223	3214	goto skip_write;
3224	3215
3225		- /* skip writing during file defragment */
3226		- if (is_inode_flag_set(inode, FI_DO_DEFRAG))
	3216	+ /* skip writing in file defragment preparing stage */
	3217	+ if (is_inode_flag_set(inode, FI_SKIP_WRITES))
3227	3218	goto skip_write;
3228	3219
3229	3220	trace_f2fs_writepages(mapping->host, wbc, DATA);
..	..	@@ -3231,8 +3222,12 @@
3231	3222	/* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
3232	3223	if (wbc->sync_mode == WB_SYNC_ALL)
3233	3224	atomic_inc(&sbi->wb_sync_req[DATA]);
3234		- else if (atomic_read(&sbi->wb_sync_req[DATA]))
	3225	+ else if (atomic_read(&sbi->wb_sync_req[DATA])) {
	3226	+ /* to avoid potential deadlock */
	3227	+ if (current->plug)
	3228	+ blk_finish_plug(current->plug);
3235	3229	goto skip_write;
	3230	+ }
3236	3231
3237	3232	if (__should_serialize_io(inode, wbc)) {
3238	3233	mutex_lock(&sbi->writepages);
..	..	@@ -3282,14 +3277,14 @@
3282	3277
3283	3278	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3284	3279	if (to > i_size && !f2fs_verity_in_progress(inode)) {
3285		- down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3286		- down_write(&F2FS_I(inode)->i_mmap_sem);
	3280	+ f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	3281	+ f2fs_down_write(&F2FS_I(inode)->i_mmap_sem);
3287	3282
3288	3283	truncate_pagecache(inode, i_size);
3289	3284	f2fs_truncate_blocks(inode, i_size, true);
3290	3285
3291		- up_write(&F2FS_I(inode)->i_mmap_sem);
3292		- up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	3286	+ f2fs_up_write(&F2FS_I(inode)->i_mmap_sem);
	3287	+ f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3293	3288	}
3294	3289	}
3295	3290
..	..	@@ -3302,7 +3297,7 @@
3302	3297	struct dnode_of_data dn;
3303	3298	struct page *ipage;
3304	3299	bool locked = false;
3305		- struct extent_info ei = {0,0,0};
	3300	+ struct extent_info ei = {0, };
3306	3301	int err = 0;
3307	3302	int flag;
3308	3303
..	..	@@ -3323,7 +3318,7 @@
3323	3318
3324	3319	if (f2fs_has_inline_data(inode) \|\|
3325	3320	(pos & PAGE_MASK) >= i_size_read(inode)) {
3326		- __do_map_lock(sbi, flag, true);
	3321	+ f2fs_do_map_lock(sbi, flag, true);
3327	3322	locked = true;
3328	3323	}
3329	3324
..	..	@@ -3342,7 +3337,7 @@
3342	3337	f2fs_do_read_inline_data(page, ipage);
3343	3338	set_inode_flag(inode, FI_DATA_EXIST);
3344	3339	if (inode->i_nlink)
3345		- set_inline_node(ipage);
	3340	+ set_page_private_inline(ipage);
3346	3341	} else {
3347	3342	err = f2fs_convert_inline_page(&dn, page);
3348	3343	if (err)
..	..	@@ -3353,14 +3348,14 @@
3353	3348	} else if (locked) {
3354	3349	err = f2fs_get_block(&dn, index);
3355	3350	} else {
3356		- if (f2fs_lookup_extent_cache(inode, index, &ei)) {
	3351	+ if (f2fs_lookup_read_extent_cache(inode, index, &ei)) {
3357	3352	dn.data_blkaddr = ei.blk + index - ei.fofs;
3358	3353	} else {
3359	3354	/* hole case */
3360	3355	err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
3361	3356	if (err \|\| dn.data_blkaddr == NULL_ADDR) {
3362	3357	f2fs_put_dnode(&dn);
3363		- __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
	3358	+ f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
3364	3359	true);
3365	3360	WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
3366	3361	locked = true;
..	..	@@ -3376,7 +3371,7 @@
3376	3371	f2fs_put_dnode(&dn);
3377	3372	unlock_out:
3378	3373	if (locked)
3379		- __do_map_lock(sbi, flag, false);
	3374	+ f2fs_do_map_lock(sbi, flag, false);
3380	3375	return err;
3381	3376	}
3382	3377
..	..	@@ -3392,7 +3387,13 @@
3392	3387	block_t blkaddr = NULL_ADDR;
3393	3388	int err = 0;
3394	3389
3395		- if (trace_android_fs_datawrite_start_enabled()) {
	3390	+ /*
	3391	+ * Should avoid quota operations which can make deadlock:
	3392	+ * kswapd -> f2fs_evict_inode -> dquot_drop ->
	3393	+ * f2fs_dquot_commit -> f2fs_write_begin ->
	3394	+ * d_obtain_alias -> __d_alloc -> kmem_cache_alloc(GFP_KERNEL)
	3395	+ */
	3396	+ if (trace_android_fs_datawrite_start_enabled() && !IS_NOQUOTA(inode)) {
3396	3397	char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
3397	3398
3398	3399	path = android_fstrace_get_pathname(pathbuf,
..	..	@@ -3433,6 +3434,9 @@
3433	3434	int ret;
3434	3435
3435	3436	*fsdata = NULL;
	3437	+
	3438	+ if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode)))
	3439	+ goto repeat;
3436	3440
3437	3441	ret = f2fs_prepare_compress_overwrite(inode, pagep,
3438	3442	index, fsdata);
..	..	@@ -3498,7 +3502,7 @@
3498	3502	err = -EFSCORRUPTED;
3499	3503	goto fail;
3500	3504	}
3501		- err = f2fs_submit_page_read(inode, page, blkaddr, true);
	3505	+ err = f2fs_submit_page_read(inode, page, blkaddr, 0, true);
3502	3506	if (err)
3503	3507	goto fail;
3504	3508
..	..	@@ -3549,6 +3553,10 @@
3549	3553	if (f2fs_compressed_file(inode) && fsdata) {
3550	3554	f2fs_compress_write_end(inode, fsdata, page->index, copied);
3551	3555	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
	3556	+
	3557	+ if (pos + copied > i_size_read(inode) &&
	3558	+ !f2fs_verity_in_progress(inode))
	3559	+ f2fs_i_size_write(inode, pos + copied);
3552	3560	return copied;
3553	3561	}
3554	3562	#endif
..	..	@@ -3600,7 +3608,7 @@
3600	3608	bio->bi_private = dio->orig_private;
3601	3609	bio->bi_end_io = dio->orig_end_io;
3602	3610
3603		- kvfree(dio);
	3611	+ kfree(dio);
3604	3612
3605	3613	bio_endio(bio);
3606	3614	}
..	..	@@ -3686,21 +3694,21 @@
3686	3694	iocb->ki_hint = WRITE_LIFE_NOT_SET;
3687	3695
3688	3696	if (iocb->ki_flags & IOCB_NOWAIT) {
3689		- if (!down_read_trylock(&fi->i_gc_rwsem[rw])) {
	3697	+ if (!f2fs_down_read_trylock(&fi->i_gc_rwsem[rw])) {
3690	3698	iocb->ki_hint = hint;
3691	3699	err = -EAGAIN;
3692	3700	goto out;
3693	3701	}
3694		- if (do_opu && !down_read_trylock(&fi->i_gc_rwsem[READ])) {
3695		- up_read(&fi->i_gc_rwsem[rw]);
	3702	+ if (do_opu && !f2fs_down_read_trylock(&fi->i_gc_rwsem[READ])) {
	3703	+ f2fs_up_read(&fi->i_gc_rwsem[rw]);
3696	3704	iocb->ki_hint = hint;
3697	3705	err = -EAGAIN;
3698	3706	goto out;
3699	3707	}
3700	3708	} else {
3701		- down_read(&fi->i_gc_rwsem[rw]);
	3709	+ f2fs_down_read(&fi->i_gc_rwsem[rw]);
3702	3710	if (do_opu)
3703		- down_read(&fi->i_gc_rwsem[READ]);
	3711	+ f2fs_down_read(&fi->i_gc_rwsem[READ]);
3704	3712	}
3705	3713
3706	3714	err = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
..	..	@@ -3710,9 +3718,9 @@
3710	3718	DIO_SKIP_HOLES);
3711	3719
3712	3720	if (do_opu)
3713		- up_read(&fi->i_gc_rwsem[READ]);
	3721	+ f2fs_up_read(&fi->i_gc_rwsem[READ]);
3714	3722
3715		- up_read(&fi->i_gc_rwsem[rw]);
	3723	+ f2fs_up_read(&fi->i_gc_rwsem[rw]);
3716	3724
3717	3725	if (rw == WRITE) {
3718	3726	if (whint_mode == WHINT_MODE_OFF)
..	..	@@ -3722,12 +3730,18 @@
3722	3730	err);
3723	3731	if (!do_opu)
3724	3732	set_inode_flag(inode, FI_UPDATE_WRITE);
	3733	+ } else if (err == -EIOCBQUEUED) {
	3734	+ f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
	3735	+ count - iov_iter_count(iter));
3725	3736	} else if (err < 0) {
3726	3737	f2fs_write_failed(mapping, offset + count);
3727	3738	}
3728	3739	} else {
3729	3740	if (err > 0)
3730	3741	f2fs_update_iostat(sbi, APP_DIRECT_READ_IO, err);
	3742	+ else if (err == -EIOCBQUEUED)
	3743	+ f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_READ_IO,
	3744	+ count - iov_iter_count(iter));
3731	3745	}
3732	3746
3733	3747	out:
..	..	@@ -3764,12 +3778,20 @@
3764	3778	}
3765	3779	}
3766	3780
3767		- clear_cold_data(page);
	3781	+ clear_page_private_gcing(page);
3768	3782
3769		- if (IS_ATOMIC_WRITTEN_PAGE(page))
	3783	+ if (test_opt(sbi, COMPRESS_CACHE)) {
	3784	+ if (f2fs_compressed_file(inode))
	3785	+ f2fs_invalidate_compress_pages(sbi, inode->i_ino);
	3786	+ if (inode->i_ino == F2FS_COMPRESS_INO(sbi))
	3787	+ clear_page_private_data(page);
	3788	+ }
	3789	+
	3790	+ if (page_private_atomic(page))
3770	3791	return f2fs_drop_inmem_page(inode, page);
3771	3792
3772		- f2fs_clear_page_private(page);
	3793	+ detach_page_private(page);
	3794	+ set_page_private(page, 0);
3773	3795	}
3774	3796
3775	3797	int f2fs_release_page(struct page *page, gfp_t wait)
..	..	@@ -3779,11 +3801,23 @@
3779	3801	return 0;
3780	3802
3781	3803	/* This is atomic written page, keep Private */
3782		- if (IS_ATOMIC_WRITTEN_PAGE(page))
	3804	+ if (page_private_atomic(page))
3783	3805	return 0;
3784	3806
3785		- clear_cold_data(page);
3786		- f2fs_clear_page_private(page);
	3807	+ if (test_opt(F2FS_P_SB(page), COMPRESS_CACHE)) {
	3808	+ struct f2fs_sb_info *sbi = F2FS_P_SB(page);
	3809	+ struct inode *inode = page->mapping->host;
	3810	+
	3811	+ if (f2fs_compressed_file(inode))
	3812	+ f2fs_invalidate_compress_pages(sbi, inode->i_ino);
	3813	+ if (inode->i_ino == F2FS_COMPRESS_INO(sbi))
	3814	+ clear_page_private_data(page);
	3815	+ }
	3816	+
	3817	+ clear_page_private_gcing(page);
	3818	+
	3819	+ detach_page_private(page);
	3820	+ set_page_private(page, 0);
3787	3821	return 1;
3788	3822	}
3789	3823
..	..	@@ -3799,7 +3833,7 @@
3799	3833	return __set_page_dirty_nobuffers(page);
3800	3834
3801	3835	if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
3802		- if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
	3836	+ if (!page_private_atomic(page)) {
3803	3837	f2fs_register_inmem_page(inode, page);
3804	3838	return 1;
3805	3839	}
..	..	@@ -3841,10 +3875,9 @@
3841	3875	}
3842	3876
3843	3877	f2fs_put_dnode(&dn);
3844		-
3845	3878	return blknr;
3846	3879	#else
3847		- return -EOPNOTSUPP;
	3880	+ return 0;
3848	3881	#endif
3849	3882	}
3850	3883
..	..	@@ -3852,18 +3885,36 @@
3852	3885	static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
3853	3886	{
3854	3887	struct inode *inode = mapping->host;
	3888	+ sector_t blknr = 0;
3855	3889
3856	3890	if (f2fs_has_inline_data(inode))
3857		- return 0;
	3891	+ goto out;
3858	3892
3859	3893	/* make sure allocating whole blocks */
3860	3894	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
3861	3895	filemap_write_and_wait(mapping);
3862	3896
3863		- if (f2fs_compressed_file(inode))
3864		- return f2fs_bmap_compress(inode, block);
	3897	+ /* Block number less than F2FS MAX BLOCKS */
	3898	+ if (unlikely(block >= max_file_blocks(inode)))
	3899	+ goto out;
3865	3900
3866		- return generic_block_bmap(mapping, block, get_data_block_bmap);
	3901	+ if (f2fs_compressed_file(inode)) {
	3902	+ blknr = f2fs_bmap_compress(inode, block);
	3903	+ } else {
	3904	+ struct f2fs_map_blocks map;
	3905	+
	3906	+ memset(&map, 0, sizeof(map));
	3907	+ map.m_lblk = block;
	3908	+ map.m_len = 1;
	3909	+ map.m_next_pgofs = NULL;
	3910	+ map.m_seg_type = NO_CHECK_TYPE;
	3911	+
	3912	+ if (!f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_BMAP))
	3913	+ blknr = map.m_pblk;
	3914	+ }
	3915	+out:
	3916	+ trace_f2fs_bmap(inode, block, blknr);
	3917	+ return blknr;
3867	3918	}
3868	3919
3869	3920	#ifdef CONFIG_MIGRATION
..	..	@@ -3874,7 +3925,7 @@
3874	3925	{
3875	3926	int rc, extra_count;
3876	3927	struct f2fs_inode_info *fi = F2FS_I(mapping->host);
3877		- bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
	3928	+ bool atomic_written = page_private_atomic(page);
3878	3929
3879	3930	BUG_ON(PageWriteback(page));
3880	3931
..	..	@@ -3889,7 +3940,7 @@
3889	3940	/* one extra reference was held for atomic_write page */
3890	3941	extra_count = atomic_written ? 1 : 0;
3891	3942	rc = migrate_page_move_mapping(mapping, newpage,
3892		- page, NULL, mode, extra_count);
	3943	+ page, extra_count);
3893	3944	if (rc != MIGRATEPAGE_SUCCESS) {
3894	3945	if (atomic_written)
3895	3946	mutex_unlock(&fi->inmem_lock);
..	..	@@ -3898,6 +3949,7 @@
3898	3949
3899	3950	if (atomic_written) {
3900	3951	struct inmem_pages *cur;
	3952	+
3901	3953	list_for_each_entry(cur, &fi->inmem_pages, list)
3902	3954	if (cur->page == page) {
3903	3955	cur->page = newpage;
..	..	@@ -3908,9 +3960,16 @@
3908	3960	get_page(newpage);
3909	3961	}
3910	3962
	3963	+ /* guarantee to start from no stale private field */
	3964	+ set_page_private(newpage, 0);
3911	3965	if (PagePrivate(page)) {
3912		- f2fs_set_page_private(newpage, page_private(page));
3913		- f2fs_clear_page_private(page);
	3966	+ set_page_private(newpage, page_private(page));
	3967	+ SetPagePrivate(newpage);
	3968	+ get_page(newpage);
	3969	+
	3970	+ set_page_private(page, 0);
	3971	+ ClearPagePrivate(page);
	3972	+ put_page(page);
3914	3973	}
3915	3974
3916	3975	if (mode != MIGRATE_SYNC_NO_COPY)
..	..	@@ -3923,97 +3982,172 @@
3923	3982	#endif
3924	3983
3925	3984	#ifdef CONFIG_SWAP
3926		-/* Copied from generic_swapfile_activate() to check any holes */
	3985	+static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk,
	3986	+ unsigned int blkcnt)
	3987	+{
	3988	+ struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	3989	+ unsigned int blkofs;
	3990	+ unsigned int blk_per_sec = BLKS_PER_SEC(sbi);
	3991	+ unsigned int secidx = start_blk / blk_per_sec;
	3992	+ unsigned int end_sec = secidx + blkcnt / blk_per_sec;
	3993	+ int ret = 0;
	3994	+
	3995	+ f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	3996	+ f2fs_down_write(&F2FS_I(inode)->i_mmap_sem);
	3997	+
	3998	+ set_inode_flag(inode, FI_ALIGNED_WRITE);
	3999	+ set_inode_flag(inode, FI_OPU_WRITE);
	4000	+
	4001	+ for (; secidx < end_sec; secidx++) {
	4002	+ f2fs_down_write(&sbi->pin_sem);
	4003	+
	4004	+ f2fs_lock_op(sbi);
	4005	+ f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
	4006	+ f2fs_unlock_op(sbi);
	4007	+
	4008	+ set_inode_flag(inode, FI_SKIP_WRITES);
	4009	+
	4010	+ for (blkofs = 0; blkofs < blk_per_sec; blkofs++) {
	4011	+ struct page *page;
	4012	+ unsigned int blkidx = secidx * blk_per_sec + blkofs;
	4013	+
	4014	+ page = f2fs_get_lock_data_page(inode, blkidx, true);
	4015	+ if (IS_ERR(page)) {
	4016	+ f2fs_up_write(&sbi->pin_sem);
	4017	+ ret = PTR_ERR(page);
	4018	+ goto done;
	4019	+ }
	4020	+
	4021	+ set_page_dirty(page);
	4022	+ f2fs_put_page(page, 1);
	4023	+ }
	4024	+
	4025	+ clear_inode_flag(inode, FI_SKIP_WRITES);
	4026	+
	4027	+ ret = filemap_fdatawrite(inode->i_mapping);
	4028	+
	4029	+ f2fs_up_write(&sbi->pin_sem);
	4030	+
	4031	+ if (ret)
	4032	+ break;
	4033	+ }
	4034	+
	4035	+done:
	4036	+ clear_inode_flag(inode, FI_SKIP_WRITES);
	4037	+ clear_inode_flag(inode, FI_OPU_WRITE);
	4038	+ clear_inode_flag(inode, FI_ALIGNED_WRITE);
	4039	+
	4040	+ f2fs_up_write(&F2FS_I(inode)->i_mmap_sem);
	4041	+ f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	4042	+
	4043	+ return ret;
	4044	+}
	4045	+
3927	4046	static int check_swap_activate(struct swap_info_struct *sis,
3928	4047	struct file swap_file, sector_t span)
3929	4048	{
3930	4049	struct address_space *mapping = swap_file->f_mapping;
3931	4050	struct inode *inode = mapping->host;
3932		- unsigned blocks_per_page;
3933		- unsigned long page_no;
3934		- unsigned blkbits;
3935		- sector_t probe_block;
3936		- sector_t last_block;
3937		- sector_t lowest_block = -1;
3938		- sector_t highest_block = 0;
	4051	+ struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	4052	+ sector_t cur_lblock;
	4053	+ sector_t last_lblock;
	4054	+ sector_t pblock;
	4055	+ sector_t lowest_pblock = -1;
	4056	+ sector_t highest_pblock = 0;
3939	4057	int nr_extents = 0;
3940		- int ret;
3941		-
3942		- blkbits = inode->i_blkbits;
3943		- blocks_per_page = PAGE_SIZE >> blkbits;
	4058	+ unsigned long nr_pblocks;
	4059	+ unsigned int blks_per_sec = BLKS_PER_SEC(sbi);
	4060	+ unsigned int sec_blks_mask = BLKS_PER_SEC(sbi) - 1;
	4061	+ unsigned int not_aligned = 0;
	4062	+ int ret = 0;
3944	4063
3945	4064	/*
3946	4065	* Map all the blocks into the extent list. This code doesn't try
3947	4066	* to be very smart.
3948	4067	*/
3949		- probe_block = 0;
3950		- page_no = 0;
3951		- last_block = i_size_read(inode) >> blkbits;
3952		- while ((probe_block + blocks_per_page) <= last_block &&
3953		- page_no < sis->max) {
3954		- unsigned block_in_page;
3955		- sector_t first_block;
	4068	+ cur_lblock = 0;
	4069	+ last_lblock = bytes_to_blks(inode, i_size_read(inode));
3956	4070
	4071	+ while (cur_lblock < last_lblock && cur_lblock < sis->max) {
	4072	+ struct f2fs_map_blocks map;
	4073	+retry:
3957	4074	cond_resched();
3958	4075
3959		- first_block = bmap(inode, probe_block);
3960		- if (first_block == 0)
3961		- goto bad_bmap;
	4076	+ memset(&map, 0, sizeof(map));
	4077	+ map.m_lblk = cur_lblock;
	4078	+ map.m_len = last_lblock - cur_lblock;
	4079	+ map.m_next_pgofs = NULL;
	4080	+ map.m_next_extent = NULL;
	4081	+ map.m_seg_type = NO_CHECK_TYPE;
	4082	+ map.m_may_create = false;
3962	4083
3963		- /*
3964		- * It must be PAGE_SIZE aligned on-disk
3965		- */
3966		- if (first_block & (blocks_per_page - 1)) {
3967		- probe_block++;
3968		- goto reprobe;
	4084	+ ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP);
	4085	+ if (ret)
	4086	+ goto out;
	4087	+
	4088	+ /* hole */
	4089	+ if (!(map.m_flags & F2FS_MAP_FLAGS)) {
	4090	+ f2fs_err(sbi, "Swapfile has holes");
	4091	+ ret = -EINVAL;
	4092	+ goto out;
3969	4093	}
3970	4094
3971		- for (block_in_page = 1; block_in_page < blocks_per_page;
3972		- block_in_page++) {
3973		- sector_t block;
	4095	+ pblock = map.m_pblk;
	4096	+ nr_pblocks = map.m_len;
3974	4097
3975		- block = bmap(inode, probe_block + block_in_page);
3976		- if (block == 0)
3977		- goto bad_bmap;
3978		- if (block != first_block + block_in_page) {
3979		- /* Discontiguity */
3980		- probe_block++;
3981		- goto reprobe;
	4098	+ if ((pblock - SM_I(sbi)->main_blkaddr) & sec_blks_mask \|\|
	4099	+ nr_pblocks & sec_blks_mask) {
	4100	+ not_aligned++;
	4101	+
	4102	+ nr_pblocks = roundup(nr_pblocks, blks_per_sec);
	4103	+ if (cur_lblock + nr_pblocks > sis->max)
	4104	+ nr_pblocks -= blks_per_sec;
	4105	+
	4106	+ if (!nr_pblocks) {
	4107	+ /* this extent is last one */
	4108	+ nr_pblocks = map.m_len;
	4109	+ f2fs_warn(sbi, "Swapfile: last extent is not aligned to section");
	4110	+ goto next;
3982	4111	}
3983		- }
3984	4112
3985		- first_block >>= (PAGE_SHIFT - blkbits);
3986		- if (page_no) { /* exclude the header page */
3987		- if (first_block < lowest_block)
3988		- lowest_block = first_block;
3989		- if (first_block > highest_block)
3990		- highest_block = first_block;
	4113	+ ret = f2fs_migrate_blocks(inode, cur_lblock,
	4114	+ nr_pblocks);
	4115	+ if (ret)
	4116	+ goto out;
	4117	+ goto retry;
	4118	+ }
	4119	+next:
	4120	+ if (cur_lblock + nr_pblocks >= sis->max)
	4121	+ nr_pblocks = sis->max - cur_lblock;
	4122	+
	4123	+ if (cur_lblock) { /* exclude the header page */
	4124	+ if (pblock < lowest_pblock)
	4125	+ lowest_pblock = pblock;
	4126	+ if (pblock + nr_pblocks - 1 > highest_pblock)
	4127	+ highest_pblock = pblock + nr_pblocks - 1;
3991	4128	}
3992	4129
3993	4130	/*
3994	4131	* We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
3995	4132	*/
3996		- ret = add_swap_extent(sis, page_no, 1, first_block);
	4133	+ ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock);
3997	4134	if (ret < 0)
3998	4135	goto out;
3999	4136	nr_extents += ret;
4000		- page_no++;
4001		- probe_block += blocks_per_page;
4002		-reprobe:
4003		- continue;
	4137	+ cur_lblock += nr_pblocks;
4004	4138	}
4005	4139	ret = nr_extents;
4006		- *span = 1 + highest_block - lowest_block;
4007		- if (page_no == 0)
4008		- page_no = 1; /* force Empty message */
4009		- sis->max = page_no;
4010		- sis->pages = page_no - 1;
4011		- sis->highest_bit = page_no - 1;
	4140	+ *span = 1 + highest_pblock - lowest_pblock;
	4141	+ if (cur_lblock == 0)
	4142	+ cur_lblock = 1; /* force Empty message */
	4143	+ sis->max = cur_lblock;
	4144	+ sis->pages = cur_lblock - 1;
	4145	+ sis->highest_bit = cur_lblock - 1;
4012	4146	out:
	4147	+ if (not_aligned)
	4148	+ f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%u * N)",
	4149	+ not_aligned, blks_per_sec * F2FS_BLKSIZE);
4013	4150	return ret;
4014		-bad_bmap:
4015		- pr_err("swapon: swapfile has holes\n");
4016		- return -EINVAL;
4017	4151	}
4018	4152
4019	4153	static int f2fs_swap_activate(struct swap_info_struct sis, struct file file,
..	..	@@ -4028,19 +4162,26 @@
4028	4162	if (f2fs_readonly(F2FS_I_SB(inode)->sb))
4029	4163	return -EROFS;
4030	4164
	4165	+ if (f2fs_lfs_mode(F2FS_I_SB(inode))) {
	4166	+ f2fs_err(F2FS_I_SB(inode),
	4167	+ "Swapfile not supported in LFS mode");
	4168	+ return -EINVAL;
	4169	+ }
	4170	+
4031	4171	ret = f2fs_convert_inline_inode(inode);
4032	4172	if (ret)
4033	4173	return ret;
4034	4174
4035		- if (f2fs_disable_compressed_file(inode))
	4175	+ if (!f2fs_disable_compressed_file(inode))
4036	4176	return -EINVAL;
	4177	+
	4178	+ f2fs_precache_extents(inode);
4037	4179
4038	4180	ret = check_swap_activate(sis, file, span);
4039	4181	if (ret < 0)
4040	4182	return ret;
4041	4183
4042	4184	set_inode_flag(inode, FI_PIN_FILE);
4043		- f2fs_precache_extents(inode);
4044	4185	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
4045	4186	return ret;
4046	4187	}
..	..	@@ -4065,7 +4206,7 @@
4065	4206
4066	4207	const struct address_space_operations f2fs_dblock_aops = {
4067	4208	.readpage = f2fs_read_data_page,
4068		- .readpages = f2fs_read_data_pages,
	4209	+ .readahead = f2fs_readahead,
4069	4210	.writepage = f2fs_write_data_page,
4070	4211	.writepages = f2fs_write_data_pages,
4071	4212	.write_begin = f2fs_write_begin,
..	..	@@ -4082,13 +4223,13 @@
4082	4223	#endif
4083	4224	};
4084	4225
4085		-void f2fs_clear_radix_tree_dirty_tag(struct page *page)
	4226	+void f2fs_clear_page_cache_dirty_tag(struct page *page)
4086	4227	{
4087	4228	struct address_space *mapping = page_mapping(page);
4088	4229	unsigned long flags;
4089	4230
4090	4231	xa_lock_irqsave(&mapping->i_pages, flags);
4091		- radix_tree_tag_clear(&mapping->i_pages, page_index(page),
	4232	+ __xa_clear_mark(&mapping->i_pages, page_index(page),
4092	4233	PAGECACHE_TAG_DIRTY);
4093	4234	xa_unlock_irqrestore(&mapping->i_pages, flags);
4094	4235	}