~hc/RK356X_SDK_RELEASE.git

..	..	@@ -20,7 +20,6 @@
20	20	#include "segment.h"
21	21	#include "node.h"
22	22	#include "gc.h"
23		-#include "trace.h"
24	23	#include <trace/events/f2fs.h>
25	24
26	25	#define __reverse_ffz(x) __reverse_ffs(~(x))
..	..	@@ -174,7 +173,7 @@
174	173
175	174	if (f2fs_lfs_mode(sbi))
176	175	return false;
177		- if (sbi->gc_mode == GC_URGENT)
	176	+ if (sbi->gc_mode == GC_URGENT_HIGH)
178	177	return true;
179	178	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
180	179	return true;
..	..	@@ -187,9 +186,7 @@
187	186	{
188	187	struct inmem_pages *new;
189	188
190		- f2fs_trace_pid(page);
191		-
192		- f2fs_set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
	189	+ set_page_private_atomic(page);
193	190
194	191	new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
195	192
..	..	@@ -254,7 +251,7 @@
254	251	goto next;
255	252	}
256	253
257		- err = f2fs_get_node_info(sbi, dn.nid, &ni);
	254	+ err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
258	255	if (err) {
259	256	f2fs_put_dnode(&dn);
260	257	return err;
..	..	@@ -272,9 +269,10 @@
272	269	/* we don't need to invalidate this in the sccessful status */
273	270	if (drop \|\| recover) {
274	271	ClearPageUptodate(page);
275		- clear_cold_data(page);
	272	+ clear_page_private_gcing(page);
276	273	}
277		- f2fs_clear_page_private(page);
	274	+ detach_page_private(page);
	275	+ set_page_private(page, 0);
278	276	f2fs_put_page(page, 1);
279	277
280	278	list_del(&cur->list);
..	..	@@ -327,23 +325,27 @@
327	325	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
328	326	struct f2fs_inode_info *fi = F2FS_I(inode);
329	327
330		- while (!list_empty(&fi->inmem_pages)) {
	328	+ do {
331	329	mutex_lock(&fi->inmem_lock);
	330	+ if (list_empty(&fi->inmem_pages)) {
	331	+ fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
	332	+
	333	+ spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
	334	+ if (!list_empty(&fi->inmem_ilist))
	335	+ list_del_init(&fi->inmem_ilist);
	336	+ if (f2fs_is_atomic_file(inode)) {
	337	+ clear_inode_flag(inode, FI_ATOMIC_FILE);
	338	+ sbi->atomic_files--;
	339	+ }
	340	+ spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
	341	+
	342	+ mutex_unlock(&fi->inmem_lock);
	343	+ break;
	344	+ }
332	345	__revoke_inmem_pages(inode, &fi->inmem_pages,
333	346	true, false, true);
334	347	mutex_unlock(&fi->inmem_lock);
335		- }
336		-
337		- fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
338		-
339		- spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
340		- if (!list_empty(&fi->inmem_ilist))
341		- list_del_init(&fi->inmem_ilist);
342		- if (f2fs_is_atomic_file(inode)) {
343		- clear_inode_flag(inode, FI_ATOMIC_FILE);
344		- sbi->atomic_files--;
345		- }
346		- spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
	348	+ } while (1);
347	349	}
348	350
349	351	void f2fs_drop_inmem_page(struct inode inode, struct page page)
..	..	@@ -352,16 +354,19 @@
352	354	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
353	355	struct list_head *head = &fi->inmem_pages;
354	356	struct inmem_pages *cur = NULL;
	357	+ struct inmem_pages *tmp;
355	358
356		- f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page));
	359	+ f2fs_bug_on(sbi, !page_private_atomic(page));
357	360
358	361	mutex_lock(&fi->inmem_lock);
359		- list_for_each_entry(cur, head, list) {
360		- if (cur->page == page)
	362	+ list_for_each_entry(tmp, head, list) {
	363	+ if (tmp->page == page) {
	364	+ cur = tmp;
361	365	break;
	366	+ }
362	367	}
363	368
364		- f2fs_bug_on(sbi, list_empty(head) \|\| cur->page != page);
	369	+ f2fs_bug_on(sbi, !cur);
365	370	list_del(&cur->list);
366	371	mutex_unlock(&fi->inmem_lock);
367	372
..	..	@@ -369,8 +374,11 @@
369	374	kmem_cache_free(inmem_entry_slab, cur);
370	375
371	376	ClearPageUptodate(page);
372		- f2fs_clear_page_private(page);
	377	+ clear_page_private_atomic(page);
373	378	f2fs_put_page(page, 0);
	379	+
	380	+ detach_page_private(page);
	381	+ set_page_private(page, 0);
374	382
375	383	trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
376	384	}
..	..	@@ -466,7 +474,7 @@
466	474
467	475	f2fs_balance_fs(sbi, true);
468	476
469		- down_write(&fi->i_gc_rwsem[WRITE]);
	477	+ f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
470	478
471	479	f2fs_lock_op(sbi);
472	480	set_inode_flag(inode, FI_ATOMIC_COMMIT);
..	..	@@ -478,7 +486,7 @@
478	486	clear_inode_flag(inode, FI_ATOMIC_COMMIT);
479	487
480	488	f2fs_unlock_op(sbi);
481		- up_write(&fi->i_gc_rwsem[WRITE]);
	489	+ f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
482	490
483	491	return err;
484	492	}
..	..	@@ -491,7 +499,7 @@
491	499	{
492	500	if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
493	501	f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
494		- f2fs_stop_checkpoint(sbi, false);
	502	+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
495	503	}
496	504
497	505	/* balance_fs_bg is able to be pending */
..	..	@@ -506,8 +514,19 @@
506	514	* dir/node pages without enough free segments.
507	515	*/
508	516	if (has_not_enough_free_secs(sbi, 0, 0)) {
509		- down_write(&sbi->gc_lock);
510		- f2fs_gc(sbi, false, false, NULL_SEGNO);
	517	+ if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
	518	+ sbi->gc_thread->f2fs_gc_task) {
	519	+ DEFINE_WAIT(wait);
	520	+
	521	+ prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
	522	+ TASK_UNINTERRUPTIBLE);
	523	+ wake_up(&sbi->gc_thread->gc_wait_queue_head);
	524	+ io_schedule();
	525	+ finish_wait(&sbi->gc_thread->fggc_wq, &wait);
	526	+ } else {
	527	+ f2fs_down_write(&sbi->gc_lock);
	528	+ f2fs_gc(sbi, false, false, false, NULL_SEGNO);
	529	+ }
511	530	}
512	531	}
513	532
..	..	@@ -517,8 +536,14 @@
517	536	return;
518	537
519	538	/* try to shrink extent cache when there is no enough memory */
520		- if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
521		- f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
	539	+ if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))
	540	+ f2fs_shrink_read_extent_tree(sbi,
	541	+ READ_EXTENT_CACHE_SHRINK_NUMBER);
	542	+
	543	+ /* try to shrink age extent cache when there is no enough memory */
	544	+ if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))
	545	+ f2fs_shrink_age_extent_tree(sbi,
	546	+ AGE_EXTENT_CACHE_SHRINK_NUMBER);
522	547
523	548	/* check the # of cached NAT entries */
524	549	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
..	..	@@ -529,47 +554,44 @@
529	554	else
530	555	f2fs_build_free_nids(sbi, false, false);
531	556
532		- if (!is_idle(sbi, REQ_TIME) &&
533		- (!excess_dirty_nats(sbi) && !excess_dirty_nodes(sbi)))
	557	+ if (excess_dirty_nats(sbi) \|\| excess_dirty_nodes(sbi) \|\|
	558	+ excess_prefree_segs(sbi))
	559	+ goto do_sync;
	560	+
	561	+ /* there is background inflight IO or foreground operation recently */
	562	+ if (is_inflight_io(sbi, REQ_TIME) \|\|
	563	+ (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
534	564	return;
535	565
	566	+ /* exceed periodical checkpoint timeout threshold */
	567	+ if (f2fs_time_over(sbi, CP_TIME))
	568	+ goto do_sync;
	569	+
536	570	/* checkpoint is the only way to shrink partial cached entries */
537		- if (!f2fs_available_free_memory(sbi, NAT_ENTRIES) \|\|
538		- !f2fs_available_free_memory(sbi, INO_ENTRIES) \|\|
539		- excess_prefree_segs(sbi) \|\|
540		- excess_dirty_nats(sbi) \|\|
541		- excess_dirty_nodes(sbi) \|\|
542		- f2fs_time_over(sbi, CP_TIME)) {
543		- if (test_opt(sbi, DATA_FLUSH) && from_bg) {
544		- struct blk_plug plug;
	571	+ if (f2fs_available_free_memory(sbi, NAT_ENTRIES) \|\|
	572	+ f2fs_available_free_memory(sbi, INO_ENTRIES))
	573	+ return;
545	574
546		- mutex_lock(&sbi->flush_lock);
	575	+do_sync:
	576	+ if (test_opt(sbi, DATA_FLUSH) && from_bg) {
	577	+ struct blk_plug plug;
547	578
548		- blk_start_plug(&plug);
549		- f2fs_sync_dirty_inodes(sbi, FILE_INODE);
550		- blk_finish_plug(&plug);
	579	+ mutex_lock(&sbi->flush_lock);
551	580
552		- mutex_unlock(&sbi->flush_lock);
553		- }
554		- f2fs_sync_fs(sbi->sb, true);
555		- stat_inc_bg_cp_count(sbi->stat_info);
	581	+ blk_start_plug(&plug);
	582	+ f2fs_sync_dirty_inodes(sbi, FILE_INODE, NULL);
	583	+ blk_finish_plug(&plug);
	584	+
	585	+ mutex_unlock(&sbi->flush_lock);
556	586	}
	587	+ f2fs_sync_fs(sbi->sb, true);
	588	+ stat_inc_bg_cp_count(sbi->stat_info);
557	589	}
558	590
559	591	static int __submit_flush_wait(struct f2fs_sb_info *sbi,
560	592	struct block_device *bdev)
561	593	{
562		- struct bio *bio;
563		- int ret;
564		-
565		- bio = f2fs_bio_alloc(sbi, 0, false);
566		- if (!bio)
567		- return -ENOMEM;
568		-
569		- bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_PREFLUSH;
570		- bio_set_dev(bio, bdev);
571		- ret = submit_bio_wait(bio);
572		- bio_put(bio);
	594	+ int ret = blkdev_issue_flush(bdev, GFP_NOFS);
573	595
574	596	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
575	597	test_opt(sbi, FLUSH_MERGE), ret);
..	..	@@ -603,8 +625,6 @@
603	625	if (kthread_should_stop())
604	626	return 0;
605	627
606		- sb_start_intwrite(sbi->sb);
607		-
608	628	if (!llist_empty(&fcc->issue_list)) {
609	629	struct flush_cmd cmd, next;
610	630	int ret;
..	..	@@ -624,8 +644,6 @@
624	644	}
625	645	fcc->dispatch_list = NULL;
626	646	}
627		-
628		- sb_end_intwrite(sbi->sb);
629	647
630	648	wait_event_interruptible(*q,
631	649	kthread_should_stop() \|\| !llist_empty(&fcc->issue_list));
..	..	@@ -663,7 +681,11 @@
663	681
664	682	llist_add(&cmd.llnode, &fcc->issue_list);
665	683
666		- /* update issue_list before we wake up issue_flush thread */
	684	+ /*
	685	+ * update issue_list before we wake up issue_flush thread, this
	686	+ * smp_mb() pairs with another barrier in ___wait_event(), see
	687	+ * more details in comments of waitqueue_active().
	688	+ */
667	689	smp_mb();
668	690
669	691	if (waitqueue_active(&fcc->flush_wait_queue))
..	..	@@ -728,7 +750,7 @@
728	750	"f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
729	751	if (IS_ERR(fcc->f2fs_issue_flush)) {
730	752	err = PTR_ERR(fcc->f2fs_issue_flush);
731		- kvfree(fcc);
	753	+ kfree(fcc);
732	754	SM_I(sbi)->fcc_info = NULL;
733	755	return err;
734	756	}
..	..	@@ -747,7 +769,7 @@
747	769	kthread_stop(flush_thread);
748	770	}
749	771	if (free) {
750		- kvfree(fcc);
	772	+ kfree(fcc);
751	773	SM_I(sbi)->fcc_info = NULL;
752	774	}
753	775	}
..	..	@@ -759,12 +781,18 @@
759	781	if (!f2fs_is_multi_device(sbi))
760	782	return 0;
761	783
	784	+ if (test_opt(sbi, NOBARRIER))
	785	+ return 0;
	786	+
762	787	for (i = 1; i < sbi->s_ndevs; i++) {
763	788	if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
764	789	continue;
765	790	ret = __submit_flush_wait(sbi, FDEV(i).bdev);
766		- if (ret)
	791	+ if (ret) {
	792	+ f2fs_stop_checkpoint(sbi, false,
	793	+ STOP_CP_REASON_FLUSH_FAIL);
767	794	break;
	795	+ }
768	796
769	797	spin_lock(&sbi->dev_lock);
770	798	f2fs_clear_bit(i, (char *)&sbi->dirty_device);
..	..	@@ -796,6 +824,18 @@
796	824	}
797	825	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
798	826	dirty_i->nr_dirty[t]++;
	827	+
	828	+ if (__is_large_section(sbi)) {
	829	+ unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
	830	+ block_t valid_blocks =
	831	+ get_valid_blocks(sbi, segno, true);
	832	+
	833	+ f2fs_bug_on(sbi, unlikely(!valid_blocks \|\|
	834	+ valid_blocks == BLKS_PER_SEC(sbi)));
	835	+
	836	+ if (!IS_CURSEC(sbi, secno))
	837	+ set_bit(secno, dirty_i->dirty_secmap);
	838	+ }
799	839	}
800	840	}
801	841
..	..	@@ -803,6 +843,7 @@
803	843	enum dirty_type dirty_type)
804	844	{
805	845	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
	846	+ block_t valid_blocks;
806	847
807	848	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
808	849	dirty_i->nr_dirty[dirty_type]--;
..	..	@@ -814,12 +855,25 @@
814	855	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
815	856	dirty_i->nr_dirty[t]--;
816	857
817		- if (get_valid_blocks(sbi, segno, true) == 0) {
	858	+ valid_blocks = get_valid_blocks(sbi, segno, true);
	859	+ if (valid_blocks == 0) {
818	860	clear_bit(GET_SEC_FROM_SEG(sbi, segno),
819	861	dirty_i->victim_secmap);
820	862	#ifdef CONFIG_F2FS_CHECK_FS
821	863	clear_bit(segno, SIT_I(sbi)->invalid_segmap);
822	864	#endif
	865	+ }
	866	+ if (__is_large_section(sbi)) {
	867	+ unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
	868	+
	869	+ if (!valid_blocks \|\|
	870	+ valid_blocks == BLKS_PER_SEC(sbi)) {
	871	+ clear_bit(secno, dirty_i->dirty_secmap);
	872	+ return;
	873	+ }
	874	+
	875	+ if (!IS_CURSEC(sbi, secno))
	876	+ set_bit(secno, dirty_i->dirty_secmap);
823	877	}
824	878	}
825	879	}
..	..	@@ -833,20 +887,22 @@
833	887	{
834	888	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
835	889	unsigned short valid_blocks, ckpt_valid_blocks;
	890	+ unsigned int usable_blocks;
836	891
837	892	if (segno == NULL_SEGNO \|\| IS_CURSEG(sbi, segno))
838	893	return;
839	894
	895	+ usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
840	896	mutex_lock(&dirty_i->seglist_lock);
841	897
842	898	valid_blocks = get_valid_blocks(sbi, segno, false);
843		- ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno);
	899	+ ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
844	900
845	901	if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) \|\|
846		- ckpt_valid_blocks == sbi->blocks_per_seg)) {
	902	+ ckpt_valid_blocks == usable_blocks)) {
847	903	__locate_dirty_segment(sbi, segno, PRE);
848	904	__remove_dirty_segment(sbi, segno, DIRTY);
849		- } else if (valid_blocks < sbi->blocks_per_seg) {
	905	+ } else if (valid_blocks < usable_blocks) {
850	906	__locate_dirty_segment(sbi, segno, DIRTY);
851	907	} else {
852	908	/* Recovery routine with SSR needs this */
..	..	@@ -889,9 +945,11 @@
889	945	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
890	946	se = get_seg_entry(sbi, segno);
891	947	if (IS_NODESEG(se->type))
892		- holes[NODE] += sbi->blocks_per_seg - se->valid_blocks;
	948	+ holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
	949	+ se->valid_blocks;
893	950	else
894		- holes[DATA] += sbi->blocks_per_seg - se->valid_blocks;
	951	+ holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
	952	+ se->valid_blocks;
895	953	}
896	954	mutex_unlock(&dirty_i->seglist_lock);
897	955
..	..	@@ -923,7 +981,7 @@
923	981	for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
924	982	if (get_valid_blocks(sbi, segno, false))
925	983	continue;
926		- if (get_ckpt_valid_blocks(sbi, segno))
	984	+ if (get_ckpt_valid_blocks(sbi, segno, false))
927	985	continue;
928	986	mutex_unlock(&dirty_i->seglist_lock);
929	987	return segno;
..	..	@@ -1072,6 +1130,8 @@
1072	1130	struct discard_policy *dpolicy,
1073	1131	int discard_type, unsigned int granularity)
1074	1132	{
	1133	+ struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
	1134	+
1075	1135	/* common policy */
1076	1136	dpolicy->type = discard_type;
1077	1137	dpolicy->sync = true;
..	..	@@ -1091,7 +1151,9 @@
1091	1151	dpolicy->ordered = true;
1092	1152	if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1093	1153	dpolicy->granularity = 1;
1094		- dpolicy->max_interval = DEF_MIN_DISCARD_ISSUE_TIME;
	1154	+ if (atomic_read(&dcc->discard_cmd_cnt))
	1155	+ dpolicy->max_interval =
	1156	+ DEF_MIN_DISCARD_ISSUE_TIME;
1095	1157	}
1096	1158	} else if (discard_type == DPOLICY_FORCE) {
1097	1159	dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
..	..	@@ -1485,7 +1547,7 @@
1485	1547	if (i + 1 < dpolicy->granularity)
1486	1548	break;
1487	1549
1488		- if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
	1550	+ if (i + 1 < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1489	1551	return __issue_discard_cmd_orderly(sbi, dpolicy);
1490	1552
1491	1553	pend_list = &dcc->pend_list[i];
..	..	@@ -1495,7 +1557,7 @@
1495	1557	goto next;
1496	1558	if (unlikely(dcc->rbtree_check))
1497	1559	f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1498		- &dcc->root));
	1560	+ &dcc->root, false));
1499	1561	blk_start_plug(&plug);
1500	1562	list_for_each_entry_safe(dc, tmp, pend_list, list) {
1501	1563	f2fs_bug_on(sbi, dc->state != D_PREP);
..	..	@@ -1707,8 +1769,15 @@
1707	1769	set_freezable();
1708	1770
1709	1771	do {
1710		- __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1711		- dcc->discard_granularity);
	1772	+ if (sbi->gc_mode == GC_URGENT_HIGH \|\|
	1773	+ !f2fs_available_free_memory(sbi, DISCARD_CACHE))
	1774	+ __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
	1775	+ else
	1776	+ __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
	1777	+ dcc->discard_granularity);
	1778	+
	1779	+ if (!atomic_read(&dcc->discard_cmd_cnt))
	1780	+ wait_ms = dpolicy.max_interval;
1712	1781
1713	1782	wait_event_interruptible_timeout(*q,
1714	1783	kthread_should_stop() \|\| freezing(current) \|\|
..	..	@@ -1732,9 +1801,8 @@
1732	1801	wait_ms = dpolicy.max_interval;
1733	1802	continue;
1734	1803	}
1735		-
1736		- if (sbi->gc_mode == GC_URGENT)
1737		- __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
	1804	+ if (!atomic_read(&dcc->discard_cmd_cnt))
	1805	+ continue;
1738	1806
1739	1807	sb_start_intwrite(sbi->sb);
1740	1808
..	..	@@ -1742,7 +1810,7 @@
1742	1810	if (issued > 0) {
1743	1811	__wait_all_discard_cmd(sbi, &dpolicy);
1744	1812	wait_ms = dpolicy.min_interval;
1745		- } else if (issued == -1){
	1813	+ } else if (issued == -1) {
1746	1814	wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1747	1815	if (!wait_ms)
1748	1816	wait_ms = dpolicy.mid_interval;
..	..	@@ -1787,7 +1855,8 @@
1787	1855	return -EIO;
1788	1856	}
1789	1857	trace_f2fs_issue_reset_zone(bdev, blkstart);
1790		- return blkdev_reset_zones(bdev, sector, nr_sects, GFP_NOFS);
	1858	+ return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
	1859	+ sector, nr_sects, GFP_NOFS);
1791	1860	}
1792	1861
1793	1862	/* For conventional zones, use regular discard if supported */
..	..	@@ -1931,7 +2000,7 @@
1931	2000
1932	2001	mutex_lock(&dirty_i->seglist_lock);
1933	2002	for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1934		- __set_test_and_free(sbi, segno);
	2003	+ __set_test_and_free(sbi, segno, false);
1935	2004	mutex_unlock(&dirty_i->seglist_lock);
1936	2005	}
1937	2006
..	..	@@ -2074,7 +2143,7 @@
2074	2143	"f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2075	2144	if (IS_ERR(dcc->f2fs_issue_discard)) {
2076	2145	err = PTR_ERR(dcc->f2fs_issue_discard);
2077		- kvfree(dcc);
	2146	+ kfree(dcc);
2078	2147	SM_I(sbi)->dcc_info = NULL;
2079	2148	return err;
2080	2149	}
..	..	@@ -2098,7 +2167,7 @@
2098	2167	if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2099	2168	f2fs_issue_discard_timeout(sbi);
2100	2169
2101		- kvfree(dcc);
	2170	+ kfree(dcc);
2102	2171	SM_I(sbi)->dcc_info = NULL;
2103	2172	}
2104	2173
..	..	@@ -2118,9 +2187,43 @@
2118	2187	unsigned int segno, int modified)
2119	2188	{
2120	2189	struct seg_entry *se = get_seg_entry(sbi, segno);
	2190	+
2121	2191	se->type = type;
2122	2192	if (modified)
2123	2193	__mark_sit_entry_dirty(sbi, segno);
	2194	+}
	2195	+
	2196	+static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
	2197	+ block_t blkaddr)
	2198	+{
	2199	+ unsigned int segno = GET_SEGNO(sbi, blkaddr);
	2200	+
	2201	+ if (segno == NULL_SEGNO)
	2202	+ return 0;
	2203	+ return get_seg_entry(sbi, segno)->mtime;
	2204	+}
	2205	+
	2206	+static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
	2207	+ unsigned long long old_mtime)
	2208	+{
	2209	+ struct seg_entry *se;
	2210	+ unsigned int segno = GET_SEGNO(sbi, blkaddr);
	2211	+ unsigned long long ctime = get_mtime(sbi, false);
	2212	+ unsigned long long mtime = old_mtime ? old_mtime : ctime;
	2213	+
	2214	+ if (segno == NULL_SEGNO)
	2215	+ return;
	2216	+
	2217	+ se = get_seg_entry(sbi, segno);
	2218	+
	2219	+ if (!se->mtime)
	2220	+ se->mtime = mtime;
	2221	+ else
	2222	+ se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
	2223	+ se->valid_blocks + 1);
	2224	+
	2225	+ if (ctime > SIT_I(sbi)->max_mtime)
	2226	+ SIT_I(sbi)->max_mtime = ctime;
2124	2227	}
2125	2228
2126	2229	static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
..	..	@@ -2139,13 +2242,10 @@
2139	2242	new_vblocks = se->valid_blocks + del;
2140	2243	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2141	2244
2142		- f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) \|\|
2143		- (new_vblocks > sbi->blocks_per_seg)));
	2245	+ f2fs_bug_on(sbi, (new_vblocks < 0 \|\|
	2246	+ (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2144	2247
2145	2248	se->valid_blocks = new_vblocks;
2146		- se->mtime = get_mtime(sbi, false);
2147		- if (se->mtime > SIT_I(sbi)->max_mtime)
2148		- SIT_I(sbi)->max_mtime = se->mtime;
2149	2249
2150	2250	/* Update valid block bitmap */
2151	2251	if (del > 0) {
..	..	@@ -2234,10 +2334,12 @@
2234	2334	return;
2235	2335
2236	2336	invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
	2337	+ f2fs_invalidate_compress_page(sbi, addr);
2237	2338
2238	2339	/* add it into sit main buffer */
2239	2340	down_write(&sit_i->sentry_lock);
2240	2341
	2342	+ update_segment_mtime(sbi, addr, 0);
2241	2343	update_sit_entry(sbi, addr, -1);
2242	2344
2243	2345	/* add it into dirty seglist */
..	..	@@ -2278,6 +2380,7 @@
2278	2380	{
2279	2381	struct curseg_info *curseg = CURSEG_I(sbi, type);
2280	2382	void *addr = curseg->sum_blk;
	2383	+
2281	2384	addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2282	2385	memcpy(addr, sum, sizeof(struct f2fs_summary));
2283	2386	}
..	..	@@ -2317,7 +2420,9 @@
2317	2420	*/
2318	2421	struct page f2fs_get_sum_page(struct f2fs_sb_info sbi, unsigned int segno)
2319	2422	{
2320		- return f2fs_get_meta_page_nofail(sbi, GET_SUM_BLOCK(sbi, segno));
	2423	+ if (unlikely(f2fs_cp_error(sbi)))
	2424	+ return ERR_PTR(-EIO);
	2425	+ return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2321	2426	}
2322	2427
2323	2428	void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
..	..	@@ -2362,9 +2467,9 @@
2362	2467	f2fs_put_page(page, 1);
2363	2468	}
2364	2469
2365		-static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
	2470	+static int is_next_segment_free(struct f2fs_sb_info *sbi,
	2471	+ struct curseg_info *curseg, int type)
2366	2472	{
2367		- struct curseg_info *curseg = CURSEG_I(sbi, type);
2368	2473	unsigned int segno = curseg->segno + 1;
2369	2474	struct free_segmap_info *free_i = FREE_I(sbi);
2370	2475
..	..	@@ -2468,7 +2573,9 @@
2468	2573	{
2469	2574	struct curseg_info *curseg = CURSEG_I(sbi, type);
2470	2575	struct summary_footer *sum_footer;
	2576	+ unsigned short seg_type = curseg->seg_type;
2471	2577
	2578	+ curseg->inited = true;
2472	2579	curseg->segno = curseg->next_segno;
2473	2580	curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2474	2581	curseg->next_blkoff = 0;
..	..	@@ -2476,24 +2583,36 @@
2476	2583
2477	2584	sum_footer = &(curseg->sum_blk->footer);
2478	2585	memset(sum_footer, 0, sizeof(struct summary_footer));
2479		- if (IS_DATASEG(type))
	2586	+
	2587	+ sanity_check_seg_type(sbi, seg_type);
	2588	+
	2589	+ if (IS_DATASEG(seg_type))
2480	2590	SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2481		- if (IS_NODESEG(type))
	2591	+ if (IS_NODESEG(seg_type))
2482	2592	SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2483		- __set_sit_entry_type(sbi, type, curseg->segno, modified);
	2593	+ __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2484	2594	}
2485	2595
2486	2596	static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2487	2597	{
	2598	+ struct curseg_info *curseg = CURSEG_I(sbi, type);
	2599	+ unsigned short seg_type = curseg->seg_type;
	2600	+
	2601	+ sanity_check_seg_type(sbi, seg_type);
	2602	+
2488	2603	/* if segs_per_sec is large than 1, we need to keep original policy. */
2489	2604	if (__is_large_section(sbi))
2490		- return CURSEG_I(sbi, type)->segno;
	2605	+ return curseg->segno;
	2606	+
	2607	+ /* inmem log may not locate on any segment after mount */
	2608	+ if (!curseg->inited)
	2609	+ return 0;
2491	2610
2492	2611	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2493	2612	return 0;
2494	2613
2495	2614	if (test_opt(sbi, NOHEAP) &&
2496		- (type == CURSEG_HOT_DATA \|\| IS_NODESEG(type)))
	2615	+ (seg_type == CURSEG_HOT_DATA \|\| IS_NODESEG(seg_type)))
2497	2616	return 0;
2498	2617
2499	2618	if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
..	..	@@ -2503,7 +2622,7 @@
2503	2622	if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2504	2623	return 0;
2505	2624
2506		- return CURSEG_I(sbi, type)->segno;
	2625	+ return curseg->segno;
2507	2626	}
2508	2627
2509	2628	/*
..	..	@@ -2513,12 +2632,14 @@
2513	2632	static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2514	2633	{
2515	2634	struct curseg_info *curseg = CURSEG_I(sbi, type);
	2635	+ unsigned short seg_type = curseg->seg_type;
2516	2636	unsigned int segno = curseg->segno;
2517	2637	int dir = ALLOC_LEFT;
2518	2638
2519		- write_sum_page(sbi, curseg->sum_blk,
	2639	+ if (curseg->inited)
	2640	+ write_sum_page(sbi, curseg->sum_blk,
2520	2641	GET_SUM_BLOCK(sbi, segno));
2521		- if (type == CURSEG_WARM_DATA \|\| type == CURSEG_COLD_DATA)
	2642	+ if (seg_type == CURSEG_WARM_DATA \|\| seg_type == CURSEG_COLD_DATA)
2522	2643	dir = ALLOC_RIGHT;
2523	2644
2524	2645	if (test_opt(sbi, NOHEAP))
..	..	@@ -2531,22 +2652,20 @@
2531	2652	curseg->alloc_type = LFS;
2532	2653	}
2533	2654
2534		-static void __next_free_blkoff(struct f2fs_sb_info *sbi,
2535		- struct curseg_info *seg, block_t start)
	2655	+static int __next_free_blkoff(struct f2fs_sb_info *sbi,
	2656	+ int segno, block_t start)
2536	2657	{
2537		- struct seg_entry *se = get_seg_entry(sbi, seg->segno);
	2658	+ struct seg_entry *se = get_seg_entry(sbi, segno);
2538	2659	int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2539	2660	unsigned long *target_map = SIT_I(sbi)->tmp_map;
2540	2661	unsigned long ckpt_map = (unsigned long )se->ckpt_valid_map;
2541	2662	unsigned long cur_map = (unsigned long )se->cur_valid_map;
2542		- int i, pos;
	2663	+ int i;
2543	2664
2544	2665	for (i = 0; i < entries; i++)
2545	2666	target_map[i] = ckpt_map[i] \| cur_map[i];
2546	2667
2547		- pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2548		-
2549		- seg->next_blkoff = pos;
	2668	+ return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2550	2669	}
2551	2670
2552	2671	/*
..	..	@@ -2558,16 +2677,23 @@
2558	2677	struct curseg_info *seg)
2559	2678	{
2560	2679	if (seg->alloc_type == SSR)
2561		- __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
	2680	+ seg->next_blkoff =
	2681	+ __next_free_blkoff(sbi, seg->segno,
	2682	+ seg->next_blkoff + 1);
2562	2683	else
2563	2684	seg->next_blkoff++;
	2685	+}
	2686	+
	2687	+bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
	2688	+{
	2689	+ return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2564	2690	}
2565	2691
2566	2692	/*
2567	2693	* This function always allocates a used segment(from dirty seglist) by SSR
2568	2694	* manner, so it should recover the existing segment information of valid blocks
2569	2695	*/
2570		-static void change_curseg(struct f2fs_sb_info *sbi, int type)
	2696	+static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2571	2697	{
2572	2698	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2573	2699	struct curseg_info *curseg = CURSEG_I(sbi, type);
..	..	@@ -2575,8 +2701,10 @@
2575	2701	struct f2fs_summary_block *sum_node;
2576	2702	struct page *sum_page;
2577	2703
2578		- write_sum_page(sbi, curseg->sum_blk,
2579		- GET_SUM_BLOCK(sbi, curseg->segno));
	2704	+ if (flush)
	2705	+ write_sum_page(sbi, curseg->sum_blk,
	2706	+ GET_SUM_BLOCK(sbi, curseg->segno));
	2707	+
2580	2708	__set_test_and_inuse(sbi, new_segno);
2581	2709
2582	2710	mutex_lock(&dirty_i->seglist_lock);
..	..	@@ -2586,32 +2714,142 @@
2586	2714
2587	2715	reset_curseg(sbi, type, 1);
2588	2716	curseg->alloc_type = SSR;
2589		- __next_free_blkoff(sbi, curseg, 0);
	2717	+ curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2590	2718
2591	2719	sum_page = f2fs_get_sum_page(sbi, new_segno);
2592		- f2fs_bug_on(sbi, IS_ERR(sum_page));
	2720	+ if (IS_ERR(sum_page)) {
	2721	+ /* GC won't be able to use stale summary pages by cp_error */
	2722	+ memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
	2723	+ return;
	2724	+ }
2593	2725	sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2594	2726	memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2595	2727	f2fs_put_page(sum_page, 1);
2596	2728	}
2597	2729
2598		-static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
	2730	+static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
	2731	+ int alloc_mode, unsigned long long age);
	2732	+
	2733	+static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
	2734	+ int target_type, int alloc_mode,
	2735	+ unsigned long long age)
	2736	+{
	2737	+ struct curseg_info *curseg = CURSEG_I(sbi, type);
	2738	+
	2739	+ curseg->seg_type = target_type;
	2740	+
	2741	+ if (get_ssr_segment(sbi, type, alloc_mode, age)) {
	2742	+ struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
	2743	+
	2744	+ curseg->seg_type = se->type;
	2745	+ change_curseg(sbi, type, true);
	2746	+ } else {
	2747	+ /* allocate cold segment by default */
	2748	+ curseg->seg_type = CURSEG_COLD_DATA;
	2749	+ new_curseg(sbi, type, true);
	2750	+ }
	2751	+ stat_inc_seg_type(sbi, curseg);
	2752	+}
	2753	+
	2754	+static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
	2755	+{
	2756	+ struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
	2757	+
	2758	+ if (!sbi->am.atgc_enabled)
	2759	+ return;
	2760	+
	2761	+ f2fs_down_read(&SM_I(sbi)->curseg_lock);
	2762	+
	2763	+ mutex_lock(&curseg->curseg_mutex);
	2764	+ down_write(&SIT_I(sbi)->sentry_lock);
	2765	+
	2766	+ get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
	2767	+
	2768	+ up_write(&SIT_I(sbi)->sentry_lock);
	2769	+ mutex_unlock(&curseg->curseg_mutex);
	2770	+
	2771	+ f2fs_up_read(&SM_I(sbi)->curseg_lock);
	2772	+
	2773	+}
	2774	+void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
	2775	+{
	2776	+ __f2fs_init_atgc_curseg(sbi);
	2777	+}
	2778	+
	2779	+static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
	2780	+{
	2781	+ struct curseg_info *curseg = CURSEG_I(sbi, type);
	2782	+
	2783	+ mutex_lock(&curseg->curseg_mutex);
	2784	+ if (!curseg->inited)
	2785	+ goto out;
	2786	+
	2787	+ if (get_valid_blocks(sbi, curseg->segno, false)) {
	2788	+ write_sum_page(sbi, curseg->sum_blk,
	2789	+ GET_SUM_BLOCK(sbi, curseg->segno));
	2790	+ } else {
	2791	+ mutex_lock(&DIRTY_I(sbi)->seglist_lock);
	2792	+ __set_test_and_free(sbi, curseg->segno, true);
	2793	+ mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
	2794	+ }
	2795	+out:
	2796	+ mutex_unlock(&curseg->curseg_mutex);
	2797	+}
	2798	+
	2799	+void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
	2800	+{
	2801	+ __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
	2802	+
	2803	+ if (sbi->am.atgc_enabled)
	2804	+ __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
	2805	+}
	2806	+
	2807	+static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
	2808	+{
	2809	+ struct curseg_info *curseg = CURSEG_I(sbi, type);
	2810	+
	2811	+ mutex_lock(&curseg->curseg_mutex);
	2812	+ if (!curseg->inited)
	2813	+ goto out;
	2814	+ if (get_valid_blocks(sbi, curseg->segno, false))
	2815	+ goto out;
	2816	+
	2817	+ mutex_lock(&DIRTY_I(sbi)->seglist_lock);
	2818	+ __set_test_and_inuse(sbi, curseg->segno);
	2819	+ mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
	2820	+out:
	2821	+ mutex_unlock(&curseg->curseg_mutex);
	2822	+}
	2823	+
	2824	+void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
	2825	+{
	2826	+ __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
	2827	+
	2828	+ if (sbi->am.atgc_enabled)
	2829	+ __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
	2830	+}
	2831	+
	2832	+static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
	2833	+ int alloc_mode, unsigned long long age)
2599	2834	{
2600	2835	struct curseg_info *curseg = CURSEG_I(sbi, type);
2601	2836	const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2602	2837	unsigned segno = NULL_SEGNO;
	2838	+ unsigned short seg_type = curseg->seg_type;
2603	2839	int i, cnt;
2604	2840	bool reversed = false;
2605	2841
	2842	+ sanity_check_seg_type(sbi, seg_type);
	2843	+
2606	2844	/* f2fs_need_SSR() already forces to do this */
2607		- if (v_ops->get_victim(sbi, &segno, BG_GC, type, SSR)) {
	2845	+ if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2608	2846	curseg->next_segno = segno;
2609	2847	return 1;
2610	2848	}
2611	2849
2612	2850	/* For node segments, let's do SSR more intensively */
2613		- if (IS_NODESEG(type)) {
2614		- if (type >= CURSEG_WARM_NODE) {
	2851	+ if (IS_NODESEG(seg_type)) {
	2852	+ if (seg_type >= CURSEG_WARM_NODE) {
2615	2853	reversed = true;
2616	2854	i = CURSEG_COLD_NODE;
2617	2855	} else {
..	..	@@ -2619,7 +2857,7 @@
2619	2857	}
2620	2858	cnt = NR_CURSEG_NODE_TYPE;
2621	2859	} else {
2622		- if (type >= CURSEG_WARM_DATA) {
	2860	+ if (seg_type >= CURSEG_WARM_DATA) {
2623	2861	reversed = true;
2624	2862	i = CURSEG_COLD_DATA;
2625	2863	} else {
..	..	@@ -2629,9 +2867,9 @@
2629	2867	}
2630	2868
2631	2869	for (; cnt-- > 0; reversed ? i-- : i++) {
2632		- if (i == type)
	2870	+ if (i == seg_type)
2633	2871	continue;
2634		- if (v_ops->get_victim(sbi, &segno, BG_GC, i, SSR)) {
	2872	+ if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2635	2873	curseg->next_segno = segno;
2636	2874	return 1;
2637	2875	}
..	..	@@ -2660,26 +2898,28 @@
2660	2898	if (force)
2661	2899	new_curseg(sbi, type, true);
2662	2900	else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2663		- type == CURSEG_WARM_NODE)
	2901	+ curseg->seg_type == CURSEG_WARM_NODE)
2664	2902	new_curseg(sbi, type, false);
2665		- else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type) &&
	2903	+ else if (curseg->alloc_type == LFS &&
	2904	+ is_next_segment_free(sbi, curseg, type) &&
2666	2905	likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2667	2906	new_curseg(sbi, type, false);
2668		- else if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type))
2669		- change_curseg(sbi, type);
	2907	+ else if (f2fs_need_SSR(sbi) &&
	2908	+ get_ssr_segment(sbi, type, SSR, 0))
	2909	+ change_curseg(sbi, type, true);
2670	2910	else
2671	2911	new_curseg(sbi, type, false);
2672	2912
2673	2913	stat_inc_seg_type(sbi, curseg);
2674	2914	}
2675	2915
2676		-void allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
	2916	+void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2677	2917	unsigned int start, unsigned int end)
2678	2918	{
2679	2919	struct curseg_info *curseg = CURSEG_I(sbi, type);
2680	2920	unsigned int segno;
2681	2921
2682		- down_read(&SM_I(sbi)->curseg_lock);
	2922	+ f2fs_down_read(&SM_I(sbi)->curseg_lock);
2683	2923	mutex_lock(&curseg->curseg_mutex);
2684	2924	down_write(&SIT_I(sbi)->sentry_lock);
2685	2925
..	..	@@ -2687,8 +2927,8 @@
2687	2927	if (segno < start \|\| segno > end)
2688	2928	goto unlock;
2689	2929
2690		- if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type))
2691		- change_curseg(sbi, type);
	2930	+ if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
	2931	+ change_curseg(sbi, type, true);
2692	2932	else
2693	2933	new_curseg(sbi, type, true);
2694	2934
..	..	@@ -2703,32 +2943,55 @@
2703	2943	type, segno, curseg->segno);
2704	2944
2705	2945	mutex_unlock(&curseg->curseg_mutex);
2706		- up_read(&SM_I(sbi)->curseg_lock);
	2946	+ f2fs_up_read(&SM_I(sbi)->curseg_lock);
2707	2947	}
2708	2948
2709		-void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi, int type)
	2949	+static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
	2950	+ bool new_sec, bool force)
2710	2951	{
2711		- struct curseg_info *curseg;
	2952	+ struct curseg_info *curseg = CURSEG_I(sbi, type);
2712	2953	unsigned int old_segno;
	2954	+
	2955	+ if (!curseg->inited)
	2956	+ goto alloc;
	2957	+
	2958	+ if (force \|\| curseg->next_blkoff \|\|
	2959	+ get_valid_blocks(sbi, curseg->segno, new_sec))
	2960	+ goto alloc;
	2961	+
	2962	+ if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
	2963	+ return;
	2964	+alloc:
	2965	+ old_segno = curseg->segno;
	2966	+ SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
	2967	+ locate_dirty_segment(sbi, old_segno);
	2968	+}
	2969	+
	2970	+static void __allocate_new_section(struct f2fs_sb_info *sbi,
	2971	+ int type, bool force)
	2972	+{
	2973	+ __allocate_new_segment(sbi, type, true, force);
	2974	+}
	2975	+
	2976	+void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
	2977	+{
	2978	+ f2fs_down_read(&SM_I(sbi)->curseg_lock);
	2979	+ down_write(&SIT_I(sbi)->sentry_lock);
	2980	+ __allocate_new_section(sbi, type, force);
	2981	+ up_write(&SIT_I(sbi)->sentry_lock);
	2982	+ f2fs_up_read(&SM_I(sbi)->curseg_lock);
	2983	+}
	2984	+
	2985	+void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
	2986	+{
2713	2987	int i;
2714	2988
	2989	+ f2fs_down_read(&SM_I(sbi)->curseg_lock);
2715	2990	down_write(&SIT_I(sbi)->sentry_lock);
2716		-
2717		- for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2718		- if (type != NO_CHECK_TYPE && i != type)
2719		- continue;
2720		-
2721		- curseg = CURSEG_I(sbi, i);
2722		- if (type == NO_CHECK_TYPE \|\| curseg->next_blkoff \|\|
2723		- get_valid_blocks(sbi, curseg->segno, false) \|\|
2724		- get_ckpt_valid_blocks(sbi, curseg->segno)) {
2725		- old_segno = curseg->segno;
2726		- SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
2727		- locate_dirty_segment(sbi, old_segno);
2728		- }
2729		- }
2730		-
	2991	+ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
	2992	+ __allocate_new_segment(sbi, i, false, false);
2731	2993	up_write(&SIT_I(sbi)->sentry_lock);
	2994	+ f2fs_up_read(&SM_I(sbi)->curseg_lock);
2732	2995	}
2733	2996
2734	2997	static const struct segment_allocation default_salloc_ops = {
..	..	@@ -2772,7 +3035,7 @@
2772	3035	mutex_lock(&dcc->cmd_lock);
2773	3036	if (unlikely(dcc->rbtree_check))
2774	3037	f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
2775		- &dcc->root));
	3038	+ &dcc->root, false));
2776	3039
2777	3040	dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
2778	3041	NULL, start,
..	..	@@ -2866,9 +3129,9 @@
2866	3129	if (sbi->discard_blks == 0)
2867	3130	goto out;
2868	3131
2869		- down_write(&sbi->gc_lock);
	3132	+ f2fs_down_write(&sbi->gc_lock);
2870	3133	err = f2fs_write_checkpoint(sbi, &cpc);
2871		- up_write(&sbi->gc_lock);
	3134	+ f2fs_up_write(&sbi->gc_lock);
2872	3135	if (err)
2873	3136	goto out;
2874	3137
..	..	@@ -2896,12 +3159,11 @@
2896	3159	return err;
2897	3160	}
2898	3161
2899		-static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
	3162	+static bool __has_curseg_space(struct f2fs_sb_info *sbi,
	3163	+ struct curseg_info *curseg)
2900	3164	{
2901		- struct curseg_info *curseg = CURSEG_I(sbi, type);
2902		- if (curseg->next_blkoff < sbi->blocks_per_seg)
2903		- return true;
2904		- return false;
	3165	+ return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
	3166	+ curseg->segno);
2905	3167	}
2906	3168
2907	3169	int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
..	..	@@ -3036,14 +3298,47 @@
3036	3298	}
3037	3299	}
3038	3300
	3301	+static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
	3302	+{
	3303	+ struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
	3304	+ struct extent_info ei = {};
	3305	+
	3306	+ if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) {
	3307	+ if (!ei.age)
	3308	+ return NO_CHECK_TYPE;
	3309	+ if (ei.age <= sbi->hot_data_age_threshold)
	3310	+ return CURSEG_HOT_DATA;
	3311	+ if (ei.age <= sbi->warm_data_age_threshold)
	3312	+ return CURSEG_WARM_DATA;
	3313	+ return CURSEG_COLD_DATA;
	3314	+ }
	3315	+ return NO_CHECK_TYPE;
	3316	+}
	3317	+
3039	3318	static int __get_segment_type_6(struct f2fs_io_info *fio)
3040	3319	{
3041	3320	if (fio->type == DATA) {
3042	3321	struct inode *inode = fio->page->mapping->host;
	3322	+ int type;
3043	3323
3044		- if (is_cold_data(fio->page) \|\| file_is_cold(inode) \|\|
3045		- f2fs_compressed_file(inode))
	3324	+ if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
	3325	+ return CURSEG_COLD_DATA_PINNED;
	3326	+
	3327	+ if (page_private_gcing(fio->page)) {
	3328	+ if (fio->sbi->am.atgc_enabled &&
	3329	+ (fio->io_type == FS_DATA_IO) &&
	3330	+ (fio->sbi->gc_mode != GC_URGENT_HIGH))
	3331	+ return CURSEG_ALL_DATA_ATGC;
	3332	+ else
	3333	+ return CURSEG_COLD_DATA;
	3334	+ }
	3335	+ if (file_is_cold(inode) \|\| f2fs_need_compress_data(inode))
3046	3336	return CURSEG_COLD_DATA;
	3337	+
	3338	+ type = __get_age_segment_type(inode, fio->page->index);
	3339	+ if (type != NO_CHECK_TYPE)
	3340	+ return type;
	3341	+
3047	3342	if (file_is_hot(inode) \|\|
3048	3343	is_inode_flag_set(inode, FI_HOT_DATA) \|\|
3049	3344	f2fs_is_atomic_file(inode) \|\|
..	..	@@ -3088,38 +3383,28 @@
3088	3383	void f2fs_allocate_data_block(struct f2fs_sb_info sbi, struct page page,
3089	3384	block_t old_blkaddr, block_t *new_blkaddr,
3090	3385	struct f2fs_summary *sum, int type,
3091		- struct f2fs_io_info *fio, bool add_list)
	3386	+ struct f2fs_io_info *fio)
3092	3387	{
3093	3388	struct sit_info *sit_i = SIT_I(sbi);
3094	3389	struct curseg_info *curseg = CURSEG_I(sbi, type);
3095		- bool put_pin_sem = false;
	3390	+ unsigned long long old_mtime;
	3391	+ bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
	3392	+ struct seg_entry *se = NULL;
3096	3393
3097		- if (type == CURSEG_COLD_DATA) {
3098		- /* GC during CURSEG_COLD_DATA_PINNED allocation */
3099		- if (down_read_trylock(&sbi->pin_sem)) {
3100		- put_pin_sem = true;
3101		- } else {
3102		- type = CURSEG_WARM_DATA;
3103		- curseg = CURSEG_I(sbi, type);
3104		- }
3105		- } else if (type == CURSEG_COLD_DATA_PINNED) {
3106		- type = CURSEG_COLD_DATA;
3107		- }
3108		-
3109		- /*
3110		- * We need to wait for node_write to avoid block allocation during
3111		- * checkpoint. This can only happen to quota writes which can cause
3112		- * the below discard race condition.
3113		- */
3114		- if (IS_DATASEG(type))
3115		- down_write(&sbi->node_write);
3116		-
3117		- down_read(&SM_I(sbi)->curseg_lock);
	3394	+ f2fs_down_read(&SM_I(sbi)->curseg_lock);
3118	3395
3119	3396	mutex_lock(&curseg->curseg_mutex);
3120	3397	down_write(&sit_i->sentry_lock);
3121	3398
	3399	+ if (from_gc) {
	3400	+ f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
	3401	+ se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
	3402	+ sanity_check_seg_type(sbi, se->type);
	3403	+ f2fs_bug_on(sbi, IS_NODESEG(se->type));
	3404	+ }
3122	3405	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
	3406	+
	3407	+ f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3123	3408
3124	3409	f2fs_wait_discard_bio(sbi, *new_blkaddr);
3125	3410
..	..	@@ -3134,6 +3419,14 @@
3134	3419
3135	3420	stat_inc_block_count(sbi, curseg);
3136	3421
	3422	+ if (from_gc) {
	3423	+ old_mtime = get_segment_mtime(sbi, old_blkaddr);
	3424	+ } else {
	3425	+ update_segment_mtime(sbi, old_blkaddr, 0);
	3426	+ old_mtime = 0;
	3427	+ }
	3428	+ update_segment_mtime(sbi, *new_blkaddr, old_mtime);
	3429	+
3137	3430	/*
3138	3431	* SIT information should be updated before segment allocation,
3139	3432	* since SSR needs latest valid block information.
..	..	@@ -3142,9 +3435,13 @@
3142	3435	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3143	3436	update_sit_entry(sbi, old_blkaddr, -1);
3144	3437
3145		- if (!__has_curseg_space(sbi, type))
3146		- sit_i->s_ops->allocate_segment(sbi, type, false);
3147		-
	3438	+ if (!__has_curseg_space(sbi, curseg)) {
	3439	+ if (from_gc)
	3440	+ get_atssr_segment(sbi, type, se->type,
	3441	+ AT_SSR, se->mtime);
	3442	+ else
	3443	+ sit_i->s_ops->allocate_segment(sbi, type, false);
	3444	+ }
3148	3445	/*
3149	3446	* segment dirty status should be updated after segment allocation,
3150	3447	* so we just need to update status only one time after previous
..	..	@@ -3152,6 +3449,9 @@
3152	3449	*/
3153	3450	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3154	3451	locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
	3452	+
	3453	+ if (IS_DATASEG(type))
	3454	+ atomic64_inc(&sbi->allocated_data_blocks);
3155	3455
3156	3456	up_write(&sit_i->sentry_lock);
3157	3457
..	..	@@ -3161,11 +3461,11 @@
3161	3461	f2fs_inode_chksum_set(sbi, page);
3162	3462	}
3163	3463
3164		- if (F2FS_IO_ALIGNED(sbi))
3165		- fio->retry = false;
3166		-
3167		- if (add_list) {
	3464	+ if (fio) {
3168	3465	struct f2fs_bio_info *io;
	3466	+
	3467	+ if (F2FS_IO_ALIGNED(sbi))
	3468	+ fio->retry = false;
3169	3469
3170	3470	INIT_LIST_HEAD(&fio->list);
3171	3471	fio->in_list = true;
..	..	@@ -3177,13 +3477,7 @@
3177	3477
3178	3478	mutex_unlock(&curseg->curseg_mutex);
3179	3479
3180		- up_read(&SM_I(sbi)->curseg_lock);
3181		-
3182		- if (IS_DATASEG(type))
3183		- up_write(&sbi->node_write);
3184		-
3185		- if (put_pin_sem)
3186		- up_read(&sbi->pin_sem);
	3480	+ f2fs_up_read(&SM_I(sbi)->curseg_lock);
3187	3481	}
3188	3482
3189	3483	static void update_device_state(struct f2fs_io_info *fio)
..	..	@@ -3213,13 +3507,15 @@
3213	3507	bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3214	3508
3215	3509	if (keep_order)
3216		- down_read(&fio->sbi->io_order_lock);
	3510	+ f2fs_down_read(&fio->sbi->io_order_lock);
3217	3511	reallocate:
3218	3512	f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3219		- &fio->new_blkaddr, sum, type, fio, true);
3220		- if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
	3513	+ &fio->new_blkaddr, sum, type, fio);
	3514	+ if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3221	3515	invalidate_mapping_pages(META_MAPPING(fio->sbi),
3222	3516	fio->old_blkaddr, fio->old_blkaddr);
	3517	+ f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
	3518	+ }
3223	3519
3224	3520	/* writeout dirty page into bdev */
3225	3521	f2fs_submit_page_write(fio);
..	..	@@ -3231,7 +3527,7 @@
3231	3527	update_device_state(fio);
3232	3528
3233	3529	if (keep_order)
3234		- up_read(&fio->sbi->io_order_lock);
	3530	+ f2fs_up_read(&fio->sbi->io_order_lock);
3235	3531	}
3236	3532
3237	3533	void f2fs_do_write_meta_page(struct f2fs_sb_info sbi, struct page page,
..	..	@@ -3278,6 +3574,8 @@
3278	3574	struct f2fs_summary sum;
3279	3575
3280	3576	f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
	3577	+ if (fio->io_type == FS_DATA_IO \|\| fio->io_type == FS_CP_DATA_IO)
	3578	+ f2fs_update_age_extent_cache(dn);
3281	3579	set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3282	3580	do_write_page(&sum, fio);
3283	3581	f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
..	..	@@ -3301,8 +3599,18 @@
3301	3599	set_sbi_flag(sbi, SBI_NEED_FSCK);
3302	3600	f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3303	3601	__func__, segno);
3304		- return -EFSCORRUPTED;
	3602	+ err = -EFSCORRUPTED;
	3603	+ goto drop_bio;
3305	3604	}
	3605	+
	3606	+ if (f2fs_cp_error(sbi)) {
	3607	+ err = -EIO;
	3608	+ goto drop_bio;
	3609	+ }
	3610	+
	3611	+ if (fio->post_read)
	3612	+ invalidate_mapping_pages(META_MAPPING(sbi),
	3613	+ fio->new_blkaddr, fio->new_blkaddr);
3306	3614
3307	3615	stat_inc_inplace_blocks(fio->sbi);
3308	3616
..	..	@@ -3315,6 +3623,15 @@
3315	3623	f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3316	3624	}
3317	3625
	3626	+ return err;
	3627	+drop_bio:
	3628	+ if (fio->bio && *(fio->bio)) {
	3629	+ struct bio bio = (fio->bio);
	3630	+
	3631	+ bio->bi_status = BLK_STS_IOERR;
	3632	+ bio_endio(bio);
	3633	+ *(fio->bio) = NULL;
	3634	+ }
3318	3635	return err;
3319	3636	}
3320	3637
..	..	@@ -3332,7 +3649,8 @@
3332	3649
3333	3650	void f2fs_do_replace_block(struct f2fs_sb_info sbi, struct f2fs_summary sum,
3334	3651	block_t old_blkaddr, block_t new_blkaddr,
3335		- bool recover_curseg, bool recover_newaddr)
	3652	+ bool recover_curseg, bool recover_newaddr,
	3653	+ bool from_gc)
3336	3654	{
3337	3655	struct sit_info *sit_i = SIT_I(sbi);
3338	3656	struct curseg_info *curseg;
..	..	@@ -3340,12 +3658,13 @@
3340	3658	struct seg_entry *se;
3341	3659	int type;
3342	3660	unsigned short old_blkoff;
	3661	+ unsigned char old_alloc_type;
3343	3662
3344	3663	segno = GET_SEGNO(sbi, new_blkaddr);
3345	3664	se = get_seg_entry(sbi, segno);
3346	3665	type = se->type;
3347	3666
3348		- down_write(&SM_I(sbi)->curseg_lock);
	3667	+ f2fs_down_write(&SM_I(sbi)->curseg_lock);
3349	3668
3350	3669	if (!recover_curseg) {
3351	3670	/* for recovery flow */
..	..	@@ -3373,21 +3692,28 @@
3373	3692
3374	3693	old_cursegno = curseg->segno;
3375	3694	old_blkoff = curseg->next_blkoff;
	3695	+ old_alloc_type = curseg->alloc_type;
3376	3696
3377	3697	/* change the current segment */
3378	3698	if (segno != curseg->segno) {
3379	3699	curseg->next_segno = segno;
3380		- change_curseg(sbi, type);
	3700	+ change_curseg(sbi, type, true);
3381	3701	}
3382	3702
3383	3703	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3384	3704	__add_sum_entry(sbi, type, sum);
3385	3705
3386		- if (!recover_curseg \|\| recover_newaddr)
	3706	+ if (!recover_curseg \|\| recover_newaddr) {
	3707	+ if (!from_gc)
	3708	+ update_segment_mtime(sbi, new_blkaddr, 0);
3387	3709	update_sit_entry(sbi, new_blkaddr, 1);
	3710	+ }
3388	3711	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3389	3712	invalidate_mapping_pages(META_MAPPING(sbi),
3390	3713	old_blkaddr, old_blkaddr);
	3714	+ f2fs_invalidate_compress_page(sbi, old_blkaddr);
	3715	+ if (!from_gc)
	3716	+ update_segment_mtime(sbi, old_blkaddr, 0);
3391	3717	update_sit_entry(sbi, old_blkaddr, -1);
3392	3718	}
3393	3719
..	..	@@ -3399,14 +3725,15 @@
3399	3725	if (recover_curseg) {
3400	3726	if (old_cursegno != curseg->segno) {
3401	3727	curseg->next_segno = old_cursegno;
3402		- change_curseg(sbi, type);
	3728	+ change_curseg(sbi, type, true);
3403	3729	}
3404	3730	curseg->next_blkoff = old_blkoff;
	3731	+ curseg->alloc_type = old_alloc_type;
3405	3732	}
3406	3733
3407	3734	up_write(&sit_i->sentry_lock);
3408	3735	mutex_unlock(&curseg->curseg_mutex);
3409		- up_write(&SM_I(sbi)->curseg_lock);
	3736	+ f2fs_up_write(&SM_I(sbi)->curseg_lock);
3410	3737	}
3411	3738
3412	3739	void f2fs_replace_block(struct f2fs_sb_info sbi, struct dnode_of_data dn,
..	..	@@ -3419,7 +3746,7 @@
3419	3746	set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3420	3747
3421	3748	f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3422		- recover_curseg, recover_newaddr);
	3749	+ recover_curseg, recover_newaddr, false);
3423	3750
3424	3751	f2fs_update_data_blkaddr(dn, new_addr);
3425	3752	}
..	..	@@ -3464,10 +3791,16 @@
3464	3791	void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3465	3792	block_t len)
3466	3793	{
	3794	+ struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3467	3795	block_t i;
	3796	+
	3797	+ if (!f2fs_post_read_required(inode))
	3798	+ return;
3468	3799
3469	3800	for (i = 0; i < len; i++)
3470	3801	f2fs_wait_on_block_writeback(inode, blkaddr + i);
	3802	+
	3803	+ invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1);
3471	3804	}
3472	3805
3473	3806	static int read_compacted_summaries(struct f2fs_sb_info *sbi)
..	..	@@ -3513,6 +3846,7 @@
3513	3846
3514	3847	for (j = 0; j < blk_off; j++) {
3515	3848	struct f2fs_summary *s;
	3849	+
3516	3850	s = (struct f2fs_summary *)(kaddr + offset);
3517	3851	seg_i->sum_blk->entries[j] = *s;
3518	3852	offset += SUMMARY_SIZE;
..	..	@@ -3551,7 +3885,7 @@
3551	3885	blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3552	3886	CURSEG_HOT_DATA]);
3553	3887	if (__exist_node_summaries(sbi))
3554		- blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
	3888	+ blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3555	3889	else
3556	3890	blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3557	3891	} else {
..	..	@@ -3575,6 +3909,7 @@
3575	3909	if (__exist_node_summaries(sbi)) {
3576	3910	struct f2fs_summary *ns = &sum->entries[0];
3577	3911	int i;
	3912	+
3578	3913	for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3579	3914	ns->version = 0;
3580	3915	ns->ofs_in_node = 0;
..	..	@@ -3629,8 +3964,9 @@
3629	3964	}
3630	3965
3631	3966	if (__exist_node_summaries(sbi))
3632		- f2fs_ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
3633		- NR_CURSEG_TYPE - type, META_CP, true);
	3967	+ f2fs_ra_meta_pages(sbi,
	3968	+ sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
	3969	+ NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
3634	3970
3635	3971	for (; type <= CURSEG_COLD_NODE; type++) {
3636	3972	err = read_normal_summaries(sbi, type);
..	..	@@ -3641,7 +3977,7 @@
3641	3977	/* sanity check for summary blocks */
3642	3978	if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES \|\|
3643	3979	sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3644		- f2fs_err(sbi, "invalid journal entries nats %u sits %u\n",
	3980	+ f2fs_err(sbi, "invalid journal entries nats %u sits %u",
3645	3981	nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3646	3982	return -EINVAL;
3647	3983	}
..	..	@@ -3675,6 +4011,7 @@
3675	4011	/* Step 3: write summary entries */
3676	4012	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3677	4013	unsigned short blkoff;
	4014	+
3678	4015	seg_i = CURSEG_I(sbi, i);
3679	4016	if (sbi->ckpt->alloc_type[i] == SSR)
3680	4017	blkoff = sbi->blocks_per_seg;
..	..	@@ -3711,6 +4048,7 @@
3711	4048	block_t blkaddr, int type)
3712	4049	{
3713	4050	int i, end;
	4051	+
3714	4052	if (IS_DATASEG(type))
3715	4053	end = type + NR_CURSEG_DATA_TYPE;
3716	4054	else
..	..	@@ -3758,7 +4096,7 @@
3758	4096	static struct page get_current_sit_page(struct f2fs_sb_info sbi,
3759	4097	unsigned int segno)
3760	4098	{
3761		- return f2fs_get_meta_page_nofail(sbi, current_sit_addr(sbi, segno));
	4099	+ return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
3762	4100	}
3763	4101
3764	4102	static struct page get_next_sit_page(struct f2fs_sb_info sbi,
..	..	@@ -4132,14 +4470,14 @@
4132	4470	struct curseg_info *array;
4133	4471	int i;
4134	4472
4135		- array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, sizeof(*array)),
4136		- GFP_KERNEL);
	4473	+ array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
	4474	+ sizeof(*array)), GFP_KERNEL);
4137	4475	if (!array)
4138	4476	return -ENOMEM;
4139	4477
4140	4478	SM_I(sbi)->curseg_array = array;
4141	4479
4142		- for (i = 0; i < NR_CURSEG_TYPE; i++) {
	4480	+ for (i = 0; i < NO_CHECK_TYPE; i++) {
4143	4481	mutex_init(&array[i].curseg_mutex);
4144	4482	array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4145	4483	if (!array[i].sum_blk)
..	..	@@ -4149,8 +4487,15 @@
4149	4487	sizeof(struct f2fs_journal), GFP_KERNEL);
4150	4488	if (!array[i].journal)
4151	4489	return -ENOMEM;
	4490	+ if (i < NR_PERSISTENT_LOG)
	4491	+ array[i].seg_type = CURSEG_HOT_DATA + i;
	4492	+ else if (i == CURSEG_COLD_DATA_PINNED)
	4493	+ array[i].seg_type = CURSEG_COLD_DATA;
	4494	+ else if (i == CURSEG_ALL_DATA_ATGC)
	4495	+ array[i].seg_type = CURSEG_COLD_DATA;
4152	4496	array[i].segno = NULL_SEGNO;
4153	4497	array[i].next_blkoff = 0;
	4498	+ array[i].inited = false;
4154	4499	}
4155	4500	return restore_curseg_summaries(sbi);
4156	4501	}
..	..	@@ -4166,7 +4511,7 @@
4166	4511	unsigned int i, start, end;
4167	4512	unsigned int readed, start_blk = 0;
4168	4513	int err = 0;
4169		- block_t total_node_blocks = 0;
	4514	+ block_t sit_valid_blocks[2] = {0, 0};
4170	4515
4171	4516	do {
4172	4517	readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
..	..	@@ -4191,8 +4536,14 @@
4191	4536	if (err)
4192	4537	return err;
4193	4538	seg_info_from_raw_sit(se, &sit);
4194		- if (IS_NODESEG(se->type))
4195		- total_node_blocks += se->valid_blocks;
	4539	+
	4540	+ if (se->type >= NR_PERSISTENT_LOG) {
	4541	+ f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
	4542	+ se->type, start);
	4543	+ return -EFSCORRUPTED;
	4544	+ }
	4545	+
	4546	+ sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4196	4547
4197	4548	/* build discard map only one time */
4198	4549	if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
..	..	@@ -4230,15 +4581,22 @@
4230	4581	sit = sit_in_journal(journal, i);
4231	4582
4232	4583	old_valid_blocks = se->valid_blocks;
4233		- if (IS_NODESEG(se->type))
4234		- total_node_blocks -= old_valid_blocks;
	4584	+
	4585	+ sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4235	4586
4236	4587	err = check_block_count(sbi, start, &sit);
4237	4588	if (err)
4238	4589	break;
4239	4590	seg_info_from_raw_sit(se, &sit);
4240		- if (IS_NODESEG(se->type))
4241		- total_node_blocks += se->valid_blocks;
	4591	+
	4592	+ if (se->type >= NR_PERSISTENT_LOG) {
	4593	+ f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
	4594	+ se->type, start);
	4595	+ err = -EFSCORRUPTED;
	4596	+ break;
	4597	+ }
	4598	+
	4599	+ sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4242	4600
4243	4601	if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4244	4602	memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
..	..	@@ -4258,22 +4616,36 @@
4258	4616	}
4259	4617	up_read(&curseg->journal_rwsem);
4260	4618
4261		- if (!err && total_node_blocks != valid_node_count(sbi)) {
	4619	+ if (err)
	4620	+ return err;
	4621	+
	4622	+ if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
4262	4623	f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4263		- total_node_blocks, valid_node_count(sbi));
4264		- err = -EFSCORRUPTED;
	4624	+ sit_valid_blocks[NODE], valid_node_count(sbi));
	4625	+ return -EFSCORRUPTED;
4265	4626	}
4266	4627
4267		- return err;
	4628	+ if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
	4629	+ valid_user_blocks(sbi)) {
	4630	+ f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
	4631	+ sit_valid_blocks[DATA], sit_valid_blocks[NODE],
	4632	+ valid_user_blocks(sbi));
	4633	+ return -EFSCORRUPTED;
	4634	+ }
	4635	+
	4636	+ return 0;
4268	4637	}
4269	4638
4270	4639	static void init_free_segmap(struct f2fs_sb_info *sbi)
4271	4640	{
4272	4641	unsigned int start;
4273	4642	int type;
	4643	+ struct seg_entry *sentry;
4274	4644
4275	4645	for (start = 0; start < MAIN_SEGS(sbi); start++) {
4276		- struct seg_entry *sentry = get_seg_entry(sbi, start);
	4646	+ if (f2fs_usable_blks_in_seg(sbi, start) == 0)
	4647	+ continue;
	4648	+ sentry = get_seg_entry(sbi, start);
4277	4649	if (!sentry->valid_blocks)
4278	4650	__set_free(sbi, start);
4279	4651	else
..	..	@@ -4284,6 +4656,7 @@
4284	4656	/* set use the current segments */
4285	4657	for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4286	4658	struct curseg_info *curseg_t = CURSEG_I(sbi, type);
	4659	+
4287	4660	__set_test_and_inuse(sbi, curseg_t->segno);
4288	4661	}
4289	4662	}
..	..	@@ -4292,8 +4665,9 @@
4292	4665	{
4293	4666	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4294	4667	struct free_segmap_info *free_i = FREE_I(sbi);
4295		- unsigned int segno = 0, offset = 0;
4296		- unsigned short valid_blocks;
	4668	+ unsigned int segno = 0, offset = 0, secno;
	4669	+ block_t valid_blocks, usable_blks_in_seg;
	4670	+ block_t blks_per_sec = BLKS_PER_SEC(sbi);
4297	4671
4298	4672	while (1) {
4299	4673	/* find dirty segment based on free segmap */
..	..	@@ -4302,9 +4676,10 @@
4302	4676	break;
4303	4677	offset = segno + 1;
4304	4678	valid_blocks = get_valid_blocks(sbi, segno, false);
4305		- if (valid_blocks == sbi->blocks_per_seg \|\| !valid_blocks)
	4679	+ usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
	4680	+ if (valid_blocks == usable_blks_in_seg \|\| !valid_blocks)
4306	4681	continue;
4307		- if (valid_blocks > sbi->blocks_per_seg) {
	4682	+ if (valid_blocks > usable_blks_in_seg) {
4308	4683	f2fs_bug_on(sbi, 1);
4309	4684	continue;
4310	4685	}
..	..	@@ -4312,6 +4687,22 @@
4312	4687	__locate_dirty_segment(sbi, segno, DIRTY);
4313	4688	mutex_unlock(&dirty_i->seglist_lock);
4314	4689	}
	4690	+
	4691	+ if (!__is_large_section(sbi))
	4692	+ return;
	4693	+
	4694	+ mutex_lock(&dirty_i->seglist_lock);
	4695	+ for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
	4696	+ valid_blocks = get_valid_blocks(sbi, segno, true);
	4697	+ secno = GET_SEC_FROM_SEG(sbi, segno);
	4698	+
	4699	+ if (!valid_blocks \|\| valid_blocks == blks_per_sec)
	4700	+ continue;
	4701	+ if (IS_CURSEC(sbi, secno))
	4702	+ continue;
	4703	+ set_bit(secno, dirty_i->dirty_secmap);
	4704	+ }
	4705	+ mutex_unlock(&dirty_i->seglist_lock);
4315	4706	}
4316	4707
4317	4708	static int init_victim_secmap(struct f2fs_sb_info *sbi)
..	..	@@ -4322,6 +4713,13 @@
4322	4713	dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4323	4714	if (!dirty_i->victim_secmap)
4324	4715	return -ENOMEM;
	4716	+
	4717	+ dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
	4718	+ if (!dirty_i->pinned_secmap)
	4719	+ return -ENOMEM;
	4720	+
	4721	+ dirty_i->pinned_secmap_cnt = 0;
	4722	+ dirty_i->enable_pin_section = true;
4325	4723	return 0;
4326	4724	}
4327	4725
..	..	@@ -4348,6 +4746,14 @@
4348	4746	return -ENOMEM;
4349	4747	}
4350	4748
	4749	+ if (__is_large_section(sbi)) {
	4750	+ bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
	4751	+ dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
	4752	+ bitmap_size, GFP_KERNEL);
	4753	+ if (!dirty_i->dirty_secmap)
	4754	+ return -ENOMEM;
	4755	+ }
	4756	+
4351	4757	init_dirty_segmap(sbi);
4352	4758	return init_victim_secmap(sbi);
4353	4759	}
..	..	@@ -4360,10 +4766,23 @@
4360	4766	* In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4361	4767	* In LFS curseg, all blkaddr after .next_blkoff should be unused.
4362	4768	*/
4363		- for (i = 0; i < NO_CHECK_TYPE; i++) {
	4769	+ for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4364	4770	struct curseg_info *curseg = CURSEG_I(sbi, i);
4365	4771	struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4366	4772	unsigned int blkofs = curseg->next_blkoff;
	4773	+
	4774	+ if (f2fs_sb_has_readonly(sbi) &&
	4775	+ i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
	4776	+ continue;
	4777	+
	4778	+ sanity_check_seg_type(sbi, curseg->seg_type);
	4779	+
	4780	+ if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
	4781	+ f2fs_err(sbi,
	4782	+ "Current segment has invalid alloc_type:%d",
	4783	+ curseg->alloc_type);
	4784	+ return -EFSCORRUPTED;
	4785	+ }
4367	4786
4368	4787	if (f2fs_test_bit(blkofs, se->cur_valid_map))
4369	4788	goto out;
..	..	@@ -4383,6 +4802,327 @@
4383	4802	}
4384	4803	}
4385	4804	return 0;
	4805	+}
	4806	+
	4807	+#ifdef CONFIG_BLK_DEV_ZONED
	4808	+
	4809	+static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
	4810	+ struct f2fs_dev_info *fdev,
	4811	+ struct blk_zone *zone)
	4812	+{
	4813	+ unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
	4814	+ block_t zone_block, wp_block, last_valid_block;
	4815	+ unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
	4816	+ int i, s, b, ret;
	4817	+ struct seg_entry *se;
	4818	+
	4819	+ if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
	4820	+ return 0;
	4821	+
	4822	+ wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
	4823	+ wp_segno = GET_SEGNO(sbi, wp_block);
	4824	+ wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
	4825	+ zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
	4826	+ zone_segno = GET_SEGNO(sbi, zone_block);
	4827	+ zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
	4828	+
	4829	+ if (zone_segno >= MAIN_SEGS(sbi))
	4830	+ return 0;
	4831	+
	4832	+ /*
	4833	+ * Skip check of zones cursegs point to, since
	4834	+ * fix_curseg_write_pointer() checks them.
	4835	+ */
	4836	+ for (i = 0; i < NO_CHECK_TYPE; i++)
	4837	+ if (zone_secno == GET_SEC_FROM_SEG(sbi,
	4838	+ CURSEG_I(sbi, i)->segno))
	4839	+ return 0;
	4840	+
	4841	+ /*
	4842	+ * Get last valid block of the zone.
	4843	+ */
	4844	+ last_valid_block = zone_block - 1;
	4845	+ for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
	4846	+ segno = zone_segno + s;
	4847	+ se = get_seg_entry(sbi, segno);
	4848	+ for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
	4849	+ if (f2fs_test_bit(b, se->cur_valid_map)) {
	4850	+ last_valid_block = START_BLOCK(sbi, segno) + b;
	4851	+ break;
	4852	+ }
	4853	+ if (last_valid_block >= zone_block)
	4854	+ break;
	4855	+ }
	4856	+
	4857	+ /*
	4858	+ * If last valid block is beyond the write pointer, report the
	4859	+ * inconsistency. This inconsistency does not cause write error
	4860	+ * because the zone will not be selected for write operation until
	4861	+ * it get discarded. Just report it.
	4862	+ */
	4863	+ if (last_valid_block >= wp_block) {
	4864	+ f2fs_notice(sbi, "Valid block beyond write pointer: "
	4865	+ "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
	4866	+ GET_SEGNO(sbi, last_valid_block),
	4867	+ GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
	4868	+ wp_segno, wp_blkoff);
	4869	+ return 0;
	4870	+ }
	4871	+
	4872	+ /*
	4873	+ * If there is no valid block in the zone and if write pointer is
	4874	+ * not at zone start, reset the write pointer.
	4875	+ */
	4876	+ if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
	4877	+ f2fs_notice(sbi,
	4878	+ "Zone without valid block has non-zero write "
	4879	+ "pointer. Reset the write pointer: wp[0x%x,0x%x]",
	4880	+ wp_segno, wp_blkoff);
	4881	+ ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
	4882	+ zone->len >> log_sectors_per_block);
	4883	+ if (ret) {
	4884	+ f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
	4885	+ fdev->path, ret);
	4886	+ return ret;
	4887	+ }
	4888	+ }
	4889	+
	4890	+ return 0;
	4891	+}
	4892	+
	4893	+static struct f2fs_dev_info get_target_zoned_dev(struct f2fs_sb_info sbi,
	4894	+ block_t zone_blkaddr)
	4895	+{
	4896	+ int i;
	4897	+
	4898	+ for (i = 0; i < sbi->s_ndevs; i++) {
	4899	+ if (!bdev_is_zoned(FDEV(i).bdev))
	4900	+ continue;
	4901	+ if (sbi->s_ndevs == 1 \|\| (FDEV(i).start_blk <= zone_blkaddr &&
	4902	+ zone_blkaddr <= FDEV(i).end_blk))
	4903	+ return &FDEV(i);
	4904	+ }
	4905	+
	4906	+ return NULL;
	4907	+}
	4908	+
	4909	+static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
	4910	+ void *data)
	4911	+{
	4912	+ memcpy(data, zone, sizeof(struct blk_zone));
	4913	+ return 0;
	4914	+}
	4915	+
	4916	+static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
	4917	+{
	4918	+ struct curseg_info *cs = CURSEG_I(sbi, type);
	4919	+ struct f2fs_dev_info *zbd;
	4920	+ struct blk_zone zone;
	4921	+ unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
	4922	+ block_t cs_zone_block, wp_block;
	4923	+ unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
	4924	+ sector_t zone_sector;
	4925	+ int err;
	4926	+
	4927	+ cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
	4928	+ cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
	4929	+
	4930	+ zbd = get_target_zoned_dev(sbi, cs_zone_block);
	4931	+ if (!zbd)
	4932	+ return 0;
	4933	+
	4934	+ /* report zone for the sector the curseg points to */
	4935	+ zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
	4936	+ << log_sectors_per_block;
	4937	+ err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
	4938	+ report_one_zone_cb, &zone);
	4939	+ if (err != 1) {
	4940	+ f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
	4941	+ zbd->path, err);
	4942	+ return err;
	4943	+ }
	4944	+
	4945	+ if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
	4946	+ return 0;
	4947	+
	4948	+ wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
	4949	+ wp_segno = GET_SEGNO(sbi, wp_block);
	4950	+ wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
	4951	+ wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
	4952	+
	4953	+ if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
	4954	+ wp_sector_off == 0)
	4955	+ return 0;
	4956	+
	4957	+ f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
	4958	+ "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
	4959	+ type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
	4960	+
	4961	+ f2fs_notice(sbi, "Assign new section to curseg[%d]: "
	4962	+ "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
	4963	+
	4964	+ f2fs_allocate_new_section(sbi, type, true);
	4965	+
	4966	+ /* check consistency of the zone curseg pointed to */
	4967	+ if (check_zone_write_pointer(sbi, zbd, &zone))
	4968	+ return -EIO;
	4969	+
	4970	+ /* check newly assigned zone */
	4971	+ cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
	4972	+ cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
	4973	+
	4974	+ zbd = get_target_zoned_dev(sbi, cs_zone_block);
	4975	+ if (!zbd)
	4976	+ return 0;
	4977	+
	4978	+ zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
	4979	+ << log_sectors_per_block;
	4980	+ err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
	4981	+ report_one_zone_cb, &zone);
	4982	+ if (err != 1) {
	4983	+ f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
	4984	+ zbd->path, err);
	4985	+ return err;
	4986	+ }
	4987	+
	4988	+ if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
	4989	+ return 0;
	4990	+
	4991	+ if (zone.wp != zone.start) {
	4992	+ f2fs_notice(sbi,
	4993	+ "New zone for curseg[%d] is not yet discarded. "
	4994	+ "Reset the zone: curseg[0x%x,0x%x]",
	4995	+ type, cs->segno, cs->next_blkoff);
	4996	+ err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
	4997	+ zone_sector >> log_sectors_per_block,
	4998	+ zone.len >> log_sectors_per_block);
	4999	+ if (err) {
	5000	+ f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
	5001	+ zbd->path, err);
	5002	+ return err;
	5003	+ }
	5004	+ }
	5005	+
	5006	+ return 0;
	5007	+}
	5008	+
	5009	+int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
	5010	+{
	5011	+ int i, ret;
	5012	+
	5013	+ for (i = 0; i < NR_PERSISTENT_LOG; i++) {
	5014	+ ret = fix_curseg_write_pointer(sbi, i);
	5015	+ if (ret)
	5016	+ return ret;
	5017	+ }
	5018	+
	5019	+ return 0;
	5020	+}
	5021	+
	5022	+struct check_zone_write_pointer_args {
	5023	+ struct f2fs_sb_info *sbi;
	5024	+ struct f2fs_dev_info *fdev;
	5025	+};
	5026	+
	5027	+static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
	5028	+ void *data)
	5029	+{
	5030	+ struct check_zone_write_pointer_args *args;
	5031	+
	5032	+ args = (struct check_zone_write_pointer_args *)data;
	5033	+
	5034	+ return check_zone_write_pointer(args->sbi, args->fdev, zone);
	5035	+}
	5036	+
	5037	+int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
	5038	+{
	5039	+ int i, ret;
	5040	+ struct check_zone_write_pointer_args args;
	5041	+
	5042	+ for (i = 0; i < sbi->s_ndevs; i++) {
	5043	+ if (!bdev_is_zoned(FDEV(i).bdev))
	5044	+ continue;
	5045	+
	5046	+ args.sbi = sbi;
	5047	+ args.fdev = &FDEV(i);
	5048	+ ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
	5049	+ check_zone_write_pointer_cb, &args);
	5050	+ if (ret < 0)
	5051	+ return ret;
	5052	+ }
	5053	+
	5054	+ return 0;
	5055	+}
	5056	+
	5057	+/*
	5058	+ * Return the number of usable blocks in a segment. The number of blocks
	5059	+ * returned is always equal to the number of blocks in a segment for
	5060	+ * segments fully contained within a sequential zone capacity or a
	5061	+ * conventional zone. For segments partially contained in a sequential
	5062	+ * zone capacity, the number of usable blocks up to the zone capacity
	5063	+ * is returned. 0 is returned in all other cases.
	5064	+ */
	5065	+static inline unsigned int f2fs_usable_zone_blks_in_seg(
	5066	+ struct f2fs_sb_info *sbi, unsigned int segno)
	5067	+{
	5068	+ block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
	5069	+ unsigned int secno;
	5070	+
	5071	+ if (!sbi->unusable_blocks_per_sec)
	5072	+ return sbi->blocks_per_seg;
	5073	+
	5074	+ secno = GET_SEC_FROM_SEG(sbi, segno);
	5075	+ seg_start = START_BLOCK(sbi, segno);
	5076	+ sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
	5077	+ sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
	5078	+
	5079	+ /*
	5080	+ * If segment starts before zone capacity and spans beyond
	5081	+ * zone capacity, then usable blocks are from seg start to
	5082	+ * zone capacity. If the segment starts after the zone capacity,
	5083	+ * then there are no usable blocks.
	5084	+ */
	5085	+ if (seg_start >= sec_cap_blkaddr)
	5086	+ return 0;
	5087	+ if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
	5088	+ return sec_cap_blkaddr - seg_start;
	5089	+
	5090	+ return sbi->blocks_per_seg;
	5091	+}
	5092	+#else
	5093	+int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
	5094	+{
	5095	+ return 0;
	5096	+}
	5097	+
	5098	+int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
	5099	+{
	5100	+ return 0;
	5101	+}
	5102	+
	5103	+static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
	5104	+ unsigned int segno)
	5105	+{
	5106	+ return 0;
	5107	+}
	5108	+
	5109	+#endif
	5110	+unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
	5111	+ unsigned int segno)
	5112	+{
	5113	+ if (f2fs_sb_has_blkzoned(sbi))
	5114	+ return f2fs_usable_zone_blks_in_seg(sbi, segno);
	5115	+
	5116	+ return sbi->blocks_per_seg;
	5117	+}
	5118	+
	5119	+unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
	5120	+ unsigned int segno)
	5121	+{
	5122	+ if (f2fs_sb_has_blkzoned(sbi))
	5123	+ return CAP_SEGS_PER_SEC(sbi);
	5124	+
	5125	+ return sbi->segs_per_sec;
4386	5126	}
4387	5127
4388	5128	/*
..	..	@@ -4410,6 +5150,7 @@
4410	5150	sit_i->min_mtime = mtime;
4411	5151	}
4412	5152	sit_i->max_mtime = get_mtime(sbi, false);
	5153	+ sit_i->dirty_max_mtime = 0;
4413	5154	up_write(&sit_i->sentry_lock);
4414	5155	}
4415	5156
..	..	@@ -4448,7 +5189,7 @@
4448	5189
4449	5190	INIT_LIST_HEAD(&sm_info->sit_entry_set);
4450	5191
4451		- init_rwsem(&sm_info->curseg_lock);
	5192	+ init_f2fs_rwsem(&sm_info->curseg_lock);
4452	5193
4453	5194	if (!f2fs_readonly(sbi->sb)) {
4454	5195	err = f2fs_create_flush_cmd_control(sbi);
..	..	@@ -4502,6 +5243,8 @@
4502	5243	static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
4503	5244	{
4504	5245	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
	5246	+
	5247	+ kvfree(dirty_i->pinned_secmap);
4505	5248	kvfree(dirty_i->victim_secmap);
4506	5249	}
4507	5250
..	..	@@ -4517,9 +5260,15 @@
4517	5260	for (i = 0; i < NR_DIRTY_TYPE; i++)
4518	5261	discard_dirty_segmap(sbi, i);
4519	5262
	5263	+ if (__is_large_section(sbi)) {
	5264	+ mutex_lock(&dirty_i->seglist_lock);
	5265	+ kvfree(dirty_i->dirty_secmap);
	5266	+ mutex_unlock(&dirty_i->seglist_lock);
	5267	+ }
	5268	+
4520	5269	destroy_victim_secmap(sbi);
4521	5270	SM_I(sbi)->dirty_info = NULL;
4522		- kvfree(dirty_i);
	5271	+ kfree(dirty_i);
4523	5272	}
4524	5273
4525	5274	static void destroy_curseg(struct f2fs_sb_info *sbi)
..	..	@@ -4531,21 +5280,22 @@
4531	5280	return;
4532	5281	SM_I(sbi)->curseg_array = NULL;
4533	5282	for (i = 0; i < NR_CURSEG_TYPE; i++) {
4534		- kvfree(array[i].sum_blk);
4535		- kvfree(array[i].journal);
	5283	+ kfree(array[i].sum_blk);
	5284	+ kfree(array[i].journal);
4536	5285	}
4537		- kvfree(array);
	5286	+ kfree(array);
4538	5287	}
4539	5288
4540	5289	static void destroy_free_segmap(struct f2fs_sb_info *sbi)
4541	5290	{
4542	5291	struct free_segmap_info *free_i = SM_I(sbi)->free_info;
	5292	+
4543	5293	if (!free_i)
4544	5294	return;
4545	5295	SM_I(sbi)->free_info = NULL;
4546	5296	kvfree(free_i->free_segmap);
4547	5297	kvfree(free_i->free_secmap);
4548		- kvfree(free_i);
	5298	+ kfree(free_i);
4549	5299	}
4550	5300
4551	5301	static void destroy_sit_info(struct f2fs_sb_info *sbi)
..	..	@@ -4557,7 +5307,7 @@
4557	5307
4558	5308	if (sit_i->sentries)
4559	5309	kvfree(sit_i->bitmap);
4560		- kvfree(sit_i->tmp_map);
	5310	+ kfree(sit_i->tmp_map);
4561	5311
4562	5312	kvfree(sit_i->sentries);
4563	5313	kvfree(sit_i->sec_entries);
..	..	@@ -4569,7 +5319,7 @@
4569	5319	kvfree(sit_i->sit_bitmap_mir);
4570	5320	kvfree(sit_i->invalid_segmap);
4571	5321	#endif
4572		- kvfree(sit_i);
	5322	+ kfree(sit_i);
4573	5323	}
4574	5324
4575	5325	void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
..	..	@@ -4585,7 +5335,7 @@
4585	5335	destroy_free_segmap(sbi);
4586	5336	destroy_sit_info(sbi);
4587	5337	sbi->sm_info = NULL;
4588		- kvfree(sm_info);
	5338	+ kfree(sm_info);
4589	5339	}
4590	5340
4591	5341	int __init f2fs_create_segment_manager_caches(void)