~hc/RK356X_SDK_RELEASE.git

..	..	@@ -21,23 +21,33 @@
21	21	#include "gc.h"
22	22	#include <trace/events/f2fs.h>
23	23
	24	+static struct kmem_cache *victim_entry_slab;
	25	+
	26	+static unsigned int count_bits(const unsigned long *addr,
	27	+ unsigned int offset, unsigned int len);
	28	+
24	29	static int gc_thread_func(void *data)
25	30	{
26	31	struct f2fs_sb_info *sbi = data;
27	32	struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
28	33	wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head;
	34	+ wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq;
29	35	unsigned int wait_ms;
30	36
31	37	wait_ms = gc_th->min_sleep_time;
32	38
33	39	set_freezable();
34	40	do {
35		- bool sync_mode;
	41	+ bool sync_mode, foreground = false;
36	42
37	43	wait_event_interruptible_timeout(*wq,
38	44	kthread_should_stop() \|\| freezing(current) \|\|
	45	+ waitqueue_active(fggc_wq) \|\|
39	46	gc_th->gc_wake,
40	47	msecs_to_jiffies(wait_ms));
	48	+
	49	+ if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq))
	50	+ foreground = true;
41	51
42	52	/* give it a try one time */
43	53	if (gc_th->gc_wake)
..	..	@@ -58,7 +68,8 @@
58	68
59	69	if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
60	70	f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
61		- f2fs_stop_checkpoint(sbi, false);
	71	+ f2fs_stop_checkpoint(sbi, false,
	72	+ STOP_CP_REASON_FAULT_INJECT);
62	73	}
63	74
64	75	if (!sb_start_write_trylock(sbi->sb)) {
..	..	@@ -79,20 +90,23 @@
79	90	* invalidated soon after by user update or deletion.
80	91	* So, I'd like to wait some time to collect dirty segments.
81	92	*/
82		- if (sbi->gc_mode == GC_URGENT) {
	93	+ if (sbi->gc_mode == GC_URGENT_HIGH) {
83	94	wait_ms = gc_th->urgent_sleep_time;
84		- down_write(&sbi->gc_lock);
	95	+ f2fs_down_write(&sbi->gc_lock);
85	96	goto do_gc;
86	97	}
87	98
88		- if (!down_write_trylock(&sbi->gc_lock)) {
	99	+ if (foreground) {
	100	+ f2fs_down_write(&sbi->gc_lock);
	101	+ goto do_gc;
	102	+ } else if (!f2fs_down_write_trylock(&sbi->gc_lock)) {
89	103	stat_other_skip_bggc_count(sbi);
90	104	goto next;
91	105	}
92	106
93	107	if (!is_idle(sbi, GC_TIME)) {
94	108	increase_sleep_time(gc_th, &wait_ms);
95		- up_write(&sbi->gc_lock);
	109	+ f2fs_up_write(&sbi->gc_lock);
96	110	stat_io_skip_bggc_count(sbi);
97	111	goto next;
98	112	}
..	..	@@ -102,13 +116,21 @@
102	116	else
103	117	increase_sleep_time(gc_th, &wait_ms);
104	118	do_gc:
105		- stat_inc_bggc_count(sbi->stat_info);
	119	+ if (!foreground)
	120	+ stat_inc_bggc_count(sbi->stat_info);
106	121
107	122	sync_mode = F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC;
108	123
	124	+ /* foreground GC was been triggered via f2fs_balance_fs() */
	125	+ if (foreground)
	126	+ sync_mode = false;
	127	+
109	128	/* if return value is not zero, no victim was selected */
110		- if (f2fs_gc(sbi, sync_mode, true, NULL_SEGNO))
	129	+ if (f2fs_gc(sbi, sync_mode, !foreground, false, NULL_SEGNO))
111	130	wait_ms = gc_th->no_gc_sleep_time;
	131	+
	132	+ if (foreground)
	133	+ wake_up_all(&gc_th->fggc_wq);
112	134
113	135	trace_f2fs_background_gc(sbi->sb, wait_ms,
114	136	prefree_segments(sbi), free_segments(sbi));
..	..	@@ -139,15 +161,16 @@
139	161	gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME;
140	162	gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME;
141	163
142		- gc_th->gc_wake= 0;
	164	+ gc_th->gc_wake = 0;
143	165
144	166	sbi->gc_thread = gc_th;
145	167	init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head);
	168	+ init_waitqueue_head(&sbi->gc_thread->fggc_wq);
146	169	sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi,
147	170	"f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev));
148	171	if (IS_ERR(gc_th->f2fs_gc_task)) {
149	172	err = PTR_ERR(gc_th->f2fs_gc_task);
150		- kvfree(gc_th);
	173	+ kfree(gc_th);
151	174	sbi->gc_thread = NULL;
152	175	}
153	176	out:
..	..	@@ -157,26 +180,41 @@
157	180	void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi)
158	181	{
159	182	struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
	183	+
160	184	if (!gc_th)
161	185	return;
162	186	kthread_stop(gc_th->f2fs_gc_task);
163		- kvfree(gc_th);
	187	+ wake_up_all(&gc_th->fggc_wq);
	188	+ kfree(gc_th);
164	189	sbi->gc_thread = NULL;
165	190	}
166	191
167	192	static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type)
168	193	{
169		- int gc_mode = (gc_type == BG_GC) ? GC_CB : GC_GREEDY;
	194	+ int gc_mode;
	195	+
	196	+ if (gc_type == BG_GC) {
	197	+ if (sbi->am.atgc_enabled)
	198	+ gc_mode = GC_AT;
	199	+ else
	200	+ gc_mode = GC_CB;
	201	+ } else {
	202	+ gc_mode = GC_GREEDY;
	203	+ }
170	204
171	205	switch (sbi->gc_mode) {
172	206	case GC_IDLE_CB:
173	207	gc_mode = GC_CB;
174	208	break;
175	209	case GC_IDLE_GREEDY:
176		- case GC_URGENT:
	210	+ case GC_URGENT_HIGH:
177	211	gc_mode = GC_GREEDY;
178	212	break;
	213	+ case GC_IDLE_AT:
	214	+ gc_mode = GC_AT;
	215	+ break;
179	216	}
	217	+
180	218	return gc_mode;
181	219	}
182	220
..	..	@@ -187,14 +225,25 @@
187	225
188	226	if (p->alloc_mode == SSR) {
189	227	p->gc_mode = GC_GREEDY;
190		- p->dirty_segmap = dirty_i->dirty_segmap[type];
	228	+ p->dirty_bitmap = dirty_i->dirty_segmap[type];
	229	+ p->max_search = dirty_i->nr_dirty[type];
	230	+ p->ofs_unit = 1;
	231	+ } else if (p->alloc_mode == AT_SSR) {
	232	+ p->gc_mode = GC_GREEDY;
	233	+ p->dirty_bitmap = dirty_i->dirty_segmap[type];
191	234	p->max_search = dirty_i->nr_dirty[type];
192	235	p->ofs_unit = 1;
193	236	} else {
194	237	p->gc_mode = select_gc_type(sbi, gc_type);
195		- p->dirty_segmap = dirty_i->dirty_segmap[DIRTY];
196		- p->max_search = dirty_i->nr_dirty[DIRTY];
197	238	p->ofs_unit = sbi->segs_per_sec;
	239	+ if (__is_large_section(sbi)) {
	240	+ p->dirty_bitmap = dirty_i->dirty_secmap;
	241	+ p->max_search = count_bits(p->dirty_bitmap,
	242	+ 0, MAIN_SECS(sbi));
	243	+ } else {
	244	+ p->dirty_bitmap = dirty_i->dirty_segmap[DIRTY];
	245	+ p->max_search = dirty_i->nr_dirty[DIRTY];
	246	+ }
198	247	}
199	248
200	249	/*
..	..	@@ -202,7 +251,8 @@
202	251	* foreground GC and urgent GC cases.
203	252	*/
204	253	if (gc_type != FG_GC &&
205		- (sbi->gc_mode != GC_URGENT) &&
	254	+ (sbi->gc_mode != GC_URGENT_HIGH) &&
	255	+ (p->gc_mode != GC_AT && p->alloc_mode != AT_SSR) &&
206	256	p->max_search > sbi->max_victim_search)
207	257	p->max_search = sbi->max_victim_search;
208	258
..	..	@@ -220,9 +270,15 @@
220	270	/* SSR allocates in a segment unit */
221	271	if (p->alloc_mode == SSR)
222	272	return sbi->blocks_per_seg;
	273	+ else if (p->alloc_mode == AT_SSR)
	274	+ return UINT_MAX;
	275	+
	276	+ /* LFS */
223	277	if (p->gc_mode == GC_GREEDY)
224	278	return 2 * sbi->blocks_per_seg * p->ofs_unit;
225	279	else if (p->gc_mode == GC_CB)
	280	+ return UINT_MAX;
	281	+ else if (p->gc_mode == GC_AT)
226	282	return UINT_MAX;
227	283	else /* No other gc_mode */
228	284	return 0;
..	..	@@ -257,13 +313,14 @@
257	313	unsigned char age = 0;
258	314	unsigned char u;
259	315	unsigned int i;
	316	+ unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi, segno);
260	317
261		- for (i = 0; i < sbi->segs_per_sec; i++)
	318	+ for (i = 0; i < usable_segs_per_sec; i++)
262	319	mtime += get_seg_entry(sbi, start + i)->mtime;
263	320	vblocks = get_valid_blocks(sbi, segno, true);
264	321
265		- mtime = div_u64(mtime, sbi->segs_per_sec);
266		- vblocks = div_u64(vblocks, sbi->segs_per_sec);
	322	+ mtime = div_u64(mtime, usable_segs_per_sec);
	323	+ vblocks = div_u64(vblocks, usable_segs_per_sec);
267	324
268	325	u = (vblocks * 100) >> sbi->log_blocks_per_seg;
269	326
..	..	@@ -288,8 +345,11 @@
288	345	/* alloc_mode == LFS */
289	346	if (p->gc_mode == GC_GREEDY)
290	347	return get_valid_blocks(sbi, segno, true);
291		- else
	348	+ else if (p->gc_mode == GC_CB)
292	349	return get_cb_cost(sbi, segno);
	350	+
	351	+ f2fs_bug_on(sbi, 1);
	352	+ return 0;
293	353	}
294	354
295	355	static unsigned int count_bits(const unsigned long *addr,
..	..	@@ -304,6 +364,269 @@
304	364	return sum;
305	365	}
306	366
	367	+static struct victim_entry attach_victim_entry(struct f2fs_sb_info sbi,
	368	+ unsigned long long mtime, unsigned int segno,
	369	+ struct rb_node parent, struct rb_node *p,
	370	+ bool left_most)
	371	+{
	372	+ struct atgc_management *am = &sbi->am;
	373	+ struct victim_entry *ve;
	374	+
	375	+ ve = f2fs_kmem_cache_alloc(victim_entry_slab, GFP_NOFS);
	376	+
	377	+ ve->mtime = mtime;
	378	+ ve->segno = segno;
	379	+
	380	+ rb_link_node(&ve->rb_node, parent, p);
	381	+ rb_insert_color_cached(&ve->rb_node, &am->root, left_most);
	382	+
	383	+ list_add_tail(&ve->list, &am->victim_list);
	384	+
	385	+ am->victim_count++;
	386	+
	387	+ return ve;
	388	+}
	389	+
	390	+static void insert_victim_entry(struct f2fs_sb_info *sbi,
	391	+ unsigned long long mtime, unsigned int segno)
	392	+{
	393	+ struct atgc_management *am = &sbi->am;
	394	+ struct rb_node **p;
	395	+ struct rb_node *parent = NULL;
	396	+ bool left_most = true;
	397	+
	398	+ p = f2fs_lookup_rb_tree_ext(sbi, &am->root, &parent, mtime, &left_most);
	399	+ attach_victim_entry(sbi, mtime, segno, parent, p, left_most);
	400	+}
	401	+
	402	+static void add_victim_entry(struct f2fs_sb_info *sbi,
	403	+ struct victim_sel_policy *p, unsigned int segno)
	404	+{
	405	+ struct sit_info *sit_i = SIT_I(sbi);
	406	+ unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
	407	+ unsigned int start = GET_SEG_FROM_SEC(sbi, secno);
	408	+ unsigned long long mtime = 0;
	409	+ unsigned int i;
	410	+
	411	+ if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
	412	+ if (p->gc_mode == GC_AT &&
	413	+ get_valid_blocks(sbi, segno, true) == 0)
	414	+ return;
	415	+ }
	416	+
	417	+ for (i = 0; i < sbi->segs_per_sec; i++)
	418	+ mtime += get_seg_entry(sbi, start + i)->mtime;
	419	+ mtime = div_u64(mtime, sbi->segs_per_sec);
	420	+
	421	+ /* Handle if the system time has changed by the user */
	422	+ if (mtime < sit_i->min_mtime)
	423	+ sit_i->min_mtime = mtime;
	424	+ if (mtime > sit_i->max_mtime)
	425	+ sit_i->max_mtime = mtime;
	426	+ if (mtime < sit_i->dirty_min_mtime)
	427	+ sit_i->dirty_min_mtime = mtime;
	428	+ if (mtime > sit_i->dirty_max_mtime)
	429	+ sit_i->dirty_max_mtime = mtime;
	430	+
	431	+ /* don't choose young section as candidate */
	432	+ if (sit_i->dirty_max_mtime - mtime < p->age_threshold)
	433	+ return;
	434	+
	435	+ insert_victim_entry(sbi, mtime, segno);
	436	+}
	437	+
	438	+static struct rb_node lookup_central_victim(struct f2fs_sb_info sbi,
	439	+ struct victim_sel_policy *p)
	440	+{
	441	+ struct atgc_management *am = &sbi->am;
	442	+ struct rb_node *parent = NULL;
	443	+ bool left_most;
	444	+
	445	+ f2fs_lookup_rb_tree_ext(sbi, &am->root, &parent, p->age, &left_most);
	446	+
	447	+ return parent;
	448	+}
	449	+
	450	+static void atgc_lookup_victim(struct f2fs_sb_info *sbi,
	451	+ struct victim_sel_policy *p)
	452	+{
	453	+ struct sit_info *sit_i = SIT_I(sbi);
	454	+ struct atgc_management *am = &sbi->am;
	455	+ struct rb_root_cached *root = &am->root;
	456	+ struct rb_node *node;
	457	+ struct rb_entry *re;
	458	+ struct victim_entry *ve;
	459	+ unsigned long long total_time;
	460	+ unsigned long long age, u, accu;
	461	+ unsigned long long max_mtime = sit_i->dirty_max_mtime;
	462	+ unsigned long long min_mtime = sit_i->dirty_min_mtime;
	463	+ unsigned int sec_blocks = BLKS_PER_SEC(sbi);
	464	+ unsigned int vblocks;
	465	+ unsigned int dirty_threshold = max(am->max_candidate_count,
	466	+ am->candidate_ratio *
	467	+ am->victim_count / 100);
	468	+ unsigned int age_weight = am->age_weight;
	469	+ unsigned int cost;
	470	+ unsigned int iter = 0;
	471	+
	472	+ if (max_mtime < min_mtime)
	473	+ return;
	474	+
	475	+ max_mtime += 1;
	476	+ total_time = max_mtime - min_mtime;
	477	+
	478	+ accu = div64_u64(ULLONG_MAX, total_time);
	479	+ accu = min_t(unsigned long long, div_u64(accu, 100),
	480	+ DEFAULT_ACCURACY_CLASS);
	481	+
	482	+ node = rb_first_cached(root);
	483	+next:
	484	+ re = rb_entry_safe(node, struct rb_entry, rb_node);
	485	+ if (!re)
	486	+ return;
	487	+
	488	+ ve = (struct victim_entry *)re;
	489	+
	490	+ if (ve->mtime >= max_mtime \|\| ve->mtime < min_mtime)
	491	+ goto skip;
	492	+
	493	+ /* age = 10000 * x% * 60 */
	494	+ age = div64_u64(accu * (max_mtime - ve->mtime), total_time) *
	495	+ age_weight;
	496	+
	497	+ vblocks = get_valid_blocks(sbi, ve->segno, true);
	498	+ f2fs_bug_on(sbi, !vblocks \|\| vblocks == sec_blocks);
	499	+
	500	+ /* u = 10000 * x% * 40 */
	501	+ u = div64_u64(accu * (sec_blocks - vblocks), sec_blocks) *
	502	+ (100 - age_weight);
	503	+
	504	+ f2fs_bug_on(sbi, age + u >= UINT_MAX);
	505	+
	506	+ cost = UINT_MAX - (age + u);
	507	+ iter++;
	508	+
	509	+ if (cost < p->min_cost \|\|
	510	+ (cost == p->min_cost && age > p->oldest_age)) {
	511	+ p->min_cost = cost;
	512	+ p->oldest_age = age;
	513	+ p->min_segno = ve->segno;
	514	+ }
	515	+skip:
	516	+ if (iter < dirty_threshold) {
	517	+ node = rb_next(node);
	518	+ goto next;
	519	+ }
	520	+}
	521	+
	522	+/*
	523	+ * select candidates around source section in range of
	524	+ * [target - dirty_threshold, target + dirty_threshold]
	525	+ */
	526	+static void atssr_lookup_victim(struct f2fs_sb_info *sbi,
	527	+ struct victim_sel_policy *p)
	528	+{
	529	+ struct sit_info *sit_i = SIT_I(sbi);
	530	+ struct atgc_management *am = &sbi->am;
	531	+ struct rb_node *node;
	532	+ struct rb_entry *re;
	533	+ struct victim_entry *ve;
	534	+ unsigned long long age;
	535	+ unsigned long long max_mtime = sit_i->dirty_max_mtime;
	536	+ unsigned long long min_mtime = sit_i->dirty_min_mtime;
	537	+ unsigned int seg_blocks = sbi->blocks_per_seg;
	538	+ unsigned int vblocks;
	539	+ unsigned int dirty_threshold = max(am->max_candidate_count,
	540	+ am->candidate_ratio *
	541	+ am->victim_count / 100);
	542	+ unsigned int cost;
	543	+ unsigned int iter = 0;
	544	+ int stage = 0;
	545	+
	546	+ if (max_mtime < min_mtime)
	547	+ return;
	548	+ max_mtime += 1;
	549	+next_stage:
	550	+ node = lookup_central_victim(sbi, p);
	551	+next_node:
	552	+ re = rb_entry_safe(node, struct rb_entry, rb_node);
	553	+ if (!re) {
	554	+ if (stage == 0)
	555	+ goto skip_stage;
	556	+ return;
	557	+ }
	558	+
	559	+ ve = (struct victim_entry *)re;
	560	+
	561	+ if (ve->mtime >= max_mtime \|\| ve->mtime < min_mtime)
	562	+ goto skip_node;
	563	+
	564	+ age = max_mtime - ve->mtime;
	565	+
	566	+ vblocks = get_seg_entry(sbi, ve->segno)->ckpt_valid_blocks;
	567	+ f2fs_bug_on(sbi, !vblocks);
	568	+
	569	+ /* rare case */
	570	+ if (vblocks == seg_blocks)
	571	+ goto skip_node;
	572	+
	573	+ iter++;
	574	+
	575	+ age = max_mtime - abs(p->age - age);
	576	+ cost = UINT_MAX - vblocks;
	577	+
	578	+ if (cost < p->min_cost \|\|
	579	+ (cost == p->min_cost && age > p->oldest_age)) {
	580	+ p->min_cost = cost;
	581	+ p->oldest_age = age;
	582	+ p->min_segno = ve->segno;
	583	+ }
	584	+skip_node:
	585	+ if (iter < dirty_threshold) {
	586	+ if (stage == 0)
	587	+ node = rb_prev(node);
	588	+ else if (stage == 1)
	589	+ node = rb_next(node);
	590	+ goto next_node;
	591	+ }
	592	+skip_stage:
	593	+ if (stage < 1) {
	594	+ stage++;
	595	+ iter = 0;
	596	+ goto next_stage;
	597	+ }
	598	+}
	599	+static void lookup_victim_by_age(struct f2fs_sb_info *sbi,
	600	+ struct victim_sel_policy *p)
	601	+{
	602	+ f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
	603	+ &sbi->am.root, true));
	604	+
	605	+ if (p->gc_mode == GC_AT)
	606	+ atgc_lookup_victim(sbi, p);
	607	+ else if (p->alloc_mode == AT_SSR)
	608	+ atssr_lookup_victim(sbi, p);
	609	+ else
	610	+ f2fs_bug_on(sbi, 1);
	611	+}
	612	+
	613	+static void release_victim_entry(struct f2fs_sb_info *sbi)
	614	+{
	615	+ struct atgc_management *am = &sbi->am;
	616	+ struct victim_entry ve, tmp;
	617	+
	618	+ list_for_each_entry_safe(ve, tmp, &am->victim_list, list) {
	619	+ list_del(&ve->list);
	620	+ kmem_cache_free(victim_entry_slab, ve);
	621	+ am->victim_count--;
	622	+ }
	623	+
	624	+ am->root = RB_ROOT_CACHED;
	625	+
	626	+ f2fs_bug_on(sbi, am->victim_count);
	627	+ f2fs_bug_on(sbi, !list_empty(&am->victim_list));
	628	+}
	629	+
307	630	/*
308	631	* This function is called from two paths.
309	632	* One is garbage collection and the other is SSR segment selection.
..	..	@@ -313,31 +636,51 @@
313	636	* which has minimum valid blocks and removes it from dirty seglist.
314	637	*/
315	638	static int get_victim_by_default(struct f2fs_sb_info *sbi,
316		- unsigned int *result, int gc_type, int type, char alloc_mode)
	639	+ unsigned int *result, int gc_type, int type,
	640	+ char alloc_mode, unsigned long long age)
317	641	{
318	642	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
319	643	struct sit_info *sm = SIT_I(sbi);
320	644	struct victim_sel_policy p;
321	645	unsigned int secno, last_victim;
322	646	unsigned int last_segment;
323		- unsigned int nsearched = 0;
	647	+ unsigned int nsearched;
	648	+ bool is_atgc;
	649	+ int ret = 0;
324	650
325	651	mutex_lock(&dirty_i->seglist_lock);
326	652	last_segment = MAIN_SECS(sbi) * sbi->segs_per_sec;
327	653
328	654	p.alloc_mode = alloc_mode;
329		- select_policy(sbi, gc_type, type, &p);
	655	+ p.age = age;
	656	+ p.age_threshold = sbi->am.age_threshold;
330	657
	658	+retry:
	659	+ select_policy(sbi, gc_type, type, &p);
331	660	p.min_segno = NULL_SEGNO;
	661	+ p.oldest_age = 0;
332	662	p.min_cost = get_max_cost(sbi, &p);
333	663
	664	+ is_atgc = (p.gc_mode == GC_AT \|\| p.alloc_mode == AT_SSR);
	665	+ nsearched = 0;
	666	+
	667	+ if (is_atgc)
	668	+ SIT_I(sbi)->dirty_min_mtime = ULLONG_MAX;
	669	+
334	670	if (*result != NULL_SEGNO) {
335		- if (get_valid_blocks(sbi, *result, false) &&
336		- !sec_usage_check(sbi, GET_SEC_FROM_SEG(sbi, *result)))
	671	+ if (!get_valid_blocks(sbi, *result, false)) {
	672	+ ret = -ENODATA;
	673	+ goto out;
	674	+ }
	675	+
	676	+ if (sec_usage_check(sbi, GET_SEC_FROM_SEG(sbi, *result)))
	677	+ ret = -EBUSY;
	678	+ else
337	679	p.min_segno = *result;
338	680	goto out;
339	681	}
340	682
	683	+ ret = -ENODATA;
341	684	if (p.max_search == 0)
342	685	goto out;
343	686
..	..	@@ -365,10 +708,14 @@
365	708	}
366	709
367	710	while (1) {
368		- unsigned long cost;
369		- unsigned int segno;
	711	+ unsigned long cost, *dirty_bitmap;
	712	+ unsigned int unit_no, segno;
370	713
371		- segno = find_next_bit(p.dirty_segmap, last_segment, p.offset);
	714	+ dirty_bitmap = p.dirty_bitmap;
	715	+ unit_no = find_next_bit(dirty_bitmap,
	716	+ last_segment / p.ofs_unit,
	717	+ p.offset / p.ofs_unit);
	718	+ segno = unit_no * p.ofs_unit;
372	719	if (segno >= last_segment) {
373	720	if (sm->last_victim[p.gc_mode]) {
374	721	last_segment =
..	..	@@ -381,14 +728,7 @@
381	728	}
382	729
383	730	p.offset = segno + p.ofs_unit;
384		- if (p.ofs_unit > 1) {
385		- p.offset -= segno % p.ofs_unit;
386		- nsearched += count_bits(p.dirty_segmap,
387		- p.offset - p.ofs_unit,
388		- p.ofs_unit);
389		- } else {
390		- nsearched++;
391		- }
	731	+ nsearched++;
392	732
393	733	#ifdef CONFIG_F2FS_CHECK_FS
394	734	/*
..	..	@@ -404,13 +744,34 @@
404	744
405	745	if (sec_usage_check(sbi, secno))
406	746	goto next;
	747	+
407	748	/* Don't touch checkpointed data */
408		- if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
409		- get_ckpt_valid_blocks(sbi, segno) &&
410		- p.alloc_mode != SSR))
411		- goto next;
	749	+ if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
	750	+ if (p.alloc_mode == LFS) {
	751	+ /*
	752	+ * LFS is set to find source section during GC.
	753	+ * The victim should have no checkpointed data.
	754	+ */
	755	+ if (get_ckpt_valid_blocks(sbi, segno, true))
	756	+ goto next;
	757	+ } else {
	758	+ /*
	759	+ * SSR \| AT_SSR are set to find target segment
	760	+ * for writes which can be full by checkpointed
	761	+ * and newly written blocks.
	762	+ */
	763	+ if (!f2fs_segment_has_free_slot(sbi, segno))
	764	+ goto next;
	765	+ }
	766	+ }
	767	+
412	768	if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap))
413	769	goto next;
	770	+
	771	+ if (is_atgc) {
	772	+ add_victim_entry(sbi, &p, segno);
	773	+ goto next;
	774	+ }
414	775
415	776	cost = get_gc_cost(sbi, segno, &p);
416	777
..	..	@@ -421,14 +782,28 @@
421	782	next:
422	783	if (nsearched >= p.max_search) {
423	784	if (!sm->last_victim[p.gc_mode] && segno <= last_victim)
424		- sm->last_victim[p.gc_mode] = last_victim + 1;
	785	+ sm->last_victim[p.gc_mode] =
	786	+ last_victim + p.ofs_unit;
425	787	else
426		- sm->last_victim[p.gc_mode] = segno + 1;
	788	+ sm->last_victim[p.gc_mode] = segno + p.ofs_unit;
427	789	sm->last_victim[p.gc_mode] %=
428	790	(MAIN_SECS(sbi) * sbi->segs_per_sec);
429	791	break;
430	792	}
431	793	}
	794	+
	795	+ /* get victim for GC_AT/AT_SSR */
	796	+ if (is_atgc) {
	797	+ lookup_victim_by_age(sbi, &p);
	798	+ release_victim_entry(sbi);
	799	+ }
	800	+
	801	+ if (is_atgc && p.min_segno == NULL_SEGNO &&
	802	+ sm->elapsed_time < p.age_threshold) {
	803	+ p.age_threshold = 0;
	804	+ goto retry;
	805	+ }
	806	+
432	807	if (p.min_segno != NULL_SEGNO) {
433	808	got_it:
434	809	result = (p.min_segno / p.ofs_unit) p.ofs_unit;
..	..	@@ -440,6 +815,7 @@
440	815	else
441	816	set_bit(secno, dirty_i->victim_secmap);
442	817	}
	818	+ ret = 0;
443	819
444	820	}
445	821	out:
..	..	@@ -449,7 +825,7 @@
449	825	prefree_segments(sbi), free_segments(sbi));
450	826	mutex_unlock(&dirty_i->seglist_lock);
451	827
452		- return (p.min_segno == NULL_SEGNO) ? 0 : 1;
	828	+ return ret;
453	829	}
454	830
455	831	static const struct victim_selection default_v_ops = {
..	..	@@ -484,6 +860,7 @@
484	860	static void put_gc_inode(struct gc_inode_list *gc_list)
485	861	{
486	862	struct inode_entry ie, next_ie;
	863	+
487	864	list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) {
488	865	radix_tree_delete(&gc_list->iroot, ie->inode->i_ino);
489	866	iput(ie->inode);
..	..	@@ -520,6 +897,7 @@
520	897	int phase = 0;
521	898	bool fggc = (gc_type == FG_GC);
522	899	int submitted = 0;
	900	+ unsigned int usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
523	901
524	902	start_addr = START_BLOCK(sbi, segno);
525	903
..	..	@@ -529,7 +907,7 @@
529	907	if (fggc && phase == 2)
530	908	atomic_inc(&sbi->wb_sync_req[NODE]);
531	909
532		- for (off = 0; off < sbi->blocks_per_seg; off++, entry++) {
	910	+ for (off = 0; off < usable_blks_in_seg; off++, entry++) {
533	911	nid_t nid = le32_to_cpu(entry->nid);
534	912	struct page *node_page;
535	913	struct node_info ni;
..	..	@@ -564,7 +942,7 @@
564	942	continue;
565	943	}
566	944
567		- if (f2fs_get_node_info(sbi, nid, &ni)) {
	945	+ if (f2fs_get_node_info(sbi, nid, &ni, false)) {
568	946	f2fs_put_page(node_page, 1);
569	947	continue;
570	948	}
..	..	@@ -607,9 +985,11 @@
607	985	bidx = node_ofs - 1;
608	986	} else if (node_ofs <= indirect_blks) {
609	987	int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1);
	988	+
610	989	bidx = node_ofs - 2 - dec;
611	990	} else {
612	991	int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1);
	992	+
613	993	bidx = node_ofs - 5 - dec;
614	994	}
615	995	return bidx * ADDRS_PER_BLOCK(inode) + ADDRS_PER_INODE(inode);
..	..	@@ -620,7 +1000,7 @@
620	1000	{
621	1001	struct page *node_page;
622	1002	nid_t nid;
623		- unsigned int ofs_in_node;
	1003	+ unsigned int ofs_in_node, max_addrs;
624	1004	block_t source_blkaddr;
625	1005
626	1006	nid = le32_to_cpu(sum->nid);
..	..	@@ -630,7 +1010,7 @@
630	1010	if (IS_ERR(node_page))
631	1011	return false;
632	1012
633		- if (f2fs_get_node_info(sbi, nid, dni)) {
	1013	+ if (f2fs_get_node_info(sbi, nid, dni, false)) {
634	1014	f2fs_put_page(node_page, 1);
635	1015	return false;
636	1016	}
..	..	@@ -641,8 +1021,18 @@
641	1021	set_sbi_flag(sbi, SBI_NEED_FSCK);
642	1022	}
643	1023
644		- if (f2fs_check_nid_range(sbi, dni->ino))
	1024	+ if (f2fs_check_nid_range(sbi, dni->ino)) {
	1025	+ f2fs_put_page(node_page, 1);
645	1026	return false;
	1027	+ }
	1028	+
	1029	+ max_addrs = IS_INODE(node_page) ? DEF_ADDRS_PER_INODE :
	1030	+ DEF_ADDRS_PER_BLOCK;
	1031	+ if (ofs_in_node >= max_addrs) {
	1032	+ f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%u, nid:%u, max:%u",
	1033	+ ofs_in_node, dni->ino, dni->nid, max_addrs);
	1034	+ return false;
	1035	+ }
646	1036
647	1037	*nofs = ofs_of_node(node_page);
648	1038	source_blkaddr = data_blkaddr(NULL, node_page, ofs_in_node);
..	..	@@ -655,8 +1045,8 @@
655	1045
656	1046	if (unlikely(check_valid_map(sbi, segno, offset))) {
657	1047	if (!test_and_set_bit(segno, SIT_I(sbi)->invalid_segmap)) {
658		- f2fs_err(sbi, "mismatched blkaddr %u (source_blkaddr %u) in seg %u\n",
659		- blkaddr, source_blkaddr, segno);
	1048	+ f2fs_err(sbi, "mismatched blkaddr %u (source_blkaddr %u) in seg %u",
	1049	+ blkaddr, source_blkaddr, segno);
660	1050	f2fs_bug_on(sbi, 1);
661	1051	}
662	1052	}
..	..	@@ -672,7 +1062,7 @@
672	1062	struct address_space *mapping = inode->i_mapping;
673	1063	struct dnode_of_data dn;
674	1064	struct page *page;
675		- struct extent_info ei = {0, 0, 0};
	1065	+ struct extent_info ei = {0, };
676	1066	struct f2fs_io_info fio = {
677	1067	.sbi = sbi,
678	1068	.ino = inode->i_ino,
..	..	@@ -690,7 +1080,7 @@
690	1080	if (!page)
691	1081	return -ENOMEM;
692	1082
693		- if (f2fs_lookup_extent_cache(inode, index, &ei)) {
	1083	+ if (f2fs_lookup_read_extent_cache(inode, index, &ei)) {
694	1084	dn.data_blkaddr = ei.blk + index - ei.fofs;
695	1085	if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
696	1086	DATA_GENERIC_ENHANCE_READ))) {
..	..	@@ -778,6 +1168,9 @@
778	1168	block_t newaddr;
779	1169	int err = 0;
780	1170	bool lfs_mode = f2fs_lfs_mode(fio.sbi);
	1171	+ int type = fio.sbi->am.atgc_enabled && (gc_type == BG_GC) &&
	1172	+ (fio.sbi->gc_mode != GC_URGENT_HIGH) ?
	1173	+ CURSEG_ALL_DATA_ATGC : CURSEG_COLD_DATA;
781	1174
782	1175	/* do not read out */
783	1176	page = f2fs_grab_cache_page(inode->i_mapping, bidx, false);
..	..	@@ -797,7 +1190,8 @@
797	1190	}
798	1191
799	1192	if (f2fs_is_pinned_file(inode)) {
800		- f2fs_pin_file_control(inode, true);
	1193	+ if (gc_type == FG_GC)
	1194	+ f2fs_pin_file_control(inode, true);
801	1195	err = -EAGAIN;
802	1196	goto out;
803	1197	}
..	..	@@ -821,23 +1215,23 @@
821	1215
822	1216	f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
823	1217
824		- err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
	1218	+ err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false);
825	1219	if (err)
826	1220	goto put_out;
827		-
828		- set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version);
829	1221
830	1222	/* read page */
831	1223	fio.page = page;
832	1224	fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
833	1225
834	1226	if (lfs_mode)
835		- down_write(&fio.sbi->io_order_lock);
	1227	+ f2fs_down_write(&fio.sbi->io_order_lock);
836	1228
837	1229	mpage = f2fs_grab_cache_page(META_MAPPING(fio.sbi),
838	1230	fio.old_blkaddr, false);
839		- if (!mpage)
	1231	+ if (!mpage) {
	1232	+ err = -ENOMEM;
840	1233	goto up_out;
	1234	+ }
841	1235
842	1236	fio.encrypted_page = mpage;
843	1237
..	..	@@ -861,8 +1255,11 @@
861	1255	}
862	1256	}
863	1257
	1258	+ set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version);
	1259	+
	1260	+ /* allocate block address */
864	1261	f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
865		- &sum, CURSEG_COLD_DATA, NULL, false);
	1262	+ &sum, type, NULL);
866	1263
867	1264	fio.encrypted_page = f2fs_pagecache_get_page(META_MAPPING(fio.sbi),
868	1265	newaddr, FGP_LOCK \| FGP_CREAT, GFP_NOFS);
..	..	@@ -879,6 +1276,7 @@
879	1276	f2fs_put_page(mpage, 1);
880	1277	invalidate_mapping_pages(META_MAPPING(fio.sbi),
881	1278	fio.old_blkaddr, fio.old_blkaddr);
	1279	+ f2fs_invalidate_compress_page(fio.sbi, fio.old_blkaddr);
882	1280
883	1281	set_page_dirty(fio.encrypted_page);
884	1282	if (clear_page_dirty_for_io(fio.encrypted_page))
..	..	@@ -886,9 +1284,6 @@
886	1284
887	1285	set_page_writeback(fio.encrypted_page);
888	1286	ClearPageError(page);
889		-
890		- /* allocate block address */
891		- f2fs_wait_on_page_writeback(dn.node_page, NODE, true, true);
892	1287
893	1288	fio.op = REQ_OP_WRITE;
894	1289	fio.op_flags = REQ_SYNC;
..	..	@@ -912,10 +1307,10 @@
912	1307	recover_block:
913	1308	if (err)
914	1309	f2fs_do_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr,
915		- true, true);
	1310	+ true, true, true);
916	1311	up_out:
917	1312	if (lfs_mode)
918		- up_write(&fio.sbi->io_order_lock);
	1313	+ f2fs_up_write(&fio.sbi->io_order_lock);
919	1314	put_out:
920	1315	f2fs_put_dnode(&dn);
921	1316	out:
..	..	@@ -957,7 +1352,7 @@
957	1352	goto out;
958	1353	}
959	1354	set_page_dirty(page);
960		- set_cold_data(page);
	1355	+ set_page_private_gcing(page);
961	1356	} else {
962	1357	struct f2fs_io_info fio = {
963	1358	.sbi = F2FS_I_SB(inode),
..	..	@@ -983,11 +1378,11 @@
983	1378	f2fs_remove_dirty_inode(inode);
984	1379	}
985	1380
986		- set_cold_data(page);
	1381	+ set_page_private_gcing(page);
987	1382
988	1383	err = f2fs_do_write_data_page(&fio);
989	1384	if (err) {
990		- clear_cold_data(page);
	1385	+ clear_page_private_gcing(page);
991	1386	if (err == -ENOMEM) {
992	1387	congestion_wait(BLK_RW_ASYNC,
993	1388	DEFAULT_IO_TIMEOUT);
..	..	@@ -1010,7 +1405,8 @@
1010	1405	* the victim data block is ignored.
1011	1406	*/
1012	1407	static int gc_data_segment(struct f2fs_sb_info sbi, struct f2fs_summary sum,
1013		- struct gc_inode_list *gc_list, unsigned int segno, int gc_type)
	1408	+ struct gc_inode_list *gc_list, unsigned int segno, int gc_type,
	1409	+ bool force_migrate)
1014	1410	{
1015	1411	struct super_block *sb = sbi->sb;
1016	1412	struct f2fs_summary *entry;
..	..	@@ -1018,13 +1414,14 @@
1018	1414	int off;
1019	1415	int phase = 0;
1020	1416	int submitted = 0;
	1417	+ unsigned int usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
1021	1418
1022	1419	start_addr = START_BLOCK(sbi, segno);
1023	1420
1024	1421	next_step:
1025	1422	entry = sum;
1026	1423
1027		- for (off = 0; off < sbi->blocks_per_seg; off++, entry++) {
	1424	+ for (off = 0; off < usable_blks_in_seg; off++, entry++) {
1028	1425	struct page *data_page;
1029	1426	struct inode *inode;
1030	1427	struct node_info dni; /* dnode info for the data */
..	..	@@ -1038,8 +1435,8 @@
1038	1435	* race condition along with SSR block allocation.
1039	1436	*/
1040	1437	if ((gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) \|\|
1041		- get_valid_blocks(sbi, segno, true) ==
1042		- BLKS_PER_SEC(sbi))
	1438	+ (!force_migrate && get_valid_blocks(sbi, segno, true) ==
	1439	+ BLKS_PER_SEC(sbi)))
1043	1440	return submitted;
1044	1441
1045	1442	if (check_valid_map(sbi, segno, off) == 0)
..	..	@@ -1069,12 +1466,10 @@
1069	1466
1070	1467	if (phase == 3) {
1071	1468	inode = f2fs_iget(sb, dni.ino);
1072		- if (IS_ERR(inode) \|\| is_bad_inode(inode)) {
1073		- set_sbi_flag(sbi, SBI_NEED_FSCK);
	1469	+ if (IS_ERR(inode) \|\| is_bad_inode(inode))
1074	1470	continue;
1075		- }
1076	1471
1077		- if (!down_write_trylock(
	1472	+ if (!f2fs_down_write_trylock(
1078	1473	&F2FS_I(inode)->i_gc_rwsem[WRITE])) {
1079	1474	iput(inode);
1080	1475	sbi->skipped_gc_rwsem++;
..	..	@@ -1087,7 +1482,7 @@
1087	1482	if (f2fs_post_read_required(inode)) {
1088	1483	int err = ra_data_block(inode, start_bidx);
1089	1484
1090		- up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	1485	+ f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1091	1486	if (err) {
1092	1487	iput(inode);
1093	1488	continue;
..	..	@@ -1098,7 +1493,7 @@
1098	1493
1099	1494	data_page = f2fs_get_read_data_page(inode,
1100	1495	start_bidx, REQ_RAHEAD, true);
1101		- up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
	1496	+ f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1102	1497	if (IS_ERR(data_page)) {
1103	1498	iput(inode);
1104	1499	continue;
..	..	@@ -1117,14 +1512,14 @@
1117	1512	int err;
1118	1513
1119	1514	if (S_ISREG(inode->i_mode)) {
1120		- if (!down_write_trylock(&fi->i_gc_rwsem[READ])) {
	1515	+ if (!f2fs_down_write_trylock(&fi->i_gc_rwsem[READ])) {
1121	1516	sbi->skipped_gc_rwsem++;
1122	1517	continue;
1123	1518	}
1124		- if (!down_write_trylock(
	1519	+ if (!f2fs_down_write_trylock(
1125	1520	&fi->i_gc_rwsem[WRITE])) {
1126	1521	sbi->skipped_gc_rwsem++;
1127		- up_write(&fi->i_gc_rwsem[READ]);
	1522	+ f2fs_up_write(&fi->i_gc_rwsem[READ]);
1128	1523	continue;
1129	1524	}
1130	1525	locked = true;
..	..	@@ -1147,8 +1542,8 @@
1147	1542	submitted++;
1148	1543
1149	1544	if (locked) {
1150		- up_write(&fi->i_gc_rwsem[WRITE]);
1151		- up_write(&fi->i_gc_rwsem[READ]);
	1545	+ f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
	1546	+ f2fs_up_write(&fi->i_gc_rwsem[READ]);
1152	1547	}
1153	1548
1154	1549	stat_inc_data_blk_count(sbi, 1, gc_type);
..	..	@@ -1169,14 +1564,15 @@
1169	1564
1170	1565	down_write(&sit_i->sentry_lock);
1171	1566	ret = DIRTY_I(sbi)->v_ops->get_victim(sbi, victim, gc_type,
1172		- NO_CHECK_TYPE, LFS);
	1567	+ NO_CHECK_TYPE, LFS, 0);
1173	1568	up_write(&sit_i->sentry_lock);
1174	1569	return ret;
1175	1570	}
1176	1571
1177	1572	static int do_garbage_collect(struct f2fs_sb_info *sbi,
1178	1573	unsigned int start_segno,
1179		- struct gc_inode_list *gc_list, int gc_type)
	1574	+ struct gc_inode_list *gc_list, int gc_type,
	1575	+ bool force_migrate)
1180	1576	{
1181	1577	struct page *sum_page;
1182	1578	struct f2fs_summary_block *sum;
..	..	@@ -1190,6 +1586,17 @@
1190	1586
1191	1587	if (__is_large_section(sbi))
1192	1588	end_segno = rounddown(end_segno, sbi->segs_per_sec);
	1589	+
	1590	+ /*
	1591	+ * zone-capacity can be less than zone-size in zoned devices,
	1592	+ * resulting in less than expected usable segments in the zone,
	1593	+ * calculate the end segno in the zone which can be garbage collected
	1594	+ */
	1595	+ if (f2fs_sb_has_blkzoned(sbi))
	1596	+ end_segno -= sbi->segs_per_sec -
	1597	+ f2fs_usable_segs_in_sec(sbi, segno);
	1598	+
	1599	+ sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type);
1193	1600
1194	1601	/* readahead multi ssa blocks those have contiguous address */
1195	1602	if (__is_large_section(sbi))
..	..	@@ -1236,7 +1643,8 @@
1236	1643	f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT",
1237	1644	segno, type, GET_SUM_TYPE((&sum->footer)));
1238	1645	set_sbi_flag(sbi, SBI_NEED_FSCK);
1239		- f2fs_stop_checkpoint(sbi, false);
	1646	+ f2fs_stop_checkpoint(sbi, false,
	1647	+ STOP_CP_REASON_CORRUPTED_SUMMARY);
1240	1648	goto skip;
1241	1649	}
1242	1650
..	..	@@ -1252,9 +1660,11 @@
1252	1660	gc_type);
1253	1661	else
1254	1662	submitted += gc_data_segment(sbi, sum->entries, gc_list,
1255		- segno, gc_type);
	1663	+ segno, gc_type,
	1664	+ force_migrate);
1256	1665
1257	1666	stat_inc_seg_count(sbi, type, gc_type);
	1667	+ sbi->gc_reclaimed_segs[sbi->gc_mode]++;
1258	1668	migrated++;
1259	1669
1260	1670	freed:
..	..	@@ -1280,7 +1690,7 @@
1280	1690	}
1281	1691
1282	1692	int f2fs_gc(struct f2fs_sb_info *sbi, bool sync,
1283		- bool background, unsigned int segno)
	1693	+ bool background, bool force, unsigned int segno)
1284	1694	{
1285	1695	int gc_type = sync ? FG_GC : BG_GC;
1286	1696	int sec_freed = 0, seg_freed = 0, total_freed = 0;
..	..	@@ -1338,13 +1748,13 @@
1338	1748	ret = -EINVAL;
1339	1749	goto stop;
1340	1750	}
1341		- if (!__get_victim(sbi, &segno, gc_type)) {
1342		- ret = -ENODATA;
	1751	+ ret = __get_victim(sbi, &segno, gc_type);
	1752	+ if (ret)
1343	1753	goto stop;
1344		- }
1345	1754
1346		- seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type);
1347		- if (gc_type == FG_GC && seg_freed == sbi->segs_per_sec)
	1755	+ seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type, force);
	1756	+ if (gc_type == FG_GC &&
	1757	+ seg_freed == f2fs_usable_segs_in_sec(sbi, segno))
1348	1758	sec_freed++;
1349	1759	total_freed += seg_freed;
1350	1760
..	..	@@ -1362,23 +1772,31 @@
1362	1772	if (sync)
1363	1773	goto stop;
1364	1774
1365		- if (has_not_enough_free_secs(sbi, sec_freed, 0)) {
1366		- if (skipped_round <= MAX_SKIP_GC_COUNT \|\|
1367		- skipped_round * 2 < round) {
1368		- segno = NULL_SEGNO;
1369		- goto gc_more;
1370		- }
	1775	+ if (!has_not_enough_free_secs(sbi, sec_freed, 0))
	1776	+ goto stop;
1371	1777
1372		- if (first_skipped < last_skipped &&
1373		- (last_skipped - first_skipped) >
1374		- sbi->skipped_gc_rwsem) {
1375		- f2fs_drop_inmem_pages_all(sbi, true);
1376		- segno = NULL_SEGNO;
1377		- goto gc_more;
1378		- }
1379		- if (gc_type == FG_GC && !is_sbi_flag_set(sbi, SBI_CP_DISABLED))
	1778	+ if (skipped_round <= MAX_SKIP_GC_COUNT \|\| skipped_round * 2 < round) {
	1779	+
	1780	+ /* Write checkpoint to reclaim prefree segments */
	1781	+ if (free_sections(sbi) < NR_CURSEG_PERSIST_TYPE &&
	1782	+ prefree_segments(sbi) &&
	1783	+ !is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
1380	1784	ret = f2fs_write_checkpoint(sbi, &cpc);
	1785	+ if (ret)
	1786	+ goto stop;
	1787	+ }
	1788	+ segno = NULL_SEGNO;
	1789	+ goto gc_more;
1381	1790	}
	1791	+ if (first_skipped < last_skipped &&
	1792	+ (last_skipped - first_skipped) >
	1793	+ sbi->skipped_gc_rwsem) {
	1794	+ f2fs_drop_inmem_pages_all(sbi, true);
	1795	+ segno = NULL_SEGNO;
	1796	+ goto gc_more;
	1797	+ }
	1798	+ if (gc_type == FG_GC && !is_sbi_flag_set(sbi, SBI_CP_DISABLED))
	1799	+ ret = f2fs_write_checkpoint(sbi, &cpc);
1382	1800	stop:
1383	1801	SIT_I(sbi)->last_victim[ALLOC_NEXT] = 0;
1384	1802	SIT_I(sbi)->last_victim[FLUSH_DEVICE] = init_segno;
..	..	@@ -1392,13 +1810,45 @@
1392	1810	reserved_segments(sbi),
1393	1811	prefree_segments(sbi));
1394	1812
1395		- up_write(&sbi->gc_lock);
	1813	+ f2fs_up_write(&sbi->gc_lock);
1396	1814
1397	1815	put_gc_inode(&gc_list);
1398	1816
1399	1817	if (sync && !ret)
1400	1818	ret = sec_freed ? 0 : -EAGAIN;
1401	1819	return ret;
	1820	+}
	1821	+
	1822	+int __init f2fs_create_garbage_collection_cache(void)
	1823	+{
	1824	+ victim_entry_slab = f2fs_kmem_cache_create("f2fs_victim_entry",
	1825	+ sizeof(struct victim_entry));
	1826	+ if (!victim_entry_slab)
	1827	+ return -ENOMEM;
	1828	+ return 0;
	1829	+}
	1830	+
	1831	+void f2fs_destroy_garbage_collection_cache(void)
	1832	+{
	1833	+ kmem_cache_destroy(victim_entry_slab);
	1834	+}
	1835	+
	1836	+static void init_atgc_management(struct f2fs_sb_info *sbi)
	1837	+{
	1838	+ struct atgc_management *am = &sbi->am;
	1839	+
	1840	+ if (test_opt(sbi, ATGC) &&
	1841	+ SIT_I(sbi)->elapsed_time >= DEF_GC_THREAD_AGE_THRESHOLD)
	1842	+ am->atgc_enabled = true;
	1843	+
	1844	+ am->root = RB_ROOT_CACHED;
	1845	+ INIT_LIST_HEAD(&am->victim_list);
	1846	+ am->victim_count = 0;
	1847	+
	1848	+ am->candidate_ratio = DEF_GC_THREAD_CANDIDATE_RATIO;
	1849	+ am->max_candidate_count = DEF_GC_THREAD_MAX_CANDIDATE_COUNT;
	1850	+ am->age_weight = DEF_GC_THREAD_AGE_WEIGHT;
	1851	+ am->age_threshold = DEF_GC_THREAD_AGE_THRESHOLD;
1402	1852	}
1403	1853
1404	1854	void f2fs_build_gc_manager(struct f2fs_sb_info *sbi)
..	..	@@ -1411,6 +1861,8 @@
1411	1861	if (f2fs_is_multi_device(sbi) && !__is_large_section(sbi))
1412	1862	SIT_I(sbi)->last_victim[ALLOC_NEXT] =
1413	1863	GET_SEGNO(sbi, FDEV(0).end_blk) + 1;
	1864	+
	1865	+ init_atgc_management(sbi);
1414	1866	}
1415	1867
1416	1868	static int free_segment_range(struct f2fs_sb_info *sbi,
..	..	@@ -1438,8 +1890,8 @@
1438	1890	mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
1439	1891
1440	1892	/* Move out cursegs from the target range */
1441		- for (type = CURSEG_HOT_DATA; type < NR_CURSEG_TYPE; type++)
1442		- allocate_segment_for_resize(sbi, type, start, end);
	1893	+ for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++)
	1894	+ f2fs_allocate_segment_for_resize(sbi, type, start, end);
1443	1895
1444	1896	/* do GC to move out valid blocks in the range */
1445	1897	for (segno = start; segno <= end; segno += sbi->segs_per_sec) {
..	..	@@ -1448,7 +1900,7 @@
1448	1900	.iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS),
1449	1901	};
1450	1902
1451		- do_garbage_collect(sbi, segno, &gc_list, FG_GC);
	1903	+ do_garbage_collect(sbi, segno, &gc_list, FG_GC, true);
1452	1904	put_gc_inode(&gc_list);
1453	1905
1454	1906	if (!gc_only && get_valid_blocks(sbi, segno, true)) {
..	..	@@ -1487,7 +1939,7 @@
1487	1939	long long block_count;
1488	1940	int segs = secs * sbi->segs_per_sec;
1489	1941
1490		- down_write(&sbi->sb_lock);
	1942	+ f2fs_down_write(&sbi->sb_lock);
1491	1943
1492	1944	section_count = le32_to_cpu(raw_sb->section_count);
1493	1945	segment_count = le32_to_cpu(raw_sb->segment_count);
..	..	@@ -1508,7 +1960,7 @@
1508	1960	cpu_to_le32(dev_segs + segs);
1509	1961	}
1510	1962
1511		- up_write(&sbi->sb_lock);
	1963	+ f2fs_up_write(&sbi->sb_lock);
1512	1964	}
1513	1965
1514	1966	static void update_fs_metadata(struct f2fs_sb_info *sbi, int secs)
..	..	@@ -1582,22 +2034,35 @@
1582	2034	secs = div_u64(shrunk_blocks, BLKS_PER_SEC(sbi));
1583	2035
1584	2036	/* stop other GC */
1585		- if (!down_write_trylock(&sbi->gc_lock))
	2037	+ if (!f2fs_down_write_trylock(&sbi->gc_lock))
1586	2038	return -EAGAIN;
1587	2039
1588	2040	/* stop CP to protect MAIN_SEC in free_segment_range */
1589	2041	f2fs_lock_op(sbi);
	2042	+
	2043	+ spin_lock(&sbi->stat_lock);
	2044	+ if (shrunk_blocks + valid_user_blocks(sbi) +
	2045	+ sbi->current_reserved_blocks + sbi->unusable_block_count +
	2046	+ F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count)
	2047	+ err = -ENOSPC;
	2048	+ spin_unlock(&sbi->stat_lock);
	2049	+
	2050	+ if (err)
	2051	+ goto out_unlock;
	2052	+
1590	2053	err = free_segment_range(sbi, secs, true);
	2054	+
	2055	+out_unlock:
1591	2056	f2fs_unlock_op(sbi);
1592		- up_write(&sbi->gc_lock);
	2057	+ f2fs_up_write(&sbi->gc_lock);
1593	2058	if (err)
1594	2059	return err;
1595	2060
1596	2061	set_sbi_flag(sbi, SBI_IS_RESIZEFS);
1597	2062
1598	2063	freeze_super(sbi->sb);
1599		- down_write(&sbi->gc_lock);
1600		- mutex_lock(&sbi->cp_mutex);
	2064	+ f2fs_down_write(&sbi->gc_lock);
	2065	+ f2fs_down_write(&sbi->cp_global_sem);
1601	2066
1602	2067	spin_lock(&sbi->stat_lock);
1603	2068	if (shrunk_blocks + valid_user_blocks(sbi) +
..	..	@@ -1642,8 +2107,8 @@
1642	2107	spin_unlock(&sbi->stat_lock);
1643	2108	}
1644	2109	out_err:
1645		- mutex_unlock(&sbi->cp_mutex);
1646		- up_write(&sbi->gc_lock);
	2110	+ f2fs_up_write(&sbi->cp_global_sem);
	2111	+ f2fs_up_write(&sbi->gc_lock);
1647	2112	thaw_super(sbi->sb);
1648	2113	clear_sbi_flag(sbi, SBI_IS_RESIZEFS);
1649	2114	return err;