~hc/RK356X_SDK_RELEASE.git

..	..	@@ -18,9 +18,14 @@
18	18	#include "extent_io.h"
19	19	#include "inode-map.h"
20	20	#include "volumes.h"
	21	+#include "space-info.h"
	22	+#include "delalloc-space.h"
	23	+#include "block-group.h"
	24	+#include "discard.h"
21	25
22	26	#define BITS_PER_BITMAP (PAGE_SIZE * 8UL)
23		-#define MAX_CACHE_BYTES_PER_GIG SZ_32K
	27	+#define MAX_CACHE_BYTES_PER_GIG SZ_64K
	28	+#define FORCE_EXTENT_THRESHOLD SZ_1M
24	29
25	30	struct btrfs_trim_range {
26	31	u64 start;
..	..	@@ -28,6 +33,8 @@
28	33	struct list_head list;
29	34	};
30	35
	36	+static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
	37	+ struct btrfs_free_space *bitmap_info);
31	38	static int link_free_space(struct btrfs_free_space_ctl *ctl,
32	39	struct btrfs_free_space *info);
33	40	static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
..	..	@@ -75,7 +82,7 @@
75	82	* sure NOFS is set to keep us from deadlocking.
76	83	*/
77	84	nofs_flag = memalloc_nofs_save();
78		- inode = btrfs_iget_path(fs_info->sb, &location, root, NULL, path);
	85	+ inode = btrfs_iget_path(fs_info->sb, location.objectid, root, path);
79	86	btrfs_release_path(path);
80	87	memalloc_nofs_restore(nofs_flag);
81	88	if (IS_ERR(inode))
..	..	@@ -88,10 +95,10 @@
88	95	return inode;
89	96	}
90	97
91		-struct inode lookup_free_space_inode(struct btrfs_fs_info fs_info,
92		- struct btrfs_block_group_cache
93		- block_group, struct btrfs_path path)
	98	+struct inode lookup_free_space_inode(struct btrfs_block_group block_group,
	99	+ struct btrfs_path *path)
94	100	{
	101	+ struct btrfs_fs_info *fs_info = block_group->fs_info;
95	102	struct inode *inode = NULL;
96	103	u32 flags = BTRFS_INODE_NODATASUM \| BTRFS_INODE_NODATACOW;
97	104
..	..	@@ -103,7 +110,7 @@
103	110	return inode;
104	111
105	112	inode = __lookup_free_space_inode(fs_info->tree_root, path,
106		- block_group->key.objectid);
	113	+ block_group->start);
107	114	if (IS_ERR(inode))
108	115	return inode;
109	116
..	..	@@ -185,20 +192,19 @@
185	192	return 0;
186	193	}
187	194
188		-int create_free_space_inode(struct btrfs_fs_info *fs_info,
189		- struct btrfs_trans_handle *trans,
190		- struct btrfs_block_group_cache *block_group,
	195	+int create_free_space_inode(struct btrfs_trans_handle *trans,
	196	+ struct btrfs_block_group *block_group,
191	197	struct btrfs_path *path)
192	198	{
193	199	int ret;
194	200	u64 ino;
195	201
196		- ret = btrfs_find_free_objectid(fs_info->tree_root, &ino);
	202	+ ret = btrfs_find_free_objectid(trans->fs_info->tree_root, &ino);
197	203	if (ret < 0)
198	204	return ret;
199	205
200		- return __create_free_space_inode(fs_info->tree_root, trans, path, ino,
201		- block_group->key.objectid);
	206	+ return __create_free_space_inode(trans->fs_info->tree_root, trans, path,
	207	+ ino, block_group->start);
202	208	}
203	209
204	210	int btrfs_check_trunc_cache_free_space(struct btrfs_fs_info *fs_info,
..	..	@@ -208,8 +214,8 @@
208	214	int ret;
209	215
210	216	/* 1 for slack space, 1 for updating the inode */
211		- needed_bytes = btrfs_calc_trunc_metadata_size(fs_info, 1) +
212		- btrfs_calc_trans_metadata_size(fs_info, 1);
	217	+ needed_bytes = btrfs_calc_insert_metadata_size(fs_info, 1) +
	218	+ btrfs_calc_metadata_size(fs_info, 1);
213	219
214	220	spin_lock(&rsv->lock);
215	221	if (rsv->reserved < needed_bytes)
..	..	@@ -221,7 +227,7 @@
221	227	}
222	228
223	229	int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans,
224		- struct btrfs_block_group_cache *block_group,
	230	+ struct btrfs_block_group *block_group,
225	231	struct inode *inode)
226	232	{
227	233	struct btrfs_root *root = BTRFS_I(inode)->root;
..	..	@@ -365,10 +371,10 @@
365	371	}
366	372	}
367	373
368		-static int io_ctl_prepare_pages(struct btrfs_io_ctl io_ctl, struct inode inode,
369		- int uptodate)
	374	+static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)
370	375	{
371	376	struct page *page;
	377	+ struct inode *inode = io_ctl->inode;
372	378	gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
373	379	int i;
374	380
..	..	@@ -407,8 +413,6 @@
407	413
408	414	static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
409	415	{
410		- __le64 *val;
411		-
412	416	io_ctl_map_page(io_ctl, 1);
413	417
414	418	/*
..	..	@@ -423,14 +427,13 @@
423	427	io_ctl->size -= sizeof(u64) * 2;
424	428	}
425	429
426		- val = io_ctl->cur;
427		- *val = cpu_to_le64(generation);
	430	+ put_unaligned_le64(generation, io_ctl->cur);
428	431	io_ctl->cur += sizeof(u64);
429	432	}
430	433
431	434	static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
432	435	{
433		- __le64 *gen;
	436	+ u64 cache_gen;
434	437
435	438	/*
436	439	* Skip the crc area. If we don't check crcs then we just have a 64bit
..	..	@@ -445,11 +448,11 @@
445	448	io_ctl->size -= sizeof(u64) * 2;
446	449	}
447	450
448		- gen = io_ctl->cur;
449		- if (le64_to_cpu(*gen) != generation) {
	451	+ cache_gen = get_unaligned_le64(io_ctl->cur);
	452	+ if (cache_gen != generation) {
450	453	btrfs_err_rl(io_ctl->fs_info,
451	454	"space cache generation (%llu) does not match inode (%llu)",
452		- *gen, generation);
	455	+ cache_gen, generation);
453	456	io_ctl_unmap_page(io_ctl);
454	457	return -EIO;
455	458	}
..	..	@@ -471,9 +474,8 @@
471	474	if (index == 0)
472	475	offset = sizeof(u32) * io_ctl->num_pages;
473	476
474		- crc = btrfs_csum_data(io_ctl->orig + offset, crc,
475		- PAGE_SIZE - offset);
476		- btrfs_csum_final(crc, (u8 *)&crc);
	477	+ crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);
	478	+ btrfs_crc32c_final(crc, (u8 *)&crc);
477	479	io_ctl_unmap_page(io_ctl);
478	480	tmp = page_address(io_ctl->pages[0]);
479	481	tmp += index;
..	..	@@ -499,9 +501,8 @@
499	501	val = *tmp;
500	502
501	503	io_ctl_map_page(io_ctl, 0);
502		- crc = btrfs_csum_data(io_ctl->orig + offset, crc,
503		- PAGE_SIZE - offset);
504		- btrfs_csum_final(crc, (u8 *)&crc);
	504	+ crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);
	505	+ btrfs_crc32c_final(crc, (u8 *)&crc);
505	506	if (val != crc) {
506	507	btrfs_err_rl(io_ctl->fs_info,
507	508	"csum mismatch on free space cache");
..	..	@@ -521,8 +522,8 @@
521	522	return -ENOSPC;
522	523
523	524	entry = io_ctl->cur;
524		- entry->offset = cpu_to_le64(offset);
525		- entry->bytes = cpu_to_le64(bytes);
	525	+ put_unaligned_le64(offset, &entry->offset);
	526	+ put_unaligned_le64(bytes, &entry->bytes);
526	527	entry->type = (bitmap) ? BTRFS_FREE_SPACE_BITMAP :
527	528	BTRFS_FREE_SPACE_EXTENT;
528	529	io_ctl->cur += sizeof(struct btrfs_free_space_entry);
..	..	@@ -595,8 +596,8 @@
595	596	}
596	597
597	598	e = io_ctl->cur;
598		- entry->offset = le64_to_cpu(e->offset);
599		- entry->bytes = le64_to_cpu(e->bytes);
	599	+ entry->offset = get_unaligned_le64(&e->offset);
	600	+ entry->bytes = get_unaligned_le64(&e->bytes);
600	601	*type = e->type;
601	602	io_ctl->cur += sizeof(struct btrfs_free_space_entry);
602	603	io_ctl->size -= sizeof(struct btrfs_free_space_entry);
..	..	@@ -728,7 +729,7 @@
728	729
729	730	readahead_cache(inode);
730	731
731		- ret = io_ctl_prepare_pages(&io_ctl, inode, 1);
	732	+ ret = io_ctl_prepare_pages(&io_ctl, true);
732	733	if (ret)
733	734	goto out;
734	735
..	..	@@ -753,6 +754,16 @@
753	754	kmem_cache_free(btrfs_free_space_cachep, e);
754	755	goto free_cache;
755	756	}
	757	+
	758	+ /*
	759	+ * Sync discard ensures that the free space cache is always
	760	+ * trimmed. So when reading this in, the state should reflect
	761	+ * that. We also do this for async as a stop gap for lack of
	762	+ * persistence.
	763	+ */
	764	+ if (btrfs_test_opt(fs_info, DISCARD_SYNC) \|\|
	765	+ btrfs_test_opt(fs_info, DISCARD_ASYNC))
	766	+ e->trim_state = BTRFS_TRIM_STATE_TRIMMED;
756	767
757	768	if (!e->bytes) {
758	769	ret = -1;
..	..	@@ -809,12 +820,19 @@
809	820	ret = io_ctl_read_bitmap(&io_ctl, e);
810	821	if (ret)
811	822	goto free_cache;
	823	+ e->bitmap_extents = count_bitmap_extents(ctl, e);
	824	+ if (!btrfs_free_space_trimmed(e)) {
	825	+ ctl->discardable_extents[BTRFS_STAT_CURR] +=
	826	+ e->bitmap_extents;
	827	+ ctl->discardable_bytes[BTRFS_STAT_CURR] += e->bytes;
	828	+ }
812	829	}
813	830
814	831	io_ctl_drop_pages(&io_ctl);
815	832	merge_space_tree(ctl);
816	833	ret = 1;
817	834	out:
	835	+ btrfs_discard_update_discardable(ctl->private, ctl);
818	836	io_ctl_free(&io_ctl);
819	837	return ret;
820	838	free_cache:
..	..	@@ -823,15 +841,15 @@
823	841	goto out;
824	842	}
825	843
826		-int load_free_space_cache(struct btrfs_fs_info *fs_info,
827		- struct btrfs_block_group_cache *block_group)
	844	+int load_free_space_cache(struct btrfs_block_group *block_group)
828	845	{
	846	+ struct btrfs_fs_info *fs_info = block_group->fs_info;
829	847	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
830	848	struct inode *inode;
831	849	struct btrfs_path *path;
832	850	int ret = 0;
833	851	bool matched;
834		- u64 used = btrfs_block_group_used(&block_group->item);
	852	+ u64 used = block_group->used;
835	853
836	854	/*
837	855	* If this block group has been marked to be cleared for one reason or
..	..	@@ -869,7 +887,7 @@
869	887	* once created get their ->cached field set to BTRFS_CACHE_FINISHED so
870	888	* we will never try to read their inode item while the fs is mounted.
871	889	*/
872		- inode = lookup_free_space_inode(fs_info, block_group, path);
	890	+ inode = lookup_free_space_inode(block_group, path);
873	891	if (IS_ERR(inode)) {
874	892	btrfs_free_path(path);
875	893	return 0;
..	..	@@ -885,13 +903,13 @@
885	903	spin_unlock(&block_group->lock);
886	904
887	905	ret = __load_free_space_cache(fs_info->tree_root, inode, ctl,
888		- path, block_group->key.objectid);
	906	+ path, block_group->start);
889	907	btrfs_free_path(path);
890	908	if (ret <= 0)
891	909	goto out;
892	910
893	911	spin_lock(&ctl->tree_lock);
894		- matched = (ctl->free_space == (block_group->key.offset - used -
	912	+ matched = (ctl->free_space == (block_group->length - used -
895	913	block_group->bytes_super));
896	914	spin_unlock(&ctl->tree_lock);
897	915
..	..	@@ -899,7 +917,7 @@
899	917	__btrfs_remove_free_space_cache(ctl);
900	918	btrfs_warn(fs_info,
901	919	"block group %llu has wrong amount of free space",
902		- block_group->key.objectid);
	920	+ block_group->start);
903	921	ret = -1;
904	922	}
905	923	out:
..	..	@@ -912,7 +930,7 @@
912	930
913	931	btrfs_warn(fs_info,
914	932	"failed to load free space cache for block group %llu, rebuilding it now",
915		- block_group->key.objectid);
	933	+ block_group->start);
916	934	}
917	935
918	936	iput(inode);
..	..	@@ -922,7 +940,7 @@
922	940	static noinline_for_stack
923	941	int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,
924	942	struct btrfs_free_space_ctl *ctl,
925		- struct btrfs_block_group_cache *block_group,
	943	+ struct btrfs_block_group *block_group,
926	944	int entries, int bitmaps,
927	945	struct list_head *bitmap_list)
928	946	{
..	..	@@ -1015,7 +1033,7 @@
1015	1033	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
1016	1034	if (ret < 0) {
1017	1035	clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1,
1018		- EXTENT_DIRTY \| EXTENT_DELALLOC, 0, 0, NULL);
	1036	+ EXTENT_DELALLOC, 0, 0, NULL);
1019	1037	goto fail;
1020	1038	}
1021	1039	leaf = path->nodes[0];
..	..	@@ -1027,9 +1045,8 @@
1027	1045	if (found_key.objectid != BTRFS_FREE_SPACE_OBJECTID \|\|
1028	1046	found_key.offset != offset) {
1029	1047	clear_extent_bit(&BTRFS_I(inode)->io_tree, 0,
1030		- inode->i_size - 1,
1031		- EXTENT_DIRTY \| EXTENT_DELALLOC, 0, 0,
1032		- NULL);
	1048	+ inode->i_size - 1, EXTENT_DELALLOC, 0,
	1049	+ 0, NULL);
1033	1050	btrfs_release_path(path);
1034	1051	goto fail;
1035	1052	}
..	..	@@ -1050,9 +1067,9 @@
1050	1067	return -1;
1051	1068	}
1052	1069
1053		-static noinline_for_stack int
1054		-write_pinned_extent_entries(struct btrfs_fs_info *fs_info,
1055		- struct btrfs_block_group_cache *block_group,
	1070	+static noinline_for_stack int write_pinned_extent_entries(
	1071	+ struct btrfs_trans_handle *trans,
	1072	+ struct btrfs_block_group *block_group,
1056	1073	struct btrfs_io_ctl *io_ctl,
1057	1074	int *entries)
1058	1075	{
..	..	@@ -1070,11 +1087,11 @@
1070	1087	* We shouldn't have switched the pinned extents yet so this is the
1071	1088	* right one
1072	1089	*/
1073		- unpin = fs_info->pinned_extents;
	1090	+ unpin = &trans->transaction->pinned_extents;
1074	1091
1075		- start = block_group->key.objectid;
	1092	+ start = block_group->start;
1076	1093
1077		- while (start < block_group->key.objectid + block_group->key.offset) {
	1094	+ while (start < block_group->start + block_group->length) {
1078	1095	ret = find_first_extent_bit(unpin, start,
1079	1096	&extent_start, &extent_end,
1080	1097	EXTENT_DIRTY, NULL);
..	..	@@ -1082,13 +1099,12 @@
1082	1099	return 0;
1083	1100
1084	1101	/* This pinned extent is out of our range */
1085		- if (extent_start >= block_group->key.objectid +
1086		- block_group->key.offset)
	1102	+ if (extent_start >= block_group->start + block_group->length)
1087	1103	return 0;
1088	1104
1089	1105	extent_start = max(extent_start, start);
1090		- extent_end = min(block_group->key.objectid +
1091		- block_group->key.offset, extent_end + 1);
	1106	+ extent_end = min(block_group->start + block_group->length,
	1107	+ extent_end + 1);
1092	1108	len = extent_end - extent_start;
1093	1109
1094	1110	*entries += 1;
..	..	@@ -1126,7 +1142,7 @@
1126	1142	ret = btrfs_wait_ordered_range(inode, 0, (u64)-1);
1127	1143	if (ret)
1128	1144	clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1,
1129		- EXTENT_DIRTY \| EXTENT_DELALLOC, 0, 0, NULL);
	1145	+ EXTENT_DELALLOC, 0, 0, NULL);
1130	1146
1131	1147	return ret;
1132	1148	}
..	..	@@ -1152,7 +1168,7 @@
1152	1168
1153	1169	static int __btrfs_wait_cache_io(struct btrfs_root *root,
1154	1170	struct btrfs_trans_handle *trans,
1155		- struct btrfs_block_group_cache *block_group,
	1171	+ struct btrfs_block_group *block_group,
1156	1172	struct btrfs_io_ctl *io_ctl,
1157	1173	struct btrfs_path *path, u64 offset)
1158	1174	{
..	..	@@ -1174,13 +1190,10 @@
1174	1190	if (ret) {
1175	1191	invalidate_inode_pages2(inode->i_mapping);
1176	1192	BTRFS_I(inode)->generation = 0;
1177		- if (block_group) {
1178		-#ifdef DEBUG
1179		- btrfs_err(root->fs_info,
1180		- "failed to write free space cache for block group %llu",
1181		- block_group->key.objectid);
1182		-#endif
1183		- }
	1193	+ if (block_group)
	1194	+ btrfs_debug(root->fs_info,
	1195	+ "failed to write free space cache for block group %llu error %d",
	1196	+ block_group->start, ret);
1184	1197	}
1185	1198	btrfs_update_inode(trans, root, inode);
1186	1199
..	..	@@ -1220,12 +1233,12 @@
1220	1233	}
1221	1234
1222	1235	int btrfs_wait_cache_io(struct btrfs_trans_handle *trans,
1223		- struct btrfs_block_group_cache *block_group,
	1236	+ struct btrfs_block_group *block_group,
1224	1237	struct btrfs_path *path)
1225	1238	{
1226	1239	return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans,
1227	1240	block_group, &block_group->io_ctl,
1228		- path, block_group->key.objectid);
	1241	+ path, block_group->start);
1229	1242	}
1230	1243
1231	1244	/**
..	..	@@ -1241,11 +1254,10 @@
1241	1254	*/
1242	1255	static int __btrfs_write_out_cache(struct btrfs_root root, struct inode inode,
1243	1256	struct btrfs_free_space_ctl *ctl,
1244		- struct btrfs_block_group_cache *block_group,
	1257	+ struct btrfs_block_group *block_group,
1245	1258	struct btrfs_io_ctl *io_ctl,
1246	1259	struct btrfs_trans_handle *trans)
1247	1260	{
1248		- struct btrfs_fs_info *fs_info = root->fs_info;
1249	1261	struct extent_state *cached_state = NULL;
1250	1262	LIST_HEAD(bitmap_list);
1251	1263	int entries = 0;
..	..	@@ -1277,7 +1289,7 @@
1277	1289	}
1278	1290
1279	1291	/* Lock all pages first so we can lock the extent safely. */
1280		- ret = io_ctl_prepare_pages(io_ctl, inode, 0);
	1292	+ ret = io_ctl_prepare_pages(io_ctl, false);
1281	1293	if (ret)
1282	1294	goto out_unlock;
1283	1295
..	..	@@ -1303,8 +1315,7 @@
1303	1315	* If this changes while we are working we'll get added back to
1304	1316	* the dirty list and redo it. No locking needed
1305	1317	*/
1306		- ret = write_pinned_extent_entries(fs_info, block_group,
1307		- io_ctl, &entries);
	1318	+ ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);
1308	1319	if (ret)
1309	1320	goto out_nospc_locked;
1310	1321
..	..	@@ -1323,8 +1334,9 @@
1323	1334	io_ctl_zero_remaining_pages(io_ctl);
1324	1335
1325	1336	/* Everything is written out, now we dirty the pages in the file. */
1326		- ret = btrfs_dirty_pages(inode, io_ctl->pages, io_ctl->num_pages, 0,
1327		- i_size_read(inode), &cached_state);
	1337	+ ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,
	1338	+ io_ctl->num_pages, 0, i_size_read(inode),
	1339	+ &cached_state);
1328	1340	if (ret)
1329	1341	goto out_nospc;
1330	1342
..	..	@@ -1342,7 +1354,7 @@
1342	1354
1343	1355	/*
1344	1356	* at this point the pages are under IO and we're happy,
1345		- * The caller is responsible for waiting on them and updating the
	1357	+ * The caller is responsible for waiting on them and updating
1346	1358	* the cache and the inode
1347	1359	*/
1348	1360	io_ctl->entries = entries;
..	..	@@ -1353,18 +1365,6 @@
1353	1365	goto out;
1354	1366
1355	1367	return 0;
1356		-
1357		-out:
1358		- io_ctl->inode = NULL;
1359		- io_ctl_free(io_ctl);
1360		- if (ret) {
1361		- invalidate_inode_pages2(inode->i_mapping);
1362		- BTRFS_I(inode)->generation = 0;
1363		- }
1364		- btrfs_update_inode(trans, root, inode);
1365		- if (must_iput)
1366		- iput(inode);
1367		- return ret;
1368	1368
1369	1369	out_nospc_locked:
1370	1370	cleanup_bitmap_list(&bitmap_list);
..	..	@@ -1378,14 +1378,24 @@
1378	1378	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))
1379	1379	up_write(&block_group->data_rwsem);
1380	1380
1381		- goto out;
	1381	+out:
	1382	+ io_ctl->inode = NULL;
	1383	+ io_ctl_free(io_ctl);
	1384	+ if (ret) {
	1385	+ invalidate_inode_pages2(inode->i_mapping);
	1386	+ BTRFS_I(inode)->generation = 0;
	1387	+ }
	1388	+ btrfs_update_inode(trans, root, inode);
	1389	+ if (must_iput)
	1390	+ iput(inode);
	1391	+ return ret;
1382	1392	}
1383	1393
1384		-int btrfs_write_out_cache(struct btrfs_fs_info *fs_info,
1385		- struct btrfs_trans_handle *trans,
1386		- struct btrfs_block_group_cache *block_group,
	1394	+int btrfs_write_out_cache(struct btrfs_trans_handle *trans,
	1395	+ struct btrfs_block_group *block_group,
1387	1396	struct btrfs_path *path)
1388	1397	{
	1398	+ struct btrfs_fs_info *fs_info = trans->fs_info;
1389	1399	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
1390	1400	struct inode *inode;
1391	1401	int ret = 0;
..	..	@@ -1397,18 +1407,16 @@
1397	1407	}
1398	1408	spin_unlock(&block_group->lock);
1399	1409
1400		- inode = lookup_free_space_inode(fs_info, block_group, path);
	1410	+ inode = lookup_free_space_inode(block_group, path);
1401	1411	if (IS_ERR(inode))
1402	1412	return 0;
1403	1413
1404	1414	ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl,
1405	1415	block_group, &block_group->io_ctl, trans);
1406	1416	if (ret) {
1407		-#ifdef DEBUG
1408		- btrfs_err(fs_info,
1409		- "failed to write free space cache for block group %llu",
1410		- block_group->key.objectid);
1411		-#endif
	1417	+ btrfs_debug(fs_info,
	1418	+ "failed to write free space cache for block group %llu error %d",
	1419	+ block_group->start, ret);
1412	1420	spin_lock(&block_group->lock);
1413	1421	block_group->disk_cache_state = BTRFS_DC_ERROR;
1414	1422	spin_unlock(&block_group->lock);
..	..	@@ -1633,6 +1641,11 @@
1633	1641	{
1634	1642	rb_erase(&info->offset_index, &ctl->free_space_offset);
1635	1643	ctl->free_extents--;
	1644	+
	1645	+ if (!info->bitmap && !btrfs_free_space_trimmed(info)) {
	1646	+ ctl->discardable_extents[BTRFS_STAT_CURR]--;
	1647	+ ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes;
	1648	+ }
1636	1649	}
1637	1650
1638	1651	static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
..	..	@@ -1653,6 +1666,11 @@
1653	1666	if (ret)
1654	1667	return ret;
1655	1668
	1669	+ if (!info->bitmap && !btrfs_free_space_trimmed(info)) {
	1670	+ ctl->discardable_extents[BTRFS_STAT_CURR]++;
	1671	+ ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes;
	1672	+ }
	1673	+
1656	1674	ctl->free_space += info->bytes;
1657	1675	ctl->free_extents++;
1658	1676	return ret;
..	..	@@ -1660,11 +1678,11 @@
1660	1678
1661	1679	static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl)
1662	1680	{
1663		- struct btrfs_block_group_cache *block_group = ctl->private;
	1681	+ struct btrfs_block_group *block_group = ctl->private;
1664	1682	u64 max_bytes;
1665	1683	u64 bitmap_bytes;
1666	1684	u64 extent_bytes;
1667		- u64 size = block_group->key.offset;
	1685	+ u64 size = block_group->length;
1668	1686	u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit;
1669	1687	u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg);
1670	1688
..	..	@@ -1673,26 +1691,17 @@
1673	1691	ASSERT(ctl->total_bitmaps <= max_bitmaps);
1674	1692
1675	1693	/*
1676		- * The goal is to keep the total amount of memory used per 1gb of space
1677		- * at or below 32k, so we need to adjust how much memory we allow to be
1678		- * used by extent based free space tracking
	1694	+ * We are trying to keep the total amount of memory used per 1GiB of
	1695	+ * space to be MAX_CACHE_BYTES_PER_GIG. However, with a reclamation
	1696	+ * mechanism of pulling extents >= FORCE_EXTENT_THRESHOLD out of
	1697	+ * bitmaps, we may end up using more memory than this.
1679	1698	*/
1680	1699	if (size < SZ_1G)
1681	1700	max_bytes = MAX_CACHE_BYTES_PER_GIG;
1682	1701	else
1683	1702	max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G);
1684	1703
1685		- /*
1686		- * we want to account for 1 more bitmap than what we have so we can make
1687		- * sure we don't go over our overall goal of MAX_CACHE_BYTES_PER_GIG as
1688		- * we add more bitmaps.
1689		- */
1690		- bitmap_bytes = (ctl->total_bitmaps + 1) * ctl->unit;
1691		-
1692		- if (bitmap_bytes >= max_bytes) {
1693		- ctl->extents_thresh = 0;
1694		- return;
1695		- }
	1704	+ bitmap_bytes = ctl->total_bitmaps * ctl->unit;
1696	1705
1697	1706	/*
1698	1707	* we want the extent entry threshold to always be at most 1/2 the max
..	..	@@ -1709,17 +1718,31 @@
1709	1718	struct btrfs_free_space *info,
1710	1719	u64 offset, u64 bytes)
1711	1720	{
1712		- unsigned long start, count;
	1721	+ unsigned long start, count, end;
	1722	+ int extent_delta = -1;
1713	1723
1714	1724	start = offset_to_bit(info->offset, ctl->unit, offset);
1715	1725	count = bytes_to_bits(bytes, ctl->unit);
1716		- ASSERT(start + count <= BITS_PER_BITMAP);
	1726	+ end = start + count;
	1727	+ ASSERT(end <= BITS_PER_BITMAP);
1717	1728
1718	1729	bitmap_clear(info->bitmap, start, count);
1719	1730
1720	1731	info->bytes -= bytes;
1721	1732	if (info->max_extent_size > ctl->unit)
1722	1733	info->max_extent_size = 0;
	1734	+
	1735	+ if (start && test_bit(start - 1, info->bitmap))
	1736	+ extent_delta++;
	1737	+
	1738	+ if (end < BITS_PER_BITMAP && test_bit(end, info->bitmap))
	1739	+ extent_delta++;
	1740	+
	1741	+ info->bitmap_extents += extent_delta;
	1742	+ if (!btrfs_free_space_trimmed(info)) {
	1743	+ ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta;
	1744	+ ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes;
	1745	+ }
1723	1746	}
1724	1747
1725	1748	static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
..	..	@@ -1734,16 +1757,30 @@
1734	1757	struct btrfs_free_space *info, u64 offset,
1735	1758	u64 bytes)
1736	1759	{
1737		- unsigned long start, count;
	1760	+ unsigned long start, count, end;
	1761	+ int extent_delta = 1;
1738	1762
1739	1763	start = offset_to_bit(info->offset, ctl->unit, offset);
1740	1764	count = bytes_to_bits(bytes, ctl->unit);
1741		- ASSERT(start + count <= BITS_PER_BITMAP);
	1765	+ end = start + count;
	1766	+ ASSERT(end <= BITS_PER_BITMAP);
1742	1767
1743	1768	bitmap_set(info->bitmap, start, count);
1744	1769
1745	1770	info->bytes += bytes;
1746	1771	ctl->free_space += bytes;
	1772	+
	1773	+ if (start && test_bit(start - 1, info->bitmap))
	1774	+ extent_delta--;
	1775	+
	1776	+ if (end < BITS_PER_BITMAP && test_bit(end, info->bitmap))
	1777	+ extent_delta--;
	1778	+
	1779	+ info->bitmap_extents += extent_delta;
	1780	+ if (!btrfs_free_space_trimmed(info)) {
	1781	+ ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta;
	1782	+ ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes;
	1783	+ }
1747	1784	}
1748	1785
1749	1786	/*
..	..	@@ -1879,11 +1916,35 @@
1879	1916	return NULL;
1880	1917	}
1881	1918
	1919	+static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
	1920	+ struct btrfs_free_space *bitmap_info)
	1921	+{
	1922	+ struct btrfs_block_group *block_group = ctl->private;
	1923	+ u64 bytes = bitmap_info->bytes;
	1924	+ unsigned int rs, re;
	1925	+ int count = 0;
	1926	+
	1927	+ if (!block_group \|\| !bytes)
	1928	+ return count;
	1929	+
	1930	+ bitmap_for_each_set_region(bitmap_info->bitmap, rs, re, 0,
	1931	+ BITS_PER_BITMAP) {
	1932	+ bytes -= (rs - re) * ctl->unit;
	1933	+ count++;
	1934	+
	1935	+ if (!bytes)
	1936	+ break;
	1937	+ }
	1938	+
	1939	+ return count;
	1940	+}
	1941	+
1882	1942	static void add_new_bitmap(struct btrfs_free_space_ctl *ctl,
1883	1943	struct btrfs_free_space *info, u64 offset)
1884	1944	{
1885	1945	info->offset = offset_to_bitmap(ctl, offset);
1886	1946	info->bytes = 0;
	1947	+ info->bitmap_extents = 0;
1887	1948	INIT_LIST_HEAD(&info->list);
1888	1949	link_free_space(ctl, info);
1889	1950	ctl->total_bitmaps++;
..	..	@@ -1894,6 +1955,18 @@
1894	1955	static void free_bitmap(struct btrfs_free_space_ctl *ctl,
1895	1956	struct btrfs_free_space *bitmap_info)
1896	1957	{
	1958	+ /*
	1959	+ * Normally when this is called, the bitmap is completely empty. However,
	1960	+ * if we are blowing up the free space cache for one reason or another
	1961	+ * via __btrfs_remove_free_space_cache(), then it may not be freed and
	1962	+ * we may leave stats on the table.
	1963	+ */
	1964	+ if (bitmap_info->bytes && !btrfs_free_space_trimmed(bitmap_info)) {
	1965	+ ctl->discardable_extents[BTRFS_STAT_CURR] -=
	1966	+ bitmap_info->bitmap_extents;
	1967	+ ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes;
	1968	+
	1969	+ }
1897	1970	unlink_free_space(ctl, bitmap_info);
1898	1971	kmem_cache_free(btrfs_free_space_bitmap_cachep, bitmap_info->bitmap);
1899	1972	kmem_cache_free(btrfs_free_space_cachep, bitmap_info);
..	..	@@ -1980,10 +2053,23 @@
1980	2053
1981	2054	static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl,
1982	2055	struct btrfs_free_space *info, u64 offset,
1983		- u64 bytes)
	2056	+ u64 bytes, enum btrfs_trim_state trim_state)
1984	2057	{
1985	2058	u64 bytes_to_set = 0;
1986	2059	u64 end;
	2060	+
	2061	+ /*
	2062	+ * This is a tradeoff to make bitmap trim state minimal. We mark the
	2063	+ * whole bitmap untrimmed if at any point we add untrimmed regions.
	2064	+ */
	2065	+ if (trim_state == BTRFS_TRIM_STATE_UNTRIMMED) {
	2066	+ if (btrfs_free_space_trimmed(info)) {
	2067	+ ctl->discardable_extents[BTRFS_STAT_CURR] +=
	2068	+ info->bitmap_extents;
	2069	+ ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes;
	2070	+ }
	2071	+ info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
	2072	+ }
1987	2073
1988	2074	end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit);
1989	2075
..	..	@@ -2004,7 +2090,7 @@
2004	2090	static bool use_bitmap(struct btrfs_free_space_ctl *ctl,
2005	2091	struct btrfs_free_space *info)
2006	2092	{
2007		- struct btrfs_block_group_cache *block_group = ctl->private;
	2093	+ struct btrfs_block_group *block_group = ctl->private;
2008	2094	struct btrfs_fs_info *fs_info = block_group->fs_info;
2009	2095	bool forced = false;
2010	2096
..	..	@@ -2012,6 +2098,10 @@
2012	2098	if (btrfs_should_fragment_free_space(block_group))
2013	2099	forced = true;
2014	2100	#endif
	2101	+
	2102	+ /* This is a way to reclaim large regions from the bitmaps. */
	2103	+ if (!forced && info->bytes >= FORCE_EXTENT_THRESHOLD)
	2104	+ return false;
2015	2105
2016	2106	/*
2017	2107	* If we are below the extents threshold then we can add this as an
..	..	@@ -2025,8 +2115,8 @@
2025	2115	* of cache left then go ahead an dadd them, no sense in adding
2026	2116	* the overhead of a bitmap if we don't have to.
2027	2117	*/
2028		- if (info->bytes <= fs_info->sectorsize * 4) {
2029		- if (ctl->free_extents * 2 <= ctl->extents_thresh)
	2118	+ if (info->bytes <= fs_info->sectorsize * 8) {
	2119	+ if (ctl->free_extents * 3 <= ctl->extents_thresh)
2030	2120	return false;
2031	2121	} else {
2032	2122	return false;
..	..	@@ -2041,7 +2131,7 @@
2041	2131	* so allow those block groups to still be allowed to have a bitmap
2042	2132	* entry.
2043	2133	*/
2044		- if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->key.offset)
	2134	+ if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length)
2045	2135	return false;
2046	2136
2047	2137	return true;
..	..	@@ -2056,13 +2146,15 @@
2056	2146	struct btrfs_free_space *info)
2057	2147	{
2058	2148	struct btrfs_free_space *bitmap_info;
2059		- struct btrfs_block_group_cache *block_group = NULL;
	2149	+ struct btrfs_block_group *block_group = NULL;
2060	2150	int added = 0;
2061	2151	u64 bytes, offset, bytes_added;
	2152	+ enum btrfs_trim_state trim_state;
2062	2153	int ret;
2063	2154
2064	2155	bytes = info->bytes;
2065	2156	offset = info->offset;
	2157	+ trim_state = info->trim_state;
2066	2158
2067	2159	if (!ctl->op->use_bitmap(ctl, info))
2068	2160	return 0;
..	..	@@ -2097,8 +2189,8 @@
2097	2189	}
2098	2190
2099	2191	if (entry->offset == offset_to_bitmap(ctl, offset)) {
2100		- bytes_added = add_bytes_to_bitmap(ctl, entry,
2101		- offset, bytes);
	2192	+ bytes_added = add_bytes_to_bitmap(ctl, entry, offset,
	2193	+ bytes, trim_state);
2102	2194	bytes -= bytes_added;
2103	2195	offset += bytes_added;
2104	2196	}
..	..	@@ -2117,7 +2209,8 @@
2117	2209	goto new_bitmap;
2118	2210	}
2119	2211
2120		- bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes);
	2212	+ bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes,
	2213	+ trim_state);
2121	2214	bytes -= bytes_added;
2122	2215	offset += bytes_added;
2123	2216	added = 0;
..	..	@@ -2151,6 +2244,7 @@
2151	2244	/* allocate the bitmap */
2152	2245	info->bitmap = kmem_cache_zalloc(btrfs_free_space_bitmap_cachep,
2153	2246	GFP_NOFS);
	2247	+ info->trim_state = BTRFS_TRIM_STATE_TRIMMED;
2154	2248	spin_lock(&ctl->tree_lock);
2155	2249	if (!info->bitmap) {
2156	2250	ret = -ENOMEM;
..	..	@@ -2170,6 +2264,22 @@
2170	2264	return ret;
2171	2265	}
2172	2266
	2267	+/*
	2268	+ * Free space merging rules:
	2269	+ * 1) Merge trimmed areas together
	2270	+ * 2) Let untrimmed areas coalesce with trimmed areas
	2271	+ * 3) Always pull neighboring regions from bitmaps
	2272	+ *
	2273	+ * The above rules are for when we merge free space based on btrfs_trim_state.
	2274	+ * Rules 2 and 3 are subtle because they are suboptimal, but are done for the
	2275	+ * same reason: to promote larger extent regions which makes life easier for
	2276	+ * find_free_extent(). Rule 2 enables coalescing based on the common path
	2277	+ * being returning free space from btrfs_finish_extent_commit(). So when free
	2278	+ * space is trimmed, it will prevent aggregating trimmed new region and
	2279	+ * untrimmed regions in the rb_tree. Rule 3 is purely to obtain larger extents
	2280	+ * and provide find_free_extent() with the largest extents possible hoping for
	2281	+ * the reuse path.
	2282	+ */
2173	2283	static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl,
2174	2284	struct btrfs_free_space *info, bool update_stat)
2175	2285	{
..	..	@@ -2178,6 +2288,7 @@
2178	2288	bool merged = false;
2179	2289	u64 offset = info->offset;
2180	2290	u64 bytes = info->bytes;
	2291	+ const bool is_trimmed = btrfs_free_space_trimmed(info);
2181	2292
2182	2293	/*
2183	2294	* first we want to see if there is free space adjacent to the range we
..	..	@@ -2191,7 +2302,9 @@
2191	2302	else if (!right_info)
2192	2303	left_info = tree_search_offset(ctl, offset - 1, 0, 0);
2193	2304
2194		- if (right_info && !right_info->bitmap) {
	2305	+ /* See try_merge_free_space() comment. */
	2306	+ if (right_info && !right_info->bitmap &&
	2307	+ (!is_trimmed \|\| btrfs_free_space_trimmed(right_info))) {
2195	2308	if (update_stat)
2196	2309	unlink_free_space(ctl, right_info);
2197	2310	else
..	..	@@ -2201,8 +2314,10 @@
2201	2314	merged = true;
2202	2315	}
2203	2316
	2317	+ /* See try_merge_free_space() comment. */
2204	2318	if (left_info && !left_info->bitmap &&
2205		- left_info->offset + left_info->bytes == offset) {
	2319	+ left_info->offset + left_info->bytes == offset &&
	2320	+ (!is_trimmed \|\| btrfs_free_space_trimmed(left_info))) {
2206	2321	if (update_stat)
2207	2322	unlink_free_space(ctl, left_info);
2208	2323	else
..	..	@@ -2237,6 +2352,10 @@
2237	2352	return false;
2238	2353	bytes = (j - i) * ctl->unit;
2239	2354	info->bytes += bytes;
	2355	+
	2356	+ /* See try_merge_free_space() comment. */
	2357	+ if (!btrfs_free_space_trimmed(bitmap))
	2358	+ info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
2240	2359
2241	2360	if (update_stat)
2242	2361	bitmap_clear_bits(ctl, bitmap, end, bytes);
..	..	@@ -2291,6 +2410,10 @@
2291	2410	info->offset -= bytes;
2292	2411	info->bytes += bytes;
2293	2412
	2413	+ /* See try_merge_free_space() comment. */
	2414	+ if (!btrfs_free_space_trimmed(bitmap))
	2415	+ info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
	2416	+
2294	2417	if (update_stat)
2295	2418	bitmap_clear_bits(ctl, bitmap, info->offset, bytes);
2296	2419	else
..	..	@@ -2340,10 +2463,13 @@
2340	2463
2341	2464	int __btrfs_add_free_space(struct btrfs_fs_info *fs_info,
2342	2465	struct btrfs_free_space_ctl *ctl,
2343		- u64 offset, u64 bytes)
	2466	+ u64 offset, u64 bytes,
	2467	+ enum btrfs_trim_state trim_state)
2344	2468	{
	2469	+ struct btrfs_block_group *block_group = ctl->private;
2345	2470	struct btrfs_free_space *info;
2346	2471	int ret = 0;
	2472	+ u64 filter_bytes = bytes;
2347	2473
2348	2474	info = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
2349	2475	if (!info)
..	..	@@ -2351,6 +2477,7 @@
2351	2477
2352	2478	info->offset = offset;
2353	2479	info->bytes = bytes;
	2480	+ info->trim_state = trim_state;
2354	2481	RB_CLEAR_NODE(&info->offset_index);
2355	2482
2356	2483	spin_lock(&ctl->tree_lock);
..	..	@@ -2379,10 +2506,13 @@
2379	2506	*/
2380	2507	steal_from_bitmap(ctl, info, true);
2381	2508
	2509	+ filter_bytes = max(filter_bytes, info->bytes);
	2510	+
2382	2511	ret = link_free_space(ctl, info);
2383	2512	if (ret)
2384	2513	kmem_cache_free(btrfs_free_space_cachep, info);
2385	2514	out:
	2515	+ btrfs_discard_update_discardable(block_group, ctl);
2386	2516	spin_unlock(&ctl->tree_lock);
2387	2517
2388	2518	if (ret) {
..	..	@@ -2390,10 +2520,47 @@
2390	2520	ASSERT(ret != -EEXIST);
2391	2521	}
2392	2522
	2523	+ if (trim_state != BTRFS_TRIM_STATE_TRIMMED) {
	2524	+ btrfs_discard_check_filter(block_group, filter_bytes);
	2525	+ btrfs_discard_queue_work(&fs_info->discard_ctl, block_group);
	2526	+ }
	2527	+
2393	2528	return ret;
2394	2529	}
2395	2530
2396		-int btrfs_remove_free_space(struct btrfs_block_group_cache *block_group,
	2531	+int btrfs_add_free_space(struct btrfs_block_group *block_group,
	2532	+ u64 bytenr, u64 size)
	2533	+{
	2534	+ enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
	2535	+
	2536	+ if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC))
	2537	+ trim_state = BTRFS_TRIM_STATE_TRIMMED;
	2538	+
	2539	+ return __btrfs_add_free_space(block_group->fs_info,
	2540	+ block_group->free_space_ctl,
	2541	+ bytenr, size, trim_state);
	2542	+}
	2543	+
	2544	+/*
	2545	+ * This is a subtle distinction because when adding free space back in general,
	2546	+ * we want it to be added as untrimmed for async. But in the case where we add
	2547	+ * it on loading of a block group, we want to consider it trimmed.
	2548	+ */
	2549	+int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group,
	2550	+ u64 bytenr, u64 size)
	2551	+{
	2552	+ enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
	2553	+
	2554	+ if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC) \|\|
	2555	+ btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC))
	2556	+ trim_state = BTRFS_TRIM_STATE_TRIMMED;
	2557	+
	2558	+ return __btrfs_add_free_space(block_group->fs_info,
	2559	+ block_group->free_space_ctl,
	2560	+ bytenr, size, trim_state);
	2561	+}
	2562	+
	2563	+int btrfs_remove_free_space(struct btrfs_block_group *block_group,
2397	2564	u64 offset, u64 bytes)
2398	2565	{
2399	2566	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
..	..	@@ -2465,8 +2632,10 @@
2465	2632	}
2466	2633	spin_unlock(&ctl->tree_lock);
2467	2634
2468		- ret = btrfs_add_free_space(block_group, offset + bytes,
2469		- old_end - (offset + bytes));
	2635	+ ret = __btrfs_add_free_space(block_group->fs_info, ctl,
	2636	+ offset + bytes,
	2637	+ old_end - (offset + bytes),
	2638	+ info->trim_state);
2470	2639	WARN_ON(ret);
2471	2640	goto out;
2472	2641	}
..	..	@@ -2478,12 +2647,13 @@
2478	2647	goto again;
2479	2648	}
2480	2649	out_lock:
	2650	+ btrfs_discard_update_discardable(block_group, ctl);
2481	2651	spin_unlock(&ctl->tree_lock);
2482	2652	out:
2483	2653	return ret;
2484	2654	}
2485	2655
2486		-void btrfs_dump_free_space(struct btrfs_block_group_cache *block_group,
	2656	+void btrfs_dump_free_space(struct btrfs_block_group *block_group,
2487	2657	u64 bytes)
2488	2658	{
2489	2659	struct btrfs_fs_info *fs_info = block_group->fs_info;
..	..	@@ -2508,14 +2678,14 @@
2508	2678	"%d blocks of free space at or bigger than bytes is", count);
2509	2679	}
2510	2680
2511		-void btrfs_init_free_space_ctl(struct btrfs_block_group_cache *block_group)
	2681	+void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group)
2512	2682	{
2513	2683	struct btrfs_fs_info *fs_info = block_group->fs_info;
2514	2684	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
2515	2685
2516	2686	spin_lock_init(&ctl->tree_lock);
2517	2687	ctl->unit = fs_info->sectorsize;
2518		- ctl->start = block_group->key.objectid;
	2688	+ ctl->start = block_group->start;
2519	2689	ctl->private = block_group;
2520	2690	ctl->op = &free_space_op;
2521	2691	INIT_LIST_HEAD(&ctl->trimming_ranges);
..	..	@@ -2535,9 +2705,8 @@
2535	2705	* pointed to by the cluster, someone else raced in and freed the
2536	2706	* cluster already. In that case, we just return without changing anything
2537	2707	*/
2538		-static int
2539		-__btrfs_return_cluster_to_free_space(
2540		- struct btrfs_block_group_cache *block_group,
	2708	+static void __btrfs_return_cluster_to_free_space(
	2709	+ struct btrfs_block_group *block_group,
2541	2710	struct btrfs_free_cluster *cluster)
2542	2711	{
2543	2712	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
..	..	@@ -2545,8 +2714,10 @@
2545	2714	struct rb_node *node;
2546	2715
2547	2716	spin_lock(&cluster->lock);
2548		- if (cluster->block_group != block_group)
2549		- goto out;
	2717	+ if (cluster->block_group != block_group) {
	2718	+ spin_unlock(&cluster->lock);
	2719	+ return;
	2720	+ }
2550	2721
2551	2722	cluster->block_group = NULL;
2552	2723	cluster->window_start = 0;
..	..	@@ -2563,18 +2734,29 @@
2563	2734
2564	2735	bitmap = (entry->bitmap != NULL);
2565	2736	if (!bitmap) {
	2737	+ /* Merging treats extents as if they were new */
	2738	+ if (!btrfs_free_space_trimmed(entry)) {
	2739	+ ctl->discardable_extents[BTRFS_STAT_CURR]--;
	2740	+ ctl->discardable_bytes[BTRFS_STAT_CURR] -=
	2741	+ entry->bytes;
	2742	+ }
	2743	+
2566	2744	try_merge_free_space(ctl, entry, false);
2567	2745	steal_from_bitmap(ctl, entry, false);
	2746	+
	2747	+ /* As we insert directly, update these statistics */
	2748	+ if (!btrfs_free_space_trimmed(entry)) {
	2749	+ ctl->discardable_extents[BTRFS_STAT_CURR]++;
	2750	+ ctl->discardable_bytes[BTRFS_STAT_CURR] +=
	2751	+ entry->bytes;
	2752	+ }
2568	2753	}
2569	2754	tree_insert_offset(&ctl->free_space_offset,
2570	2755	entry->offset, &entry->offset_index, bitmap);
2571	2756	}
2572	2757	cluster->root = RB_ROOT;
2573		-
2574		-out:
2575	2758	spin_unlock(&cluster->lock);
2576	2759	btrfs_put_block_group(block_group);
2577		- return 0;
2578	2760	}
2579	2761
2580	2762	static void __btrfs_remove_free_space_cache_locked(
..	..	@@ -2600,10 +2782,12 @@
2600	2782	{
2601	2783	spin_lock(&ctl->tree_lock);
2602	2784	__btrfs_remove_free_space_cache_locked(ctl);
	2785	+ if (ctl->private)
	2786	+ btrfs_discard_update_discardable(ctl->private, ctl);
2603	2787	spin_unlock(&ctl->tree_lock);
2604	2788	}
2605	2789
2606		-void btrfs_remove_free_space_cache(struct btrfs_block_group_cache *block_group)
	2790	+void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group)
2607	2791	{
2608	2792	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
2609	2793	struct btrfs_free_cluster *cluster;
..	..	@@ -2621,20 +2805,55 @@
2621	2805	cond_resched_lock(&ctl->tree_lock);
2622	2806	}
2623	2807	__btrfs_remove_free_space_cache_locked(ctl);
	2808	+ btrfs_discard_update_discardable(block_group, ctl);
2624	2809	spin_unlock(&ctl->tree_lock);
2625	2810
2626	2811	}
2627	2812
2628		-u64 btrfs_find_space_for_alloc(struct btrfs_block_group_cache *block_group,
	2813	+/**
	2814	+ * btrfs_is_free_space_trimmed - see if everything is trimmed
	2815	+ * @block_group: block_group of interest
	2816	+ *
	2817	+ * Walk @block_group's free space rb_tree to determine if everything is trimmed.
	2818	+ */
	2819	+bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group)
	2820	+{
	2821	+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
	2822	+ struct btrfs_free_space *info;
	2823	+ struct rb_node *node;
	2824	+ bool ret = true;
	2825	+
	2826	+ spin_lock(&ctl->tree_lock);
	2827	+ node = rb_first(&ctl->free_space_offset);
	2828	+
	2829	+ while (node) {
	2830	+ info = rb_entry(node, struct btrfs_free_space, offset_index);
	2831	+
	2832	+ if (!btrfs_free_space_trimmed(info)) {
	2833	+ ret = false;
	2834	+ break;
	2835	+ }
	2836	+
	2837	+ node = rb_next(node);
	2838	+ }
	2839	+
	2840	+ spin_unlock(&ctl->tree_lock);
	2841	+ return ret;
	2842	+}
	2843	+
	2844	+u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group,
2629	2845	u64 offset, u64 bytes, u64 empty_size,
2630	2846	u64 *max_extent_size)
2631	2847	{
2632	2848	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
	2849	+ struct btrfs_discard_ctl *discard_ctl =
	2850	+ &block_group->fs_info->discard_ctl;
2633	2851	struct btrfs_free_space *entry = NULL;
2634	2852	u64 bytes_search = bytes + empty_size;
2635	2853	u64 ret = 0;
2636	2854	u64 align_gap = 0;
2637	2855	u64 align_gap_len = 0;
	2856	+ enum btrfs_trim_state align_gap_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
2638	2857
2639	2858	spin_lock(&ctl->tree_lock);
2640	2859	entry = find_free_space(ctl, &offset, &bytes_search,
..	..	@@ -2645,12 +2864,20 @@
2645	2864	ret = offset;
2646	2865	if (entry->bitmap) {
2647	2866	bitmap_clear_bits(ctl, entry, offset, bytes);
	2867	+
	2868	+ if (!btrfs_free_space_trimmed(entry))
	2869	+ atomic64_add(bytes, &discard_ctl->discard_bytes_saved);
	2870	+
2648	2871	if (!entry->bytes)
2649	2872	free_bitmap(ctl, entry);
2650	2873	} else {
2651	2874	unlink_free_space(ctl, entry);
2652	2875	align_gap_len = offset - entry->offset;
2653	2876	align_gap = entry->offset;
	2877	+ align_gap_trim_state = entry->trim_state;
	2878	+
	2879	+ if (!btrfs_free_space_trimmed(entry))
	2880	+ atomic64_add(bytes, &discard_ctl->discard_bytes_saved);
2654	2881
2655	2882	entry->offset = offset + bytes;
2656	2883	WARN_ON(entry->bytes < bytes + align_gap_len);
..	..	@@ -2662,11 +2889,13 @@
2662	2889	link_free_space(ctl, entry);
2663	2890	}
2664	2891	out:
	2892	+ btrfs_discard_update_discardable(block_group, ctl);
2665	2893	spin_unlock(&ctl->tree_lock);
2666	2894
2667	2895	if (align_gap_len)
2668	2896	__btrfs_add_free_space(block_group->fs_info, ctl,
2669		- align_gap, align_gap_len);
	2897	+ align_gap, align_gap_len,
	2898	+ align_gap_trim_state);
2670	2899	return ret;
2671	2900	}
2672	2901
..	..	@@ -2678,12 +2907,11 @@
2678	2907	* Otherwise, it'll get a reference on the block group pointed to by the
2679	2908	* cluster and remove the cluster from it.
2680	2909	*/
2681		-int btrfs_return_cluster_to_free_space(
2682		- struct btrfs_block_group_cache *block_group,
	2910	+void btrfs_return_cluster_to_free_space(
	2911	+ struct btrfs_block_group *block_group,
2683	2912	struct btrfs_free_cluster *cluster)
2684	2913	{
2685	2914	struct btrfs_free_space_ctl *ctl;
2686		- int ret;
2687	2915
2688	2916	/* first, get a safe pointer to the block group */
2689	2917	spin_lock(&cluster->lock);
..	..	@@ -2691,29 +2919,30 @@
2691	2919	block_group = cluster->block_group;
2692	2920	if (!block_group) {
2693	2921	spin_unlock(&cluster->lock);
2694		- return 0;
	2922	+ return;
2695	2923	}
2696	2924	} else if (cluster->block_group != block_group) {
2697	2925	/* someone else has already freed it don't redo their work */
2698	2926	spin_unlock(&cluster->lock);
2699		- return 0;
	2927	+ return;
2700	2928	}
2701		- atomic_inc(&block_group->count);
	2929	+ btrfs_get_block_group(block_group);
2702	2930	spin_unlock(&cluster->lock);
2703	2931
2704	2932	ctl = block_group->free_space_ctl;
2705	2933
2706	2934	/* now return any extents the cluster had on it */
2707	2935	spin_lock(&ctl->tree_lock);
2708		- ret = __btrfs_return_cluster_to_free_space(block_group, cluster);
	2936	+ __btrfs_return_cluster_to_free_space(block_group, cluster);
2709	2937	spin_unlock(&ctl->tree_lock);
	2938	+
	2939	+ btrfs_discard_queue_work(&block_group->fs_info->discard_ctl, block_group);
2710	2940
2711	2941	/* finally drop our ref */
2712	2942	btrfs_put_block_group(block_group);
2713		- return ret;
2714	2943	}
2715	2944
2716		-static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group_cache *block_group,
	2945	+static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group *block_group,
2717	2946	struct btrfs_free_cluster *cluster,
2718	2947	struct btrfs_free_space *entry,
2719	2948	u64 bytes, u64 min_start,
..	..	@@ -2746,11 +2975,13 @@
2746	2975	* if it couldn't find anything suitably large, or a logical disk offset
2747	2976	* if things worked out
2748	2977	*/
2749		-u64 btrfs_alloc_from_cluster(struct btrfs_block_group_cache *block_group,
	2978	+u64 btrfs_alloc_from_cluster(struct btrfs_block_group *block_group,
2750	2979	struct btrfs_free_cluster *cluster, u64 bytes,
2751	2980	u64 min_start, u64 *max_extent_size)
2752	2981	{
2753	2982	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
	2983	+ struct btrfs_discard_ctl *discard_ctl =
	2984	+ &block_group->fs_info->discard_ctl;
2754	2985	struct btrfs_free_space *entry = NULL;
2755	2986	struct rb_node *node;
2756	2987	u64 ret = 0;
..	..	@@ -2803,8 +3034,6 @@
2803	3034	entry->bytes -= bytes;
2804	3035	}
2805	3036
2806		- if (entry->bytes == 0)
2807		- rb_erase(&entry->offset_index, &cluster->root);
2808	3037	break;
2809	3038	}
2810	3039	out:
..	..	@@ -2815,24 +3044,35 @@
2815	3044
2816	3045	spin_lock(&ctl->tree_lock);
2817	3046
	3047	+ if (!btrfs_free_space_trimmed(entry))
	3048	+ atomic64_add(bytes, &discard_ctl->discard_bytes_saved);
	3049	+
2818	3050	ctl->free_space -= bytes;
	3051	+ if (!entry->bitmap && !btrfs_free_space_trimmed(entry))
	3052	+ ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes;
	3053	+
	3054	+ spin_lock(&cluster->lock);
2819	3055	if (entry->bytes == 0) {
	3056	+ rb_erase(&entry->offset_index, &cluster->root);
2820	3057	ctl->free_extents--;
2821	3058	if (entry->bitmap) {
2822	3059	kmem_cache_free(btrfs_free_space_bitmap_cachep,
2823	3060	entry->bitmap);
2824	3061	ctl->total_bitmaps--;
2825	3062	ctl->op->recalc_thresholds(ctl);
	3063	+ } else if (!btrfs_free_space_trimmed(entry)) {
	3064	+ ctl->discardable_extents[BTRFS_STAT_CURR]--;
2826	3065	}
2827	3066	kmem_cache_free(btrfs_free_space_cachep, entry);
2828	3067	}
2829	3068
	3069	+ spin_unlock(&cluster->lock);
2830	3070	spin_unlock(&ctl->tree_lock);
2831	3071
2832	3072	return ret;
2833	3073	}
2834	3074
2835		-static int btrfs_bitmap_cluster(struct btrfs_block_group_cache *block_group,
	3075	+static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group,
2836	3076	struct btrfs_free_space *entry,
2837	3077	struct btrfs_free_cluster *cluster,
2838	3078	u64 offset, u64 bytes,
..	..	@@ -2914,7 +3154,7 @@
2914	3154	* extent of cont1_bytes, and other clusters of at least min_bytes.
2915	3155	*/
2916	3156	static noinline int
2917		-setup_cluster_no_bitmap(struct btrfs_block_group_cache *block_group,
	3157	+setup_cluster_no_bitmap(struct btrfs_block_group *block_group,
2918	3158	struct btrfs_free_cluster *cluster,
2919	3159	struct list_head *bitmaps, u64 offset, u64 bytes,
2920	3160	u64 cont1_bytes, u64 min_bytes)
..	..	@@ -3005,7 +3245,7 @@
3005	3245	* that we have already failed to find extents that will work.
3006	3246	*/
3007	3247	static noinline int
3008		-setup_cluster_bitmap(struct btrfs_block_group_cache *block_group,
	3248	+setup_cluster_bitmap(struct btrfs_block_group *block_group,
3009	3249	struct btrfs_free_cluster *cluster,
3010	3250	struct list_head *bitmaps, u64 offset, u64 bytes,
3011	3251	u64 cont1_bytes, u64 min_bytes)
..	..	@@ -3055,11 +3295,11 @@
3055	3295	* returns zero and sets up cluster if things worked out, otherwise
3056	3296	* it returns -enospc
3057	3297	*/
3058		-int btrfs_find_space_cluster(struct btrfs_fs_info *fs_info,
3059		- struct btrfs_block_group_cache *block_group,
	3298	+int btrfs_find_space_cluster(struct btrfs_block_group *block_group,
3060	3299	struct btrfs_free_cluster *cluster,
3061	3300	u64 offset, u64 bytes, u64 empty_size)
3062	3301	{
	3302	+ struct btrfs_fs_info *fs_info = block_group->fs_info;
3063	3303	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
3064	3304	struct btrfs_free_space entry, tmp;
3065	3305	LIST_HEAD(bitmaps);
..	..	@@ -3118,7 +3358,7 @@
3118	3358	list_del_init(&entry->list);
3119	3359
3120	3360	if (!ret) {
3121		- atomic_inc(&block_group->count);
	3361	+ btrfs_get_block_group(block_group);
3122	3362	list_add_tail(&cluster->block_group_list,
3123	3363	&block_group->cluster_list);
3124	3364	cluster->block_group = block_group;
..	..	@@ -3146,9 +3386,10 @@
3146	3386	cluster->block_group = NULL;
3147	3387	}
3148	3388
3149		-static int do_trimming(struct btrfs_block_group_cache *block_group,
	3389	+static int do_trimming(struct btrfs_block_group *block_group,
3150	3390	u64 *total_trimmed, u64 start, u64 bytes,
3151	3391	u64 reserved_start, u64 reserved_bytes,
	3392	+ enum btrfs_trim_state reserved_trim_state,
3152	3393	struct btrfs_trim_range *trim_entry)
3153	3394	{
3154	3395	struct btrfs_space_info *space_info = block_group->space_info;
..	..	@@ -3156,6 +3397,9 @@
3156	3397	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
3157	3398	int ret;
3158	3399	int update = 0;
	3400	+ const u64 end = start + bytes;
	3401	+ const u64 reserved_end = reserved_start + reserved_bytes;
	3402	+ enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
3159	3403	u64 trimmed = 0;
3160	3404
3161	3405	spin_lock(&space_info->lock);
..	..	@@ -3169,11 +3413,20 @@
3169	3413	spin_unlock(&space_info->lock);
3170	3414
3171	3415	ret = btrfs_discard_extent(fs_info, start, bytes, &trimmed);
3172		- if (!ret)
	3416	+ if (!ret) {
3173	3417	*total_trimmed += trimmed;
	3418	+ trim_state = BTRFS_TRIM_STATE_TRIMMED;
	3419	+ }
3174	3420
3175	3421	mutex_lock(&ctl->cache_writeout_mutex);
3176		- btrfs_add_free_space(block_group, reserved_start, reserved_bytes);
	3422	+ if (reserved_start < start)
	3423	+ __btrfs_add_free_space(fs_info, ctl, reserved_start,
	3424	+ start - reserved_start,
	3425	+ reserved_trim_state);
	3426	+ if (start + bytes < reserved_start + reserved_bytes)
	3427	+ __btrfs_add_free_space(fs_info, ctl, end, reserved_end - end,
	3428	+ reserved_trim_state);
	3429	+ __btrfs_add_free_space(fs_info, ctl, start, bytes, trim_state);
3177	3430	list_del(&trim_entry->list);
3178	3431	mutex_unlock(&ctl->cache_writeout_mutex);
3179	3432
..	..	@@ -3184,23 +3437,31 @@
3184	3437	space_info->bytes_readonly += reserved_bytes;
3185	3438	block_group->reserved -= reserved_bytes;
3186	3439	space_info->bytes_reserved -= reserved_bytes;
3187		- spin_unlock(&space_info->lock);
3188	3440	spin_unlock(&block_group->lock);
	3441	+ spin_unlock(&space_info->lock);
3189	3442	}
3190	3443
3191	3444	return ret;
3192	3445	}
3193	3446
3194		-static int trim_no_bitmap(struct btrfs_block_group_cache *block_group,
3195		- u64 *total_trimmed, u64 start, u64 end, u64 minlen)
	3447	+/*
	3448	+ * If @async is set, then we will trim 1 region and return.
	3449	+ */
	3450	+static int trim_no_bitmap(struct btrfs_block_group *block_group,
	3451	+ u64 *total_trimmed, u64 start, u64 end, u64 minlen,
	3452	+ bool async)
3196	3453	{
	3454	+ struct btrfs_discard_ctl *discard_ctl =
	3455	+ &block_group->fs_info->discard_ctl;
3197	3456	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
3198	3457	struct btrfs_free_space *entry;
3199	3458	struct rb_node *node;
3200	3459	int ret = 0;
3201	3460	u64 extent_start;
3202	3461	u64 extent_bytes;
	3462	+ enum btrfs_trim_state extent_trim_state;
3203	3463	u64 bytes;
	3464	+ const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size);
3204	3465
3205	3466	while (start < end) {
3206	3467	struct btrfs_trim_range trim_entry;
..	..	@@ -3208,49 +3469,66 @@
3208	3469	mutex_lock(&ctl->cache_writeout_mutex);
3209	3470	spin_lock(&ctl->tree_lock);
3210	3471
3211		- if (ctl->free_space < minlen) {
3212		- spin_unlock(&ctl->tree_lock);
3213		- mutex_unlock(&ctl->cache_writeout_mutex);
3214		- break;
3215		- }
	3472	+ if (ctl->free_space < minlen)
	3473	+ goto out_unlock;
3216	3474
3217	3475	entry = tree_search_offset(ctl, start, 0, 1);
3218		- if (!entry) {
3219		- spin_unlock(&ctl->tree_lock);
3220		- mutex_unlock(&ctl->cache_writeout_mutex);
3221		- break;
3222		- }
	3476	+ if (!entry)
	3477	+ goto out_unlock;
3223	3478
3224		- /* skip bitmaps */
3225		- while (entry->bitmap) {
	3479	+ /* Skip bitmaps and if async, already trimmed entries */
	3480	+ while (entry->bitmap \|\|
	3481	+ (async && btrfs_free_space_trimmed(entry))) {
3226	3482	node = rb_next(&entry->offset_index);
3227		- if (!node) {
3228		- spin_unlock(&ctl->tree_lock);
3229		- mutex_unlock(&ctl->cache_writeout_mutex);
3230		- goto out;
3231		- }
	3483	+ if (!node)
	3484	+ goto out_unlock;
3232	3485	entry = rb_entry(node, struct btrfs_free_space,
3233	3486	offset_index);
3234	3487	}
3235	3488
3236		- if (entry->offset >= end) {
3237		- spin_unlock(&ctl->tree_lock);
3238		- mutex_unlock(&ctl->cache_writeout_mutex);
3239		- break;
3240		- }
	3489	+ if (entry->offset >= end)
	3490	+ goto out_unlock;
3241	3491
3242	3492	extent_start = entry->offset;
3243	3493	extent_bytes = entry->bytes;
3244		- start = max(start, extent_start);
3245		- bytes = min(extent_start + extent_bytes, end) - start;
3246		- if (bytes < minlen) {
3247		- spin_unlock(&ctl->tree_lock);
3248		- mutex_unlock(&ctl->cache_writeout_mutex);
3249		- goto next;
3250		- }
	3494	+ extent_trim_state = entry->trim_state;
	3495	+ if (async) {
	3496	+ start = entry->offset;
	3497	+ bytes = entry->bytes;
	3498	+ if (bytes < minlen) {
	3499	+ spin_unlock(&ctl->tree_lock);
	3500	+ mutex_unlock(&ctl->cache_writeout_mutex);
	3501	+ goto next;
	3502	+ }
	3503	+ unlink_free_space(ctl, entry);
	3504	+ /*
	3505	+ * Let bytes = BTRFS_MAX_DISCARD_SIZE + X.
	3506	+ * If X < BTRFS_ASYNC_DISCARD_MIN_FILTER, we won't trim
	3507	+ * X when we come back around. So trim it now.
	3508	+ */
	3509	+ if (max_discard_size &&
	3510	+ bytes >= (max_discard_size +
	3511	+ BTRFS_ASYNC_DISCARD_MIN_FILTER)) {
	3512	+ bytes = max_discard_size;
	3513	+ extent_bytes = max_discard_size;
	3514	+ entry->offset += max_discard_size;
	3515	+ entry->bytes -= max_discard_size;
	3516	+ link_free_space(ctl, entry);
	3517	+ } else {
	3518	+ kmem_cache_free(btrfs_free_space_cachep, entry);
	3519	+ }
	3520	+ } else {
	3521	+ start = max(start, extent_start);
	3522	+ bytes = min(extent_start + extent_bytes, end) - start;
	3523	+ if (bytes < minlen) {
	3524	+ spin_unlock(&ctl->tree_lock);
	3525	+ mutex_unlock(&ctl->cache_writeout_mutex);
	3526	+ goto next;
	3527	+ }
3251	3528
3252		- unlink_free_space(ctl, entry);
3253		- kmem_cache_free(btrfs_free_space_cachep, entry);
	3529	+ unlink_free_space(ctl, entry);
	3530	+ kmem_cache_free(btrfs_free_space_cachep, entry);
	3531	+ }
3254	3532
3255	3533	spin_unlock(&ctl->tree_lock);
3256	3534	trim_entry.start = extent_start;
..	..	@@ -3259,11 +3537,17 @@
3259	3537	mutex_unlock(&ctl->cache_writeout_mutex);
3260	3538
3261	3539	ret = do_trimming(block_group, total_trimmed, start, bytes,
3262		- extent_start, extent_bytes, &trim_entry);
3263		- if (ret)
	3540	+ extent_start, extent_bytes, extent_trim_state,
	3541	+ &trim_entry);
	3542	+ if (ret) {
	3543	+ block_group->discard_cursor = start + bytes;
3264	3544	break;
	3545	+ }
3265	3546	next:
3266	3547	start += bytes;
	3548	+ block_group->discard_cursor = start;
	3549	+ if (async && *total_trimmed)
	3550	+ break;
3267	3551
3268	3552	if (fatal_signal_pending(current)) {
3269	3553	ret = -ERESTARTSYS;
..	..	@@ -3272,19 +3556,76 @@
3272	3556
3273	3557	cond_resched();
3274	3558	}
3275		-out:
	3559	+
	3560	+ return ret;
	3561	+
	3562	+out_unlock:
	3563	+ block_group->discard_cursor = btrfs_block_group_end(block_group);
	3564	+ spin_unlock(&ctl->tree_lock);
	3565	+ mutex_unlock(&ctl->cache_writeout_mutex);
	3566	+
3276	3567	return ret;
3277	3568	}
3278	3569
3279		-static int trim_bitmaps(struct btrfs_block_group_cache *block_group,
3280		- u64 *total_trimmed, u64 start, u64 end, u64 minlen)
	3570	+/*
	3571	+ * If we break out of trimming a bitmap prematurely, we should reset the
	3572	+ * trimming bit. In a rather contrieved case, it's possible to race here so
	3573	+ * reset the state to BTRFS_TRIM_STATE_UNTRIMMED.
	3574	+ *
	3575	+ * start = start of bitmap
	3576	+ * end = near end of bitmap
	3577	+ *
	3578	+ * Thread 1: Thread 2:
	3579	+ * trim_bitmaps(start)
	3580	+ * trim_bitmaps(end)
	3581	+ * end_trimming_bitmap()
	3582	+ * reset_trimming_bitmap()
	3583	+ */
	3584	+static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset)
3281	3585	{
	3586	+ struct btrfs_free_space *entry;
	3587	+
	3588	+ spin_lock(&ctl->tree_lock);
	3589	+ entry = tree_search_offset(ctl, offset, 1, 0);
	3590	+ if (entry) {
	3591	+ if (btrfs_free_space_trimmed(entry)) {
	3592	+ ctl->discardable_extents[BTRFS_STAT_CURR] +=
	3593	+ entry->bitmap_extents;
	3594	+ ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes;
	3595	+ }
	3596	+ entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
	3597	+ }
	3598	+
	3599	+ spin_unlock(&ctl->tree_lock);
	3600	+}
	3601	+
	3602	+static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl,
	3603	+ struct btrfs_free_space *entry)
	3604	+{
	3605	+ if (btrfs_free_space_trimming_bitmap(entry)) {
	3606	+ entry->trim_state = BTRFS_TRIM_STATE_TRIMMED;
	3607	+ ctl->discardable_extents[BTRFS_STAT_CURR] -=
	3608	+ entry->bitmap_extents;
	3609	+ ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes;
	3610	+ }
	3611	+}
	3612	+
	3613	+/*
	3614	+ * If @async is set, then we will trim 1 region and return.
	3615	+ */
	3616	+static int trim_bitmaps(struct btrfs_block_group *block_group,
	3617	+ u64 *total_trimmed, u64 start, u64 end, u64 minlen,
	3618	+ u64 maxlen, bool async)
	3619	+{
	3620	+ struct btrfs_discard_ctl *discard_ctl =
	3621	+ &block_group->fs_info->discard_ctl;
3282	3622	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
3283	3623	struct btrfs_free_space *entry;
3284	3624	int ret = 0;
3285	3625	int ret2;
3286	3626	u64 bytes;
3287	3627	u64 offset = offset_to_bitmap(ctl, start);
	3628	+ const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size);
3288	3629
3289	3630	while (offset < end) {
3290	3631	bool next_bitmap = false;
..	..	@@ -3294,34 +3635,83 @@
3294	3635	spin_lock(&ctl->tree_lock);
3295	3636
3296	3637	if (ctl->free_space < minlen) {
	3638	+ block_group->discard_cursor =
	3639	+ btrfs_block_group_end(block_group);
3297	3640	spin_unlock(&ctl->tree_lock);
3298	3641	mutex_unlock(&ctl->cache_writeout_mutex);
3299	3642	break;
3300	3643	}
3301	3644
3302	3645	entry = tree_search_offset(ctl, offset, 1, 0);
3303		- if (!entry) {
	3646	+ /*
	3647	+ * Bitmaps are marked trimmed lossily now to prevent constant
	3648	+ * discarding of the same bitmap (the reason why we are bound
	3649	+ * by the filters). So, retrim the block group bitmaps when we
	3650	+ * are preparing to punt to the unused_bgs list. This uses
	3651	+ * @minlen to determine if we are in BTRFS_DISCARD_INDEX_UNUSED
	3652	+ * which is the only discard index which sets minlen to 0.
	3653	+ */
	3654	+ if (!entry \|\| (async && minlen && start == offset &&
	3655	+ btrfs_free_space_trimmed(entry))) {
3304	3656	spin_unlock(&ctl->tree_lock);
3305	3657	mutex_unlock(&ctl->cache_writeout_mutex);
3306	3658	next_bitmap = true;
3307	3659	goto next;
3308	3660	}
	3661	+
	3662	+ /*
	3663	+ * Async discard bitmap trimming begins at by setting the start
	3664	+ * to be key.objectid and the offset_to_bitmap() aligns to the
	3665	+ * start of the bitmap. This lets us know we are fully
	3666	+ * scanning the bitmap rather than only some portion of it.
	3667	+ */
	3668	+ if (start == offset)
	3669	+ entry->trim_state = BTRFS_TRIM_STATE_TRIMMING;
3309	3670
3310	3671	bytes = minlen;
3311	3672	ret2 = search_bitmap(ctl, entry, &start, &bytes, false);
3312	3673	if (ret2 \|\| start >= end) {
	3674	+ /*
	3675	+ * We lossily consider a bitmap trimmed if we only skip
	3676	+ * over regions <= BTRFS_ASYNC_DISCARD_MIN_FILTER.
	3677	+ */
	3678	+ if (ret2 && minlen <= BTRFS_ASYNC_DISCARD_MIN_FILTER)
	3679	+ end_trimming_bitmap(ctl, entry);
	3680	+ else
	3681	+ entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
3313	3682	spin_unlock(&ctl->tree_lock);
3314	3683	mutex_unlock(&ctl->cache_writeout_mutex);
3315	3684	next_bitmap = true;
3316	3685	goto next;
3317	3686	}
3318	3687
	3688	+ /*
	3689	+ * We already trimmed a region, but are using the locking above
	3690	+ * to reset the trim_state.
	3691	+ */
	3692	+ if (async && *total_trimmed) {
	3693	+ spin_unlock(&ctl->tree_lock);
	3694	+ mutex_unlock(&ctl->cache_writeout_mutex);
	3695	+ goto out;
	3696	+ }
	3697	+
3319	3698	bytes = min(bytes, end - start);
3320		- if (bytes < minlen) {
	3699	+ if (bytes < minlen \|\| (async && maxlen && bytes > maxlen)) {
3321	3700	spin_unlock(&ctl->tree_lock);
3322	3701	mutex_unlock(&ctl->cache_writeout_mutex);
3323	3702	goto next;
3324	3703	}
	3704	+
	3705	+ /*
	3706	+ * Let bytes = BTRFS_MAX_DISCARD_SIZE + X.
	3707	+ * If X < @minlen, we won't trim X when we come back around.
	3708	+ * So trim it now. We differ here from trimming extents as we
	3709	+ * don't keep individual state per bit.
	3710	+ */
	3711	+ if (async &&
	3712	+ max_discard_size &&
	3713	+ bytes > (max_discard_size + minlen))
	3714	+ bytes = max_discard_size;
3325	3715
3326	3716	bitmap_clear_bits(ctl, entry, start, bytes);
3327	3717	if (entry->bytes == 0)
..	..	@@ -3334,19 +3724,25 @@
3334	3724	mutex_unlock(&ctl->cache_writeout_mutex);
3335	3725
3336	3726	ret = do_trimming(block_group, total_trimmed, start, bytes,
3337		- start, bytes, &trim_entry);
3338		- if (ret)
	3727	+ start, bytes, 0, &trim_entry);
	3728	+ if (ret) {
	3729	+ reset_trimming_bitmap(ctl, offset);
	3730	+ block_group->discard_cursor =
	3731	+ btrfs_block_group_end(block_group);
3339	3732	break;
	3733	+ }
3340	3734	next:
3341	3735	if (next_bitmap) {
3342	3736	offset += BITS_PER_BITMAP * ctl->unit;
	3737	+ start = offset;
3343	3738	} else {
3344	3739	start += bytes;
3345		- if (start >= offset + BITS_PER_BITMAP * ctl->unit)
3346		- offset += BITS_PER_BITMAP * ctl->unit;
3347	3740	}
	3741	+ block_group->discard_cursor = start;
3348	3742
3349	3743	if (fatal_signal_pending(current)) {
	3744	+ if (start != offset)
	3745	+ reset_trimming_bitmap(ctl, offset);
3350	3746	ret = -ERESTARTSYS;
3351	3747	break;
3352	3748	}
..	..	@@ -3354,55 +3750,47 @@
3354	3750	cond_resched();
3355	3751	}
3356	3752
	3753	+ if (offset >= end)
	3754	+ block_group->discard_cursor = end;
	3755	+
	3756	+out:
3357	3757	return ret;
3358	3758	}
3359	3759
3360		-void btrfs_get_block_group_trimming(struct btrfs_block_group_cache *cache)
	3760	+int btrfs_trim_block_group(struct btrfs_block_group *block_group,
	3761	+ u64 *trimmed, u64 start, u64 end, u64 minlen)
3361	3762	{
3362		- atomic_inc(&cache->trimming);
3363		-}
	3763	+ struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
	3764	+ int ret;
	3765	+ u64 rem = 0;
3364	3766
3365		-void btrfs_put_block_group_trimming(struct btrfs_block_group_cache *block_group)
3366		-{
3367		- struct btrfs_fs_info *fs_info = block_group->fs_info;
3368		- struct extent_map_tree *em_tree;
3369		- struct extent_map *em;
3370		- bool cleanup;
	3767	+ *trimmed = 0;
3371	3768
3372	3769	spin_lock(&block_group->lock);
3373		- cleanup = (atomic_dec_and_test(&block_group->trimming) &&
3374		- block_group->removed);
	3770	+ if (block_group->removed) {
	3771	+ spin_unlock(&block_group->lock);
	3772	+ return 0;
	3773	+ }
	3774	+ btrfs_freeze_block_group(block_group);
3375	3775	spin_unlock(&block_group->lock);
3376	3776
3377		- if (cleanup) {
3378		- mutex_lock(&fs_info->chunk_mutex);
3379		- em_tree = &fs_info->mapping_tree.map_tree;
3380		- write_lock(&em_tree->lock);
3381		- em = lookup_extent_mapping(em_tree, block_group->key.objectid,
3382		- 1);
3383		- BUG_ON(!em); /* logic error, can't happen */
3384		- /*
3385		- * remove_extent_mapping() will delete us from the pinned_chunks
3386		- * list, which is protected by the chunk mutex.
3387		- */
3388		- remove_extent_mapping(em_tree, em);
3389		- write_unlock(&em_tree->lock);
3390		- mutex_unlock(&fs_info->chunk_mutex);
	3777	+ ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, false);
	3778	+ if (ret)
	3779	+ goto out;
3391	3780
3392		- /* once for us and once for the tree */
3393		- free_extent_map(em);
3394		- free_extent_map(em);
3395		-
3396		- /*
3397		- * We've left one free space entry and other tasks trimming
3398		- * this block group have left 1 entry each one. Free them.
3399		- */
3400		- __btrfs_remove_free_space_cache(block_group->free_space_ctl);
3401		- }
	3781	+ ret = trim_bitmaps(block_group, trimmed, start, end, minlen, 0, false);
	3782	+ div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem);
	3783	+ /* If we ended in the middle of a bitmap, reset the trimming flag */
	3784	+ if (rem)
	3785	+ reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end));
	3786	+out:
	3787	+ btrfs_unfreeze_block_group(block_group);
	3788	+ return ret;
3402	3789	}
3403	3790
3404		-int btrfs_trim_block_group(struct btrfs_block_group_cache *block_group,
3405		- u64 *trimmed, u64 start, u64 end, u64 minlen)
	3791	+int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group,
	3792	+ u64 *trimmed, u64 start, u64 end, u64 minlen,
	3793	+ bool async)
3406	3794	{
3407	3795	int ret;
3408	3796
..	..	@@ -3413,16 +3801,36 @@
3413	3801	spin_unlock(&block_group->lock);
3414	3802	return 0;
3415	3803	}
3416		- btrfs_get_block_group_trimming(block_group);
	3804	+ btrfs_freeze_block_group(block_group);
3417	3805	spin_unlock(&block_group->lock);
3418	3806
3419		- ret = trim_no_bitmap(block_group, trimmed, start, end, minlen);
3420		- if (ret)
3421		- goto out;
	3807	+ ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, async);
	3808	+ btrfs_unfreeze_block_group(block_group);
3422	3809
3423		- ret = trim_bitmaps(block_group, trimmed, start, end, minlen);
3424		-out:
3425		- btrfs_put_block_group_trimming(block_group);
	3810	+ return ret;
	3811	+}
	3812	+
	3813	+int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group,
	3814	+ u64 *trimmed, u64 start, u64 end, u64 minlen,
	3815	+ u64 maxlen, bool async)
	3816	+{
	3817	+ int ret;
	3818	+
	3819	+ *trimmed = 0;
	3820	+
	3821	+ spin_lock(&block_group->lock);
	3822	+ if (block_group->removed) {
	3823	+ spin_unlock(&block_group->lock);
	3824	+ return 0;
	3825	+ }
	3826	+ btrfs_freeze_block_group(block_group);
	3827	+ spin_unlock(&block_group->lock);
	3828	+
	3829	+ ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen,
	3830	+ async);
	3831	+
	3832	+ btrfs_unfreeze_block_group(block_group);
	3833	+
3426	3834	return ret;
3427	3835	}
3428	3836
..	..	@@ -3582,11 +3990,9 @@
3582	3990	if (release_metadata)
3583	3991	btrfs_delalloc_release_metadata(BTRFS_I(inode),
3584	3992	inode->i_size, true);
3585		-#ifdef DEBUG
3586		- btrfs_err(fs_info,
3587		- "failed to write free ino cache for root %llu",
3588		- root->root_key.objectid);
3589		-#endif
	3993	+ btrfs_debug(fs_info,
	3994	+ "failed to write free ino cache for root %llu error %d",
	3995	+ root->root_key.objectid, ret);
3590	3996	}
3591	3997
3592	3998	return ret;
..	..	@@ -3599,12 +4005,13 @@
3599	4005	* how the free space cache loading stuff works, so you can get really weird
3600	4006	* configurations.
3601	4007	*/
3602		-int test_add_free_space_entry(struct btrfs_block_group_cache *cache,
	4008	+int test_add_free_space_entry(struct btrfs_block_group *cache,
3603	4009	u64 offset, u64 bytes, bool bitmap)
3604	4010	{
3605	4011	struct btrfs_free_space_ctl *ctl = cache->free_space_ctl;
3606	4012	struct btrfs_free_space info = NULL, bitmap_info;
3607	4013	void *map = NULL;
	4014	+ enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_TRIMMED;
3608	4015	u64 bytes_added;
3609	4016	int ret;
3610	4017
..	..	@@ -3646,7 +4053,8 @@
3646	4053	info = NULL;
3647	4054	}
3648	4055
3649		- bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes);
	4056	+ bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes,
	4057	+ trim_state);
3650	4058
3651	4059	bytes -= bytes_added;
3652	4060	offset += bytes_added;
..	..	@@ -3667,7 +4075,7 @@
3667	4075	* just used to check the absence of space, so if there is free space in the
3668	4076	* range at all we will return 1.
3669	4077	*/
3670		-int test_check_exists(struct btrfs_block_group_cache *cache,
	4078	+int test_check_exists(struct btrfs_block_group *cache,
3671	4079	u64 offset, u64 bytes)
3672	4080	{
3673	4081	struct btrfs_free_space_ctl *ctl = cache->free_space_ctl;