~hc/RK356X_SDK_RELEASE.git

..	..	@@ -4,10 +4,13 @@
4	4	*/
5	5	#include <linux/crc32.h>
6	6	#include <linux/file.h>
	7	+#include <linux/fsverity.h>
7	8	#include <linux/gfp.h>
	9	+#include <linux/kobject.h>
8	10	#include <linux/ktime.h>
9	11	#include <linux/lz4.h>
10	12	#include <linux/mm.h>
	13	+#include <linux/namei.h>
11	14	#include <linux/pagemap.h>
12	15	#include <linux/slab.h>
13	16	#include <linux/types.h>
..	..	@@ -16,6 +19,10 @@
16	19	#include "data_mgmt.h"
17	20	#include "format.h"
18	21	#include "integrity.h"
	22	+#include "sysfs.h"
	23	+#include "verity.h"
	24	+
	25	+static int incfs_scan_metadata_chain(struct data_file *df);
19	26
20	27	static void log_wake_up_all(struct work_struct *work)
21	28	{
..	..	@@ -24,12 +31,26 @@
24	31	wake_up_all(&rl->ml_notif_wq);
25	32	}
26	33
	34	+static void zstd_free_workspace(struct work_struct *work)
	35	+{
	36	+ struct delayed_work *dw = container_of(work, struct delayed_work, work);
	37	+ struct mount_info *mi =
	38	+ container_of(dw, struct mount_info, mi_zstd_cleanup_work);
	39	+
	40	+ mutex_lock(&mi->mi_zstd_workspace_mutex);
	41	+ kvfree(mi->mi_zstd_workspace);
	42	+ mi->mi_zstd_workspace = NULL;
	43	+ mi->mi_zstd_stream = NULL;
	44	+ mutex_unlock(&mi->mi_zstd_workspace_mutex);
	45	+}
	46	+
27	47	struct mount_info incfs_alloc_mount_info(struct super_block sb,
28	48	struct mount_options *options,
29	49	struct path *backing_dir_path)
30	50	{
31	51	struct mount_info *mi = NULL;
32	52	int error = 0;
	53	+ struct incfs_sysfs_node *node;
33	54
34	55	mi = kzalloc(sizeof(*mi), GFP_NOFS);
35	56	if (!mi)
..	..	@@ -40,12 +61,25 @@
40	61	mi->mi_owner = get_current_cred();
41	62	path_get(&mi->mi_backing_dir_path);
42	63	mutex_init(&mi->mi_dir_struct_mutex);
43		- mutex_init(&mi->mi_pending_reads_mutex);
44	64	init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
45	65	init_waitqueue_head(&mi->mi_log.ml_notif_wq);
	66	+ init_waitqueue_head(&mi->mi_blocks_written_notif_wq);
	67	+ atomic_set(&mi->mi_blocks_written, 0);
46	68	INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);
47	69	spin_lock_init(&mi->mi_log.rl_lock);
	70	+ spin_lock_init(&mi->pending_read_lock);
48	71	INIT_LIST_HEAD(&mi->mi_reads_list_head);
	72	+ spin_lock_init(&mi->mi_per_uid_read_timeouts_lock);
	73	+ mutex_init(&mi->mi_zstd_workspace_mutex);
	74	+ INIT_DELAYED_WORK(&mi->mi_zstd_cleanup_work, zstd_free_workspace);
	75	+ mutex_init(&mi->mi_le_mutex);
	76	+
	77	+ node = incfs_add_sysfs_node(options->sysfs_name, mi);
	78	+ if (IS_ERR(node)) {
	79	+ error = PTR_ERR(node);
	80	+ goto err;
	81	+ }
	82	+ mi->mi_sysfs_node = node;
49	83
50	84	error = incfs_realloc_mount_info(mi, options);
51	85	if (error)
..	..	@@ -95,38 +129,144 @@
95	129	kfree(old_buffer);
96	130	}
97	131
	132	+ if (options->sysfs_name && !mi->mi_sysfs_node)
	133	+ mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
	134	+ mi);
	135	+ else if (!options->sysfs_name && mi->mi_sysfs_node) {
	136	+ incfs_free_sysfs_node(mi->mi_sysfs_node);
	137	+ mi->mi_sysfs_node = NULL;
	138	+ } else if (options->sysfs_name &&
	139	+ strcmp(options->sysfs_name,
	140	+ kobject_name(&mi->mi_sysfs_node->isn_sysfs_node))) {
	141	+ incfs_free_sysfs_node(mi->mi_sysfs_node);
	142	+ mi->mi_sysfs_node = incfs_add_sysfs_node(options->sysfs_name,
	143	+ mi);
	144	+ }
	145	+
	146	+ if (IS_ERR(mi->mi_sysfs_node)) {
	147	+ int err = PTR_ERR(mi->mi_sysfs_node);
	148	+
	149	+ mi->mi_sysfs_node = NULL;
	150	+ return err;
	151	+ }
	152	+
98	153	mi->mi_options = *options;
99	154	return 0;
100	155	}
101	156
102	157	void incfs_free_mount_info(struct mount_info *mi)
103	158	{
	159	+ int i;
104	160	if (!mi)
105	161	return;
106	162
107	163	flush_delayed_work(&mi->mi_log.ml_wakeup_work);
	164	+ flush_delayed_work(&mi->mi_zstd_cleanup_work);
108	165
109	166	dput(mi->mi_index_dir);
	167	+ dput(mi->mi_incomplete_dir);
110	168	path_put(&mi->mi_backing_dir_path);
111	169	mutex_destroy(&mi->mi_dir_struct_mutex);
112		- mutex_destroy(&mi->mi_pending_reads_mutex);
	170	+ mutex_destroy(&mi->mi_zstd_workspace_mutex);
113	171	put_cred(mi->mi_owner);
114	172	kfree(mi->mi_log.rl_ring_buf);
115		- kfree(mi->log_xattr);
116		- kfree(mi->pending_read_xattr);
	173	+ for (i = 0; i < ARRAY_SIZE(mi->pseudo_file_xattr); ++i)
	174	+ kfree(mi->pseudo_file_xattr[i].data);
	175	+ kfree(mi->mi_per_uid_read_timeouts);
	176	+ incfs_free_sysfs_node(mi->mi_sysfs_node);
117	177	kfree(mi);
118	178	}
119	179
120	180	static void data_file_segment_init(struct data_file_segment *segment)
121	181	{
122	182	init_waitqueue_head(&segment->new_data_arrival_wq);
123		- mutex_init(&segment->blockmap_mutex);
	183	+ init_rwsem(&segment->rwsem);
124	184	INIT_LIST_HEAD(&segment->reads_list_head);
125	185	}
126	186
127		-static void data_file_segment_destroy(struct data_file_segment *segment)
	187	+char *file_id_to_str(incfs_uuid_t id)
128	188	{
129		- mutex_destroy(&segment->blockmap_mutex);
	189	+ char result = kmalloc(1 + sizeof(id.bytes) 2, GFP_NOFS);
	190	+ char *end;
	191	+
	192	+ if (!result)
	193	+ return NULL;
	194	+
	195	+ end = bin2hex(result, id.bytes, sizeof(id.bytes));
	196	+ *end = 0;
	197	+ return result;
	198	+}
	199	+
	200	+struct dentry incfs_lookup_dentry(struct dentry parent, const char *name)
	201	+{
	202	+ struct inode *inode;
	203	+ struct dentry *result = NULL;
	204	+
	205	+ if (!parent)
	206	+ return ERR_PTR(-EFAULT);
	207	+
	208	+ inode = d_inode(parent);
	209	+ inode_lock_nested(inode, I_MUTEX_PARENT);
	210	+ result = lookup_one_len(name, parent, strlen(name));
	211	+ inode_unlock(inode);
	212	+
	213	+ if (IS_ERR(result))
	214	+ pr_warn("%s err:%ld\n", __func__, PTR_ERR(result));
	215	+
	216	+ return result;
	217	+}
	218	+
	219	+static struct data_file handle_mapped_file(struct mount_info mi,
	220	+ struct data_file *df)
	221	+{
	222	+ char *file_id_str;
	223	+ struct dentry *index_file_dentry;
	224	+ struct path path;
	225	+ struct file *bf;
	226	+ struct data_file *result = NULL;
	227	+ const struct cred *old_cred;
	228	+
	229	+ file_id_str = file_id_to_str(df->df_id);
	230	+ if (!file_id_str)
	231	+ return ERR_PTR(-ENOENT);
	232	+
	233	+ index_file_dentry = incfs_lookup_dentry(mi->mi_index_dir,
	234	+ file_id_str);
	235	+ kfree(file_id_str);
	236	+ if (!index_file_dentry)
	237	+ return ERR_PTR(-ENOENT);
	238	+ if (IS_ERR(index_file_dentry))
	239	+ return (struct data_file *)index_file_dentry;
	240	+ if (!d_really_is_positive(index_file_dentry)) {
	241	+ result = ERR_PTR(-ENOENT);
	242	+ goto out;
	243	+ }
	244	+
	245	+ path = (struct path) {
	246	+ .mnt = mi->mi_backing_dir_path.mnt,
	247	+ .dentry = index_file_dentry
	248	+ };
	249	+
	250	+ old_cred = override_creds(mi->mi_owner);
	251	+ bf = dentry_open(&path, O_RDWR \| O_NOATIME \| O_LARGEFILE,
	252	+ current_cred());
	253	+ revert_creds(old_cred);
	254	+
	255	+ if (IS_ERR(bf)) {
	256	+ result = (struct data_file *)bf;
	257	+ goto out;
	258	+ }
	259	+
	260	+ result = incfs_open_data_file(mi, bf);
	261	+ fput(bf);
	262	+ if (IS_ERR(result))
	263	+ goto out;
	264	+
	265	+ result->df_mapped_offset = df->df_metadata_off;
	266	+
	267	+out:
	268	+ dput(index_file_dentry);
	269	+ return result;
130	270	}
131	271
132	272	struct data_file incfs_open_data_file(struct mount_info mi, struct file *bf)
..	..	@@ -154,17 +294,15 @@
154	294	goto out;
155	295	}
156	296
	297	+ mutex_init(&df->df_enable_verity);
	298	+
157	299	df->df_backing_file_context = bfc;
158	300	df->df_mount_info = mi;
159	301	for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
160	302	data_file_segment_init(&df->df_segments[i]);
161	303
162		- error = mutex_lock_interruptible(&bfc->bc_mutex);
163		- if (error)
164		- goto out;
165	304	error = incfs_read_file_header(bfc, &df->df_metadata_off, &df->df_id,
166	305	&size, &df->df_header_flags);
167		- mutex_unlock(&bfc->bc_mutex);
168	306
169	307	if (error)
170	308	goto out;
..	..	@@ -172,6 +310,13 @@
172	310	df->df_size = size;
173	311	if (size > 0)
174	312	df->df_data_block_count = get_blocks_count_for_size(size);
	313	+
	314	+ if (df->df_header_flags & INCFS_FILE_MAPPED) {
	315	+ struct data_file *mapped_df = handle_mapped_file(mi, df);
	316	+
	317	+ incfs_free_data_file(df);
	318	+ return mapped_df;
	319	+ }
175	320
176	321	md_records = incfs_scan_metadata_chain(df);
177	322	if (md_records < 0)
..	..	@@ -190,16 +335,39 @@
190	335
191	336	void incfs_free_data_file(struct data_file *df)
192	337	{
193		- int i;
	338	+ u32 data_blocks_written, hash_blocks_written;
194	339
195	340	if (!df)
196	341	return;
197	342
	343	+ data_blocks_written = atomic_read(&df->df_data_blocks_written);
	344	+ hash_blocks_written = atomic_read(&df->df_hash_blocks_written);
	345	+
	346	+ if (data_blocks_written != df->df_initial_data_blocks_written \|\|
	347	+ hash_blocks_written != df->df_initial_hash_blocks_written) {
	348	+ struct backing_file_context *bfc = df->df_backing_file_context;
	349	+ int error = -1;
	350	+
	351	+ if (bfc && !mutex_lock_interruptible(&bfc->bc_mutex)) {
	352	+ error = incfs_write_status_to_backing_file(
	353	+ df->df_backing_file_context,
	354	+ df->df_status_offset,
	355	+ data_blocks_written,
	356	+ hash_blocks_written);
	357	+ mutex_unlock(&bfc->bc_mutex);
	358	+ }
	359	+
	360	+ if (error)
	361	+ /* Nothing can be done, just warn */
	362	+ pr_warn("incfs: failed to write status to backing file\n");
	363	+ }
	364	+
198	365	incfs_free_mtree(df->df_hash_tree);
199		- for (i = 0; i < ARRAY_SIZE(df->df_segments); i++)
200		- data_file_segment_destroy(&df->df_segments[i]);
201	366	incfs_free_bfc(df->df_backing_file_context);
202	367	kfree(df->df_signature);
	368	+ kfree(df->df_verity_file_digest.data);
	369	+ kfree(df->df_verity_signature);
	370	+ mutex_destroy(&df->df_enable_verity);
203	371	kfree(df);
204	372	}
205	373
..	..	@@ -252,14 +420,71 @@
252	420	kfree(dir);
253	421	}
254	422
255		-static ssize_t decompress(struct mem_range src, struct mem_range dst)
	423	+static ssize_t zstd_decompress_safe(struct mount_info *mi,
	424	+ struct mem_range src, struct mem_range dst)
256	425	{
257		- int result = LZ4_decompress_safe(src.data, dst.data, src.len, dst.len);
	426	+ ssize_t result;
	427	+ ZSTD_inBuffer inbuf = {.src = src.data, .size = src.len};
	428	+ ZSTD_outBuffer outbuf = {.dst = dst.data, .size = dst.len};
258	429
259		- if (result < 0)
260		- return -EBADMSG;
	430	+ result = mutex_lock_interruptible(&mi->mi_zstd_workspace_mutex);
	431	+ if (result)
	432	+ return result;
261	433
	434	+ if (!mi->mi_zstd_stream) {
	435	+ unsigned int workspace_size = ZSTD_DStreamWorkspaceBound(
	436	+ INCFS_DATA_FILE_BLOCK_SIZE);
	437	+ void *workspace = kvmalloc(workspace_size, GFP_NOFS);
	438	+ ZSTD_DStream *stream;
	439	+
	440	+ if (!workspace) {
	441	+ result = -ENOMEM;
	442	+ goto out;
	443	+ }
	444	+
	445	+ stream = ZSTD_initDStream(INCFS_DATA_FILE_BLOCK_SIZE, workspace,
	446	+ workspace_size);
	447	+ if (!stream) {
	448	+ kvfree(workspace);
	449	+ result = -EIO;
	450	+ goto out;
	451	+ }
	452	+
	453	+ mi->mi_zstd_workspace = workspace;
	454	+ mi->mi_zstd_stream = stream;
	455	+ }
	456	+
	457	+ result = ZSTD_decompressStream(mi->mi_zstd_stream, &outbuf, &inbuf) ?
	458	+ -EBADMSG : outbuf.pos;
	459	+
	460	+ mod_delayed_work(system_wq, &mi->mi_zstd_cleanup_work,
	461	+ msecs_to_jiffies(5000));
	462	+
	463	+out:
	464	+ mutex_unlock(&mi->mi_zstd_workspace_mutex);
262	465	return result;
	466	+}
	467	+
	468	+static ssize_t decompress(struct mount_info *mi,
	469	+ struct mem_range src, struct mem_range dst, int alg)
	470	+{
	471	+ int result;
	472	+
	473	+ switch (alg) {
	474	+ case INCFS_BLOCK_COMPRESSED_LZ4:
	475	+ result = LZ4_decompress_safe(src.data, dst.data, src.len,
	476	+ dst.len);
	477	+ if (result < 0)
	478	+ return -EBADMSG;
	479	+ return result;
	480	+
	481	+ case INCFS_BLOCK_COMPRESSED_ZSTD:
	482	+ return zstd_decompress_safe(mi, src, dst);
	483	+
	484	+ default:
	485	+ WARN_ON(true);
	486	+ return -EOPNOTSUPP;
	487	+ }
263	488	}
264	489
265	490	static void log_read_one_record(struct read_log rl, struct read_log_state rs)
..	..	@@ -276,10 +501,27 @@
276	501
277	502	case SAME_FILE:
278	503	rs->base_record.block_index =
279		- record->same_file_record.block_index;
	504	+ record->same_file.block_index;
280	505	rs->base_record.absolute_ts_us +=
281		- record->same_file_record.relative_ts_us;
282		- record_size = sizeof(record->same_file_record);
	506	+ record->same_file.relative_ts_us;
	507	+ rs->base_record.uid = record->same_file.uid;
	508	+ record_size = sizeof(record->same_file);
	509	+ break;
	510	+
	511	+ case SAME_FILE_CLOSE_BLOCK:
	512	+ rs->base_record.block_index +=
	513	+ record->same_file_close_block.block_index_delta;
	514	+ rs->base_record.absolute_ts_us +=
	515	+ record->same_file_close_block.relative_ts_us;
	516	+ record_size = sizeof(record->same_file_close_block);
	517	+ break;
	518	+
	519	+ case SAME_FILE_CLOSE_BLOCK_SHORT:
	520	+ rs->base_record.block_index +=
	521	+ record->same_file_close_block_short.block_index_delta;
	522	+ rs->base_record.absolute_ts_us +=
	523	+ record->same_file_close_block_short.relative_ts_tens_us * 10;
	524	+ record_size = sizeof(record->same_file_close_block_short);
283	525	break;
284	526
285	527	case SAME_FILE_NEXT_BLOCK:
..	..	@@ -292,7 +534,7 @@
292	534	case SAME_FILE_NEXT_BLOCK_SHORT:
293	535	++rs->base_record.block_index;
294	536	rs->base_record.absolute_ts_us +=
295		- record->same_file_next_block_short.relative_ts_us;
	537	+ record->same_file_next_block_short.relative_ts_tens_us * 10;
296	538	record_size = sizeof(record->same_file_next_block_short);
297	539	break;
298	540	}
..	..	@@ -314,6 +556,11 @@
314	556	s64 relative_us;
315	557	union log_record record;
316	558	size_t record_size;
	559	+ uid_t uid = current_uid().val;
	560	+ int block_delta;
	561	+ bool same_file, same_uid;
	562	+ bool next_block, close_block, very_close_block;
	563	+ bool close_time, very_close_time, very_very_close_time;
317	564
318	565	/*
319	566	* This may read the old value, but it's OK to delay the logging start
..	..	@@ -334,37 +581,66 @@
334	581	tail = &log->rl_tail;
335	582	relative_us = now_us - head->base_record.absolute_ts_us;
336	583
337		- if (memcmp(id, &head->base_record.file_id, sizeof(incfs_uuid_t)) \|\|
338		- relative_us >= 1ll << 32) {
339		- record.full_record = (struct full_record){
340		- .type = FULL,
341		- .block_index = block_index,
342		- .file_id = *id,
343		- .absolute_ts_us = now_us,
344		- };
345		- head->base_record.file_id = *id;
346		- record_size = sizeof(struct full_record);
347		- } else if (block_index != head->base_record.block_index + 1 \|\|
348		- relative_us >= 1 << 30) {
349		- record.same_file_record = (struct same_file_record){
350		- .type = SAME_FILE,
351		- .block_index = block_index,
352		- .relative_ts_us = relative_us,
353		- };
354		- record_size = sizeof(struct same_file_record);
355		- } else if (relative_us >= 1 << 14) {
	584	+ same_file = !memcmp(id, &head->base_record.file_id,
	585	+ sizeof(incfs_uuid_t));
	586	+ same_uid = uid == head->base_record.uid;
	587	+
	588	+ block_delta = block_index - head->base_record.block_index;
	589	+ next_block = block_delta == 1;
	590	+ very_close_block = block_delta >= S8_MIN && block_delta <= S8_MAX;
	591	+ close_block = block_delta >= S16_MIN && block_delta <= S16_MAX;
	592	+
	593	+ very_very_close_time = relative_us < (1 << 5) * 10;
	594	+ very_close_time = relative_us < (1 << 13);
	595	+ close_time = relative_us < (1 << 16);
	596	+
	597	+ if (same_file && same_uid && next_block && very_very_close_time) {
	598	+ record.same_file_next_block_short =
	599	+ (struct same_file_next_block_short){
	600	+ .type = SAME_FILE_NEXT_BLOCK_SHORT,
	601	+ .relative_ts_tens_us = div_s64(relative_us, 10),
	602	+ };
	603	+ record_size = sizeof(struct same_file_next_block_short);
	604	+ } else if (same_file && same_uid && next_block && very_close_time) {
356	605	record.same_file_next_block = (struct same_file_next_block){
357	606	.type = SAME_FILE_NEXT_BLOCK,
358	607	.relative_ts_us = relative_us,
359	608	};
360	609	record_size = sizeof(struct same_file_next_block);
361		- } else {
362		- record.same_file_next_block_short =
363		- (struct same_file_next_block_short){
364		- .type = SAME_FILE_NEXT_BLOCK_SHORT,
365		- .relative_ts_us = relative_us,
	610	+ } else if (same_file && same_uid && very_close_block &&
	611	+ very_very_close_time) {
	612	+ record.same_file_close_block_short =
	613	+ (struct same_file_close_block_short){
	614	+ .type = SAME_FILE_CLOSE_BLOCK_SHORT,
	615	+ .relative_ts_tens_us = div_s64(relative_us, 10),
	616	+ .block_index_delta = block_delta,
366	617	};
367		- record_size = sizeof(struct same_file_next_block_short);
	618	+ record_size = sizeof(struct same_file_close_block_short);
	619	+ } else if (same_file && same_uid && close_block && very_close_time) {
	620	+ record.same_file_close_block = (struct same_file_close_block){
	621	+ .type = SAME_FILE_CLOSE_BLOCK,
	622	+ .relative_ts_us = relative_us,
	623	+ .block_index_delta = block_delta,
	624	+ };
	625	+ record_size = sizeof(struct same_file_close_block);
	626	+ } else if (same_file && close_time) {
	627	+ record.same_file = (struct same_file){
	628	+ .type = SAME_FILE,
	629	+ .block_index = block_index,
	630	+ .relative_ts_us = relative_us,
	631	+ .uid = uid,
	632	+ };
	633	+ record_size = sizeof(struct same_file);
	634	+ } else {
	635	+ record.full_record = (struct full_record){
	636	+ .type = FULL,
	637	+ .block_index = block_index,
	638	+ .file_id = *id,
	639	+ .absolute_ts_us = now_us,
	640	+ .uid = uid,
	641	+ };
	642	+ head->base_record.file_id = *id;
	643	+ record_size = sizeof(struct full_record);
368	644	}
369	645
370	646	head->base_record.block_index = block_index;
..	..	@@ -405,7 +681,11 @@
405	681	int hash_per_block;
406	682	pgoff_t file_pages;
407	683
408		- tree = df->df_hash_tree;
	684	+ /*
	685	+ * Memory barrier to make sure tree is fully present if added via enable
	686	+ * verity
	687	+ */
	688	+ tree = smp_load_acquire(&df->df_hash_tree);
409	689	sig = df->df_signature;
410	690	if (!tree \|\| !sig)
411	691	return 0;
..	..	@@ -462,7 +742,7 @@
462	742	int i;
463	743	bool zero = true;
464	744
465		- pr_debug("incfs: Hash mismatch lvl:%d blk:%d\n",
	745	+ pr_warn("incfs: Hash mismatch lvl:%d blk:%d\n",
466	746	lvl, block_index);
467	747	for (i = 0; i < digest_size; i++)
468	748	if (stored_digest[i]) {
..	..	@@ -471,7 +751,7 @@
471	751	}
472	752
473	753	if (zero)
474		- pr_debug("incfs: Note saved_digest all zero - did you forget to load the hashes?\n");
	754	+ pr_debug("Note saved_digest all zero - did you forget to load the hashes?\n");
475	755	return -EBADMSG;
476	756	}
477	757
..	..	@@ -496,7 +776,7 @@
496	776	return res;
497	777
498	778	if (memcmp(stored_digest, calculated_digest, digest_size)) {
499		- pr_debug("incfs: Leaf hash mismatch blk:%d\n", block_index);
	779	+ pr_debug("Leaf hash mismatch blk:%d\n", block_index);
500	780	return -EBADMSG;
501	781	}
502	782
..	..	@@ -528,9 +808,7 @@
528	808	res_block->db_backing_file_data_offset \|=
529	809	le32_to_cpu(bme->me_data_offset_lo);
530	810	res_block->db_stored_size = le16_to_cpu(bme->me_data_size);
531		- res_block->db_comp_alg = (flags & INCFS_BLOCK_COMPRESSED_LZ4) ?
532		- COMPRESSION_LZ4 :
533		- COMPRESSION_NONE;
	811	+ res_block->db_comp_alg = flags & INCFS_BLOCK_COMPRESSED_MASK;
534	812	}
535	813
536	814	static int get_data_file_block(struct data_file *df, int index,
..	..	@@ -580,36 +858,9 @@
580	858	return 0;
581	859	}
582	860
583		-static int update_file_header_flags(struct data_file *df, u32 bits_to_reset,
584		- u32 bits_to_set)
585		-{
586		- int result;
587		- u32 new_flags;
588		- struct backing_file_context *bfc;
589		-
590		- if (!df)
591		- return -EFAULT;
592		- bfc = df->df_backing_file_context;
593		- if (!bfc)
594		- return -EFAULT;
595		-
596		- result = mutex_lock_interruptible(&bfc->bc_mutex);
597		- if (result)
598		- return result;
599		-
600		- new_flags = (df->df_header_flags & ~bits_to_reset) \| bits_to_set;
601		- if (new_flags != df->df_header_flags) {
602		- df->df_header_flags = new_flags;
603		- result = incfs_write_file_header_flags(bfc, new_flags);
604		- }
605		-
606		- mutex_unlock(&bfc->bc_mutex);
607		-
608		- return result;
609		-}
610		-
611	861	#define READ_BLOCKMAP_ENTRIES 512
612	862	int incfs_get_filled_blocks(struct data_file *df,
	863	+ struct incfs_file_data *fd,
613	864	struct incfs_get_filled_blocks_args *arg)
614	865	{
615	866	int error = 0;
..	..	@@ -623,6 +874,8 @@
623	874	int i = READ_BLOCKMAP_ENTRIES - 1;
624	875	int entries_read = 0;
625	876	struct incfs_blockmap_entry *bme;
	877	+ int data_blocks_filled = 0;
	878	+ int hash_blocks_filled = 0;
626	879
627	880	*size_out = 0;
628	881	if (end_index > df->df_total_block_count)
..	..	@@ -630,7 +883,8 @@
630	883	arg->total_blocks_out = df->df_total_block_count;
631	884	arg->data_blocks_out = df->df_data_block_count;
632	885
633		- if (df->df_header_flags & INCFS_FILE_COMPLETE) {
	886	+ if (atomic_read(&df->df_data_blocks_written) ==
	887	+ df->df_data_block_count) {
634	888	pr_debug("File marked full, fast get_filled_blocks");
635	889	if (arg->start_index > end_index) {
636	890	arg->index_out = arg->start_index;
..	..	@@ -683,6 +937,13 @@
683	937
684	938	convert_data_file_block(bme + i, &dfb);
685	939
	940	+ if (is_data_block_present(&dfb)) {
	941	+ if (arg->index_out >= df->df_data_block_count)
	942	+ ++hash_blocks_filled;
	943	+ else
	944	+ ++data_blocks_filled;
	945	+ }
	946	+
686	947	if (is_data_block_present(&dfb) == in_range)
687	948	continue;
688	949
..	..	@@ -712,13 +973,28 @@
712	973	arg->index_out = range.begin;
713	974	}
714	975
715		- if (!error && in_range && arg->start_index == 0 &&
716		- end_index == df->df_total_block_count &&
717		- *size_out == sizeof(struct incfs_filled_range)) {
718		- int result =
719		- update_file_header_flags(df, 0, INCFS_FILE_COMPLETE);
720		- /* Log failure only, since it's just a failed optimization */
721		- pr_debug("Marked file full with result %d", result);
	976	+ if (arg->start_index == 0) {
	977	+ fd->fd_get_block_pos = 0;
	978	+ fd->fd_filled_data_blocks = 0;
	979	+ fd->fd_filled_hash_blocks = 0;
	980	+ }
	981	+
	982	+ if (arg->start_index == fd->fd_get_block_pos) {
	983	+ fd->fd_get_block_pos = arg->index_out + 1;
	984	+ fd->fd_filled_data_blocks += data_blocks_filled;
	985	+ fd->fd_filled_hash_blocks += hash_blocks_filled;
	986	+ }
	987	+
	988	+ if (fd->fd_get_block_pos == df->df_total_block_count + 1) {
	989	+ if (fd->fd_filled_data_blocks >
	990	+ atomic_read(&df->df_data_blocks_written))
	991	+ atomic_set(&df->df_data_blocks_written,
	992	+ fd->fd_filled_data_blocks);
	993	+
	994	+ if (fd->fd_filled_hash_blocks >
	995	+ atomic_read(&df->df_hash_blocks_written))
	996	+ atomic_set(&df->df_hash_blocks_written,
	997	+ fd->fd_filled_hash_blocks);
722	998	}
723	999
724	1000	kfree(bme);
..	..	@@ -756,18 +1032,29 @@
756	1032	result->file_id = df->df_id;
757	1033	result->block_index = block_index;
758	1034	result->timestamp_us = ktime_to_us(ktime_get());
	1035	+ result->uid = current_uid().val;
759	1036
760		- mutex_lock(&mi->mi_pending_reads_mutex);
	1037	+ spin_lock(&mi->pending_read_lock);
761	1038
762	1039	result->serial_number = ++mi->mi_last_pending_read_number;
763	1040	mi->mi_pending_reads_count++;
764	1041
765		- list_add(&result->mi_reads_list, &mi->mi_reads_list_head);
766		- list_add(&result->segment_reads_list, &segment->reads_list_head);
767		- mutex_unlock(&mi->mi_pending_reads_mutex);
	1042	+ list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);
	1043	+ list_add_rcu(&result->segment_reads_list, &segment->reads_list_head);
	1044	+
	1045	+ spin_unlock(&mi->pending_read_lock);
768	1046
769	1047	wake_up_all(&mi->mi_pending_reads_notif_wq);
770	1048	return result;
	1049	+}
	1050	+
	1051	+static void free_pending_read_entry(struct rcu_head *entry)
	1052	+{
	1053	+ struct pending_read *read;
	1054	+
	1055	+ read = container_of(entry, struct pending_read, rcu);
	1056	+
	1057	+ kfree(read);
771	1058	}
772	1059
773	1060	/* Notifies a given data file that pending read is completed. */
..	..	@@ -783,14 +1070,17 @@
783	1070
784	1071	mi = df->df_mount_info;
785	1072
786		- mutex_lock(&mi->mi_pending_reads_mutex);
787		- list_del(&read->mi_reads_list);
788		- list_del(&read->segment_reads_list);
	1073	+ spin_lock(&mi->pending_read_lock);
	1074	+
	1075	+ list_del_rcu(&read->mi_reads_list);
	1076	+ list_del_rcu(&read->segment_reads_list);
789	1077
790	1078	mi->mi_pending_reads_count--;
791		- mutex_unlock(&mi->mi_pending_reads_mutex);
792	1079
793		- kfree(read);
	1080	+ spin_unlock(&mi->pending_read_lock);
	1081	+
	1082	+ /* Don't free. Wait for readers */
	1083	+ call_rcu(&read->rcu, free_pending_read_entry);
794	1084	}
795	1085
796	1086	static void notify_pending_reads(struct mount_info *mi,
..	..	@@ -800,26 +1090,32 @@
800	1090	struct pending_read *entry = NULL;
801	1091
802	1092	/* Notify pending reads waiting for this block. */
803		- mutex_lock(&mi->mi_pending_reads_mutex);
804		- list_for_each_entry(entry, &segment->reads_list_head,
	1093	+ rcu_read_lock();
	1094	+ list_for_each_entry_rcu(entry, &segment->reads_list_head,
805	1095	segment_reads_list) {
806	1096	if (entry->block_index == index)
807	1097	set_read_done(entry);
808	1098	}
809		- mutex_unlock(&mi->mi_pending_reads_mutex);
	1099	+ rcu_read_unlock();
810	1100	wake_up_all(&segment->new_data_arrival_wq);
	1101	+
	1102	+ atomic_inc(&mi->mi_blocks_written);
	1103	+ wake_up_all(&mi->mi_blocks_written_notif_wq);
811	1104	}
812	1105
813	1106	static int wait_for_data_block(struct data_file *df, int block_index,
814		- int timeout_ms,
815		- struct data_file_block *res_block)
	1107	+ struct data_file_block *res_block,
	1108	+ struct incfs_read_data_file_timeouts *timeouts,
	1109	+ unsigned int *delayed_min_us)
816	1110	{
817	1111	struct data_file_block block = {};
818	1112	struct data_file_segment *segment = NULL;
819	1113	struct pending_read *read = NULL;
820	1114	struct mount_info *mi = NULL;
821		- int error = 0;
	1115	+ int error;
822	1116	int wait_res = 0;
	1117	+ unsigned int delayed_pending_us = 0;
	1118	+ bool delayed_pending = false;
823	1119
824	1120	if (!df \|\| !res_block)
825	1121	return -EFAULT;
..	..	@@ -827,50 +1123,58 @@
827	1123	if (block_index < 0 \|\| block_index >= df->df_data_block_count)
828	1124	return -EINVAL;
829	1125
830		- if (df->df_blockmap_off <= 0)
	1126	+ if (df->df_blockmap_off <= 0 \|\| !df->df_mount_info)
831	1127	return -ENODATA;
832	1128
	1129	+ mi = df->df_mount_info;
833	1130	segment = get_file_segment(df, block_index);
834		- error = mutex_lock_interruptible(&segment->blockmap_mutex);
	1131	+
	1132	+ error = down_read_killable(&segment->rwsem);
835	1133	if (error)
836	1134	return error;
837	1135
838	1136	/* Look up the given block */
839	1137	error = get_data_file_block(df, block_index, &block);
840	1138
841		- /* If it's not found, create a pending read */
842		- if (!error && !is_data_block_present(&block) && timeout_ms != 0)
843		- read = add_pending_read(df, block_index);
	1139	+ up_read(&segment->rwsem);
844	1140
845		- mutex_unlock(&segment->blockmap_mutex);
846	1141	if (error)
847	1142	return error;
848	1143
849	1144	/* If the block was found, just return it. No need to wait. */
850	1145	if (is_data_block_present(&block)) {
851	1146	*res_block = block;
	1147	+ if (timeouts && timeouts->min_time_us) {
	1148	+ *delayed_min_us = timeouts->min_time_us;
	1149	+ goto out;
	1150	+ }
852	1151	return 0;
	1152	+ } else {
	1153	+ /* If it's not found, create a pending read */
	1154	+ if (timeouts && timeouts->max_pending_time_us) {
	1155	+ read = add_pending_read(df, block_index);
	1156	+ if (!read)
	1157	+ return -ENOMEM;
	1158	+ } else {
	1159	+ log_block_read(mi, &df->df_id, block_index);
	1160	+ return -ETIME;
	1161	+ }
853	1162	}
854	1163
855		- mi = df->df_mount_info;
856		-
857		- if (timeout_ms == 0) {
858		- log_block_read(mi, &df->df_id, block_index);
859		- return -ETIME;
	1164	+ /* Rest of function only applies if timeouts != NULL */
	1165	+ if (!timeouts) {
	1166	+ pr_warn("incfs: timeouts unexpectedly NULL\n");
	1167	+ return -EFSCORRUPTED;
860	1168	}
861		-
862		- if (!read)
863		- return -ENOMEM;
864	1169
865	1170	/* Wait for notifications about block's arrival */
866	1171	wait_res =
867	1172	wait_event_interruptible_timeout(segment->new_data_arrival_wq,
868		- (is_read_done(read)),
869		- msecs_to_jiffies(timeout_ms));
	1173	+ (is_read_done(read)),
	1174	+ usecs_to_jiffies(timeouts->max_pending_time_us));
870	1175
871	1176	/* Woke up, the pending read is no longer needed. */
872	1177	remove_pending_read(df, read);
873		- read = NULL;
874	1178
875	1179	if (wait_res == 0) {
876	1180	/* Wait has timed out */
..	..	@@ -885,12 +1189,19 @@
885	1189	return wait_res;
886	1190	}
887	1191
888		- error = mutex_lock_interruptible(&segment->blockmap_mutex);
	1192	+ delayed_pending = true;
	1193	+ delayed_pending_us = timeouts->max_pending_time_us -
	1194	+ jiffies_to_usecs(wait_res);
	1195	+ if (timeouts->min_pending_time_us > delayed_pending_us)
	1196	+ *delayed_min_us = timeouts->min_pending_time_us -
	1197	+ delayed_pending_us;
	1198	+
	1199	+ error = down_read_killable(&segment->rwsem);
889	1200	if (error)
890	1201	return error;
891	1202
892	1203	/*
893		- * Re-read block's info now, it has just arrived and
	1204	+ * Re-read blocks info now, it has just arrived and
894	1205	* should be available.
895	1206	*/
896	1207	error = get_data_file_block(df, block_index, &block);
..	..	@@ -899,21 +1210,59 @@
899	1210	*res_block = block;
900	1211	else {
901	1212	/*
902		- * Somehow wait finished successfully bug block still
	1213	+ * Somehow wait finished successfully but block still
903	1214	* can't be found. It's not normal.
904	1215	*/
905		- pr_warn("incfs:Wait succeeded, but block not found.\n");
	1216	+ pr_warn("incfs: Wait succeeded but block not found.\n");
906	1217	error = -ENODATA;
907	1218	}
908	1219	}
	1220	+ up_read(&segment->rwsem);
909	1221
910		- mutex_unlock(&segment->blockmap_mutex);
911		- return error;
	1222	+out:
	1223	+ if (error)
	1224	+ return error;
	1225	+
	1226	+ if (delayed_pending) {
	1227	+ mi->mi_reads_delayed_pending++;
	1228	+ mi->mi_reads_delayed_pending_us +=
	1229	+ delayed_pending_us;
	1230	+ }
	1231	+
	1232	+ if (delayed_min_us && *delayed_min_us) {
	1233	+ mi->mi_reads_delayed_min++;
	1234	+ mi->mi_reads_delayed_min_us += *delayed_min_us;
	1235	+ }
	1236	+
	1237	+ return 0;
	1238	+}
	1239	+
	1240	+static int incfs_update_sysfs_error(struct file *file, int index, int result,
	1241	+ struct mount_info mi, struct data_file df)
	1242	+{
	1243	+ int error;
	1244	+
	1245	+ if (result >= 0)
	1246	+ return 0;
	1247	+
	1248	+ error = mutex_lock_interruptible(&mi->mi_le_mutex);
	1249	+ if (error)
	1250	+ return error;
	1251	+
	1252	+ mi->mi_le_file_id = df->df_id;
	1253	+ mi->mi_le_time_us = ktime_to_us(ktime_get());
	1254	+ mi->mi_le_page = index;
	1255	+ mi->mi_le_errno = result;
	1256	+ mi->mi_le_uid = current_uid().val;
	1257	+ mutex_unlock(&mi->mi_le_mutex);
	1258	+
	1259	+ return 0;
912	1260	}
913	1261
914	1262	ssize_t incfs_read_data_file_block(struct mem_range dst, struct file *f,
915		- int index, int timeout_ms,
916		- struct mem_range tmp)
	1263	+ int index, struct mem_range tmp,
	1264	+ struct incfs_read_data_file_timeouts *timeouts,
	1265	+ unsigned int *delayed_min_us)
917	1266	{
918	1267	loff_t pos;
919	1268	ssize_t result;
..	..	@@ -923,7 +1272,7 @@
923	1272	struct data_file_block block = {};
924	1273	struct data_file *df = get_incfs_data_file(f);
925	1274
926		- if (!dst.data \|\| !df)
	1275	+ if (!dst.data \|\| !df \|\| !tmp.data)
927	1276	return -EFAULT;
928	1277
929	1278	if (tmp.len < 2 * INCFS_DATA_FILE_BLOCK_SIZE)
..	..	@@ -932,7 +1281,8 @@
932	1281	mi = df->df_mount_info;
933	1282	bfc = df->df_backing_file_context;
934	1283
935		- result = wait_for_data_block(df, index, timeout_ms, &block);
	1284	+ result = wait_for_data_block(df, index, &block, timeouts,
	1285	+ delayed_min_us);
936	1286	if (result < 0)
937	1287	goto out;
938	1288
..	..	@@ -949,7 +1299,8 @@
949	1299	result = incfs_kread(bfc, tmp.data, bytes_to_read, pos);
950	1300	if (result == bytes_to_read) {
951	1301	result =
952		- decompress(range(tmp.data, bytes_to_read), dst);
	1302	+ decompress(mi, range(tmp.data, bytes_to_read),
	1303	+ dst, block.db_comp_alg);
953	1304	if (result < 0) {
954	1305	const char *name =
955	1306	bfc->bc_file->f_path.dentry->d_name.name;
..	..	@@ -974,11 +1325,43 @@
974	1325	log_block_read(mi, &df->df_id, index);
975	1326
976	1327	out:
	1328	+ if (result == -ETIME)
	1329	+ mi->mi_reads_failed_timed_out++;
	1330	+ else if (result == -EBADMSG)
	1331	+ mi->mi_reads_failed_hash_verification++;
	1332	+ else if (result < 0)
	1333	+ mi->mi_reads_failed_other++;
	1334	+
	1335	+ incfs_update_sysfs_error(f, index, result, mi, df);
	1336	+
977	1337	return result;
978	1338	}
979	1339
	1340	+ssize_t incfs_read_merkle_tree_blocks(struct mem_range dst,
	1341	+ struct data_file *df, size_t offset)
	1342	+{
	1343	+ struct backing_file_context *bfc = NULL;
	1344	+ struct incfs_df_signature *sig = NULL;
	1345	+ size_t to_read = dst.len;
	1346	+
	1347	+ if (!dst.data \|\| !df)
	1348	+ return -EFAULT;
	1349	+
	1350	+ sig = df->df_signature;
	1351	+ bfc = df->df_backing_file_context;
	1352	+
	1353	+ if (offset > sig->hash_size)
	1354	+ return -ERANGE;
	1355	+
	1356	+ if (offset + to_read > sig->hash_size)
	1357	+ to_read = sig->hash_size - offset;
	1358	+
	1359	+ return incfs_kread(bfc, dst.data, to_read, sig->hash_offset + offset);
	1360	+}
	1361	+
980	1362	int incfs_process_new_data_block(struct data_file *df,
981		- struct incfs_fill_block block, u8 data)
	1363	+ struct incfs_fill_block block, u8 data,
	1364	+ bool *complete)
982	1365	{
983	1366	struct mount_info *mi = NULL;
984	1367	struct backing_file_context *bfc = NULL;
..	..	@@ -999,33 +1382,62 @@
999	1382	segment = get_file_segment(df, block->block_index);
1000	1383	if (!segment)
1001	1384	return -EFAULT;
	1385	+
1002	1386	if (block->compression == COMPRESSION_LZ4)
1003	1387	flags \|= INCFS_BLOCK_COMPRESSED_LZ4;
	1388	+ else if (block->compression == COMPRESSION_ZSTD)
	1389	+ flags \|= INCFS_BLOCK_COMPRESSED_ZSTD;
	1390	+ else if (block->compression)
	1391	+ return -EINVAL;
1004	1392
1005		- error = mutex_lock_interruptible(&segment->blockmap_mutex);
	1393	+ error = down_read_killable(&segment->rwsem);
1006	1394	if (error)
1007	1395	return error;
1008	1396
1009	1397	error = get_data_file_block(df, block->block_index, &existing_block);
	1398	+
	1399	+ up_read(&segment->rwsem);
	1400	+
1010	1401	if (error)
1011		- goto unlock;
1012		- if (is_data_block_present(&existing_block)) {
	1402	+ return error;
	1403	+ if (is_data_block_present(&existing_block))
1013	1404	/* Block is already present, nothing to do here */
1014		- goto unlock;
1015		- }
	1405	+ return 0;
	1406	+
	1407	+ error = down_write_killable(&segment->rwsem);
	1408	+ if (error)
	1409	+ return error;
	1410	+
	1411	+ /* Recheck inside write lock */
	1412	+ error = get_data_file_block(df, block->block_index, &existing_block);
	1413	+ if (error)
	1414	+ goto out_up_write;
	1415	+
	1416	+ if (is_data_block_present(&existing_block))
	1417	+ goto out_up_write;
1016	1418
1017	1419	error = mutex_lock_interruptible(&bfc->bc_mutex);
1018		- if (!error) {
1019		- error = incfs_write_data_block_to_backing_file(
1020		- bfc, range(data, block->data_len), block->block_index,
	1420	+ if (error)
	1421	+ goto out_up_write;
	1422	+
	1423	+ error = incfs_write_data_block_to_backing_file(bfc,
	1424	+ range(data, block->data_len), block->block_index,
1021	1425	df->df_blockmap_off, flags);
1022		- mutex_unlock(&bfc->bc_mutex);
1023		- }
	1426	+ if (error)
	1427	+ goto out_mutex_unlock;
	1428	+
	1429	+ if (atomic_inc_return(&df->df_data_blocks_written)
	1430	+ >= df->df_data_block_count)
	1431	+ *complete = true;
	1432	+
	1433	+out_mutex_unlock:
	1434	+ mutex_unlock(&bfc->bc_mutex);
1024	1435	if (!error)
1025	1436	notify_pending_reads(mi, segment, block->block_index);
1026	1437
1027		-unlock:
1028		- mutex_unlock(&segment->blockmap_mutex);
	1438	+out_up_write:
	1439	+ up_write(&segment->rwsem);
	1440	+
1029	1441	if (error)
1030	1442	pr_debug("%d error: %d\n", block->block_index, error);
1031	1443	return error;
..	..	@@ -1101,6 +1513,9 @@
1101	1513	hash_area_base, df->df_blockmap_off, df->df_size);
1102	1514	mutex_unlock(&bfc->bc_mutex);
1103	1515	}
	1516	+ if (!error)
	1517	+ atomic_inc(&df->df_hash_blocks_written);
	1518	+
1104	1519	return error;
1105	1520	}
1106	1521
..	..	@@ -1121,25 +1536,6 @@
1121	1536	df->df_total_block_count = block_count;
1122	1537	df->df_blockmap_off = base_off;
1123	1538	return error;
1124		-}
1125		-
1126		-static int process_file_attr_md(struct incfs_file_attr *fa,
1127		- struct metadata_handler *handler)
1128		-{
1129		- struct data_file *df = handler->context;
1130		- u16 attr_size = le16_to_cpu(fa->fa_size);
1131		-
1132		- if (!df)
1133		- return -EFAULT;
1134		-
1135		- if (attr_size > INCFS_MAX_FILE_ATTR_SIZE)
1136		- return -E2BIG;
1137		-
1138		- df->n_attr.fa_value_offset = le64_to_cpu(fa->fa_offset);
1139		- df->n_attr.fa_value_size = attr_size;
1140		- df->n_attr.fa_crc = le32_to_cpu(fa->fa_crc);
1141		-
1142		- return 0;
1143	1539	}
1144	1540
1145	1541	static int process_file_signature_md(struct incfs_file_signature *sg,
..	..	@@ -1217,13 +1613,58 @@
1217	1613	return error;
1218	1614	}
1219	1615
1220		-int incfs_scan_metadata_chain(struct data_file *df)
	1616	+static int process_status_md(struct incfs_status *is,
	1617	+ struct metadata_handler *handler)
	1618	+{
	1619	+ struct data_file *df = handler->context;
	1620	+
	1621	+ df->df_initial_data_blocks_written =
	1622	+ le32_to_cpu(is->is_data_blocks_written);
	1623	+ atomic_set(&df->df_data_blocks_written,
	1624	+ df->df_initial_data_blocks_written);
	1625	+
	1626	+ df->df_initial_hash_blocks_written =
	1627	+ le32_to_cpu(is->is_hash_blocks_written);
	1628	+ atomic_set(&df->df_hash_blocks_written,
	1629	+ df->df_initial_hash_blocks_written);
	1630	+
	1631	+ df->df_status_offset = handler->md_record_offset;
	1632	+ return 0;
	1633	+}
	1634	+
	1635	+static int process_file_verity_signature_md(
	1636	+ struct incfs_file_verity_signature *vs,
	1637	+ struct metadata_handler *handler)
	1638	+{
	1639	+ struct data_file *df = handler->context;
	1640	+ struct incfs_df_verity_signature *verity_signature;
	1641	+
	1642	+ if (!df)
	1643	+ return -EFAULT;
	1644	+
	1645	+ verity_signature = kzalloc(sizeof(*verity_signature), GFP_NOFS);
	1646	+ if (!verity_signature)
	1647	+ return -ENOMEM;
	1648	+
	1649	+ verity_signature->offset = le64_to_cpu(vs->vs_offset);
	1650	+ verity_signature->size = le32_to_cpu(vs->vs_size);
	1651	+ if (verity_signature->size > FS_VERITY_MAX_SIGNATURE_SIZE) {
	1652	+ kfree(verity_signature);
	1653	+ return -EFAULT;
	1654	+ }
	1655	+
	1656	+ df->df_verity_signature = verity_signature;
	1657	+ return 0;
	1658	+}
	1659	+
	1660	+static int incfs_scan_metadata_chain(struct data_file *df)
1221	1661	{
1222	1662	struct metadata_handler *handler = NULL;
1223	1663	int result = 0;
1224	1664	int records_count = 0;
1225	1665	int error = 0;
1226	1666	struct backing_file_context *bfc = NULL;
	1667	+ int nondata_block_count;
1227	1668
1228	1669	if (!df \|\| !df->df_backing_file_context)
1229	1670	return -EFAULT;
..	..	@@ -1234,20 +1675,13 @@
1234	1675	if (!handler)
1235	1676	return -ENOMEM;
1236	1677
1237		- /* No writing to the backing file while it's being scanned. */
1238		- error = mutex_lock_interruptible(&bfc->bc_mutex);
1239		- if (error)
1240		- goto out;
1241		-
1242		- /* Reading superblock */
1243	1678	handler->md_record_offset = df->df_metadata_off;
1244	1679	handler->context = df;
1245	1680	handler->handle_blockmap = process_blockmap_md;
1246		- handler->handle_file_attr = process_file_attr_md;
1247	1681	handler->handle_signature = process_file_signature_md;
	1682	+ handler->handle_status = process_status_md;
	1683	+ handler->handle_verity_signature = process_file_verity_signature_md;
1248	1684
1249		- pr_debug("incfs: Starting reading incfs-metadata records at offset %lld\n",
1250		- handler->md_record_offset);
1251	1685	while (handler->md_record_offset > 0) {
1252	1686	error = incfs_read_next_metadata_record(bfc, handler);
1253	1687	if (error) {
..	..	@@ -1259,27 +1693,32 @@
1259	1693	records_count++;
1260	1694	}
1261	1695	if (error) {
1262		- pr_debug("incfs: Error %d after reading %d incfs-metadata records.\n",
	1696	+ pr_warn("incfs: Error %d after reading %d incfs-metadata records.\n",
1263	1697	-error, records_count);
1264	1698	result = error;
1265		- } else {
1266		- pr_debug("incfs: Finished reading %d incfs-metadata records.\n",
1267		- records_count);
	1699	+ } else
1268	1700	result = records_count;
1269		- }
1270		- mutex_unlock(&bfc->bc_mutex);
1271	1701
	1702	+ nondata_block_count = df->df_total_block_count -
	1703	+ df->df_data_block_count;
1272	1704	if (df->df_hash_tree) {
1273	1705	int hash_block_count = get_blocks_count_for_size(
1274	1706	df->df_hash_tree->hash_tree_area_size);
1275	1707
1276		- if (df->df_data_block_count + hash_block_count !=
1277		- df->df_total_block_count)
	1708	+ /*
	1709	+ * Files that were created with a hash tree have the hash tree
	1710	+ * included in the block map, i.e. nondata_block_count ==
	1711	+ * hash_block_count. Files whose hash tree was added by
	1712	+ * FS_IOC_ENABLE_VERITY will still have the original block
	1713	+ * count, i.e. nondata_block_count == 0.
	1714	+ */
	1715	+ if (nondata_block_count != hash_block_count &&
	1716	+ nondata_block_count != 0)
1278	1717	result = -EINVAL;
1279		- } else if (df->df_data_block_count != df->df_total_block_count)
	1718	+ } else if (nondata_block_count != 0) {
1280	1719	result = -EINVAL;
	1720	+ }
1281	1721
1282		-out:
1283	1722	kfree(handler);
1284	1723	return result;
1285	1724	}
..	..	@@ -1292,16 +1731,17 @@
1292	1731	{
1293	1732	bool result = false;
1294	1733
1295		- mutex_lock(&mi->mi_pending_reads_mutex);
	1734	+ spin_lock(&mi->pending_read_lock);
1296	1735	result = (mi->mi_last_pending_read_number > last_number) &&
1297		- (mi->mi_pending_reads_count > 0);
1298		- mutex_unlock(&mi->mi_pending_reads_mutex);
	1736	+ (mi->mi_pending_reads_count > 0);
	1737	+ spin_unlock(&mi->pending_read_lock);
1299	1738	return result;
1300	1739	}
1301	1740
1302	1741	int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
1303	1742	struct incfs_pending_read_info *reads,
1304		- int reads_size)
	1743	+ struct incfs_pending_read_info2 *reads2,
	1744	+ int reads_size, int *new_max_sn)
1305	1745	{
1306	1746	int reported_reads = 0;
1307	1747	struct pending_read *entry = NULL;
..	..	@@ -1312,29 +1752,43 @@
1312	1752	if (reads_size <= 0)
1313	1753	return 0;
1314	1754
1315		- mutex_lock(&mi->mi_pending_reads_mutex);
	1755	+ if (!incfs_fresh_pending_reads_exist(mi, sn_lowerbound))
	1756	+ return 0;
1316	1757
1317		- if (mi->mi_last_pending_read_number <= sn_lowerbound
1318		- \|\| mi->mi_pending_reads_count == 0)
1319		- goto unlock;
	1758	+ rcu_read_lock();
1320	1759
1321		- list_for_each_entry(entry, &mi->mi_reads_list_head, mi_reads_list) {
	1760	+ list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {
1322	1761	if (entry->serial_number <= sn_lowerbound)
1323	1762	continue;
1324	1763
1325		- reads[reported_reads].file_id = entry->file_id;
1326		- reads[reported_reads].block_index = entry->block_index;
1327		- reads[reported_reads].serial_number = entry->serial_number;
1328		- reads[reported_reads].timestamp_us = entry->timestamp_us;
1329		- /* reads[reported_reads].kind = INCFS_READ_KIND_PENDING; */
	1764	+ if (reads) {
	1765	+ reads[reported_reads].file_id = entry->file_id;
	1766	+ reads[reported_reads].block_index = entry->block_index;
	1767	+ reads[reported_reads].serial_number =
	1768	+ entry->serial_number;
	1769	+ reads[reported_reads].timestamp_us =
	1770	+ entry->timestamp_us;
	1771	+ }
	1772	+
	1773	+ if (reads2) {
	1774	+ reads2[reported_reads].file_id = entry->file_id;
	1775	+ reads2[reported_reads].block_index = entry->block_index;
	1776	+ reads2[reported_reads].serial_number =
	1777	+ entry->serial_number;
	1778	+ reads2[reported_reads].timestamp_us =
	1779	+ entry->timestamp_us;
	1780	+ reads2[reported_reads].uid = entry->uid;
	1781	+ }
	1782	+
	1783	+ if (entry->serial_number > *new_max_sn)
	1784	+ *new_max_sn = entry->serial_number;
1330	1785
1331	1786	reported_reads++;
1332	1787	if (reported_reads >= reads_size)
1333	1788	break;
1334	1789	}
1335	1790
1336		-unlock:
1337		- mutex_unlock(&mi->mi_pending_reads_mutex);
	1791	+ rcu_read_unlock();
1338	1792
1339	1793	return reported_reads;
1340	1794	}
..	..	@@ -1370,8 +1824,9 @@
1370	1824	}
1371	1825
1372	1826	int incfs_collect_logged_reads(struct mount_info *mi,
1373		- struct read_log_state *reader_state,
	1827	+ struct read_log_state *state,
1374	1828	struct incfs_pending_read_info *reads,
	1829	+ struct incfs_pending_read_info2 *reads2,
1375	1830	int reads_size)
1376	1831	{
1377	1832	int dst_idx;
..	..	@@ -1382,45 +1837,51 @@
1382	1837	head = &log->rl_head;
1383	1838	tail = &log->rl_tail;
1384	1839
1385		- if (reader_state->generation_id != head->generation_id) {
	1840	+ if (state->generation_id != head->generation_id) {
1386	1841	pr_debug("read ptr is wrong generation: %u/%u",
1387		- reader_state->generation_id, head->generation_id);
	1842	+ state->generation_id, head->generation_id);
1388	1843
1389		- *reader_state = (struct read_log_state){
	1844	+ *state = (struct read_log_state){
1390	1845	.generation_id = head->generation_id,
1391	1846	};
1392	1847	}
1393	1848
1394		- if (reader_state->current_record_no < tail->current_record_no) {
	1849	+ if (state->current_record_no < tail->current_record_no) {
1395	1850	pr_debug("read ptr is behind, moving: %u/%u -> %u/%u\n",
1396		- (u32)reader_state->next_offset,
1397		- (u32)reader_state->current_pass_no,
	1851	+ (u32)state->next_offset,
	1852	+ (u32)state->current_pass_no,
1398	1853	(u32)tail->next_offset, (u32)tail->current_pass_no);
1399	1854
1400		- reader_state = tail;
	1855	+ state = tail;
1401	1856	}
1402	1857
1403	1858	for (dst_idx = 0; dst_idx < reads_size; dst_idx++) {
1404		- if (reader_state->current_record_no == head->current_record_no)
	1859	+ if (state->current_record_no == head->current_record_no)
1405	1860	break;
1406	1861
1407		- log_read_one_record(log, reader_state);
	1862	+ log_read_one_record(log, state);
1408	1863
1409		- reads[dst_idx] = (struct incfs_pending_read_info){
1410		- .file_id = reader_state->base_record.file_id,
1411		- .block_index = reader_state->base_record.block_index,
1412		- .serial_number = reader_state->current_record_no,
1413		- .timestamp_us = reader_state->base_record.absolute_ts_us
1414		- };
	1864	+ if (reads)
	1865	+ reads[dst_idx] = (struct incfs_pending_read_info) {
	1866	+ .file_id = state->base_record.file_id,
	1867	+ .block_index = state->base_record.block_index,
	1868	+ .serial_number = state->current_record_no,
	1869	+ .timestamp_us =
	1870	+ state->base_record.absolute_ts_us,
	1871	+ };
	1872	+
	1873	+ if (reads2)
	1874	+ reads2[dst_idx] = (struct incfs_pending_read_info2) {
	1875	+ .file_id = state->base_record.file_id,
	1876	+ .block_index = state->base_record.block_index,
	1877	+ .serial_number = state->current_record_no,
	1878	+ .timestamp_us =
	1879	+ state->base_record.absolute_ts_us,
	1880	+ .uid = state->base_record.uid,
	1881	+ };
1415	1882	}
1416	1883
1417	1884	spin_unlock(&log->rl_lock);
1418	1885	return dst_idx;
1419	1886	}
1420	1887
1421		-bool incfs_equal_ranges(struct mem_range lhs, struct mem_range rhs)
1422		-{
1423		- if (lhs.len != rhs.len)
1424		- return false;
1425		- return memcmp(lhs.data, rhs.data, lhs.len) == 0;
1426		-}