~hc/RK356X_SDK_RELEASE.git

..	..	@@ -18,12 +18,25 @@
18	18	* information about these ioctls.
19	19	*/
20	20
	21	+#include <asm/unaligned.h>
21	22	#include <crypto/skcipher.h>
22	23	#include <linux/key-type.h>
23	24	#include <linux/random.h>
24	25	#include <linux/seq_file.h>
25	26
26	27	#include "fscrypt_private.h"
	28	+
	29	+/* The master encryption keys for a filesystem (->s_master_keys) */
	30	+struct fscrypt_keyring {
	31	+ /*
	32	+ * Lock that protects ->key_hashtable. It does not protect the
	33	+ * fscrypt_master_key structs themselves.
	34	+ */
	35	+ spinlock_t lock;
	36	+
	37	+ /* Hash table that maps fscrypt_key_specifier to fscrypt_master_key */
	38	+ struct hlist_head key_hashtable[128];
	39	+};
27	40
28	41	static void wipe_master_key_secret(struct fscrypt_master_key_secret *secret)
29	42	{
..	..	@@ -38,20 +51,70 @@
38	51	memzero_explicit(src, sizeof(*src));
39	52	}
40	53
41		-static void free_master_key(struct fscrypt_master_key *mk)
	54	+static void fscrypt_free_master_key(struct rcu_head *head)
42	55	{
	56	+ struct fscrypt_master_key *mk =
	57	+ container_of(head, struct fscrypt_master_key, mk_rcu_head);
	58	+ /*
	59	+ * The master key secret and any embedded subkeys should have already
	60	+ * been wiped when the last active reference to the fscrypt_master_key
	61	+ * struct was dropped; doing it here would be unnecessarily late.
	62	+ * Nevertheless, use kfree_sensitive() in case anything was missed.
	63	+ */
	64	+ kfree_sensitive(mk);
	65	+}
	66	+
	67	+void fscrypt_put_master_key(struct fscrypt_master_key *mk)
	68	+{
	69	+ if (!refcount_dec_and_test(&mk->mk_struct_refs))
	70	+ return;
	71	+ /*
	72	+ * No structural references left, so free ->mk_users, and also free the
	73	+ * fscrypt_master_key struct itself after an RCU grace period ensures
	74	+ * that concurrent keyring lookups can no longer find it.
	75	+ */
	76	+ WARN_ON(refcount_read(&mk->mk_active_refs) != 0);
	77	+ key_put(mk->mk_users);
	78	+ mk->mk_users = NULL;
	79	+ call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key);
	80	+}
	81	+
	82	+void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk)
	83	+{
	84	+ struct super_block *sb = mk->mk_sb;
	85	+ struct fscrypt_keyring *keyring = sb->s_master_keys;
43	86	size_t i;
44	87
45		- wipe_master_key_secret(&mk->mk_secret);
	88	+ if (!refcount_dec_and_test(&mk->mk_active_refs))
	89	+ return;
	90	+ /*
	91	+ * No active references left, so complete the full removal of this
	92	+ * fscrypt_master_key struct by removing it from the keyring and
	93	+ * destroying any subkeys embedded in it.
	94	+ */
46	95
47		- for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) {
	96	+ spin_lock(&keyring->lock);
	97	+ hlist_del_rcu(&mk->mk_node);
	98	+ spin_unlock(&keyring->lock);
	99	+
	100	+ /*
	101	+ * ->mk_active_refs == 0 implies that ->mk_secret is not present and
	102	+ * that ->mk_decrypted_inodes is empty.
	103	+ */
	104	+ WARN_ON(is_master_key_secret_present(&mk->mk_secret));
	105	+ WARN_ON(!list_empty(&mk->mk_decrypted_inodes));
	106	+
	107	+ for (i = 0; i <= FSCRYPT_MODE_MAX; i++) {
48	108	fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]);
49	109	fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]);
50	110	fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_32_keys[i]);
51	111	}
	112	+ memzero_explicit(&mk->mk_ino_hash_key,
	113	+ sizeof(mk->mk_ino_hash_key));
	114	+ mk->mk_ino_hash_key_initialized = false;
52	115
53		- key_put(mk->mk_users);
54		- kzfree(mk);
	116	+ /* Drop the structural ref associated with the active refs. */
	117	+ fscrypt_put_master_key(mk);
55	118	}
56	119
57	120	static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec)
..	..	@@ -60,44 +123,6 @@
60	123	return false;
61	124	return master_key_spec_len(spec) != 0;
62	125	}
63		-
64		-static int fscrypt_key_instantiate(struct key *key,
65		- struct key_preparsed_payload *prep)
66		-{
67		- key->payload.data[0] = (struct fscrypt_master_key *)prep->data;
68		- return 0;
69		-}
70		-
71		-static void fscrypt_key_destroy(struct key *key)
72		-{
73		- free_master_key(key->payload.data[0]);
74		-}
75		-
76		-static void fscrypt_key_describe(const struct key key, struct seq_file m)
77		-{
78		- seq_puts(m, key->description);
79		-
80		- if (key_is_positive(key)) {
81		- const struct fscrypt_master_key *mk = key->payload.data[0];
82		-
83		- if (!is_master_key_secret_present(&mk->mk_secret))
84		- seq_puts(m, ": secret removed");
85		- }
86		-}
87		-
88		-/*
89		- * Type of key in ->s_master_keys. Each key of this type represents a master
90		- * key which has been added to the filesystem. Its payload is a
91		- * 'struct fscrypt_master_key'. The "." prefix in the key type name prevents
92		- * users from adding keys of this type via the keyrings syscalls rather than via
93		- * the intended method of FS_IOC_ADD_ENCRYPTION_KEY.
94		- */
95		-static struct key_type key_type_fscrypt = {
96		- .name = "._fscrypt",
97		- .instantiate = fscrypt_key_instantiate,
98		- .destroy = fscrypt_key_destroy,
99		- .describe = fscrypt_key_describe,
100		-};
101	126
102	127	static int fscrypt_user_key_instantiate(struct key *key,
103	128	struct key_preparsed_payload *prep)
..	..	@@ -131,53 +156,12 @@
131	156	.describe = fscrypt_user_key_describe,
132	157	};
133	158
134		-/* Search ->s_master_keys or ->mk_users */
135		-static struct key search_fscrypt_keyring(struct key keyring,
136		- struct key_type *type,
137		- const char *description)
138		-{
139		- /*
140		- * We need to mark the keyring reference as "possessed" so that we
141		- * acquire permission to search it, via the KEY_POS_SEARCH permission.
142		- */
143		- key_ref_t keyref = make_key_ref(keyring, true /* possessed */);
144		-
145		- keyref = keyring_search(keyref, type, description);
146		- if (IS_ERR(keyref)) {
147		- if (PTR_ERR(keyref) == -EAGAIN \|\| /* not found */
148		- PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */
149		- keyref = ERR_PTR(-ENOKEY);
150		- return ERR_CAST(keyref);
151		- }
152		- return key_ref_to_ptr(keyref);
153		-}
154		-
155		-#define FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE \
156		- (CONST_STRLEN("fscrypt-") + FIELD_SIZEOF(struct super_block, s_id))
157		-
158		-#define FSCRYPT_MK_DESCRIPTION_SIZE (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + 1)
159		-
160	159	#define FSCRYPT_MK_USERS_DESCRIPTION_SIZE \
161	160	(CONST_STRLEN("fscrypt-") + 2 * FSCRYPT_KEY_IDENTIFIER_SIZE + \
162	161	CONST_STRLEN("-users") + 1)
163	162
164	163	#define FSCRYPT_MK_USER_DESCRIPTION_SIZE \
165	164	(2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1)
166		-
167		-static void format_fs_keyring_description(
168		- char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE],
169		- const struct super_block *sb)
170		-{
171		- sprintf(description, "fscrypt-%s", sb->s_id);
172		-}
173		-
174		-static void format_mk_description(
175		- char description[FSCRYPT_MK_DESCRIPTION_SIZE],
176		- const struct fscrypt_key_specifier *mk_spec)
177		-{
178		- sprintf(description, "%*phN",
179		- master_key_spec_len(mk_spec), (u8 *)&mk_spec->u);
180		-}
181	165
182	166	static void format_mk_users_keyring_description(
183	167	char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE],
..	..	@@ -199,48 +183,139 @@
199	183	/* Create ->s_master_keys if needed. Synchronized by fscrypt_add_key_mutex. */
200	184	static int allocate_filesystem_keyring(struct super_block *sb)
201	185	{
202		- char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE];
203		- struct key *keyring;
	186	+ struct fscrypt_keyring *keyring;
204	187
205	188	if (sb->s_master_keys)
206	189	return 0;
207	190
208		- format_fs_keyring_description(description, sb);
209		- keyring = keyring_alloc(description, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
210		- current_cred(), KEY_POS_SEARCH \|
211		- KEY_USR_SEARCH \| KEY_USR_READ \| KEY_USR_VIEW,
212		- KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL);
213		- if (IS_ERR(keyring))
214		- return PTR_ERR(keyring);
215		-
216		- /* Pairs with READ_ONCE() in fscrypt_find_master_key() */
	191	+ keyring = kzalloc(sizeof(*keyring), GFP_KERNEL);
	192	+ if (!keyring)
	193	+ return -ENOMEM;
	194	+ spin_lock_init(&keyring->lock);
	195	+ /*
	196	+ * Pairs with the smp_load_acquire() in fscrypt_find_master_key().
	197	+ * I.e., here we publish ->s_master_keys with a RELEASE barrier so that
	198	+ * concurrent tasks can ACQUIRE it.
	199	+ */
217	200	smp_store_release(&sb->s_master_keys, keyring);
218	201	return 0;
219	202	}
220	203
221		-void fscrypt_sb_free(struct super_block *sb)
	204	+/*
	205	+ * Release all encryption keys that have been added to the filesystem, along
	206	+ * with the keyring that contains them.
	207	+ *
	208	+ * This is called at unmount time. The filesystem's underlying block device(s)
	209	+ * are still available at this time; this is important because after user file
	210	+ * accesses have been allowed, this function may need to evict keys from the
	211	+ * keyslots of an inline crypto engine, which requires the block device(s).
	212	+ *
	213	+ * This is also called when the super_block is being freed. This is needed to
	214	+ * avoid a memory leak if mounting fails after the "test_dummy_encryption"
	215	+ * option was processed, as in that case the unmount-time call isn't made.
	216	+ */
	217	+void fscrypt_destroy_keyring(struct super_block *sb)
222	218	{
223		- key_put(sb->s_master_keys);
	219	+ struct fscrypt_keyring *keyring = sb->s_master_keys;
	220	+ size_t i;
	221	+
	222	+ if (!keyring)
	223	+ return;
	224	+
	225	+ for (i = 0; i < ARRAY_SIZE(keyring->key_hashtable); i++) {
	226	+ struct hlist_head *bucket = &keyring->key_hashtable[i];
	227	+ struct fscrypt_master_key *mk;
	228	+ struct hlist_node *tmp;
	229	+
	230	+ hlist_for_each_entry_safe(mk, tmp, bucket, mk_node) {
	231	+ /*
	232	+ * Since all inodes were already evicted, every key
	233	+ * remaining in the keyring should have an empty inode
	234	+ * list, and should only still be in the keyring due to
	235	+ * the single active ref associated with ->mk_secret.
	236	+ * There should be no structural refs beyond the one
	237	+ * associated with the active ref.
	238	+ */
	239	+ WARN_ON(refcount_read(&mk->mk_active_refs) != 1);
	240	+ WARN_ON(refcount_read(&mk->mk_struct_refs) != 1);
	241	+ WARN_ON(!is_master_key_secret_present(&mk->mk_secret));
	242	+ wipe_master_key_secret(&mk->mk_secret);
	243	+ fscrypt_put_master_key_activeref(mk);
	244	+ }
	245	+ }
	246	+ kfree_sensitive(keyring);
224	247	sb->s_master_keys = NULL;
225	248	}
226	249
227		-/*
228		- * Find the specified master key in ->s_master_keys.
229		- * Returns ERR_PTR(-ENOKEY) if not found.
230		- */
231		-struct key fscrypt_find_master_key(struct super_block sb,
232		- const struct fscrypt_key_specifier *mk_spec)
	250	+static struct hlist_head *
	251	+fscrypt_mk_hash_bucket(struct fscrypt_keyring *keyring,
	252	+ const struct fscrypt_key_specifier *mk_spec)
233	253	{
234		- struct key *keyring;
235		- char description[FSCRYPT_MK_DESCRIPTION_SIZE];
	254	+ /*
	255	+ * Since key specifiers should be "random" values, it is sufficient to
	256	+ * use a trivial hash function that just takes the first several bits of
	257	+ * the key specifier.
	258	+ */
	259	+ unsigned long i = get_unaligned((unsigned long *)&mk_spec->u);
236	260
237		- /* pairs with smp_store_release() in allocate_filesystem_keyring() */
238		- keyring = READ_ONCE(sb->s_master_keys);
	261	+ return &keyring->key_hashtable[i % ARRAY_SIZE(keyring->key_hashtable)];
	262	+}
	263	+
	264	+/*
	265	+ * Find the specified master key struct in ->s_master_keys and take a structural
	266	+ * ref to it. The structural ref guarantees that the key struct continues to
	267	+ * exist, but it does not guarantee that ->s_master_keys continues to contain
	268	+ * the key struct. The structural ref needs to be dropped by
	269	+ * fscrypt_put_master_key(). Returns NULL if the key struct is not found.
	270	+ */
	271	+struct fscrypt_master_key *
	272	+fscrypt_find_master_key(struct super_block *sb,
	273	+ const struct fscrypt_key_specifier *mk_spec)
	274	+{
	275	+ struct fscrypt_keyring *keyring;
	276	+ struct hlist_head *bucket;
	277	+ struct fscrypt_master_key *mk;
	278	+
	279	+ /*
	280	+ * Pairs with the smp_store_release() in allocate_filesystem_keyring().
	281	+ * I.e., another task can publish ->s_master_keys concurrently,
	282	+ * executing a RELEASE barrier. We need to use smp_load_acquire() here
	283	+ * to safely ACQUIRE the memory the other task published.
	284	+ */
	285	+ keyring = smp_load_acquire(&sb->s_master_keys);
239	286	if (keyring == NULL)
240		- return ERR_PTR(-ENOKEY); /* No keyring yet, so no keys yet. */
	287	+ return NULL; /* No keyring yet, so no keys yet. */
241	288
242		- format_mk_description(description, mk_spec);
243		- return search_fscrypt_keyring(keyring, &key_type_fscrypt, description);
	289	+ bucket = fscrypt_mk_hash_bucket(keyring, mk_spec);
	290	+ rcu_read_lock();
	291	+ switch (mk_spec->type) {
	292	+ case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR:
	293	+ hlist_for_each_entry_rcu(mk, bucket, mk_node) {
	294	+ if (mk->mk_spec.type ==
	295	+ FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR &&
	296	+ memcmp(mk->mk_spec.u.descriptor,
	297	+ mk_spec->u.descriptor,
	298	+ FSCRYPT_KEY_DESCRIPTOR_SIZE) == 0 &&
	299	+ refcount_inc_not_zero(&mk->mk_struct_refs))
	300	+ goto out;
	301	+ }
	302	+ break;
	303	+ case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER:
	304	+ hlist_for_each_entry_rcu(mk, bucket, mk_node) {
	305	+ if (mk->mk_spec.type ==
	306	+ FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER &&
	307	+ memcmp(mk->mk_spec.u.identifier,
	308	+ mk_spec->u.identifier,
	309	+ FSCRYPT_KEY_IDENTIFIER_SIZE) == 0 &&
	310	+ refcount_inc_not_zero(&mk->mk_struct_refs))
	311	+ goto out;
	312	+ }
	313	+ break;
	314	+ }
	315	+ mk = NULL;
	316	+out:
	317	+ rcu_read_unlock();
	318	+ return mk;
244	319	}
245	320
246	321	static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk)
..	..	@@ -268,17 +343,30 @@
268	343	static struct key find_master_key_user(struct fscrypt_master_key mk)
269	344	{
270	345	char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE];
	346	+ key_ref_t keyref;
271	347
272	348	format_mk_user_description(description, mk->mk_spec.u.identifier);
273		- return search_fscrypt_keyring(mk->mk_users, &key_type_fscrypt_user,
274		- description);
	349	+
	350	+ /*
	351	+ * We need to mark the keyring reference as "possessed" so that we
	352	+ * acquire permission to search it, via the KEY_POS_SEARCH permission.
	353	+ */
	354	+ keyref = keyring_search(make_key_ref(mk->mk_users, true /possessed/),
	355	+ &key_type_fscrypt_user, description, false);
	356	+ if (IS_ERR(keyref)) {
	357	+ if (PTR_ERR(keyref) == -EAGAIN \|\| /* not found */
	358	+ PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */
	359	+ keyref = ERR_PTR(-ENOKEY);
	360	+ return ERR_CAST(keyref);
	361	+ }
	362	+ return key_ref_to_ptr(keyref);
275	363	}
276	364
277	365	/*
278	366	* Give the current user a "key" in ->mk_users. This charges the user's quota
279	367	* and marks the master key as added by the current user, so that it cannot be
280		- * removed by another user with the key. Either the master key's key->sem must
281		- * be held for write, or the master key must be still undergoing initialization.
	368	+ * removed by another user with the key. Either ->mk_sem must be held for
	369	+ * write, or the master key must be still undergoing initialization.
282	370	*/
283	371	static int add_master_key_user(struct fscrypt_master_key *mk)
284	372	{
..	..	@@ -300,7 +388,7 @@
300	388
301	389	/*
302	390	* Remove the current user's "key" from ->mk_users.
303		- * The master key's key->sem must be held for write.
	391	+ * ->mk_sem must be held for write.
304	392	*
305	393	* Returns 0 if removed, -ENOKEY if not found, or another -errno code.
306	394	*/
..	..	@@ -318,64 +406,49 @@
318	406	}
319	407
320	408	/*
321		- * Allocate a new fscrypt_master_key which contains the given secret, set it as
322		- * the payload of a new 'struct key' of type fscrypt, and link the 'struct key'
323		- * into the given keyring. Synchronized by fscrypt_add_key_mutex.
	409	+ * Allocate a new fscrypt_master_key, transfer the given secret over to it, and
	410	+ * insert it into sb->s_master_keys.
324	411	*/
325		-static int add_new_master_key(struct fscrypt_master_key_secret *secret,
326		- const struct fscrypt_key_specifier *mk_spec,
327		- struct key *keyring)
	412	+static int add_new_master_key(struct super_block *sb,
	413	+ struct fscrypt_master_key_secret *secret,
	414	+ const struct fscrypt_key_specifier *mk_spec)
328	415	{
	416	+ struct fscrypt_keyring *keyring = sb->s_master_keys;
329	417	struct fscrypt_master_key *mk;
330		- char description[FSCRYPT_MK_DESCRIPTION_SIZE];
331		- struct key *key;
332	418	int err;
333	419
334	420	mk = kzalloc(sizeof(*mk), GFP_KERNEL);
335	421	if (!mk)
336	422	return -ENOMEM;
337	423
	424	+ mk->mk_sb = sb;
	425	+ init_rwsem(&mk->mk_sem);
	426	+ refcount_set(&mk->mk_struct_refs, 1);
338	427	mk->mk_spec = *mk_spec;
339	428
340		- move_master_key_secret(&mk->mk_secret, secret);
341		- init_rwsem(&mk->mk_secret_sem);
342		-
343		- refcount_set(&mk->mk_refcount, 1); /* secret is present */
344	429	INIT_LIST_HEAD(&mk->mk_decrypted_inodes);
345	430	spin_lock_init(&mk->mk_decrypted_inodes_lock);
346	431
347	432	if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) {
348	433	err = allocate_master_key_users_keyring(mk);
349	434	if (err)
350		- goto out_free_mk;
	435	+ goto out_put;
351	436	err = add_master_key_user(mk);
352	437	if (err)
353		- goto out_free_mk;
	438	+ goto out_put;
354	439	}
355	440
356		- /*
357		- * Note that we don't charge this key to anyone's quota, since when
358		- * ->mk_users is in use those keys are charged instead, and otherwise
359		- * (when ->mk_users isn't in use) only root can add these keys.
360		- */
361		- format_mk_description(description, mk_spec);
362		- key = key_alloc(&key_type_fscrypt, description,
363		- GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(),
364		- KEY_POS_SEARCH \| KEY_USR_SEARCH \| KEY_USR_VIEW,
365		- KEY_ALLOC_NOT_IN_QUOTA, NULL);
366		- if (IS_ERR(key)) {
367		- err = PTR_ERR(key);
368		- goto out_free_mk;
369		- }
370		- err = key_instantiate_and_link(key, mk, sizeof(*mk), keyring, NULL);
371		- key_put(key);
372		- if (err)
373		- goto out_free_mk;
	441	+ move_master_key_secret(&mk->mk_secret, secret);
	442	+ refcount_set(&mk->mk_active_refs, 1); /* ->mk_secret is present */
374	443
	444	+ spin_lock(&keyring->lock);
	445	+ hlist_add_head_rcu(&mk->mk_node,
	446	+ fscrypt_mk_hash_bucket(keyring, mk_spec));
	447	+ spin_unlock(&keyring->lock);
375	448	return 0;
376	449
377		-out_free_mk:
378		- free_master_key(mk);
	450	+out_put:
	451	+ fscrypt_put_master_key(mk);
379	452	return err;
380	453	}
381	454
..	..	@@ -384,45 +457,34 @@
384	457	static int add_existing_master_key(struct fscrypt_master_key *mk,
385	458	struct fscrypt_master_key_secret *secret)
386	459	{
387		- struct key *mk_user;
388		- bool rekey;
389	460	int err;
390	461
391	462	/*
392	463	* If the current user is already in ->mk_users, then there's nothing to
393		- * do. (Not applicable for v1 policy keys, which have NULL ->mk_users.)
	464	+ * do. Otherwise, we need to add the user to ->mk_users. (Neither is
	465	+ * applicable for v1 policy keys, which have NULL ->mk_users.)
394	466	*/
395	467	if (mk->mk_users) {
396		- mk_user = find_master_key_user(mk);
	468	+ struct key *mk_user = find_master_key_user(mk);
	469	+
397	470	if (mk_user != ERR_PTR(-ENOKEY)) {
398	471	if (IS_ERR(mk_user))
399	472	return PTR_ERR(mk_user);
400	473	key_put(mk_user);
401	474	return 0;
402	475	}
403		- }
404		-
405		- /* If we'll be re-adding ->mk_secret, try to take the reference. */
406		- rekey = !is_master_key_secret_present(&mk->mk_secret);
407		- if (rekey && !refcount_inc_not_zero(&mk->mk_refcount))
408		- return KEY_DEAD;
409		-
410		- /* Add the current user to ->mk_users, if applicable. */
411		- if (mk->mk_users) {
412	476	err = add_master_key_user(mk);
413		- if (err) {
414		- if (rekey && refcount_dec_and_test(&mk->mk_refcount))
415		- return KEY_DEAD;
	477	+ if (err)
416	478	return err;
417		- }
418	479	}
419	480
420	481	/* Re-add the secret if needed. */
421		- if (rekey) {
422		- down_write(&mk->mk_secret_sem);
	482	+ if (!is_master_key_secret_present(&mk->mk_secret)) {
	483	+ if (!refcount_inc_not_zero(&mk->mk_active_refs))
	484	+ return KEY_DEAD;
423	485	move_master_key_secret(&mk->mk_secret, secret);
424		- up_write(&mk->mk_secret_sem);
425	486	}
	487	+
426	488	return 0;
427	489	}
428	490
..	..	@@ -431,38 +493,36 @@
431	493	const struct fscrypt_key_specifier *mk_spec)
432	494	{
433	495	static DEFINE_MUTEX(fscrypt_add_key_mutex);
434		- struct key *key;
	496	+ struct fscrypt_master_key *mk;
435	497	int err;
436	498
437	499	mutex_lock(&fscrypt_add_key_mutex); /* serialize find + link */
438		-retry:
439		- key = fscrypt_find_master_key(sb, mk_spec);
440		- if (IS_ERR(key)) {
441		- err = PTR_ERR(key);
442		- if (err != -ENOKEY)
443		- goto out_unlock;
	500	+
	501	+ mk = fscrypt_find_master_key(sb, mk_spec);
	502	+ if (!mk) {
444	503	/* Didn't find the key in ->s_master_keys. Add it. */
445	504	err = allocate_filesystem_keyring(sb);
446		- if (err)
447		- goto out_unlock;
448		- err = add_new_master_key(secret, mk_spec, sb->s_master_keys);
	505	+ if (!err)
	506	+ err = add_new_master_key(sb, secret, mk_spec);
449	507	} else {
450	508	/*
451	509	* Found the key in ->s_master_keys. Re-add the secret if
452	510	* needed, and add the user to ->mk_users if needed.
453	511	*/
454		- down_write(&key->sem);
455		- err = add_existing_master_key(key->payload.data[0], secret);
456		- up_write(&key->sem);
	512	+ down_write(&mk->mk_sem);
	513	+ err = add_existing_master_key(mk, secret);
	514	+ up_write(&mk->mk_sem);
457	515	if (err == KEY_DEAD) {
458		- /* Key being removed or needs to be removed */
459		- key_invalidate(key);
460		- key_put(key);
461		- goto retry;
	516	+ /*
	517	+ * We found a key struct, but it's already been fully
	518	+ * removed. Ignore the old struct and add a new one.
	519	+ * fscrypt_add_key_mutex means we don't need to worry
	520	+ * about concurrent adds.
	521	+ */
	522	+ err = add_new_master_key(sb, secret, mk_spec);
462	523	}
463		- key_put(key);
	524	+ fscrypt_put_master_key(mk);
464	525	}
465		-out_unlock:
466	526	mutex_unlock(&fscrypt_add_key_mutex);
467	527	return err;
468	528	}
..	..	@@ -538,7 +598,7 @@
538	598	static void fscrypt_provisioning_key_free_preparse(
539	599	struct key_preparsed_payload *prep)
540	600	{
541		- kzfree(prep->payload.data[0]);
	601	+ kfree_sensitive(prep->payload.data[0]);
542	602	}
543	603
544	604	static void fscrypt_provisioning_key_describe(const struct key *key,
..	..	@@ -555,7 +615,7 @@
555	615
556	616	static void fscrypt_provisioning_key_destroy(struct key *key)
557	617	{
558		- kzfree(key->payload.data[0]);
	618	+ kfree_sensitive(key->payload.data[0]);
559	619	}
560	620
561	621	static struct key_type key_type_fscrypt_provisioning = {
..	..	@@ -756,19 +816,19 @@
756	816	const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE])
757	817	{
758	818	struct fscrypt_key_specifier mk_spec;
759		- struct key key, mk_user;
760	819	struct fscrypt_master_key *mk;
	820	+ struct key *mk_user;
761	821	int err;
762	822
763	823	mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER;
764	824	memcpy(mk_spec.u.identifier, identifier, FSCRYPT_KEY_IDENTIFIER_SIZE);
765	825
766		- key = fscrypt_find_master_key(sb, &mk_spec);
767		- if (IS_ERR(key)) {
768		- err = PTR_ERR(key);
	826	+ mk = fscrypt_find_master_key(sb, &mk_spec);
	827	+ if (!mk) {
	828	+ err = -ENOKEY;
769	829	goto out;
770	830	}
771		- mk = key->payload.data[0];
	831	+ down_read(&mk->mk_sem);
772	832	mk_user = find_master_key_user(mk);
773	833	if (IS_ERR(mk_user)) {
774	834	err = PTR_ERR(mk_user);
..	..	@@ -776,7 +836,8 @@
776	836	key_put(mk_user);
777	837	err = 0;
778	838	}
779		- key_put(key);
	839	+ up_read(&mk->mk_sem);
	840	+ fscrypt_put_master_key(mk);
780	841	out:
781	842	if (err == -ENOKEY && capable(CAP_FOWNER))
782	843	err = 0;
..	..	@@ -838,6 +899,7 @@
838	899	struct list_head *pos;
839	900	size_t busy_count = 0;
840	901	unsigned long ino;
	902	+ char ino_str[50] = "";
841	903
842	904	spin_lock(&mk->mk_decrypted_inodes_lock);
843	905
..	..	@@ -859,41 +921,23 @@
859	921	}
860	922	spin_unlock(&mk->mk_decrypted_inodes_lock);
861	923
	924	+ /* If the inode is currently being created, ino may still be 0. */
	925	+ if (ino)
	926	+ snprintf(ino_str, sizeof(ino_str), ", including ino %lu", ino);
	927	+
862	928	fscrypt_warn(NULL,
863		- "%s: %zu inode(s) still busy after removing key with %s %*phN, including ino %lu",
	929	+ "%s: %zu inode(s) still busy after removing key with %s %*phN%s",
864	930	sb->s_id, busy_count, master_key_spec_type(&mk->mk_spec),
865	931	master_key_spec_len(&mk->mk_spec), (u8 *)&mk->mk_spec.u,
866		- ino);
	932	+ ino_str);
867	933	return -EBUSY;
868	934	}
869		-
870		-static BLOCKING_NOTIFIER_HEAD(fscrypt_key_removal_notifiers);
871		-
872		-/*
873		- * Register a function to be executed when the FS_IOC_REMOVE_ENCRYPTION_KEY
874		- * ioctl has removed a key and is about to try evicting inodes.
875		- */
876		-int fscrypt_register_key_removal_notifier(struct notifier_block *nb)
877		-{
878		- return blocking_notifier_chain_register(&fscrypt_key_removal_notifiers,
879		- nb);
880		-}
881		-EXPORT_SYMBOL_GPL(fscrypt_register_key_removal_notifier);
882		-
883		-int fscrypt_unregister_key_removal_notifier(struct notifier_block *nb)
884		-{
885		- return blocking_notifier_chain_unregister(&fscrypt_key_removal_notifiers,
886		- nb);
887		-}
888		-EXPORT_SYMBOL_GPL(fscrypt_unregister_key_removal_notifier);
889	935
890	936	static int try_to_lock_encrypted_files(struct super_block *sb,
891	937	struct fscrypt_master_key *mk)
892	938	{
893	939	int err1;
894	940	int err2;
895		-
896		- blocking_notifier_call_chain(&fscrypt_key_removal_notifiers, 0, NULL);
897	941
898	942	/*
899	943	* An inode can't be evicted while it is dirty or has dirty pages.
..	..	@@ -955,11 +999,10 @@
955	999	struct super_block *sb = file_inode(filp)->i_sb;
956	1000	struct fscrypt_remove_key_arg __user *uarg = _uarg;
957	1001	struct fscrypt_remove_key_arg arg;
958		- struct key *key;
959	1002	struct fscrypt_master_key *mk;
960	1003	u32 status_flags = 0;
961	1004	int err;
962		- bool dead;
	1005	+ bool inodes_remain;
963	1006
964	1007	if (copy_from_user(&arg, uarg, sizeof(arg)))
965	1008	return -EFAULT;
..	..	@@ -979,12 +1022,10 @@
979	1022	return -EACCES;
980	1023
981	1024	/* Find the key being removed. */
982		- key = fscrypt_find_master_key(sb, &arg.key_spec);
983		- if (IS_ERR(key))
984		- return PTR_ERR(key);
985		- mk = key->payload.data[0];
986		-
987		- down_write(&key->sem);
	1025	+ mk = fscrypt_find_master_key(sb, &arg.key_spec);
	1026	+ if (!mk)
	1027	+ return -ENOKEY;
	1028	+ down_write(&mk->mk_sem);
988	1029
989	1030	/* If relevant, remove current user's (or all users) claim to the key */
990	1031	if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) {
..	..	@@ -993,7 +1034,7 @@
993	1034	else
994	1035	err = remove_master_key_user(mk);
995	1036	if (err) {
996		- up_write(&key->sem);
	1037	+ up_write(&mk->mk_sem);
997	1038	goto out_put_key;
998	1039	}
999	1040	if (mk->mk_users->keys.nr_leaves_on_tree != 0) {
..	..	@@ -1005,28 +1046,22 @@
1005	1046	status_flags \|=
1006	1047	FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS;
1007	1048	err = 0;
1008		- up_write(&key->sem);
	1049	+ up_write(&mk->mk_sem);
1009	1050	goto out_put_key;
1010	1051	}
1011	1052	}
1012	1053
1013	1054	/* No user claims remaining. Go ahead and wipe the secret. */
1014		- dead = false;
	1055	+ err = -ENOKEY;
1015	1056	if (is_master_key_secret_present(&mk->mk_secret)) {
1016		- down_write(&mk->mk_secret_sem);
1017	1057	wipe_master_key_secret(&mk->mk_secret);
1018		- dead = refcount_dec_and_test(&mk->mk_refcount);
1019		- up_write(&mk->mk_secret_sem);
1020		- }
1021		- up_write(&key->sem);
1022		- if (dead) {
1023		- /*
1024		- * No inodes reference the key, and we wiped the secret, so the
1025		- * key object is free to be removed from the keyring.
1026		- */
1027		- key_invalidate(key);
	1058	+ fscrypt_put_master_key_activeref(mk);
1028	1059	err = 0;
1029		- } else {
	1060	+ }
	1061	+ inodes_remain = refcount_read(&mk->mk_active_refs) > 0;
	1062	+ up_write(&mk->mk_sem);
	1063	+
	1064	+ if (inodes_remain) {
1030	1065	/* Some inodes still reference this key; try to evict them. */
1031	1066	err = try_to_lock_encrypted_files(sb, mk);
1032	1067	if (err == -EBUSY) {
..	..	@@ -1042,7 +1077,7 @@
1042	1077	* has been fully removed including all files locked.
1043	1078	*/
1044	1079	out_put_key:
1045		- key_put(key);
	1080	+ fscrypt_put_master_key(mk);
1046	1081	if (err == 0)
1047	1082	err = put_user(status_flags, &uarg->removal_status_flags);
1048	1083	return err;
..	..	@@ -1089,7 +1124,6 @@
1089	1124	{
1090	1125	struct super_block *sb = file_inode(filp)->i_sb;
1091	1126	struct fscrypt_get_key_status_arg arg;
1092		- struct key *key;
1093	1127	struct fscrypt_master_key *mk;
1094	1128	int err;
1095	1129
..	..	@@ -1106,19 +1140,18 @@
1106	1140	arg.user_count = 0;
1107	1141	memset(arg.__out_reserved, 0, sizeof(arg.__out_reserved));
1108	1142
1109		- key = fscrypt_find_master_key(sb, &arg.key_spec);
1110		- if (IS_ERR(key)) {
1111		- if (key != ERR_PTR(-ENOKEY))
1112		- return PTR_ERR(key);
	1143	+ mk = fscrypt_find_master_key(sb, &arg.key_spec);
	1144	+ if (!mk) {
1113	1145	arg.status = FSCRYPT_KEY_STATUS_ABSENT;
1114	1146	err = 0;
1115	1147	goto out;
1116	1148	}
1117		- mk = key->payload.data[0];
1118		- down_read(&key->sem);
	1149	+ down_read(&mk->mk_sem);
1119	1150
1120	1151	if (!is_master_key_secret_present(&mk->mk_secret)) {
1121		- arg.status = FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED;
	1152	+ arg.status = refcount_read(&mk->mk_active_refs) > 0 ?
	1153	+ FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED :
	1154	+ FSCRYPT_KEY_STATUS_ABSENT /* raced with full removal */;
1122	1155	err = 0;
1123	1156	goto out_release_key;
1124	1157	}
..	..	@@ -1140,8 +1173,8 @@
1140	1173	}
1141	1174	err = 0;
1142	1175	out_release_key:
1143		- up_read(&key->sem);
1144		- key_put(key);
	1176	+ up_read(&mk->mk_sem);
	1177	+ fscrypt_put_master_key(mk);
1145	1178	out:
1146	1179	if (!err && copy_to_user(uarg, &arg, sizeof(arg)))
1147	1180	err = -EFAULT;
..	..	@@ -1153,13 +1186,9 @@
1153	1186	{
1154	1187	int err;
1155	1188
1156		- err = register_key_type(&key_type_fscrypt);
1157		- if (err)
1158		- return err;
1159		-
1160	1189	err = register_key_type(&key_type_fscrypt_user);
1161	1190	if (err)
1162		- goto err_unregister_fscrypt;
	1191	+ return err;
1163	1192
1164	1193	err = register_key_type(&key_type_fscrypt_provisioning);
1165	1194	if (err)
..	..	@@ -1169,7 +1198,5 @@
1169	1198
1170	1199	err_unregister_fscrypt_user:
1171	1200	unregister_key_type(&key_type_fscrypt_user);
1172		-err_unregister_fscrypt:
1173		- unregister_key_type(&key_type_fscrypt);
1174	1201	return err;
1175	1202	}