forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/kernel/futex.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  *  Fast Userspace Mutexes (which I call "Futexes!").
34  *  (C) Rusty Russell, IBM 2002
....@@ -29,49 +30,20 @@
2930  *
3031  *  "The futexes are also cursed."
3132  *  "But they come in a choice of three flavours!"
32
- *
33
- *  This program is free software; you can redistribute it and/or modify
34
- *  it under the terms of the GNU General Public License as published by
35
- *  the Free Software Foundation; either version 2 of the License, or
36
- *  (at your option) any later version.
37
- *
38
- *  This program is distributed in the hope that it will be useful,
39
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
40
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
41
- *  GNU General Public License for more details.
42
- *
43
- *  You should have received a copy of the GNU General Public License
44
- *  along with this program; if not, write to the Free Software
45
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
4633  */
4734 #include <linux/compat.h>
48
-#include <linux/slab.h>
49
-#include <linux/poll.h>
50
-#include <linux/fs.h>
51
-#include <linux/file.h>
5235 #include <linux/jhash.h>
53
-#include <linux/init.h>
54
-#include <linux/futex.h>
55
-#include <linux/mount.h>
5636 #include <linux/pagemap.h>
5737 #include <linux/syscalls.h>
58
-#include <linux/signal.h>
59
-#include <linux/export.h>
60
-#include <linux/magic.h>
61
-#include <linux/pid.h>
62
-#include <linux/nsproxy.h>
63
-#include <linux/ptrace.h>
64
-#include <linux/sched/rt.h>
65
-#include <linux/sched/wake_q.h>
66
-#include <linux/sched/mm.h>
67
-#include <linux/hugetlb.h>
6838 #include <linux/freezer.h>
69
-#include <linux/bootmem.h>
39
+#include <linux/memblock.h>
7040 #include <linux/fault-inject.h>
41
+#include <linux/time_namespace.h>
7142 
7243 #include <asm/futex.h>
7344 
7445 #include "locking/rtmutex_common.h"
46
+#include <trace/hooks/futex.h>
7547 
7648 /*
7749  * READ this before attempting to hack on futexes!
....@@ -147,8 +119,7 @@
147119  *
148120  * Where (A) orders the waiters increment and the futex value read through
149121  * atomic operations (see hb_waiters_inc) and where (B) orders the write
150
- * to futex and the waiters read -- this is done by the barriers for both
151
- * shared and private futexes in get_futex_key_refs().
122
+ * to futex and the waiters read (see hb_waiters_pending()).
152123  *
153124  * This yields the following case (where X:=waiters, Y:=futex):
154125  *
....@@ -212,7 +183,7 @@
212183 	struct rt_mutex pi_mutex;
213184 
214185 	struct task_struct *owner;
215
-	atomic_t refcount;
186
+	refcount_t refcount;
216187 
217188 	union futex_key key;
218189 } __randomize_layout;
....@@ -321,12 +292,8 @@
321292 	if (IS_ERR(dir))
322293 		return PTR_ERR(dir);
323294 
324
-	if (!debugfs_create_bool("ignore-private", mode, dir,
325
-				 &fail_futex.ignore_private)) {
326
-		debugfs_remove_recursive(dir);
327
-		return -ENOMEM;
328
-	}
329
-
295
+	debugfs_create_bool("ignore-private", mode, dir,
296
+			    &fail_futex.ignore_private);
330297 	return 0;
331298 }
332299 
....@@ -346,17 +313,6 @@
346313 #else
347314 static inline void compat_exit_robust_list(struct task_struct *curr) { }
348315 #endif
349
-
350
-static inline void futex_get_mm(union futex_key *key)
351
-{
352
-	mmgrab(key->private.mm);
353
-	/*
354
-	 * Ensure futex_get_mm() implies a full barrier such that
355
-	 * get_futex_key() implies a full barrier. This is relied upon
356
-	 * as smp_mb(); (B), see the ordering comment above.
357
-	 */
358
-	smp_mb__after_atomic();
359
-}
360316 
361317 /*
362318  * Reflects a new waiter being added to the waitqueue.
....@@ -386,6 +342,10 @@
386342 static inline int hb_waiters_pending(struct futex_hash_bucket *hb)
387343 {
388344 #ifdef CONFIG_SMP
345
+	/*
346
+	 * Full barrier (B), see the ordering comment above.
347
+	 */
348
+	smp_mb();
389349 	return atomic_read(&hb->waiters);
390350 #else
391351 	return 1;
....@@ -423,67 +383,38 @@
423383 		&& key1->both.offset == key2->both.offset);
424384 }
425385 
426
-/*
427
- * Take a reference to the resource addressed by a key.
428
- * Can be called while holding spinlocks.
386
+enum futex_access {
387
+	FUTEX_READ,
388
+	FUTEX_WRITE
389
+};
390
+
391
+/**
392
+ * futex_setup_timer - set up the sleeping hrtimer.
393
+ * @time:	ptr to the given timeout value
394
+ * @timeout:	the hrtimer_sleeper structure to be set up
395
+ * @flags:	futex flags
396
+ * @range_ns:	optional range in ns
429397  *
398
+ * Return: Initialized hrtimer_sleeper structure or NULL if no timeout
399
+ *	   value given
430400  */
431
-static void get_futex_key_refs(union futex_key *key)
401
+static inline struct hrtimer_sleeper *
402
+futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
403
+		  int flags, u64 range_ns)
432404 {
433
-	if (!key->both.ptr)
434
-		return;
405
+	if (!time)
406
+		return NULL;
435407 
408
+	hrtimer_init_sleeper_on_stack(timeout, (flags & FLAGS_CLOCKRT) ?
409
+				      CLOCK_REALTIME : CLOCK_MONOTONIC,
410
+				      HRTIMER_MODE_ABS);
436411 	/*
437
-	 * On MMU less systems futexes are always "private" as there is no per
438
-	 * process address space. We need the smp wmb nevertheless - yes,
439
-	 * arch/blackfin has MMU less SMP ...
412
+	 * If range_ns is 0, calling hrtimer_set_expires_range_ns() is
413
+	 * effectively the same as calling hrtimer_set_expires().
440414 	 */
441
-	if (!IS_ENABLED(CONFIG_MMU)) {
442
-		smp_mb(); /* explicit smp_mb(); (B) */
443
-		return;
444
-	}
415
+	hrtimer_set_expires_range_ns(&timeout->timer, *time, range_ns);
445416 
446
-	switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
447
-	case FUT_OFF_INODE:
448
-		smp_mb();		/* explicit smp_mb(); (B) */
449
-		break;
450
-	case FUT_OFF_MMSHARED:
451
-		futex_get_mm(key); /* implies smp_mb(); (B) */
452
-		break;
453
-	default:
454
-		/*
455
-		 * Private futexes do not hold reference on an inode or
456
-		 * mm, therefore the only purpose of calling get_futex_key_refs
457
-		 * is because we need the barrier for the lockless waiter check.
458
-		 */
459
-		smp_mb(); /* explicit smp_mb(); (B) */
460
-	}
461
-}
462
-
463
-/*
464
- * Drop a reference to the resource addressed by a key.
465
- * The hash bucket spinlock must not be held. This is
466
- * a no-op for private futexes, see comment in the get
467
- * counterpart.
468
- */
469
-static void drop_futex_key_refs(union futex_key *key)
470
-{
471
-	if (!key->both.ptr) {
472
-		/* If we're here then we tried to put a key we failed to get */
473
-		WARN_ON_ONCE(1);
474
-		return;
475
-	}
476
-
477
-	if (!IS_ENABLED(CONFIG_MMU))
478
-		return;
479
-
480
-	switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
481
-	case FUT_OFF_INODE:
482
-		break;
483
-	case FUT_OFF_MMSHARED:
484
-		mmdrop(key->private.mm);
485
-		break;
486
-	}
417
+	return timeout;
487418 }
488419 
489420 /*
....@@ -529,20 +460,23 @@
529460 /**
530461  * get_futex_key() - Get parameters which are the keys for a futex
531462  * @uaddr:	virtual address of the futex
532
- * @fshared:	0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED
463
+ * @fshared:	false for a PROCESS_PRIVATE futex, true for PROCESS_SHARED
533464  * @key:	address where result is stored.
534
- * @rw:		mapping needs to be read/write (values: VERIFY_READ,
535
- *              VERIFY_WRITE)
465
+ * @rw:		mapping needs to be read/write (values: FUTEX_READ,
466
+ *              FUTEX_WRITE)
536467  *
537468  * Return: a negative error code or 0
538469  *
539470  * The key words are stored in @key on success.
540471  *
541472  * For shared mappings (when @fshared), the key is:
473
+ *
542474  *   ( inode->i_sequence, page->index, offset_within_page )
475
+ *
543476  * [ also see get_inode_sequence_number() ]
544477  *
545478  * For private mappings (or when !@fshared), the key is:
479
+ *
546480  *   ( current->mm, address, 0 )
547481  *
548482  * This allows (cross process, where applicable) identification of the futex
....@@ -550,8 +484,8 @@
550484  *
551485  * lock_page() might sleep, the caller should not hold a spinlock.
552486  */
553
-static int
554
-get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
487
+static int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
488
+			 enum futex_access rw)
555489 {
556490 	unsigned long address = (unsigned long)uaddr;
557491 	struct mm_struct *mm = current->mm;
....@@ -567,7 +501,7 @@
567501 		return -EINVAL;
568502 	address -= key->both.offset;
569503 
570
-	if (unlikely(!access_ok(rw, uaddr, sizeof(u32))))
504
+	if (unlikely(!access_ok(uaddr, sizeof(u32))))
571505 		return -EFAULT;
572506 
573507 	if (unlikely(should_fail_futex(fshared)))
....@@ -583,21 +517,20 @@
583517 	if (!fshared) {
584518 		key->private.mm = mm;
585519 		key->private.address = address;
586
-		get_futex_key_refs(key);  /* implies smp_mb(); (B) */
587520 		return 0;
588521 	}
589522 
590523 again:
591524 	/* Ignore any VERIFY_READ mapping (futex common case) */
592
-	if (unlikely(should_fail_futex(fshared)))
525
+	if (unlikely(should_fail_futex(true)))
593526 		return -EFAULT;
594527 
595
-	err = get_user_pages_fast(address, 1, 1, &page);
528
+	err = get_user_pages_fast(address, 1, FOLL_WRITE, &page);
596529 	/*
597530 	 * If write access is not required (eg. FUTEX_WAIT), try
598531 	 * and get read-only access.
599532 	 */
600
-	if (err == -EFAULT && rw == VERIFY_READ) {
533
+	if (err == -EFAULT && rw == FUTEX_READ) {
601534 		err = get_user_pages_fast(address, 1, 0, &page);
602535 		ro = 1;
603536 	}
....@@ -654,7 +587,7 @@
654587 		lock_page(page);
655588 		shmem_swizzled = PageSwapCache(page) || page->mapping;
656589 		unlock_page(page);
657
-		put_page(page);
590
+		put_user_page(page);
658591 
659592 		if (shmem_swizzled)
660593 			goto again;
....@@ -677,7 +610,7 @@
677610 		 * A RO anonymous page will never change and thus doesn't make
678611 		 * sense for futex operations.
679612 		 */
680
-		if (unlikely(should_fail_futex(fshared)) || ro) {
613
+		if (unlikely(should_fail_futex(true)) || ro) {
681614 			err = -EFAULT;
682615 			goto out;
683616 		}
....@@ -704,7 +637,7 @@
704637 
705638 		if (READ_ONCE(page->mapping) != mapping) {
706639 			rcu_read_unlock();
707
-			put_page(page);
640
+			put_user_page(page);
708641 
709642 			goto again;
710643 		}
....@@ -712,7 +645,7 @@
712645 		inode = READ_ONCE(mapping->host);
713646 		if (!inode) {
714647 			rcu_read_unlock();
715
-			put_page(page);
648
+			put_user_page(page);
716649 
717650 			goto again;
718651 		}
....@@ -723,16 +656,9 @@
723656 		rcu_read_unlock();
724657 	}
725658 
726
-	get_futex_key_refs(key); /* implies smp_mb(); (B) */
727
-
728659 out:
729
-	put_page(page);
660
+	put_user_page(page);
730661 	return err;
731
-}
732
-
733
-static inline void put_futex_key(union futex_key *key)
734
-{
735
-	drop_futex_key_refs(key);
736662 }
737663 
738664 /**
....@@ -752,10 +678,10 @@
752678 	struct mm_struct *mm = current->mm;
753679 	int ret;
754680 
755
-	down_read(&mm->mmap_sem);
756
-	ret = fixup_user_fault(current, mm, (unsigned long)uaddr,
681
+	mmap_read_lock(mm);
682
+	ret = fixup_user_fault(mm, (unsigned long)uaddr,
757683 			       FAULT_FLAG_WRITE, NULL);
758
-	up_read(&mm->mmap_sem);
684
+	mmap_read_unlock(mm);
759685 
760686 	return ret < 0 ? ret : 0;
761687 }
....@@ -821,7 +747,7 @@
821747 	INIT_LIST_HEAD(&pi_state->list);
822748 	/* pi_mutex gets initialized later */
823749 	pi_state->owner = NULL;
824
-	atomic_set(&pi_state->refcount, 1);
750
+	refcount_set(&pi_state->refcount, 1);
825751 	pi_state->key = FUTEX_KEY_INIT;
826752 
827753 	current->pi_state_cache = pi_state;
....@@ -864,7 +790,7 @@
864790 
865791 static void get_pi_state(struct futex_pi_state *pi_state)
866792 {
867
-	WARN_ON_ONCE(!atomic_inc_not_zero(&pi_state->refcount));
793
+	WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));
868794 }
869795 
870796 /*
....@@ -876,7 +802,7 @@
876802 	if (!pi_state)
877803 		return;
878804 
879
-	if (!atomic_dec_and_test(&pi_state->refcount))
805
+	if (!refcount_dec_and_test(&pi_state->refcount))
880806 		return;
881807 
882808 	/*
....@@ -901,7 +827,7 @@
901827 		 * refcount is at 0 - put it back to 1.
902828 		 */
903829 		pi_state->owner = NULL;
904
-		atomic_set(&pi_state->refcount, 1);
830
+		refcount_set(&pi_state->refcount, 1);
905831 		current->pi_state_cache = pi_state;
906832 	}
907833 }
....@@ -944,7 +870,7 @@
944870 		 * In that case; drop the locks to let put_pi_state() make
945871 		 * progress and retry the loop.
946872 		 */
947
-		if (!atomic_inc_not_zero(&pi_state->refcount)) {
873
+		if (!refcount_inc_not_zero(&pi_state->refcount)) {
948874 			raw_spin_unlock_irq(&curr->pi_lock);
949875 			cpu_relax();
950876 			raw_spin_lock_irq(&curr->pi_lock);
....@@ -962,9 +888,7 @@
962888 		if (head->next != next) {
963889 			/* retain curr->pi_lock for the loop invariant */
964890 			raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
965
-			raw_spin_unlock_irq(&curr->pi_lock);
966891 			spin_unlock(&hb->lock);
967
-			raw_spin_lock_irq(&curr->pi_lock);
968892 			put_pi_state(pi_state);
969893 			continue;
970894 		}
....@@ -1011,7 +935,7 @@
1011935  * [10] Found  | Found    | task      | !=taskTID | 0/1    | Invalid
1012936  *
1013937  * [1]	Indicates that the kernel can acquire the futex atomically. We
1014
- *	came came here due to a stale FUTEX_WAITERS/FUTEX_OWNER_DIED bit.
938
+ *	came here due to a stale FUTEX_WAITERS/FUTEX_OWNER_DIED bit.
1015939  *
1016940  * [2]	Valid, if TID does not belong to a kernel thread. If no matching
1017941  *      thread is found then it indicates that the owner TID has died.
....@@ -1104,7 +1028,7 @@
11041028 	 * and futex_wait_requeue_pi() as it cannot go to 0 and consequently
11051029 	 * free pi_state before we can take a reference ourselves.
11061030 	 */
1107
-	WARN_ON(!atomic_read(&pi_state->refcount));
1031
+	WARN_ON(!refcount_read(&pi_state->refcount));
11081032 
11091033 	/*
11101034 	 * Now that we have a pi_state, we can acquire wait_lock
....@@ -1198,6 +1122,7 @@
11981122 
11991123 /**
12001124  * wait_for_owner_exiting - Block until the owner has exited
1125
+ * @ret: owner's current futex lock status
12011126  * @exiting:	Pointer to the exiting task
12021127  *
12031128  * Caller must hold a refcount on @exiting.
....@@ -1400,7 +1325,7 @@
14001325 static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
14011326 {
14021327 	int err;
1403
-	u32 uninitialized_var(curval);
1328
+	u32 curval;
14041329 
14051330 	if (unlikely(should_fail_futex(true)))
14061331 		return -EFAULT;
....@@ -1525,9 +1450,9 @@
15251450 {
15261451 	struct futex_hash_bucket *hb;
15271452 
1528
-	if (WARN_ON_SMP(!q->lock_ptr || !spin_is_locked(q->lock_ptr))
1529
-	    || WARN_ON(plist_node_empty(&q->list)))
1453
+	if (WARN_ON_SMP(!q->lock_ptr) || WARN_ON(plist_node_empty(&q->list)))
15301454 		return;
1455
+	lockdep_assert_held(q->lock_ptr);
15311456 
15321457 	hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
15331458 	plist_del(&q->list, &hb->chain);
....@@ -1560,10 +1485,9 @@
15601485 
15611486 	/*
15621487 	 * Queue the task for later wakeup for after we've released
1563
-	 * the hb->lock. wake_q_add() grabs reference to p.
1488
+	 * the hb->lock.
15641489 	 */
1565
-	wake_q_add(wake_q, p);
1566
-	put_task_struct(p);
1490
+	wake_q_add_safe(wake_q, p);
15671491 }
15681492 
15691493 /*
....@@ -1571,7 +1495,7 @@
15711495  */
15721496 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)
15731497 {
1574
-	u32 uninitialized_var(curval), newval;
1498
+	u32 curval, newval;
15751499 	struct task_struct *new_owner;
15761500 	bool postunlock = false;
15771501 	DEFINE_WAKE_Q(wake_q);
....@@ -1672,23 +1596,25 @@
16721596 	struct futex_q *this, *next;
16731597 	union futex_key key = FUTEX_KEY_INIT;
16741598 	int ret;
1599
+	int target_nr;
16751600 	DEFINE_WAKE_Q(wake_q);
16761601 
16771602 	if (!bitset)
16781603 		return -EINVAL;
16791604 
1680
-	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_READ);
1605
+	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
16811606 	if (unlikely(ret != 0))
1682
-		goto out;
1607
+		return ret;
16831608 
16841609 	hb = hash_futex(&key);
16851610 
16861611 	/* Make sure we really have tasks to wakeup */
16871612 	if (!hb_waiters_pending(hb))
1688
-		goto out_put_key;
1613
+		return ret;
16891614 
16901615 	spin_lock(&hb->lock);
16911616 
1617
+	trace_android_vh_futex_wake_traverse_plist(&hb->chain, &target_nr, key, bitset);
16921618 	plist_for_each_entry_safe(this, next, &hb->chain, list) {
16931619 		if (match_futex (&this->key, &key)) {
16941620 			if (this->pi_state || this->rt_waiter) {
....@@ -1700,6 +1626,7 @@
17001626 			if (!(this->bitset & bitset))
17011627 				continue;
17021628 
1629
+			trace_android_vh_futex_wake_this(ret, nr_wake, target_nr, this->task);
17031630 			mark_wake_futex(&wake_q, this);
17041631 			if (++ret >= nr_wake)
17051632 				break;
....@@ -1708,9 +1635,7 @@
17081635 
17091636 	spin_unlock(&hb->lock);
17101637 	wake_up_q(&wake_q);
1711
-out_put_key:
1712
-	put_futex_key(&key);
1713
-out:
1638
+	trace_android_vh_futex_wake_up_q_finish(nr_wake, target_nr);
17141639 	return ret;
17151640 }
17161641 
....@@ -1736,10 +1661,9 @@
17361661 		oparg = 1 << oparg;
17371662 	}
17381663 
1739
-	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
1740
-		return -EFAULT;
1741
-
1664
+	pagefault_disable();
17421665 	ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
1666
+	pagefault_enable();
17431667 	if (ret)
17441668 		return ret;
17451669 
....@@ -1776,12 +1700,12 @@
17761700 	DEFINE_WAKE_Q(wake_q);
17771701 
17781702 retry:
1779
-	ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ);
1703
+	ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
17801704 	if (unlikely(ret != 0))
1781
-		goto out;
1782
-	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
1705
+		return ret;
1706
+	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
17831707 	if (unlikely(ret != 0))
1784
-		goto out_put_key1;
1708
+		return ret;
17851709 
17861710 	hb1 = hash_futex(&key1);
17871711 	hb2 = hash_futex(&key2);
....@@ -1799,13 +1723,13 @@
17991723 			 * an MMU, but we might get them from range checking
18001724 			 */
18011725 			ret = op_ret;
1802
-			goto out_put_keys;
1726
+			return ret;
18031727 		}
18041728 
18051729 		if (op_ret == -EFAULT) {
18061730 			ret = fault_in_user_writeable(uaddr2);
18071731 			if (ret)
1808
-				goto out_put_keys;
1732
+				return ret;
18091733 		}
18101734 
18111735 		if (!(flags & FLAGS_SHARED)) {
....@@ -1813,8 +1737,6 @@
18131737 			goto retry_private;
18141738 		}
18151739 
1816
-		put_futex_key(&key2);
1817
-		put_futex_key(&key1);
18181740 		cond_resched();
18191741 		goto retry;
18201742 	}
....@@ -1850,11 +1772,6 @@
18501772 out_unlock:
18511773 	double_unlock_hb(hb1, hb2);
18521774 	wake_up_q(&wake_q);
1853
-out_put_keys:
1854
-	put_futex_key(&key2);
1855
-out_put_key1:
1856
-	put_futex_key(&key1);
1857
-out:
18581775 	return ret;
18591776 }
18601777 
....@@ -1881,7 +1798,6 @@
18811798 		plist_add(&q->list, &hb2->chain);
18821799 		q->lock_ptr = &hb2->lock;
18831800 	}
1884
-	get_futex_key_refs(key2);
18851801 	q->key = *key2;
18861802 }
18871803 
....@@ -1903,7 +1819,6 @@
19031819 void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
19041820 			   struct futex_hash_bucket *hb)
19051821 {
1906
-	get_futex_key_refs(key);
19071822 	q->key = *key;
19081823 
19091824 	__unqueue_futex(q);
....@@ -2014,7 +1929,7 @@
20141929 			 u32 *cmpval, int requeue_pi)
20151930 {
20161931 	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
2017
-	int drop_count = 0, task_count = 0, ret;
1932
+	int task_count = 0, ret;
20181933 	struct futex_pi_state *pi_state = NULL;
20191934 	struct futex_hash_bucket *hb1, *hb2;
20201935 	struct futex_q *this, *next;
....@@ -2061,22 +1976,20 @@
20611976 	}
20621977 
20631978 retry:
2064
-	ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ);
1979
+	ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
20651980 	if (unlikely(ret != 0))
2066
-		goto out;
1981
+		return ret;
20671982 	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2,
2068
-			    requeue_pi ? VERIFY_WRITE : VERIFY_READ);
1983
+			    requeue_pi ? FUTEX_WRITE : FUTEX_READ);
20691984 	if (unlikely(ret != 0))
2070
-		goto out_put_key1;
1985
+		return ret;
20711986 
20721987 	/*
20731988 	 * The check above which compares uaddrs is not sufficient for
20741989 	 * shared futexes. We need to compare the keys:
20751990 	 */
2076
-	if (requeue_pi && match_futex(&key1, &key2)) {
2077
-		ret = -EINVAL;
2078
-		goto out_put_keys;
2079
-	}
1991
+	if (requeue_pi && match_futex(&key1, &key2))
1992
+		return -EINVAL;
20801993 
20811994 	hb1 = hash_futex(&key1);
20821995 	hb2 = hash_futex(&key2);
....@@ -2096,13 +2009,11 @@
20962009 
20972010 			ret = get_user(curval, uaddr1);
20982011 			if (ret)
2099
-				goto out_put_keys;
2012
+				return ret;
21002013 
21012014 			if (!(flags & FLAGS_SHARED))
21022015 				goto retry_private;
21032016 
2104
-			put_futex_key(&key2);
2105
-			put_futex_key(&key1);
21062017 			goto retry;
21072018 		}
21082019 		if (curval != *cmpval) {
....@@ -2135,7 +2046,6 @@
21352046 		 */
21362047 		if (ret > 0) {
21372048 			WARN_ON(pi_state);
2138
-			drop_count++;
21392049 			task_count++;
21402050 			/*
21412051 			 * If we acquired the lock, then the user space value
....@@ -2162,12 +2072,10 @@
21622072 		case -EFAULT:
21632073 			double_unlock_hb(hb1, hb2);
21642074 			hb_waiters_dec(hb2);
2165
-			put_futex_key(&key2);
2166
-			put_futex_key(&key1);
21672075 			ret = fault_in_user_writeable(uaddr2);
21682076 			if (!ret)
21692077 				goto retry;
2170
-			goto out;
2078
+			return ret;
21712079 		case -EBUSY:
21722080 		case -EAGAIN:
21732081 			/*
....@@ -2178,8 +2086,6 @@
21782086 			 */
21792087 			double_unlock_hb(hb1, hb2);
21802088 			hb_waiters_dec(hb2);
2181
-			put_futex_key(&key2);
2182
-			put_futex_key(&key1);
21832089 			/*
21842090 			 * Handle the case where the owner is in the middle of
21852091 			 * exiting. Wait for the exit to complete otherwise
....@@ -2255,17 +2161,6 @@
22552161 				 * doing so.
22562162 				 */
22572163 				requeue_pi_wake_futex(this, &key2, hb2);
2258
-				drop_count++;
2259
-				continue;
2260
-			} else if (ret == -EAGAIN) {
2261
-				/*
2262
-				 * Waiter was woken by timeout or
2263
-				 * signal and has set pi_blocked_on to
2264
-				 * PI_WAKEUP_INPROGRESS before we
2265
-				 * tried to enqueue it on the rtmutex.
2266
-				 */
2267
-				this->pi_state = NULL;
2268
-				put_pi_state(pi_state);
22692164 				continue;
22702165 			} else if (ret) {
22712166 				/*
....@@ -2286,7 +2181,6 @@
22862181 			}
22872182 		}
22882183 		requeue_futex(this, hb1, hb2, &key2);
2289
-		drop_count++;
22902184 	}
22912185 
22922186 	/*
....@@ -2300,21 +2194,6 @@
23002194 	double_unlock_hb(hb1, hb2);
23012195 	wake_up_q(&wake_q);
23022196 	hb_waiters_dec(hb2);
2303
-
2304
-	/*
2305
-	 * drop_futex_key_refs() must be called outside the spinlocks. During
2306
-	 * the requeue we moved futex_q's from the hash bucket at key1 to the
2307
-	 * one at key2 and updated their key pointer.  We no longer need to
2308
-	 * hold the references to key1.
2309
-	 */
2310
-	while (--drop_count >= 0)
2311
-		drop_futex_key_refs(&key1);
2312
-
2313
-out_put_keys:
2314
-	put_futex_key(&key2);
2315
-out_put_key1:
2316
-	put_futex_key(&key1);
2317
-out:
23182197 	return ret ? ret : task_count;
23192198 }
23202199 
....@@ -2334,11 +2213,11 @@
23342213 	 * decrement the counter at queue_unlock() when some error has
23352214 	 * occurred and we don't end up adding the task to the list.
23362215 	 */
2337
-	hb_waiters_inc(hb);
2216
+	hb_waiters_inc(hb); /* implies smp_mb(); (A) */
23382217 
23392218 	q->lock_ptr = &hb->lock;
23402219 
2341
-	spin_lock(&hb->lock); /* implies smp_mb(); (A) */
2220
+	spin_lock(&hb->lock);
23422221 	return hb;
23432222 }
23442223 
....@@ -2353,6 +2232,7 @@
23532232 static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
23542233 {
23552234 	int prio;
2235
+	bool already_on_hb = false;
23562236 
23572237 	/*
23582238 	 * The priority used to register this element is
....@@ -2365,7 +2245,9 @@
23652245 	prio = min(current->normal_prio, MAX_RT_PRIO);
23662246 
23672247 	plist_node_init(&q->list, prio);
2368
-	plist_add(&q->list, &hb->chain);
2248
+	trace_android_vh_alter_futex_plist_add(&q->list, &hb->chain, &already_on_hb);
2249
+	if (!already_on_hb)
2250
+		plist_add(&q->list, &hb->chain);
23692251 	q->task = current;
23702252 }
23712253 
....@@ -2439,7 +2321,6 @@
24392321 		ret = 1;
24402322 	}
24412323 
2442
-	drop_futex_key_refs(&q->key);
24432324 	return ret;
24442325 }
24452326 
....@@ -2463,9 +2344,9 @@
24632344 static int __fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
24642345 				  struct task_struct *argowner)
24652346 {
2466
-	u32 uval, uninitialized_var(curval), newval, newtid;
24672347 	struct futex_pi_state *pi_state = q->pi_state;
24682348 	struct task_struct *oldowner, *newowner;
2349
+	u32 uval, curval, newval, newtid;
24692350 	int err = 0;
24702351 
24712352 	oldowner = pi_state->owner;
....@@ -2720,7 +2601,7 @@
27202601 
27212602 	/* Arm the timer */
27222603 	if (timeout)
2723
-		hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
2604
+		hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
27242605 
27252606 	/*
27262607 	 * If we have been removed from the hash list, then another task
....@@ -2732,8 +2613,10 @@
27322613 		 * flagged for rescheduling. Only call schedule if there
27332614 		 * is no timeout, or if it has yet to expire.
27342615 		 */
2735
-		if (!timeout || timeout->task)
2616
+		if (!timeout || timeout->task) {
2617
+			trace_android_vh_futex_sleep_start(current);
27362618 			freezable_schedule();
2619
+		}
27372620 	}
27382621 	__set_current_state(TASK_RUNNING);
27392622 }
....@@ -2780,7 +2663,7 @@
27802663 	 * while the syscall executes.
27812664 	 */
27822665 retry:
2783
-	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ);
2666
+	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
27842667 	if (unlikely(ret != 0))
27852668 		return ret;
27862669 
....@@ -2794,12 +2677,11 @@
27942677 
27952678 		ret = get_user(uval, uaddr);
27962679 		if (ret)
2797
-			goto out;
2680
+			return ret;
27982681 
27992682 		if (!(flags & FLAGS_SHARED))
28002683 			goto retry_private;
28012684 
2802
-		put_futex_key(&q->key);
28032685 		goto retry;
28042686 	}
28052687 
....@@ -2808,16 +2690,13 @@
28082690 		ret = -EWOULDBLOCK;
28092691 	}
28102692 
2811
-out:
2812
-	if (ret)
2813
-		put_futex_key(&q->key);
28142693 	return ret;
28152694 }
28162695 
28172696 static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
28182697 		      ktime_t *abs_time, u32 bitset)
28192698 {
2820
-	struct hrtimer_sleeper timeout, *to = NULL;
2699
+	struct hrtimer_sleeper timeout, *to;
28212700 	struct restart_block *restart;
28222701 	struct futex_hash_bucket *hb;
28232702 	struct futex_q q = futex_q_init;
....@@ -2826,17 +2705,10 @@
28262705 	if (!bitset)
28272706 		return -EINVAL;
28282707 	q.bitset = bitset;
2708
+	trace_android_vh_futex_wait_start(flags, bitset);
28292709 
2830
-	if (abs_time) {
2831
-		to = &timeout;
2832
-
2833
-		hrtimer_init_sleeper_on_stack(to, (flags & FLAGS_CLOCKRT) ?
2834
-					      CLOCK_REALTIME : CLOCK_MONOTONIC,
2835
-					      HRTIMER_MODE_ABS, current);
2836
-		hrtimer_set_expires_range_ns(&to->timer, *abs_time,
2837
-					     current->timer_slack_ns);
2838
-	}
2839
-
2710
+	to = futex_setup_timer(abs_time, &timeout, flags,
2711
+			       current->timer_slack_ns);
28402712 retry:
28412713 	/*
28422714 	 * Prepare to wait on uaddr. On success, holds hb lock and increments
....@@ -2883,6 +2755,7 @@
28832755 		hrtimer_cancel(&to->timer);
28842756 		destroy_hrtimer_on_stack(&to->timer);
28852757 	}
2758
+	trace_android_vh_futex_wait_end(flags, bitset);
28862759 	return ret;
28872760 }
28882761 
....@@ -2915,7 +2788,7 @@
29152788 static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
29162789 			 ktime_t *time, int trylock)
29172790 {
2918
-	struct hrtimer_sleeper timeout, *to = NULL;
2791
+	struct hrtimer_sleeper timeout, *to;
29192792 	struct task_struct *exiting = NULL;
29202793 	struct rt_mutex_waiter rt_waiter;
29212794 	struct futex_hash_bucket *hb;
....@@ -2928,15 +2801,10 @@
29282801 	if (refill_pi_state_cache())
29292802 		return -ENOMEM;
29302803 
2931
-	if (time) {
2932
-		to = &timeout;
2933
-		hrtimer_init_sleeper_on_stack(to, CLOCK_REALTIME,
2934
-					      HRTIMER_MODE_ABS, current);
2935
-		hrtimer_set_expires(&to->timer, *time);
2936
-	}
2804
+	to = futex_setup_timer(time, &timeout, FLAGS_CLOCKRT, 0);
29372805 
29382806 retry:
2939
-	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, VERIFY_WRITE);
2807
+	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE);
29402808 	if (unlikely(ret != 0))
29412809 		goto out;
29422810 
....@@ -2966,7 +2834,6 @@
29662834 			 * - EAGAIN: The user space value changed.
29672835 			 */
29682836 			queue_unlock(hb);
2969
-			put_futex_key(&q.key);
29702837 			/*
29712838 			 * Handle the case where the owner is in the middle of
29722839 			 * exiting. Wait for the exit to complete otherwise
....@@ -3010,14 +2877,6 @@
30102877 	 * before __rt_mutex_start_proxy_lock() is done.
30112878 	 */
30122879 	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
3013
-	/*
3014
-	 * the migrate_disable() here disables migration in the in_atomic() fast
3015
-	 * path which is enabled again in the following spin_unlock(). We have
3016
-	 * one migrate_disable() pending in the slow-path which is reversed
3017
-	 * after the raw_spin_unlock_irq() where we leave the atomic context.
3018
-	 */
3019
-	migrate_disable();
3020
-
30212880 	spin_unlock(q.lock_ptr);
30222881 	/*
30232882 	 * __rt_mutex_start_proxy_lock() unconditionally enqueues the @rt_waiter
....@@ -3026,7 +2885,6 @@
30262885 	 */
30272886 	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);
30282887 	raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);
3029
-	migrate_enable();
30302888 
30312889 	if (ret) {
30322890 		if (ret == 1)
....@@ -3035,7 +2893,7 @@
30352893 	}
30362894 
30372895 	if (unlikely(to))
3038
-		hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
2896
+		hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
30392897 
30402898 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
30412899 
....@@ -3068,14 +2926,11 @@
30682926 
30692927 	/* Unqueue and drop the lock */
30702928 	unqueue_me_pi(&q);
3071
-
3072
-	goto out_put_key;
2929
+	goto out;
30732930 
30742931 out_unlock_put_key:
30752932 	queue_unlock(hb);
30762933 
3077
-out_put_key:
3078
-	put_futex_key(&q.key);
30792934 out:
30802935 	if (to) {
30812936 		hrtimer_cancel(&to->timer);
....@@ -3088,12 +2943,11 @@
30882943 
30892944 	ret = fault_in_user_writeable(uaddr);
30902945 	if (ret)
3091
-		goto out_put_key;
2946
+		goto out;
30922947 
30932948 	if (!(flags & FLAGS_SHARED))
30942949 		goto retry_private;
30952950 
3096
-	put_futex_key(&q.key);
30972951 	goto retry;
30982952 }
30992953 
....@@ -3104,7 +2958,7 @@
31042958  */
31052959 static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
31062960 {
3107
-	u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
2961
+	u32 curval, uval, vpid = task_pid_vnr(current);
31082962 	union futex_key key = FUTEX_KEY_INIT;
31092963 	struct futex_hash_bucket *hb;
31102964 	struct futex_q *top_waiter;
....@@ -3122,7 +2976,7 @@
31222976 	if ((uval & FUTEX_TID_MASK) != vpid)
31232977 		return -EPERM;
31242978 
3125
-	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE);
2979
+	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE);
31262980 	if (ret)
31272981 		return ret;
31282982 
....@@ -3161,19 +3015,10 @@
31613015 		 * rt_waiter. Also see the WARN in wake_futex_pi().
31623016 		 */
31633017 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
3164
-		/*
3165
-		 * Magic trickery for now to make the RT migrate disable
3166
-		 * logic happy. The following spin_unlock() happens with
3167
-		 * interrupts disabled so the internal migrate_enable()
3168
-		 * won't undo the migrate_disable() which was issued when
3169
-		 * locking hb->lock.
3170
-		 */
3171
-		migrate_disable();
31723018 		spin_unlock(&hb->lock);
31733019 
31743020 		/* drops pi_state->pi_mutex.wait_lock */
31753021 		ret = wake_futex_pi(uaddr, uval, pi_state);
3176
-		migrate_enable();
31773022 
31783023 		put_pi_state(pi_state);
31793024 
....@@ -3231,16 +3076,13 @@
32313076 out_unlock:
32323077 	spin_unlock(&hb->lock);
32333078 out_putkey:
3234
-	put_futex_key(&key);
32353079 	return ret;
32363080 
32373081 pi_retry:
3238
-	put_futex_key(&key);
32393082 	cond_resched();
32403083 	goto retry;
32413084 
32423085 pi_faulted:
3243
-	put_futex_key(&key);
32443086 
32453087 	ret = fault_in_user_writeable(uaddr);
32463088 	if (!ret)
....@@ -3342,9 +3184,9 @@
33423184 				 u32 val, ktime_t *abs_time, u32 bitset,
33433185 				 u32 __user *uaddr2)
33443186 {
3345
-	struct hrtimer_sleeper timeout, *to = NULL;
3187
+	struct hrtimer_sleeper timeout, *to;
33463188 	struct rt_mutex_waiter rt_waiter;
3347
-	struct futex_hash_bucket *hb, *hb2;
3189
+	struct futex_hash_bucket *hb;
33483190 	union futex_key key2 = FUTEX_KEY_INIT;
33493191 	struct futex_q q = futex_q_init;
33503192 	int res, ret;
....@@ -3358,14 +3200,8 @@
33583200 	if (!bitset)
33593201 		return -EINVAL;
33603202 
3361
-	if (abs_time) {
3362
-		to = &timeout;
3363
-		hrtimer_init_sleeper_on_stack(to, (flags & FLAGS_CLOCKRT) ?
3364
-					      CLOCK_REALTIME : CLOCK_MONOTONIC,
3365
-					      HRTIMER_MODE_ABS, current);
3366
-		hrtimer_set_expires_range_ns(&to->timer, *abs_time,
3367
-					     current->timer_slack_ns);
3368
-	}
3203
+	to = futex_setup_timer(abs_time, &timeout, flags,
3204
+			       current->timer_slack_ns);
33693205 
33703206 	/*
33713207 	 * The waiter is allocated on our stack, manipulated by the requeue
....@@ -3373,7 +3209,7 @@
33733209 	 */
33743210 	rt_mutex_init_waiter(&rt_waiter, false);
33753211 
3376
-	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
3212
+	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
33773213 	if (unlikely(ret != 0))
33783214 		goto out;
33793215 
....@@ -3387,7 +3223,7 @@
33873223 	 */
33883224 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
33893225 	if (ret)
3390
-		goto out_key2;
3226
+		goto out;
33913227 
33923228 	/*
33933229 	 * The check above which compares uaddrs is not sufficient for
....@@ -3396,61 +3232,26 @@
33963232 	if (match_futex(&q.key, &key2)) {
33973233 		queue_unlock(hb);
33983234 		ret = -EINVAL;
3399
-		goto out_put_keys;
3235
+		goto out;
34003236 	}
34013237 
34023238 	/* Queue the futex_q, drop the hb lock, wait for wakeup. */
34033239 	futex_wait_queue_me(hb, &q, to);
34043240 
3405
-	/*
3406
-	 * On RT we must avoid races with requeue and trying to block
3407
-	 * on two mutexes (hb->lock and uaddr2's rtmutex) by
3408
-	 * serializing access to pi_blocked_on with pi_lock.
3409
-	 */
3410
-	raw_spin_lock_irq(&current->pi_lock);
3411
-	if (current->pi_blocked_on) {
3412
-		/*
3413
-		 * We have been requeued or are in the process of
3414
-		 * being requeued.
3415
-		 */
3416
-		raw_spin_unlock_irq(&current->pi_lock);
3417
-	} else {
3418
-		/*
3419
-		 * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS
3420
-		 * prevents a concurrent requeue from moving us to the
3421
-		 * uaddr2 rtmutex. After that we can safely acquire
3422
-		 * (and possibly block on) hb->lock.
3423
-		 */
3424
-		current->pi_blocked_on = PI_WAKEUP_INPROGRESS;
3425
-		raw_spin_unlock_irq(&current->pi_lock);
3426
-
3427
-		spin_lock(&hb->lock);
3428
-
3429
-		/*
3430
-		 * Clean up pi_blocked_on. We might leak it otherwise
3431
-		 * when we succeeded with the hb->lock in the fast
3432
-		 * path.
3433
-		 */
3434
-		raw_spin_lock_irq(&current->pi_lock);
3435
-		current->pi_blocked_on = NULL;
3436
-		raw_spin_unlock_irq(&current->pi_lock);
3437
-
3438
-		ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
3439
-		spin_unlock(&hb->lock);
3440
-		if (ret)
3441
-			goto out_put_keys;
3442
-	}
3241
+	spin_lock(&hb->lock);
3242
+	ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
3243
+	spin_unlock(&hb->lock);
3244
+	if (ret)
3245
+		goto out;
34433246 
34443247 	/*
3445
-	 * In order to be here, we have either been requeued, are in
3446
-	 * the process of being requeued, or requeue successfully
3447
-	 * acquired uaddr2 on our behalf.  If pi_blocked_on was
3448
-	 * non-null above, we may be racing with a requeue.  Do not
3449
-	 * rely on q->lock_ptr to be hb2->lock until after blocking on
3450
-	 * hb->lock or hb2->lock. The futex_requeue dropped our key1
3451
-	 * reference and incremented our key2 reference count.
3248
+	 * In order for us to be here, we know our q.key == key2, and since
3249
+	 * we took the hb->lock above, we also know that futex_requeue() has
3250
+	 * completed and we no longer have to concern ourselves with a wakeup
3251
+	 * race with the atomic proxy lock acquisition by the requeue code. The
3252
+	 * futex_requeue dropped our key1 reference and incremented our key2
3253
+	 * reference count.
34523254 	 */
3453
-	hb2 = hash_futex(&key2);
34543255 
34553256 	/* Check if the requeue code acquired the second futex for us. */
34563257 	if (!q.rt_waiter) {
....@@ -3459,15 +3260,14 @@
34593260 		 * did a lock-steal - fix up the PI-state in that case.
34603261 		 */
34613262 		if (q.pi_state && (q.pi_state->owner != current)) {
3462
-			spin_lock(&hb2->lock);
3463
-			BUG_ON(&hb2->lock != q.lock_ptr);
3263
+			spin_lock(q.lock_ptr);
34643264 			ret = fixup_pi_state_owner(uaddr2, &q, current);
34653265 			/*
34663266 			 * Drop the reference to the pi state which
34673267 			 * the requeue_pi() code acquired for us.
34683268 			 */
34693269 			put_pi_state(q.pi_state);
3470
-			spin_unlock(&hb2->lock);
3270
+			spin_unlock(q.lock_ptr);
34713271 			/*
34723272 			 * Adjust the return value. It's either -EFAULT or
34733273 			 * success (1) but the caller expects 0 for success.
....@@ -3486,8 +3286,7 @@
34863286 		pi_mutex = &q.pi_state->pi_mutex;
34873287 		ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
34883288 
3489
-		spin_lock(&hb2->lock);
3490
-		BUG_ON(&hb2->lock != q.lock_ptr);
3289
+		spin_lock(q.lock_ptr);
34913290 		if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
34923291 			ret = 0;
34933292 
....@@ -3518,11 +3317,6 @@
35183317 		 */
35193318 		ret = -EWOULDBLOCK;
35203319 	}
3521
-
3522
-out_put_keys:
3523
-	put_futex_key(&q.key);
3524
-out_key2:
3525
-	put_futex_key(&key2);
35263320 
35273321 out:
35283322 	if (to) {
....@@ -3624,7 +3418,7 @@
36243418 static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
36253419 			      bool pi, bool pending_op)
36263420 {
3627
-	u32 uval, uninitialized_var(nval), mval;
3421
+	u32 uval, nval, mval;
36283422 	int err;
36293423 
36303424 	/* Futex address must be 32bit aligned */
....@@ -3754,7 +3548,7 @@
37543548 	struct robust_list_head __user *head = curr->robust_list;
37553549 	struct robust_list __user *entry, *next_entry, *pending;
37563550 	unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
3757
-	unsigned int uninitialized_var(next_pi);
3551
+	unsigned int next_pi;
37583552 	unsigned long futex_offset;
37593553 	int rc;
37603554 
....@@ -3947,15 +3741,16 @@
39473741 			return -ENOSYS;
39483742 	}
39493743 
3744
+	trace_android_vh_do_futex(cmd, &flags, uaddr2);
39503745 	switch (cmd) {
39513746 	case FUTEX_WAIT:
39523747 		val3 = FUTEX_BITSET_MATCH_ANY;
3953
-		/* fall through */
3748
+		fallthrough;
39543749 	case FUTEX_WAIT_BITSET:
39553750 		return futex_wait(uaddr, flags, val, timeout, val3);
39563751 	case FUTEX_WAKE:
39573752 		val3 = FUTEX_BITSET_MATCH_ANY;
3958
-		/* fall through */
3753
+		fallthrough;
39593754 	case FUTEX_WAKE_BITSET:
39603755 		return futex_wake(uaddr, flags, val, val3);
39613756 	case FUTEX_REQUEUE:
....@@ -3982,10 +3777,10 @@
39823777 
39833778 
39843779 SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
3985
-		struct timespec __user *, utime, u32 __user *, uaddr2,
3780
+		struct __kernel_timespec __user *, utime, u32 __user *, uaddr2,
39863781 		u32, val3)
39873782 {
3988
-	struct timespec ts;
3783
+	struct timespec64 ts;
39893784 	ktime_t t, *tp = NULL;
39903785 	u32 val2 = 0;
39913786 	int cmd = op & FUTEX_CMD_MASK;
....@@ -3995,14 +3790,16 @@
39953790 		      cmd == FUTEX_WAIT_REQUEUE_PI)) {
39963791 		if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
39973792 			return -EFAULT;
3998
-		if (copy_from_user(&ts, utime, sizeof(ts)) != 0)
3793
+		if (get_timespec64(&ts, utime))
39993794 			return -EFAULT;
4000
-		if (!timespec_valid(&ts))
3795
+		if (!timespec64_valid(&ts))
40013796 			return -EINVAL;
40023797 
4003
-		t = timespec_to_ktime(ts);
3798
+		t = timespec64_to_ktime(ts);
40043799 		if (cmd == FUTEX_WAIT)
40053800 			t = ktime_add_safe(ktime_get(), t);
3801
+		else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))
3802
+			t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
40063803 		tp = &t;
40073804 	}
40083805 	/*
....@@ -4053,7 +3850,7 @@
40533850 	struct compat_robust_list_head __user *head = curr->compat_robust_list;
40543851 	struct robust_list __user *entry, *next_entry, *pending;
40553852 	unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
4056
-	unsigned int uninitialized_var(next_pi);
3853
+	unsigned int next_pi;
40573854 	compat_uptr_t uentry, next_uentry, upending;
40583855 	compat_long_t futex_offset;
40593856 	int rc;
....@@ -4172,12 +3969,14 @@
41723969 
41733970 	return ret;
41743971 }
3972
+#endif /* CONFIG_COMPAT */
41753973 
4176
-COMPAT_SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
3974
+#ifdef CONFIG_COMPAT_32BIT_TIME
3975
+SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
41773976 		struct old_timespec32 __user *, utime, u32 __user *, uaddr2,
41783977 		u32, val3)
41793978 {
4180
-	struct timespec ts;
3979
+	struct timespec64 ts;
41813980 	ktime_t t, *tp = NULL;
41823981 	int val2 = 0;
41833982 	int cmd = op & FUTEX_CMD_MASK;
....@@ -4185,14 +3984,16 @@
41853984 	if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
41863985 		      cmd == FUTEX_WAIT_BITSET ||
41873986 		      cmd == FUTEX_WAIT_REQUEUE_PI)) {
4188
-		if (compat_get_timespec(&ts, utime))
3987
+		if (get_old_timespec32(&ts, utime))
41893988 			return -EFAULT;
4190
-		if (!timespec_valid(&ts))
3989
+		if (!timespec64_valid(&ts))
41913990 			return -EINVAL;
41923991 
4193
-		t = timespec_to_ktime(ts);
3992
+		t = timespec64_to_ktime(ts);
41943993 		if (cmd == FUTEX_WAIT)
41953994 			t = ktime_add_safe(ktime_get(), t);
3995
+		else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))
3996
+			t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
41963997 		tp = &t;
41973998 	}
41983999 	if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
....@@ -4201,7 +4002,7 @@
42014002 
42024003 	return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
42034004 }
4204
-#endif /* CONFIG_COMPAT */
4005
+#endif /* CONFIG_COMPAT_32BIT_TIME */
42054006 
42064007 static void __init futex_detect_cmpxchg(void)
42074008 {