...
 
Commits (3)
......@@ -21,6 +21,7 @@
#define FUTEX_WAKE_BITSET 10
#define FUTEX_WAIT_REQUEUE_PI 11
#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_WAIT_MULTIPLE 13
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
......@@ -150,4 +151,10 @@ struct robust_list_head {
(((op & 0xf) << 28) | ((cmp & 0xf) << 24) \
| ((oparg & 0xfff) << 12) | (cmparg & 0xfff))
struct futex_wait_block {
__u32 __user *uaddr;
__u32 val;
__u32 bitset;
};
#endif /* _UAPI_LINUX_FUTEX_H */
......@@ -183,6 +183,7 @@ static int __read_mostly futex_cmpxchg_enabled;
#endif
#define FLAGS_CLOCKRT 0x02
#define FLAGS_HAS_TIMEOUT 0x04
#define FLAGS_WAKE_MULTIPLE 0x08
/*
* Priority Inheritance state:
......@@ -2631,6 +2632,39 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
__set_current_state(TASK_RUNNING);
}
static int __futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
struct futex_q *q, struct futex_hash_bucket **hb)
{
u32 uval;
int ret;
retry_private:
*hb = queue_lock(q);
ret = get_futex_value_locked(&uval, uaddr);
if (ret) {
queue_unlock(*hb);
ret = get_user(uval, uaddr);
if (ret)
return ret;
if (!(flags & FLAGS_SHARED))
goto retry_private;
return 1;
}
if (uval != val) {
queue_unlock(*hb);
ret = -EWOULDBLOCK;
}
return ret;
}
/**
* futex_wait_setup() - Prepare to wait on a futex
* @uaddr: the futex userspace address
......@@ -2651,7 +2685,6 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
struct futex_q *q, struct futex_hash_bucket **hb)
{
u32 uval;
int ret;
/*
......@@ -2672,74 +2705,102 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
* absorb a wakeup if *uaddr does not match the desired values
* while the syscall executes.
*/
retry:
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
if (unlikely(ret != 0))
return ret;
do {
ret = get_futex_key(uaddr, flags & FLAGS_SHARED,
&q->key, FUTEX_READ);
if (unlikely(ret != 0))
return ret;
retry_private:
*hb = queue_lock(q);
ret = get_futex_value_locked(&uval, uaddr);
ret = __futex_wait_setup(uaddr, val, flags, q, hb);
if (ret) {
queue_unlock(*hb);
ret = get_user(uval, uaddr);
/* Drop key reference if retry or error. */
if (ret)
goto out;
if (!(flags & FLAGS_SHARED))
goto retry_private;
put_futex_key(&q->key);
goto retry;
}
if (uval != val) {
queue_unlock(*hb);
ret = -EWOULDBLOCK;
}
put_futex_key(&q->key);
} while (ret > 0);
out:
if (ret)
put_futex_key(&q->key);
return ret;
}
static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
ktime_t *abs_time, u32 bitset)
static int do_futex_wait_multiple(struct futex_wait_block *wb,
u32 count, unsigned int flags,
ktime_t *abs_time)
{
struct hrtimer_sleeper timeout, *to;
struct restart_block *restart;
struct futex_hash_bucket *hb;
struct futex_q q = futex_q_init;
struct futex_q *qs = NULL;
int ret;
int i;
if (!bitset)
return -EINVAL;
q.bitset = bitset;
qs = kcalloc(count, sizeof(struct futex_q), GFP_KERNEL);
if (!qs)
return -ENOMEM;
to = futex_setup_timer(abs_time, &timeout, flags,
current->timer_slack_ns);
retry:
/*
* Prepare to wait on uaddr. On success, holds hb lock and increments
* q.key refs.
retry:
for (i = 0; i < count; i++) {
qs[i].key = FUTEX_KEY_INIT;
qs[i].bitset = wb[i].bitset;
ret = get_futex_key(wb[i].uaddr, flags & FLAGS_SHARED,
&qs[i].key, FUTEX_READ);
if (unlikely(ret != 0)) {
for (--i; i >= 0; i--)
put_futex_key(&qs[i].key);
goto out;
}
}
set_current_state(TASK_INTERRUPTIBLE);
for (i = 0; i < count; i++) {
ret = __futex_wait_setup(wb[i].uaddr, wb[i].val,
flags, &qs[i], &hb);
if (ret) {
/* Drop the failed key directly. keys 0..(i-1)
* will be put by unqueue_me. */
put_futex_key(&qs[i].key);
/* Undo the partial work we did. */
for (--i; i >= 0; i--)
unqueue_me(&qs[i]);
__set_current_state(TASK_RUNNING);
if (ret > 0)
goto retry;
goto out;
}
/* We can't hold to the bucket lock when dealing with
* the next futex. Queue ourselves now so we can unlock
* it before moving on. */
queue_me(&qs[i], hb);
}
if (to)
hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
/* There is no easy to way to check if we are wake already on
* multiple futexes without waking through each one of them. So
* just sleep and let the scheduler handle it.
*/
ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
if (ret)
goto out;
if (!to || to->task)
freezable_schedule();
/* queue_me and wait for wakeup, timeout, or a signal. */
futex_wait_queue_me(hb, &q, to);
__set_current_state(TASK_RUNNING);
/* If we were woken (and unqueued), we succeeded, whatever. */
ret = 0;
/* unqueue_me() drops q.key ref */
if (!unqueue_me(&q))
goto out;
ret = -ETIMEDOUT;
/* If we were woken (and unqueued), we succeeded. */
for (i = 0; i < count; i++)
if (!unqueue_me(&qs[i]))
ret = i;
/* Succeed wakeup */
if (ret >= 0)
goto out;
/* Woken by triggered timeout */
if (to && !to->task)
goto out;
......@@ -2754,21 +2815,79 @@ static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
if (!abs_time)
goto out;
restart = &current->restart_block;
restart->fn = futex_wait_restart;
restart->futex.uaddr = uaddr;
restart->futex.val = val;
restart->futex.time = *abs_time;
restart->futex.bitset = bitset;
restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
ret = -ERESTART_RESTARTBLOCK;
out:
out:
if (to) {
hrtimer_cancel(&to->timer);
destroy_hrtimer_on_stack(&to->timer);
}
kfree(qs);
return ret;
}
static int futex_wait_multiple(u32 __user *uaddr, unsigned int flags,
u32 count, ktime_t *abs_time)
{
struct futex_wait_block *wb;
struct restart_block *restart;
int ret;
if (!count)
return -EINVAL;
wb = kcalloc(count, sizeof(struct futex_wait_block), GFP_KERNEL);
if (!wb)
return -ENOMEM;
if (copy_from_user(wb, uaddr,
count * sizeof(struct futex_wait_block))) {
ret = -EFAULT;
goto out;
}
ret = do_futex_wait_multiple(wb, count, flags, abs_time);
if (ret == -ERESTART_RESTARTBLOCK) {
restart = &current->restart_block;
restart->fn = futex_wait_restart;
restart->futex.uaddr = uaddr;
restart->futex.val = count;
restart->futex.time = *abs_time;
restart->futex.flags = (flags | FLAGS_HAS_TIMEOUT |
FLAGS_WAKE_MULTIPLE);
}
out:
kfree(wb);
return ret;
}
static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
ktime_t *abs_time, u32 bitset)
{
struct futex_wait_block wb;
struct restart_block *restart;
int ret;
if (!bitset)
return -EINVAL;
wb.uaddr = uaddr;
wb.val = val;
wb.bitset = bitset;
ret = do_futex_wait_multiple(&wb, 1, flags, abs_time);
if (ret == -ERESTART_RESTARTBLOCK) {
restart = &current->restart_block;
restart->fn = futex_wait_restart;
restart->futex.uaddr = uaddr;
restart->futex.val = val;
restart->futex.time = *abs_time;
restart->futex.bitset = bitset;
restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
}
return ret;
}
......@@ -2784,6 +2903,10 @@ static long futex_wait_restart(struct restart_block *restart)
}
restart->fn = do_no_restart_syscall;
if (restart->futex.flags & FLAGS_WAKE_MULTIPLE)
return (long)futex_wait_multiple(uaddr, restart->futex.flags,
restart->futex.val, tp);
return (long)futex_wait(uaddr, restart->futex.flags,
restart->futex.val, tp, restart->futex.bitset);
}
......@@ -3667,6 +3790,8 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
uaddr2);
case FUTEX_CMP_REQUEUE_PI:
return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
case FUTEX_WAIT_MULTIPLE:
return futex_wait_multiple(uaddr, flags, val, timeout);
}
return -ENOSYS;
}
......@@ -3683,7 +3808,8 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
cmd == FUTEX_WAIT_REQUEUE_PI ||
cmd == FUTEX_WAIT_MULTIPLE)) {
if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
return -EFAULT;
if (get_timespec64(&ts, utime))
......@@ -3692,7 +3818,7 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
return -EINVAL;
t = timespec64_to_ktime(ts);
if (cmd == FUTEX_WAIT)
if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE)
t = ktime_add_safe(ktime_get(), t);
tp = &t;
}
......@@ -3876,14 +4002,15 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
cmd == FUTEX_WAIT_REQUEUE_PI ||
cmd == FUTEX_WAIT_MULTIPLE)) {
if (get_old_timespec32(&ts, utime))
return -EFAULT;
if (!timespec64_valid(&ts))
return -EINVAL;
t = timespec64_to_ktime(ts);
if (cmd == FUTEX_WAIT)
if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE)
t = ktime_add_safe(ktime_get(), t);
tp = &t;
}
......