upstream_fixes

parent 2ae0e721
......@@ -151,6 +151,8 @@ struct robust_list_head {
(((op & 0xf) << 28) | ((cmp & 0xf) << 24) \
| ((oparg & 0xfff) << 12) | (cmparg & 0xfff))
#define FUTEX_MULTIPLE_MAX_COUNT 128
struct futex_wait_block {
__u32 __user *uaddr;
__u32 val;
......
......@@ -238,6 +238,8 @@ struct futex_q {
struct rt_mutex_waiter *rt_waiter;
union futex_key *requeue_pi_key;
u32 bitset;
u32 __user *uaddr;
u32 uval;
} __randomize_layout;
static const struct futex_q futex_q_init = {
......@@ -2345,6 +2347,29 @@ static int unqueue_me(struct futex_q *q)
return ret;
}
/**
* unqueue_multiple() - Remove several futexes from their futex_hash_bucket
* @q: The list of futexes to unqueue
* @count: Number of futexes in the list
*
* Helper to unqueue a list of futexes. This can't fail.
*
* Return:
* - >=0 - Index in the of the last futex that was awoken;
* - -1 - if no futex was awoken
*/
static int unqueue_multiple(struct futex_q *q, int count)
{
int ret = -1;
int i;
for (i = 0; i < count; i++) {
if (!unqueue_me(&q[i]))
ret = i;
}
return ret;
}
/*
* PI futexes can not be requeued and must remove themself from the
* hash bucket. The hash bucket lock (i.e. lock_ptr) is held on entry
......@@ -2721,147 +2746,174 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
return ret;
}
static int do_futex_wait_multiple(struct futex_wait_block *wb,
u32 count, unsigned int flags,
ktime_t *abs_time)
/**
* futex_setup_queue_multiple() - Prepare to wait on and enqueue multiple futexes
* @wb: The userspace list of objects to wait on
* @qs: The corresponding futex list
* @count: The size of the lists
* @flags futex flags (FLAGS_SHARED, etc.)
*
* This is a helper to enqueue multiple futex objects in a single step.
* Enqueing multiple futexes are tricky, because we need to enqueue each
* futex in the list before dealing with the next one to avoid
* deadlocking on the hash bucket. In addition, before enqueing, we
* need to make sure that current->state is TASK_INTERRUPTIBLE, but this
* cannot be done before the get_futex_key of the next key, because it
* calls get_user_pages, which can sleep. Thus, we fetch the list of
* futexes keys in two steps, by first pinning all the memory keys in
* the futex key, and only then we read each the key and queue the
* correspoding futex.
*
* Return:
* - 1 - One of the futexes was awaken by another thread
* - 0 - Success
* - <1 - -EFAULT or -EWOULDBLOCK
*/
static int futex_setup_queue_multiple(struct futex_q *qs, int count,
unsigned int flags, int *awaken)
{
struct hrtimer_sleeper timeout, *to;
struct futex_hash_bucket *hb;
struct futex_q *qs = NULL;
int ret;
int i;
int ret, i;
qs = kcalloc(count, sizeof(struct futex_q), GFP_KERNEL);
if (!qs)
return -ENOMEM;
to = futex_setup_timer(abs_time, &timeout, flags,
current->timer_slack_ns);
retry:
retry:
for (i = 0; i < count; i++) {
qs[i].key = FUTEX_KEY_INIT;
qs[i].bitset = wb[i].bitset;
ret = get_futex_key(wb[i].uaddr, flags & FLAGS_SHARED,
ret = get_futex_key(qs[i].uaddr, flags & FLAGS_SHARED,
&qs[i].key, FUTEX_READ);
if (unlikely(ret != 0)) {
if (unlikely(ret)) {
for (--i; i >= 0; i--)
put_futex_key(&qs[i].key);
goto out;
return ret;
}
}
set_current_state(TASK_INTERRUPTIBLE);
for (i = 0; i < count; i++) {
ret = __futex_wait_setup(wb[i].uaddr, wb[i].val,
flags, &qs[i], &hb);
ret = __futex_wait_setup(qs[i].uaddr, qs[i].uval,
flags, &qs[i], &hb);
if (ret) {
/* Drop the failed key directly. keys 0..(i-1)
* will be put by unqueue_me.
/*
* Keys 0..(i-1) are implicitly put by
* unqueue_me.
*/
put_futex_key(&qs[i].key);
/* Undo the partial work we did. */
for (--i; i >= 0; i--)
unqueue_me(&qs[i]);
/*
* If something was already awaken, ignore the
* futex_wait_setup error and succeed the
* syscall.
*/
*awaken = unqueue_multiple(qs, i);
__set_current_state(TASK_RUNNING);
if (*awaken >= 0)
return 1;
if (ret > 0)
goto retry;
goto out;
return ret;
}
/* We can't hold to the bucket lock when dealing with
* the next futex. Queue ourselves now so we can unlock
/*
* This bucket lock can't be held while dealing with
* the next futex. Queue this futex now so we can unlock
* it before moving on.
*/
queue_me(&qs[i], hb);
}
if (to)
hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
/* There is no easy to way to check if we are wake already on
* multiple futexes without waking through each one of them. So
* just sleep and let the scheduler handle it.
*/
if (!to || to->task)
freezable_schedule();
__set_current_state(TASK_RUNNING);
ret = -ETIMEDOUT;
/* If we were woken (and unqueued), we succeeded. */
for (i = 0; i < count; i++)
if (!unqueue_me(&qs[i]))
ret = i;
/* Succeed wakeup */
if (ret >= 0)
goto out;
/* Woken by triggered timeout */
if (to && !to->task)
goto out;
/*
* We expect signal_pending(current), but we might be the
* victim of a spurious wakeup as well.
*/
if (!signal_pending(current))
goto retry;
ret = -ERESTARTSYS;
if (!abs_time)
goto out;
ret = -ERESTART_RESTARTBLOCK;
out:
if (to) {
hrtimer_cancel(&to->timer);
destroy_hrtimer_on_stack(&to->timer);
}
kfree(qs);
return ret;
return 0;
}
/**
* futex_wait_multiple() - Prepare to wait on and enqueue several futexes
* @uaddr: The userspace list of objects to wait on
* @flags: Futex flags (FLAGS_SHARED, etc.)
* @count: The number of objects
* @abs_time: Timeout before giving up and returning to userspace
*
* Entry point for the FUTEX_WAIT_MULTIPLE futex operation, this fuction
* sleeps on a group of futexes and returns on the first futex that
* triggered, or after the timeout has elapsed.
*
* Return:
* - >=0 - Hint to the futex that was awoken
* - <0 - On error
*/
static int futex_wait_multiple(u32 __user *uaddr, unsigned int flags,
u32 count, ktime_t *abs_time)
{
struct futex_wait_block *wb;
struct restart_block *restart;
int ret;
struct hrtimer_sleeper timeout, *to;
struct futex_q *qs = NULL;
int ret, hint = 0, i;
struct futex_wait_block entry, *fwb;
if (!count)
if (!count || count > FUTEX_MULTIPLE_MAX_COUNT)
return -EINVAL;
wb = kcalloc(count, sizeof(struct futex_wait_block), GFP_KERNEL);
if (!wb)
qs = kcalloc(count, sizeof(struct futex_q), GFP_KERNEL);
if (!qs)
return -ENOMEM;
if (copy_from_user(wb, uaddr,
count * sizeof(struct futex_wait_block))) {
ret = -EFAULT;
goto out;
to = futex_setup_timer(abs_time, &timeout, flags,
current->timer_slack_ns);
fwb = (struct futex_wait_block *) __user uaddr;
for (i = 0; i < count; i++) {
if(copy_from_user(&entry, &fwb[i], sizeof(entry)))
return -EFAULT;
qs[i].uval = entry.val;
qs[i].uaddr = entry.uaddr;
qs[i].bitset = entry.bitset;
}
ret = do_futex_wait_multiple(wb, count, flags, abs_time);
do {
ret = futex_setup_queue_multiple(qs, count, flags, &hint);
if (ret) {
/* A futex was awaken during setup */
if (ret > 0)
ret = hint;
break;
}
if (to)
hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
/*
* Avoid sleeping if another thread already tried to
* wake us.
*/
for (i = 0; i < count; i++) {
if (plist_node_empty(&qs[i].list))
break;
}
if (i == count && (!to || to->task))
freezable_schedule();
__set_current_state(TASK_RUNNING);
ret = unqueue_multiple(qs, count);
if (ret < 0) {
if (to && !to->task) {
ret = -ETIMEDOUT;
break;
} else if (signal_pending(current)) {
ret = -ERESTARTSYS;
break;
}
/*
* The final case is a spurious wakeup, for
* which we can just retry.
*/
}
} while (ret < 0);
if (ret == -ERESTART_RESTARTBLOCK) {
restart = &current->restart_block;
restart->fn = futex_wait_restart;
restart->futex.uaddr = uaddr;
restart->futex.val = count;
restart->futex.time = *abs_time;
restart->futex.flags = (flags | FLAGS_HAS_TIMEOUT |
FLAGS_WAKE_MULTIPLE);
if (to) {
hrtimer_cancel(&to->timer);
destroy_hrtimer_on_stack(&to->timer);
}
out:
kfree(wb);
kfree(qs);
return ret;
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment