...
 
Commits (4)
  • Gabriel Krisman Bertazi's avatar
    futex: Implement mechanism to wait on any of several futexes · f2f386e4
    Gabriel Krisman Bertazi authored
    This is a new futex operation, called FUTEX_WAIT_MULTIPLE, which allows
    a thread to wait on several futexes at the same time, and be awoken by
    any of them.  In a sense, it implements one of the features that was
    supported by pooling on the old FUTEX_FD interface.
    
    My use case for this operation lies in Wine, where we want to implement
    a similar interface available in Windows, used mainly for event
    handling.  The wine folks have an implementation that uses eventfd, but
    it suffers from FD exhaustion (I was told they have application that go
    to the order of multi-milion FDs), and higher CPU utilization.
    
    In time, we are also proposing modifications to glibc and libpthread to
    make this feature available for Linux native multithreaded applications
    using libpthread, which can benefit from the behavior of waiting on any
    of a group of futexes.
    
    In particular, using futexes in our Wine use case reduced the CPU
    utilization by 4% for the game Beat Saber and by 1.5% for the game
    Shadow of Tomb Raider, both running over Proton (a wine based solution
    for Windows emulation), when compared to the eventfd interface. This
    implementation also doesn't rely of file descriptors, so it doesn't risk
    overflowing the resource.
    
    Technically, the existing FUTEX_WAIT implementation can be easily
    reworked by using do_futex_wait_multiple with a count of one, and I
    have a patch showing how it works.  I'm not proposing it, since
    futex is such a tricky code, that I'd be more confortable to have
    FUTEX_WAIT_MULTIPLE running upstream for a couple development cycles,
    before considering modifying FUTEX_WAIT.
    
    From an implementation perspective, the futex list is passed as an array
    of (pointer,value,bitset) to the kernel, which will enqueue all of them
    and sleep if none was already triggered. It returns a hint of which
    futex caused the wake up event to userspace, but the hint doesn't
    guarantee that is the only futex triggered.  Before calling the syscall
    again, userspace should traverse the list, trying to re-acquire any of
    the other futexes, to prevent an immediate -EWOULDBLOCK return code from
    the kernel.
    
    This was tested using three mechanisms:
    
    1) By reimplementing FUTEX_WAIT in terms of FUTEX_WAIT_MULTIPLE and
    running the unmodified tools/testing/selftests/futex and a full linux
    distro on top of this kernel.
    
    2) By an example code that exercises the FUTEX_WAIT_MULTIPLE path on a
    multi-threaded, event-handling setup.
    
    3) By running the Wine fsync implementation and executing multi-threaded
    applications, in particular the modern games mentioned above, on top of
    this implementation.
    
    Changes since RFC:
      - Limit waitlist to 128 futexes
      - Simplify wait loop
      - Document functions
      - Reduce allocated space
      - Return hint if a futex was awoken during setup
      - Check if any futex was awoken prior to sleep
      - Drop relative timer logic
    
    Cc: Richard Yao <ryao@gentoo.org>
    Cc: Thomas Gleixner <tglx@linutronix.de>
    Cc: Peter Zijlstra <peterz@infradead.org>
    Co-developed-by: default avatarZebediah Figura <z.figura12@gmail.com>
    Signed-off-by: default avatarZebediah Figura <z.figura12@gmail.com>
    Co-developed-by: default avatarSteven Noonan <steven@valvesoftware.com>
    Signed-off-by: default avatarSteven Noonan <steven@valvesoftware.com>
    Co-developed-by: Pierre-Loup Griffais's avatarPierre-Loup A. Griffais <pgriffais@valvesoftware.com>
    Signed-off-by: Pierre-Loup Griffais's avatarPierre-Loup A. Griffais <pgriffais@valvesoftware.com>
    Signed-off-by: default avatarGabriel Krisman Bertazi <krisman@collabora.com>
    f2f386e4
  • Gabriel Krisman Bertazi's avatar
  • Gabriel Krisman Bertazi's avatar
  • Gabriel Krisman Bertazi's avatar
    futex wait multiple selftest · 7986c58e
    Gabriel Krisman Bertazi authored
    7986c58e
......@@ -21,6 +21,7 @@
#define FUTEX_WAKE_BITSET 10
#define FUTEX_WAIT_REQUEUE_PI 11
#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_WAIT_MULTIPLE 13
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
......@@ -150,4 +151,12 @@ struct robust_list_head {
(((op & 0xf) << 28) | ((cmp & 0xf) << 24) \
| ((oparg & 0xfff) << 12) | (cmparg & 0xfff))
#define FUTEX_MULTIPLE_MAX_COUNT 128
struct futex_wait_block {
__u32 __user *uaddr;
__u32 val;
__u32 bitset;
};
#endif /* _UAPI_LINUX_FUTEX_H */
......@@ -237,6 +237,8 @@ struct futex_q {
struct rt_mutex_waiter *rt_waiter;
union futex_key *requeue_pi_key;
u32 bitset;
u32 __user *uaddr;
u32 uval;
} __randomize_layout;
static const struct futex_q futex_q_init = {
......@@ -2344,6 +2346,29 @@ static int unqueue_me(struct futex_q *q)
return ret;
}
/**
* unqueue_multiple() - Remove several futexes from their futex_hash_bucket
* @q: The list of futexes to unqueue
* @count: Number of futexes in the list
*
* Helper to unqueue a list of futexes. This can't fail.
*
* Return:
* - >=0 - Index in the of the last futex that was awoken;
* - -1 - if no futex was awoken
*/
static int unqueue_multiple(struct futex_q *q, int count)
{
int ret = -1;
int i;
for (i = 0; i < count; i++) {
if (!unqueue_me(&q[i]))
ret = i;
}
return ret;
}
/*
* PI futexes can not be requeued and must remove themself from the
* hash bucket. The hash bucket lock (i.e. lock_ptr) is held on entry
......@@ -2631,22 +2656,21 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
__set_current_state(TASK_RUNNING);
}
static int __futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
struct futex_q *q, struct futex_hash_bucket **hb)
static int __futex_wait_setup(struct futex_q *q, unsigned int flags,
struct futex_hash_bucket **hb)
{
u32 uval;
int ret;
retry_private:
*hb = queue_lock(q);
ret = get_futex_value_locked(&uval, uaddr);
ret = get_futex_value_locked(&uval, q->uaddr);
if (ret) {
queue_unlock(*hb);
ret = get_user(uval, uaddr);
ret = get_user(uval, q->uaddr);
if (ret)
return ret;
......@@ -2656,7 +2680,7 @@ static int __futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
return 1;
}
if (uval != val) {
if (uval != q->uval) {
queue_unlock(*hb);
ret = -EWOULDBLOCK;
}
......@@ -2710,7 +2734,9 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
if (unlikely(ret != 0))
return ret;
ret = __futex_wait_setup(uaddr, val, flags, q, hb);
q->uaddr = uaddr;
q->uval = val;
ret = __futex_wait_setup(q, flags, hb);
/* Drop key reference if retry or error. */
if (ret)
......@@ -2720,6 +2746,176 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
return ret;
}
/**
* futex_setup_queue_multiple() - Prepare to wait on and enqueue multiple futexes
* @wb: The userspace list of objects to wait on
* @qs: The corresponding futex list
* @count: The size of the lists
* @flags futex flags (FLAGS_SHARED, etc.)
*
* This is a helper to enqueue multiple futex objects in a single step.
* Enqueing multiple futexes are tricky, because we need to enqueue each
* futex in the list before dealing with the next one to avoid
* deadlocking on the hash bucket. In addition, before enqueing, we
* need to make sure that current->state is TASK_INTERRUPTIBLE, but this
* cannot be done before the get_futex_key of the next key, because it
* calls get_user_pages, which can sleep. Thus, we fetch the list of
* futexes keys in two steps, by first pinning all the memory keys in
* the futex key, and only then we read each the key and queue the
* correspoding futex.
*
* Return:
* - 1 - One of the futexes was awaken by another thread
* - 0 - Success
* - <1 - -EFAULT or -EWOULDBLOCK
*/
static int futex_setup_queue_multiple(struct futex_q *qs, int count,
unsigned int flags, int *awaken)
{
struct futex_hash_bucket *hb;
int ret, i;
retry:
for (i = 0; i < count; i++) {
qs[i].key = FUTEX_KEY_INIT;
ret = get_futex_key(qs[i].uaddr, flags & FLAGS_SHARED,
&qs[i].key, FUTEX_READ);
if (unlikely(ret)) {
for (--i; i >= 0; i--)
put_futex_key(&qs[i].key);
return ret;
}
}
set_current_state(TASK_INTERRUPTIBLE);
for (i = 0; i < count; i++) {
ret = __futex_wait_setup(&qs[i], flags, &hb);
if (ret) {
/*
* Keys 0..(i-1) are implicitly put afterwards
* on unqueue_multiple.
*/
put_futex_key(&qs[i].key);
/*
* If something was already awaken, we can
* safely ignore the error and succeed.
*/
*awaken = unqueue_multiple(qs, i);
__set_current_state(TASK_RUNNING);
if (*awaken >= 0)
return 1;
if (ret > 0)
goto retry;
return ret;
}
/*
* The bucket lock can't be held while dealing with the
* next futex. Queue each futex at this moment so hb can
* be unlocked.
*/
queue_me(&qs[i], hb);
}
return 0;
}
/**
* futex_wait_multiple() - Prepare to wait on and enqueue several futexes
* @uaddr: The userspace list of objects to wait on
* @flags: Futex flags (FLAGS_SHARED, etc.)
* @count: The number of objects
* @abs_time: Timeout before giving up and returning to userspace
*
* Entry point for the FUTEX_WAIT_MULTIPLE futex operation, this fuction
* sleeps on a group of futexes and returns on the first futex that
* triggered, or after the timeout has elapsed.
*
* Return:
* - >=0 - Hint to the futex that was awoken
* - <0 - On error
*/
static int futex_wait_multiple(u32 __user *uaddr, unsigned int flags,
u32 count, ktime_t *abs_time)
{
struct hrtimer_sleeper timeout, *to;
struct futex_q *qs = NULL;
int ret, i, hint = 0;
struct futex_wait_block entry, *fwb;
if (!count || count > FUTEX_MULTIPLE_MAX_COUNT)
return -EINVAL;
qs = kcalloc(count, sizeof(*qs), GFP_KERNEL);
if (!qs)
return -ENOMEM;
fwb = (struct futex_wait_block *) __user uaddr;
for (i = 0; i < count; i++) {
if (copy_from_user(&entry, &fwb[i], sizeof(entry)))
return -EFAULT;
qs[i].uval = entry.val;
qs[i].uaddr = entry.uaddr;
qs[i].bitset = entry.bitset;
}
to = futex_setup_timer(abs_time, &timeout, FLAGS_CLOCKRT, 0);
do {
ret = futex_setup_queue_multiple(qs, count, flags, &hint);
if (ret) {
if (ret > 0) {
/* A futex was awaken during setup */
ret = hint;
}
break;
}
if (to)
hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
/*
* Avoid sleeping if another thread already tried to
* wake us.
*/
for (i = 0; i < count; i++) {
if (plist_node_empty(&qs[i].list))
break;
}
if (i == count && (!to || to->task)) {
freezable_schedule();
}
ret = unqueue_multiple(qs, count);
__set_current_state(TASK_RUNNING);
if (ret < 0) {
if (to && !to->task) {
ret = -ETIMEDOUT;
break;
} else if (signal_pending(current)) {
ret = -ERESTARTSYS;
break;
}
/*
* The final case is a spurious wakeup, for
* which we can just retry.
*/
}
} while (ret < 0);
if (to) {
hrtimer_cancel(&to->timer);
destroy_hrtimer_on_stack(&to->timer);
}
kfree(qs);
return ret;
}
static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
ktime_t *abs_time, u32 bitset)
{
......@@ -3680,6 +3876,8 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
uaddr2);
case FUTEX_CMP_REQUEUE_PI:
return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
case FUTEX_WAIT_MULTIPLE:
return futex_wait_multiple(uaddr, flags, val, timeout);
}
return -ENOSYS;
}
......@@ -3696,7 +3894,8 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
cmd == FUTEX_WAIT_REQUEUE_PI ||
cmd == FUTEX_WAIT_MULTIPLE)) {
if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
return -EFAULT;
if (get_timespec64(&ts, utime))
......@@ -3889,7 +4088,8 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
cmd == FUTEX_WAIT_BITSET ||
cmd == FUTEX_WAIT_REQUEUE_PI)) {
cmd == FUTEX_WAIT_REQUEUE_PI ||
cmd == FUTEX_WAIT_MULTIPLE)) {
if (get_old_timespec32(&ts, utime))
return -EFAULT;
if (!timespec64_valid(&ts))
......
......@@ -14,7 +14,8 @@ TEST_GEN_FILES := \
futex_requeue_pi_signal_restart \
futex_requeue_pi_mismatched_ops \
futex_wait_uninitialized_heap \
futex_wait_private_mapped_file
futex_wait_private_mapped_file \
futex_wait_multiple
TEST_PROGS := run.sh
......
// SPDX-License-Identifier: GPL-2.0-or-later
/******************************************************************************
*
* Copyright © International Business Machines Corp., 2009
*
* DESCRIPTION
* Test if FUTEX_WAIT op returns -EWOULDBLOCK if the futex value differs
* from the expected one.
*
* AUTHOR
* Gowrishankar <gowrishankar.m@in.ibm.com>
*
* HISTORY
* 2009-Nov-14: Initial version by Gowrishankar <gowrishankar.m@in.ibm.com>
*
*****************************************************************************/
#include <errno.h>
#include <getopt.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <pthread.h>
#include "futextest.h"
#include "logging.h"
#define TEST_NAME "futex-wait-multiple"
#define timeout_ns 100000
#define MAX_COUNT 1
#define WAKE_WAIT_US 3000000
char *progname;
futex_t f[MAX_COUNT];
struct futex_wait_block fwb[MAX_COUNT];
void usage(char *prog)
{
printf("Usage: %s\n", prog);
printf(" -c Use color\n");
printf(" -h Display this help message\n");
printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n",
VQUIET, VCRITICAL, VINFO);
}
void test_count_overflow()
{
futex_t f = FUTEX_INITIALIZER;
struct futex_wait_block fwb[MAX_COUNT+1];
int ret, i;
return;
ksft_print_msg("%s: Test a too big number of futexes\n",
progname);
for (i = 0; i < MAX_COUNT+1; i++) {
fwb[i].uaddr = &f;
fwb[i].val = f;
fwb[i].bitset = 0;
}
ret = futex_wait_multiple(fwb, MAX_COUNT+1, NULL,
FUTEX_PRIVATE_FLAG);
if (ret) {
ksft_test_result_fail("futex_wait_multiple returned %d\n",
ret < 0 ? errno : ret);
}
ksft_test_result_pass("futex_wait_multiple count overflow succeed\n");
}
void *waiterfn(void *arg)
{
int ret;
info("futex wait\n");
// ret = futex_wait(fwb[0].uaddr, fwb[0].val, NULL, FUTEX_PRIVATE_FLAG);
ret = futex_wait_multiple(fwb, 1, NULL, FUTEX_PRIVATE_FLAG);
if (ret < 0)
ksft_print_msg("waiter failed %d\n", errno);
ksft_print_msg("futex_wait: ret = %d, errno = %d\n", ret, errno);
return NULL;
}
void test_fwb_wakeup()
{
int ret, i;
pthread_t waiter;
ksft_print_msg("%s: Test wake up in a list of futex\n", progname);
for (i = 0; i < MAX_COUNT; i++) {
f[i] = 127;
fwb[i].uaddr = &f[i];
fwb[i].val = f[i];
fwb[i].bitset = 0;
}
ret = pthread_create(&waiter, NULL, waiterfn, NULL);
if (ret) {
ksft_test_result_fail("Creating waiting thread failed");
ksft_exit_fail();
}
usleep(WAKE_WAIT_US);
f[0] = 2;
ksft_print_msg("futex wake %lx %d \n", f, f[0]);
ret = futex_wake(f, 1, FUTEX_PRIVATE_FLAG);
if (ret != 1) {
ksft_test_result_fail("FUTEX_WAKE didn't find "
"the waiting thread errno=%d,.\n", errno);
ksft_exit_fail();
}
pthread_join(waiter, NULL);
ksft_test_result_pass("test_fwb_wakeup succeed\n");
}
int main(int argc, char *argv[])
{
int c;
while ((c = getopt(argc, argv, "cht:v:")) != -1) {
switch (c) {
case 'c':
log_color(1);
break;
case 'h':
usage(basename(argv[0]));
exit(0);
case 'v':
log_verbosity(atoi(optarg));
break;
default:
usage(basename(argv[0]));
exit(1);
}
}
progname = basename(argv[0]);
ksft_print_header();
ksft_set_plan(1);
test_count_overflow();
test_fwb_wakeup();
ksft_print_cnts();
return 0;
}
......@@ -41,6 +41,8 @@ int main(int argc, char *argv[])
{
futex_t f1 = FUTEX_INITIALIZER;
struct timespec to;
time_t secs;
struct futex_wait_block fwb = {&f1, f1, 0};
int res, ret = RET_PASS;
int c;
......@@ -65,7 +67,7 @@ int main(int argc, char *argv[])
}
ksft_print_header();
ksft_set_plan(1);
ksft_set_plan(2);
ksft_print_msg("%s: Block on a futex and wait for timeout\n",
basename(argv[0]));
ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns);
......@@ -77,10 +79,30 @@ int main(int argc, char *argv[])
info("Calling futex_wait on f1: %u @ %p\n", f1, &f1);
res = futex_wait(&f1, f1, &to, FUTEX_PRIVATE_FLAG);
if (!res || errno != ETIMEDOUT) {
fail("futex_wait returned %d\n", ret < 0 ? errno : ret);
ret = RET_FAIL;
ksft_test_result_fail("futex_wait returned %d\n",
ret < 0 ? errno : ret);
return RET_FAIL;
}
ksft_test_result_pass("futex_wait timeout succeeds\n");
print_result(TEST_NAME, ret);
return ret;
info("Calling futex_wait_multiple on f1: %u @ %p\n", f1, &f1);
/* Setup absolute time */
ret = clock_gettime(CLOCK_REALTIME, &to);
secs = (to.tv_nsec + timeout_ns) / 1000000000;
to.tv_nsec = ((int64_t)to.tv_nsec + timeout_ns) % 1000000000;
to.tv_sec += secs;
info("to.tv_sec = %ld\n", to.tv_sec);
info("to.tv_nsec = %ld\n", to.tv_nsec);
res = futex_wait_multiple(&fwb, 1, &to, FUTEX_PRIVATE_FLAG);
if (!res || errno != ETIMEDOUT) {
ksft_test_result_fail("futex_wait_multiple returned %d\n",
res < 0 ? errno : res);
return RET_FAIL;
}
ksft_test_result_pass("futex_wait_multiple timeout succeeds\n");
ksft_print_cnts();
return 0;
}
......@@ -40,6 +40,7 @@ int main(int argc, char *argv[])
{
struct timespec to = {.tv_sec = 0, .tv_nsec = timeout_ns};
futex_t f1 = FUTEX_INITIALIZER;
struct futex_wait_block fwb = {&f1, f1+1, 0};
int res, ret = RET_PASS;
int c;
......@@ -61,18 +62,29 @@ int main(int argc, char *argv[])
}
ksft_print_header();
ksft_set_plan(1);
ksft_set_plan(2);
ksft_print_msg("%s: Test the unexpected futex value in FUTEX_WAIT\n",
basename(argv[0]));
info("Calling futex_wait on f1: %u @ %p with val=%u\n", f1, &f1, f1+1);
res = futex_wait(&f1, f1+1, &to, FUTEX_PRIVATE_FLAG);
if (!res || errno != EWOULDBLOCK) {
fail("futex_wait returned: %d %s\n",
res ? errno : res, res ? strerror(errno) : "");
ret = RET_FAIL;
ksft_test_result_fail("futex_wait_multiple returned %d\n",
res < 0 ? errno : res);
return RET_FAIL;
}
ksft_test_result_pass("futex_wait wouldblock succeeds\n");
print_result(TEST_NAME, ret);
info("Calling futex_wait_multiple on f1: %u @ %p with val=%u\n",
f1, &f1, f1+1);
res = futex_wait_multiple(&fwb, 1, NULL, FUTEX_PRIVATE_FLAG);
if (!res || errno != EWOULDBLOCK) {
ksft_test_result_fail("futex_wait_multiple returned %d\n",
res < 0 ? errno : res);
return RET_FAIL;
}
ksft_test_result_pass("futex_wait_multiple wouldblock succeeds\n");
ksft_print_cnts();
return ret;
}
......@@ -73,3 +73,6 @@ echo
echo
./futex_wait_uninitialized_heap $COLOR
./futex_wait_private_mapped_file $COLOR
echo
./futex_wait_multiple $COLOR
......@@ -38,6 +38,9 @@ typedef volatile u_int32_t futex_t;
#ifndef FUTEX_CMP_REQUEUE_PI
#define FUTEX_CMP_REQUEUE_PI 12
#endif
#ifndef FUTEX_WAIT_MULTIPLE
#define FUTEX_WAIT_MULTIPLE 13
#endif
#ifndef FUTEX_WAIT_REQUEUE_PI_PRIVATE
#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | \
FUTEX_PRIVATE_FLAG)
......@@ -47,6 +50,12 @@ typedef volatile u_int32_t futex_t;
FUTEX_PRIVATE_FLAG)
#endif
struct futex_wait_block {
futex_t *uaddr;
futex_t val;
__u32 bitset;
};
/**
* futex() - SYS_futex syscall wrapper
* @uaddr: address of first futex
......@@ -80,6 +89,18 @@ futex_wait(futex_t *uaddr, futex_t val, struct timespec *timeout, int opflags)
return futex(uaddr, FUTEX_WAIT, val, timeout, NULL, 0, opflags);
}
/**
* futex_wait_multiple() - block on several futexes with optional timeout
* @timeout: absolute timeout
*/
static inline int
futex_wait_multiple(struct futex_wait_block *fwb, int count,
struct timespec *timeout, int opflags)
{
return futex(fwb, FUTEX_WAIT_MULTIPLE, count, timeout, NULL, 0,
opflags);
}
/**
* futex_wake() - wake one or more tasks blocked on uaddr
* @nr_wake: wake up to this many tasks
......