Commit 253be348 authored by Philip Withnall's avatar Philip Withnall Committed by Olivier Crête
Browse files

agent: Add support for vectored I/O for receives

Add two new public functions:
 • nice_agent_recv_messages()
 • nice_agent_recv_messages_nonblocking()
which allow receiving multiple messages in a single call, and support
vectors of buffers to receive the messages into.

The existing nice_agent_recv[_nonblocking]() APIs have been left
untouched.

This tidies up a lot of the message handling code internally, and
eliminates a couple of memcpy()s. There are still a few more memcpy()s
on the critical path, which could be eliminated with further work.

In the reliable agent case, every message is memcpy()ed twice: once into
the pseudo-TCP receive buffer, and once out of it. The copy on input
could be eliminated (in the case of in-order delivery of packets) by
receiving directly into the receive buffer. The copy on output can’t be
eliminated except in the I/O callback case (when
nice_agent_attach_recv() has been used), in which case the callback
could be invoked with a pointer directly into the pseudo-TCP receive
buffer.

In the non-reliable agent case, zero memcpy()s are used.

A couple of the more complex socket implementations (TURN and HTTP) have
slow paths during setup, and partially also during normal use. These
could be optimised further, and FIXME comments have been added.
parent 9661150d
......@@ -52,11 +52,44 @@
#include <glib.h>
#include "agent.h"
/**
* NiceInputMessageIter:
* @message: index of the message currently being written into
* @buffer: index of the buffer currently being written into
* @offset: byte offset into the buffer
*
* Iterator for sequentially writing into an array of #NiceInputMessages,
* tracking the current write position (i.e. the index of the next byte to be
* written).
*
* If @message is equal to the number of messages in the associated
* #NiceInputMessage array, and @buffer and @offset are zero, the iterator is at
* the end of the messages array, and the array is (presumably) full.
*
* Since: 0.1.5
*/
typedef struct {
guint message;
guint buffer;
gsize offset;
} NiceInputMessageIter;
void
nice_input_message_iter_reset (NiceInputMessageIter *iter);
gboolean
nice_input_message_iter_is_at_end (NiceInputMessageIter *iter,
NiceInputMessage *messages, guint n_messages);
guint
nice_input_message_iter_get_n_valid_messages (NiceInputMessageIter *iter);
#include "socket.h"
#include "candidate.h"
#include "stream.h"
#include "conncheck.h"
#include "component.h"
#include "random.h"
#include "stun/stunagent.h"
#include "stun/usages/turn.h"
#include "stun/usages/ice.h"
......@@ -178,9 +211,6 @@ component_io_cb (
GIOCondition condition,
gpointer data);
gssize agent_recv_locked (NiceAgent *agent, Stream *stream,
Component *component, NiceSocket *socket, guint8 *buf, gsize buf_len);
gsize
memcpy_buffer_to_input_message (NiceInputMessage *message,
const guint8 *buffer, gsize buffer_length);
......
......@@ -83,6 +83,9 @@
#define MAX_TCP_MTU 1400 /* Use 1400 because of VPNs and we assume IEE 802.3 */
static void
nice_debug_message_composition (NiceInputMessage *messages, guint n_messages);
G_DEFINE_TYPE (NiceAgent, nice_agent, G_TYPE_OBJECT);
enum
......@@ -1025,6 +1028,73 @@ pseudo_tcp_socket_opened (PseudoTcpSocket *sock, gpointer user_data)
stream->id, component->id);
}
/* Will fill up @messages from the first free byte onwards (as determined using
* @iter). This is always used in reliable mode, so it essentially treats
* @messages as a massive flat array of buffers.
*
* Updates @iter in place. @iter and @messages are left in invalid states if
* an error is returned.
*
* Returns the number of valid messages in @messages on success (which may be
* zero if reading into the first buffer of the message would have blocked), or
* a negative number on error. */
static gint
pseudo_tcp_socket_recv_messages (PseudoTcpSocket *self,
NiceInputMessage *messages, guint n_messages, NiceInputMessageIter *iter,
GError **error)
{
for (; iter->message < n_messages; iter->message++) {
NiceInputMessage *message = &messages[iter->message];
if (iter->buffer == 0 && iter->offset == 0) {
message->length = 0;
}
for (;
(message->n_buffers >= 0 && iter->buffer < (guint) message->n_buffers) ||
(message->n_buffers < 0 && message->buffers[iter->buffer].buffer != NULL);
iter->buffer++) {
GInputVector *buffer = &message->buffers[iter->buffer];
do {
gssize len;
len = pseudo_tcp_socket_recv (self,
(gchar *) buffer->buffer + iter->offset,
buffer->size - iter->offset);
nice_debug ("%s: Received %" G_GSSIZE_FORMAT " bytes into "
"buffer %p (offset %" G_GSIZE_FORMAT ", length %" G_GSIZE_FORMAT
").", G_STRFUNC, len, buffer->buffer, iter->offset, buffer->size);
if (len < 0 && pseudo_tcp_socket_get_error (self) == EWOULDBLOCK) {
len = 0;
goto done;
} else if (len < 0 && pseudo_tcp_socket_get_error (self) == ENOTCONN) {
g_set_error (error, G_IO_ERROR, G_IO_ERROR_WOULD_BLOCK,
"Error reading data from pseudo-TCP socket: not connected.");
return len;
} else if (len < 0) {
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Error reading data from pseudo-TCP socket.");
return len;
} else {
/* Got some data! */
message->length += len;
iter->offset += len;
}
} while (iter->offset < buffer->size);
iter->offset = 0;
}
iter->buffer = 0;
}
done:
return nice_input_message_iter_get_n_valid_messages (iter);
}
/* This is called with the agent lock held. */
static void
pseudo_tcp_socket_readable (PseudoTcpSocket *sock, gpointer user_data)
......@@ -1032,8 +1102,6 @@ pseudo_tcp_socket_readable (PseudoTcpSocket *sock, gpointer user_data)
Component *component = user_data;
NiceAgent *agent = component->agent;
Stream *stream = component->stream;
guint8 buf[MAX_BUFFER_SIZE];
gssize len;
gboolean has_io_callback;
nice_debug ("Agent %p: s%d:%d pseudo Tcp socket readable", agent,
......@@ -1045,59 +1113,81 @@ pseudo_tcp_socket_readable (PseudoTcpSocket *sock, gpointer user_data)
g_object_add_weak_pointer (G_OBJECT (agent), (gpointer *)&agent);
has_io_callback = component_has_io_callback (component);
do {
/* Only dequeue pseudo-TCP data if we can reliably inform the client. The
* agent lock is held here, so has_io_callback can only change during
* component_emit_io_callback(), after which it’s re-queried. This ensures
* no data loss of packets already received and dequeued. */
if (has_io_callback) {
/* Only dequeue pseudo-TCP data if we can reliably inform the client. The
* agent lock is held here, so has_io_callback can only change during
* component_emit_io_callback(), after which it’s re-queried. This ensures
* no data loss of packets already received and dequeued. */
if (has_io_callback) {
do {
guint8 buf[MAX_BUFFER_SIZE];
gssize len;
/* FIXME: Why copy into a temporary buffer here? Why can’t the I/O
* callbacks be emitted directly from the pseudo-TCP receive buffer? */
len = pseudo_tcp_socket_recv (sock, (gchar *) buf, sizeof(buf));
} else if (component->recv_buf != NULL) {
len = pseudo_tcp_socket_recv (sock,
(gchar *) component->recv_buf + component->recv_buf_valid_len,
component->recv_buf_len - component->recv_buf_valid_len);
} else {
len = 0;
}
nice_debug ("%s: received %" G_GSSIZE_FORMAT " bytes", G_STRFUNC, len);
nice_debug ("%s: I/O callback case: Received %" G_GSSIZE_FORMAT " bytes",
G_STRFUNC, len);
if (len == 0) {
component->tcp_readable = FALSE;
break;
} else if (len <= 0) {
/* Handle errors. */
if (pseudo_tcp_socket_get_error (sock) != EWOULDBLOCK) {
nice_debug ("%s: calling priv_pseudo_tcp_error()", G_STRFUNC);
priv_pseudo_tcp_error (agent, stream, component);
if (component->recv_buf_error != NULL) {
GIOErrorEnum error_code;
if (pseudo_tcp_socket_get_error (sock) == ENOTCONN)
error_code = G_IO_ERROR_BROKEN_PIPE;
else
error_code = G_IO_ERROR_FAILED;
g_set_error (component->recv_buf_error, G_IO_ERROR, error_code,
"Error reading data from pseudo-TCP socket.");
}
}
break;
}
if (len > 0 && has_io_callback) {
component_emit_io_callback (component, buf, len);
if (sock == NULL) {
nice_debug ("PseudoTCP socket got destroyed in readable callback!");
break;
}
} else if (len > 0 && component->recv_buf != NULL) {
/* No callback to call. The data has been copied directly into the
* client’s receive buffer. */
component->recv_buf_valid_len += len;
} else if (len < 0 &&
pseudo_tcp_socket_get_error (sock) != EWOULDBLOCK) {
/* Signal error */
has_io_callback = component_has_io_callback (component);
} while (has_io_callback);
} else if (component->recv_messages != NULL) {
gint n_valid_messages;
/* Fill up every buffer in every message until the connection closes or an
* error occurs. Copy the data directly into the client’s receive message
* array without making any callbacks. Update component->recv_messages_iter
* as we go. */
n_valid_messages = pseudo_tcp_socket_recv_messages (sock,
component->recv_messages, component->n_recv_messages,
&component->recv_messages_iter, component->recv_buf_error);
nice_debug ("%s: Client buffers case: Received %d valid messages:",
G_STRFUNC, n_valid_messages);
nice_debug_message_composition (component->recv_messages,
component->n_recv_messages);
if (n_valid_messages < 0) {
nice_debug ("%s: calling priv_pseudo_tcp_error()", G_STRFUNC);
priv_pseudo_tcp_error (agent, stream, component);
if (component->recv_buf != NULL) {
GIOErrorEnum error_code;
if (pseudo_tcp_socket_get_error (sock) == ENOTCONN)
error_code = G_IO_ERROR_BROKEN_PIPE;
else
error_code = G_IO_ERROR_FAILED;
g_set_error (component->recv_buf_error, G_IO_ERROR, error_code,
"Error reading data from pseudo-TCP socket.");
}
} else if (len < 0 &&
pseudo_tcp_socket_get_error (sock) == EWOULDBLOCK){
} else if (n_valid_messages == 0) {
component->tcp_readable = FALSE;
}
has_io_callback = component_has_io_callback (component);
} while (len > 0 &&
(has_io_callback ||
component->recv_buf_valid_len < component->recv_buf_len));
} else {
nice_debug ("%s: no data read", G_STRFUNC);
}
if (agent) {
adjust_tcp_clock (agent, stream, component);
......@@ -2405,68 +2495,76 @@ nice_agent_set_remote_candidates (NiceAgent *agent, guint stream_id, guint compo
return added;
}
/* Return values for agent_recv_message_unlocked(). Needed purely because it
* must differentiate between RECV_OOB and RECV_SUCCESS. */
typedef enum {
RECV_ERROR = -2,
RECV_WOULD_BLOCK = -1,
RECV_OOB = 0,
RECV_SUCCESS = 1,
} RecvStatus;
/*
* agent_recv_locked:
* agent_recv_message_unlocked:
* @agent: a #NiceAgent
* @stream: the stream to receive from
* @component: the component to receive from
* @socket: the socket to receive on
* @buf: the buffer to write into (must be at least @buf_len bytes long)
* @buf_len: the length of @buf
* @message: the message to write into (must have at least 65536 bytes of buffer
* space)
*
* Receive up to @buf_len bytes of data from the given
* @stream/@component/@socket, in a non-blocking fashion. If the socket is a
* datagram socket and @buf_len is not big enough to hold an entire packet, the
* remaining bytes of the packet will be silently dropped.
* Receive a single message of data from the given @stream, @component and
* @socket tuple, in a non-blocking fashion. The caller must ensure that
* @message contains enough buffers to provide at least 65536 bytes of buffer
* space, but the buffers may be split as the caller sees fit.
*
* NOTE: Must be called with the agent’s lock held.
* This must be called with the agent’s lock held.
*
* Returns: number of bytes stored in @buf, 0 if no data is available, or -1 on
* error
* Returns: number of valid messages received on success (i.e. %RECV_SUCCESS or
* 1), %RECV_OOB if data was successfully received but was handled out-of-band
* (e.g. due to being a STUN control packet), %RECV_WOULD_BLOCK if no data is
* available and the call would block, or %RECV_ERROR on error
*/
gssize
agent_recv_locked (
static RecvStatus
agent_recv_message_unlocked (
NiceAgent *agent,
Stream *stream,
Component *component,
NiceSocket *socket,
guint8 *buf,
gsize buf_len)
NiceInputMessage *message)
{
NiceAddress from;
gssize len;
GList *item;
guint8 local_buf[MAX_BUFFER_SIZE];
gsize local_buf_len = MAX_BUFFER_SIZE;
GInputVector local_bufs = { local_buf, local_buf_len };
NiceInputMessage local_messages = { &local_bufs, 1, &from, 0 };
gint n_valid_messages;
gint retval;
/* Returns -1 on error, 0 on EWOULDBLOCK, and > 0 on success.
*
* FIXME: We have to receive into a local buffer then copy out because
* otherwise, if @buf is too small, we could lose data, even when in
* reliable mode (because reliable streams are packetised). */
n_valid_messages = nice_socket_recv_messages (socket, &local_messages, 1);
/* We need an address for packet parsing, below. */
if (message->from == NULL) {
message->from = &from;
}
len = (n_valid_messages == 1) ?
(gssize) local_messages.length : n_valid_messages;
retval = nice_socket_recv_messages (socket, message, 1);
if (len == 0) {
return 0;
} else if (len < 0) {
nice_debug ("Agent %p: %s returned %" G_GSSIZE_FORMAT ", errno (%d) : %s",
agent, G_STRFUNC, len, errno, g_strerror (errno));
nice_debug ("%s: Received %d valid messages of length %" G_GSIZE_FORMAT
" from base socket %p.", G_STRFUNC, retval, message->length, socket);
if (retval == 0) {
retval = RECV_WOULD_BLOCK; /* EWOULDBLOCK */
goto done;
} else if (retval < 0) {
nice_debug ("Agent %p: %s returned %d, errno (%d) : %s",
agent, G_STRFUNC, retval, errno, g_strerror (errno));
return len;
retval = RECV_ERROR;
goto done;
}
#ifndef NDEBUG
if (len > 0) {
if (message->length > 0) {
gchar tmpbuf[INET6_ADDRSTRLEN];
nice_address_to_string (&from, tmpbuf);
nice_address_to_string (message->from, tmpbuf);
nice_debug ("Agent %p : Packet received on local socket %d from [%s]:%u (%" G_GSSIZE_FORMAT " octets).", agent,
g_socket_get_fd (socket->fileno), tmpbuf, nice_address_get_port (&from), len);
g_socket_get_fd (socket->fileno), tmpbuf,
nice_address_get_port (message->from), message->length);
}
#endif
......@@ -2474,7 +2572,7 @@ agent_recv_locked (
TurnServer *turn = item->data;
GSList *i = NULL;
if (!nice_address_equal (&from, &turn->server))
if (!nice_address_equal (message->from, &turn->server))
continue;
#ifndef NDEBUG
......@@ -2488,8 +2586,7 @@ agent_recv_locked (
if (cand->type == NICE_CANDIDATE_TYPE_RELAYED &&
cand->stream_id == stream->id &&
cand->component_id == component->id) {
len = nice_turn_socket_parse_recv (cand->sockptr, &socket,
&from, len, (gchar *) local_buf, &from, (gchar *) local_buf, len);
nice_turn_socket_parse_recv_message (cand->sockptr, &socket, message);
}
}
}
......@@ -2498,17 +2595,39 @@ agent_recv_locked (
/* If the message’s stated length is equal to its actual length, it’s probably
* a STUN message; otherwise it’s probably data. */
if (stun_message_validate_buffer_length ((uint8_t *) local_buf, (size_t) len,
if (stun_message_validate_buffer_length_fast (
(StunInputVector *) message->buffers, message->n_buffers, message->length,
(agent->compatibility != NICE_COMPATIBILITY_OC2007 &&
agent->compatibility != NICE_COMPATIBILITY_OC2007R2)) == len &&
conn_check_handle_inbound_stun (agent, stream, component, socket,
&from, (gchar *) local_buf, len)) {
/* Handled STUN message. */
return 0;
agent->compatibility != NICE_COMPATIBILITY_OC2007R2)) == (ssize_t) message->length) {
/* Slow path: If this message isn’t obviously *not* a STUN packet, compact
* its buffers
* into a single monolithic one and parse the packet properly. */
guint8 *big_buf;
gsize big_buf_len;
big_buf = compact_input_message (message, &big_buf_len);
if (stun_message_validate_buffer_length (big_buf, big_buf_len,
(agent->compatibility != NICE_COMPATIBILITY_OC2007 &&
agent->compatibility != NICE_COMPATIBILITY_OC2007R2)) == (gint) big_buf_len &&
conn_check_handle_inbound_stun (agent, stream, component, socket,
message->from, (gchar *) big_buf, big_buf_len)) {
/* Handled STUN message. */
nice_debug ("%s: Valid STUN packet received.", G_STRFUNC);
retval = RECV_OOB;
g_free (big_buf);
goto done;
}
nice_debug ("%s: WARNING: Packet passed fast STUN validation but failed "
"slow validation.", G_STRFUNC);
g_free (big_buf);
}
/* Unhandled STUN; try handling TCP data, then pass to the client. */
if (len > 0 && component->tcp) {
if (message->length > 0 && component->tcp) {
/* If we don’t yet have an underlying selected socket, queue up the incoming
* data to handle later. This is because we can’t send ACKs (or, more
* importantly for the first few packets, SYNACKs) without an underlying
......@@ -2518,11 +2637,10 @@ agent_recv_locked (
* machine. */
if (component->selected_pair.local == NULL) {
GOutputVector *vec = g_slice_new (GOutputVector);
vec->buffer = g_memdup (local_buf, len);
vec->size = len;
vec->buffer = compact_input_message (message, &vec->size);
g_queue_push_tail (&component->queued_tcp_packets, vec);
nice_debug ("%s: Queued %" G_GSSIZE_FORMAT " bytes for agent %p.",
G_STRFUNC, len, agent);
G_STRFUNC, vec->size, agent);
return 0;
} else {
......@@ -2532,9 +2650,9 @@ agent_recv_locked (
/* Received data on a reliable connection. */
g_object_add_weak_pointer (G_OBJECT (agent), (gpointer *) &agent);
nice_debug ("%s: notifying pseudo-TCP of packet, length %" G_GSSIZE_FORMAT,
G_STRFUNC, len);
pseudo_tcp_socket_notify_packet (component->tcp, (gchar *) local_buf, len);
nice_debug ("%s: notifying pseudo-TCP of packet, length %" G_GSIZE_FORMAT,
G_STRFUNC, message->length);
pseudo_tcp_socket_notify_message (component->tcp, message);
if (agent) {
adjust_tcp_clock (agent, stream, component);
......@@ -2543,22 +2661,24 @@ agent_recv_locked (
nice_debug ("Our agent got destroyed in notify_packet!!");
}
/* Data’s already been handled, so return 0. */
return 0;
} else if (len > 0 && !component->tcp && agent->reliable) {
/* Success! Handled out-of-band. */
retval = RECV_OOB;
goto done;
} else if (message->length > 0 && !component->tcp && agent->reliable) {
/* Received data on a reliable connection which has no TCP component. */
nice_debug ("Received data on a pseudo tcp FAILED component. Ignoring.");
return 0;
retval = RECV_OOB;
goto done;
}
/* Yay for poor performance! */
if (len >= 0) {
len = MIN (buf_len, (gsize) len);
memcpy (buf, local_buf, len);
done:
/* Clear local modifications. */
if (message->from == &from) {
message->from = NULL;
}
return len;
return retval;
}
/* Print the composition of an array of messages. No-op if debugging is
......@@ -2661,6 +2781,137 @@ memcpy_buffer_to_input_message (NiceInputMessage *message,
return message->length;
}
/**
* nice_input_message_iter_reset:
* @iter: a #NiceInputMessageIter
*
* Reset the given @iter to point to the beginning of the array of messages.
* This may be used both to initialise it and to reset it after use.
*
* Since: 0.1.5
*/
void
nice_input_message_iter_reset (NiceInputMessageIter *iter)
{
iter->message = 0;
iter->buffer = 0;
iter->offset = 0;
}
/**
* nice_input_message_iter_is_at_end:
* @iter: a #NiceInputMessageIter
* @messages: (array length=n_messages): an array of #NiceInputMessages
* @n_messages: number of entries in @messages
*
* Determine whether @iter points to the end of the given @messages array. If it
* does, the array is full: every buffer in every message is full of valid
* bytes.
*
* Returns: %TRUE if the messages’ buffers are full, %FALSE otherwise
*
* Since: 0.1.5
*/
gboolean
nice_input_message_iter_is_at_end (NiceInputMessageIter *iter,
NiceInputMessage *messages, guint n_messages)
{
return (iter->message == n_messages &&
iter->buffer == 0 && iter->offset == 0);
}
/**
* nice_input_message_iter_get_n_valid_messages:
* @iter: a #NiceInputMessageIter
*
* Calculate the number of valid messages in the messages array. A valid message
* is one which contains at least one valid byte of data in its buffers.
*
* Returns: number of valid messages (may be zero)
*
* Since: 0.1.5
*/
guint
nice_input_message_iter_get_n_valid_messages (NiceInputMessageIter *iter)
{
if (iter->buffer == 0 && iter->offset == 0)
return iter->message;
else
return iter->message + 1;
}
/* Will fill up @messages from the first free byte onwards (as determined using
* @iter). This may be used in reliable or non-reliable mode; in non-reliable
* mode it will always increment the message index after each buffer is
* consumed.
*
* Updates @iter in place. No errors can occur.
*
* Returns the number of valid messages in @messages on success (which may be
* zero if reading into the first buffer of the message would have blocked).
*
* Must be called with the io_mutex held. */
static gint
pending_io_messages_recv_messages (Component *component, gboolean reliable,
NiceInputMessage *messages, guint n_messages, NiceInputMessageIter *iter)
{
gsize len;
IOCallbackData *data;
NiceInputMessage *message = &messages[iter->message];
g_assert (component->io_callback_id == 0);
data = g_queue_peek_head (&component->pending_io_messages);
if (data == NULL)
goto done;
if (iter->buffer == 0 && iter->offset == 0) {
message->length = 0;
}
for (;
(message->n_buffers >= 0 && iter->buffer < (guint) message->n_buffers) ||
(message->n_buffers < 0 && message->buffers[iter->buffer].buffer != NULL);
iter->buffer++) {
GInputVector *buffer = &message->buffers[iter->buffer];
do {
len = MIN (data->buf_len - data->offset, buffer->size - iter->offset);
memcpy ((guint8 *) buffer->buffer + iter->offset,
data->buf + data->offset, len);
nice_debug ("%s: Unbuffered %" G_GSIZE_FORMAT " bytes into "
"buffer %p (offset %" G_GSIZE_FORMAT ", length %" G_GSIZE_FORMAT
").", G_STRFUNC, len, buffer->buffer, iter->offset, buffer->size);
message->length += len;
iter->offset += len;
data->offset += len;
} while (iter->offset < buffer->size);
iter->offset = 0;
}
/* Only if we managed to consume the whole buffer should it be popped off the
* queue; otherwise we’ll have another go at it later. */
if (data->offset == data->buf_len) {