pipe.c 27.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 *  linux/fs/pipe.c
 *
 *  Copyright (C) 1991, 1992, 1999  Linus Torvalds
 */

#include <linux/mm.h>
#include <linux/file.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/fs.h>
15
#include <linux/log2.h>
Linus Torvalds's avatar
Linus Torvalds committed
16
#include <linux/mount.h>
17
#include <linux/magic.h>
Linus Torvalds's avatar
Linus Torvalds committed
18 19 20
#include <linux/pipe_fs_i.h>
#include <linux/uio.h>
#include <linux/highmem.h>
21
#include <linux/pagemap.h>
Al Viro's avatar
Al Viro committed
22
#include <linux/audit.h>
23
#include <linux/syscalls.h>
24
#include <linux/fcntl.h>
25
#include <linux/memcontrol.h>
Linus Torvalds's avatar
Linus Torvalds committed
26

27
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29
#include <asm/ioctls.h>

30 31
#include "internal.h"

32 33
/*
 * The max size that a non-root user is allowed to grow the pipe. Can
34
 * be set by root in /proc/sys/fs/pipe-max-size
35
 */
36 37 38 39 40 41
unsigned int pipe_max_size = 1048576;

/*
 * Minimum pipe size, as required by POSIX
 */
unsigned int pipe_min_size = PAGE_SIZE;
42

43 44 45 46 47 48
/* Maximum allocatable pages per user. Hard limit is unset by default, soft
 * matches default values.
 */
unsigned long pipe_user_pages_hard;
unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;

Linus Torvalds's avatar
Linus Torvalds committed
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
/*
 * We use a start+len construction, which provides full use of the 
 * allocated memory.
 * -- Florian Coosmann (FGC)
 * 
 * Reads with count = 0 should always return 0.
 * -- Julian Bradfield 1999-06-07.
 *
 * FIFOs and Pipes now generate SIGIO for both readers and writers.
 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
 *
 * pipe_read & write cleanup
 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
 */

64 65
static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
{
Al Viro's avatar
Al Viro committed
66
	if (pipe->files)
Al Viro's avatar
Al Viro committed
67
		mutex_lock_nested(&pipe->mutex, subclass);
68 69 70 71 72 73 74 75 76 77 78 79 80
}

void pipe_lock(struct pipe_inode_info *pipe)
{
	/*
	 * pipe_lock() nests non-pipe inode locks (for writing to a file)
	 */
	pipe_lock_nested(pipe, I_MUTEX_PARENT);
}
EXPORT_SYMBOL(pipe_lock);

void pipe_unlock(struct pipe_inode_info *pipe)
{
Al Viro's avatar
Al Viro committed
81
	if (pipe->files)
Al Viro's avatar
Al Viro committed
82
		mutex_unlock(&pipe->mutex);
83 84 85
}
EXPORT_SYMBOL(pipe_unlock);

86 87 88 89 90 91 92 93 94 95
static inline void __pipe_lock(struct pipe_inode_info *pipe)
{
	mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
}

static inline void __pipe_unlock(struct pipe_inode_info *pipe)
{
	mutex_unlock(&pipe->mutex);
}

96 97 98 99 100 101 102 103 104
void pipe_double_lock(struct pipe_inode_info *pipe1,
		      struct pipe_inode_info *pipe2)
{
	BUG_ON(pipe1 == pipe2);

	if (pipe1 < pipe2) {
		pipe_lock_nested(pipe1, I_MUTEX_PARENT);
		pipe_lock_nested(pipe2, I_MUTEX_CHILD);
	} else {
105 106
		pipe_lock_nested(pipe2, I_MUTEX_PARENT);
		pipe_lock_nested(pipe1, I_MUTEX_CHILD);
107 108 109
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
110
/* Drop the inode semaphore and wait for a pipe event, atomically */
111
void pipe_wait(struct pipe_inode_info *pipe)
Linus Torvalds's avatar
Linus Torvalds committed
112 113 114
{
	DEFINE_WAIT(wait);

115 116 117 118
	/*
	 * Pipes are system-local resources, so sleeping on them
	 * is considered a noninteractive wait:
	 */
119
	prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE);
120
	pipe_unlock(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
121
	schedule();
122
	finish_wait(&pipe->wait, &wait);
123
	pipe_lock(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
124 125
}

126 127
static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
				  struct pipe_buffer *buf)
Linus Torvalds's avatar
Linus Torvalds committed
128 129 130
{
	struct page *page = buf->page;

131 132 133
	/*
	 * If nobody else uses this page, and we don't already have a
	 * temporary page, let's keep track of it as a one-deep
134
	 * allocation cache. (Otherwise just release our reference to it)
135
	 */
136
	if (page_count(page) == 1 && !pipe->tmp_page)
137
		pipe->tmp_page = page;
138
	else
139
		put_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
140 141
}

142 143 144 145 146 147
static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
			       struct pipe_buffer *buf)
{
	struct page *page = buf->page;

	if (page_count(page) == 1) {
148
		if (memcg_kmem_enabled())
149 150 151 152 153 154 155
			memcg_kmem_uncharge(page, 0);
		__SetPageLocked(page);
		return 0;
	}
	return 1;
}

156
/**
157
 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
158 159 160 161
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to attempt to steal
 *
 * Description:
162
 *	This function attempts to steal the &struct page attached to
163 164
 *	@buf. If successful, this function returns 0 and returns with
 *	the page locked. The caller may then reuse the page for whatever
165
 *	he wishes; the typical use is insertion into a different file
166 167
 *	page cache.
 */
168 169
int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
			   struct pipe_buffer *buf)
170
{
171 172
	struct page *page = buf->page;

173 174 175 176 177
	/*
	 * A reference of one is golden, that means that the owner of this
	 * page is the only one holding a reference to it. lock the page
	 * and return OK.
	 */
178 179 180 181 182 183
	if (page_count(page) == 1) {
		lock_page(page);
		return 0;
	}

	return 1;
184
}
185
EXPORT_SYMBOL(generic_pipe_buf_steal);
186

187
/**
188
 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
189 190 191 192 193 194 195 196 197
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to get a reference to
 *
 * Description:
 *	This function grabs an extra reference to @buf. It's used in
 *	in the tee() system call, when we duplicate the buffers in one
 *	pipe into another.
 */
void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
198
{
199
	get_page(buf->page);
200
}
201
EXPORT_SYMBOL(generic_pipe_buf_get);
202

203 204
/**
 * generic_pipe_buf_confirm - verify contents of the pipe buffer
205
 * @info:	the pipe that the buffer belongs to
206 207 208 209 210 211
 * @buf:	the buffer to confirm
 *
 * Description:
 *	This function does nothing, because the generic pipe code uses
 *	pages that are always good when inserted into the pipe.
 */
212 213
int generic_pipe_buf_confirm(struct pipe_inode_info *info,
			     struct pipe_buffer *buf)
214 215 216
{
	return 0;
}
217
EXPORT_SYMBOL(generic_pipe_buf_confirm);
218

219 220 221 222 223 224 225 226 227 228 229
/**
 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
 * @pipe:	the pipe that the buffer belongs to
 * @buf:	the buffer to put a reference to
 *
 * Description:
 *	This function releases a reference to @buf.
 */
void generic_pipe_buf_release(struct pipe_inode_info *pipe,
			      struct pipe_buffer *buf)
{
230
	put_page(buf->page);
231
}
232
EXPORT_SYMBOL(generic_pipe_buf_release);
233

234
static const struct pipe_buf_operations anon_pipe_buf_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
235
	.can_merge = 1,
236
	.confirm = generic_pipe_buf_confirm,
Linus Torvalds's avatar
Linus Torvalds committed
237
	.release = anon_pipe_buf_release,
238
	.steal = anon_pipe_buf_steal,
239
	.get = generic_pipe_buf_get,
Linus Torvalds's avatar
Linus Torvalds committed
240 241
};

242 243 244 245
static const struct pipe_buf_operations packet_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = anon_pipe_buf_release,
246
	.steal = anon_pipe_buf_steal,
247 248 249
	.get = generic_pipe_buf_get,
};

Linus Torvalds's avatar
Linus Torvalds committed
250
static ssize_t
Al Viro's avatar
Al Viro committed
251
pipe_read(struct kiocb *iocb, struct iov_iter *to)
Linus Torvalds's avatar
Linus Torvalds committed
252
{
Al Viro's avatar
Al Viro committed
253
	size_t total_len = iov_iter_count(to);
254
	struct file *filp = iocb->ki_filp;
255
	struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
256 257 258 259 260 261 262 263 264
	int do_wakeup;
	ssize_t ret;

	/* Null read succeeds. */
	if (unlikely(total_len == 0))
		return 0;

	do_wakeup = 0;
	ret = 0;
265
	__pipe_lock(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
266
	for (;;) {
267
		int bufs = pipe->nrbufs;
Linus Torvalds's avatar
Linus Torvalds committed
268
		if (bufs) {
269 270
			int curbuf = pipe->curbuf;
			struct pipe_buffer *buf = pipe->bufs + curbuf;
Linus Torvalds's avatar
Linus Torvalds committed
271
			size_t chars = buf->len;
272 273
			size_t written;
			int error;
Linus Torvalds's avatar
Linus Torvalds committed
274 275 276 277

			if (chars > total_len)
				chars = total_len;

278
			error = pipe_buf_confirm(pipe, buf);
279
			if (error) {
280
				if (!ret)
281
					ret = error;
282 283
				break;
			}
284

Al Viro's avatar
Al Viro committed
285
			written = copy_page_to_iter(buf->page, buf->offset, chars, to);
286
			if (unlikely(written < chars)) {
287
				if (!ret)
288
					ret = -EFAULT;
Linus Torvalds's avatar
Linus Torvalds committed
289 290 291 292 293
				break;
			}
			ret += chars;
			buf->offset += chars;
			buf->len -= chars;
294 295 296 297 298 299 300

			/* Was it a packet buffer? Clean up and exit */
			if (buf->flags & PIPE_BUF_FLAG_PACKET) {
				total_len = chars;
				buf->len = 0;
			}

Linus Torvalds's avatar
Linus Torvalds committed
301
			if (!buf->len) {
302
				pipe_buf_release(pipe, buf);
303
				curbuf = (curbuf + 1) & (pipe->buffers - 1);
304 305
				pipe->curbuf = curbuf;
				pipe->nrbufs = --bufs;
Linus Torvalds's avatar
Linus Torvalds committed
306 307 308 309 310 311 312 313
				do_wakeup = 1;
			}
			total_len -= chars;
			if (!total_len)
				break;	/* common path: read succeeded */
		}
		if (bufs)	/* More to do? */
			continue;
314
		if (!pipe->writers)
Linus Torvalds's avatar
Linus Torvalds committed
315
			break;
316
		if (!pipe->waiting_writers) {
Linus Torvalds's avatar
Linus Torvalds committed
317 318 319 320 321 322 323 324 325 326 327 328 329
			/* syscall merging: Usually we must not sleep
			 * if O_NONBLOCK is set, or if we got some data.
			 * But if a writer sleeps in kernel space, then
			 * we can wait for that data without violating POSIX.
			 */
			if (ret)
				break;
			if (filp->f_flags & O_NONBLOCK) {
				ret = -EAGAIN;
				break;
			}
		}
		if (signal_pending(current)) {
330 331
			if (!ret)
				ret = -ERESTARTSYS;
Linus Torvalds's avatar
Linus Torvalds committed
332 333 334
			break;
		}
		if (do_wakeup) {
335
			wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM);
336
 			kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds's avatar
Linus Torvalds committed
337
		}
338
		pipe_wait(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
339
	}
340
	__pipe_unlock(pipe);
341 342

	/* Signal writers asynchronously that there is more room. */
Linus Torvalds's avatar
Linus Torvalds committed
343
	if (do_wakeup) {
344
		wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM);
345
		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds's avatar
Linus Torvalds committed
346 347 348 349 350 351
	}
	if (ret > 0)
		file_accessed(filp);
	return ret;
}

352 353 354 355 356
static inline int is_packetized(struct file *file)
{
	return (file->f_flags & O_DIRECT) != 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
357
static ssize_t
358
pipe_write(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds's avatar
Linus Torvalds committed
359
{
360
	struct file *filp = iocb->ki_filp;
361
	struct pipe_inode_info *pipe = filp->private_data;
362 363 364
	ssize_t ret = 0;
	int do_wakeup = 0;
	size_t total_len = iov_iter_count(from);
Linus Torvalds's avatar
Linus Torvalds committed
365 366 367 368 369 370
	ssize_t chars;

	/* Null write succeeds. */
	if (unlikely(total_len == 0))
		return 0;

371
	__pipe_lock(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
372

373
	if (!pipe->readers) {
Linus Torvalds's avatar
Linus Torvalds committed
374 375 376 377 378 379 380
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
		goto out;
	}

	/* We try to merge small writes */
	chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
381
	if (pipe->nrbufs && chars != 0) {
382
		int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) &
383
							(pipe->buffers - 1);
384
		struct pipe_buffer *buf = pipe->bufs + lastbuf;
Linus Torvalds's avatar
Linus Torvalds committed
385
		int offset = buf->offset + buf->len;
386

387 388
		if (buf->ops->can_merge && offset + chars <= PAGE_SIZE) {
			ret = pipe_buf_confirm(pipe, buf);
389
			if (ret)
390
				goto out;
391

392 393
			ret = copy_page_from_iter(buf->page, offset, chars, from);
			if (unlikely(ret < chars)) {
394
				ret = -EFAULT;
Linus Torvalds's avatar
Linus Torvalds committed
395
				goto out;
396
			}
397
			do_wakeup = 1;
398
			buf->len += ret;
399
			if (!iov_iter_count(from))
Linus Torvalds's avatar
Linus Torvalds committed
400 401 402 403 404 405
				goto out;
		}
	}

	for (;;) {
		int bufs;
406

407
		if (!pipe->readers) {
Linus Torvalds's avatar
Linus Torvalds committed
408
			send_sig(SIGPIPE, current, 0);
409 410
			if (!ret)
				ret = -EPIPE;
Linus Torvalds's avatar
Linus Torvalds committed
411 412
			break;
		}
413
		bufs = pipe->nrbufs;
414 415
		if (bufs < pipe->buffers) {
			int newbuf = (pipe->curbuf + bufs) & (pipe->buffers-1);
416 417
			struct pipe_buffer *buf = pipe->bufs + newbuf;
			struct page *page = pipe->tmp_page;
418
			int copied;
Linus Torvalds's avatar
Linus Torvalds committed
419 420

			if (!page) {
421
				page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
Linus Torvalds's avatar
Linus Torvalds committed
422 423 424 425
				if (unlikely(!page)) {
					ret = ret ? : -ENOMEM;
					break;
				}
426
				pipe->tmp_page = page;
Linus Torvalds's avatar
Linus Torvalds committed
427
			}
428
			/* Always wake up, even if the copy fails. Otherwise
Linus Torvalds's avatar
Linus Torvalds committed
429 430 431 432 433
			 * we lock up (O_NONBLOCK-)readers that sleep due to
			 * syscall merging.
			 * FIXME! Is this really true?
			 */
			do_wakeup = 1;
434 435
			copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
			if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
436
				if (!ret)
437
					ret = -EFAULT;
Linus Torvalds's avatar
Linus Torvalds committed
438 439
				break;
			}
440
			ret += copied;
Linus Torvalds's avatar
Linus Torvalds committed
441 442 443 444 445

			/* Insert it into the buffer array */
			buf->page = page;
			buf->ops = &anon_pipe_buf_ops;
			buf->offset = 0;
446
			buf->len = copied;
447 448 449 450 451
			buf->flags = 0;
			if (is_packetized(filp)) {
				buf->ops = &packet_pipe_buf_ops;
				buf->flags = PIPE_BUF_FLAG_PACKET;
			}
452 453
			pipe->nrbufs = ++bufs;
			pipe->tmp_page = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
454

455
			if (!iov_iter_count(from))
Linus Torvalds's avatar
Linus Torvalds committed
456 457
				break;
		}
458
		if (bufs < pipe->buffers)
Linus Torvalds's avatar
Linus Torvalds committed
459 460
			continue;
		if (filp->f_flags & O_NONBLOCK) {
461 462
			if (!ret)
				ret = -EAGAIN;
Linus Torvalds's avatar
Linus Torvalds committed
463 464 465
			break;
		}
		if (signal_pending(current)) {
466 467
			if (!ret)
				ret = -ERESTARTSYS;
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470
			break;
		}
		if (do_wakeup) {
471
			wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM);
472
			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Linus Torvalds's avatar
Linus Torvalds committed
473 474
			do_wakeup = 0;
		}
475 476 477
		pipe->waiting_writers++;
		pipe_wait(pipe);
		pipe->waiting_writers--;
Linus Torvalds's avatar
Linus Torvalds committed
478 479
	}
out:
480
	__pipe_unlock(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
481
	if (do_wakeup) {
482
		wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM);
483
		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
Linus Torvalds's avatar
Linus Torvalds committed
484
	}
485
	if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {
486 487 488
		int err = file_update_time(filp);
		if (err)
			ret = err;
489
		sb_end_write(file_inode(filp)->i_sb);
490
	}
Linus Torvalds's avatar
Linus Torvalds committed
491 492 493
	return ret;
}

494
static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
Linus Torvalds's avatar
Linus Torvalds committed
495
{
496
	struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
497 498 499 500
	int count, buf, nrbufs;

	switch (cmd) {
		case FIONREAD:
501
			__pipe_lock(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
502
			count = 0;
503 504
			buf = pipe->curbuf;
			nrbufs = pipe->nrbufs;
Linus Torvalds's avatar
Linus Torvalds committed
505
			while (--nrbufs >= 0) {
506
				count += pipe->bufs[buf].len;
507
				buf = (buf+1) & (pipe->buffers - 1);
Linus Torvalds's avatar
Linus Torvalds committed
508
			}
509
			__pipe_unlock(pipe);
510

Linus Torvalds's avatar
Linus Torvalds committed
511 512
			return put_user(count, (int __user *)arg);
		default:
513
			return -ENOIOCTLCMD;
Linus Torvalds's avatar
Linus Torvalds committed
514 515 516 517 518 519 520 521
	}
}

/* No kernel lock held - fine */
static unsigned int
pipe_poll(struct file *filp, poll_table *wait)
{
	unsigned int mask;
522
	struct pipe_inode_info *pipe = filp->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
523 524
	int nrbufs;

525
	poll_wait(filp, &pipe->wait, wait);
Linus Torvalds's avatar
Linus Torvalds committed
526 527

	/* Reading only -- no need for acquiring the semaphore.  */
528
	nrbufs = pipe->nrbufs;
Linus Torvalds's avatar
Linus Torvalds committed
529 530 531
	mask = 0;
	if (filp->f_mode & FMODE_READ) {
		mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
532
		if (!pipe->writers && filp->f_version != pipe->w_counter)
Linus Torvalds's avatar
Linus Torvalds committed
533 534 535 536
			mask |= POLLHUP;
	}

	if (filp->f_mode & FMODE_WRITE) {
537
		mask |= (nrbufs < pipe->buffers) ? POLLOUT | POLLWRNORM : 0;
538 539 540 541
		/*
		 * Most Unices do not set POLLERR for FIFOs but on Linux they
		 * behave exactly like pipes for poll().
		 */
542
		if (!pipe->readers)
Linus Torvalds's avatar
Linus Torvalds committed
543 544 545 546 547 548
			mask |= POLLERR;
	}

	return mask;
}

549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
{
	int kill = 0;

	spin_lock(&inode->i_lock);
	if (!--pipe->files) {
		inode->i_pipe = NULL;
		kill = 1;
	}
	spin_unlock(&inode->i_lock);

	if (kill)
		free_pipe_info(pipe);
}

Linus Torvalds's avatar
Linus Torvalds committed
564
static int
565
pipe_release(struct inode *inode, struct file *file)
Linus Torvalds's avatar
Linus Torvalds committed
566
{
567
	struct pipe_inode_info *pipe = file->private_data;
568

569
	__pipe_lock(pipe);
570 571 572 573
	if (file->f_mode & FMODE_READ)
		pipe->readers--;
	if (file->f_mode & FMODE_WRITE)
		pipe->writers--;
574

575
	if (pipe->readers || pipe->writers) {
576
		wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM | POLLERR | POLLHUP);
577 578
		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
Linus Torvalds's avatar
Linus Torvalds committed
579
	}
580
	__pipe_unlock(pipe);
581

582
	put_pipe_info(inode, pipe);
Linus Torvalds's avatar
Linus Torvalds committed
583 584 585 586
	return 0;
}

static int
587
pipe_fasync(int fd, struct file *filp, int on)
Linus Torvalds's avatar
Linus Torvalds committed
588
{
589
	struct pipe_inode_info *pipe = filp->private_data;
590
	int retval = 0;
Linus Torvalds's avatar
Linus Torvalds committed
591

592
	__pipe_lock(pipe);
593 594 595
	if (filp->f_mode & FMODE_READ)
		retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
	if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
596
		retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
597 598
		if (retval < 0 && (filp->f_mode & FMODE_READ))
			/* this can happen only if on == T */
599 600
			fasync_helper(-1, filp, 0, &pipe->fasync_readers);
	}
601
	__pipe_unlock(pipe);
602
	return retval;
Linus Torvalds's avatar
Linus Torvalds committed
603 604
}

605
static unsigned long account_pipe_buffers(struct user_struct *user,
606 607
                                 unsigned long old, unsigned long new)
{
608
	return atomic_long_add_return(new - old, &user->pipe_bufs);
609 610
}

611
static bool too_many_pipe_buffers_soft(unsigned long user_bufs)
612
{
613
	return pipe_user_pages_soft && user_bufs >= pipe_user_pages_soft;
614 615
}

616
static bool too_many_pipe_buffers_hard(unsigned long user_bufs)
617
{
618
	return pipe_user_pages_hard && user_bufs >= pipe_user_pages_hard;
619 620
}

621
struct pipe_inode_info *alloc_pipe_info(void)
622
{
623
	struct pipe_inode_info *pipe;
624 625
	unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
	struct user_struct *user = get_current_user();
626
	unsigned long user_bufs;
627

628
	pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
629 630 631
	if (pipe == NULL)
		goto out_free_uid;

632 633 634
	if (pipe_bufs * PAGE_SIZE > pipe_max_size && !capable(CAP_SYS_RESOURCE))
		pipe_bufs = pipe_max_size >> PAGE_SHIFT;

635
	user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
636

637 638
	if (too_many_pipe_buffers_soft(user_bufs)) {
		user_bufs = account_pipe_buffers(user, pipe_bufs, 1);
639
		pipe_bufs = 1;
640
	}
641

642
	if (too_many_pipe_buffers_hard(user_bufs))
643 644 645 646 647
		goto out_revert_acct;

	pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
			     GFP_KERNEL_ACCOUNT);

648 649 650 651 652 653 654
	if (pipe->bufs) {
		init_waitqueue_head(&pipe->wait);
		pipe->r_counter = pipe->w_counter = 1;
		pipe->buffers = pipe_bufs;
		pipe->user = user;
		mutex_init(&pipe->mutex);
		return pipe;
655 656
	}

657
out_revert_acct:
658
	(void) account_pipe_buffers(user, pipe_bufs, 0);
659 660 661
	kfree(pipe);
out_free_uid:
	free_uid(user);
662
	return NULL;
663 664
}

665
void free_pipe_info(struct pipe_inode_info *pipe)
Linus Torvalds's avatar
Linus Torvalds committed
666 667 668
{
	int i;

669
	(void) account_pipe_buffers(pipe->user, pipe->buffers, 0);
670
	free_uid(pipe->user);
671
	for (i = 0; i < pipe->buffers; i++) {
672
		struct pipe_buffer *buf = pipe->bufs + i;
Linus Torvalds's avatar
Linus Torvalds committed
673
		if (buf->ops)
674
			pipe_buf_release(pipe, buf);
Linus Torvalds's avatar
Linus Torvalds committed
675
	}
676 677
	if (pipe->tmp_page)
		__free_page(pipe->tmp_page);
678
	kfree(pipe->bufs);
679
	kfree(pipe);
Linus Torvalds's avatar
Linus Torvalds committed
680 681
}

682
static struct vfsmount *pipe_mnt __read_mostly;
683

684 685 686 687 688 689
/*
 * pipefs_dname() is called from d_path().
 */
static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
{
	return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
690
				d_inode(dentry)->i_ino);
691 692
}

693
static const struct dentry_operations pipefs_dentry_operations = {
694
	.d_dname	= pipefs_dname,
Linus Torvalds's avatar
Linus Torvalds committed
695 696 697 698
};

static struct inode * get_pipe_inode(void)
{
699
	struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
700
	struct pipe_inode_info *pipe;
Linus Torvalds's avatar
Linus Torvalds committed
701 702 703 704

	if (!inode)
		goto fail_inode;

705 706
	inode->i_ino = get_next_ino();

707
	pipe = alloc_pipe_info();
708
	if (!pipe)
Linus Torvalds's avatar
Linus Torvalds committed
709
		goto fail_iput;
710

711 712
	inode->i_pipe = pipe;
	pipe->files = 2;
713
	pipe->readers = pipe->writers = 1;
714
	inode->i_fop = &pipefifo_fops;
Linus Torvalds's avatar
Linus Torvalds committed
715 716 717 718 719 720 721 722 723

	/*
	 * Mark the inode dirty from the very beginning,
	 * that way it will never be moved to the dirty
	 * list because "mark_inode_dirty()" will think
	 * that it already _is_ on the dirty list.
	 */
	inode->i_state = I_DIRTY;
	inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
724 725
	inode->i_uid = current_fsuid();
	inode->i_gid = current_fsgid();
726
	inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
727

Linus Torvalds's avatar
Linus Torvalds committed
728 729 730 731
	return inode;

fail_iput:
	iput(inode);
732

Linus Torvalds's avatar
Linus Torvalds committed
733 734 735 736
fail_inode:
	return NULL;
}

Al Viro's avatar
Al Viro committed
737
int create_pipe_files(struct file **res, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
738
{
739
	int err;
Al Viro's avatar
Al Viro committed
740
	struct inode *inode = get_pipe_inode();
741
	struct file *f;
742
	struct path path;
Linus Torvalds's avatar
Linus Torvalds committed
743 744

	if (!inode)
Al Viro's avatar
Al Viro committed
745
		return -ENFILE;
Linus Torvalds's avatar
Linus Torvalds committed
746

747
	err = -ENOMEM;
748
	path.dentry = d_alloc_pseudo(pipe_mnt->mnt_sb, &empty_name);
749
	if (!path.dentry)
750
		goto err_inode;
751
	path.mnt = mntget(pipe_mnt);
752

753
	d_instantiate(path.dentry, inode);
754

755
	f = alloc_file(&path, FMODE_WRITE, &pipefifo_fops);
756 757
	if (IS_ERR(f)) {
		err = PTR_ERR(f);
758
		goto err_dentry;
759
	}
760

761
	f->f_flags = O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT));
762
	f->private_data = inode->i_pipe;
763

764
	res[0] = alloc_file(&path, FMODE_READ, &pipefifo_fops);
765 766
	if (IS_ERR(res[0])) {
		err = PTR_ERR(res[0]);
Al Viro's avatar
Al Viro committed
767
		goto err_file;
768
	}
Al Viro's avatar
Al Viro committed
769 770

	path_get(&path);
771
	res[0]->private_data = inode->i_pipe;
Al Viro's avatar
Al Viro committed
772 773 774
	res[0]->f_flags = O_RDONLY | (flags & O_NONBLOCK);
	res[1] = f;
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
775

Al Viro's avatar
Al Viro committed
776 777 778
err_file:
	put_filp(f);
err_dentry:
779
	free_pipe_info(inode->i_pipe);
780
	path_put(&path);
Al Viro's avatar
Al Viro committed
781
	return err;
782

Al Viro's avatar
Al Viro committed
783
err_inode:
784
	free_pipe_info(inode->i_pipe);
Linus Torvalds's avatar
Linus Torvalds committed
785
	iput(inode);
Al Viro's avatar
Al Viro committed
786
	return err;
787 788
}

789
static int __do_pipe_flags(int *fd, struct file **files, int flags)
790 791 792 793
{
	int error;
	int fdw, fdr;

794
	if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT))
Ulrich Drepper's avatar
Ulrich Drepper committed
795 796
		return -EINVAL;

Al Viro's avatar
Al Viro committed
797 798 799
	error = create_pipe_files(files, flags);
	if (error)
		return error;
800

Ulrich Drepper's avatar
Ulrich Drepper committed
801
	error = get_unused_fd_flags(flags);
802 803 804 805
	if (error < 0)
		goto err_read_pipe;
	fdr = error;

Ulrich Drepper's avatar
Ulrich Drepper committed
806
	error = get_unused_fd_flags(flags);
807 808 809 810
	if (error < 0)
		goto err_fdr;
	fdw = error;

Al Viro's avatar
Al Viro committed
811
	audit_fd_pair(fdr, fdw);
812 813 814 815 816 817 818
	fd[0] = fdr;
	fd[1] = fdw;
	return 0;

 err_fdr:
	put_unused_fd(fdr);
 err_read_pipe:
Al Viro's avatar
Al Viro committed
819 820
	fput(files[0]);
	fput(files[1]);
821
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
822 823
}

824 825 826 827 828 829 830 831 832 833 834
int do_pipe_flags(int *fd, int flags)
{
	struct file *files[2];
	int error = __do_pipe_flags(fd, files, flags);
	if (!error) {
		fd_install(fd[0], files[0]);
		fd_install(fd[1], files[1]);
	}
	return error;
}

835 836 837 838
/*
 * sys_pipe() is the normal C calling standard for creating
 * a pipe. It's not the way Unix traditionally does this, though.
 */
839
SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
840
{
841
	struct file *files[2];
842 843 844
	int fd[2];
	int error;

845
	error = __do_pipe_flags(fd, files, flags);
846
	if (!error) {
847 848 849 850 851
		if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
			fput(files[0]);
			fput(files[1]);
			put_unused_fd(fd[0]);
			put_unused_fd(fd[1]);
852
			error = -EFAULT;
853 854 855
		} else {
			fd_install(fd[0], files[0]);
			fd_install(fd[1], files[1]);
856
		}
857 858 859 860
	}
	return error;
}

861
SYSCALL_DEFINE1(pipe, int __user *, fildes)
Ulrich Drepper's avatar
Ulrich Drepper committed
862 863 864 865
{
	return sys_pipe2(fildes, 0);
}

866
static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
Al Viro's avatar
Al Viro committed
867 868 869 870
{
	int cur = *cnt;	

	while (cur == *cnt) {
871
		pipe_wait(pipe);
Al Viro's avatar
Al Viro committed
872 873 874 875 876 877
		if (signal_pending(current))
			break;
	}
	return cur == *cnt ? -ERESTARTSYS : 0;
}

878
static void wake_up_partner(struct pipe_inode_info *pipe)
Al Viro's avatar
Al Viro committed
879
{
880
	wake_up_interruptible(&pipe->wait);
Al Viro's avatar
Al Viro committed
881 882 883 884 885
}

static int fifo_open(struct inode *inode, struct file *filp)
{
	struct pipe_inode_info *pipe;
886
	bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;
Al Viro's avatar
Al Viro committed
887 888
	int ret;

889 890 891 892 893 894 895 896 897
	filp->f_version = 0;

	spin_lock(&inode->i_lock);
	if (inode->i_pipe) {
		pipe = inode->i_pipe;
		pipe->files++;
		spin_unlock(&inode->i_lock);
	} else {
		spin_unlock(&inode->i_lock);
898
		pipe = alloc_pipe_info();
Al Viro's avatar
Al Viro committed
899
		if (!pipe)
900 901 902 903 904 905
			return -ENOMEM;
		pipe->files = 1;
		spin_lock(&inode->i_lock);
		if (unlikely(inode->i_pipe)) {
			inode->i_pipe->files++;
			spin_unlock(&inode->i_lock);
906
			free_pipe_info(pipe);
907 908 909 910 911
			pipe = inode->i_pipe;
		} else {
			inode->i_pipe = pipe;
			spin_unlock(&inode->i_lock);
		}
Al Viro's avatar
Al Viro committed
912
	}
913
	filp->private_data = pipe;
914 915
	/* OK, we have a pipe and it's pinned down */

916
	__pipe_lock(pipe);
Al Viro's avatar
Al Viro committed
917 918 919 920 921 922 923 924 925 926 927 928 929

	/* We can only do regular read/write on fifos */
	filp->f_mode &= (FMODE_READ | FMODE_WRITE);

	switch (filp->f_mode) {
	case FMODE_READ:
	/*
	 *  O_RDONLY
	 *  POSIX.1 says that O_NONBLOCK means return with the FIFO
	 *  opened, even when there is no process writing the FIFO.
	 */
		pipe->r_counter++;
		if (pipe->readers++ == 0)
930
			wake_up_partner(pipe);
Al Viro's avatar
Al Viro committed
931

932
		if (!is_pipe && !pipe->writers) {
Al Viro's avatar
Al Viro committed
933 934 935 936 937
			if ((filp->f_flags & O_NONBLOCK)) {
				/* suppress POLLHUP until we have
				 * seen a writer */
				filp->f_version = pipe->w_counter;
			} else {
938
				if (wait_for_partner(pipe, &pipe->w_counter))
Al Viro's avatar
Al Viro committed
939 940 941 942 943 944 945 946 947 948 949 950
					goto err_rd;
			}
		}
		break;
	
	case FMODE_WRITE:
	/*
	 *  O_WRONLY
	 *  POSIX.1 says that O_NONBLOCK means return -1 with
	 *  errno=ENXIO when there is no process reading the FIFO.
	 */
		ret = -ENXIO;
951
		if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
Al Viro's avatar
Al Viro committed
952 953 954 955
			goto err;

		pipe->w_counter++;
		if (!pipe->writers++)
956
			wake_up_partner(pipe);
Al Viro's avatar
Al Viro committed
957

958
		if (!is_pipe && !pipe->readers) {
959
			if (wait_for_partner(pipe, &pipe->r_counter))
Al Viro's avatar
Al Viro committed
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
				goto err_wr;
		}
		break;
	
	case FMODE_READ | FMODE_WRITE:
	/*
	 *  O_RDWR
	 *  POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
	 *  This implementation will NEVER block on a O_RDWR open, since
	 *  the process can at least talk to itself.
	 */

		pipe->readers++;
		pipe->writers++;
		pipe->r_counter++;
		pipe->w_counter++;
		if (pipe->readers == 1 || pipe->writers == 1)
977
			wake_up_partner(pipe);
Al Viro's avatar
Al Viro committed
978 979 980 981 982 983 984 985
		break;

	default:
		ret = -EINVAL;
		goto err;
	}

	/* Ok! */
986
	__pipe_unlock(pipe);
Al Viro's avatar
Al Viro committed
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001
	return 0;

err_rd:
	if (!--pipe->readers)
		wake_up_interruptible(&pipe->wait);
	ret = -ERESTARTSYS;
	goto err;

err_wr:
	if (!--pipe->writers)
		wake_up_interruptible(&pipe->wait);
	ret = -ERESTARTSYS;
	goto err;

err:
1002
	__pipe_unlock(pipe);
1003 1004

	put_pipe_info(inode, pipe);
Al Viro's avatar
Al Viro committed
1005 1006 1007
	return ret;
}

1008 1009 1010
const struct file_operations pipefifo_fops = {
	.open		= fifo_open,
	.llseek		= no_llseek,
Al Viro's avatar
Al Viro committed
1011
	.read_iter	= pipe_read,
1012
	.write_iter	= pipe_write,
1013 1014 1015 1016
	.poll		= pipe_poll,
	.unlocked_ioctl	= pipe_ioctl,
	.release	= pipe_release,
	.fasync		= pipe_fasync,
Al Viro's avatar
Al Viro committed
1017 1018
};

1019 1020
/*
 * Currently we rely on the pipe array holding a power-of-2 number
1021
 * of pages. Returns 0 on error.
1022
 */
1023
unsigned int round_pipe_size(unsigned int size)
1024 1025 1026
{
	unsigned long nr_pages;

1027 1028 1029
	if (size < pipe_min_size)
		size = pipe_min_size;

1030
	nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1031 1032 1033
	if (nr_pages == 0)
		return 0;

1034 1035 1036
	return roundup_pow_of_two(nr_pages) << PAGE_SHIFT;
}

1037 1038 1039 1040
/*
 * Allocate a new array of pipe buffers and copy the info over. Returns the
 * pipe size if successful, or return -ERROR on error.
 */
1041
static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
1042 1043
{
	struct pipe_buffer *bufs;
1044
	unsigned int size, nr_pages;
1045
	unsigned long user_bufs;
1046
	long ret = 0;
1047 1048

	size = round_pipe_size(arg);
1049 1050
	if (size == 0)
		return -EINVAL;
1051 1052 1053 1054 1055
	nr_pages = size >> PAGE_SHIFT;

	if (!nr_pages)
		return -EINVAL;

1056 1057 1058 1059 1060 1061 1062 1063 1064
	/*
	 * If trying to increase the pipe capacity, check that an
	 * unprivileged user is not trying to exceed various limits
	 * (soft limit check here, hard limit check just below).
	 * Decreasing the pipe capacity is always permitted, even
	 * if the user is currently over a limit.
	 */
	if (nr_pages > pipe->buffers &&
			size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
1065 1066
		return -EPERM;

1067
	user_bufs = account_pipe_buffers(pipe->user, pipe->buffers, nr_pages);
1068 1069

	if (nr_pages > pipe->buffers &&
1070 1071
			(too_many_pipe_buffers_hard(user_bufs) ||
			 too_many_pipe_buffers_soft(user_bufs)) &&
1072 1073 1074 1075
			!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
		ret = -EPERM;
		goto out_revert_acct;
	}
1076 1077 1078 1079 1080 1081 1082

	/*
	 * We can shrink the pipe, if arg >= pipe->nrbufs. Since we don't
	 * expect a lot of shrink+grow operations, just free and allocate
	 * again like we would do for growing. If the pipe currently
	 * contains more buffers than arg, then return busy.
	 */
1083 1084 1085 1086
	if (nr_pages < pipe->nrbufs) {
		ret = -EBUSY;
		goto out_revert_acct;
	}
1087

1088 1089
	bufs = kcalloc(nr_pages, sizeof(*bufs),
		       GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1090 1091 1092 1093
	if (unlikely(!bufs)) {
		ret = -ENOMEM;
		goto out_revert_acct;
	}
1094 1095 1096 1097 1098 1099

	/*
	 * The pipe array wraps around, so just start the new one at zero
	 * and adjust the indexes.
	 */
	if (pipe->nrbufs) {
1100 1101
		unsigned int tail;
		unsigned int head;
1102

1103 1104 1105 1106 1107 1108 1109
		tail = pipe->curbuf + pipe->nrbufs;
		if (tail < pipe->buffers)
			tail = 0;
		else
			tail &= (pipe->buffers - 1);

		head = pipe->nrbufs - tail;
1110 1111 1112
		if (head)
			memcpy(bufs, pipe->bufs + pipe->curbuf, head * sizeof(struct pipe_buffer));
		if (tail)
1113
			memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer));
1114 1115 1116 1117 1118
	}

	pipe->curbuf = 0;
	kfree(pipe->bufs);
	pipe->bufs = bufs;
1119 1120
	pipe->buffers = nr_pages;
	return nr_pages * PAGE_SIZE;
1121 1122

out_revert_acct:
1123
	(void) account_pipe_buffers(pipe->user, nr_pages, pipe->buffers);
1124
	return ret;
1125 1126
}

1127
/*
1128
 * This should work even if CONFIG_PROC_FS isn't set, as proc_dopipe_max_size
1129 1130 1131 1132 1133
 * will return an error.
 */
int pipe_proc_fn(struct ctl_table *table, int write, void __user *buf,
		 size_t *lenp, loff_t *ppos)
{
1134
	return proc_dopipe_max_size(table, write, buf, lenp, ppos);
1135 1136
}

1137 1138 1139 1140 1141 1142 1143
/*
 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
 * location, so checking ->i_pipe is not enough to verify that this is a
 * pipe.
 */
struct pipe_inode_info *get_pipe_info(struct file *file)
{
1144
	return file->f_op == &pipefifo_fops ? file->private_data : NULL;
1145 1146
}

1147 1148 1149 1150 1151
long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
{
	struct pipe_inode_info *pipe;
	long ret;

1152
	pipe = get_pipe_info(file);
1153 1154 1155
	if (!pipe)
		return -EBADF;

1156
	__pipe_lock(pipe);
1157 1158

	switch (cmd) {
1159 1160
	case F_SETPIPE_SZ:
		ret = pipe_set_size(pipe, arg);
1161 1162
		break;
	case F_GETPIPE_SZ:
1163
		ret = pipe->buffers * PAGE_SIZE;
1164 1165 1166 1167 1168 1169
		break;
	default:
		ret = -EINVAL;
		break;
	}

1170
	__pipe_unlock(pipe);
1171 1172 1173
	return ret;
}

1174 1175
static const struct super_operations pipefs_ops = {
	.destroy_inode = free_inode_nonrcu,
1176
	.statfs = simple_statfs,
1177 1178
};

Linus Torvalds's avatar
Linus Torvalds committed
1179 1180 1181 1182 1183 1184
/*
 * pipefs should _never_ be mounted by userland - too much of security hassle,
 * no real gain from having the whole whorehouse mounted. So we don't need
 * any operations on the root directory. However, we need a non-trivial
 * d_name - pipe: will go nicely and kill the special-casing in procfs.
 */
Al Viro's avatar
Al Viro committed
1185 1186
static struct dentry *pipefs_mount(struct file_system_type *fs_type,
			 int flags, const char *dev_name, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
1187
{
1188 1189
	return mount_pseudo(fs_type, "pipe:", &pipefs_ops,
			&pipefs_dentry_operations, PIPEFS_MAGIC);
Linus Torvalds's avatar
Linus Torvalds committed
1190 1191 1192 1193
}

static struct file_system_type pipe_fs_type = {
	.name		= "pipefs",
Al Viro's avatar
Al Viro committed
1194
	.mount		= pipefs_mount,
Linus Torvalds's avatar
Linus Torvalds committed
1195 1196 1197 1198 1199 1200
	.kill_sb	= kill_anon_super,
};

static int __init init_pipe_fs(void)
{
	int err = register_filesystem(&pipe_fs_type);
1201

Linus Torvalds's avatar
Linus Torvalds committed
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
	if (!err) {
		pipe_mnt = kern_mount(&pipe_fs_type);
		if (IS_ERR(pipe_mnt)) {
			err = PTR_ERR(pipe_mnt);
			unregister_filesystem(&pipe_fs_type);
		}
	}
	return err;
}

fs_initcall(init_pipe_fs);