shm.c 33.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * linux/ipc/shm.c
 * Copyright (C) 1992, 1993 Krishna Balasubramanian
 *	 Many improvements/fixes by Bruno Haible.
 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
 *
 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
 *
Steve Grubb's avatar
Steve Grubb committed
16 17
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
Kirill Korotaev's avatar
Kirill Korotaev committed
18 19 20 21
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
22 23 24
 *
 * Better ipc lock (kern_ipc_perm.lock) handling
 * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27 28 29 30 31 32 33 34 35 36 37
 */

#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/mman.h>
#include <linux/shmem_fs.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
38
#include <linux/capability.h>
39
#include <linux/ptrace.h>
40
#include <linux/seq_file.h>
Nadia Derbey's avatar
Nadia Derbey committed
41
#include <linux/rwsem.h>
Kirill Korotaev's avatar
Kirill Korotaev committed
42
#include <linux/nsproxy.h>
43
#include <linux/mount.h>
44
#include <linux/ipc_namespace.h>
45

Paul McQuade's avatar
Paul McQuade committed
46
#include <linux/uaccess.h>
Linus Torvalds's avatar
Linus Torvalds committed
47 48 49

#include "util.h"

50 51 52 53 54 55 56 57 58
struct shm_file_data {
	int id;
	struct ipc_namespace *ns;
	struct file *file;
	const struct vm_operations_struct *vm_ops;
};

#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))

59
static const struct file_operations shm_file_operations;
60
static const struct vm_operations_struct shm_vm_ops;
Linus Torvalds's avatar
Linus Torvalds committed
61

62
#define shm_ids(ns)	((ns)->ids[IPC_SHM_IDS])
Linus Torvalds's avatar
Linus Torvalds committed
63

Kirill Korotaev's avatar
Kirill Korotaev committed
64 65
#define shm_unlock(shp)			\
	ipc_unlock(&(shp)->shm_perm)
Linus Torvalds's avatar
Linus Torvalds committed
66

Nadia Derbey's avatar
Nadia Derbey committed
67
static int newseg(struct ipc_namespace *, struct ipc_params *);
68 69
static void shm_open(struct vm_area_struct *vma);
static void shm_close(struct vm_area_struct *vma);
Manfred Spraul's avatar
Manfred Spraul committed
70
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
Linus Torvalds's avatar
Linus Torvalds committed
71
#ifdef CONFIG_PROC_FS
72
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
Linus Torvalds's avatar
Linus Torvalds committed
73 74
#endif

75
void shm_init_ns(struct ipc_namespace *ns)
Kirill Korotaev's avatar
Kirill Korotaev committed
76 77 78 79
{
	ns->shm_ctlmax = SHMMAX;
	ns->shm_ctlall = SHMALL;
	ns->shm_ctlmni = SHMMNI;
80
	ns->shm_rmid_forced = 0;
Kirill Korotaev's avatar
Kirill Korotaev committed
81
	ns->shm_tot = 0;
WANG Cong's avatar
WANG Cong committed
82
	ipc_init_ids(&shm_ids(ns));
Kirill Korotaev's avatar
Kirill Korotaev committed
83 84
}

Nadia Derbey's avatar
Nadia Derbey committed
85
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
86 87
 * Called with shm_ids.rwsem (writer) and the shp structure locked.
 * Only shm_ids.rwsem remains locked on exit.
Nadia Derbey's avatar
Nadia Derbey committed
88
 */
89
static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
Kirill Korotaev's avatar
Kirill Korotaev committed
90
{
91
	struct shmid_kernel *shp;
92

93 94
	shp = container_of(ipcp, struct shmid_kernel, shm_perm);

Manfred Spraul's avatar
Manfred Spraul committed
95
	if (shp->shm_nattch) {
Kirill Korotaev's avatar
Kirill Korotaev committed
96 97 98 99 100 101 102 103
		shp->shm_perm.mode |= SHM_DEST;
		/* Do not find it any more */
		shp->shm_perm.key = IPC_PRIVATE;
		shm_unlock(shp);
	} else
		shm_destroy(ns, shp);
}

104
#ifdef CONFIG_IPC_NS
Kirill Korotaev's avatar
Kirill Korotaev committed
105 106
void shm_exit_ns(struct ipc_namespace *ns)
{
107
	free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
108
	idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
Kirill Korotaev's avatar
Kirill Korotaev committed
109
}
110
#endif
Linus Torvalds's avatar
Linus Torvalds committed
111

112
static int __init ipc_ns_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
113
{
114
	shm_init_ns(&init_ipc_ns);
115 116 117 118 119
	return 0;
}

pure_initcall(ipc_ns_init);

Manfred Spraul's avatar
Manfred Spraul committed
120
void __init shm_init(void)
121
{
122
	ipc_init_proc_interface("sysvipc/shm",
123 124 125 126 127
#if BITS_PER_LONG <= 32
				"       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
#else
				"       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
#endif
Kirill Korotaev's avatar
Kirill Korotaev committed
128
				IPC_SHM_IDS, sysvipc_shm_proc_show);
Linus Torvalds's avatar
Linus Torvalds committed
129 130
}

131 132
static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
{
133
	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150

	if (IS_ERR(ipcp))
		return ERR_CAST(ipcp);

	return container_of(ipcp, struct shmid_kernel, shm_perm);
}

static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
{
	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);

	if (IS_ERR(ipcp))
		return ERR_CAST(ipcp);

	return container_of(ipcp, struct shmid_kernel, shm_perm);
}

Nadia Derbey's avatar
Nadia Derbey committed
151
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
152
 * shm_lock_(check_) routines are called in the paths where the rwsem
Nadia Derbey's avatar
Nadia Derbey committed
153
 * is not necessarily held.
Nadia Derbey's avatar
Nadia Derbey committed
154
 */
155
static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
Linus Torvalds's avatar
Linus Torvalds committed
156
{
Nadia Derbey's avatar
Nadia Derbey committed
157 158
	struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id);

159
	/*
160 161 162
	 * Callers of shm_lock() must validate the status of the returned ipc
	 * object pointer (as returned by ipc_lock()), and error out as
	 * appropriate.
163
	 */
164 165
	if (IS_ERR(ipcp))
		return (void *)ipcp;
Nadia Derbey's avatar
Nadia Derbey committed
166
	return container_of(ipcp, struct shmid_kernel, shm_perm);
167 168
}

169 170 171
static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
{
	rcu_read_lock();
172
	ipc_lock_object(&ipcp->shm_perm);
173 174
}

Davidlohr Bueso's avatar
Davidlohr Bueso committed
175 176
static void shm_rcu_free(struct rcu_head *head)
{
177 178 179 180
	struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,
							rcu);
	struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,
							shm_perm);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
181
	security_shm_free(shp);
182
	kvfree(shp);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
183 184
}

Nadia Derbey's avatar
Nadia Derbey committed
185
static inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
Linus Torvalds's avatar
Linus Torvalds committed
186
{
187
	list_del(&s->shm_clist);
Nadia Derbey's avatar
Nadia Derbey committed
188
	ipc_rmid(&shm_ids(ns), &s->shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
189 190 191
}


192
static int __shm_open(struct vm_area_struct *vma)
Kirill Korotaev's avatar
Kirill Korotaev committed
193
{
194 195
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds's avatar
Linus Torvalds committed
196 197
	struct shmid_kernel *shp;

198
	shp = shm_lock(sfd->ns, sfd->id);
199 200 201 202

	if (IS_ERR(shp))
		return PTR_ERR(shp);

Linus Torvalds's avatar
Linus Torvalds committed
203
	shp->shm_atim = get_seconds();
204
	shp->shm_lprid = task_tgid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
205 206
	shp->shm_nattch++;
	shm_unlock(shp);
207 208 209 210 211 212 213 214 215 216 217 218
	return 0;
}

/* This is called by fork, once for every shm attach. */
static void shm_open(struct vm_area_struct *vma)
{
	int err = __shm_open(vma);
	/*
	 * We raced in the idr lookup or with shm_destroy().
	 * Either way, the ID is busted.
	 */
	WARN_ON_ONCE(err);
Linus Torvalds's avatar
Linus Torvalds committed
219 220 221 222 223
}

/*
 * shm_destroy - free the struct shmid_kernel
 *
Nadia Derbey's avatar
Nadia Derbey committed
224
 * @ns: namespace
Linus Torvalds's avatar
Linus Torvalds committed
225 226
 * @shp: struct to free
 *
Davidlohr Bueso's avatar
Davidlohr Bueso committed
227
 * It has to be called with shp and shm_ids.rwsem (writer) locked,
Linus Torvalds's avatar
Linus Torvalds committed
228 229
 * but returns with shp unlocked and freed.
 */
Kirill Korotaev's avatar
Kirill Korotaev committed
230
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
Linus Torvalds's avatar
Linus Torvalds committed
231
{
232 233 234 235
	struct file *shm_file;

	shm_file = shp->shm_file;
	shp->shm_file = NULL;
Kirill Korotaev's avatar
Kirill Korotaev committed
236
	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
Nadia Derbey's avatar
Nadia Derbey committed
237
	shm_rmid(ns, shp);
Linus Torvalds's avatar
Linus Torvalds committed
238
	shm_unlock(shp);
239 240
	if (!is_file_hugepages(shm_file))
		shmem_lock(shm_file, 0, shp->mlock_user);
241
	else if (shp->mlock_user)
242 243
		user_shm_unlock(i_size_read(file_inode(shm_file)),
				shp->mlock_user);
244
	fput(shm_file);
245
	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
246 247
}

248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
/*
 * shm_may_destroy - identifies whether shm segment should be destroyed now
 *
 * Returns true if and only if there are no active users of the segment and
 * one of the following is true:
 *
 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 *
 * 2) sysctl kernel.shm_rmid_forced is set to 1.
 */
static bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
{
	return (shp->shm_nattch == 0) &&
	       (ns->shm_rmid_forced ||
		(shp->shm_perm.mode & SHM_DEST));
}

Linus Torvalds's avatar
Linus Torvalds committed
265
/*
266
 * remove the attach descriptor vma.
Linus Torvalds's avatar
Linus Torvalds committed
267 268 269 270
 * free memory for segment if it is marked destroyed.
 * The descriptor has already been removed from the current->mm->mmap list
 * and will later be kfree()d.
 */
271
static void shm_close(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
272
{
Manfred Spraul's avatar
Manfred Spraul committed
273
	struct file *file = vma->vm_file;
274
	struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds's avatar
Linus Torvalds committed
275
	struct shmid_kernel *shp;
276
	struct ipc_namespace *ns = sfd->ns;
Kirill Korotaev's avatar
Kirill Korotaev committed
277

Davidlohr Bueso's avatar
Davidlohr Bueso committed
278
	down_write(&shm_ids(ns).rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
279
	/* remove from the list of attaches of the shm segment */
Nadia Derbey's avatar
Nadia Derbey committed
280
	shp = shm_lock(ns, sfd->id);
281 282 283 284 285 286 287 288

	/*
	 * We raced in the idr lookup or with shm_destroy().
	 * Either way, the ID is busted.
	 */
	if (WARN_ON_ONCE(IS_ERR(shp)))
		goto done; /* no-op */

289
	shp->shm_lprid = task_tgid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
290 291
	shp->shm_dtim = get_seconds();
	shp->shm_nattch--;
292 293 294 295
	if (shm_may_destroy(ns, shp))
		shm_destroy(ns, shp);
	else
		shm_unlock(shp);
296
done:
Davidlohr Bueso's avatar
Davidlohr Bueso committed
297
	up_write(&shm_ids(ns).rwsem);
298 299
}

Davidlohr Bueso's avatar
Davidlohr Bueso committed
300
/* Called with ns->shm_ids(ns).rwsem locked */
301 302 303
static int shm_try_destroy_orphaned(int id, void *p, void *data)
{
	struct ipc_namespace *ns = data;
304 305
	struct kern_ipc_perm *ipcp = p;
	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
306 307 308 309

	/*
	 * We want to destroy segments without users and with already
	 * exit'ed originating process.
310
	 *
Davidlohr Bueso's avatar
Davidlohr Bueso committed
311
	 * As shp->* are changed under rwsem, it's safe to skip shp locking.
312
	 */
313
	if (shp->shm_creator != NULL)
314 315
		return 0;

316 317
	if (shm_may_destroy(ns, shp)) {
		shm_lock_by_ptr(shp);
Kirill Korotaev's avatar
Kirill Korotaev committed
318
		shm_destroy(ns, shp);
319
	}
320 321 322 323 324
	return 0;
}

void shm_destroy_orphaned(struct ipc_namespace *ns)
{
Davidlohr Bueso's avatar
Davidlohr Bueso committed
325
	down_write(&shm_ids(ns).rwsem);
Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
326
	if (shm_ids(ns).in_use)
327
		idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
328
	up_write(&shm_ids(ns).rwsem);
329 330
}

331
/* Locking assumes this will only be called with task == current */
332 333
void exit_shm(struct task_struct *task)
{
334
	struct ipc_namespace *ns = task->nsproxy->ipc_ns;
335
	struct shmid_kernel *shp, *n;
336

337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
	if (list_empty(&task->sysvshm.shm_clist))
		return;

	/*
	 * If kernel.shm_rmid_forced is not set then only keep track of
	 * which shmids are orphaned, so that a later set of the sysctl
	 * can clean them up.
	 */
	if (!ns->shm_rmid_forced) {
		down_read(&shm_ids(ns).rwsem);
		list_for_each_entry(shp, &task->sysvshm.shm_clist, shm_clist)
			shp->shm_creator = NULL;
		/*
		 * Only under read lock but we are only called on current
		 * so no entry on the list will be shared.
		 */
		list_del(&task->sysvshm.shm_clist);
		up_read(&shm_ids(ns).rwsem);
Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
355
		return;
356
	}
Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
357

358 359 360 361 362
	/*
	 * Destroy all already created segments, that were not yet mapped,
	 * and mark any mapped as orphan to cover the sysctl toggling.
	 * Destroy is skipped if shm_may_destroy() returns false.
	 */
Davidlohr Bueso's avatar
Davidlohr Bueso committed
363
	down_write(&shm_ids(ns).rwsem);
364 365 366 367 368 369 370 371 372 373
	list_for_each_entry_safe(shp, n, &task->sysvshm.shm_clist, shm_clist) {
		shp->shm_creator = NULL;

		if (shm_may_destroy(ns, shp)) {
			shm_lock_by_ptr(shp);
			shm_destroy(ns, shp);
		}
	}

	/* Remove the list head from any segments still attached. */
374
	list_del(&task->sysvshm.shm_clist);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
375
	up_write(&shm_ids(ns).rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
376 377
}

378
static int shm_fault(struct vm_fault *vmf)
379
{
380
	struct file *file = vmf->vma->vm_file;
381 382
	struct shm_file_data *sfd = shm_file_data(file);

383
	return sfd->vm_ops->fault(vmf);
384 385 386
}

#ifdef CONFIG_NUMA
387
static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
388 389 390 391
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
	int err = 0;
392

393 394 395 396 397
	if (sfd->vm_ops->set_policy)
		err = sfd->vm_ops->set_policy(vma, new);
	return err;
}

398 399
static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
					unsigned long addr)
400 401 402 403 404 405 406
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
	struct mempolicy *pol = NULL;

	if (sfd->vm_ops->get_policy)
		pol = sfd->vm_ops->get_policy(vma, addr);
407
	else if (vma->vm_policy)
408
		pol = vma->vm_policy;
409

410 411 412 413
	return pol;
}
#endif

Manfred Spraul's avatar
Manfred Spraul committed
414
static int shm_mmap(struct file *file, struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
415
{
416
	struct shm_file_data *sfd = shm_file_data(file);
417 418
	int ret;

419 420 421 422
	/*
	 * In case of remap_file_pages() emulation, the file can represent
	 * removed IPC ID: propogate shm_lock() error to caller.
	 */
423
	ret = __shm_open(vma);
424 425 426
	if (ret)
		return ret;

427
	ret = call_mmap(sfd->file, vma);
428 429
	if (ret) {
		shm_close(vma);
430
		return ret;
431
	}
432
	sfd->vm_ops = vma->vm_ops;
David Howells's avatar
David Howells committed
433
#ifdef CONFIG_MMU
434
	WARN_ON(!sfd->vm_ops->fault);
David Howells's avatar
David Howells committed
435
#endif
436
	vma->vm_ops = &shm_vm_ops;
437
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
438 439
}

Kirill Korotaev's avatar
Kirill Korotaev committed
440 441
static int shm_release(struct inode *ino, struct file *file)
{
442
	struct shm_file_data *sfd = shm_file_data(file);
Kirill Korotaev's avatar
Kirill Korotaev committed
443

444 445 446
	put_ipc_ns(sfd->ns);
	shm_file_data(file) = NULL;
	kfree(sfd);
Kirill Korotaev's avatar
Kirill Korotaev committed
447 448 449
	return 0;
}

450
static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
451 452 453
{
	struct shm_file_data *sfd = shm_file_data(file);

454 455
	if (!sfd->file->f_op->fsync)
		return -EINVAL;
456
	return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
457 458
}

459 460 461 462 463 464 465 466 467 468
static long shm_fallocate(struct file *file, int mode, loff_t offset,
			  loff_t len)
{
	struct shm_file_data *sfd = shm_file_data(file);

	if (!sfd->file->f_op->fallocate)
		return -EOPNOTSUPP;
	return sfd->file->f_op->fallocate(file, mode, offset, len);
}

469 470 471 472 473
static unsigned long shm_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len, unsigned long pgoff,
	unsigned long flags)
{
	struct shm_file_data *sfd = shm_file_data(file);
474

475 476
	return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
						pgoff, flags);
477 478
}

479
static const struct file_operations shm_file_operations = {
Kirill Korotaev's avatar
Kirill Korotaev committed
480
	.mmap		= shm_mmap,
481
	.fsync		= shm_fsync,
Kirill Korotaev's avatar
Kirill Korotaev committed
482
	.release	= shm_release,
David Howells's avatar
David Howells committed
483
	.get_unmapped_area	= shm_get_unmapped_area,
484
	.llseek		= noop_llseek,
485
	.fallocate	= shm_fallocate,
486 487
};

488 489 490 491
/*
 * shm_file_operations_huge is now identical to shm_file_operations,
 * but we keep it distinct for the sake of is_file_shm_hugepages().
 */
492 493 494 495
static const struct file_operations shm_file_operations_huge = {
	.mmap		= shm_mmap,
	.fsync		= shm_fsync,
	.release	= shm_release,
496
	.get_unmapped_area	= shm_get_unmapped_area,
497
	.llseek		= noop_llseek,
498
	.fallocate	= shm_fallocate,
Linus Torvalds's avatar
Linus Torvalds committed
499 500
};

501
bool is_file_shm_hugepages(struct file *file)
502 503 504 505
{
	return file->f_op == &shm_file_operations_huge;
}

506
static const struct vm_operations_struct shm_vm_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
507 508
	.open	= shm_open,	/* callback for a new vm-area open */
	.close	= shm_close,	/* callback for when the vm-area is released */
509
	.fault	= shm_fault,
510 511 512
#if defined(CONFIG_NUMA)
	.set_policy = shm_set_policy,
	.get_policy = shm_get_policy,
Linus Torvalds's avatar
Linus Torvalds committed
513 514 515
#endif
};

Nadia Derbey's avatar
Nadia Derbey committed
516 517 518 519 520
/**
 * newseg - Create a new shared memory segment
 * @ns: namespace
 * @params: ptr to the structure that contains key, size and shmflg
 *
Davidlohr Bueso's avatar
Davidlohr Bueso committed
521
 * Called with shm_ids.rwsem held as a writer.
Nadia Derbey's avatar
Nadia Derbey committed
522
 */
Nadia Derbey's avatar
Nadia Derbey committed
523
static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
Linus Torvalds's avatar
Linus Torvalds committed
524
{
Nadia Derbey's avatar
Nadia Derbey committed
525 526 527
	key_t key = params->key;
	int shmflg = params->flg;
	size_t size = params->u.size;
Linus Torvalds's avatar
Linus Torvalds committed
528 529
	int error;
	struct shmid_kernel *shp;
530
	size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Manfred Spraul's avatar
Manfred Spraul committed
531
	struct file *file;
Linus Torvalds's avatar
Linus Torvalds committed
532
	char name[13];
533
	vm_flags_t acctflag = 0;
Linus Torvalds's avatar
Linus Torvalds committed
534

Kirill Korotaev's avatar
Kirill Korotaev committed
535
	if (size < SHMMIN || size > ns->shm_ctlmax)
Linus Torvalds's avatar
Linus Torvalds committed
536 537
		return -EINVAL;

538 539 540
	if (numpages << PAGE_SHIFT < size)
		return -ENOSPC;

541 542
	if (ns->shm_tot + numpages < ns->shm_tot ||
			ns->shm_tot + numpages > ns->shm_ctlall)
Linus Torvalds's avatar
Linus Torvalds committed
543 544
		return -ENOSPC;

545 546
	shp = kvmalloc(sizeof(*shp), GFP_KERNEL);
	if (unlikely(!shp))
Linus Torvalds's avatar
Linus Torvalds committed
547 548 549
		return -ENOMEM;

	shp->shm_perm.key = key;
Andrew Morton's avatar
Andrew Morton committed
550
	shp->shm_perm.mode = (shmflg & S_IRWXUGO);
Linus Torvalds's avatar
Linus Torvalds committed
551 552 553 554 555
	shp->mlock_user = NULL;

	shp->shm_perm.security = NULL;
	error = security_shm_alloc(shp);
	if (error) {
556
		kvfree(shp);
Linus Torvalds's avatar
Linus Torvalds committed
557 558 559
		return error;
	}

Manfred Spraul's avatar
Manfred Spraul committed
560
	sprintf(name, "SYSV%08x", key);
Linus Torvalds's avatar
Linus Torvalds committed
561
	if (shmflg & SHM_HUGETLB) {
562
		struct hstate *hs;
563 564
		size_t hugesize;

565
		hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
566 567 568 569 570
		if (!hs) {
			error = -EINVAL;
			goto no_file;
		}
		hugesize = ALIGN(size, huge_page_size(hs));
571

572 573 574
		/* hugetlb_file_setup applies strict accounting */
		if (shmflg & SHM_NORESERVE)
			acctflag = VM_NORESERVE;
575
		file = hugetlb_file_setup(name, hugesize, acctflag,
576 577
				  &shp->mlock_user, HUGETLB_SHMFS_INODE,
				(shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
Linus Torvalds's avatar
Linus Torvalds committed
578
	} else {
579 580
		/*
		 * Do not allow no accounting for OVERCOMMIT_NEVER, even
Manfred Spraul's avatar
Manfred Spraul committed
581
		 * if it's asked for.
582 583 584
		 */
		if  ((shmflg & SHM_NORESERVE) &&
				sysctl_overcommit_memory != OVERCOMMIT_NEVER)
585
			acctflag = VM_NORESERVE;
586
		file = shmem_kernel_file_setup(name, size, acctflag);
Linus Torvalds's avatar
Linus Torvalds committed
587 588 589 590 591
	}
	error = PTR_ERR(file);
	if (IS_ERR(file))
		goto no_file;

592
	shp->shm_cprid = task_tgid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
593 594 595 596 597 598
	shp->shm_lprid = 0;
	shp->shm_atim = shp->shm_dtim = 0;
	shp->shm_ctim = get_seconds();
	shp->shm_segsz = size;
	shp->shm_nattch = 0;
	shp->shm_file = file;
599
	shp->shm_creator = current;
600

601 602
	error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
	if (error < 0)
603 604
		goto no_id;

605
	list_add(&shp->shm_clist, &current->sysvshm.shm_clist);
606

607 608 609 610
	/*
	 * shmid gets reported as "inode#" in /proc/pid/maps.
	 * proc-ps tools use this. Changing this will break them.
	 */
Al Viro's avatar
Al Viro committed
611
	file_inode(file)->i_ino = shp->shm_perm.id;
612

Kirill Korotaev's avatar
Kirill Korotaev committed
613
	ns->shm_tot += numpages;
Nadia Derbey's avatar
Nadia Derbey committed
614
	error = shp->shm_perm.id;
615

616
	ipc_unlock_object(&shp->shm_perm);
617
	rcu_read_unlock();
Nadia Derbey's avatar
Nadia Derbey committed
618
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
619 620

no_id:
621
	if (is_file_hugepages(file) && shp->mlock_user)
622
		user_shm_unlock(size, shp->mlock_user);
Linus Torvalds's avatar
Linus Torvalds committed
623 624
	fput(file);
no_file:
625
	call_rcu(&shp->shm_perm.rcu, shm_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
626 627 628
	return error;
}

Nadia Derbey's avatar
Nadia Derbey committed
629
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
630
 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbey's avatar
Nadia Derbey committed
631
 */
Nadia Derbey's avatar
Nadia Derbey committed
632
static inline int shm_security(struct kern_ipc_perm *ipcp, int shmflg)
Nadia Derbey's avatar
Nadia Derbey committed
633
{
Nadia Derbey's avatar
Nadia Derbey committed
634 635 636 637
	struct shmid_kernel *shp;

	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
	return security_shm_associate(shp, shmflg);
Nadia Derbey's avatar
Nadia Derbey committed
638 639
}

Nadia Derbey's avatar
Nadia Derbey committed
640
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
641
 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbey's avatar
Nadia Derbey committed
642
 */
Nadia Derbey's avatar
Nadia Derbey committed
643 644
static inline int shm_more_checks(struct kern_ipc_perm *ipcp,
				struct ipc_params *params)
Nadia Derbey's avatar
Nadia Derbey committed
645
{
Nadia Derbey's avatar
Nadia Derbey committed
646 647 648 649
	struct shmid_kernel *shp;

	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
	if (shp->shm_segsz < params->u.size)
Nadia Derbey's avatar
Nadia Derbey committed
650 651 652 653 654
		return -EINVAL;

	return 0;
}

655
SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
Linus Torvalds's avatar
Linus Torvalds committed
656
{
Kirill Korotaev's avatar
Kirill Korotaev committed
657
	struct ipc_namespace *ns;
Mathias Krause's avatar
Mathias Krause committed
658 659 660 661 662
	static const struct ipc_ops shm_ops = {
		.getnew = newseg,
		.associate = shm_security,
		.more_checks = shm_more_checks,
	};
Nadia Derbey's avatar
Nadia Derbey committed
663
	struct ipc_params shm_params;
Kirill Korotaev's avatar
Kirill Korotaev committed
664 665

	ns = current->nsproxy->ipc_ns;
Linus Torvalds's avatar
Linus Torvalds committed
666

Nadia Derbey's avatar
Nadia Derbey committed
667 668 669
	shm_params.key = key;
	shm_params.flg = shmflg;
	shm_params.u.size = size;
Linus Torvalds's avatar
Linus Torvalds committed
670

Nadia Derbey's avatar
Nadia Derbey committed
671
	return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
Linus Torvalds's avatar
Linus Torvalds committed
672 673 674 675
}

static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
{
Manfred Spraul's avatar
Manfred Spraul committed
676
	switch (version) {
Linus Torvalds's avatar
Linus Torvalds committed
677 678 679 680 681 682
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct shmid_ds out;

683
		memset(&out, 0, sizeof(out));
Linus Torvalds's avatar
Linus Torvalds committed
684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
		ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
		out.shm_segsz	= in->shm_segsz;
		out.shm_atime	= in->shm_atime;
		out.shm_dtime	= in->shm_dtime;
		out.shm_ctime	= in->shm_ctime;
		out.shm_cpid	= in->shm_cpid;
		out.shm_lpid	= in->shm_lpid;
		out.shm_nattch	= in->shm_nattch;

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

700 701
static inline unsigned long
copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
Linus Torvalds's avatar
Linus Torvalds committed
702
{
Manfred Spraul's avatar
Manfred Spraul committed
703
	switch (version) {
Linus Torvalds's avatar
Linus Torvalds committed
704
	case IPC_64:
705
		if (copy_from_user(out, buf, sizeof(*out)))
Linus Torvalds's avatar
Linus Torvalds committed
706 707 708 709 710 711 712 713 714
			return -EFAULT;
		return 0;
	case IPC_OLD:
	    {
		struct shmid_ds tbuf_old;

		if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
			return -EFAULT;

715 716 717
		out->shm_perm.uid	= tbuf_old.shm_perm.uid;
		out->shm_perm.gid	= tbuf_old.shm_perm.gid;
		out->shm_perm.mode	= tbuf_old.shm_perm.mode;
Linus Torvalds's avatar
Linus Torvalds committed
718 719 720 721 722 723 724 725 726 727

		return 0;
	    }
	default:
		return -EINVAL;
	}
}

static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
{
Manfred Spraul's avatar
Manfred Spraul committed
728
	switch (version) {
Linus Torvalds's avatar
Linus Torvalds committed
729 730 731 732 733 734
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct shminfo out;

Manfred Spraul's avatar
Manfred Spraul committed
735
		if (in->shmmax > INT_MAX)
Linus Torvalds's avatar
Linus Torvalds committed
736 737 738 739 740 741 742
			out.shmmax = INT_MAX;
		else
			out.shmmax = (int)in->shmmax;

		out.shmmin	= in->shmmin;
		out.shmmni	= in->shmmni;
		out.shmseg	= in->shmseg;
Paul McQuade's avatar
Paul McQuade committed
743
		out.shmall	= in->shmall;
Linus Torvalds's avatar
Linus Torvalds committed
744 745 746 747 748 749 750 751

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

752 753
/*
 * Calculate and add used RSS and swap pages of a shm.
Davidlohr Bueso's avatar
Davidlohr Bueso committed
754
 * Called with shm_ids.rwsem held as a reader
755 756 757 758 759 760
 */
static void shm_add_rss_swap(struct shmid_kernel *shp,
	unsigned long *rss_add, unsigned long *swp_add)
{
	struct inode *inode;

Al Viro's avatar
Al Viro committed
761
	inode = file_inode(shp->shm_file);
762 763 764 765 766 767 768 769

	if (is_file_hugepages(shp->shm_file)) {
		struct address_space *mapping = inode->i_mapping;
		struct hstate *h = hstate_file(shp->shm_file);
		*rss_add += pages_per_huge_page(h) * mapping->nrpages;
	} else {
#ifdef CONFIG_SHMEM
		struct shmem_inode_info *info = SHMEM_I(inode);
770

771
		spin_lock_irq(&info->lock);
772 773
		*rss_add += inode->i_mapping->nrpages;
		*swp_add += info->swapped;
774
		spin_unlock_irq(&info->lock);
775 776 777 778 779 780
#else
		*rss_add += inode->i_mapping->nrpages;
#endif
	}
}

Nadia Derbey's avatar
Nadia Derbey committed
781
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
782
 * Called with shm_ids.rwsem held as a reader
Nadia Derbey's avatar
Nadia Derbey committed
783
 */
Kirill Korotaev's avatar
Kirill Korotaev committed
784 785
static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
		unsigned long *swp)
Linus Torvalds's avatar
Linus Torvalds committed
786
{
Nadia Derbey's avatar
Nadia Derbey committed
787 788
	int next_id;
	int total, in_use;
Linus Torvalds's avatar
Linus Torvalds committed
789 790 791 792

	*rss = 0;
	*swp = 0;

Nadia Derbey's avatar
Nadia Derbey committed
793 794 795
	in_use = shm_ids(ns).in_use;

	for (total = 0, next_id = 0; total < in_use; next_id++) {
796
		struct kern_ipc_perm *ipc;
Linus Torvalds's avatar
Linus Torvalds committed
797 798
		struct shmid_kernel *shp;

799 800
		ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
		if (ipc == NULL)
Linus Torvalds's avatar
Linus Torvalds committed
801
			continue;
802
		shp = container_of(ipc, struct shmid_kernel, shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
803

804
		shm_add_rss_swap(shp, rss, swp);
Nadia Derbey's avatar
Nadia Derbey committed
805 806

		total++;
Linus Torvalds's avatar
Linus Torvalds committed
807 808 809
	}
}

810
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
811
 * This function handles some shmctl commands which require the rwsem
812
 * to be held in write mode.
Davidlohr Bueso's avatar
Davidlohr Bueso committed
813
 * NOTE: no locks must be held, the rwsem is taken inside this function.
814 815
 */
static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
816
		       struct shmid64_ds *shmid64)
Linus Torvalds's avatar
Linus Torvalds committed
817
{
818 819 820 821
	struct kern_ipc_perm *ipcp;
	struct shmid_kernel *shp;
	int err;

Davidlohr Bueso's avatar
Davidlohr Bueso committed
822
	down_write(&shm_ids(ns).rwsem);
823 824
	rcu_read_lock();

825
	ipcp = ipcctl_pre_down_nolock(ns, &shm_ids(ns), shmid, cmd,
826
				      &shmid64->shm_perm, 0);
827 828 829 830
	if (IS_ERR(ipcp)) {
		err = PTR_ERR(ipcp);
		goto out_unlock1;
	}
831

832
	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
833 834 835

	err = security_shm_shmctl(shp, cmd);
	if (err)
836
		goto out_unlock1;
837

838 839
	switch (cmd) {
	case IPC_RMID:
840
		ipc_lock_object(&shp->shm_perm);
841
		/* do_shm_rmid unlocks the ipc object and rcu */
842 843 844
		do_shm_rmid(ns, ipcp);
		goto out_up;
	case IPC_SET:
845
		ipc_lock_object(&shp->shm_perm);
846
		err = ipc_update_perm(&shmid64->shm_perm, ipcp);
847
		if (err)
848
			goto out_unlock0;
849 850 851 852
		shp->shm_ctim = get_seconds();
		break;
	default:
		err = -EINVAL;
853
		goto out_unlock1;
854
	}
855 856 857 858 859

out_unlock0:
	ipc_unlock_object(&shp->shm_perm);
out_unlock1:
	rcu_read_unlock();
860
out_up:
Davidlohr Bueso's avatar
Davidlohr Bueso committed
861
	up_write(&shm_ids(ns).rwsem);
862 863 864
	return err;
}

865 866
static int shmctl_ipc_info(struct ipc_namespace *ns,
			   struct shminfo64 *shminfo)
867
{
868 869 870 871 872 873 874
	int err = security_shm_shmctl(NULL, IPC_INFO);
	if (!err) {
		memset(shminfo, 0, sizeof(*shminfo));
		shminfo->shmmni = shminfo->shmseg = ns->shm_ctlmni;
		shminfo->shmmax = ns->shm_ctlmax;
		shminfo->shmall = ns->shm_ctlall;
		shminfo->shmmin = SHMMIN;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
875
		down_read(&shm_ids(ns).rwsem);
Nadia Derbey's avatar
Nadia Derbey committed
876
		err = ipc_get_maxid(&shm_ids(ns));
Davidlohr Bueso's avatar
Davidlohr Bueso committed
877
		up_read(&shm_ids(ns).rwsem);
Manfred Spraul's avatar
Manfred Spraul committed
878
		if (err < 0)
Linus Torvalds's avatar
Linus Torvalds committed
879 880
			err = 0;
	}
881 882
	return err;
}
Linus Torvalds's avatar
Linus Torvalds committed
883

884 885 886 887 888 889
static int shmctl_shm_info(struct ipc_namespace *ns,
			   struct shm_info *shm_info)
{
	int err = security_shm_shmctl(NULL, SHM_INFO);
	if (!err) {
		memset(shm_info, 0, sizeof(*shm_info));
Davidlohr Bueso's avatar
Davidlohr Bueso committed
890
		down_read(&shm_ids(ns).rwsem);
891 892 893 894 895
		shm_info->used_ids = shm_ids(ns).in_use;
		shm_get_stat(ns, &shm_info->shm_rss, &shm_info->shm_swp);
		shm_info->shm_tot = ns->shm_tot;
		shm_info->swap_attempts = 0;
		shm_info->swap_successes = 0;
Nadia Derbey's avatar
Nadia Derbey committed
896
		err = ipc_get_maxid(&shm_ids(ns));
Davidlohr Bueso's avatar
Davidlohr Bueso committed
897
		up_read(&shm_ids(ns).rwsem);
898 899
		if (err < 0)
			err = 0;
Linus Torvalds's avatar
Linus Torvalds committed
900
	}
901 902
	return err;
}
903

904 905 906 907 908 909
static int shmctl_stat(struct ipc_namespace *ns, int shmid,
			int cmd, struct shmid64_ds *tbuf)
{
	struct shmid_kernel *shp;
	int result;
	int err;
910

911 912 913 914 915 916 917 918 919 920 921 922
	rcu_read_lock();
	if (cmd == SHM_STAT) {
		shp = shm_obtain_object(ns, shmid);
		if (IS_ERR(shp)) {
			err = PTR_ERR(shp);
			goto out_unlock;
		}
		result = shp->shm_perm.id;
	} else {
		shp = shm_obtain_object_check(ns, shmid);
		if (IS_ERR(shp)) {
			err = PTR_ERR(shp);
Linus Torvalds's avatar
Linus Torvalds committed
923
			goto out_unlock;
924 925 926
		}
		result = 0;
	}
927

928 929 930
	err = -EACCES;
	if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
		goto out_unlock;
931

932 933 934 935 936 937 938 939 940 941 942 943 944 945 946
	err = security_shm_shmctl(shp, cmd);
	if (err)
		goto out_unlock;

	memset(tbuf, 0, sizeof(*tbuf));
	kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);
	tbuf->shm_segsz	= shp->shm_segsz;
	tbuf->shm_atime	= shp->shm_atim;
	tbuf->shm_dtime	= shp->shm_dtim;
	tbuf->shm_ctime	= shp->shm_ctim;
	tbuf->shm_cpid	= shp->shm_cprid;
	tbuf->shm_lpid	= shp->shm_lprid;
	tbuf->shm_nattch = shp->shm_nattch;
	rcu_read_unlock();
	return result;
947 948

out_unlock:
949
	rcu_read_unlock();
950 951 952
	return err;
}

953
static int shmctl_do_lock(struct ipc_namespace *ns, int shmid, int cmd)
954 955
{
	struct shmid_kernel *shp;
956 957
	struct file *shm_file;
	int err;
958

959 960 961 962 963 964
	rcu_read_lock();
	shp = shm_obtain_object_check(ns, shmid);
	if (IS_ERR(shp)) {
		err = PTR_ERR(shp);
		goto out_unlock1;
	}
965

966 967 968 969
	audit_ipc_obj(&(shp->shm_perm));
	err = security_shm_shmctl(shp, cmd);
	if (err)
		goto out_unlock1;
970

971
	ipc_lock_object(&shp->shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
972

973 974 975 976 977
	/* check if shm_destroy() is tearing down shp */
	if (!ipc_valid_object(&shp->shm_perm)) {
		err = -EIDRM;
		goto out_unlock0;
	}
Steve Grubb's avatar
Steve Grubb committed
978

979 980
	if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
		kuid_t euid = current_euid();
981

982 983 984
		if (!uid_eq(euid, shp->shm_perm.uid) &&
		    !uid_eq(euid, shp->shm_perm.cuid)) {
			err = -EPERM;
985 986
			goto out_unlock0;
		}
987 988 989
		if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
			err = -EPERM;
			goto out_unlock0;
Linus Torvalds's avatar
Linus Torvalds committed
990
		}
991
	}
Linus Torvalds's avatar
Linus Torvalds committed
992

993 994 995
	shm_file = shp->shm_file;
	if (is_file_hugepages(shm_file))
		goto out_unlock0;
996

997 998
	if (cmd == SHM_LOCK) {
		struct user_struct *user = current_user();
999

1000 1001 1002 1003
		err = shmem_lock(shm_file, 1, user);
		if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
			shp->shm_perm.mode |= SHM_LOCKED;
			shp->mlock_user = user;
Linus Torvalds's avatar
Linus Torvalds committed
1004
		}
1005
		goto out_unlock0;
Linus Torvalds's avatar
Linus Torvalds committed
1006 1007
	}

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	/* SHM_UNLOCK */
	if (!(shp->shm_perm.mode & SHM_LOCKED))
		goto out_unlock0;
	shmem_lock(shm_file, 0, shp->mlock_user);
	shp->shm_perm.mode &= ~SHM_LOCKED;
	shp->mlock_user = NULL;
	get_file(shm_file);
	ipc_unlock_object(&shp->shm_perm);
	rcu_read_unlock();
	shmem_unlock_mapping(shm_file->f_mapping);

	fput(shm_file);
	return err;

1022 1023 1024 1025
out_unlock0:
	ipc_unlock_object(&shp->shm_perm);
out_unlock1:
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
1026 1027 1028
	return err;
}

1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
	int err, version;
	struct ipc_namespace *ns;
	struct shmid64_ds tbuf;

	if (cmd < 0 || shmid < 0)
		return -EINVAL;

	version = ipc_parse_version(&cmd);
	ns = current->nsproxy->ipc_ns;

	switch (cmd) {
	case IPC_INFO: {
		struct shminfo64 shminfo;
		err = shmctl_ipc_info(ns, &shminfo);
		if (err < 0)
			return err;
		if (copy_shminfo_to_user(buf, &shminfo, version))
			err = -EFAULT;
		return err;
	}
	case SHM_INFO: {
		struct shm_info shm_info;
		err = shmctl_shm_info(ns, &shm_info);
		if (err < 0)
			return err;
		if (copy_to_user(buf, &shm_info, sizeof(shm_info)))
			err = -EFAULT;
		return err;
	}
	case SHM_STAT:
	case IPC_STAT: {
		err = shmctl_stat(ns, shmid, cmd, &tbuf);
		if (err < 0)
			return err;
		if (copy_shmid_to_user(buf, &tbuf, version))
			err = -EFAULT;
		return err;
	}
	case IPC_SET:
		if (copy_shmid_from_user(&tbuf, buf, version))
			return -EFAULT;
	case IPC_RMID:
		return shmctl_down(ns, shmid, cmd, &tbuf);
	case SHM_LOCK:
	case SHM_UNLOCK:
		return shmctl_do_lock(ns, shmid, cmd);
	default:
		return -EINVAL;
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
1082 1083 1084 1085 1086 1087 1088
/*
 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
 *
 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
 * "raddr" thing points to kernel space, and there has to be a wrapper around
 * this.
 */
1089 1090
long do_shmat(int shmid, char __user *shmaddr, int shmflg,
	      ulong *raddr, unsigned long shmlba)
Linus Torvalds's avatar
Linus Torvalds committed
1091 1092
{
	struct shmid_kernel *shp;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
1093
	unsigned long addr = (unsigned long)shmaddr;
Linus Torvalds's avatar
Linus Torvalds committed
1094
	unsigned long size;
Manfred Spraul's avatar
Manfred Spraul committed
1095
	struct file *file;
Linus Torvalds's avatar
Linus Torvalds committed
1096
	int    err;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
1097
	unsigned long flags = MAP_SHARED;
Linus Torvalds's avatar
Linus Torvalds committed
1098 1099
	unsigned long prot;
	int acc_mode;
Kirill Korotaev's avatar
Kirill Korotaev committed
1100
	struct ipc_namespace *ns;
1101 1102
	struct shm_file_data *sfd;
	struct path path;
1103
	fmode_t f_mode;
1104
	unsigned long populate = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1105

1106 1107
	err = -EINVAL;
	if (shmid < 0)
Linus Torvalds's avatar
Linus Torvalds committed
1108
		goto out;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
1109 1110

	if (addr) {
Will Deacon's avatar
Will Deacon committed
1111
		if (addr & (shmlba - 1)) {
1112 1113 1114 1115 1116 1117 1118
			/*
			 * Round down to the nearest multiple of shmlba.
			 * For sane do_mmap_pgoff() parameters, avoid
			 * round downs that trigger nil-page and MAP_FIXED.
			 */
			if ((shmflg & SHM_RND) && addr >= shmlba)
				addr &= ~(shmlba - 1