shm.c 31.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * linux/ipc/shm.c
 * Copyright (C) 1992, 1993 Krishna Balasubramanian
 *	 Many improvements/fixes by Bruno Haible.
 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
 *
 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
 *
Steve Grubb's avatar
Steve Grubb committed
16 17
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
Kirill Korotaev's avatar
Kirill Korotaev committed
18 19 20 21
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
22 23 24
 *
 * Better ipc lock (kern_ipc_perm.lock) handling
 * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27 28 29 30 31 32 33 34 35 36 37
 */

#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/mman.h>
#include <linux/shmem_fs.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
38
#include <linux/capability.h>
39
#include <linux/ptrace.h>
40
#include <linux/seq_file.h>
Nadia Derbey's avatar
Nadia Derbey committed
41
#include <linux/rwsem.h>
Kirill Korotaev's avatar
Kirill Korotaev committed
42
#include <linux/nsproxy.h>
43
#include <linux/mount.h>
44
#include <linux/ipc_namespace.h>
45

Linus Torvalds's avatar
Linus Torvalds committed
46 47 48 49
#include <asm/uaccess.h>

#include "util.h"

50 51 52 53 54 55 56 57 58
struct shm_file_data {
	int id;
	struct ipc_namespace *ns;
	struct file *file;
	const struct vm_operations_struct *vm_ops;
};

#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))

59
static const struct file_operations shm_file_operations;
60
static const struct vm_operations_struct shm_vm_ops;
Linus Torvalds's avatar
Linus Torvalds committed
61

62
#define shm_ids(ns)	((ns)->ids[IPC_SHM_IDS])
Linus Torvalds's avatar
Linus Torvalds committed
63

Kirill Korotaev's avatar
Kirill Korotaev committed
64 65
#define shm_unlock(shp)			\
	ipc_unlock(&(shp)->shm_perm)
Linus Torvalds's avatar
Linus Torvalds committed
66

Nadia Derbey's avatar
Nadia Derbey committed
67
static int newseg(struct ipc_namespace *, struct ipc_params *);
68 69
static void shm_open(struct vm_area_struct *vma);
static void shm_close(struct vm_area_struct *vma);
Kirill Korotaev's avatar
Kirill Korotaev committed
70
static void shm_destroy (struct ipc_namespace *ns, struct shmid_kernel *shp);
Linus Torvalds's avatar
Linus Torvalds committed
71
#ifdef CONFIG_PROC_FS
72
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
Linus Torvalds's avatar
Linus Torvalds committed
73 74
#endif

75
void shm_init_ns(struct ipc_namespace *ns)
Kirill Korotaev's avatar
Kirill Korotaev committed
76 77 78 79
{
	ns->shm_ctlmax = SHMMAX;
	ns->shm_ctlall = SHMALL;
	ns->shm_ctlmni = SHMMNI;
80
	ns->shm_rmid_forced = 0;
Kirill Korotaev's avatar
Kirill Korotaev committed
81
	ns->shm_tot = 0;
WANG Cong's avatar
WANG Cong committed
82
	ipc_init_ids(&shm_ids(ns));
Kirill Korotaev's avatar
Kirill Korotaev committed
83 84
}

Nadia Derbey's avatar
Nadia Derbey committed
85
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
86 87
 * Called with shm_ids.rwsem (writer) and the shp structure locked.
 * Only shm_ids.rwsem remains locked on exit.
Nadia Derbey's avatar
Nadia Derbey committed
88
 */
89
static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
Kirill Korotaev's avatar
Kirill Korotaev committed
90
{
91 92 93
	struct shmid_kernel *shp;
	shp = container_of(ipcp, struct shmid_kernel, shm_perm);

Kirill Korotaev's avatar
Kirill Korotaev committed
94 95 96 97 98 99 100 101 102
	if (shp->shm_nattch){
		shp->shm_perm.mode |= SHM_DEST;
		/* Do not find it any more */
		shp->shm_perm.key = IPC_PRIVATE;
		shm_unlock(shp);
	} else
		shm_destroy(ns, shp);
}

103
#ifdef CONFIG_IPC_NS
Kirill Korotaev's avatar
Kirill Korotaev committed
104 105
void shm_exit_ns(struct ipc_namespace *ns)
{
106
	free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
107
	idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
Kirill Korotaev's avatar
Kirill Korotaev committed
108
}
109
#endif
Linus Torvalds's avatar
Linus Torvalds committed
110

111
static int __init ipc_ns_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
112
{
113
	shm_init_ns(&init_ipc_ns);
114 115 116 117 118 119 120
	return 0;
}

pure_initcall(ipc_ns_init);

void __init shm_init (void)
{
121
	ipc_init_proc_interface("sysvipc/shm",
122 123 124 125 126
#if BITS_PER_LONG <= 32
				"       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
#else
				"       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
#endif
Kirill Korotaev's avatar
Kirill Korotaev committed
127
				IPC_SHM_IDS, sysvipc_shm_proc_show);
Linus Torvalds's avatar
Linus Torvalds committed
128 129
}

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
static inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
{
	struct kern_ipc_perm *ipcp = ipc_obtain_object(&shm_ids(ns), id);

	if (IS_ERR(ipcp))
		return ERR_CAST(ipcp);

	return container_of(ipcp, struct shmid_kernel, shm_perm);
}

static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
{
	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);

	if (IS_ERR(ipcp))
		return ERR_CAST(ipcp);

	return container_of(ipcp, struct shmid_kernel, shm_perm);
}

Nadia Derbey's avatar
Nadia Derbey committed
150
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
151
 * shm_lock_(check_) routines are called in the paths where the rwsem
Nadia Derbey's avatar
Nadia Derbey committed
152
 * is not necessarily held.
Nadia Derbey's avatar
Nadia Derbey committed
153
 */
154
static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
Linus Torvalds's avatar
Linus Torvalds committed
155
{
Nadia Derbey's avatar
Nadia Derbey committed
156 157
	struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id);

158 159 160
	if (IS_ERR(ipcp))
		return (struct shmid_kernel *)ipcp;

Nadia Derbey's avatar
Nadia Derbey committed
161
	return container_of(ipcp, struct shmid_kernel, shm_perm);
162 163
}

164 165 166
static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
{
	rcu_read_lock();
167
	ipc_lock_object(&ipcp->shm_perm);
168 169
}

Davidlohr Bueso's avatar
Davidlohr Bueso committed
170 171 172 173 174 175 176 177 178
static void shm_rcu_free(struct rcu_head *head)
{
	struct ipc_rcu *p = container_of(head, struct ipc_rcu, rcu);
	struct shmid_kernel *shp = ipc_rcu_to_struct(p);

	security_shm_free(shp);
	ipc_rcu_free(head);
}

Nadia Derbey's avatar
Nadia Derbey committed
179
static inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
Linus Torvalds's avatar
Linus Torvalds committed
180
{
Nadia Derbey's avatar
Nadia Derbey committed
181
	ipc_rmid(&shm_ids(ns), &s->shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
182 183 184
}


185 186
/* This is called by fork, once for every shm attach. */
static void shm_open(struct vm_area_struct *vma)
Kirill Korotaev's avatar
Kirill Korotaev committed
187
{
188 189
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds's avatar
Linus Torvalds committed
190 191
	struct shmid_kernel *shp;

192
	shp = shm_lock(sfd->ns, sfd->id);
193
	BUG_ON(IS_ERR(shp));
Linus Torvalds's avatar
Linus Torvalds committed
194
	shp->shm_atim = get_seconds();
195
	shp->shm_lprid = task_tgid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
196 197 198 199 200 201 202
	shp->shm_nattch++;
	shm_unlock(shp);
}

/*
 * shm_destroy - free the struct shmid_kernel
 *
Nadia Derbey's avatar
Nadia Derbey committed
203
 * @ns: namespace
Linus Torvalds's avatar
Linus Torvalds committed
204 205
 * @shp: struct to free
 *
Davidlohr Bueso's avatar
Davidlohr Bueso committed
206
 * It has to be called with shp and shm_ids.rwsem (writer) locked,
Linus Torvalds's avatar
Linus Torvalds committed
207 208
 * but returns with shp unlocked and freed.
 */
Kirill Korotaev's avatar
Kirill Korotaev committed
209
static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
Linus Torvalds's avatar
Linus Torvalds committed
210
{
211 212 213 214
	struct file *shm_file;

	shm_file = shp->shm_file;
	shp->shm_file = NULL;
Kirill Korotaev's avatar
Kirill Korotaev committed
215
	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
Nadia Derbey's avatar
Nadia Derbey committed
216
	shm_rmid(ns, shp);
Linus Torvalds's avatar
Linus Torvalds committed
217
	shm_unlock(shp);
218 219
	if (!is_file_hugepages(shm_file))
		shmem_lock(shm_file, 0, shp->mlock_user);
220
	else if (shp->mlock_user)
221 222
		user_shm_unlock(file_inode(shm_file)->i_size, shp->mlock_user);
	fput(shm_file);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
223
	ipc_rcu_putref(shp, shm_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
224 225
}

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
/*
 * shm_may_destroy - identifies whether shm segment should be destroyed now
 *
 * Returns true if and only if there are no active users of the segment and
 * one of the following is true:
 *
 * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
 *
 * 2) sysctl kernel.shm_rmid_forced is set to 1.
 */
static bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
{
	return (shp->shm_nattch == 0) &&
	       (ns->shm_rmid_forced ||
		(shp->shm_perm.mode & SHM_DEST));
}

Linus Torvalds's avatar
Linus Torvalds committed
243
/*
244
 * remove the attach descriptor vma.
Linus Torvalds's avatar
Linus Torvalds committed
245 246 247 248
 * free memory for segment if it is marked destroyed.
 * The descriptor has already been removed from the current->mm->mmap list
 * and will later be kfree()d.
 */
249
static void shm_close(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
250
{
251 252
	struct file * file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
Linus Torvalds's avatar
Linus Torvalds committed
253
	struct shmid_kernel *shp;
254
	struct ipc_namespace *ns = sfd->ns;
Kirill Korotaev's avatar
Kirill Korotaev committed
255

Davidlohr Bueso's avatar
Davidlohr Bueso committed
256
	down_write(&shm_ids(ns).rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
257
	/* remove from the list of attaches of the shm segment */
Nadia Derbey's avatar
Nadia Derbey committed
258
	shp = shm_lock(ns, sfd->id);
259
	BUG_ON(IS_ERR(shp));
260
	shp->shm_lprid = task_tgid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
261 262
	shp->shm_dtim = get_seconds();
	shp->shm_nattch--;
263 264 265 266
	if (shm_may_destroy(ns, shp))
		shm_destroy(ns, shp);
	else
		shm_unlock(shp);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
267
	up_write(&shm_ids(ns).rwsem);
268 269
}

Davidlohr Bueso's avatar
Davidlohr Bueso committed
270
/* Called with ns->shm_ids(ns).rwsem locked */
271 272 273
static int shm_try_destroy_current(int id, void *p, void *data)
{
	struct ipc_namespace *ns = data;
274 275
	struct kern_ipc_perm *ipcp = p;
	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
276

277
	if (shp->shm_creator != current)
278 279 280 281 282 283 284 285 286 287 288 289 290
		return 0;

	/*
	 * Mark it as orphaned to destroy the segment when
	 * kernel.shm_rmid_forced is changed.
	 * It is noop if the following shm_may_destroy() returns true.
	 */
	shp->shm_creator = NULL;

	/*
	 * Don't even try to destroy it.  If shm_rmid_forced=0 and IPC_RMID
	 * is not set, it shouldn't be deleted here.
	 */
291
	if (!ns->shm_rmid_forced)
292 293
		return 0;

294 295
	if (shm_may_destroy(ns, shp)) {
		shm_lock_by_ptr(shp);
296
		shm_destroy(ns, shp);
297
	}
298 299 300
	return 0;
}

Davidlohr Bueso's avatar
Davidlohr Bueso committed
301
/* Called with ns->shm_ids(ns).rwsem locked */
302 303 304
static int shm_try_destroy_orphaned(int id, void *p, void *data)
{
	struct ipc_namespace *ns = data;
305 306
	struct kern_ipc_perm *ipcp = p;
	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);
307 308 309 310

	/*
	 * We want to destroy segments without users and with already
	 * exit'ed originating process.
311
	 *
Davidlohr Bueso's avatar
Davidlohr Bueso committed
312
	 * As shp->* are changed under rwsem, it's safe to skip shp locking.
313
	 */
314
	if (shp->shm_creator != NULL)
315 316
		return 0;

317 318
	if (shm_may_destroy(ns, shp)) {
		shm_lock_by_ptr(shp);
Kirill Korotaev's avatar
Kirill Korotaev committed
319
		shm_destroy(ns, shp);
320
	}
321 322 323 324 325
	return 0;
}

void shm_destroy_orphaned(struct ipc_namespace *ns)
{
Davidlohr Bueso's avatar
Davidlohr Bueso committed
326
	down_write(&shm_ids(ns).rwsem);
Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
327
	if (shm_ids(ns).in_use)
328
		idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_orphaned, ns);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
329
	up_write(&shm_ids(ns).rwsem);
330 331 332 333 334
}


void exit_shm(struct task_struct *task)
{
335
	struct ipc_namespace *ns = task->nsproxy->ipc_ns;
336

Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
337 338 339
	if (shm_ids(ns).in_use == 0)
		return;

340
	/* Destroy all already created segments, but not mapped yet */
Davidlohr Bueso's avatar
Davidlohr Bueso committed
341
	down_write(&shm_ids(ns).rwsem);
Vasiliy Kulikov's avatar
Vasiliy Kulikov committed
342
	if (shm_ids(ns).in_use)
343
		idr_for_each(&shm_ids(ns).ipcs_idr, &shm_try_destroy_current, ns);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
344
	up_write(&shm_ids(ns).rwsem);
Linus Torvalds's avatar
Linus Torvalds committed
345 346
}

Nick Piggin's avatar
Nick Piggin committed
347
static int shm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
348 349 350 351
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);

Nick Piggin's avatar
Nick Piggin committed
352
	return sfd->vm_ops->fault(vma, vmf);
353 354 355
}

#ifdef CONFIG_NUMA
356
static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
357 358 359 360 361 362 363 364 365
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
	int err = 0;
	if (sfd->vm_ops->set_policy)
		err = sfd->vm_ops->set_policy(vma, new);
	return err;
}

366 367
static struct mempolicy *shm_get_policy(struct vm_area_struct *vma,
					unsigned long addr)
368 369 370 371 372 373 374
{
	struct file *file = vma->vm_file;
	struct shm_file_data *sfd = shm_file_data(file);
	struct mempolicy *pol = NULL;

	if (sfd->vm_ops->get_policy)
		pol = sfd->vm_ops->get_policy(vma, addr);
375
	else if (vma->vm_policy)
376
		pol = vma->vm_policy;
377

378 379 380 381
	return pol;
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
382 383
static int shm_mmap(struct file * file, struct vm_area_struct * vma)
{
384
	struct shm_file_data *sfd = shm_file_data(file);
385 386
	int ret;

387 388 389 390
	ret = sfd->file->f_op->mmap(sfd->file, vma);
	if (ret != 0)
		return ret;
	sfd->vm_ops = vma->vm_ops;
David Howells's avatar
David Howells committed
391
#ifdef CONFIG_MMU
392
	BUG_ON(!sfd->vm_ops->fault);
David Howells's avatar
David Howells committed
393
#endif
394 395
	vma->vm_ops = &shm_vm_ops;
	shm_open(vma);
396 397

	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
398 399
}

Kirill Korotaev's avatar
Kirill Korotaev committed
400 401
static int shm_release(struct inode *ino, struct file *file)
{
402
	struct shm_file_data *sfd = shm_file_data(file);
Kirill Korotaev's avatar
Kirill Korotaev committed
403

404 405 406
	put_ipc_ns(sfd->ns);
	shm_file_data(file) = NULL;
	kfree(sfd);
Kirill Korotaev's avatar
Kirill Korotaev committed
407 408 409
	return 0;
}

410
static int shm_fsync(struct file *file, loff_t start, loff_t end, int datasync)
411 412 413
{
	struct shm_file_data *sfd = shm_file_data(file);

414 415
	if (!sfd->file->f_op->fsync)
		return -EINVAL;
416
	return sfd->file->f_op->fsync(sfd->file, start, end, datasync);
417 418
}

419 420 421 422 423 424 425 426 427 428
static long shm_fallocate(struct file *file, int mode, loff_t offset,
			  loff_t len)
{
	struct shm_file_data *sfd = shm_file_data(file);

	if (!sfd->file->f_op->fallocate)
		return -EOPNOTSUPP;
	return sfd->file->f_op->fallocate(file, mode, offset, len);
}

429 430 431 432 433
static unsigned long shm_get_unmapped_area(struct file *file,
	unsigned long addr, unsigned long len, unsigned long pgoff,
	unsigned long flags)
{
	struct shm_file_data *sfd = shm_file_data(file);
434 435
	return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
						pgoff, flags);
436 437
}

438
static const struct file_operations shm_file_operations = {
Kirill Korotaev's avatar
Kirill Korotaev committed
439
	.mmap		= shm_mmap,
440
	.fsync		= shm_fsync,
Kirill Korotaev's avatar
Kirill Korotaev committed
441
	.release	= shm_release,
David Howells's avatar
David Howells committed
442 443 444
#ifndef CONFIG_MMU
	.get_unmapped_area	= shm_get_unmapped_area,
#endif
445
	.llseek		= noop_llseek,
446
	.fallocate	= shm_fallocate,
447 448 449 450 451 452
};

static const struct file_operations shm_file_operations_huge = {
	.mmap		= shm_mmap,
	.fsync		= shm_fsync,
	.release	= shm_release,
453
	.get_unmapped_area	= shm_get_unmapped_area,
454
	.llseek		= noop_llseek,
455
	.fallocate	= shm_fallocate,
Linus Torvalds's avatar
Linus Torvalds committed
456 457
};

458 459 460 461 462
int is_file_shm_hugepages(struct file *file)
{
	return file->f_op == &shm_file_operations_huge;
}

463
static const struct vm_operations_struct shm_vm_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
464 465
	.open	= shm_open,	/* callback for a new vm-area open */
	.close	= shm_close,	/* callback for when the vm-area is released */
466
	.fault	= shm_fault,
467 468 469
#if defined(CONFIG_NUMA)
	.set_policy = shm_set_policy,
	.get_policy = shm_get_policy,
Linus Torvalds's avatar
Linus Torvalds committed
470 471 472
#endif
};

Nadia Derbey's avatar
Nadia Derbey committed
473 474 475 476 477
/**
 * newseg - Create a new shared memory segment
 * @ns: namespace
 * @params: ptr to the structure that contains key, size and shmflg
 *
Davidlohr Bueso's avatar
Davidlohr Bueso committed
478
 * Called with shm_ids.rwsem held as a writer.
Nadia Derbey's avatar
Nadia Derbey committed
479 480
 */

Nadia Derbey's avatar
Nadia Derbey committed
481
static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
Linus Torvalds's avatar
Linus Torvalds committed
482
{
Nadia Derbey's avatar
Nadia Derbey committed
483 484 485
	key_t key = params->key;
	int shmflg = params->flg;
	size_t size = params->u.size;
Linus Torvalds's avatar
Linus Torvalds committed
486 487
	int error;
	struct shmid_kernel *shp;
488
	size_t numpages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
489 490 491
	struct file * file;
	char name[13];
	int id;
492
	vm_flags_t acctflag = 0;
Linus Torvalds's avatar
Linus Torvalds committed
493

Kirill Korotaev's avatar
Kirill Korotaev committed
494
	if (size < SHMMIN || size > ns->shm_ctlmax)
Linus Torvalds's avatar
Linus Torvalds committed
495 496
		return -EINVAL;

497
	if (ns->shm_tot + numpages > ns->shm_ctlall)
Linus Torvalds's avatar
Linus Torvalds committed
498 499 500 501 502 503 504
		return -ENOSPC;

	shp = ipc_rcu_alloc(sizeof(*shp));
	if (!shp)
		return -ENOMEM;

	shp->shm_perm.key = key;
Andrew Morton's avatar
Andrew Morton committed
505
	shp->shm_perm.mode = (shmflg & S_IRWXUGO);
Linus Torvalds's avatar
Linus Torvalds committed
506 507 508 509 510
	shp->mlock_user = NULL;

	shp->shm_perm.security = NULL;
	error = security_shm_alloc(shp);
	if (error) {
Davidlohr Bueso's avatar
Davidlohr Bueso committed
511
		ipc_rcu_putref(shp, ipc_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
512 513 514
		return error;
	}

515
	sprintf (name, "SYSV%08x", key);
Linus Torvalds's avatar
Linus Torvalds committed
516
	if (shmflg & SHM_HUGETLB) {
517
		struct hstate *hs;
518 519
		size_t hugesize;

520
		hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
521 522 523 524 525
		if (!hs) {
			error = -EINVAL;
			goto no_file;
		}
		hugesize = ALIGN(size, huge_page_size(hs));
526

527 528 529
		/* hugetlb_file_setup applies strict accounting */
		if (shmflg & SHM_NORESERVE)
			acctflag = VM_NORESERVE;
530
		file = hugetlb_file_setup(name, hugesize, acctflag,
531 532
				  &shp->mlock_user, HUGETLB_SHMFS_INODE,
				(shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
Linus Torvalds's avatar
Linus Torvalds committed
533
	} else {
534 535 536 537 538 539
		/*
		 * Do not allow no accounting for OVERCOMMIT_NEVER, even
	 	 * if it's asked for.
		 */
		if  ((shmflg & SHM_NORESERVE) &&
				sysctl_overcommit_memory != OVERCOMMIT_NEVER)
540
			acctflag = VM_NORESERVE;
541
		file = shmem_file_setup(name, size, acctflag);
Linus Torvalds's avatar
Linus Torvalds committed
542 543 544 545 546
	}
	error = PTR_ERR(file);
	if (IS_ERR(file))
		goto no_file;

547
	id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
548 549
	if (id < 0) {
		error = id;
Linus Torvalds's avatar
Linus Torvalds committed
550
		goto no_id;
551
	}
Linus Torvalds's avatar
Linus Torvalds committed
552

553
	shp->shm_cprid = task_tgid_vnr(current);
Linus Torvalds's avatar
Linus Torvalds committed
554 555 556 557 558 559
	shp->shm_lprid = 0;
	shp->shm_atim = shp->shm_dtim = 0;
	shp->shm_ctim = get_seconds();
	shp->shm_segsz = size;
	shp->shm_nattch = 0;
	shp->shm_file = file;
560
	shp->shm_creator = current;
561

562 563 564 565
	/*
	 * shmid gets reported as "inode#" in /proc/pid/maps.
	 * proc-ps tools use this. Changing this will break them.
	 */
Al Viro's avatar
Al Viro committed
566
	file_inode(file)->i_ino = shp->shm_perm.id;
567

Kirill Korotaev's avatar
Kirill Korotaev committed
568
	ns->shm_tot += numpages;
Nadia Derbey's avatar
Nadia Derbey committed
569
	error = shp->shm_perm.id;
570

571
	ipc_unlock_object(&shp->shm_perm);
572
	rcu_read_unlock();
Nadia Derbey's avatar
Nadia Derbey committed
573
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
574 575

no_id:
576
	if (is_file_hugepages(file) && shp->mlock_user)
577
		user_shm_unlock(size, shp->mlock_user);
Linus Torvalds's avatar
Linus Torvalds committed
578 579
	fput(file);
no_file:
Davidlohr Bueso's avatar
Davidlohr Bueso committed
580
	ipc_rcu_putref(shp, shm_rcu_free);
Linus Torvalds's avatar
Linus Torvalds committed
581 582 583
	return error;
}

Nadia Derbey's avatar
Nadia Derbey committed
584
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
585
 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbey's avatar
Nadia Derbey committed
586
 */
Nadia Derbey's avatar
Nadia Derbey committed
587
static inline int shm_security(struct kern_ipc_perm *ipcp, int shmflg)
Nadia Derbey's avatar
Nadia Derbey committed
588
{
Nadia Derbey's avatar
Nadia Derbey committed
589 590 591 592
	struct shmid_kernel *shp;

	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
	return security_shm_associate(shp, shmflg);
Nadia Derbey's avatar
Nadia Derbey committed
593 594
}

Nadia Derbey's avatar
Nadia Derbey committed
595
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
596
 * Called with shm_ids.rwsem and ipcp locked.
Nadia Derbey's avatar
Nadia Derbey committed
597
 */
Nadia Derbey's avatar
Nadia Derbey committed
598 599
static inline int shm_more_checks(struct kern_ipc_perm *ipcp,
				struct ipc_params *params)
Nadia Derbey's avatar
Nadia Derbey committed
600
{
Nadia Derbey's avatar
Nadia Derbey committed
601 602 603 604
	struct shmid_kernel *shp;

	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
	if (shp->shm_segsz < params->u.size)
Nadia Derbey's avatar
Nadia Derbey committed
605 606 607 608 609
		return -EINVAL;

	return 0;
}

610
SYSCALL_DEFINE3(shmget, key_t, key, size_t, size, int, shmflg)
Linus Torvalds's avatar
Linus Torvalds committed
611
{
Kirill Korotaev's avatar
Kirill Korotaev committed
612
	struct ipc_namespace *ns;
Nadia Derbey's avatar
Nadia Derbey committed
613 614
	struct ipc_ops shm_ops;
	struct ipc_params shm_params;
Kirill Korotaev's avatar
Kirill Korotaev committed
615 616

	ns = current->nsproxy->ipc_ns;
Linus Torvalds's avatar
Linus Torvalds committed
617

Nadia Derbey's avatar
Nadia Derbey committed
618 619 620
	shm_ops.getnew = newseg;
	shm_ops.associate = shm_security;
	shm_ops.more_checks = shm_more_checks;
Nadia Derbey's avatar
Nadia Derbey committed
621

Nadia Derbey's avatar
Nadia Derbey committed
622 623 624
	shm_params.key = key;
	shm_params.flg = shmflg;
	shm_params.u.size = size;
Linus Torvalds's avatar
Linus Torvalds committed
625

Nadia Derbey's avatar
Nadia Derbey committed
626
	return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params);
Linus Torvalds's avatar
Linus Torvalds committed
627 628 629 630 631 632 633 634 635 636 637
}

static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version)
{
	switch(version) {
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct shmid_ds out;

638
		memset(&out, 0, sizeof(out));
Linus Torvalds's avatar
Linus Torvalds committed
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
		ipc64_perm_to_ipc_perm(&in->shm_perm, &out.shm_perm);
		out.shm_segsz	= in->shm_segsz;
		out.shm_atime	= in->shm_atime;
		out.shm_dtime	= in->shm_dtime;
		out.shm_ctime	= in->shm_ctime;
		out.shm_cpid	= in->shm_cpid;
		out.shm_lpid	= in->shm_lpid;
		out.shm_nattch	= in->shm_nattch;

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

655 656
static inline unsigned long
copy_shmid_from_user(struct shmid64_ds *out, void __user *buf, int version)
Linus Torvalds's avatar
Linus Torvalds committed
657 658 659
{
	switch(version) {
	case IPC_64:
660
		if (copy_from_user(out, buf, sizeof(*out)))
Linus Torvalds's avatar
Linus Torvalds committed
661 662 663 664 665 666 667 668 669
			return -EFAULT;
		return 0;
	case IPC_OLD:
	    {
		struct shmid_ds tbuf_old;

		if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
			return -EFAULT;

670 671 672
		out->shm_perm.uid	= tbuf_old.shm_perm.uid;
		out->shm_perm.gid	= tbuf_old.shm_perm.gid;
		out->shm_perm.mode	= tbuf_old.shm_perm.mode;
Linus Torvalds's avatar
Linus Torvalds committed
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706

		return 0;
	    }
	default:
		return -EINVAL;
	}
}

static inline unsigned long copy_shminfo_to_user(void __user *buf, struct shminfo64 *in, int version)
{
	switch(version) {
	case IPC_64:
		return copy_to_user(buf, in, sizeof(*in));
	case IPC_OLD:
	    {
		struct shminfo out;

		if(in->shmmax > INT_MAX)
			out.shmmax = INT_MAX;
		else
			out.shmmax = (int)in->shmmax;

		out.shmmin	= in->shmmin;
		out.shmmni	= in->shmmni;
		out.shmseg	= in->shmseg;
		out.shmall	= in->shmall; 

		return copy_to_user(buf, &out, sizeof(out));
	    }
	default:
		return -EINVAL;
	}
}

707 708
/*
 * Calculate and add used RSS and swap pages of a shm.
Davidlohr Bueso's avatar
Davidlohr Bueso committed
709
 * Called with shm_ids.rwsem held as a reader
710 711 712 713 714 715
 */
static void shm_add_rss_swap(struct shmid_kernel *shp,
	unsigned long *rss_add, unsigned long *swp_add)
{
	struct inode *inode;

Al Viro's avatar
Al Viro committed
716
	inode = file_inode(shp->shm_file);
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734

	if (is_file_hugepages(shp->shm_file)) {
		struct address_space *mapping = inode->i_mapping;
		struct hstate *h = hstate_file(shp->shm_file);
		*rss_add += pages_per_huge_page(h) * mapping->nrpages;
	} else {
#ifdef CONFIG_SHMEM
		struct shmem_inode_info *info = SHMEM_I(inode);
		spin_lock(&info->lock);
		*rss_add += inode->i_mapping->nrpages;
		*swp_add += info->swapped;
		spin_unlock(&info->lock);
#else
		*rss_add += inode->i_mapping->nrpages;
#endif
	}
}

Nadia Derbey's avatar
Nadia Derbey committed
735
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
736
 * Called with shm_ids.rwsem held as a reader
Nadia Derbey's avatar
Nadia Derbey committed
737
 */
Kirill Korotaev's avatar
Kirill Korotaev committed
738 739
static void shm_get_stat(struct ipc_namespace *ns, unsigned long *rss,
		unsigned long *swp)
Linus Torvalds's avatar
Linus Torvalds committed
740
{
Nadia Derbey's avatar
Nadia Derbey committed
741 742
	int next_id;
	int total, in_use;
Linus Torvalds's avatar
Linus Torvalds committed
743 744 745 746

	*rss = 0;
	*swp = 0;

Nadia Derbey's avatar
Nadia Derbey committed
747 748 749
	in_use = shm_ids(ns).in_use;

	for (total = 0, next_id = 0; total < in_use; next_id++) {
750
		struct kern_ipc_perm *ipc;
Linus Torvalds's avatar
Linus Torvalds committed
751 752
		struct shmid_kernel *shp;

753 754
		ipc = idr_find(&shm_ids(ns).ipcs_idr, next_id);
		if (ipc == NULL)
Linus Torvalds's avatar
Linus Torvalds committed
755
			continue;
756
		shp = container_of(ipc, struct shmid_kernel, shm_perm);
Linus Torvalds's avatar
Linus Torvalds committed
757

758
		shm_add_rss_swap(shp, rss, swp);
Nadia Derbey's avatar
Nadia Derbey committed
759 760

		total++;
Linus Torvalds's avatar
Linus Torvalds committed
761 762 763
	}
}

764
/*
Davidlohr Bueso's avatar
Davidlohr Bueso committed
765
 * This function handles some shmctl commands which require the rwsem
766
 * to be held in write mode.
Davidlohr Bueso's avatar
Davidlohr Bueso committed
767
 * NOTE: no locks must be held, the rwsem is taken inside this function.
768 769 770
 */
static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
		       struct shmid_ds __user *buf, int version)
Linus Torvalds's avatar
Linus Torvalds committed
771
{
772
	struct kern_ipc_perm *ipcp;
773
	struct shmid64_ds shmid64;
774 775 776 777
	struct shmid_kernel *shp;
	int err;

	if (cmd == IPC_SET) {
778
		if (copy_shmid_from_user(&shmid64, buf, version))
779 780 781
			return -EFAULT;
	}

Davidlohr Bueso's avatar
Davidlohr Bueso committed
782
	down_write(&shm_ids(ns).rwsem);
783 784
	rcu_read_lock();

785 786
	ipcp = ipcctl_pre_down_nolock(ns, &shm_ids(ns), shmid, cmd,
				      &shmid64.shm_perm, 0);
787 788 789 790
	if (IS_ERR(ipcp)) {
		err = PTR_ERR(ipcp);
		goto out_unlock1;
	}
791

792
	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
793 794 795

	err = security_shm_shmctl(shp, cmd);
	if (err)
796
		goto out_unlock1;
797

798 799
	switch (cmd) {
	case IPC_RMID:
800
		ipc_lock_object(&shp->shm_perm);
801
		/* do_shm_rmid unlocks the ipc object and rcu */
802 803 804
		do_shm_rmid(ns, ipcp);
		goto out_up;
	case IPC_SET:
805
		ipc_lock_object(&shp->shm_perm);
806 807
		err = ipc_update_perm(&shmid64.shm_perm, ipcp);
		if (err)
808
			goto out_unlock0;
809 810 811 812
		shp->shm_ctim = get_seconds();
		break;
	default:
		err = -EINVAL;
813
		goto out_unlock1;
814
	}
815 816 817 818 819

out_unlock0:
	ipc_unlock_object(&shp->shm_perm);
out_unlock1:
	rcu_read_unlock();
820
out_up:
Davidlohr Bueso's avatar
Davidlohr Bueso committed
821
	up_write(&shm_ids(ns).rwsem);
822 823 824
	return err;
}

825 826
static int shmctl_nolock(struct ipc_namespace *ns, int shmid,
			 int cmd, int version, void __user *buf)
827
{
828
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
829 830
	struct shmid_kernel *shp;

831 832 833 834 835
	/* preliminary security checks for *_INFO */
	if (cmd == IPC_INFO || cmd == SHM_INFO) {
		err = security_shm_shmctl(NULL, cmd);
		if (err)
			return err;
Linus Torvalds's avatar
Linus Torvalds committed
836 837
	}

838
	switch (cmd) {
Linus Torvalds's avatar
Linus Torvalds committed
839 840 841 842
	case IPC_INFO:
	{
		struct shminfo64 shminfo;

WANG Cong's avatar
WANG Cong committed
843
		memset(&shminfo, 0, sizeof(shminfo));
Kirill Korotaev's avatar
Kirill Korotaev committed
844 845 846
		shminfo.shmmni = shminfo.shmseg = ns->shm_ctlmni;
		shminfo.shmmax = ns->shm_ctlmax;
		shminfo.shmall = ns->shm_ctlall;
Linus Torvalds's avatar
Linus Torvalds committed
847 848 849 850

		shminfo.shmmin = SHMMIN;
		if(copy_shminfo_to_user (buf, &shminfo, version))
			return -EFAULT;
Nadia Derbey's avatar
Nadia Derbey committed
851

Davidlohr Bueso's avatar
Davidlohr Bueso committed
852
		down_read(&shm_ids(ns).rwsem);
Nadia Derbey's avatar
Nadia Derbey committed
853
		err = ipc_get_maxid(&shm_ids(ns));
Davidlohr Bueso's avatar
Davidlohr Bueso committed
854
		up_read(&shm_ids(ns).rwsem);
Nadia Derbey's avatar
Nadia Derbey committed
855

Linus Torvalds's avatar
Linus Torvalds committed
856 857 858 859 860 861 862 863
		if(err<0)
			err = 0;
		goto out;
	}
	case SHM_INFO:
	{
		struct shm_info shm_info;

WANG Cong's avatar
WANG Cong committed
864
		memset(&shm_info, 0, sizeof(shm_info));
Davidlohr Bueso's avatar
Davidlohr Bueso committed
865
		down_read(&shm_ids(ns).rwsem);
Kirill Korotaev's avatar
Kirill Korotaev committed
866 867 868
		shm_info.used_ids = shm_ids(ns).in_use;
		shm_get_stat (ns, &shm_info.shm_rss, &shm_info.shm_swp);
		shm_info.shm_tot = ns->shm_tot;
Linus Torvalds's avatar
Linus Torvalds committed
869 870
		shm_info.swap_attempts = 0;
		shm_info.swap_successes = 0;
Nadia Derbey's avatar
Nadia Derbey committed
871
		err = ipc_get_maxid(&shm_ids(ns));
Davidlohr Bueso's avatar
Davidlohr Bueso committed
872
		up_read(&shm_ids(ns).rwsem);
WANG Cong's avatar
WANG Cong committed
873
		if (copy_to_user(buf, &shm_info, sizeof(shm_info))) {
Linus Torvalds's avatar
Linus Torvalds committed
874 875 876 877 878 879 880 881 882 883 884 885
			err = -EFAULT;
			goto out;
		}

		err = err < 0 ? 0 : err;
		goto out;
	}
	case SHM_STAT:
	case IPC_STAT:
	{
		struct shmid64_ds tbuf;
		int result;
886

887
		rcu_read_lock();
888
		if (cmd == SHM_STAT) {
889
			shp = shm_obtain_object(ns, shmid);
890 891
			if (IS_ERR(shp)) {
				err = PTR_ERR(shp);
892
				goto out_unlock;
893
			}
Nadia Derbey's avatar
Nadia Derbey committed
894
			result = shp->shm_perm.id;
Linus Torvalds's avatar
Linus Torvalds committed
895
		} else {
896
			shp = shm_obtain_object_check(ns, shmid);
897 898
			if (IS_ERR(shp)) {
				err = PTR_ERR(shp);
899
				goto out_unlock;
900
			}
Linus Torvalds's avatar
Linus Torvalds committed
901 902
			result = 0;
		}
903

WANG Cong's avatar
WANG Cong committed
904
		err = -EACCES;
905
		if (ipcperms(ns, &shp->shm_perm, S_IRUGO))
Linus Torvalds's avatar
Linus Torvalds committed
906
			goto out_unlock;
907

Linus Torvalds's avatar
Linus Torvalds committed
908 909 910
		err = security_shm_shmctl(shp, cmd);
		if (err)
			goto out_unlock;
911

912
		memset(&tbuf, 0, sizeof(tbuf));
Linus Torvalds's avatar
Linus Torvalds committed
913 914 915 916 917 918 919
		kernel_to_ipc64_perm(&shp->shm_perm, &tbuf.shm_perm);
		tbuf.shm_segsz	= shp->shm_segsz;
		tbuf.shm_atime	= shp->shm_atim;
		tbuf.shm_dtime	= shp->shm_dtim;
		tbuf.shm_ctime	= shp->shm_ctim;
		tbuf.shm_cpid	= shp->shm_cprid;
		tbuf.shm_lpid	= shp->shm_lprid;
920
		tbuf.shm_nattch	= shp->shm_nattch;
921 922 923
		rcu_read_unlock();

		if (copy_shmid_to_user(buf, &tbuf, version))
Linus Torvalds's avatar
Linus Torvalds committed
924 925 926 927 928
			err = -EFAULT;
		else
			err = result;
		goto out;
	}
929 930 931 932 933
	default:
		return -EINVAL;
	}

out_unlock:
934
	rcu_read_unlock();
935 936 937 938 939 940 941 942 943 944
out:
	return err;
}

SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
	struct shmid_kernel *shp;
	int err, version;
	struct ipc_namespace *ns;

945 946
	if (cmd < 0 || shmid < 0)
		return -EINVAL;
947 948 949 950 951 952 953 954 955 956

	version = ipc_parse_version(&cmd);
	ns = current->nsproxy->ipc_ns;

	switch (cmd) {
	case IPC_INFO:
	case SHM_INFO:
	case SHM_STAT:
	case IPC_STAT:
		return shmctl_nolock(ns, shmid, cmd, version, buf);
957 958 959
	case IPC_RMID:
	case IPC_SET:
		return shmctl_down(ns, shmid, cmd, buf, version);
Linus Torvalds's avatar
Linus Torvalds committed
960 961 962
	case SHM_LOCK:
	case SHM_UNLOCK:
	{
963
		struct file *shm_file;
964

965 966
		rcu_read_lock();
		shp = shm_obtain_object_check(ns, shmid);
967 968
		if (IS_ERR(shp)) {
			err = PTR_ERR(shp);
969
			goto out_unlock1;
Linus Torvalds's avatar
Linus Torvalds committed
970 971
		}

Al Viro's avatar
Al Viro committed
972
		audit_ipc_obj(&(shp->shm_perm));
973 974 975
		err = security_shm_shmctl(shp, cmd);
		if (err)
			goto out_unlock1;
Steve Grubb's avatar
Steve Grubb committed
976

977
		ipc_lock_object(&shp->shm_perm);
978 979 980 981 982 983 984

		/* check if shm_destroy() is tearing down shp */
		if (!ipc_valid_object(&shp->shm_perm)) {
			err = -EIDRM;
			goto out_unlock0;
		}

985
		if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
986 987
			kuid_t euid = current_euid();
			if (!uid_eq(euid, shp->shm_perm.uid) &&
988 989
			    !uid_eq(euid, shp->shm_perm.cuid)) {
				err = -EPERM;
990
				goto out_unlock0;
991 992 993
			}
			if (cmd == SHM_LOCK && !rlimit(RLIMIT_MEMLOCK)) {
				err = -EPERM;
994
				goto out_unlock0;
995
			}
Linus Torvalds's avatar
Linus Torvalds committed
996 997
		}

998 999
		shm_file = shp->shm_file;
		if (is_file_hugepages(shm_file))
1000
			goto out_unlock0;
1001 1002

		if (cmd == SHM_LOCK) {
1003
			struct user_struct *user = current_user();
1004 1005 1006 1007
			err = shmem_lock(shm_file, 1, user);
			if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
				shp->shm_perm.mode |= SHM_LOCKED;
				shp->mlock_user = user;
Linus Torvalds's avatar
Linus Torvalds committed
1008
			}
1009
			goto out_unlock0;
Linus Torvalds's avatar
Linus Torvalds committed
1010
		}
1011 1012 1013

		/* SHM_UNLOCK */
		if (!(shp->shm_perm.mode & SHM_LOCKED))
1014
			goto out_unlock0;
1015 1016 1017 1018
		shmem_lock(shm_file, 0, shp->mlock_user);
		shp->shm_perm.mode &= ~SHM_LOCKED;
		shp->mlock_user = NULL;
		get_file(shm_file);
1019 1020
		ipc_unlock_object(&shp->shm_perm);
		rcu_read_unlock();
1021
		shmem_unlock_mapping(shm_file->f_mapping);
1022

1023
		fput(shm_file);
1024
		return err;
1025
	}
Linus Torvalds's avatar
Linus Torvalds committed
1026
	default:
1027
		return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
1028 1029
	}

1030 1031 1032 1033
out_unlock0:
	ipc_unlock_object(&shp->shm_perm);
out_unlock1:
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
	return err;
}

/*
 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
 *
 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
 * "raddr" thing points to kernel space, and there has to be a wrapper around
 * this.
 */
Will Deacon's avatar
Will Deacon committed
1044 1045
long do_shmat(int shmid, char __user *shmaddr, int shmflg, ulong *raddr,
	      unsigned long shmlba)
Linus Torvalds's avatar
Linus Torvalds committed
1046 1047 1048 1049 1050 1051 1052 1053 1054
{
	struct shmid_kernel *shp;
	unsigned long addr;
	unsigned long size;
	struct file * file;
	int    err;
	unsigned long flags;
	unsigned long prot;
	int acc_mode;
Kirill Korotaev's avatar
Kirill Korotaev committed
1055
	struct ipc_namespace *ns;
1056 1057
	struct shm_file_data *sfd;
	struct path path;
1058
	fmode_t f_mode;
1059
	unsigned long populate = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1060

1061 1062
	err = -EINVAL;
	if (shmid < 0)
Linus Torvalds's avatar
Linus Torvalds committed
1063
		goto out;
1064
	else if ((addr = (ulong)shmaddr)) {
Will Deacon's avatar
Will Deacon committed
1065
		if (addr & (shmlba - 1)) {
Linus Torvalds's avatar
Linus Torvalds committed
1066
			if (shmflg & SHM_RND)
Will Deacon's avatar
Will Deacon committed
1067
				addr &= ~(shmlba - 1);	   /* round down */
Linus Torvalds's avatar
Linus Torvalds committed
1068 1069 1070 1071
			else
#ifndef __ARCH_FORCE_SHMLBA
				if (addr & ~PAGE_MASK)
#endif
1072
					goto out;
Linus Torvalds's avatar
Linus Torvalds committed
1073 1074 1075 1076
		}
		flags = MAP_SHARED | MAP_FIXED;
	} else {
		if ((shmflg & SHM_REMAP))
1077
			goto out;
Linus Torvalds's avatar
Linus Torvalds committed
1078 1079 1080 1081 1082 1083 1084

		flags = MAP_SHARED;
	}

	if (shmflg & SHM_RDONLY) {
		prot = PROT_READ;
		acc_mode = S_IRUGO;
1085
		f_mode = FMODE_READ;
Linus Torvalds's avatar
Linus Torvalds committed
1086 1087 1088
	} else {
		prot = PROT_READ | PROT_WRITE;
		acc_mode = S_IRUGO | S_IWUGO;
1089
		f_mode = FMODE_READ | FMODE_WRITE;
Linus Torvalds's avatar
Linus Torvalds committed
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
	}
	if (shmflg & SHM_EXEC) {
		prot |= PROT_EXEC;
		acc_mode |= S_IXUGO;
	}

	/*
	 * We cannot rely on the fs check since SYSV IPC does have an
	 * additional creator id...
	 */
Kirill Korotaev's avatar
Kirill Korotaev committed
1100
	ns = current->nsproxy->ipc_ns;
1101 1102
	rcu_read_lock();
	shp = shm_obtain_object_check(ns, shmid);
1103 1104
	if (IS_ERR(shp)) {
		err = PTR_ERR(shp);
1105
		goto out_unlock;
1106
	}
1107 1108

	err = -EACCES;
1109
	if (ipcperms(ns, &shp->shm_perm, acc_mode))
1110
		goto out_unlock;
Linus Torvalds's avatar
Linus Torvalds committed
1111 1112

	err = security_shm_shmat(shp, shmaddr, shmflg);
1113 1114 1115
	if (err)
		goto out_unlock;

1116
	ipc_lock_object(&shp->shm_perm);
1117 1118

	/* check if shm_destroy() is tearing down shp */
1119
	if (!ipc_valid_object(&shp->shm_perm)) {
Greg Thelen's avatar