i915_gem.c 131 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"
Chris Wilson's avatar
Chris Wilson committed
32
#include "i915_trace.h"
33
#include "intel_drv.h"
34
#include <linux/slab.h>
35
#include <linux/swap.h>
36
#include <linux/pci.h>
37
#include <linux/intel-gtt.h>
38

39
static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
40 41 42

static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
						  bool pipelined);
43 44 45 46 47 48 49 50
static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
					     int write);
static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
						     uint64_t offset,
						     uint64_t size);
static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
51 52
static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
					  bool interruptible);
53 54 55
static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
					   unsigned alignment);
static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
56 57 58
static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
				struct drm_i915_gem_pwrite *args,
				struct drm_file *file_priv);
59
static void i915_gem_free_object_tail(struct drm_gem_object *obj);
60

61 62 63 64 65 66 67
static int
i915_gem_object_get_pages(struct drm_gem_object *obj,
			  gfp_t gfpmask);

static void
i915_gem_object_put_pages(struct drm_gem_object *obj);

68 69 70
static LIST_HEAD(shrink_list);
static DEFINE_SPINLOCK(shrink_list_lock);

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
}

static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.gtt_count++;
	dev_priv->mm.gtt_memory += size;
}

static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.gtt_count--;
	dev_priv->mm.gtt_memory -= size;
}

static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.pin_count++;
	dev_priv->mm.pin_memory += size;
}

static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.pin_count--;
	dev_priv->mm.pin_memory -= size;
}

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
int
i915_gem_check_is_wedged(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct completion *x = &dev_priv->error_completion;
	unsigned long flags;
	int ret;

	if (!atomic_read(&dev_priv->mm.wedged))
		return 0;

	ret = wait_for_completion_interruptible(x);
	if (ret)
		return ret;

	/* Success, we reset the GPU! */
	if (!atomic_read(&dev_priv->mm.wedged))
		return 0;

	/* GPU is hung, bump the completion count to account for
	 * the token we just consumed so that we never hit zero and
	 * end up waiting upon a subsequent completion event that
	 * will never happen.
	 */
	spin_lock_irqsave(&x->wait.lock, flags);
	x->done++;
	spin_unlock_irqrestore(&x->wait.lock, flags);
	return -EIO;
}

144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
static int i915_mutex_lock_interruptible(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	ret = i915_gem_check_is_wedged(dev);
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

	if (atomic_read(&dev_priv->mm.wedged)) {
		mutex_unlock(&dev->struct_mutex);
		return -EAGAIN;
	}

162
	WARN_ON(i915_verify_lists(dev));
163 164
	return 0;
}
165

166 167 168 169 170 171 172 173
static inline bool
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
{
	return obj_priv->gtt_space &&
		!obj_priv->active &&
		obj_priv->pin_count == 0;
}

174 175
int i915_gem_do_init(struct drm_device *dev,
		     unsigned long start,
176
		     unsigned long end)
177 178 179
{
	drm_i915_private_t *dev_priv = dev->dev_private;

180 181 182
	if (start >= end ||
	    (start & (PAGE_SIZE - 1)) != 0 ||
	    (end & (PAGE_SIZE - 1)) != 0) {
183 184 185
		return -EINVAL;
	}

186 187
	drm_mm_init(&dev_priv->mm.gtt_space, start,
		    end - start);
188

189
	dev_priv->mm.gtt_total = end - start;
190 191 192

	return 0;
}
193

194 195 196 197 198 199 200 201 202
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
		    struct drm_file *file_priv)
{
	struct drm_i915_gem_init *args = data;
	int ret;

	mutex_lock(&dev->struct_mutex);
	ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
203 204
	mutex_unlock(&dev->struct_mutex);

205
	return ret;
206 207
}

208 209 210 211
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
			    struct drm_file *file_priv)
{
212
	struct drm_i915_private *dev_priv = dev->dev_private;
213 214 215 216 217
	struct drm_i915_gem_get_aperture *args = data;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

218 219 220 221
	mutex_lock(&dev->struct_mutex);
	args->aper_size = dev_priv->mm.gtt_total;
	args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
	mutex_unlock(&dev->struct_mutex);
222 223 224 225

	return 0;
}

226 227 228 229 230 231 232 233 234 235

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_i915_gem_create *args = data;
	struct drm_gem_object *obj;
236 237
	int ret;
	u32 handle;
238 239 240 241

	args->size = roundup(args->size, PAGE_SIZE);

	/* Allocate the new object */
242
	obj = i915_gem_alloc_object(dev, args->size);
243 244 245 246
	if (obj == NULL)
		return -ENOMEM;

	ret = drm_gem_handle_create(file_priv, obj, &handle);
247 248
	/* drop reference from allocate - handle holds it now */
	drm_gem_object_unreference_unlocked(obj);
249
	if (ret) {
250
		return ret;
251
	}
252

253
	args->handle = handle;
254 255 256
	return 0;
}

257 258 259 260 261 262 263
static inline int
fast_shmem_read(struct page **pages,
		loff_t page_base, int page_offset,
		char __user *data,
		int length)
{
	char __iomem *vaddr;
264
	int unwritten;
265 266 267 268

	vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
	if (vaddr == NULL)
		return -ENOMEM;
269
	unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
270 271
	kunmap_atomic(vaddr, KM_USER0);

272 273 274 275
	if (unwritten)
		return -EFAULT;

	return 0;
276 277
}

278 279 280
static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
{
	drm_i915_private_t *dev_priv = obj->dev->dev_private;
281
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
282 283 284 285 286

	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		obj_priv->tiling_mode != I915_TILING_NONE;
}

287
static inline void
288 289 290 291 292 293 294 295
slow_shmem_copy(struct page *dst_page,
		int dst_offset,
		struct page *src_page,
		int src_offset,
		int length)
{
	char *dst_vaddr, *src_vaddr;

296 297
	dst_vaddr = kmap(dst_page);
	src_vaddr = kmap(src_page);
298 299 300

	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);

301 302
	kunmap(src_page);
	kunmap(dst_page);
303 304
}

305
static inline void
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
slow_shmem_bit17_copy(struct page *gpu_page,
		      int gpu_offset,
		      struct page *cpu_page,
		      int cpu_offset,
		      int length,
		      int is_read)
{
	char *gpu_vaddr, *cpu_vaddr;

	/* Use the unswizzled path if this page isn't affected. */
	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
		if (is_read)
			return slow_shmem_copy(cpu_page, cpu_offset,
					       gpu_page, gpu_offset, length);
		else
			return slow_shmem_copy(gpu_page, gpu_offset,
					       cpu_page, cpu_offset, length);
	}

325 326
	gpu_vaddr = kmap(gpu_page);
	cpu_vaddr = kmap(cpu_page);
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
	 * XORing with the other bits (A9 for Y, A9 and A10 for X)
	 */
	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		if (is_read) {
			memcpy(cpu_vaddr + cpu_offset,
			       gpu_vaddr + swizzled_gpu_offset,
			       this_length);
		} else {
			memcpy(gpu_vaddr + swizzled_gpu_offset,
			       cpu_vaddr + cpu_offset,
			       this_length);
		}
		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

350 351
	kunmap(cpu_page);
	kunmap(gpu_page);
352 353
}

354 355 356 357 358 359 360 361 362 363
/**
 * This is the fast shmem pread path, which attempts to copy_from_user directly
 * from the backing pages of the object to the user's address space.  On a
 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 */
static int
i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
			  struct drm_i915_gem_pread *args,
			  struct drm_file *file_priv)
{
364
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
365 366 367 368 369 370 371 372 373
	ssize_t remain;
	loff_t offset, page_base;
	char __user *user_data;
	int page_offset, page_length;
	int ret;

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;

374 375 376
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
377

378
	ret = i915_gem_object_get_pages(obj, 0);
379 380 381 382 383 384 385 386
	if (ret != 0)
		goto fail_unlock;

	ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
							args->size);
	if (ret != 0)
		goto fail_put_pages;

387
	obj_priv = to_intel_bo(obj);
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
	offset = args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		page_base = (offset & ~(PAGE_SIZE-1));
		page_offset = offset & (PAGE_SIZE-1);
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		ret = fast_shmem_read(obj_priv->pages,
				      page_base, page_offset,
				      user_data, page_length);
		if (ret)
			goto fail_put_pages;

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

422 423 424 425 426
static int
i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
{
	int ret;

427
	ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
428 429 430 431 432 433 434

	/* If we've insufficient memory to map in the pages, attempt
	 * to make some space by throwing out some old buffers.
	 */
	if (ret == -ENOMEM) {
		struct drm_device *dev = obj->dev;

435 436
		ret = i915_gem_evict_something(dev, obj->size,
					       i915_gem_get_gtt_alignment(obj));
437 438 439
		if (ret)
			return ret;

440
		ret = i915_gem_object_get_pages(obj, 0);
441 442 443 444 445
	}

	return ret;
}

446 447 448 449 450 451 452 453 454 455 456
/**
 * This is the fallback shmem pread path, which allocates temporary storage
 * in kernel space to copy_to_user into outside of the struct_mutex, so we
 * can copy out of the object's backing pages while holding the struct mutex
 * and not take page faults.
 */
static int
i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
			  struct drm_i915_gem_pread *args,
			  struct drm_file *file_priv)
{
457
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
458 459 460 461 462 463 464 465 466 467
	struct mm_struct *mm = current->mm;
	struct page **user_pages;
	ssize_t remain;
	loff_t offset, pinned_pages, i;
	loff_t first_data_page, last_data_page, num_pages;
	int shmem_page_index, shmem_page_offset;
	int data_page_index,  data_page_offset;
	int page_length;
	int ret;
	uint64_t data_ptr = args->data_ptr;
468
	int do_bit17_swizzling;
469 470 471 472 473 474 475 476 477 478 479

	remain = args->size;

	/* Pin the user pages containing the data.  We can't fault while
	 * holding the struct mutex, yet we want to hold it while
	 * dereferencing the user data.
	 */
	first_data_page = data_ptr / PAGE_SIZE;
	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
	num_pages = last_data_page - first_data_page + 1;

480
	user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
481 482 483 484 485
	if (user_pages == NULL)
		return -ENOMEM;

	down_read(&mm->mmap_sem);
	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
486
				      num_pages, 1, 0, user_pages, NULL);
487 488 489 490 491 492
	up_read(&mm->mmap_sem);
	if (pinned_pages < num_pages) {
		ret = -EFAULT;
		goto fail_put_user_pages;
	}

493 494
	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);

495 496 497
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto fail_put_user_pages;
498

499 500
	ret = i915_gem_object_get_pages_or_evict(obj);
	if (ret)
501 502 503 504 505 506 507
		goto fail_unlock;

	ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
							args->size);
	if (ret != 0)
		goto fail_put_pages;

508
	obj_priv = to_intel_bo(obj);
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
	offset = args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * shmem_page_index = page number within shmem file
		 * shmem_page_offset = offset within page in shmem file
		 * data_page_index = page number in get_user_pages return
		 * data_page_offset = offset with data_page_index page.
		 * page_length = bytes to copy for this page
		 */
		shmem_page_index = offset / PAGE_SIZE;
		shmem_page_offset = offset & ~PAGE_MASK;
		data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		data_page_offset = data_ptr & ~PAGE_MASK;

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;
		if ((data_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - data_page_offset;

531
		if (do_bit17_swizzling) {
532
			slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
533
					      shmem_page_offset,
534 535 536 537 538 539 540 541 542 543
					      user_pages[data_page_index],
					      data_page_offset,
					      page_length,
					      1);
		} else {
			slow_shmem_copy(user_pages[data_page_index],
					data_page_offset,
					obj_priv->pages[shmem_page_index],
					shmem_page_offset,
					page_length);
544
		}
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559

		remain -= page_length;
		data_ptr += page_length;
		offset += page_length;
	}

fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
	mutex_unlock(&dev->struct_mutex);
fail_put_user_pages:
	for (i = 0; i < pinned_pages; i++) {
		SetPageDirty(user_pages[i]);
		page_cache_release(user_pages[i]);
	}
560
	drm_free_large(user_pages);
561 562 563 564

	return ret;
}

565 566 567 568 569 570 571 572 573 574 575 576
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
		     struct drm_file *file_priv)
{
	struct drm_i915_gem_pread *args = data;
	struct drm_gem_object *obj;
	struct drm_i915_gem_object *obj_priv;
577
	int ret = 0;
578 579 580

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
581
		return -ENOENT;
582
	obj_priv = to_intel_bo(obj);
583

584 585
	/* Bounds check source.  */
	if (args->offset > obj->size || args->size > obj->size - args->offset) {
586
		ret = -EINVAL;
587
		goto out;
588 589
	}

590 591 592
	if (args->size == 0)
		goto out;

593 594 595 596
	if (!access_ok(VERIFY_WRITE,
		       (char __user *)(uintptr_t)args->data_ptr,
		       args->size)) {
		ret = -EFAULT;
597
		goto out;
598 599
	}

600
	if (i915_gem_object_needs_bit17_swizzle(obj)) {
601
		ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
602 603 604 605 606 607
	} else {
		ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
		if (ret != 0)
			ret = i915_gem_shmem_pread_slow(dev, obj, args,
							file_priv);
	}
608

609
out:
610
	drm_gem_object_unreference_unlocked(obj);
611
	return ret;
612 613
}

614 615
/* This is the fast write path which cannot handle
 * page faults in the source data
616
 */
617 618 619 620 621 622

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
623 624
{
	char *vaddr_atomic;
625
	unsigned long unwritten;
626

627
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
628 629
	unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
						      user_data, length);
630
	io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
631 632 633 634 635 636 637 638 639
	if (unwritten)
		return -EFAULT;
	return 0;
}

/* Here's the write path which can sleep for
 * page faults
 */

640
static inline void
641 642 643 644
slow_kernel_write(struct io_mapping *mapping,
		  loff_t gtt_base, int gtt_offset,
		  struct page *user_page, int user_offset,
		  int length)
645
{
646 647
	char __iomem *dst_vaddr;
	char *src_vaddr;
648

649 650 651 652 653 654 655 656 657
	dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
	src_vaddr = kmap(user_page);

	memcpy_toio(dst_vaddr + gtt_offset,
		    src_vaddr + user_offset,
		    length);

	kunmap(user_page);
	io_mapping_unmap(dst_vaddr);
658 659
}

660 661 662 663 664 665 666
static inline int
fast_shmem_write(struct page **pages,
		 loff_t page_base, int page_offset,
		 char __user *data,
		 int length)
{
	char __iomem *vaddr;
667
	unsigned long unwritten;
668 669 670 671

	vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
	if (vaddr == NULL)
		return -ENOMEM;
672
	unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
673 674
	kunmap_atomic(vaddr, KM_USER0);

675 676
	if (unwritten)
		return -EFAULT;
677 678 679
	return 0;
}

680 681 682 683
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
684
static int
685 686 687
i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
			 struct drm_i915_gem_pwrite *args,
			 struct drm_file *file_priv)
688
{
689
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
690
	drm_i915_private_t *dev_priv = dev->dev_private;
691
	ssize_t remain;
692
	loff_t offset, page_base;
693
	char __user *user_data;
694 695
	int page_offset, page_length;
	int ret;
696 697 698 699

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;

700 701 702
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
703 704 705 706 707 708

	ret = i915_gem_object_pin(obj, 0);
	if (ret) {
		mutex_unlock(&dev->struct_mutex);
		return ret;
	}
709
	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
710 711 712
	if (ret)
		goto fail;

713
	obj_priv = to_intel_bo(obj);
714 715 716 717 718
	offset = obj_priv->gtt_offset + args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
719 720 721
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
722
		 */
723 724 725 726 727 728 729 730 731 732
		page_base = (offset & ~(PAGE_SIZE-1));
		page_offset = offset & (PAGE_SIZE-1);
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
				       page_offset, user_data, page_length);

		/* If we get a fault while copying data, then (presumably) our
733 734
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
735
		 */
736 737
		if (ret)
			goto fail;
738

739 740 741
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
742 743 744 745 746 747 748 749 750
	}

fail:
	i915_gem_object_unpin(obj);
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

751 752 753 754 755 756 757
/**
 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 * the memory and maps it using kmap_atomic for copying.
 *
 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 */
758
static int
759 760 761
i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
			 struct drm_i915_gem_pwrite *args,
			 struct drm_file *file_priv)
762
{
763
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
764 765 766 767 768 769 770 771
	drm_i915_private_t *dev_priv = dev->dev_private;
	ssize_t remain;
	loff_t gtt_page_base, offset;
	loff_t first_data_page, last_data_page, num_pages;
	loff_t pinned_pages, i;
	struct page **user_pages;
	struct mm_struct *mm = current->mm;
	int gtt_page_offset, data_page_offset, data_page_index, page_length;
772
	int ret;
773 774 775 776 777 778 779 780 781 782 783 784
	uint64_t data_ptr = args->data_ptr;

	remain = args->size;

	/* Pin the user pages containing the data.  We can't fault while
	 * holding the struct mutex, and all of the pwrite implementations
	 * want to hold it while dereferencing the user data.
	 */
	first_data_page = data_ptr / PAGE_SIZE;
	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
	num_pages = last_data_page - first_data_page + 1;

785
	user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
786 787 788 789 790 791 792 793 794 795 796
	if (user_pages == NULL)
		return -ENOMEM;

	down_read(&mm->mmap_sem);
	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
				      num_pages, 0, 0, user_pages, NULL);
	up_read(&mm->mmap_sem);
	if (pinned_pages < num_pages) {
		ret = -EFAULT;
		goto out_unpin_pages;
	}
797

798 799 800 801
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out_unpin_pages;

802 803 804 805 806 807 808 809
	ret = i915_gem_object_pin(obj, 0);
	if (ret)
		goto out_unlock;

	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
	if (ret)
		goto out_unpin_object;

810
	obj_priv = to_intel_bo(obj);
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
	offset = obj_priv->gtt_offset + args->offset;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * gtt_page_base = page offset within aperture
		 * gtt_page_offset = offset within page in aperture
		 * data_page_index = page number in get_user_pages return
		 * data_page_offset = offset with data_page_index page.
		 * page_length = bytes to copy for this page
		 */
		gtt_page_base = offset & PAGE_MASK;
		gtt_page_offset = offset & ~PAGE_MASK;
		data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		data_page_offset = data_ptr & ~PAGE_MASK;

		page_length = remain;
		if ((gtt_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - gtt_page_offset;
		if ((data_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - data_page_offset;

833 834 835 836 837
		slow_kernel_write(dev_priv->mm.gtt_mapping,
				  gtt_page_base, gtt_page_offset,
				  user_pages[data_page_index],
				  data_page_offset,
				  page_length);
838 839 840 841 842 843 844 845 846 847 848 849 850

		remain -= page_length;
		offset += page_length;
		data_ptr += page_length;
	}

out_unpin_object:
	i915_gem_object_unpin(obj);
out_unlock:
	mutex_unlock(&dev->struct_mutex);
out_unpin_pages:
	for (i = 0; i < pinned_pages; i++)
		page_cache_release(user_pages[i]);
851
	drm_free_large(user_pages);
852 853 854 855

	return ret;
}

856 857 858 859
/**
 * This is the fast shmem pwrite path, which attempts to directly
 * copy_from_user into the kmapped pages backing the object.
 */
860
static int
861 862 863
i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
			   struct drm_i915_gem_pwrite *args,
			   struct drm_file *file_priv)
864
{
865
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
866 867 868 869
	ssize_t remain;
	loff_t offset, page_base;
	char __user *user_data;
	int page_offset, page_length;
870
	int ret;
871 872 873

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;
874

875 876 877
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
878

879
	ret = i915_gem_object_get_pages(obj, 0);
880 881
	if (ret != 0)
		goto fail_unlock;
882

883
	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
884 885 886
	if (ret != 0)
		goto fail_put_pages;

887
	obj_priv = to_intel_bo(obj);
888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
	offset = args->offset;
	obj_priv->dirty = 1;

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		page_base = (offset & ~(PAGE_SIZE-1));
		page_offset = offset & (PAGE_SIZE-1);
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		ret = fast_shmem_write(obj_priv->pages,
				       page_base, page_offset,
				       user_data, page_length);
		if (ret)
			goto fail_put_pages;

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
	mutex_unlock(&dev->struct_mutex);

	return ret;
}

/**
 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 * the memory and maps it using kmap_atomic for copying.
 *
 * This avoids taking mmap_sem for faulting on the user's address while the
 * struct_mutex is held.
 */
static int
i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
			   struct drm_i915_gem_pwrite *args,
			   struct drm_file *file_priv)
{
935
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
936 937 938 939 940 941 942 943 944 945
	struct mm_struct *mm = current->mm;
	struct page **user_pages;
	ssize_t remain;
	loff_t offset, pinned_pages, i;
	loff_t first_data_page, last_data_page, num_pages;
	int shmem_page_index, shmem_page_offset;
	int data_page_index,  data_page_offset;
	int page_length;
	int ret;
	uint64_t data_ptr = args->data_ptr;
946
	int do_bit17_swizzling;
947 948 949 950 951 952 953 954 955 956 957

	remain = args->size;

	/* Pin the user pages containing the data.  We can't fault while
	 * holding the struct mutex, and all of the pwrite implementations
	 * want to hold it while dereferencing the user data.
	 */
	first_data_page = data_ptr / PAGE_SIZE;
	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
	num_pages = last_data_page - first_data_page + 1;

958
	user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
959 960 961 962 963 964 965 966 967 968
	if (user_pages == NULL)
		return -ENOMEM;

	down_read(&mm->mmap_sem);
	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
				      num_pages, 0, 0, user_pages, NULL);
	up_read(&mm->mmap_sem);
	if (pinned_pages < num_pages) {
		ret = -EFAULT;
		goto fail_put_user_pages;
969 970
	}

971 972
	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);

973 974 975
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto fail_put_user_pages;
976

977 978
	ret = i915_gem_object_get_pages_or_evict(obj);
	if (ret)
979 980 981 982 983 984
		goto fail_unlock;

	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
	if (ret != 0)
		goto fail_put_pages;

985
	obj_priv = to_intel_bo(obj);
986
	offset = args->offset;
987
	obj_priv->dirty = 1;
988

989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
	while (remain > 0) {
		/* Operation in this page
		 *
		 * shmem_page_index = page number within shmem file
		 * shmem_page_offset = offset within page in shmem file
		 * data_page_index = page number in get_user_pages return
		 * data_page_offset = offset with data_page_index page.
		 * page_length = bytes to copy for this page
		 */
		shmem_page_index = offset / PAGE_SIZE;
		shmem_page_offset = offset & ~PAGE_MASK;
		data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		data_page_offset = data_ptr & ~PAGE_MASK;

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;
		if ((data_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - data_page_offset;

1009
		if (do_bit17_swizzling) {
1010
			slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
1011 1012 1013
					      shmem_page_offset,
					      user_pages[data_page_index],
					      data_page_offset,
1014 1015 1016 1017 1018 1019 1020 1021
					      page_length,
					      0);
		} else {
			slow_shmem_copy(obj_priv->pages[shmem_page_index],
					shmem_page_offset,
					user_pages[data_page_index],
					data_page_offset,
					page_length);
1022
		}
1023 1024 1025 1026

		remain -= page_length;
		data_ptr += page_length;
		offset += page_length;
1027 1028
	}

1029 1030 1031
fail_put_pages:
	i915_gem_object_put_pages(obj);
fail_unlock:
1032
	mutex_unlock(&dev->struct_mutex);
1033 1034 1035
fail_put_user_pages:
	for (i = 0; i < pinned_pages; i++)
		page_cache_release(user_pages[i]);
1036
	drm_free_large(user_pages);
1037

1038
	return ret;
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_i915_gem_pwrite *args = data;
	struct drm_gem_object *obj;
	struct drm_i915_gem_object *obj_priv;
	int ret = 0;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
1057
		return -ENOENT;
1058
	obj_priv = to_intel_bo(obj);
1059

1060 1061
	/* Bounds check destination. */
	if (args->offset > obj->size || args->size > obj->size - args->offset) {
1062
		ret = -EINVAL;
1063
		goto out;
1064 1065
	}

1066 1067 1068
	if (args->size == 0)
		goto out;

1069 1070 1071 1072
	if (!access_ok(VERIFY_READ,
		       (char __user *)(uintptr_t)args->data_ptr,
		       args->size)) {
		ret = -EFAULT;
1073
		goto out;
1074 1075 1076 1077 1078 1079 1080 1081
	}

	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
1082 1083 1084
	if (obj_priv->phys_obj)
		ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
	else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1085
		 obj_priv->gtt_space &&
1086
		 obj->write_domain != I915_GEM_DOMAIN_CPU) {
1087 1088 1089 1090 1091
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
		if (ret == -EFAULT) {
			ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
						       file_priv);
		}
1092 1093
	} else if (i915_gem_object_needs_bit17_swizzle(obj)) {
		ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
1094 1095 1096 1097 1098 1099 1100
	} else {
		ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
		if (ret == -EFAULT) {
			ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
							 file_priv);
		}
	}
1101 1102 1103 1104 1105 1106

#if WATCH_PWRITE
	if (ret)
		DRM_INFO("pwrite failed %d\n", ret);
#endif

1107
out:
1108
	drm_gem_object_unreference_unlocked(obj);
1109 1110 1111 1112
	return ret;
}

/**
1113 1114
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1115 1116 1117 1118 1119
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file_priv)
{
1120
	struct drm_i915_private *dev_priv = dev->dev_private;
1121 1122
	struct drm_i915_gem_set_domain *args = data;
	struct drm_gem_object *obj;
1123
	struct drm_i915_gem_object *obj_priv;
1124 1125
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1126 1127 1128 1129 1130
	int ret;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

1131
	/* Only handle setting domains to types used by the CPU. */
1132
	if (write_domain & I915_GEM_GPU_DOMAINS)
1133 1134
		return -EINVAL;

1135
	if (read_domains & I915_GEM_GPU_DOMAINS)
1136 1137 1138 1139 1140 1141 1142 1143
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1144 1145
	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
1146
		return -ENOENT;
1147
	obj_priv = to_intel_bo(obj);
1148

1149 1150 1151 1152 1153
	ret = i915_mutex_lock_interruptible(dev);
	if (ret) {
		drm_gem_object_unreference_unlocked(obj);
		return ret;
	}
1154 1155 1156

	intel_mark_busy(dev, obj);

1157 1158
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1159

1160 1161 1162 1163
		/* Update the LRU on the fence for the CPU access that's
		 * about to occur.
		 */
		if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1164 1165 1166
			struct drm_i915_fence_reg *reg =
				&dev_priv->fence_regs[obj_priv->fence_reg];
			list_move_tail(&reg->lru_list,
1167 1168 1169
				       &dev_priv->mm.fence_list);
		}

1170 1171 1172 1173 1174 1175
		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
1176
	} else {
1177
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1178 1179
	}

1180 1181 1182 1183
	/* Maintain LRU order of "inactive" objects */
	if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
		list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);

1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
	drm_gem_object_unreference(obj);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_i915_gem_sw_finish *args = data;
	struct drm_gem_object *obj;
	int ret = 0;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1204
	if (obj == NULL)
1205
		return -ENOENT;
1206 1207 1208 1209 1210

	ret = i915_mutex_lock_interruptible(dev);
	if (ret) {
		drm_gem_object_unreference_unlocked(obj);
		return ret;
1211 1212 1213
	}

	/* Pinned buffers may be scanout, so flush the cache */
1214
	if (to_intel_bo(obj)->pin_count)
1215 1216
		i915_gem_object_flush_cpu_write_domain(obj);

1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
	drm_gem_object_unreference(obj);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
		   struct drm_file *file_priv)
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	loff_t offset;
	unsigned long addr;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
1243
		return -ENOENT;
1244 1245 1246 1247 1248 1249 1250 1251

	offset = args->offset;

	down_write(&current->mm->mmap_sem);
	addr = do_mmap(obj->filp, 0, args->size,
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
	up_write(&current->mm->mmap_sem);
1252
	drm_gem_object_unreference_unlocked(obj);
1253 1254 1255 1256 1257 1258 1259 1260
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct drm_gem_object *obj = vma->vm_private_data;
	struct drm_device *dev = obj->dev;
1281
	drm_i915_private_t *dev_priv = dev->dev_private;
1282
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1283 1284 1285
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1286
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1287 1288 1289 1290 1291 1292 1293 1294

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

	/* Now bind it into the GTT if needed */
	mutex_lock(&dev->struct_mutex);
	if (!obj_priv->gtt_space) {
1295
		ret = i915_gem_object_bind_to_gtt(obj, 0);
1296 1297
		if (ret)
			goto unlock;
1298 1299

		ret = i915_gem_object_set_to_gtt_domain(obj, write);
1300 1301
		if (ret)
			goto unlock;
1302 1303 1304
	}

	/* Need a new fence register? */
1305
	if (obj_priv->tiling_mode != I915_TILING_NONE) {
1306
		ret = i915_gem_object_get_fence_reg(obj, true);
1307 1308
		if (ret)
			goto unlock;
1309
	}
1310

1311 1312 1313
	if (i915_gem_object_is_inactive(obj_priv))
		list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);

1314 1315 1316 1317 1318
	pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
		page_offset;

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1319
unlock:
1320 1321 1322
	mutex_unlock(&dev->struct_mutex);

	switch (ret) {
1323 1324 1325
	case 0:
	case -ERESTARTSYS:
		return VM_FAULT_NOPAGE;
1326 1327 1328 1329
	case -ENOMEM:
	case -EAGAIN:
		return VM_FAULT_OOM;
	default:
1330
		return VM_FAULT_SIGBUS;
1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
	}
}

/**
 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
 * @obj: obj in question
 *
 * GEM memory mapping works by handing back to userspace a fake mmap offset
 * it can use in a subsequent mmap(2) call.  The DRM core code then looks
 * up the object based on the offset and sets up the various memory mapping
 * structures.
 *
 * This routine allocates and attaches a fake offset for @obj.
 */
static int
i915_gem_create_mmap_offset(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
	struct drm_gem_mm *mm = dev->mm_private;
1350
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1351
	struct drm_map_list *list;
1352
	struct drm_local_map *map;
1353 1354 1355 1356
	int ret = 0;

	/* Set the object up for mmap'ing */
	list = &obj->map_list;
1357
	list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
	if (!list->map)
		return -ENOMEM;

	map = list->map;
	map->type = _DRM_GEM;
	map->size = obj->size;
	map->handle = obj;

	/* Get a DRM GEM mmap offset allocated... */
	list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
						    obj->size / PAGE_SIZE, 0, 0);
	if (!list->file_offset_node) {
		DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1371
		ret = -ENOSPC;
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
		goto out_free_list;
	}

	list->file_offset_node = drm_mm_get_block(list->file_offset_node,
						  obj->size / PAGE_SIZE, 0);
	if (!list->file_offset_node) {
		ret = -ENOMEM;
		goto out_free_list;
	}

	list->hash.key = list->file_offset_node->start;
1383 1384
	ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
	if (ret) {
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
		DRM_ERROR("failed to add to map hash\n");
		goto out_free_mm;
	}

	/* By now we should be all set, any drm_mmap request on the offset
	 * below will get to our mmap & fault handler */
	obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;

	return 0;

out_free_mm:
	drm_mm_put_block(list->file_offset_node);
out_free_list:
1398
	kfree(list->map);
1399 1400 1401 1402

	return ret;
}

1403 1404 1405 1406
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1407
 * Preserve the reservation of the mmapping with the DRM core code, but
1408 1409 1410 1411 1412 1413 1414 1415 1416
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1417
void
1418 1419 1420
i915_gem_release_mmap(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
1421
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1422 1423 1424 1425 1426 1427

	if (dev->dev_mapping)
		unmap_mapping_range(dev->dev_mapping,
				    obj_priv->mmap_offset, obj->size, 1);
}

1428 1429 1430 1431
static void
i915_gem_free_mmap_offset(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
1432
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
	struct drm_gem_mm *mm = dev->mm_private;
	struct drm_map_list *list;

	list = &obj->map_list;
	drm_ht_remove_item(&mm->offset_hash, &list->hash);

	if (list->file_offset_node) {
		drm_mm_put_block(list->file_offset_node);
		list->file_offset_node = NULL;
	}

	if (list->map) {
1445
		kfree(list->map);
1446 1447 1448 1449 1450 1451
		list->map = NULL;
	}

	obj_priv->mmap_offset = 0;
}

1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
 * potential fence register mapping if needed.
 */
static uint32_t
i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
{
	struct drm_device *dev = obj->dev;
1463
	struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1464 1465 1466 1467 1468 1469
	int start, i;

	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
1470
	if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1471 1472 1473 1474 1475 1476
		return 4096;

	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
1477
	if (INTEL_INFO(dev)->gen == 3)
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
		start = 1024*1024;
	else
		start = 512*1024;

	for (i = start; i < obj->size; i <<= 1)
		;

	return i;
}

/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file_priv: GEM object info