i915_gem.c 101 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"
Chris Wilson's avatar
Chris Wilson committed
32
#include "i915_trace.h"
33
#include "intel_drv.h"
34
#include <linux/shmem_fs.h>
35
#include <linux/slab.h>
36
#include <linux/swap.h>
37
#include <linux/pci.h>
38

39
static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40 41
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42 43 44
static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
						    unsigned alignment,
						    bool map_and_fenceable);
45 46
static int i915_gem_phys_pwrite(struct drm_device *dev,
				struct drm_i915_gem_object *obj,
47
				struct drm_i915_gem_pwrite *args,
48
				struct drm_file *file);
49

50 51 52 53 54 55
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable);

56
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
57
				    struct shrink_control *sc);
58
static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
59

60 61 62 63 64 65 66 67
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
	if (obj->tiling_mode)
		i915_gem_release_mmap(obj);

	/* As we do not have an associated fence register, we will force
	 * a tiling change if we ever need to acquire one.
	 */
68
	obj->fence_dirty = false;
69 70 71
	obj->fence_reg = I915_FENCE_REG_NONE;
}

72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
}

87 88
static int
i915_gem_wait_for_error(struct drm_device *dev)
89 90 91 92 93 94 95 96 97 98 99 100 101
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct completion *x = &dev_priv->error_completion;
	unsigned long flags;
	int ret;

	if (!atomic_read(&dev_priv->mm.wedged))
		return 0;

	ret = wait_for_completion_interruptible(x);
	if (ret)
		return ret;

102 103 104 105 106 107 108 109 110 111 112
	if (atomic_read(&dev_priv->mm.wedged)) {
		/* GPU is hung, bump the completion count to account for
		 * the token we just consumed so that we never hit zero and
		 * end up waiting upon a subsequent completion event that
		 * will never happen.
		 */
		spin_lock_irqsave(&x->wait.lock, flags);
		x->done++;
		spin_unlock_irqrestore(&x->wait.lock, flags);
	}
	return 0;
113 114
}

115
int i915_mutex_lock_interruptible(struct drm_device *dev)
116 117 118
{
	int ret;

119
	ret = i915_gem_wait_for_error(dev);
120 121 122 123 124 125 126
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

127
	WARN_ON(i915_verify_lists(dev));
128 129
	return 0;
}
130

131
static inline bool
132
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
133
{
134
	return !obj->active;
135 136
}

137 138
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
139
		    struct drm_file *file)
140 141
{
	struct drm_i915_gem_init *args = data;
142

143 144 145
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return -ENODEV;

146 147 148
	if (args->gtt_start >= args->gtt_end ||
	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
		return -EINVAL;
149

150 151 152 153
	/* GEM with user mode setting was never supported on ilk and later. */
	if (INTEL_INFO(dev)->gen >= 5)
		return -ENODEV;

154
	mutex_lock(&dev->struct_mutex);
155 156
	i915_gem_init_global_gtt(dev, args->gtt_start,
				 args->gtt_end, args->gtt_end);
157 158
	mutex_unlock(&dev->struct_mutex);

159
	return 0;
160 161
}

162 163
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
164
			    struct drm_file *file)
165
{
166
	struct drm_i915_private *dev_priv = dev->dev_private;
167
	struct drm_i915_gem_get_aperture *args = data;
168 169
	struct drm_i915_gem_object *obj;
	size_t pinned;
170 171 172 173

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

174
	pinned = 0;
175
	mutex_lock(&dev->struct_mutex);
176 177 178
	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
		if (obj->pin_count)
			pinned += obj->gtt_space->size;
179
	mutex_unlock(&dev->struct_mutex);
180

181
	args->aper_size = dev_priv->mm.gtt_total;
182
	args->aper_available_size = args->aper_size - pinned;
183

184 185 186
	return 0;
}

187 188 189 190 191
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
192
{
193
	struct drm_i915_gem_object *obj;
194 195
	int ret;
	u32 handle;
196

197
	size = roundup(size, PAGE_SIZE);
198 199
	if (size == 0)
		return -EINVAL;
200 201

	/* Allocate the new object */
202
	obj = i915_gem_alloc_object(dev, size);
203 204 205
	if (obj == NULL)
		return -ENOMEM;

206
	ret = drm_gem_handle_create(file, &obj->base, &handle);
207
	if (ret) {
208 209
		drm_gem_object_release(&obj->base);
		i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
210
		kfree(obj);
211
		return ret;
212
	}
213

214
	/* drop reference from allocate - handle holds it now */
215
	drm_gem_object_unreference(&obj->base);
216 217
	trace_i915_gem_object_create(obj);

218
	*handle_p = handle;
219 220 221
	return 0;
}

222 223 224 225 226 227
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
228
	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

int i915_gem_dumb_destroy(struct drm_file *file,
			  struct drm_device *dev,
			  uint32_t handle)
{
	return drm_gem_handle_delete(file, handle);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

253
static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
254
{
255
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
256 257

	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
258
		obj->tiling_mode != I915_TILING_NONE;
259 260
}

261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

287
static inline int
288 289
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

313 314 315
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
316
static int
317 318 319 320 321 322 323
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

324
	if (unlikely(page_do_bit17_swizzling))
325 326 327 328 329 330 331 332 333 334 335 336 337 338
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

	return ret;
}

339 340 341 342
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
343
	if (unlikely(swizzled)) {
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

361 362 363 364 365 366 367 368 369 370 371 372
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
373 374 375
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
376 377 378 379 380 381 382 383 384 385 386 387 388 389

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

	return ret;
}

390
static int
391 392 393 394
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
395
{
396
	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
397
	char __user *user_data;
398
	ssize_t remain;
399
	loff_t offset;
400
	int shmem_page_offset, page_length, ret = 0;
401
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
402
	int hit_slowpath = 0;
403
	int prefaulted = 0;
404
	int needs_clflush = 0;
405
	int release_page;
406

407
	user_data = (char __user *) (uintptr_t) args->data_ptr;
408 409
	remain = args->size;

410
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
411

412 413 414 415 416 417 418 419 420 421 422
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush = 1;
		ret = i915_gem_object_set_to_gtt_domain(obj, false);
		if (ret)
			return ret;
	}
423

424
	offset = args->offset;
425 426

	while (remain > 0) {
427 428
		struct page *page;

429 430 431 432 433
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
434
		shmem_page_offset = offset_in_page(offset);
435 436 437 438
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

439 440 441 442 443 444 445 446 447 448
		if (obj->pages) {
			page = obj->pages[offset >> PAGE_SHIFT];
			release_page = 0;
		} else {
			page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
			if (IS_ERR(page)) {
				ret = PTR_ERR(page);
				goto out;
			}
			release_page = 1;
449
		}
450

451 452 453
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

454 455 456 457 458
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
459 460

		hit_slowpath = 1;
461
		page_cache_get(page);
462 463
		mutex_unlock(&dev->struct_mutex);

464
		if (!prefaulted) {
465
			ret = fault_in_multipages_writeable(user_data, remain);
466 467 468 469 470 471 472
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
473

474 475 476
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
477

478
		mutex_lock(&dev->struct_mutex);
479
		page_cache_release(page);
480
next_page:
481
		mark_page_accessed(page);
482 483
		if (release_page)
			page_cache_release(page);
484

485 486 487 488 489
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

490
		remain -= page_length;
491
		user_data += page_length;
492 493 494
		offset += page_length;
	}

495
out:
496 497 498 499 500
	if (hit_slowpath) {
		/* Fixup: Kill any reinstated backing storage pages */
		if (obj->madv == __I915_MADV_PURGED)
			i915_gem_object_truncate(obj);
	}
501 502 503 504

	return ret;
}

505 506 507 508 509 510 511
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
512
		     struct drm_file *file)
513 514
{
	struct drm_i915_gem_pread *args = data;
515
	struct drm_i915_gem_object *obj;
516
	int ret = 0;
517

518 519 520 521 522 523 524 525
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
		       (char __user *)(uintptr_t)args->data_ptr,
		       args->size))
		return -EFAULT;

526
	ret = i915_mutex_lock_interruptible(dev);
527
	if (ret)
528
		return ret;
529

530
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
531
	if (&obj->base == NULL) {
532 533
		ret = -ENOENT;
		goto unlock;
534
	}
535

536
	/* Bounds check source.  */
537 538
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
539
		ret = -EINVAL;
540
		goto out;
541 542
	}

Chris Wilson's avatar
Chris Wilson committed
543 544
	trace_i915_gem_object_pread(obj, args->offset, args->size);

545
	ret = i915_gem_shmem_pread(dev, obj, args, file);
546

547
out:
548
	drm_gem_object_unreference(&obj->base);
549
unlock:
550
	mutex_unlock(&dev->struct_mutex);
551
	return ret;
552 553
}

554 555
/* This is the fast write path which cannot handle
 * page faults in the source data
556
 */
557 558 559 560 561 562

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
563
{
564 565
	void __iomem *vaddr_atomic;
	void *vaddr;
566
	unsigned long unwritten;
567

568
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
569 570 571
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
572
						      user_data, length);
573
	io_mapping_unmap_atomic(vaddr_atomic);
574
	return unwritten;
575 576
}

577 578 579 580
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
581
static int
582 583
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
584
			 struct drm_i915_gem_pwrite *args,
585
			 struct drm_file *file)
586
{
587
	drm_i915_private_t *dev_priv = dev->dev_private;
588
	ssize_t remain;
589
	loff_t offset, page_base;
590
	char __user *user_data;
Daniel Vetter's avatar
Daniel Vetter committed
591 592 593 594 595 596 597 598 599 600 601 602 603
	int page_offset, page_length, ret;

	ret = i915_gem_object_pin(obj, 0, true);
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
604 605 606 607

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;

608
	offset = obj->gtt_offset + args->offset;
609 610 611 612

	while (remain > 0) {
		/* Operation in this page
		 *
613 614 615
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
616
		 */
617 618
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
619 620 621 622 623
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
624 625
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
626
		 */
627
		if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
Daniel Vetter's avatar
Daniel Vetter committed
628 629 630 631
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
			goto out_unpin;
		}
632

633 634 635
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
636 637
	}

Daniel Vetter's avatar
Daniel Vetter committed
638 639 640
out_unpin:
	i915_gem_object_unpin(obj);
out:
641
	return ret;
642 643
}

644 645 646 647
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
648
static int
649 650 651 652 653
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
654
{
655
	char *vaddr;
656
	int ret;
657

658
	if (unlikely(page_do_bit17_swizzling))
659
		return -EINVAL;
660

661 662 663 664 665 666 667 668 669 670 671
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
						user_data,
						page_length);
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
672 673 674 675

	return ret;
}

676 677
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
678
static int
679 680 681 682 683
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
684
{
685 686
	char *vaddr;
	int ret;
687

688
	vaddr = kmap(page);
689
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
690 691 692
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
693 694
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
695 696
						user_data,
						page_length);
697 698 699 700 701
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
702 703 704
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
705
	kunmap(page);
706

707
	return ret;
708 709 710
}

static int
711 712 713 714
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
715
{
716
	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
717
	ssize_t remain;
718 719
	loff_t offset;
	char __user *user_data;
720
	int shmem_page_offset, page_length, ret = 0;
721
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
722
	int hit_slowpath = 0;
723 724
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
725
	int release_page;
726

727
	user_data = (char __user *) (uintptr_t) args->data_ptr;
728 729
	remain = args->size;

730
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
731

732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush_after = 1;
		ret = i915_gem_object_set_to_gtt_domain(obj, true);
		if (ret)
			return ret;
	}
	/* Same trick applies for invalidate partially written cachelines before
	 * writing.  */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
	    && obj->cache_level == I915_CACHE_NONE)
		needs_clflush_before = 1;

749
	offset = args->offset;
750
	obj->dirty = 1;
751

752
	while (remain > 0) {
753
		struct page *page;
754
		int partial_cacheline_write;
755

756 757 758 759 760
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
761
		shmem_page_offset = offset_in_page(offset);
762 763 764 765 766

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

767 768 769 770 771 772 773
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

774 775 776 777 778 779 780 781 782 783
		if (obj->pages) {
			page = obj->pages[offset >> PAGE_SHIFT];
			release_page = 0;
		} else {
			page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
			if (IS_ERR(page)) {
				ret = PTR_ERR(page);
				goto out;
			}
			release_page = 1;
784 785
		}

786 787 788
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

789 790 791 792 793 794
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
795 796

		hit_slowpath = 1;
797
		page_cache_get(page);
798 799
		mutex_unlock(&dev->struct_mutex);

800 801 802 803
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
804

805
		mutex_lock(&dev->struct_mutex);
806
		page_cache_release(page);
807
next_page:
808 809
		set_page_dirty(page);
		mark_page_accessed(page);
810 811
		if (release_page)
			page_cache_release(page);
812

813 814 815 816 817
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

818
		remain -= page_length;
819
		user_data += page_length;
820
		offset += page_length;
821 822
	}

823
out:
824 825 826 827 828 829 830 831 832 833
	if (hit_slowpath) {
		/* Fixup: Kill any reinstated backing storage pages */
		if (obj->madv == __I915_MADV_PURGED)
			i915_gem_object_truncate(obj);
		/* and flush dirty cachelines in case the object isn't in the cpu write
		 * domain anymore. */
		if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
			i915_gem_clflush_object(obj);
			intel_gtt_chipset_flush();
		}
834
	}
835

836 837 838
	if (needs_clflush_after)
		intel_gtt_chipset_flush();

839
	return ret;
840 841 842 843 844 845 846 847 848
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
849
		      struct drm_file *file)
850 851
{
	struct drm_i915_gem_pwrite *args = data;
852
	struct drm_i915_gem_object *obj;
853 854 855 856 857 858 859 860 861 862
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
		       (char __user *)(uintptr_t)args->data_ptr,
		       args->size))
		return -EFAULT;

863 864
	ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
					   args->size);
865 866
	if (ret)
		return -EFAULT;
867

868
	ret = i915_mutex_lock_interruptible(dev);
869
	if (ret)
870
		return ret;
871

872
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
873
	if (&obj->base == NULL) {
874 875
		ret = -ENOENT;
		goto unlock;
876
	}
877

878
	/* Bounds check destination. */
879 880
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
881
		ret = -EINVAL;
882
		goto out;
883 884
	}

Chris Wilson's avatar
Chris Wilson committed
885 886
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

Daniel Vetter's avatar
Daniel Vetter committed
887
	ret = -EFAULT;
888 889 890 891 892 893
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
894
	if (obj->phys_obj) {
895
		ret = i915_gem_phys_pwrite(dev, obj, args, file);
896 897 898 899
		goto out;
	}

	if (obj->gtt_space &&
900
	    obj->cache_level == I915_CACHE_NONE &&
901
	    obj->tiling_mode == I915_TILING_NONE &&
902
	    obj->map_and_fenceable &&
903
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
904
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
Daniel Vetter's avatar
Daniel Vetter committed
905 906 907
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
908
	}
909

910
	if (ret == -EFAULT)
Daniel Vetter's avatar
Daniel Vetter committed
911
		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
912

913
out:
914
	drm_gem_object_unreference(&obj->base);
915
unlock:
916
	mutex_unlock(&dev->struct_mutex);
917 918 919 920
	return ret;
}

/**
921 922
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
923 924 925
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
926
			  struct drm_file *file)
927 928
{
	struct drm_i915_gem_set_domain *args = data;
929
	struct drm_i915_gem_object *obj;
930 931
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
932 933 934 935 936
	int ret;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

937
	/* Only handle setting domains to types used by the CPU. */
938
	if (write_domain & I915_GEM_GPU_DOMAINS)
939 940
		return -EINVAL;

941
	if (read_domains & I915_GEM_GPU_DOMAINS)
942 943 944 945 946 947 948 949
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

950
	ret = i915_mutex_lock_interruptible(dev);
951
	if (ret)
952
		return ret;
953

954
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
955
	if (&obj->base == NULL) {
956 957
		ret = -ENOENT;
		goto unlock;
958
	}
959

960 961
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
962 963 964 965 966 967 968

		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
969
	} else {
970
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
971 972
	}

973
	drm_gem_object_unreference(&obj->base);
974
unlock:
975 976 977 978 979 980 981 982 983
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
984
			 struct drm_file *file)
985 986
{
	struct drm_i915_gem_sw_finish *args = data;
987
	struct drm_i915_gem_object *obj;
988 989 990 991 992
	int ret = 0;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

993
	ret = i915_mutex_lock_interruptible(dev);
994
	if (ret)
995
		return ret;
996

997
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
998
	if (&obj->base == NULL) {
999 1000
		ret = -ENOENT;
		goto unlock;
1001 1002 1003
	}

	/* Pinned buffers may be scanout, so flush the cache */
1004
	if (obj->pin_count)
1005 1006
		i915_gem_object_flush_cpu_write_domain(obj);

1007
	drm_gem_object_unreference(&obj->base);
1008
unlock:
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1022
		    struct drm_file *file)
1023 1024 1025 1026 1027 1028 1029 1030
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

	if (!(dev->driver->driver_features & DRIVER_GEM))
		return -ENODEV;

1031
	obj = drm_gem_object_lookup(dev, file, args->handle);
1032
	if (obj == NULL)
1033
		return -ENOENT;
1034 1035 1036 1037 1038 1039

	down_write(&current->mm->mmap_sem);
	addr = do_mmap(obj->filp, 0, args->size,
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
	up_write(&current->mm->mmap_sem);
1040
	drm_gem_object_unreference_unlocked(obj);
1041 1042 1043 1044 1045 1046 1047 1048
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
1067 1068
	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
	struct drm_device *dev = obj->base.dev;
1069
	drm_i915_private_t *dev_priv = dev->dev_private;
1070 1071 1072
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1073
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1074 1075 1076 1077 1078

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

1079 1080 1081
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
1082

Chris Wilson's avatar
Chris Wilson committed
1083 1084
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1085
	/* Now bind it into the GTT if needed */
1086 1087 1088 1089
	if (!obj->map_and_fenceable) {
		ret = i915_gem_object_unbind(obj);
		if (ret)
			goto unlock;
1090
	}
1091
	if (!obj->gtt_space) {
1092
		ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1093 1094
		if (ret)
			goto unlock;
1095

1096 1097 1098 1099
		ret = i915_gem_object_set_to_gtt_domain(obj, write);
		if (ret)
			goto unlock;
	}
1100

1101 1102 1103
	if (!obj->has_global_gtt_mapping)
		i915_gem_gtt_bind_object(obj, obj->cache_level);

1104
	ret = i915_gem_object_get_fence(obj);
1105 1106
	if (ret)
		goto unlock;
1107

1108 1109
	if (i915_gem_object_is_inactive(obj))
		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1110

1111 1112
	obj->fault_mappable = true;

1113
	pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1114 1115 1116 1117
		page_offset;

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1118
unlock:
1119
	mutex_unlock(&dev->struct_mutex);
1120
out:
1121
	switch (ret) {
1122
	case -EIO:
1123
	case -EAGAIN:
1124 1125 1126 1127 1128 1129 1130
		/* Give the error handler a chance to run and move the
		 * objects off the GPU active list. Next time we service the
		 * fault, we should be able to transition the page into the
		 * GTT without touching the GPU (and so avoid further
		 * EIO/EGAIN). If the GPU is wedged, then there is no issue</