i915_gem.c 124 KB
Newer Older
1
/*
2
 * Copyright © 2008-2015 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28
#include <drm/drmP.h>
29
#include <drm/drm_vma_manager.h>
30
#include <drm/i915_drm.h>
31
#include "i915_drv.h"
32
#include "i915_gem_dmabuf.h"
33
#include "i915_vgpu.h"
Chris Wilson's avatar
Chris Wilson committed
34
#include "i915_trace.h"
35
#include "intel_drv.h"
36
#include "intel_frontbuffer.h"
37
#include "intel_mocs.h"
38
#include <linux/reservation.h>
39
#include <linux/shmem_fs.h>
40
#include <linux/slab.h>
41
#include <linux/swap.h>
42
#include <linux/pci.h>
43
#include <linux/dma-buf.h>
44

45
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
46
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
47

48 49 50 51 52 53
static bool cpu_cache_is_coherent(struct drm_device *dev,
				  enum i915_cache_level level)
{
	return HAS_LLC(dev) || level != I915_CACHE_NONE;
}

54 55
static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
{
56 57 58
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return false;

59 60 61 62 63 64
	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		return true;

	return obj->pin_display;
}

65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
static int
insert_mappable_node(struct drm_i915_private *i915,
                     struct drm_mm_node *node, u32 size)
{
	memset(node, 0, sizeof(*node));
	return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
						   size, 0, 0, 0,
						   i915->ggtt.mappable_end,
						   DRM_MM_SEARCH_DEFAULT,
						   DRM_MM_CREATE_DEFAULT);
}

static void
remove_mappable_node(struct drm_mm_node *node)
{
	drm_mm_remove_node(node);
}

83 84 85 86
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
87
	spin_lock(&dev_priv->mm.object_stat_lock);
88 89
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
90
	spin_unlock(&dev_priv->mm.object_stat_lock);
91 92 93 94 95
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
96
	spin_lock(&dev_priv->mm.object_stat_lock);
97 98
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
99
	spin_unlock(&dev_priv->mm.object_stat_lock);
100 101
}

102
static int
103
i915_gem_wait_for_error(struct i915_gpu_error *error)
104 105 106
{
	int ret;

107
	if (!i915_reset_in_progress(error))
108 109
		return 0;

110 111 112 113 114
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
115
	ret = wait_event_interruptible_timeout(error->reset_queue,
116
					       !i915_reset_in_progress(error),
117
					       10*HZ);
118 119 120 121
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
122
		return ret;
123 124
	} else {
		return 0;
125
	}
126 127
}

128
int i915_mutex_lock_interruptible(struct drm_device *dev)
129
{
130
	struct drm_i915_private *dev_priv = to_i915(dev);
131 132
	int ret;

133
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
134 135 136 137 138 139 140 141 142
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

	return 0;
}
143

144 145
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
146
			    struct drm_file *file)
147
{
148
	struct drm_i915_private *dev_priv = to_i915(dev);
149
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
150
	struct drm_i915_gem_get_aperture *args = data;
151
	struct i915_vma *vma;
152
	size_t pinned;
153

154
	pinned = 0;
155
	mutex_lock(&dev->struct_mutex);
156
	list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
157
		if (i915_vma_is_pinned(vma))
158
			pinned += vma->node.size;
159
	list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
160
		if (i915_vma_is_pinned(vma))
161
			pinned += vma->node.size;
162
	mutex_unlock(&dev->struct_mutex);
163

164
	args->aper_size = ggtt->base.total;
165
	args->aper_available_size = args->aper_size - pinned;
166

167 168 169
	return 0;
}

170 171
static int
i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
172
{
173 174 175 176 177
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
	char *vaddr = obj->phys_handle->vaddr;
	struct sg_table *st;
	struct scatterlist *sg;
	int i;
178

179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
		return -EINVAL;

	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
		struct page *page;
		char *src;

		page = shmem_read_mapping_page(mapping, i);
		if (IS_ERR(page))
			return PTR_ERR(page);

		src = kmap_atomic(page);
		memcpy(vaddr, src, PAGE_SIZE);
		drm_clflush_virt_range(vaddr, PAGE_SIZE);
		kunmap_atomic(src);

195
		put_page(page);
196 197 198
		vaddr += PAGE_SIZE;
	}

199
	i915_gem_chipset_flush(to_i915(obj->base.dev));
200 201 202 203 204 205 206 207 208 209 210 211 212

	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
		kfree(st);
		return -ENOMEM;
	}

	sg = st->sgl;
	sg->offset = 0;
	sg->length = obj->base.size;
213

214 215 216 217 218 219 220 221 222 223 224 225 226
	sg_dma_address(sg) = obj->phys_handle->busaddr;
	sg_dma_len(sg) = obj->base.size;

	obj->pages = st;
	return 0;
}

static void
i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
{
	int ret;

	BUG_ON(obj->madv == __I915_MADV_PURGED);
227

228
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
229
	if (WARN_ON(ret)) {
230 231 232 233 234 235 236 237 238 239
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;

	if (obj->dirty) {
240
		struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
241
		char *vaddr = obj->phys_handle->vaddr;
242 243 244
		int i;

		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
245 246 247 248 249 250 251 252 253 254 255 256 257 258
			struct page *page;
			char *dst;

			page = shmem_read_mapping_page(mapping, i);
			if (IS_ERR(page))
				continue;

			dst = kmap_atomic(page);
			drm_clflush_virt_range(vaddr, PAGE_SIZE);
			memcpy(dst, vaddr, PAGE_SIZE);
			kunmap_atomic(dst);

			set_page_dirty(page);
			if (obj->madv == I915_MADV_WILLNEED)
259
				mark_page_accessed(page);
260
			put_page(page);
261 262
			vaddr += PAGE_SIZE;
		}
263
		obj->dirty = 0;
264 265
	}

266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
	sg_free_table(obj->pages);
	kfree(obj->pages);
}

static void
i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
{
	drm_pci_free(obj->base.dev, obj->phys_handle);
}

static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
	.get_pages = i915_gem_object_get_pages_phys,
	.put_pages = i915_gem_object_put_pages_phys,
	.release = i915_gem_object_release_phys,
};

282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
int
i915_gem_object_unbind(struct drm_i915_gem_object *obj)
{
	struct i915_vma *vma;
	LIST_HEAD(still_in_list);
	int ret;

	/* The vma will only be freed if it is marked as closed, and if we wait
	 * upon rendering to the vma, we may unbind anything in the list.
	 */
	while ((vma = list_first_entry_or_null(&obj->vma_list,
					       struct i915_vma,
					       obj_link))) {
		list_move_tail(&vma->obj_link, &still_in_list);
		ret = i915_vma_unbind(vma);
		if (ret)
			break;
	}
	list_splice(&still_in_list, &obj->vma_list);

	return ret;
}

305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
/**
 * Ensures that all rendering to the object has completed and the object is
 * safe to unbind from the GTT or access from the CPU.
 * @obj: i915 gem object
 * @readonly: waiting for just read access or read-write access
 */
int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
			       bool readonly)
{
	struct reservation_object *resv;
	struct i915_gem_active *active;
	unsigned long active_mask;
	int idx;

	lockdep_assert_held(&obj->base.dev->struct_mutex);

	if (!readonly) {
		active = obj->last_read;
		active_mask = i915_gem_object_get_active(obj);
	} else {
		active_mask = 1;
		active = &obj->last_write;
	}

	for_each_active(active_mask, idx) {
		int ret;

		ret = i915_gem_active_wait(&active[idx],
					   &obj->base.dev->struct_mutex);
		if (ret)
			return ret;
	}

	resv = i915_gem_object_get_dmabuf_resv(obj);
	if (resv) {
		long err;

		err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
							  MAX_SCHEDULE_TIMEOUT);
		if (err < 0)
			return err;
	}

	return 0;
}

352 353 354
/* A nonblocking variant of the above wait. Must be called prior to
 * acquiring the mutex for the object, as the object state may change
 * during this call. A reference must be held by the caller for the object.
355 356
 */
static __must_check int
357 358 359
__unsafe_wait_rendering(struct drm_i915_gem_object *obj,
			struct intel_rps_client *rps,
			bool readonly)
360 361 362
{
	struct i915_gem_active *active;
	unsigned long active_mask;
363
	int idx;
364

365
	active_mask = __I915_BO_ACTIVE(obj);
366 367 368 369 370 371 372 373 374 375
	if (!active_mask)
		return 0;

	if (!readonly) {
		active = obj->last_read;
	} else {
		active_mask = 1;
		active = &obj->last_write;
	}

376 377
	for_each_active(active_mask, idx) {
		int ret;
378

379 380 381 382
		ret = i915_gem_active_wait_unlocked(&active[idx],
						    true, NULL, rps);
		if (ret)
			return ret;
383 384
	}

385
	return 0;
386 387 388 389 390 391 392 393 394
}

static struct intel_rps_client *to_rps_client(struct drm_file *file)
{
	struct drm_i915_file_private *fpriv = file->driver_priv;

	return &fpriv->rps;
}

395 396 397 398 399
int
i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
			    int align)
{
	drm_dma_handle_t *phys;
400
	int ret;
401 402 403 404 405 406 407 408 409 410 411 412 413 414

	if (obj->phys_handle) {
		if ((unsigned long)obj->phys_handle->vaddr & (align -1))
			return -EBUSY;

		return 0;
	}

	if (obj->madv != I915_MADV_WILLNEED)
		return -EFAULT;

	if (obj->base.filp == NULL)
		return -EINVAL;

Chris Wilson's avatar
Chris Wilson committed
415 416 417 418 419
	ret = i915_gem_object_unbind(obj);
	if (ret)
		return ret;

	ret = i915_gem_object_put_pages(obj);
420 421 422
	if (ret)
		return ret;

423 424 425 426 427 428
	/* create a new object */
	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
	if (!phys)
		return -ENOMEM;

	obj->phys_handle = phys;
429 430 431
	obj->ops = &i915_gem_phys_ops;

	return i915_gem_object_get_pages(obj);
432 433 434 435 436 437 438 439 440
}

static int
i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pwrite *args,
		     struct drm_file *file_priv)
{
	struct drm_device *dev = obj->base.dev;
	void *vaddr = obj->phys_handle->vaddr + args->offset;
441
	char __user *user_data = u64_to_user_ptr(args->data_ptr);
442
	int ret = 0;
443 444 445 446 447 448 449

	/* We manually control the domain here and pretend that it
	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
	 */
	ret = i915_gem_object_wait_rendering(obj, false);
	if (ret)
		return ret;
450

451
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
452 453 454 455 456 457 458 459 460 461
	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		unsigned long unwritten;

		/* The physical object once assigned is fixed for the lifetime
		 * of the obj, so we can safely drop the lock and continue
		 * to access vaddr.
		 */
		mutex_unlock(&dev->struct_mutex);
		unwritten = copy_from_user(vaddr, user_data, args->size);
		mutex_lock(&dev->struct_mutex);
462 463 464 465
		if (unwritten) {
			ret = -EFAULT;
			goto out;
		}
466 467
	}

468
	drm_clflush_virt_range(vaddr, args->size);
469
	i915_gem_chipset_flush(to_i915(dev));
470 471

out:
472
	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
473
	return ret;
474 475
}

476 477
void *i915_gem_object_alloc(struct drm_device *dev)
{
478
	struct drm_i915_private *dev_priv = to_i915(dev);
479
	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
480 481 482 483
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
484
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
485
	kmem_cache_free(dev_priv->objects, obj);
486 487
}

488 489 490 491 492
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
493
{
494
	struct drm_i915_gem_object *obj;
495 496
	int ret;
	u32 handle;
497

498
	size = roundup(size, PAGE_SIZE);
499 500
	if (size == 0)
		return -EINVAL;
501 502

	/* Allocate the new object */
503
	obj = i915_gem_object_create(dev, size);
504 505
	if (IS_ERR(obj))
		return PTR_ERR(obj);
506

507
	ret = drm_gem_handle_create(file, &obj->base, &handle);
508
	/* drop reference from allocate - handle holds it now */
509
	i915_gem_object_put_unlocked(obj);
510 511
	if (ret)
		return ret;
512

513
	*handle_p = handle;
514 515 516
	return 0;
}

517 518 519 520 521 522
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
523
	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
524 525
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
526
			       args->size, &args->handle);
527 528 529 530
}

/**
 * Creates a new mm object and returns a handle to it.
531 532 533
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
534 535 536 537 538 539
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
540

541
	return i915_gem_create(file, dev,
542
			       args->size, &args->handle);
543 544
}

545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

571
static inline int
572 573
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

597 598 599 600 601 602 603 604 605 606 607 608
/*
 * Pins the specified object's pages and synchronizes the object with
 * GPU accesses. Sets needs_clflush to non-zero if the caller should
 * flush the object from the CPU cache.
 */
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
				    int *needs_clflush)
{
	int ret;

	*needs_clflush = 0;

609
	if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
610 611
		return -EINVAL;

612 613 614 615
	ret = i915_gem_object_wait_rendering(obj, true);
	if (ret)
		return ret;

616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
							obj->cache_level);
	}

	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

	return ret;
}

634 635 636
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
637
static int
638 639 640 641 642 643 644
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

645
	if (unlikely(page_do_bit17_swizzling))
646 647 648 649 650 651 652 653 654 655 656
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

657
	return ret ? -EFAULT : 0;
658 659
}

660 661 662 663
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
664
	if (unlikely(swizzled)) {
665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

682 683 684 685 686 687 688 689 690 691 692 693
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
694 695 696
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
697 698 699 700 701 702 703 704 705 706 707

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

708
	return ret ? - EFAULT : 0;
709 710
}

711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
static inline unsigned long
slow_user_access(struct io_mapping *mapping,
		 uint64_t page_base, int page_offset,
		 char __user *user_data,
		 unsigned long length, bool pwrite)
{
	void __iomem *ioaddr;
	void *vaddr;
	uint64_t unwritten;

	ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force *)ioaddr + page_offset;
	if (pwrite)
		unwritten = __copy_from_user(vaddr, user_data, length);
	else
		unwritten = __copy_to_user(user_data, vaddr, length);

	io_mapping_unmap(ioaddr);
	return unwritten;
}

static int
i915_gem_gtt_pread(struct drm_device *dev,
		   struct drm_i915_gem_object *obj, uint64_t size,
		   uint64_t data_offset, uint64_t data_ptr)
{
738
	struct drm_i915_private *dev_priv = to_i915(dev);
739 740 741 742 743 744 745
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
	struct drm_mm_node node;
	char __user *user_data;
	uint64_t remain;
	uint64_t offset;
	int ret;

746
	ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
	if (ret) {
		ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
		if (ret)
			goto out;

		ret = i915_gem_object_get_pages(obj);
		if (ret) {
			remove_mappable_node(&node);
			goto out;
		}

		i915_gem_object_pin_pages(obj);
	} else {
		node.start = i915_gem_obj_ggtt_offset(obj);
		node.allocated = false;
		ret = i915_gem_object_put_fence(obj);
		if (ret)
			goto out_unpin;
	}

	ret = i915_gem_object_set_to_gtt_domain(obj, false);
	if (ret)
		goto out_unpin;

	user_data = u64_to_user_ptr(data_ptr);
	remain = size;
	offset = data_offset;

	mutex_unlock(&dev->struct_mutex);
	if (likely(!i915.prefault_disable)) {
		ret = fault_in_multipages_writeable(user_data, remain);
		if (ret) {
			mutex_lock(&dev->struct_mutex);
			goto out_unpin;
		}
	}

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		u32 page_base = node.start;
		unsigned page_offset = offset_in_page(offset);
		unsigned page_length = PAGE_SIZE - page_offset;
		page_length = remain < page_length ? remain : page_length;
		if (node.allocated) {
			wmb();
			ggtt->base.insert_page(&ggtt->base,
					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
					       node.start,
					       I915_CACHE_NONE, 0);
			wmb();
		} else {
			page_base += offset & PAGE_MASK;
		}
		/* This is a slow read/write as it tries to read from
		 * and write to user memory which may result into page
		 * faults, and so we cannot perform this under struct_mutex.
		 */
		if (slow_user_access(ggtt->mappable, page_base,
				     page_offset, user_data,
				     page_length, false)) {
			ret = -EFAULT;
			break;
		}

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

	mutex_lock(&dev->struct_mutex);
	if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
		/* The user has modified the object whilst we tried
		 * reading from it, and we now have no idea what domain
		 * the pages should be in. As we have just been touching
		 * them directly, flush everything back to the GTT
		 * domain.
		 */
		ret = i915_gem_object_set_to_gtt_domain(obj, false);
	}

out_unpin:
	if (node.allocated) {
		wmb();
		ggtt->base.clear_range(&ggtt->base,
				       node.start, node.size,
				       true);
		i915_gem_object_unpin_pages(obj);
		remove_mappable_node(&node);
	} else {
		i915_gem_object_ggtt_unpin(obj);
	}
out:
	return ret;
}

847
static int
848 849 850 851
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
852
{
853
	char __user *user_data;
854
	ssize_t remain;
855
	loff_t offset;
856
	int shmem_page_offset, page_length, ret = 0;
857
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
858
	int prefaulted = 0;
859
	int needs_clflush = 0;
860
	struct sg_page_iter sg_iter;
861

862
	if (!i915_gem_object_has_struct_page(obj))
863 864
		return -ENODEV;

865
	user_data = u64_to_user_ptr(args->data_ptr);
866 867
	remain = args->size;

868
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
869

870
	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
871 872 873
	if (ret)
		return ret;

874
	offset = args->offset;
875

876 877
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
878
		struct page *page = sg_page_iter_page(&sg_iter);
879 880 881 882

		if (remain <= 0)
			break;

883 884 885 886 887
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
888
		shmem_page_offset = offset_in_page(offset);
889 890 891 892
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

893 894 895
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

896 897 898 899 900
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
901 902 903

		mutex_unlock(&dev->struct_mutex);

904
		if (likely(!i915.prefault_disable) && !prefaulted) {
905
			ret = fault_in_multipages_writeable(user_data, remain);
906 907 908 909 910 911 912
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
913

914 915 916
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
917

918
		mutex_lock(&dev->struct_mutex);
919 920

		if (ret)
921 922
			goto out;

923
next_page:
924
		remain -= page_length;
925
		user_data += page_length;
926 927 928
		offset += page_length;
	}

929
out:
930 931
	i915_gem_object_unpin_pages(obj);

932 933 934
	return ret;
}

935 936
/**
 * Reads data from the object referenced by handle.
937 938 939
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
940 941 942 943 944
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
945
		     struct drm_file *file)
946 947
{
	struct drm_i915_gem_pread *args = data;
948
	struct drm_i915_gem_object *obj;
949
	int ret = 0;
950

951 952 953 954
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
955
		       u64_to_user_ptr(args->data_ptr),
956 957 958
		       args->size))
		return -EFAULT;

959
	obj = i915_gem_object_lookup(file, args->handle);
960 961
	if (!obj)
		return -ENOENT;
962

963
	/* Bounds check source.  */
964 965
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
966
		ret = -EINVAL;
967
		goto err;
968 969
	}

Chris Wilson's avatar
Chris Wilson committed
970 971
	trace_i915_gem_object_pread(obj, args->offset, args->size);

972 973 974 975 976 977 978 979
	ret = __unsafe_wait_rendering(obj, to_rps_client(file), true);
	if (ret)
		goto err;

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto err;

980
	ret = i915_gem_shmem_pread(dev, obj, args, file);
981

982
	/* pread for non shmem backed objects */
983 984
	if (ret == -EFAULT || ret == -ENODEV) {
		intel_runtime_pm_get(to_i915(dev));
985 986
		ret = i915_gem_gtt_pread(dev, obj, args->size,
					args->offset, args->data_ptr);
987 988
		intel_runtime_pm_put(to_i915(dev));
	}
989

990
	i915_gem_object_put(obj);
991
	mutex_unlock(&dev->struct_mutex);
992 993 994 995 996

	return ret;

err:
	i915_gem_object_put_unlocked(obj);
997
	return ret;
998 999
}

1000 1001
/* This is the fast write path which cannot handle
 * page faults in the source data
1002
 */
1003 1004 1005 1006 1007 1008

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
1009
{
1010 1011
	void __iomem *vaddr_atomic;
	void *vaddr;
1012
	unsigned long unwritten;
1013

1014
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
1015 1016 1017
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
1018
						      user_data, length);
1019
	io_mapping_unmap_atomic(vaddr_atomic);
1020
	return unwritten;
1021 1022
}

1023 1024 1025
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
1026
 * @i915: i915 device private data
1027 1028 1029
 * @obj: i915 gem object
 * @args: pwrite arguments structure
 * @file: drm file pointer
1030
 */
1031
static int
1032
i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
1033
			 struct drm_i915_gem_object *obj,
1034
			 struct drm_i915_gem_pwrite *args,
1035
			 struct drm_file *file)
1036
{
1037
	struct i915_ggtt *ggtt = &i915->ggtt;
1038
	struct drm_device *dev = obj->base.dev;
1039 1040
	struct drm_mm_node node;
	uint64_t remain, offset;
1041
	char __user *user_data;
1042
	int ret;
1043 1044
	bool hit_slow_path = false;

1045
	if (i915_gem_object_is_tiled(obj))
1046
		return -EFAULT;
Daniel Vetter's avatar
Daniel Vetter committed
1047

1048 1049
	ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
				       PIN_MAPPABLE | PIN_NONBLOCK);
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
	if (ret) {
		ret = insert_mappable_node(i915, &node, PAGE_SIZE);
		if (ret)
			goto out;

		ret = i915_gem_object_get_pages(obj);
		if (ret) {
			remove_mappable_node(&node);
			goto out;
		}

		i915_gem_object_pin_pages(obj);
	} else {
		node.start = i915_gem_obj_ggtt_offset(obj);
		node.allocated = false;
1065 1066 1067
		ret = i915_gem_object_put_fence(obj);
		if (ret)
			goto out_unpin;
1068
	}
Daniel Vetter's avatar
Daniel Vetter committed
1069 1070 1071 1072 1073

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

1074
	intel_fb_obj_invalidate(obj, ORIGIN_GTT);
1075
	obj->dirty = true;
1076

1077 1078 1079 1080
	user_data = u64_to_user_ptr(args->data_ptr);
	offset = args->offset;
	remain = args->size;
	while (remain) {
1081 1082
		/* Operation in this page
		 *
1083 1084 1085
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
1086
		 */
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
		u32 page_base = node.start;
		unsigned page_offset = offset_in_page(offset);
		unsigned page_length = PAGE_SIZE - page_offset;
		page_length = remain < page_length ? remain : page_length;
		if (node.allocated) {
			wmb(); /* flush the write before we modify the GGTT */
			ggtt->base.insert_page(&ggtt->base,
					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
					       node.start, I915_CACHE_NONE, 0);
			wmb(); /* flush modifications to the GGTT (insert_page) */
		} else {
			page_base += offset & PAGE_MASK;
		}
1100
		/* If we get a fault while copying data, then (presumably) our
1101 1102
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
1103 1104
		 * If the object is non-shmem backed, we retry again with the
		 * path that handles page fault.
1105
		 */
1106
		if (fast_user_write(ggtt->mappable, page_base,
Daniel Vetter's avatar
Daniel Vetter committed
1107
				    page_offset, user_data, page_length)) {
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
			hit_slow_path = true;
			mutex_unlock(&dev->struct_mutex);
			if (slow_user_access(ggtt->mappable,
					     page_base,
					     page_offset, user_data,
					     page_length, true)) {
				ret = -EFAULT;
				mutex_lock(&dev->struct_mutex);
				goto out_flush;
			}

			mutex_lock(&dev->struct_mutex);
Daniel Vetter's avatar
Daniel Vetter committed
1120
		}
1121

1122 1123 1124
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
1125 1126
	}

1127
out_flush:
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
	if (hit_slow_path) {
		if (ret == 0 &&
		    (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
			/* The user has modified the object whilst we tried
			 * reading from it, and we now have no idea what domain
			 * the pages should be in. As we have just been touching
			 * them directly, flush everything back to the GTT
			 * domain.
			 */
			ret = i915_gem_object_set_to_gtt_domain(obj, false);
		}
	}

1141
	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
Daniel Vetter's avatar
Daniel Vetter committed
1142
out_unpin:
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
	if (node.allocated) {
		wmb();
		ggtt->base.clear_range(&ggtt->base,
				       node.start, node.size,
				       true);
		i915_gem_object_unpin_pages(obj);
		remove_mappable_node(&node);
	} else {
		i915_gem_object_ggtt_unpin(obj);
	}
Daniel Vetter's avatar
Daniel Vetter committed
1153
out:
1154
	return ret;
1155 1156
}

1157 1158 1159 1160
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
1161
static int
1162 1163 1164 1165 1166
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
1167
{
1168
	char *vaddr;
1169
	int ret;
1170

1171
	if (unlikely(page_do_bit17_swizzling))
1172
		return -EINVAL;
1173

1174 1175 1176 1177
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
1178 1179
	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
					user_data, page_length);
1180 1181 1182 1183
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
1184

1185
	return ret ? -EFAULT : 0;
1186 1187
}

1188 1189
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
1190
static int
1191 1192 1193 1194 1195
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
1196
{
1197 1198
	char *vaddr;
	int ret;
1199

1200
	vaddr = kmap(page);
1201
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1202 1203 1204
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
1205 1206
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1207 1208
						user_data,
						page_length);
1209 1210 1211 1212 1213
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
1214 1215 1216
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
1217
	kunmap(page);
1218

1219
	return ret ? -EFAULT : 0;
1220 1221 1222
}

static int
1223 1224 1225 1226
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
1227 1228
{
	ssize_t remain;
1229 1230
	loff_t offset;
	char __user *user_data;
1231
	int shmem_page_offset, page_length, ret = 0;
1232
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1233
	int hit_slowpath = 0;
1234 1235
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
1236
	struct sg_page_iter sg_iter;
1237

1238
	user_data = u64_to_user_ptr(args->data_ptr);
1239 1240
	remain = args->size;

1241
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1242

1243 1244 1245 1246
	ret = i915_gem_object_wait_rendering(obj, false);
	if (ret)
		return ret;

1247 1248 1249 1250 1251
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
1252
		needs_clflush_after = cpu_write_needs_clflush(obj);
1253
	}
1254 1255 1256 1257 1258
	/* Same trick applies to invalidate partially written cachelines read
	 * before writing. */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		needs_clflush_before =
			!cpu_cache_is_coherent(dev, obj->cache_level);
1259

1260 1261 1262 1263
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

1264
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1265

1266 1267
	i915_gem_object_pin_pages(obj);

1268
	offset = args->offset;
1269
	obj->dirty = 1;
1270

1271 1272
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
1273
		struct page *page = sg_page_iter_page(&sg_iter);
1274
		int partial_cacheline_write;
1275

1276 1277 1278
		if (remain <= 0)
			break;

1279 1280 1281 1282 1283
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
1284
		shmem_page_offset = offset_in_page(offset);
1285 1286 1287 1288 1289

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;