i915_gem.c 139 KB
Newer Older
1
/*
2
 * Copyright © 2008-2015 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28
#include <drm/drmP.h>
29
#include <drm/drm_vma_manager.h>
30
#include <drm/i915_drm.h>
31
#include "i915_drv.h"
32
#include "i915_gem_clflush.h"
33
#include "i915_vgpu.h"
Chris Wilson's avatar
Chris Wilson committed
34
#include "i915_trace.h"
35
#include "intel_drv.h"
36
#include "intel_frontbuffer.h"
37
#include "intel_mocs.h"
38
#include <linux/dma-fence-array.h>
39
#include <linux/kthread.h>
40
#include <linux/reservation.h>
41
#include <linux/shmem_fs.h>
42
#include <linux/slab.h>
43
#include <linux/stop_machine.h>
44
#include <linux/swap.h>
45
#include <linux/pci.h>
46
#include <linux/dma-buf.h>
47

48
static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
49
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
50
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
51

52 53
static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
{
54 55 56
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return false;

57
	if (!i915_gem_object_is_coherent(obj))
58 59 60 61 62
		return true;

	return obj->pin_display;
}

63
static int
64
insert_mappable_node(struct i915_ggtt *ggtt,
65 66 67
                     struct drm_mm_node *node, u32 size)
{
	memset(node, 0, sizeof(*node));
68 69 70 71
	return drm_mm_insert_node_in_range(&ggtt->base.mm, node,
					   size, 0, I915_COLOR_UNEVICTABLE,
					   0, ggtt->mappable_end,
					   DRM_MM_INSERT_LOW);
72 73 74 75 76 77 78 79
}

static void
remove_mappable_node(struct drm_mm_node *node)
{
	drm_mm_remove_node(node);
}

80 81
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
82
				  u64 size)
83
{
84
	spin_lock(&dev_priv->mm.object_stat_lock);
85 86
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
87
	spin_unlock(&dev_priv->mm.object_stat_lock);
88 89 90
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
91
				     u64 size)
92
{
93
	spin_lock(&dev_priv->mm.object_stat_lock);
94 95
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
96
	spin_unlock(&dev_priv->mm.object_stat_lock);
97 98
}

99
static int
100
i915_gem_wait_for_error(struct i915_gpu_error *error)
101 102 103
{
	int ret;

104 105
	might_sleep();

106 107 108 109 110
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
111
	ret = wait_event_interruptible_timeout(error->reset_queue,
112
					       !i915_reset_backoff(error),
113
					       I915_RESET_TIMEOUT);
114 115 116 117
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
118
		return ret;
119 120
	} else {
		return 0;
121
	}
122 123
}

124
int i915_mutex_lock_interruptible(struct drm_device *dev)
125
{
126
	struct drm_i915_private *dev_priv = to_i915(dev);
127 128
	int ret;

129
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
130 131 132 133 134 135 136 137 138
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

	return 0;
}
139

140 141
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
142
			    struct drm_file *file)
143
{
144
	struct drm_i915_private *dev_priv = to_i915(dev);
145
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
146
	struct drm_i915_gem_get_aperture *args = data;
147
	struct i915_vma *vma;
148
	size_t pinned;
149

150
	pinned = 0;
151
	mutex_lock(&dev->struct_mutex);
152
	list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
153
		if (i915_vma_is_pinned(vma))
154
			pinned += vma->node.size;
155
	list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
156
		if (i915_vma_is_pinned(vma))
157
			pinned += vma->node.size;
158
	mutex_unlock(&dev->struct_mutex);
159

160
	args->aper_size = ggtt->base.total;
161
	args->aper_available_size = args->aper_size - pinned;
162

163 164 165
	return 0;
}

166
static struct sg_table *
167
i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
168
{
169
	struct address_space *mapping = obj->base.filp->f_mapping;
170
	drm_dma_handle_t *phys;
171 172
	struct sg_table *st;
	struct scatterlist *sg;
173
	char *vaddr;
174
	int i;
175

176
	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
177
		return ERR_PTR(-EINVAL);
178

179 180 181 182 183 184 185 186 187 188 189
	/* Always aligning to the object size, allows a single allocation
	 * to handle all possible callers, and given typical object sizes,
	 * the alignment of the buddy allocation will naturally match.
	 */
	phys = drm_pci_alloc(obj->base.dev,
			     obj->base.size,
			     roundup_pow_of_two(obj->base.size));
	if (!phys)
		return ERR_PTR(-ENOMEM);

	vaddr = phys->vaddr;
190 191 192 193 194
	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
		struct page *page;
		char *src;

		page = shmem_read_mapping_page(mapping, i);
195 196 197 198
		if (IS_ERR(page)) {
			st = ERR_CAST(page);
			goto err_phys;
		}
199 200 201 202 203 204

		src = kmap_atomic(page);
		memcpy(vaddr, src, PAGE_SIZE);
		drm_clflush_virt_range(vaddr, PAGE_SIZE);
		kunmap_atomic(src);

205
		put_page(page);
206 207 208
		vaddr += PAGE_SIZE;
	}

209
	i915_gem_chipset_flush(to_i915(obj->base.dev));
210 211

	st = kmalloc(sizeof(*st), GFP_KERNEL);
212 213 214 215
	if (!st) {
		st = ERR_PTR(-ENOMEM);
		goto err_phys;
	}
216 217 218

	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
		kfree(st);
219 220
		st = ERR_PTR(-ENOMEM);
		goto err_phys;
221 222 223 224 225
	}

	sg = st->sgl;
	sg->offset = 0;
	sg->length = obj->base.size;
226

227
	sg_dma_address(sg) = phys->busaddr;
228 229
	sg_dma_len(sg) = obj->base.size;

230 231 232 233 234
	obj->phys_handle = phys;
	return st;

err_phys:
	drm_pci_free(obj->base.dev, phys);
235
	return st;
236 237 238
}

static void
239
__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
240 241
				struct sg_table *pages,
				bool needs_clflush)
242
{
243
	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
244

245 246
	if (obj->mm.madv == I915_MADV_DONTNEED)
		obj->mm.dirty = false;
247

248 249
	if (needs_clflush &&
	    (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
250
	    !i915_gem_object_is_coherent(obj))
251
		drm_clflush_sg(pages);
252 253 254 255 256 257 258 259 260

	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
}

static void
i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
			       struct sg_table *pages)
{
261
	__i915_gem_object_release_shmem(obj, pages, false);
262

263
	if (obj->mm.dirty) {
264
		struct address_space *mapping = obj->base.filp->f_mapping;
265
		char *vaddr = obj->phys_handle->vaddr;
266 267 268
		int i;

		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
269 270 271 272 273 274 275 276 277 278 279 280 281
			struct page *page;
			char *dst;

			page = shmem_read_mapping_page(mapping, i);
			if (IS_ERR(page))
				continue;

			dst = kmap_atomic(page);
			drm_clflush_virt_range(vaddr, PAGE_SIZE);
			memcpy(dst, vaddr, PAGE_SIZE);
			kunmap_atomic(dst);

			set_page_dirty(page);
282
			if (obj->mm.madv == I915_MADV_WILLNEED)
283
				mark_page_accessed(page);
284
			put_page(page);
285 286
			vaddr += PAGE_SIZE;
		}
287
		obj->mm.dirty = false;
288 289
	}

290 291
	sg_free_table(pages);
	kfree(pages);
292 293

	drm_pci_free(obj->base.dev, obj->phys_handle);
294 295 296 297 298
}

static void
i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
{
299
	i915_gem_object_unpin_pages(obj);
300 301 302 303 304 305 306 307
}

static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
	.get_pages = i915_gem_object_get_pages_phys,
	.put_pages = i915_gem_object_put_pages_phys,
	.release = i915_gem_object_release_phys,
};

308 309
static const struct drm_i915_gem_object_ops i915_gem_object_ops;

310
int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
311 312 313
{
	struct i915_vma *vma;
	LIST_HEAD(still_in_list);
314 315 316
	int ret;

	lockdep_assert_held(&obj->base.dev->struct_mutex);
317

318 319 320 321
	/* Closed vma are removed from the obj->vma_list - but they may
	 * still have an active binding on the object. To remove those we
	 * must wait for all rendering to complete to the object (as unbinding
	 * must anyway), and retire the requests.
322
	 */
323 324 325 326 327 328
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   I915_WAIT_ALL,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
329 330 331 332 333
	if (ret)
		return ret;

	i915_gem_retire_requests(to_i915(obj->base.dev));

334 335 336 337 338 339 340 341 342 343 344 345 346
	while ((vma = list_first_entry_or_null(&obj->vma_list,
					       struct i915_vma,
					       obj_link))) {
		list_move_tail(&vma->obj_link, &still_in_list);
		ret = i915_vma_unbind(vma);
		if (ret)
			break;
	}
	list_splice(&still_in_list, &obj->vma_list);

	return ret;
}

347 348 349 350 351
static long
i915_gem_object_wait_fence(struct dma_fence *fence,
			   unsigned int flags,
			   long timeout,
			   struct intel_rps_client *rps)
352
{
353
	struct drm_i915_gem_request *rq;
354

355
	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
356

357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
		return timeout;

	if (!dma_fence_is_i915(fence))
		return dma_fence_wait_timeout(fence,
					      flags & I915_WAIT_INTERRUPTIBLE,
					      timeout);

	rq = to_request(fence);
	if (i915_gem_request_completed(rq))
		goto out;

	/* This client is about to stall waiting for the GPU. In many cases
	 * this is undesirable and limits the throughput of the system, as
	 * many clients cannot continue processing user input/output whilst
	 * blocked. RPS autotuning may take tens of milliseconds to respond
	 * to the GPU load and thus incurs additional latency for the client.
	 * We can circumvent that by promoting the GPU frequency to maximum
	 * before we wait. This makes the GPU throttle up much more quickly
	 * (good for benchmarks and user experience, e.g. window animations),
	 * but at a cost of spending more power processing the workload
	 * (bad for battery). Not all clients even want their results
	 * immediately and for them we should just let the GPU select its own
	 * frequency to maximise efficiency. To prevent a single client from
	 * forcing the clocks too high for the whole system, we only allow
	 * each client to waitboost once in a busy period.
	 */
	if (rps) {
		if (INTEL_GEN(rq->i915) >= 6)
			gen6_rps_boost(rq->i915, rps, rq->emitted_jiffies);
		else
			rps = NULL;
389 390
	}

391 392 393 394 395 396
	timeout = i915_wait_request(rq, flags, timeout);

out:
	if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq))
		i915_gem_request_retire_upto(rq);

397
	if (rps && i915_gem_request_global_seqno(rq) == intel_engine_last_submit(rq->engine)) {
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
		/* The GPU is now idle and this client has stalled.
		 * Since no other client has submitted a request in the
		 * meantime, assume that this client is the only one
		 * supplying work to the GPU but is unable to keep that
		 * work supplied because it is waiting. Since the GPU is
		 * then never kept fully busy, RPS autoclocking will
		 * keep the clocks relatively low, causing further delays.
		 * Compensate by giving the synchronous client credit for
		 * a waitboost next time.
		 */
		spin_lock(&rq->i915->rps.client_lock);
		list_del_init(&rps->link);
		spin_unlock(&rq->i915->rps.client_lock);
	}

	return timeout;
}

static long
i915_gem_object_wait_reservation(struct reservation_object *resv,
				 unsigned int flags,
				 long timeout,
				 struct intel_rps_client *rps)
{
422
	unsigned int seq = __read_seqcount_begin(&resv->seq);
423
	struct dma_fence *excl;
424
	bool prune_fences = false;
425 426 427 428

	if (flags & I915_WAIT_ALL) {
		struct dma_fence **shared;
		unsigned int count, i;
429 430
		int ret;

431 432
		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
433 434 435
		if (ret)
			return ret;

436 437 438 439
		for (i = 0; i < count; i++) {
			timeout = i915_gem_object_wait_fence(shared[i],
							     flags, timeout,
							     rps);
440
			if (timeout < 0)
441
				break;
442

443 444 445 446 447 448
			dma_fence_put(shared[i]);
		}

		for (; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
449 450

		prune_fences = count && timeout >= 0;
451 452
	} else {
		excl = reservation_object_get_excl_rcu(resv);
453 454
	}

455
	if (excl && timeout >= 0) {
456
		timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps);
457 458
		prune_fences = timeout >= 0;
	}
459 460 461

	dma_fence_put(excl);

462 463 464 465
	/* Oportunistically prune the fences iff we know they have *all* been
	 * signaled and that the reservation object has not been changed (i.e.
	 * no new fences have been added).
	 */
466
	if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
467 468 469 470 471
		if (reservation_object_trylock(resv)) {
			if (!__read_seqcount_retry(&resv->seq, seq))
				reservation_object_add_excl_fence(resv, NULL);
			reservation_object_unlock(resv);
		}
472 473
	}

474
	return timeout;
475 476
}

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
static void __fence_set_priority(struct dma_fence *fence, int prio)
{
	struct drm_i915_gem_request *rq;
	struct intel_engine_cs *engine;

	if (!dma_fence_is_i915(fence))
		return;

	rq = to_request(fence);
	engine = rq->engine;
	if (!engine->schedule)
		return;

	engine->schedule(rq, prio);
}

static void fence_set_priority(struct dma_fence *fence, int prio)
{
	/* Recurse once into a fence-array */
	if (dma_fence_is_array(fence)) {
		struct dma_fence_array *array = to_dma_fence_array(fence);
		int i;

		for (i = 0; i < array->num_fences; i++)
			__fence_set_priority(array->fences[i], prio);
	} else {
		__fence_set_priority(fence, prio);
	}
}

int
i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
			      unsigned int flags,
			      int prio)
{
	struct dma_fence *excl;

	if (flags & I915_WAIT_ALL) {
		struct dma_fence **shared;
		unsigned int count, i;
		int ret;

		ret = reservation_object_get_fences_rcu(obj->resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			fence_set_priority(shared[i], prio);
			dma_fence_put(shared[i]);
		}

		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(obj->resv);
	}

	if (excl) {
		fence_set_priority(excl, prio);
		dma_fence_put(excl);
	}
	return 0;
}

541 542 543 544 545 546
/**
 * Waits for rendering to the object to be completed
 * @obj: i915 gem object
 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 * @timeout: how long to wait
 * @rps: client (user process) to charge for any waitboosting
547
 */
548 549 550 551 552
int
i915_gem_object_wait(struct drm_i915_gem_object *obj,
		     unsigned int flags,
		     long timeout,
		     struct intel_rps_client *rps)
553
{
554 555 556 557 558 559 560
	might_sleep();
#if IS_ENABLED(CONFIG_LOCKDEP)
	GEM_BUG_ON(debug_locks &&
		   !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
		   !!(flags & I915_WAIT_LOCKED));
#endif
	GEM_BUG_ON(timeout < 0);
561

562 563 564
	timeout = i915_gem_object_wait_reservation(obj->resv,
						   flags, timeout,
						   rps);
565
	return timeout < 0 ? timeout : 0;
566 567 568 569 570 571 572 573 574
}

static struct intel_rps_client *to_rps_client(struct drm_file *file)
{
	struct drm_i915_file_private *fpriv = file->driver_priv;

	return &fpriv->rps;
}

575 576 577 578
int
i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
			    int align)
{
579
	int ret;
580

581 582
	if (align > obj->base.size)
		return -EINVAL;
583

584
	if (obj->ops == &i915_gem_phys_ops)
585 586
		return 0;

587
	if (obj->mm.madv != I915_MADV_WILLNEED)
588 589 590 591 592
		return -EFAULT;

	if (obj->base.filp == NULL)
		return -EINVAL;

Chris Wilson's avatar
Chris Wilson committed
593 594 595 596
	ret = i915_gem_object_unbind(obj);
	if (ret)
		return ret;

597
	__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
598 599
	if (obj->mm.pages)
		return -EBUSY;
600

601
	GEM_BUG_ON(obj->ops != &i915_gem_object_ops);
602 603
	obj->ops = &i915_gem_phys_ops;

604 605 606 607 608 609 610 611 612
	ret = i915_gem_object_pin_pages(obj);
	if (ret)
		goto err_xfer;

	return 0;

err_xfer:
	obj->ops = &i915_gem_object_ops;
	return ret;
613 614 615 616 617
}

static int
i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pwrite *args,
618
		     struct drm_file *file)
619 620
{
	void *vaddr = obj->phys_handle->vaddr + args->offset;
621
	char __user *user_data = u64_to_user_ptr(args->data_ptr);
622 623 624 625

	/* We manually control the domain here and pretend that it
	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
	 */
626
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
627 628
	if (copy_from_user(vaddr, user_data, args->size))
		return -EFAULT;
629

630
	drm_clflush_virt_range(vaddr, args->size);
631
	i915_gem_chipset_flush(to_i915(obj->base.dev));
632

633
	intel_fb_obj_flush(obj, ORIGIN_CPU);
634
	return 0;
635 636
}

637
void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
638
{
639
	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
640 641 642 643
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
644
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
645
	kmem_cache_free(dev_priv->objects, obj);
646 647
}

648 649
static int
i915_gem_create(struct drm_file *file,
650
		struct drm_i915_private *dev_priv,
651 652
		uint64_t size,
		uint32_t *handle_p)
653
{
654
	struct drm_i915_gem_object *obj;
655 656
	int ret;
	u32 handle;
657

658
	size = roundup(size, PAGE_SIZE);
659 660
	if (size == 0)
		return -EINVAL;
661 662

	/* Allocate the new object */
663
	obj = i915_gem_object_create(dev_priv, size);
664 665
	if (IS_ERR(obj))
		return PTR_ERR(obj);
666

667
	ret = drm_gem_handle_create(file, &obj->base, &handle);
668
	/* drop reference from allocate - handle holds it now */
669
	i915_gem_object_put(obj);
670 671
	if (ret)
		return ret;
672

673
	*handle_p = handle;
674 675 676
	return 0;
}

677 678 679 680 681 682
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
683
	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
684
	args->size = args->pitch * args->height;
685
	return i915_gem_create(file, to_i915(dev),
686
			       args->size, &args->handle);
687 688 689 690
}

/**
 * Creates a new mm object and returns a handle to it.
691 692 693
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
694 695 696 697 698
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
699
	struct drm_i915_private *dev_priv = to_i915(dev);
700
	struct drm_i915_gem_create *args = data;
701

702
	i915_gem_flush_free_objects(dev_priv);
703

704
	return i915_gem_create(file, dev_priv,
705
			       args->size, &args->handle);
706 707
}

708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

734
static inline int
735 736
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

760 761 762 763 764 765
/*
 * Pins the specified object's pages and synchronizes the object with
 * GPU accesses. Sets needs_clflush to non-zero if the caller should
 * flush the object from the CPU cache.
 */
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
766
				    unsigned int *needs_clflush)
767 768 769
{
	int ret;

770
	lockdep_assert_held(&obj->base.dev->struct_mutex);
771

772
	*needs_clflush = 0;
773 774
	if (!i915_gem_object_has_struct_page(obj))
		return -ENODEV;
775

776 777 778 779 780
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
781 782 783
	if (ret)
		return ret;

784
	ret = i915_gem_object_pin_pages(obj);
785 786 787
	if (ret)
		return ret;

788 789 790 791 792 793 794 795 796
	if (i915_gem_object_is_coherent(obj) ||
	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
		ret = i915_gem_object_set_to_cpu_domain(obj, false);
		if (ret)
			goto err_unpin;
		else
			goto out;
	}

797 798
	i915_gem_object_flush_gtt_write_domain(obj);

799 800 801 802 803 804
	/* If we're not in the cpu read domain, set ourself into the gtt
	 * read domain and manually flush cachelines (if required). This
	 * optimizes for the case when the gpu will dirty the data
	 * anyway again before the next pread happens.
	 */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
805
		*needs_clflush = CLFLUSH_BEFORE;
806

807
out:
808
	/* return with the pages pinned */
809
	return 0;
810 811 812 813

err_unpin:
	i915_gem_object_unpin_pages(obj);
	return ret;
814 815 816 817 818 819 820
}

int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
				     unsigned int *needs_clflush)
{
	int ret;

821 822
	lockdep_assert_held(&obj->base.dev->struct_mutex);

823 824 825 826
	*needs_clflush = 0;
	if (!i915_gem_object_has_struct_page(obj))
		return -ENODEV;

827 828 829 830 831 832
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   I915_WAIT_ALL,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
833 834 835
	if (ret)
		return ret;

836
	ret = i915_gem_object_pin_pages(obj);
837 838 839
	if (ret)
		return ret;

840 841 842 843 844 845 846 847 848
	if (i915_gem_object_is_coherent(obj) ||
	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
		ret = i915_gem_object_set_to_cpu_domain(obj, true);
		if (ret)
			goto err_unpin;
		else
			goto out;
	}

849 850
	i915_gem_object_flush_gtt_write_domain(obj);

851 852 853 854 855 856
	/* If we're not in the cpu write domain, set ourself into the
	 * gtt write domain and manually flush cachelines (as required).
	 * This optimizes for the case when the gpu will use the data
	 * right away and we therefore have to clflush anyway.
	 */
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
857
		*needs_clflush |= CLFLUSH_AFTER;
858 859 860 861 862

	/* Same trick applies to invalidate partially written cachelines read
	 * before writing.
	 */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
863
		*needs_clflush |= CLFLUSH_BEFORE;
864

865
out:
866
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
867
	obj->mm.dirty = true;
868
	/* return with the pages pinned */
869
	return 0;
870 871 872 873

err_unpin:
	i915_gem_object_unpin_pages(obj);
	return ret;
874 875
}

876 877 878 879
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
880
	if (unlikely(swizzled)) {
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

898 899 900
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
901
shmem_pread_slow(struct page *page, int offset, int length,
902 903 904 905 906 907 908 909
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
910
		shmem_clflush_swizzled_range(vaddr + offset, length,
911
					     page_do_bit17_swizzling);
912 913

	if (page_do_bit17_swizzling)
914
		ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
915
	else
916
		ret = __copy_to_user(user_data, vaddr + offset, length);
917 918
	kunmap(page);

919
	return ret ? - EFAULT : 0;
920 921
}

922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
static int
shmem_pread(struct page *page, int offset, int length, char __user *user_data,
	    bool page_do_bit17_swizzling, bool needs_clflush)
{
	int ret;

	ret = -ENODEV;
	if (!page_do_bit17_swizzling) {
		char *vaddr = kmap_atomic(page);

		if (needs_clflush)
			drm_clflush_virt_range(vaddr + offset, length);
		ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
		kunmap_atomic(vaddr);
	}
	if (ret == 0)
		return 0;

	return shmem_pread_slow(page, offset, length, user_data,
				page_do_bit17_swizzling, needs_clflush);
}

static int
i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args)
{
	char __user *user_data;
	u64 remain;
	unsigned int obj_do_bit17_swizzling;
	unsigned int needs_clflush;
	unsigned int idx, offset;
	int ret;

	obj_do_bit17_swizzling = 0;
	if (i915_gem_object_needs_bit17_swizzle(obj))
		obj_do_bit17_swizzling = BIT(17);

	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
	if (ret)
		return ret;

	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
	mutex_unlock(&obj->base.dev->struct_mutex);
	if (ret)
		return ret;

	remain = args->size;
	user_data = u64_to_user_ptr(args->data_ptr);
	offset = offset_in_page(args->offset);
	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
		struct page *page = i915_gem_object_get_page(obj, idx);
		int length;

		length = remain;
		if (offset + length > PAGE_SIZE)
			length = PAGE_SIZE - offset;

		ret = shmem_pread(page, offset, length, user_data,
				  page_to_phys(page) & obj_do_bit17_swizzling,
				  needs_clflush);
		if (ret)
			break;

		remain -= length;
		user_data += length;
		offset = 0;
	}

	i915_gem_obj_finish_shmem_access(obj);
	return ret;
}

static inline bool
gtt_user_read(struct io_mapping *mapping,
	      loff_t base, int offset,
	      char __user *user_data, int length)
998 999
{
	void *vaddr;
1000
	unsigned long unwritten;
1001 1002

	/* We can use the cpu mem copy function because this is X86. */
1003 1004 1005 1006 1007 1008 1009 1010 1011
	vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base);
	unwritten = __copy_to_user_inatomic(user_data, vaddr + offset, length);
	io_mapping_unmap_atomic(vaddr);
	if (unwritten) {
		vaddr = (void __force *)
			io_mapping_map_wc(mapping, base, PAGE_SIZE);
		unwritten = copy_to_user(user_data, vaddr + offset, length);
		io_mapping_unmap(vaddr);
	}
1012 1013 1014 1015
	return unwritten;
}

static int
1016 1017
i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
		   const struct drm_i915_gem_pread *args)
1018
{
1019 1020
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	struct i915_ggtt *ggtt = &i915->ggtt;
1021
	struct drm_mm_node node;
1022 1023 1024
	struct i915_vma *vma;
	void __user *user_data;
	u64 remain, offset;
1025 1026
	int ret;

1027 1028 1029 1030 1031 1032 1033
	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
	if (ret)
		return ret;

	intel_runtime_pm_get(i915);
	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
				       PIN_MAPPABLE | PIN_NONBLOCK);
1034 1035 1036
	if (!IS_ERR(vma)) {
		node.start = i915_ggtt_offset(vma);
		node.allocated = false;
1037
		ret = i915_vma_put_fence(vma);
1038 1039 1040 1041 1042
		if (ret) {
			i915_vma_unpin(vma);
			vma = ERR_PTR(ret);
		}
	}
Chris Wilson's avatar
Chris Wilson committed
1043
	if (IS_ERR(vma)) {
1044
		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1045
		if (ret)
1046 1047
			goto out_unlock;
		GEM_BUG_ON(!node.allocated);
1048 1049 1050 1051 1052 1053
	}

	ret = i915_gem_object_set_to_gtt_domain(obj, false);
	if (ret)
		goto out_unpin;

1054
	mutex_unlock(&i915->drm.struct_mutex);
1055

1056 1057 1058
	user_data = u64_to_user_ptr(args->data_ptr);
	remain = args->size;
	offset = args->offset;
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		u32 page_base = node.start;
		unsigned page_offset = offset_in_page(offset);
		unsigned page_length = PAGE_SIZE - page_offset;
		page_length = remain < page_length ? remain : page_length;
		if (node.allocated) {
			wmb();
			ggtt->base.insert_page(&ggtt->base,
					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1075
					       node.start, I915_CACHE_NONE, 0);
1076 1077 1078 1079
			wmb();
		} else {
			page_base += offset & PAGE_MASK;
		}
1080 1081 1082

		if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
				  user_data, page_length)) {
1083 1084 1085 1086 1087 1088 1089 1090 1091
			ret = -EFAULT;
			break;
		}

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

1092
	mutex_lock(&i915->drm.struct_mutex);
1093 1094 1095 1096
out_unpin:
	if (node.allocated) {
		wmb();
		ggtt->base.clear_range(&ggtt->base,
1097
				       node.start, node.size);
1098 1099
		remove_mappable_node(&node);
	} else {
Chris Wilson's avatar
Chris Wilson committed
1100
		i915_vma_unpin(vma);
1101
	}
1102 1103 1104
out_unlock:
	intel_runtime_pm_put(i915);
	mutex_unlock(&i915->drm.struct_mutex);
1105

1106 1107 1108
	return ret;
}

1109 1110
/**
 * Reads data from the object referenced by handle.
1111 1112 1113
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
1114 1115 1116 1117 1118
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1119
		     struct drm_file *file)
1120 1121
{
	struct drm_i915_gem_pread *args = data;
1122
	struct drm_i915_gem_object *obj;
1123
	int ret;
1124

1125 1126 1127 1128
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
1129
		       u64_to_user_ptr(args->data_ptr),
1130 1131 1132
		       args->size))
		return -EFAULT;

1133
	obj = i915_gem_object_lookup(file, args->handle);
1134 1135
	if (!obj)
		return -ENOENT;
1136

1137
	/* Bounds check source.  */
1138
	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
1139
		ret = -EINVAL;
1140
		goto out;
1141 1142
	}

Chris Wilson's avatar
Chris Wilson committed
1143 1144
	trace_i915_gem_object_pread(obj, args->offset, args->size);

1145 1146 1147 1148
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE,
				   MAX_SCHEDULE_TIMEOUT,
				   to_rps_client(file));
1149
	if (ret)
1150
		goto out;
1151

1152
	ret = i915_gem_object_pin_pages(obj);
1153
	if (ret)
1154
		goto out;
1155

1156
	ret = i915_gem_shmem_pread(obj, args);
1157
	if (ret == -EFAULT || ret == -ENODEV)
1158
		ret = i915_gem_gtt_pread(obj, args);
1159

1160 1161
	i915_gem_object_unpin_pages(obj);
out:
1162
	i915_gem_object_put(obj);
1163
	return ret;
1164 1165
}

1166 1167
/* This is the fast write path which cannot handle
 * page faults in the source data
1168
 */
1169

1170 1171 1172 1173
static inline bool
ggtt_write(struct io_mapping *mapping,
	   loff_t base, int offset,
	   char __user *user_data, int length)
1174
{
1175
	void *vaddr;
1176
	unsigned long unwritten;
1177

1178
	/* We can use the cpu mem copy function because this is X86. */
1179 1180
	vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base);
	unwritten = __copy_from_user_inatomic_nocache(vaddr + offset,
1181
						      user_data, length);
1182 1183 1184 1185 1186 1187 1188
	io_mapping_unmap_atomic(vaddr);
	if (unwritten) {
		vaddr = (void __force *)
			io_mapping_map_wc(mapping, base, PAGE_SIZE);
		unwritten = copy_from_user(vaddr + offset, user_data, length);
		io_mapping_unmap(vaddr);
	}
1189 1190 1191 1192

	return unwritten;
}

1193 1194 1195
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
1196
 * @obj: i915 GEM object
1197
 * @args: pwrite arguments structure
1198
 */
1199
static int
1200 1201
i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
			 const struct drm_i915_gem_pwrite *args)
1202
{
1203
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1204 1205
	struct i915_ggtt *ggtt = &i915->ggtt;
	struct drm_mm_node node;
1206