Commit f1f6a7dd authored by John Hubbard's avatar John Hubbard Committed by Linus Torvalds

mm, tree-wide: rename put_user_page*() to unpin_user_page*()

In order to provide a clearer, more symmetric API for pinning and
unpinning DMA pages.  This way, pin_user_pages*() calls match up with
unpin_user_pages*() calls, and the API is a lot closer to being
self-explanatory.

Link: http://lkml.kernel.org/r/20200107224558.2362728-23-jhubbard@nvidia.comSigned-off-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Reviewed-by: default avatarJan Kara <jack@suse.cz>
Cc: Alex Williamson <alex.williamson@redhat.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Björn Töpel <bjorn.topel@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Hans Verkuil <hverkuil-cisco@xs4all.nl>
Cc: Ira Weiny <ira.weiny@intel.com>
Cc: Jason Gunthorpe <jgg@mellanox.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Kirill A. Shutemov <kirill@shutemov.name>
Cc: Leon Romanovsky <leonro@mellanox.com>
Cc: Mauro Carvalho Chehab <mchehab@kernel.org>
Cc: Mike Rapoport <rppt@linux.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent bdffe23e
......@@ -219,7 +219,7 @@ since the system was booted, via two new /proc/vmstat entries: ::
/proc/vmstat/nr_foll_pin_requested
Those are both going to show zero, unless CONFIG_DEBUG_VM is set. This is
because there is a noticeable performance drop in put_user_page(), when they
because there is a noticeable performance drop in unpin_user_page(), when they
are activated.
References
......
......@@ -168,7 +168,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
free_exit:
/* free the references taken */
put_user_pages(mem->hpages, pinned);
unpin_user_pages(mem->hpages, pinned);
vfree(mem->hpas);
kfree(mem);
......@@ -214,7 +214,7 @@ static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem)
if (mem->hpas[i] & MM_IOMMU_TABLE_GROUP_PAGE_DIRTY)
SetPageDirty(page);
put_user_page(page);
unpin_user_page(page);
mem->hpas[i] = 0;
}
......
......@@ -188,8 +188,8 @@ via_free_sg_info(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
kfree(vsg->desc_pages);
/* fall through */
case dr_via_pages_locked:
put_user_pages_dirty_lock(vsg->pages, vsg->num_pages,
(vsg->direction == DMA_FROM_DEVICE));
unpin_user_pages_dirty_lock(vsg->pages, vsg->num_pages,
(vsg->direction == DMA_FROM_DEVICE));
/* fall through */
case dr_via_pages_alloc:
vfree(vsg->pages);
......
......@@ -54,7 +54,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->sg_nents, 0) {
page = sg_page_iter_page(&sg_iter);
put_user_pages_dirty_lock(&page, 1, umem->writable && dirty);
unpin_user_pages_dirty_lock(&page, 1, umem->writable && dirty);
}
sg_free_table(&umem->sg_head);
......
......@@ -118,7 +118,7 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np
void hfi1_release_user_pages(struct mm_struct *mm, struct page **p,
size_t npages, bool dirty)
{
put_user_pages_dirty_lock(p, npages, dirty);
unpin_user_pages_dirty_lock(p, npages, dirty);
if (mm) { /* during close after signal, mm can be NULL */
atomic64_sub(npages, &mm->pinned_vm);
......
......@@ -482,7 +482,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
if (ret < 0) {
put_user_page(pages[0]);
unpin_user_page(pages[0]);
goto out;
}
......@@ -490,7 +490,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
mthca_uarc_virt(dev, uar, i));
if (ret) {
pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
put_user_page(sg_page(&db_tab->page[i].mem));
unpin_user_page(sg_page(&db_tab->page[i].mem));
goto out;
}
......@@ -556,7 +556,7 @@ void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,
if (db_tab->page[i].uvirt) {
mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1);
pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
put_user_page(sg_page(&db_tab->page[i].mem));
unpin_user_page(sg_page(&db_tab->page[i].mem));
}
}
......
......@@ -40,7 +40,7 @@
static void __qib_release_user_pages(struct page **p, size_t num_pages,
int dirty)
{
put_user_pages_dirty_lock(p, num_pages, dirty);
unpin_user_pages_dirty_lock(p, num_pages, dirty);
}
/**
......
......@@ -317,7 +317,7 @@ static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
* the caller can ignore this page.
*/
if (put) {
put_user_page(page);
unpin_user_page(page);
} else {
/* coalesce case */
kunmap(page);
......@@ -631,7 +631,7 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev,
kunmap(pkt->addr[i].page);
if (pkt->addr[i].put_page)
put_user_page(pkt->addr[i].page);
unpin_user_page(pkt->addr[i].page);
else
__free_page(pkt->addr[i].page);
} else if (pkt->addr[i].kvaddr) {
......@@ -706,7 +706,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
/* if error, return all pages not managed by pkt */
free_pages:
while (i < j)
put_user_page(pages[i++]);
unpin_user_page(pages[i++]);
done:
return ret;
......
......@@ -75,7 +75,7 @@ static void usnic_uiom_put_pages(struct list_head *chunk_list, int dirty)
for_each_sg(chunk->page_list, sg, chunk->nents, i) {
page = sg_page(sg);
pa = sg_phys(sg);
put_user_pages_dirty_lock(&page, 1, dirty);
unpin_user_pages_dirty_lock(&page, 1, dirty);
usnic_dbg("pa: %pa\n", &pa);
}
kfree(chunk);
......
......@@ -63,7 +63,7 @@ struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index)
static void siw_free_plist(struct siw_page_chunk *chunk, int num_pages,
bool dirty)
{
put_user_pages_dirty_lock(chunk->plist, num_pages, dirty);
unpin_user_pages_dirty_lock(chunk->plist, num_pages, dirty);
}
void siw_umem_release(struct siw_umem *umem, bool dirty)
......
......@@ -349,8 +349,8 @@ int videobuf_dma_free(struct videobuf_dmabuf *dma)
BUG_ON(dma->sglen);
if (dma->pages) {
put_user_pages_dirty_lock(dma->pages, dma->nr_pages,
dma->direction == DMA_FROM_DEVICE);
unpin_user_pages_dirty_lock(dma->pages, dma->nr_pages,
dma->direction == DMA_FROM_DEVICE);
kfree(dma->pages);
dma->pages = NULL;
}
......
......@@ -360,8 +360,8 @@ static int transfer_max_buffers(struct goldfish_pipe *pipe,
*consumed_size = pipe->command_buffer->rw_params.consumed_size;
put_user_pages_dirty_lock(pipe->pages, pages_count,
!is_write && *consumed_size > 0);
unpin_user_pages_dirty_lock(pipe->pages, pages_count,
!is_write && *consumed_size > 0);
mutex_unlock(&pipe->lock);
return 0;
......
......@@ -310,7 +310,7 @@ static int put_pfn(unsigned long pfn, int prot)
if (!is_invalid_reserved_pfn(pfn)) {
struct page *page = pfn_to_page(pfn);
put_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE);
unpin_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE);
return 1;
}
return 0;
......
......@@ -6005,7 +6005,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
for (j = 0; j < imu->nr_bvecs; j++)
put_user_page(imu->bvec[j].bv_page);
unpin_user_page(imu->bvec[j].bv_page);
if (ctx->account_mem)
io_unaccount_mem(ctx->user, imu->nr_bvecs);
......@@ -6150,7 +6150,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
* release any pages we did get
*/
if (pret > 0)
put_user_pages(pages, pret);
unpin_user_pages(pages, pret);
if (ctx->account_mem)
io_unaccount_mem(ctx->user, nr_pages);
kvfree(imu->bvec);
......
......@@ -1039,27 +1039,27 @@ static inline void put_page(struct page *page)
}
/**
* put_user_page() - release a gup-pinned page
* unpin_user_page() - release a gup-pinned page
* @page: pointer to page to be released
*
* Pages that were pinned via pin_user_pages*() must be released via either
* put_user_page(), or one of the put_user_pages*() routines. This is so that
* eventually such pages can be separately tracked and uniquely handled. In
* unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
* that eventually such pages can be separately tracked and uniquely handled. In
* particular, interactions with RDMA and filesystems need special handling.
*
* put_user_page() and put_page() are not interchangeable, despite this early
* implementation that makes them look the same. put_user_page() calls must
* unpin_user_page() and put_page() are not interchangeable, despite this early
* implementation that makes them look the same. unpin_user_page() calls must
* be perfectly matched up with pin*() calls.
*/
static inline void put_user_page(struct page *page)
static inline void unpin_user_page(struct page *page)
{
put_page(page);
}
void put_user_pages_dirty_lock(struct page **pages, unsigned long npages,
bool make_dirty);
void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
bool make_dirty);
void put_user_pages(struct page **pages, unsigned long npages);
void unpin_user_pages(struct page **pages, unsigned long npages);
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
#define SECTION_IN_PAGE_FLAGS
......@@ -2590,7 +2590,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
#define FOLL_ANON 0x8000 /* don't do file mappings */
#define FOLL_LONGTERM 0x10000 /* mapping lifetime is indefinite: see below */
#define FOLL_SPLIT_PMD 0x20000 /* split huge pmd before returning */
#define FOLL_PIN 0x40000 /* pages must be released via put_user_page() */
#define FOLL_PIN 0x40000 /* pages must be released via unpin_user_page */
/*
* FOLL_PIN and FOLL_LONGTERM may be used in various combinations with each
......@@ -2625,7 +2625,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
* Direct IO). This lets the filesystem know that some non-file-system entity is
* potentially changing the pages' data. In contrast to FOLL_GET (whose pages
* are released via put_page()), FOLL_PIN pages must be released, ultimately, by
* a call to put_user_page().
* a call to unpin_user_page().
*
* FOLL_PIN is similar to FOLL_GET: both of these pin pages. They use different
* and separate refcounting mechanisms, however, and that means that each has
......@@ -2633,7 +2633,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
*
* FOLL_GET: get_user_pages*() to acquire, and put_page() to release.
*
* FOLL_PIN: pin_user_pages*() to acquire, and put_user_pages to release.
* FOLL_PIN: pin_user_pages*() to acquire, and unpin_user_pages to release.
*
* FOLL_PIN and FOLL_GET are mutually exclusive for a given function call.
* (The underlying pages may experience both FOLL_GET-based and FOLL_PIN-based
......@@ -2643,7 +2643,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
* FOLL_PIN should be set internally by the pin_user_pages*() APIs, never
* directly by the caller. That's in order to help avoid mismatches when
* releasing pages: get_user_pages*() pages must be released via put_page(),
* while pin_user_pages*() pages must be released via put_user_page().
* while pin_user_pages*() pages must be released via unpin_user_page().
*
* Please see Documentation/vm/pin_user_pages.rst for more information.
*/
......
......@@ -45,7 +45,7 @@ static inline struct page *try_get_compound_head(struct page *page, int refs)
}
/**
* put_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
* unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
* @pages: array of pages to be maybe marked dirty, and definitely released.
* @npages: number of pages in the @pages array.
* @make_dirty: whether to mark the pages dirty
......@@ -55,19 +55,19 @@ static inline struct page *try_get_compound_head(struct page *page, int refs)
*
* For each page in the @pages array, make that page (or its head page, if a
* compound page) dirty, if @make_dirty is true, and if the page was previously
* listed as clean. In any case, releases all pages using put_user_page(),
* possibly via put_user_pages(), for the non-dirty case.
* listed as clean. In any case, releases all pages using unpin_user_page(),
* possibly via unpin_user_pages(), for the non-dirty case.
*
* Please see the put_user_page() documentation for details.
* Please see the unpin_user_page() documentation for details.
*
* set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
* required, then the caller should a) verify that this is really correct,
* because _lock() is usually required, and b) hand code it:
* set_page_dirty_lock(), put_user_page().
* set_page_dirty_lock(), unpin_user_page().
*
*/
void put_user_pages_dirty_lock(struct page **pages, unsigned long npages,
bool make_dirty)
void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
bool make_dirty)
{
unsigned long index;
......@@ -78,7 +78,7 @@ void put_user_pages_dirty_lock(struct page **pages, unsigned long npages,
*/
if (!make_dirty) {
put_user_pages(pages, npages);
unpin_user_pages(pages, npages);
return;
}
......@@ -106,21 +106,21 @@ void put_user_pages_dirty_lock(struct page **pages, unsigned long npages,
*/
if (!PageDirty(page))
set_page_dirty_lock(page);
put_user_page(page);
unpin_user_page(page);
}
}
EXPORT_SYMBOL(put_user_pages_dirty_lock);
EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
/**
* put_user_pages() - release an array of gup-pinned pages.
* unpin_user_pages() - release an array of gup-pinned pages.
* @pages: array of pages to be marked dirty and released.
* @npages: number of pages in the @pages array.
*
* For each page in the @pages array, release the page using put_user_page().
* For each page in the @pages array, release the page using unpin_user_page().
*
* Please see the put_user_page() documentation for details.
* Please see the unpin_user_page() documentation for details.
*/
void put_user_pages(struct page **pages, unsigned long npages)
void unpin_user_pages(struct page **pages, unsigned long npages)
{
unsigned long index;
......@@ -130,9 +130,9 @@ void put_user_pages(struct page **pages, unsigned long npages)
* single operation to the head page should suffice.
*/
for (index = 0; index < npages; index++)
put_user_page(pages[index]);
unpin_user_page(pages[index]);
}
EXPORT_SYMBOL(put_user_pages);
EXPORT_SYMBOL(unpin_user_pages);
#ifdef CONFIG_MMU
static struct page *no_page_table(struct vm_area_struct *vma,
......
......@@ -126,8 +126,8 @@ static int process_vm_rw_single_vec(unsigned long addr,
pa += pinned_pages * PAGE_SIZE;
/* If vm_write is set, the pages need to be made dirty: */
put_user_pages_dirty_lock(process_pages, pinned_pages,
vm_write);
unpin_user_pages_dirty_lock(process_pages, pinned_pages,
vm_write);
}
return rc;
......
......@@ -212,7 +212,7 @@ static int xdp_umem_map_pages(struct xdp_umem *umem)
static void xdp_umem_unpin_pages(struct xdp_umem *umem)
{
put_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
kfree(umem->pgs);
umem->pgs = NULL;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment