Skip to content
Snippets Groups Projects
Select Git revision
  • f99f5f3ea7efd54ba0529c4f2d7c72712918a522
  • drm-misc-templates default
  • wip/final/kci-gitlab-lava-v1
  • wip/vignesh/kci-lava-gitlab-runner
  • kci-gitlab-igt-v8
  • kci-gitlab-igt-v4
  • drm-misc-fixes-2024-10-02
  • drm-misc-next-2024-09-26
  • drm-misc-fixes-2024-09-26
  • drm-misc-next-2024-09-20
  • drm-misc-fixes-2024-09-12
  • drm-misc-fixes-2024-09-05
  • drm-misc-next-fixes-2024-09-05
  • drm-misc-fixes-2024-08-29
  • drm-misc-next-2024-08-29
  • drm-misc-next-2024-08-22
  • drm-misc-fixes-2024-08-22
  • drm-misc-next-2024-08-16
  • drm-misc-fixes-2024-08-15
  • drm-misc-next-2024-08-09
  • drm-misc-fixes-2024-08-08
  • drm-misc-next-2024-08-01
  • drm-misc-fixes-2024-08-01
  • drm-misc-next-fixes-2024-07-25
  • drm-misc-next-fixes-2024-07-19
  • drm-misc-next-fixes-2024-07-11
26 results

pvr_drv.c

Blame
  • pvr_drv.c 29.30 KiB
    // SPDX-License-Identifier: GPL-2.0-only OR MIT
    /* Copyright (c) 2023 Imagination Technologies Ltd. */
    
    #include "pvr_device.h"
    #include "pvr_drv.h"
    #include "pvr_rogue_defs.h"
    #include "pvr_rogue_fwif_client.h"
    #include "pvr_rogue_fwif_shared.h"
    
    #include <uapi/drm/pvr_drm.h>
    
    #include <drm/drm_device.h>
    #include <drm/drm_drv.h>
    #include <drm/drm_file.h>
    #include <drm/drm_gem.h>
    #include <drm/drm_ioctl.h>
    
    #include <linux/err.h>
    #include <linux/export.h>
    #include <linux/fs.h>
    #include <linux/kernel.h>
    #include <linux/mod_devicetable.h>
    #include <linux/module.h>
    #include <linux/moduleparam.h>
    #include <linux/of_device.h>
    #include <linux/of_platform.h>
    #include <linux/platform_device.h>
    
    /**
     * DOC: PowerVR (Series 6 and later) and IMG Graphics Driver
     *
     * This driver supports the following PowerVR/IMG graphics cores from Imagination Technologies:
     *
     * * AXE-1-16M (found in Texas Instruments AM62)
     */
    
    /**
     * pvr_ioctl_create_bo() - IOCTL to create a GEM buffer object.
     * @drm_dev: [IN] Target DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     * &struct drm_pvr_ioctl_create_bo_args.
     * @file: [IN] DRM file-private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_CREATE_BO.
     *
     * Return:
     *  * 0 on success,
     *  * -%EINVAL if the value of &drm_pvr_ioctl_create_bo_args.size is zero
     *    or wider than &typedef size_t,
     *  * -%EINVAL if any bits in &drm_pvr_ioctl_create_bo_args.flags that are
     *    reserved or undefined are set,
     *  * -%EINVAL if any padding fields in &drm_pvr_ioctl_create_bo_args are not
     *    zero,
     *  * Any error encountered while creating the object (see
     *    pvr_gem_object_create()), or
     *  * Any error encountered while transferring ownership of the object into a
     *    userspace-accessible handle (see pvr_gem_object_into_handle()).
     */
    static int
    pvr_ioctl_create_bo(struct drm_device *drm_dev, void *raw_args,
    		    struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_get_bo_mmap_offset() - IOCTL to generate a "fake" offset to be
     * used when calling mmap() from userspace to map the given GEM buffer object
     * @drm_dev: [IN] DRM device (unused).
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_get_bo_mmap_offset_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_GET_BO_MMAP_OFFSET.
     *
     * This IOCTL does *not* perform an mmap. See the docs on
     * &struct drm_pvr_ioctl_get_bo_mmap_offset_args for details.
     *
     * Return:
     *  * 0 on success,
     *  * -%ENOENT if the handle does not reference a valid GEM buffer object,
     *  * -%EINVAL if any padding fields in &struct
     *    drm_pvr_ioctl_get_bo_mmap_offset_args are not zero, or
     *  * Any error returned by drm_gem_create_mmap_offset().
     */
    static int
    pvr_ioctl_get_bo_mmap_offset(struct drm_device *drm_dev, void *raw_args,
    			     struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    static __always_inline u64
    pvr_fw_version_packed(u32 major, u32 minor)
    {
    	return ((u64)major << 32) | minor;
    }
    
    static u32
    rogue_get_common_store_partition_space_size(struct pvr_device *pvr_dev)
    {
    	u32 max_partitions = 0;
    	u32 tile_size_x = 0;
    	u32 tile_size_y = 0;
    
    	PVR_FEATURE_VALUE(pvr_dev, tile_size_x, &tile_size_x);
    	PVR_FEATURE_VALUE(pvr_dev, tile_size_y, &tile_size_y);
    	PVR_FEATURE_VALUE(pvr_dev, max_partitions, &max_partitions);
    
    	if (tile_size_x == 16 && tile_size_y == 16) {
    		u32 usc_min_output_registers_per_pix = 0;
    
    		PVR_FEATURE_VALUE(pvr_dev, usc_min_output_registers_per_pix,
    				  &usc_min_output_registers_per_pix);
    
    		return tile_size_x * tile_size_y * max_partitions *
    		       usc_min_output_registers_per_pix;
    	}
    
    	return max_partitions * 1024;
    }
    
    static u32
    rogue_get_common_store_alloc_region_size(struct pvr_device *pvr_dev)
    {
    	u32 common_store_size_in_dwords = 512 * 4 * 4;
    	u32 alloc_region_size;
    
    	PVR_FEATURE_VALUE(pvr_dev, common_store_size_in_dwords, &common_store_size_in_dwords);
    
    	alloc_region_size = common_store_size_in_dwords - (256U * 4U) -
    			    rogue_get_common_store_partition_space_size(pvr_dev);
    
    	if (PVR_HAS_QUIRK(pvr_dev, 44079)) {
    		u32 common_store_split_point = (768U * 4U * 4U);
    
    		return min(common_store_split_point - (256U * 4U), alloc_region_size);
    	}
    
    	return alloc_region_size;
    }
    
    static inline u32
    rogue_get_num_phantoms(struct pvr_device *pvr_dev)
    {
    	u32 num_clusters = 1;
    
    	PVR_FEATURE_VALUE(pvr_dev, num_clusters, &num_clusters);
    
    	return ROGUE_REQ_NUM_PHANTOMS(num_clusters);
    }
    
    static inline u32
    rogue_get_max_coeffs(struct pvr_device *pvr_dev)
    {
    	u32 max_coeff_additional_portion = ROGUE_MAX_VERTEX_SHARED_REGISTERS;
    	u32 pending_allocation_shared_regs = 2U * 1024U;
    	u32 pending_allocation_coeff_regs = 0U;
    	u32 num_phantoms = rogue_get_num_phantoms(pvr_dev);
    	u32 tiles_in_flight = 0;
    	u32 max_coeff_pixel_portion;
    
    	PVR_FEATURE_VALUE(pvr_dev, isp_max_tiles_in_flight, &tiles_in_flight);
    	max_coeff_pixel_portion = DIV_ROUND_UP(tiles_in_flight, num_phantoms);
    	max_coeff_pixel_portion *= ROGUE_MAX_PIXEL_SHARED_REGISTERS;
    
    	/*
    	 * Compute tasks on cores with BRN48492 and without compute overlap may lock
    	 * up without two additional lines of coeffs.
    	 */
    	if (PVR_HAS_QUIRK(pvr_dev, 48492) && !PVR_HAS_FEATURE(pvr_dev, compute_overlap))
    		pending_allocation_coeff_regs = 2U * 1024U;
    
    	if (PVR_HAS_ENHANCEMENT(pvr_dev, 38748))
    		pending_allocation_shared_regs = 0;
    
    	if (PVR_HAS_ENHANCEMENT(pvr_dev, 38020))
    		max_coeff_additional_portion += ROGUE_MAX_COMPUTE_SHARED_REGISTERS;
    
    	return rogue_get_common_store_alloc_region_size(pvr_dev) + pending_allocation_coeff_regs -
    		(max_coeff_pixel_portion + max_coeff_additional_portion +
    		 pending_allocation_shared_regs);
    }
    
    static inline u32
    rogue_get_cdm_max_local_mem_size_regs(struct pvr_device *pvr_dev)
    {
    	u32 available_coeffs_in_dwords = rogue_get_max_coeffs(pvr_dev);
    
    	if (PVR_HAS_QUIRK(pvr_dev, 48492) && PVR_HAS_FEATURE(pvr_dev, roguexe) &&
    	    !PVR_HAS_FEATURE(pvr_dev, compute_overlap)) {
    		/* Driver must not use the 2 reserved lines. */
    		available_coeffs_in_dwords -= ROGUE_CSRM_LINE_SIZE_IN_DWORDS * 2;
    	}
    
    	/*
    	 * The maximum amount of local memory available to a kernel is the minimum
    	 * of the total number of coefficient registers available and the max common
    	 * store allocation size which can be made by the CDM.
    	 *
    	 * If any coeff lines are reserved for tessellation or pixel then we need to
    	 * subtract those too.
    	 */
    	return min(available_coeffs_in_dwords, (u32)ROGUE_MAX_PER_KERNEL_LOCAL_MEM_SIZE_REGS);
    }
    
    /**
     * pvr_dev_query_gpu_info_get()
     * @pvr_dev: Device pointer.
     * @args: [IN] Device query arguments containing a pointer to a userspace
     *        struct drm_pvr_dev_query_gpu_info.
     *
     * If the query object pointer is NULL, the size field is updated with the
     * expected size of the query object.
     *
     * Returns:
     *  * 0 on success, or if size is requested using a NULL pointer, or
     *  * -%E2BIG if the indicated length of the allocation is less than is
     *    required to contain the copied data, or
     *  * -%EFAULT if local memory could not be copied to userspace.
     */
    static int
    pvr_dev_query_gpu_info_get(struct pvr_device *pvr_dev,
    			   struct drm_pvr_ioctl_dev_query_args *args)
    {
    	struct drm_pvr_dev_query_gpu_info gpu_info = {0};
    	int err;
    
    	if (!args->pointer) {
    		args->size = sizeof(struct drm_pvr_dev_query_gpu_info);
    		return 0;
    	}
    
    	gpu_info.gpu_id =
    		pvr_gpu_id_to_packed_bvnc(&pvr_dev->gpu_id);
    	gpu_info.num_phantoms = rogue_get_num_phantoms(pvr_dev);
    
    	err = PVR_UOBJ_SET(args->pointer, args->size, gpu_info);
    	if (err < 0)
    		return err;
    
    	if (args->size > sizeof(gpu_info))
    		args->size = sizeof(gpu_info);
    	return 0;
    }
    
    /**
     * pvr_dev_query_runtime_info_get()
     * @pvr_dev: Device pointer.
     * @args: [IN] Device query arguments containing a pointer to a userspace
     *        struct drm_pvr_dev_query_runtime_info.
     *
     * If the query object pointer is NULL, the size field is updated with the
     * expected size of the query object.
     *
     * Returns:
     *  * 0 on success, or if size is requested using a NULL pointer, or
     *  * -%E2BIG if the indicated length of the allocation is less than is
     *    required to contain the copied data, or
     *  * -%EFAULT if local memory could not be copied to userspace.
     */
    static int
    pvr_dev_query_runtime_info_get(struct pvr_device *pvr_dev,
    			       struct drm_pvr_ioctl_dev_query_args *args)
    {
    	struct drm_pvr_dev_query_runtime_info runtime_info = {0};
    	int err;
    
    	if (!args->pointer) {
    		args->size = sizeof(struct drm_pvr_dev_query_runtime_info);
    		return 0;
    	}
    
    	runtime_info.free_list_min_pages = 0; /* FIXME */
    	runtime_info.free_list_max_pages =
    		ROGUE_PM_MAX_FREELIST_SIZE / ROGUE_PM_PAGE_SIZE;
    	runtime_info.common_store_alloc_region_size =
    		rogue_get_common_store_alloc_region_size(pvr_dev);
    	runtime_info.common_store_partition_space_size =
    		rogue_get_common_store_partition_space_size(pvr_dev);
    	runtime_info.max_coeffs = rogue_get_max_coeffs(pvr_dev);
    	runtime_info.cdm_max_local_mem_size_regs =
    		rogue_get_cdm_max_local_mem_size_regs(pvr_dev);
    
    	err = PVR_UOBJ_SET(args->pointer, args->size, runtime_info);
    	if (err < 0)
    		return err;
    
    	if (args->size > sizeof(runtime_info))
    		args->size = sizeof(runtime_info);
    	return 0;
    }
    
    /**
     * pvr_dev_query_quirks_get() - Unpack array of quirks at the address given
     * in a struct drm_pvr_dev_query_quirks, or gets the amount of space required
     * for it.
     * @pvr_dev: Device pointer.
     * @args: [IN] Device query arguments containing a pointer to a userspace
     *        struct drm_pvr_dev_query_query_quirks.
     *
     * If the query object pointer is NULL, the size field is updated with the
     * expected size of the query object.
     * If the userspace pointer in the query object is NULL, or the count is
     * short, no data is copied.
     * The count field will be updated to that copied, or if either pointer is
     * NULL, that which would have been copied.
     * The size field in the query object will be updated to the size copied.
     *
     * Returns:
     *  * 0 on success, or if size/count is requested using a NULL pointer, or
     *  * -%EINVAL if args contained non-zero reserved fields, or
     *  * -%E2BIG if the indicated length of the allocation is less than is
     *    required to contain the copied data, or
     *  * -%EFAULT if local memory could not be copied to userspace.
     */
    static int
    pvr_dev_query_quirks_get(struct pvr_device *pvr_dev,
    			 struct drm_pvr_ioctl_dev_query_args *args)
    {
    	/*
    	 * @FIXME - hardcoding of numbers here is intended as an
    	 * intermediate step so the UAPI can be fixed, but requires a
    	 * a refactor in the future to store them in a more appropriate
    	 * location
    	 */
    	static const u32 umd_quirks_musthave[] = {
    		47217,
    		49927,
    		62269,
    	};
    	static const u32 umd_quirks[] = {
    		48545,
    		51764,
    	};
    	struct drm_pvr_dev_query_quirks query;
    	u32 out[ARRAY_SIZE(umd_quirks_musthave) + ARRAY_SIZE(umd_quirks)];
    	size_t out_musthave_count = 0;
    	size_t out_count = 0;
    	int err;
    
    	if (!args->pointer) {
    		args->size = sizeof(struct drm_pvr_dev_query_quirks);
    		return 0;
    	}
    
    	err = PVR_UOBJ_GET(query, args->size, args->pointer);
    
    	if (err < 0)
    		return err;
    	if (query._padding_c)
    		return -EINVAL;
    
    	for (int i = 0; i < ARRAY_SIZE(umd_quirks_musthave); i++) {
    		if (pvr_device_has_uapi_quirk(pvr_dev, umd_quirks_musthave[i])) {
    			out[out_count++] = umd_quirks_musthave[i];
    			out_musthave_count++;
    		}
    	}
    
    	for (int i = 0; i < ARRAY_SIZE(umd_quirks); i++) {
    		if (pvr_device_has_uapi_quirk(pvr_dev, umd_quirks[i]))
    			out[out_count++] = umd_quirks[i];
    	}
    
    	if (!query.quirks)
    		goto copy_out;
    	if (query.count < out_count)
    		return -E2BIG;
    
    	if (copy_to_user(u64_to_user_ptr(query.quirks), out,
    			 out_count * sizeof(u32))) {
    		return -EFAULT;
    	}
    
    	query.musthave_count = out_musthave_count;
    
    copy_out:
    	query.count = out_count;
    	err = PVR_UOBJ_SET(args->pointer, args->size, query);
    	if (err < 0)
    		return err;
    
    	args->size = sizeof(query);
    	return 0;
    }
    
    /**
     * pvr_dev_query_enhancements_get() - Unpack array of enhancements at the
     * address given in a struct drm_pvr_dev_query_enhancements, or gets the amount
     * of space required for it.
     * @pvr_dev: Device pointer.
     * @args: [IN] Device query arguments containing a pointer to a userspace
     *        struct drm_pvr_dev_query_enhancements.
     *
     * If the query object pointer is NULL, the size field is updated with the
     * expected size of the query object.
     * If the userspace pointer in the query object is NULL, or the count is
     * short, no data is copied.
     * The count field will be updated to that copied, or if either pointer is
     * NULL, that which would have been copied.
     * The size field in the query object will be updated to the size copied.
     *
     * Returns:
     *  * 0 on success, or if size/count is requested using a NULL pointer, or
     *  * -%EINVAL if args contained non-zero reserved fields, or
     *  * -%E2BIG if the indicated length of the allocation is less than is
     *    required to contain the copied data, or
     *  * -%EFAULT if local memory could not be copied to userspace.
     */
    static int
    pvr_dev_query_enhancements_get(struct pvr_device *pvr_dev,
    			       struct drm_pvr_ioctl_dev_query_args *args)
    {
    	/*
    	 * @FIXME - hardcoding of numbers here is intended as an
    	 * intermediate step so the UAPI can be fixed, but requires a
    	 * a refactor in the future to store them in a more appropriate
    	 * location
    	 */
    	const u32 umd_enhancements[] = {
    		35421,
    		42064,
    	};
    	struct drm_pvr_dev_query_enhancements query;
    	u32 out[ARRAY_SIZE(umd_enhancements)];
    	size_t out_idx = 0;
    	int err;
    
    	if (!args->pointer) {
    		args->size = sizeof(struct drm_pvr_dev_query_enhancements);
    		return 0;
    	}
    
    	err = PVR_UOBJ_GET(query, args->size, args->pointer);
    
    	if (err < 0)
    		return err;
    	if (query._padding_a)
    		return -EINVAL;
    	if (query._padding_c)
    		return -EINVAL;
    
    	for (int i = 0; i < ARRAY_SIZE(umd_enhancements); i++) {
    		if (pvr_device_has_uapi_enhancement(pvr_dev, umd_enhancements[i]))
    			out[out_idx++] = umd_enhancements[i];
    	}
    
    	if (!query.enhancements)
    		goto copy_out;
    	if (query.count < out_idx)
    		return -E2BIG;
    
    	if (copy_to_user(u64_to_user_ptr(query.enhancements), out,
    			 out_idx * sizeof(u32))) {
    		return -EFAULT;
    	}
    
    copy_out:
    	query.count = out_idx;
    	err = PVR_UOBJ_SET(args->pointer, args->size, query);
    	if (err < 0)
    		return err;
    
    	args->size = sizeof(query);
    	return 0;
    }
    
    /**
     * pvr_ioctl_dev_query() - IOCTL to copy information about a device
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_dev_query_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_DEV_QUERY.
     * If the given receiving struct pointer is NULL, or the indicated size is too
     * small, the expected size of the struct type will be returned in the size
     * argument field.
     *
     * Return:
     *  * 0 on success or when fetching the size with args->pointer == NULL, or
     *  * -%E2BIG if the indicated size of the receiving struct is less than is
     *    required to contain the copied data, or
     *  * -%EINVAL if the indicated struct type is unknown, or
     *  * -%ENOMEM if local memory could not be allocated, or
     *  * -%EFAULT if local memory could not be copied to userspace.
     */
    static int
    pvr_ioctl_dev_query(struct drm_device *drm_dev, void *raw_args,
    		    struct drm_file *file)
    {
    	struct pvr_device *pvr_dev = to_pvr_device(drm_dev);
    	struct drm_pvr_ioctl_dev_query_args *args = raw_args;
    	int idx;
    	int ret = -EINVAL;
    
    	if (!drm_dev_enter(drm_dev, &idx))
    		return -EIO;
    
    	switch ((enum drm_pvr_dev_query)args->type) {
    	case DRM_PVR_DEV_QUERY_GPU_INFO_GET:
    		ret = pvr_dev_query_gpu_info_get(pvr_dev, args);
    		break;
    
    	case DRM_PVR_DEV_QUERY_RUNTIME_INFO_GET:
    		ret = pvr_dev_query_runtime_info_get(pvr_dev, args);
    		break;
    
    	case DRM_PVR_DEV_QUERY_QUIRKS_GET:
    		ret = pvr_dev_query_quirks_get(pvr_dev, args);
    		break;
    
    	case DRM_PVR_DEV_QUERY_ENHANCEMENTS_GET:
    		ret = pvr_dev_query_enhancements_get(pvr_dev, args);
    		break;
    
    	case DRM_PVR_DEV_QUERY_HEAP_INFO_GET:
    		return -EINVAL;
    
    	case DRM_PVR_DEV_QUERY_STATIC_DATA_AREAS_GET:
    		return -EINVAL;
    	}
    
    	drm_dev_exit(idx);
    
    	return ret;
    }
    
    /**
     * pvr_ioctl_create_context() - IOCTL to create a context
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_create_context_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_CREATE_CONTEXT.
     *
     * Return:
     *  * 0 on success, or
     *  * -%EINVAL if provided arguments are invalid, or
     *  * -%EFAULT if arguments can't be copied from userspace, or
     *  * Any error returned by pvr_create_render_context().
     */
    static int
    pvr_ioctl_create_context(struct drm_device *drm_dev, void *raw_args,
    			 struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_destroy_context() - IOCTL to destroy a context
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_destroy_context_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_DESTROY_CONTEXT.
     *
     * Return:
     *  * 0 on success, or
     *  * -%EINVAL if context not in context list.
     */
    static int
    pvr_ioctl_destroy_context(struct drm_device *drm_dev, void *raw_args,
    			  struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_create_free_list() - IOCTL to create a free list
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_create_free_list_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_CREATE_FREE_LIST.
     *
     * Return:
     *  * 0 on success, or
     *  * Any error returned by pvr_free_list_create().
     */
    static int
    pvr_ioctl_create_free_list(struct drm_device *drm_dev, void *raw_args,
    			   struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_destroy_free_list() - IOCTL to destroy a free list
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN] Arguments passed to this IOCTL. This must be of type
     *                 &struct drm_pvr_ioctl_destroy_free_list_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_DESTROY_FREE_LIST.
     *
     * Return:
     *  * 0 on success, or
     *  * -%EINVAL if free list not in object list.
     */
    static int
    pvr_ioctl_destroy_free_list(struct drm_device *drm_dev, void *raw_args,
    			    struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_create_hwrt_dataset() - IOCTL to create a HWRT dataset
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_create_hwrt_dataset_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_CREATE_HWRT_DATASET.
     *
     * Return:
     *  * 0 on success, or
     *  * Any error returned by pvr_hwrt_dataset_create().
     */
    static int
    pvr_ioctl_create_hwrt_dataset(struct drm_device *drm_dev, void *raw_args,
    			      struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_destroy_hwrt_dataset() - IOCTL to destroy a HWRT dataset
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN] Arguments passed to this IOCTL. This must be of type
     *                 &struct drm_pvr_ioctl_destroy_hwrt_dataset_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_DESTROY_HWRT_DATASET.
     *
     * Return:
     *  * 0 on success, or
     *  * -%EINVAL if HWRT dataset not in object list.
     */
    static int
    pvr_ioctl_destroy_hwrt_dataset(struct drm_device *drm_dev, void *raw_args,
    			       struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_create_vm_context() - IOCTL to create a VM context
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN/OUT] Arguments passed to this IOCTL. This must be of type
     *                     &struct drm_pvr_ioctl_create_vm_context_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_CREATE_VM_CONTEXT.
     *
     * Return:
     *  * 0 on success, or
     *  * Any error returned by pvr_vm_create_context().
     */
    static int
    pvr_ioctl_create_vm_context(struct drm_device *drm_dev, void *raw_args,
    			    struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_destroy_vm_context() - IOCTL to destroy a VM context
    * @drm_dev: [IN] DRM device.
    * @raw_args: [IN] Arguments passed to this IOCTL. This must be of type
    *                 &struct drm_pvr_ioctl_destroy_vm_context_args.
    * @file: [IN] DRM file private data.
    *
    * Called from userspace with %DRM_IOCTL_PVR_DESTROY_VM_CONTEXT.
    *
    * Return:
    *  * 0 on success, or
    *  * -%EINVAL if object not in object list.
     */
    static int
    pvr_ioctl_destroy_vm_context(struct drm_device *drm_dev, void *raw_args,
    			     struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_vm_map() - IOCTL to map buffer to GPU address space.
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN] Arguments passed to this IOCTL. This must be of type
     *                 &struct drm_pvr_ioctl_vm_map_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_VM_MAP.
     *
     * Return:
     *  * 0 on success,
     *  * -%EINVAL if &drm_pvr_ioctl_vm_op_map_args.flags is not zero,
     *  * -%EINVAL if the bounds specified by &drm_pvr_ioctl_vm_op_map_args.offset
     *    and &drm_pvr_ioctl_vm_op_map_args.size are not valid or do not fall
     *    within the buffer object specified by
     *    &drm_pvr_ioctl_vm_op_map_args.handle,
     *  * -%EINVAL if the bounds specified by
     *    &drm_pvr_ioctl_vm_op_map_args.device_addr and
     *    &drm_pvr_ioctl_vm_op_map_args.size do not form a valid device-virtual
     *    address range which falls entirely within a single heap, or
     *  * -%ENOENT if &drm_pvr_ioctl_vm_op_map_args.handle does not refer to a
     *    valid PowerVR buffer object.
     */
    static int
    pvr_ioctl_vm_map(struct drm_device *drm_dev, void *raw_args,
    		 struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /**
     * pvr_ioctl_vm_unmap() - IOCTL to unmap buffer from GPU address space.
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN] Arguments passed to this IOCTL. This must be of type
     *                 &struct drm_pvr_ioctl_vm_unmap_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_VM_UNMAP.
     *
     * Return:
     *  * 0 on success,
     *  * -%EINVAL if &drm_pvr_ioctl_vm_op_unmap_args.device_addr is not a valid
     *    device page-aligned device-virtual address, or
     *  * -%ENOENT if there is currently no PowerVR buffer object mapped at
     *    &drm_pvr_ioctl_vm_op_unmap_args.device_addr.
     */
    static int
    pvr_ioctl_vm_unmap(struct drm_device *drm_dev, void *raw_args,
    		   struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    /*
     * pvr_ioctl_submit_job() - IOCTL to submit a job to the GPU
     * @drm_dev: [IN] DRM device.
     * @raw_args: [IN] Arguments passed to this IOCTL. This must be of type
     *                 &struct drm_pvr_ioctl_submit_job_args.
     * @file: [IN] DRM file private data.
     *
     * Called from userspace with %DRM_IOCTL_PVR_SUBMIT_JOB.
     *
     * Return:
     *  * 0 on success, or
     *  * -%EINVAL if arguments are invalid.
     */
    static int
    pvr_ioctl_submit_jobs(struct drm_device *drm_dev, void *raw_args,
    		      struct drm_file *file)
    {
    	return -ENOTTY;
    }
    
    int
    pvr_get_uobj(u64 usr_ptr, u32 usr_stride, u32 min_stride, u32 obj_size, void *out)
    {
    	if (usr_stride < min_stride)
    		return -EINVAL;
    
    	return copy_struct_from_user(out, obj_size, u64_to_user_ptr(usr_ptr), usr_stride);
    }
    
    int
    pvr_set_uobj(u64 usr_ptr, u32 usr_stride, u32 min_stride, u32 obj_size, const void *in)
    {
    	if (usr_stride < min_stride)
    		return -EINVAL;
    
    	if (copy_to_user(u64_to_user_ptr(usr_ptr), in, min_t(u32, usr_stride, obj_size)))
    		return -EFAULT;
    
    	if (usr_stride > obj_size &&
    	    clear_user(u64_to_user_ptr(usr_ptr + obj_size), usr_stride - obj_size)) {
    		return -EFAULT;
    	}
    
    	return 0;
    }
    
    int
    pvr_get_uobj_array(const struct drm_pvr_obj_array *in, u32 min_stride, u32 obj_size, void **out)
    {
    	int ret = 0;
    	void *out_alloc;
    
    	if (in->stride < min_stride)
    		return -EINVAL;
    
    	if (!in->count)
    		return 0;
    
    	out_alloc = kvmalloc_array(in->count, obj_size, GFP_KERNEL);
    	if (!out_alloc)
    		return -ENOMEM;
    
    	if (obj_size == in->stride) {
    		if (copy_from_user(out_alloc, u64_to_user_ptr(in->array),
    				   (unsigned long)obj_size * in->count))
    			ret = -EFAULT;
    	} else {
    		void __user *in_ptr = u64_to_user_ptr(in->array);
    		void *out_ptr = out_alloc;
    
    		for (u32 i = 0; i < in->count; i++) {
    			ret = copy_struct_from_user(out_ptr, obj_size, in_ptr, in->stride);
    			if (ret)
    				break;
    
    			out_ptr += obj_size;
    			in_ptr += in->stride;
    		}
    	}
    
    	if (ret) {
    		kvfree(out_alloc);
    		return ret;
    	}
    
    	*out = out_alloc;
    	return 0;
    }
    
    int
    pvr_set_uobj_array(const struct drm_pvr_obj_array *out, u32 min_stride, u32 obj_size,
    		   const void *in)
    {
    	if (out->stride < min_stride)
    		return -EINVAL;
    
    	if (!out->count)
    		return 0;
    
    	if (obj_size == out->stride) {
    		if (copy_to_user(u64_to_user_ptr(out->array), in,
    				 (unsigned long)obj_size * out->count))
    			return -EFAULT;
    	} else {
    		u32 cpy_elem_size = min_t(u32, out->stride, obj_size);
    		void __user *out_ptr = u64_to_user_ptr(out->array);
    		const void *in_ptr = in;
    
    		for (u32 i = 0; i < out->count; i++) {
    			if (copy_to_user(out_ptr, in_ptr, cpy_elem_size))
    				return -EFAULT;
    
    			out_ptr += obj_size;
    			in_ptr += out->stride;
    		}
    
    		if (out->stride > obj_size &&
    		    clear_user(u64_to_user_ptr(out->array + obj_size),
    			       out->stride - obj_size)) {
    			return -EFAULT;
    		}
    	}
    
    	return 0;
    }
    
    #define DRM_PVR_IOCTL(_name, _func, _flags) \
    	DRM_IOCTL_DEF_DRV(PVR_##_name, pvr_ioctl_##_func, _flags)
    
    /* clang-format off */
    
    static const struct drm_ioctl_desc pvr_drm_driver_ioctls[] = {
    	DRM_PVR_IOCTL(DEV_QUERY, dev_query, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(GET_BO_MMAP_OFFSET, get_bo_mmap_offset, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(CREATE_VM_CONTEXT, create_vm_context, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(DESTROY_VM_CONTEXT, destroy_vm_context, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(VM_MAP, vm_map, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(VM_UNMAP, vm_unmap, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(CREATE_CONTEXT, create_context, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(DESTROY_CONTEXT, destroy_context, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(CREATE_FREE_LIST, create_free_list, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(DESTROY_FREE_LIST, destroy_free_list, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(CREATE_HWRT_DATASET, create_hwrt_dataset, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(DESTROY_HWRT_DATASET, destroy_hwrt_dataset, DRM_RENDER_ALLOW),
    	DRM_PVR_IOCTL(SUBMIT_JOBS, submit_jobs, DRM_RENDER_ALLOW),
    };
    
    /* clang-format on */
    
    #undef DRM_PVR_IOCTL
    
    /**
     * pvr_drm_driver_open() - Driver callback when a new &struct drm_file is opened
     * @drm_dev: [IN] DRM device.
     * @file: [IN] DRM file private data.
     *
     * Allocates powervr-specific file private data (&struct pvr_file).
     *
     * Registered in &pvr_drm_driver.
     *
     * Return:
     *  * 0 on success,
     *  * -%ENOMEM if the allocation of a &struct ipvr_file fails, or
     *  * Any error returned by pvr_memory_context_init().
     */
    static int
    pvr_drm_driver_open(struct drm_device *drm_dev, struct drm_file *file)
    {
    	struct pvr_device *pvr_dev = to_pvr_device(drm_dev);
    	struct pvr_file *pvr_file;
    
    	pvr_file = kzalloc(sizeof(*pvr_file), GFP_KERNEL);
    	if (!pvr_file)
    		return -ENOMEM;
    
    	/*
    	 * Store reference to base DRM file private data for use by
    	 * from_pvr_file.
    	 */
    	pvr_file->file = file;
    
    	/*
    	 * Store reference to powervr-specific outer device struct in file
    	 * private data for convenient access.
    	 */
    	pvr_file->pvr_dev = pvr_dev;
    
    	/*
    	 * Store reference to powervr-specific file private data in DRM file
    	 * private data.
    	 */
    	file->driver_priv = pvr_file;
    
    	return 0;
    }
    
    /**
     * pvr_drm_driver_postclose() - One of the driver callbacks when a &struct
     * drm_file is closed.
     * @drm_dev: [IN] DRM device (unused).
     * @file: [IN] DRM file private data.
     *
     * Frees powervr-specific file private data (&struct pvr_file).
     *
     * Registered in &pvr_drm_driver.
     */
    static void
    pvr_drm_driver_postclose(__always_unused struct drm_device *drm_dev,
    			 struct drm_file *file)
    {
    	struct pvr_file *pvr_file = to_pvr_file(file);
    
    	kfree(pvr_file);
    	file->driver_priv = NULL;
    }
    
    DEFINE_DRM_GEM_FOPS(pvr_drm_driver_fops);
    
    static struct drm_driver pvr_drm_driver = {
    	.driver_features = DRIVER_RENDER,
    	.open = pvr_drm_driver_open,
    	.postclose = pvr_drm_driver_postclose,
    	.ioctls = pvr_drm_driver_ioctls,
    	.num_ioctls = ARRAY_SIZE(pvr_drm_driver_ioctls),
    	.fops = &pvr_drm_driver_fops,
    
    	.name = PVR_DRIVER_NAME,
    	.desc = PVR_DRIVER_DESC,
    	.date = PVR_DRIVER_DATE,
    	.major = PVR_DRIVER_MAJOR,
    	.minor = PVR_DRIVER_MINOR,
    	.patchlevel = PVR_DRIVER_PATCHLEVEL,
    
    };
    
    static int
    pvr_probe(struct platform_device *plat_dev)
    {
    	struct pvr_device *pvr_dev;
    	struct drm_device *drm_dev;
    	int err;
    
    	pvr_dev = devm_drm_dev_alloc(&plat_dev->dev, &pvr_drm_driver,
    				     struct pvr_device, base);
    	if (IS_ERR(pvr_dev))
    		return PTR_ERR(pvr_dev);
    
    	drm_dev = &pvr_dev->base;
    
    	platform_set_drvdata(plat_dev, drm_dev);
    
    	err = pvr_device_init(pvr_dev);
    	if (err)
    		return err;
    
    	err = drm_dev_register(drm_dev, 0);
    	if (err)
    		goto err_device_fini;
    
    	return 0;
    
    err_device_fini:
    	pvr_device_fini(pvr_dev);
    
    	return err;
    }
    
    static int
    pvr_remove(struct platform_device *plat_dev)
    {
    	struct drm_device *drm_dev = platform_get_drvdata(plat_dev);
    	struct pvr_device *pvr_dev = to_pvr_device(drm_dev);
    
    	pvr_device_fini(pvr_dev);
    	drm_dev_unplug(drm_dev);
    
    	return 0;
    }
    
    static const struct of_device_id dt_match[] = {
    	{ .compatible = "img,img-axe", .data = NULL },
    	{}
    };
    MODULE_DEVICE_TABLE(of, dt_match);
    
    static struct platform_driver pvr_driver = {
    	.probe = pvr_probe,
    	.remove = pvr_remove,
    	.driver = {
    		.name = PVR_DRIVER_NAME,
    		.of_match_table = dt_match,
    	},
    };
    module_platform_driver(pvr_driver);
    
    MODULE_AUTHOR("Imagination Technologies Ltd.");
    MODULE_DESCRIPTION(PVR_DRIVER_DESC);
    MODULE_LICENSE("Dual MIT/GPL");
    MODULE_IMPORT_NS(DMA_BUF);