Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:a_faerber:machinelearning
degirum
lincdadrv_whitespace.diff
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File lincdadrv_whitespace.diff of Package degirum
diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a9f06af --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text=auto + +*.c text +*.h text +Makefile text +*.sh text diff --git a/src/cdadrv.c b/src/cdadrv.c index 7e31633..e4c7b58 100644 --- a/src/cdadrv.c +++ b/src/cdadrv.c @@ -1,405 +1,410 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. -// -// CDA linux driver mem blocks/mem maps and interrupt request handler -// -// This program is free software; you can redistribute it and/or modify it -// under the terms and conditions of the GNU General Public License, -// version 2, as published by the Free Software Foundation. -// -#include <linux/module.h> -#include <linux/errno.h> -#include <linux/init.h> -#include <linux/fs.h> - -#include "cdadrv.h" -#include "cdaioctl.h" - -MODULE_AUTHOR("DeGirum Corp., Egor Pomozov"); -MODULE_DESCRIPTION("CDA linux driver to access pci devices"); -MODULE_LICENSE("GPL"); -MODULE_VERSION("0.5.0.3"); -// The version has to be in the format n.n.n.n, where each n is a single digit - -#if LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0) -#error Too old kernel -#endif - -static dev_t cdadev_first; -static const char cda_name[] = "cda"; -static int req_pci_did = 0; -static int req_pci_vid = 0; -static int test_probe = 0; - -#define CDA_DEV_MINOR_MAX 32 -static DEFINE_SPINLOCK(cdadevlist_sl); -static DEFINE_IDA(cdaminor_ida); -static LIST_HEAD(cdadevs); - -// Module parameters -module_param_named(did, req_pci_did, int, 0644); -MODULE_PARM_DESC(did, "Set required PCI device ID"); -module_param_named(vid, req_pci_vid, int, 0644); -MODULE_PARM_DESC(vid, "Set required PCI vendor ID"); -module_param_named(test_probe, test_probe, int, 0644); -MODULE_PARM_DESC(test_probe, "Check permissions to load driver"); - -static void cdadev_release(struct device *kdev); -static void cda_pci_remove(struct pci_dev *pcidev); -static int cda_pci_probe(struct pci_dev *pcidev, - const struct pci_device_id *id); - -static int cda_cdev_open(struct inode *ino, struct file *file); -static int cda_cdev_release(struct inode *ino, struct file *file); -static long cda_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg); - -static struct pci_device_id cda_pci_ids[] = { - { PCI_DEVICE(0x1f0d, 0x0100) }, - { PCI_DEVICE(0x1f0d, 0x8101) }, - { PCI_DEVICE(0x1f0d, 0x0101) }, - { PCI_DEVICE(0x10ee, 0x8011) }, - { PCI_DEVICE(0, 0) }, - { PCI_DEVICE(0, 0) }, -}; - -static struct pci_driver cda_pci = { - .name = cda_name, - .probe = cda_pci_probe, - .remove = cda_pci_remove, - .id_table = cda_pci_ids, -}; - -static struct file_operations cda_fileops = { - .owner = THIS_MODULE, - .open = cda_cdev_open, - .release = cda_cdev_release, - .unlocked_ioctl = cda_cdev_ioctl, - -}; - -static struct class cda_class = { - .name = cda_name, - .dev_release = cdadev_release, -}; -/* -static inline bool cda_kernel_is_locked_down(void) -{ -#ifdef CONFIG_LOCK_DOWN_KERNEL -#ifdef CONFIG_LOCK_DOWN_IN_EFI_SECURE_BOOT / * fedora * / - return kernel_is_locked_down(NULL); -#elif CONFIG_EFI_SECURE_BOOT_LOCK_DOWN / * ubuntu * / - return kernel_is_locked_down(); -#else - return false; -#endif -#else - return false; -#endif -} -*/ -static void cdadev_free(struct cda_dev *cdadev) -{ - ida_simple_remove(&cdaminor_ida, cdadev->minor); - device_del(&cdadev->dev); - put_device(&cdadev->dev); -} - -static int cdadev_init(struct cda_dev *cdadev) -{ - // Create and initialize device structures - int ret = -ENOMEM; - struct device *dev = &cdadev->dev; - device_initialize(dev); - - dev->class = &cda_class; - dev->parent = &cdadev->pcidev->dev; - - cdadev->dummy_blk = kzalloc(sizeof(*cdadev->dummy_blk), in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - if (!cdadev->dummy_blk) { - dev_err(&cdadev->pcidev->dev, "Can't alloc dummy blk\n"); - goto alloc_dummy; - } - idr_init(&cdadev->mblk_idr); - ret = ida_simple_get(&cdaminor_ida, 0, CDA_DEV_MINOR_MAX, GFP_KERNEL); - if( ret < 0 ) - goto err_minor_get; - - cdadev->minor = ret; - dev->devt = MKDEV(MAJOR(cdadev_first), cdadev->minor); - ret = dev_set_name(dev, "cda%02d", cdadev->minor); - if (ret) - goto err_set_name; - - ret = device_add(dev); - if (ret) { - dev_err(&cdadev->pcidev->dev, "Unable to create device. Error 0x%x\n", ret); - goto err_device_add; - } - - INIT_LIST_HEAD(&cdadev->mem_maps); - INIT_LIST_HEAD(&cdadev->mem_blocks); - spin_lock_init(&cdadev->mblk_sl); - - mutex_init(&cdadev->ilock); - cdadev->ints = NULL; - - return 0; -err_device_add: -err_set_name: - ida_simple_remove(&cdaminor_ida, cdadev->minor); -err_minor_get: -alloc_dummy: - put_device(dev); - return ret; -} - -static int cda_pci_init(struct pci_dev *pcidev) -{ - // PCI initialization - int ret; - ret = pci_enable_device_mem(pcidev); - if( ret ) { - dev_err(&pcidev->dev, "Cannot enable PCI device mem\n"); - goto err_en_devmem; - } - - if( dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(64)) && - (ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(32))) ) { - dev_err(&pcidev->dev, "Set DMA mask 32/64 failed: 0x%x\n", ret); - goto err_dma_set_mask; - } - - ret = pci_request_regions(pcidev, cda_name); - if( ret ) { - dev_err(&pcidev->dev, "Fail request regions: 0x%x\n", ret); - goto err_req_regions; - } - - pci_set_master(pcidev); - return 0; -err_req_regions: -err_dma_set_mask: - pci_disable_device(pcidev); -err_en_devmem: - return ret; -} - -static int cda_cdev_init(struct cda_dev *cdadev) -{ - int ret; - struct cdev *cdev = &cdadev->cdev; - - cdev_init(cdev, &cda_fileops); - cdev->owner = THIS_MODULE; - kobject_set_name(&cdev->kobj, "%s%d", cda_name, cdadev->minor); - ret = cdev_add(cdev, MKDEV(MAJOR(cdadev_first), cdadev->minor), CDA_DEV_MINOR_MAX); - if (ret) - return ret; - return 0; -} - -static int cda_pci_probe(struct pci_dev *pcidev, - const struct pci_device_id *id) -{ - int ret; - struct cda_dev *cdadev = kzalloc(sizeof(*cdadev), in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - if (!cdadev) { - return -ENOMEM; - } - - cdadev->pcidev = pcidev; - ret = cdadev_init(cdadev); - if( ret ) - goto err_cdadev_init; - - ret = cda_pci_init(pcidev); - if( ret ) - goto err_pci_init; - - ret = cda_mems_create(cdadev); - if( ret ) - goto err_sysfsmem; - - ret = cda_open_bars(cdadev); - if( ret ) - goto err_check_bar; - - ret = cda_cdev_init(cdadev); - if( ret ) - goto err_cdev_init; - - spin_lock(&cdadevlist_sl); - list_add(&cdadev->devices, &cdadevs); - spin_unlock(&cdadevlist_sl); - - pci_set_drvdata(pcidev, cdadev); - return 0; -err_cdev_init: - cda_release_bars(cdadev); -err_check_bar: - cda_mems_release(cdadev); -err_sysfsmem: - pci_release_regions(pcidev); - pci_disable_device(pcidev); -err_pci_init: - cdadev_free(cdadev); -err_cdadev_init: - return ret; -} - -static void cda_pci_remove(struct pci_dev *pcidev) -{ - struct cda_dev *cdadev = pci_get_drvdata(pcidev); - - if (!cdadev) - return; - - spin_lock(&cdadevlist_sl); - list_del(&cdadev->devices); - spin_unlock(&cdadevlist_sl); - - cdev_del(&cdadev->cdev); - cda_release_bars(cdadev); - - cda_mems_release(cdadev); - - cda_free_irqs(cdadev, NULL); - cda_unmmap_dev_mem(cdadev, NULL); - cda_free_dev_mem(cdadev, NULL); - - pci_release_regions(pcidev); - pci_disable_device(pcidev); - - cdadev_free(cdadev); -} - -static int cda_cdev_open(struct inode *ino, struct file *file) -{ - int ret; - struct cda_dev *cdadev = - container_of(ino->i_cdev, - struct cda_dev, - cdev); - if (!cdadev) { - ret = -ENODEV; - goto out; - } - get_device(&cdadev->dev); - file->private_data = cdadev; - return nonseekable_open(ino, file); -out: - return ret; -} - -static int cda_cdev_release(struct inode *ino, struct file *file) -{ - struct cda_dev *cdadev = file->private_data; - if (!cdadev) - return -ENODEV; - - cda_cancel_req(cdadev, (void *)file); - cda_free_irqs(cdadev, (void *)file); - cda_unmmap_dev_mem(cdadev, (void *)file); - cda_free_dev_mem(cdadev, (void *)file); - cda_sem_rel_by_owner(cdadev, (void *)file); - - put_device(&cdadev->dev); - return 0; -} - -static long cda_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) -{ - struct cda_dev *cdadev = file->private_data; - switch (cmd) { - - case CDA_ALLOC_MEM: - return cda_alloc_mem(cdadev, (void *)file, (void __user *)arg); - - case CDA_FREE_MEM: - return cda_free_mem_by_idx(cdadev, (void *)file, (void __user *)arg); - - case CDA_MAP_MEM: - return cda_map_mem(cdadev, (void *)file, (void __user *)arg); - - case CDA_UNMAP_MEM: - return cda_unmap_mem_by_idx(cdadev, (void *)file, (void __user *)arg); - - case CDA_INIT_INT: - return cda_init_interrupts(cdadev, (void *)file, (void __user *)arg); - - case CDA_FREE_INT: - return cda_free_irqs(cdadev, (void *)file); - - case CDA_REQ_INT: - return cda_req_int(cdadev, (void *)file, (void __user *) arg); - - case CDA_INT_CANCEL: - return cda_cancel_req(cdadev, (void *)file); - - case CDA_SEM_AQ: - return cda_sem_aq(cdadev, (void *)file, (void __user *) arg); - - case CDA_SEM_REL: - return cda_sem_rel(cdadev, (void *)file, (void __user *) arg); - - default: - return -ENOTTY; - } -} - -static void cdadev_release(struct device *dev) -{ - struct cda_dev *cdadev = container_of(dev, struct cda_dev, dev); - kfree(cdadev); -} - -static int __init cdadrv_init(void) -{ - int ret; - size_t pci_id_table_size = ARRAY_SIZE(cda_pci_ids); - if( test_probe ) { - printk("Test run. Nothing initialized\n"); - return 0; - } - - ret = alloc_chrdev_region(&cdadev_first, 0, CDA_DEV_MINOR_MAX, cda_name); - if( ret ) - goto err_alloc_cdev_reg; - - ret = class_register(&cda_class); - if( ret ) - goto err_cls_reg; - - if( (req_pci_did || req_pci_vid) && pci_id_table_size >= 2 ) { - // Last table element is 0,0 - // Update pre-last item - cda_pci_ids[pci_id_table_size-2].vendor = req_pci_vid; - cda_pci_ids[pci_id_table_size-2].device = req_pci_did; - } - ret = pci_register_driver(&cda_pci); - if( ret ) - goto err_pci_reg_drv; - - return 0; - -err_pci_reg_drv: - class_unregister(&cda_class); -err_cls_reg: - unregister_chrdev_region(cdadev_first, CDA_DEV_MINOR_MAX); -err_alloc_cdev_reg: - return ret; -} - -static void __exit dcadrv_exit(void) -{ - if( test_probe ) { - printk("Stop test run. Nothing initialized\n"); - return; - } - pci_unregister_driver(&cda_pci); - class_unregister(&cda_class); - unregister_chrdev_region(cdadev_first, CDA_DEV_MINOR_MAX); -} - -module_init(cdadrv_init); -module_exit(dcadrv_exit); +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. +// +// CDA linux driver mem blocks/mem maps and interrupt request handler +// +// This program is free software; you can redistribute it and/or modify it +// under the terms and conditions of the GNU General Public License, +// version 2, as published by the Free Software Foundation. +// +#include <linux/module.h> +#include <linux/errno.h> +#include <linux/init.h> +#include <linux/fs.h> + +#include "cdadrv.h" +#include "cdaioctl.h" + +MODULE_AUTHOR("DeGirum Corp., Egor Pomozov"); +MODULE_DESCRIPTION("CDA linux driver to access pci devices"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("0.5.0.3"); +// The version has to be in the format n.n.n.n, where each n is a single digit + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) +#error Too old kernel +#endif + +static dev_t cdadev_first; +static const char cda_name[] = "cda"; +static int req_pci_did; +static int req_pci_vid; +static int test_probe; + +#define CDA_DEV_MINOR_MAX 32 +static DEFINE_SPINLOCK(cdadevlist_sl); +static DEFINE_IDA(cdaminor_ida); +static LIST_HEAD(cdadevs); + +// Module parameters +module_param_named(did, req_pci_did, int, 0644); +MODULE_PARM_DESC(did, "Set required PCI device ID"); +module_param_named(vid, req_pci_vid, int, 0644); +MODULE_PARM_DESC(vid, "Set required PCI vendor ID"); +module_param_named(test_probe, test_probe, int, 0644); +MODULE_PARM_DESC(test_probe, "Check permissions to load driver"); + +static void cdadev_release(struct device *kdev); +static void cda_pci_remove(struct pci_dev *pcidev); +static int cda_pci_probe(struct pci_dev *pcidev, + const struct pci_device_id *id); + +static int cda_cdev_open(struct inode *ino, struct file *file); +static int cda_cdev_release(struct inode *ino, struct file *file); +static long cda_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + +static struct pci_device_id cda_pci_ids[] = { + { PCI_DEVICE(0x1f0d, 0x0100) }, + { PCI_DEVICE(0x1f0d, 0x8101) }, + { PCI_DEVICE(0x1f0d, 0x0101) }, + { PCI_DEVICE(0x10ee, 0x8011) }, + { PCI_DEVICE(0, 0) }, + { PCI_DEVICE(0, 0) }, +}; + +static struct pci_driver cda_pci = { + .name = cda_name, + .probe = cda_pci_probe, + .remove = cda_pci_remove, + .id_table = cda_pci_ids, +}; + +static const struct file_operations cda_fileops = { + .owner = THIS_MODULE, + .open = cda_cdev_open, + .release = cda_cdev_release, + .unlocked_ioctl = cda_cdev_ioctl, +}; + +static struct class cda_class = { + .name = cda_name, + .dev_release = cdadev_release, +}; +/* +static inline bool cda_kernel_is_locked_down(void) +{ +#ifdef CONFIG_LOCK_DOWN_KERNEL +#ifdef CONFIG_LOCK_DOWN_IN_EFI_SECURE_BOOT / * fedora * / + return kernel_is_locked_down(NULL); +#elif CONFIG_EFI_SECURE_BOOT_LOCK_DOWN / * ubuntu * / + return kernel_is_locked_down(); +#else + return false; +#endif +#else + return false; +#endif +} +*/ +static void cdadev_free(struct cda_dev *cdadev) +{ + ida_simple_remove(&cdaminor_ida, cdadev->minor); + device_del(&cdadev->dev); + put_device(&cdadev->dev); +} + +static int cdadev_init(struct cda_dev *cdadev) +{ + // Create and initialize device structures + int ret = -ENOMEM; + struct device *dev = &cdadev->dev; + + device_initialize(dev); + + dev->class = &cda_class; + dev->parent = &cdadev->pcidev->dev; + + cdadev->dummy_blk = kzalloc(sizeof(*cdadev->dummy_blk), GFP_KERNEL); + if (!cdadev->dummy_blk) + goto alloc_dummy; + idr_init(&cdadev->mblk_idr); + ret = ida_simple_get(&cdaminor_ida, 0, CDA_DEV_MINOR_MAX, GFP_KERNEL); + if (ret < 0) + goto err_minor_get; + + cdadev->minor = ret; + dev->devt = MKDEV(MAJOR(cdadev_first), cdadev->minor); + ret = dev_set_name(dev, "cda%02d", cdadev->minor); + if (ret) + goto err_set_name; + + ret = device_add(dev); + if (ret) { + dev_err(&cdadev->pcidev->dev, "Unable to create device. Error 0x%x\n", ret); + goto err_device_add; + } + + INIT_LIST_HEAD(&cdadev->mem_maps); + INIT_LIST_HEAD(&cdadev->mem_blocks); + spin_lock_init(&cdadev->mblk_sl); + + mutex_init(&cdadev->ilock); + cdadev->ints = NULL; + + return 0; +err_device_add: +err_set_name: + ida_simple_remove(&cdaminor_ida, cdadev->minor); +err_minor_get: +alloc_dummy: + put_device(dev); + return ret; +} + +static int cda_pci_init(struct pci_dev *pcidev) +{ + // PCI initialization + int ret; + + ret = pci_enable_device_mem(pcidev); + if (ret) { + dev_err(&pcidev->dev, "Cannot enable PCI device mem\n"); + goto err_en_devmem; + } + + ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(64)); + if (ret) { + ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(&pcidev->dev, "Set DMA mask 32/64 failed: 0x%x\n", ret); + goto err_dma_set_mask; + } + } + + ret = pci_request_regions(pcidev, cda_name); + if (ret) { + dev_err(&pcidev->dev, "Fail request regions: 0x%x\n", ret); + goto err_req_regions; + } + + pci_set_master(pcidev); + return 0; +err_req_regions: +err_dma_set_mask: + pci_disable_device(pcidev); +err_en_devmem: + return ret; +} + +static int cda_cdev_init(struct cda_dev *cdadev) +{ + int ret; + struct cdev *cdev = &cdadev->cdev; + + cdev_init(cdev, &cda_fileops); + cdev->owner = THIS_MODULE; + kobject_set_name(&cdev->kobj, "%s%d", cda_name, cdadev->minor); + ret = cdev_add(cdev, MKDEV(MAJOR(cdadev_first), cdadev->minor), CDA_DEV_MINOR_MAX); + if (ret) + return ret; + return 0; +} + +static int cda_pci_probe(struct pci_dev *pcidev, + const struct pci_device_id *id) +{ + int ret; + struct cda_dev *cdadev = kzalloc(sizeof(*cdadev), GFP_KERNEL); + + if (!cdadev) + return -ENOMEM; + + cdadev->pcidev = pcidev; + ret = cdadev_init(cdadev); + if (ret) + goto err_cdadev_init; + + ret = cda_pci_init(pcidev); + if (ret) + goto err_pci_init; + + ret = cda_mems_create(cdadev); + if (ret) + goto err_sysfsmem; + + ret = cda_open_bars(cdadev); + if (ret) + goto err_check_bar; + + ret = cda_cdev_init(cdadev); + if (ret) + goto err_cdev_init; + + spin_lock(&cdadevlist_sl); + list_add(&cdadev->devices, &cdadevs); + spin_unlock(&cdadevlist_sl); + + pci_set_drvdata(pcidev, cdadev); + return 0; +err_cdev_init: + cda_release_bars(cdadev); +err_check_bar: + cda_mems_release(cdadev); +err_sysfsmem: + pci_release_regions(pcidev); + pci_disable_device(pcidev); +err_pci_init: + cdadev_free(cdadev); +err_cdadev_init: + return ret; +} + +static void cda_pci_remove(struct pci_dev *pcidev) +{ + struct cda_dev *cdadev = pci_get_drvdata(pcidev); + + if (!cdadev) + return; + + spin_lock(&cdadevlist_sl); + list_del(&cdadev->devices); + spin_unlock(&cdadevlist_sl); + + cdev_del(&cdadev->cdev); + cda_release_bars(cdadev); + + cda_mems_release(cdadev); + + cda_free_irqs(cdadev, NULL); + cda_unmmap_dev_mem(cdadev, NULL); + cda_free_dev_mem(cdadev, NULL); + + pci_release_regions(pcidev); + pci_disable_device(pcidev); + + cdadev_free(cdadev); +} + +static int cda_cdev_open(struct inode *ino, struct file *file) +{ + int ret; + struct cda_dev *cdadev = + container_of(ino->i_cdev, + struct cda_dev, + cdev); + if (!cdadev) { + ret = -ENODEV; + goto out; + } + get_device(&cdadev->dev); + file->private_data = cdadev; + return nonseekable_open(ino, file); +out: + return ret; +} + +static int cda_cdev_release(struct inode *ino, struct file *file) +{ + struct cda_dev *cdadev = file->private_data; + + if (!cdadev) + return -ENODEV; + + cda_cancel_req(cdadev, (void *)file); + cda_free_irqs(cdadev, (void *)file); + cda_unmmap_dev_mem(cdadev, (void *)file); + cda_free_dev_mem(cdadev, (void *)file); + cda_sem_rel_by_owner(cdadev, (void *)file); + + put_device(&cdadev->dev); + return 0; +} + +static long cda_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct cda_dev *cdadev = file->private_data; + + switch (cmd) { + case CDA_ALLOC_MEM: + return cda_alloc_mem(cdadev, (void *)file, (void __user *)arg); + + case CDA_FREE_MEM: + return cda_free_mem_by_idx(cdadev, (void *)file, (void __user *)arg); + + case CDA_MAP_MEM: + return cda_map_mem(cdadev, (void *)file, (void __user *)arg); + + case CDA_UNMAP_MEM: + return cda_unmap_mem_by_idx(cdadev, (void *)file, (void __user *)arg); + + case CDA_INIT_INT: + return cda_init_interrupts(cdadev, (void *)file, (void __user *)arg); + + case CDA_FREE_INT: + return cda_free_irqs(cdadev, (void *)file); + + case CDA_REQ_INT: + return cda_req_int(cdadev, (void *)file, (void __user *) arg); + + case CDA_INT_CANCEL: + return cda_cancel_req(cdadev, (void *)file); + + case CDA_SEM_AQ: + return cda_sem_aq(cdadev, (void *)file, (void __user *) arg); + + case CDA_SEM_REL: + return cda_sem_rel(cdadev, (void *)file, (void __user *) arg); + + default: + return -ENOTTY; + } +} + +static void cdadev_release(struct device *dev) +{ + struct cda_dev *cdadev = container_of(dev, struct cda_dev, dev); + + kfree(cdadev); +} + +static int __init cdadrv_init(void) +{ + int ret; + size_t pci_id_table_size = ARRAY_SIZE(cda_pci_ids); + + if (test_probe) { + printk("Test run. Nothing initialized\n"); + return 0; + } + + ret = alloc_chrdev_region(&cdadev_first, 0, CDA_DEV_MINOR_MAX, cda_name); + if (ret) + goto err_alloc_cdev_reg; + + ret = class_register(&cda_class); + if (ret) + goto err_cls_reg; + + if ((req_pci_did || req_pci_vid) && pci_id_table_size >= 2) { + // Last table element is 0,0 + // Update pre-last item + cda_pci_ids[pci_id_table_size-2].vendor = req_pci_vid; + cda_pci_ids[pci_id_table_size-2].device = req_pci_did; + } + ret = pci_register_driver(&cda_pci); + if (ret) + goto err_pci_reg_drv; + + return 0; + +err_pci_reg_drv: + class_unregister(&cda_class); +err_cls_reg: + unregister_chrdev_region(cdadev_first, CDA_DEV_MINOR_MAX); +err_alloc_cdev_reg: + return ret; +} + +static void __exit dcadrv_exit(void) +{ + if (test_probe) { + printk("Stop test run. Nothing initialized\n"); + return; + } + pci_unregister_driver(&cda_pci); + class_unregister(&cda_class); + unregister_chrdev_region(cdadev_first, CDA_DEV_MINOR_MAX); +} + +module_init(cdadrv_init); +module_exit(dcadrv_exit); diff --git a/src/cdadrv.h b/src/cdadrv.h index 3a8ca2d..67a7e2d 100644 --- a/src/cdadrv.h +++ b/src/cdadrv.h @@ -1,71 +1,71 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. -// -// CDA linux driver mem blocks/mem maps and interrupt request handler -// -// This program is free software; you can redistribute it and/or modify it -// under the terms and conditions of the GNU General Public License, -// version 2, as published by the Free Software Foundation. -// -#include <linux/pci.h> -#include <linux/cdev.h> -#include <linux/version.h> - - -#define CDA_MAX_DRV_SEMAPHORES (16) - -struct cda_interrupts; -struct cda_bar; -struct cda_dev; -// Dummy block for fast releasing -struct cda_dummy_blk { - struct cda_dev *dev; - int index; -}; - -struct cda_dev { - struct cdev cdev; - struct device dev; - - int minor; - struct list_head devices; - - struct pci_dev *pcidev; - unsigned long stored_flags[PCI_ROM_RESOURCE]; - - struct mutex ilock; - struct cda_interrupts *ints; - - struct kobject *kobj_mems; - struct cda_dummy_blk *dummy_blk; - struct idr mblk_idr; - spinlock_t mblk_sl; - struct list_head mem_blocks; - struct list_head mem_maps; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) - // Security kernel lock needs w/a to access BARs - struct kobject *kobj_bars; - struct cda_bar *sysfs_bar[PCI_ROM_RESOURCE]; // 6 BARs excl. ROM -#endif - u64 semaphores[CDA_MAX_DRV_SEMAPHORES]; - void *sem_owner[CDA_MAX_DRV_SEMAPHORES]; -}; - -int cda_alloc_mem(struct cda_dev *dev, void *owner, void __user *arg); -int cda_free_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq); -int cda_map_mem(struct cda_dev *dev, void *owner, void __user *arg); -int cda_unmap_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq); -void cda_unmmap_dev_mem(struct cda_dev *dev, void *owner); -void cda_free_dev_mem(struct cda_dev *dev, void *owner); -int cda_init_interrupts(struct cda_dev *dev, void *owner, void __user *ureq); -int cda_mems_create(struct cda_dev *dev); -int cda_free_irqs(struct cda_dev *dev, void *owner); -void cda_mems_release(struct cda_dev *dev); -int cda_req_int(struct cda_dev *dev, void *owner, void __user *ureq); -int cda_cancel_req(struct cda_dev *dev, void *owner); -int cda_open_bars(struct cda_dev *cdadev); -void cda_release_bars(struct cda_dev *cdadev); -int cda_sem_aq(struct cda_dev *cdadev, void *owner, void __user *ureq); -int cda_sem_rel(struct cda_dev *cdadev, void *owner, void __user *ureq); -void cda_sem_rel_by_owner(struct cda_dev *dev, void *owner); +/* SPDX-License-Identifier: GPL-2.0 */ +// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. +// +// CDA linux driver mem blocks/mem maps and interrupt request handler +// +// This program is free software; you can redistribute it and/or modify it +// under the terms and conditions of the GNU General Public License, +// version 2, as published by the Free Software Foundation. +// +#include <linux/pci.h> +#include <linux/cdev.h> +#include <linux/version.h> + + +#define CDA_MAX_DRV_SEMAPHORES (16) + +struct cda_interrupts; +struct cda_bar; +struct cda_dev; +// Dummy block for fast releasing +struct cda_dummy_blk { + struct cda_dev *dev; + int index; +}; + +struct cda_dev { + struct cdev cdev; + struct device dev; + + int minor; + struct list_head devices; + + struct pci_dev *pcidev; + unsigned long stored_flags[PCI_ROM_RESOURCE]; + + struct mutex ilock; + struct cda_interrupts *ints; + + struct kobject *kobj_mems; + struct cda_dummy_blk *dummy_blk; + struct idr mblk_idr; + spinlock_t mblk_sl; + struct list_head mem_blocks; + struct list_head mem_maps; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) + // Security kernel lock needs w/a to access BARs + struct kobject *kobj_bars; + struct cda_bar *sysfs_bar[PCI_ROM_RESOURCE]; // 6 BARs excl. ROM +#endif + u64 semaphores[CDA_MAX_DRV_SEMAPHORES]; + void *sem_owner[CDA_MAX_DRV_SEMAPHORES]; +}; + +int cda_alloc_mem(struct cda_dev *dev, void *owner, void __user *arg); +int cda_free_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq); +int cda_map_mem(struct cda_dev *dev, void *owner, void __user *arg); +int cda_unmap_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq); +void cda_unmmap_dev_mem(struct cda_dev *dev, void *owner); +void cda_free_dev_mem(struct cda_dev *dev, void *owner); +int cda_init_interrupts(struct cda_dev *dev, void *owner, void __user *ureq); +int cda_mems_create(struct cda_dev *dev); +int cda_free_irqs(struct cda_dev *dev, void *owner); +void cda_mems_release(struct cda_dev *dev); +int cda_req_int(struct cda_dev *dev, void *owner, void __user *ureq); +int cda_cancel_req(struct cda_dev *dev, void *owner); +int cda_open_bars(struct cda_dev *cdadev); +void cda_release_bars(struct cda_dev *cdadev); +int cda_sem_aq(struct cda_dev *cdadev, void *owner, void __user *ureq); +int cda_sem_rel(struct cda_dev *cdadev, void *owner, void __user *ureq); +void cda_sem_rel_by_owner(struct cda_dev *dev, void *owner); diff --git a/src/cdaioctl.h b/src/cdaioctl.h index bb10a9c..6d1cec4 100644 --- a/src/cdaioctl.h +++ b/src/cdaioctl.h @@ -1,62 +1,62 @@ -// SPDX-License-Identifier: LGPL-3.0 -// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. -// -// CDA linux driver mem blocks/mem maps and interrupt request handler -// -// This program is free software; you can redistribute it and/or modify it -// under the terms and conditions of the Lesser GNU General Public License, -// version 3, as published by the Free Software Foundation. -// -#ifndef LINUX_VERSION_CODE -#include <stdint.h> // only include if we're not compiling against the kernel -#endif - -#define CDA_IOCTL_MAGIC 'C' -#define CDA_ALLOC_MEM _IOWR(CDA_IOCTL_MAGIC, 0x1, long) -#define CDA_FREE_MEM _IOW(CDA_IOCTL_MAGIC, 0x2, long) -#define CDA_MAP_MEM _IOWR(CDA_IOCTL_MAGIC, 0x3, long) -#define CDA_UNMAP_MEM _IOW(CDA_IOCTL_MAGIC, 0x4, long) -#define CDA_INIT_INT _IOWR(CDA_IOCTL_MAGIC, 0x5, long) -#define CDA_FREE_INT _IOW(CDA_IOCTL_MAGIC, 0x6, long) -#define CDA_REQ_INT _IOWR(CDA_IOCTL_MAGIC, 0x7, long) -#define CDA_INT_CANCEL _IOWR(CDA_IOCTL_MAGIC, 0x8, long) -#define CDA_SEM_AQ _IOW(CDA_IOCTL_MAGIC, 0x9, long) -#define CDA_SEM_REL _IOW(CDA_IOCTL_MAGIC, 0xA, long) - -struct cda_alloc_mem { - uint32_t size; - uint32_t index; -}; - -struct cda_map_mem { - uintptr_t vaddr; - uint32_t size; - uint32_t index; -}; - -struct cda_drv_sg_item { - uint64_t paddr; - uint32_t size; -}; - -struct cda_req_int { - uint32_t vector; - uint64_t timeout; - uint32_t reset; -}; - -enum int_type { - LEGACY_INTERRUPT = 0, - MSI = 1, - MSIX = 2 -}; - -struct cda_int_lock { - uint32_t inttype; - uint32_t vectors; -}; - -struct cda_sem_aq { - uint32_t sem_id; - uint64_t time_ns; -}; +/* SPDX-License-Identifier: LGPL-3.0 */ +// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. +// +// CDA linux driver mem blocks/mem maps and interrupt request handler +// +// This program is free software; you can redistribute it and/or modify it +// under the terms and conditions of the Lesser GNU General Public License, +// version 3, as published by the Free Software Foundation. +// +#ifndef LINUX_VERSION_CODE +#include <stdint.h> // only include if we're not compiling against the kernel +#endif + +#define CDA_IOCTL_MAGIC 'C' +#define CDA_ALLOC_MEM _IOWR(CDA_IOCTL_MAGIC, 0x1, long) +#define CDA_FREE_MEM _IOW(CDA_IOCTL_MAGIC, 0x2, long) +#define CDA_MAP_MEM _IOWR(CDA_IOCTL_MAGIC, 0x3, long) +#define CDA_UNMAP_MEM _IOW(CDA_IOCTL_MAGIC, 0x4, long) +#define CDA_INIT_INT _IOWR(CDA_IOCTL_MAGIC, 0x5, long) +#define CDA_FREE_INT _IOW(CDA_IOCTL_MAGIC, 0x6, long) +#define CDA_REQ_INT _IOWR(CDA_IOCTL_MAGIC, 0x7, long) +#define CDA_INT_CANCEL _IOWR(CDA_IOCTL_MAGIC, 0x8, long) +#define CDA_SEM_AQ _IOW(CDA_IOCTL_MAGIC, 0x9, long) +#define CDA_SEM_REL _IOW(CDA_IOCTL_MAGIC, 0xA, long) + +struct cda_alloc_mem { + uint32_t size; + uint32_t index; +}; + +struct cda_map_mem { + uintptr_t vaddr; + uint32_t size; + uint32_t index; +}; + +struct cda_drv_sg_item { + uint64_t paddr; + uint32_t size; +}; + +struct cda_req_int { + uint32_t vector; + uint64_t timeout; + uint32_t reset; +}; + +enum int_type { + LEGACY_INTERRUPT = 0, + MSI = 1, + MSIX = 2 +}; + +struct cda_int_lock { + uint32_t inttype; + uint32_t vectors; +}; + +struct cda_sem_aq { + uint32_t sem_id; + uint64_t time_ns; +}; diff --git a/src/cdamem.c b/src/cdamem.c index 5280185..ed39573 100644 --- a/src/cdamem.c +++ b/src/cdamem.c @@ -1,861 +1,858 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright(c) 2020 Egor Pomozov. -// -// Originally memalloc sequence was designed for simple driver -// in Aquantia Corp by Vadim Solomin -// Later was updated by QA team in Aquantia Corp. -// Later it was additionally modifyied by Egor Pomozov -// -// CDA linux driver memory request handler -// -// This program is free software; you can redistribute it and/or modify it -// under the terms and conditions of the GNU General Public License, -// version 2, as published by the Free Software Foundation. -// - -#include <linux/fs.h> -#include <linux/uaccess.h> -#include <linux/mm.h> -#include <linux/dma-mapping.h> - -#include "cdadrv.h" -#include "cdaioctl.h" - -#include <linux/version.h> - - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5,6,0) -/** - * pin_user_pages_fast() - pin user pages in memory without taking locks - * - * For now, this is a placeholder function, until various call sites are - * converted to use the correct get_user_pages*() or pin_user_pages*() API. So, - * this is identical to get_user_pages_fast(). - * - * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It - * is NOT intended for Case 2 (RDMA: long-term pins). - */ -static int pin_user_pages_fast(unsigned long start, int nr_pages, - unsigned int gup_flags, struct page **pages) -{ - /* - * This is a placeholder, until the pin functionality is activated. - * Until then, just behave like the corresponding get_user_pages*() - * routine. - */ - return get_user_pages_fast(start, nr_pages, gup_flags, pages); -} - -/** - * unpin_user_page() - release a gup-pinned page - * @page: pointer to page to be released - * - * Pages that were pinned via pin_user_pages*() must be released via either - * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so - * that eventually such pages can be separately tracked and uniquely handled. In - * particular, interactions with RDMA and filesystems need special handling. - * - * unpin_user_page() and put_page() are not interchangeable, despite this early - * implementation that makes them look the same. unpin_user_page() calls must - * be perfectly matched up with pin*() calls. - */ -static inline void unpin_user_page(struct page *page) -{ - put_page(page); -} - -/** - * unpin_user_pages() - release an array of gup-pinned pages. - * @pages: array of pages to be marked dirty and released. - * @npages: number of pages in the @pages array. - * - * For each page in the @pages array, release the page using unpin_user_page(). - * - * Please see the unpin_user_page() documentation for details. - */ -static void unpin_user_pages(struct page **pages, unsigned long npages) -{ - unsigned long index; - - /* - * TODO: this can be optimized for huge pages: if a series of pages is - * physically contiguous and part of the same compound page, then a - * single operation to the head page should suffice. - */ - for (index = 0; index < npages; index++) - unpin_user_page(pages[index]); -} - -/** - * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages - * @pages: array of pages to be maybe marked dirty, and definitely released. - * @npages: number of pages in the @pages array. - * @make_dirty: whether to mark the pages dirty - * - * "gup-pinned page" refers to a page that has had one of the get_user_pages() - * variants called on that page. - * - * For each page in the @pages array, make that page (or its head page, if a - * compound page) dirty, if @make_dirty is true, and if the page was previously - * listed as clean. In any case, releases all pages using unpin_user_page(), - * possibly via unpin_user_pages(), for the non-dirty case. - * - * Please see the unpin_user_page() documentation for details. - * - * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is - * required, then the caller should a) verify that this is really correct, - * because _lock() is usually required, and b) hand code it: - * set_page_dirty_lock(), unpin_user_page(). - * - */ -static void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, - bool make_dirty) -{ - unsigned long index; - - /* - * TODO: this can be optimized for huge pages: if a series of pages is - * physically contiguous and part of the same compound page, then a - * single operation to the head page should suffice. - */ - - if (!make_dirty) { - unpin_user_pages(pages, npages); - return; - } - - for (index = 0; index < npages; index++) { - struct page *page = compound_head(pages[index]); - /* - * Checking PageDirty at this point may race with - * clear_page_dirty_for_io(), but that's OK. Two key - * cases: - * - * 1) This code sees the page as already dirty, so it - * skips the call to set_page_dirty(). That could happen - * because clear_page_dirty_for_io() called - * page_mkclean(), followed by set_page_dirty(). - * However, now the page is going to get written back, - * which meets the original intention of setting it - * dirty, so all is well: clear_page_dirty_for_io() goes - * on to call TestClearPageDirty(), and write the page - * back. - * - * 2) This code sees the page as clean, so it calls - * set_page_dirty(). The page stays dirty, despite being - * written back, so it gets written back again in the - * next writeback cycle. This is harmless. - */ - if (!PageDirty(page)) - set_page_dirty_lock(page); - unpin_user_page(page); - } -} - -#endif - -static ssize_t name_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct cda_dev *cdadev = container_of((dev), struct cda_dev, dev); - return sprintf(buf, "cda%d\n", cdadev->minor); -} -static DEVICE_ATTR_RO(name); - -static struct attribute *cda_attrs[] = { - &dev_attr_name.attr, - NULL, -}; - -static struct attribute_group cda_attr_grp = { - .attrs = cda_attrs, -}; - -static ssize_t mblk_attr_show( - struct kobject *kobj, - struct attribute *attr, - char *buf); - -static void mblk_release(struct kobject *kobj); - -struct cda_mblk { - struct cda_dev *dev; - int index; - - struct kobject kobj; - uint32_t req_size; - void *vaddr; //kernel - uint32_t size; - dma_addr_t paddr; - void *owner; - struct list_head list; - struct bin_attribute mmap_attr; -}; - -struct cda_mmap { - struct cda_dev *dev; - int index; - - struct kobject kobj; - void *owner; - - void *vaddr; //original user - uint32_t size; //original user - uint32_t blk_cnt; - uint32_t mapped_blk_cnt; - uint32_t show_cnt; - struct sg_table sgt; - struct page **pages; - struct cda_drv_sg_item *sg_list; - struct list_head list; - struct bin_attribute mmap_attr; -}; - -struct mblkitem_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct cda_mblk *, char *); - ssize_t (*store)(struct cda_mblk *, char*, size_t); -}; - -#define cda_dev_mblk_attr(_field, _fmt) \ - static ssize_t \ - mblk_##_field##_show(struct cda_mblk *mblk, char *buf) \ - { \ - return sprintf(buf, _fmt, mblk->_field); \ - } \ - static struct mblkitem_sysfs_entry mblk_##_field##_attr = \ - __ATTR(_field, S_IRUGO, mblk_##_field##_show, NULL); - -#pragma GCC diagnostic ignored "-Wformat" -cda_dev_mblk_attr(vaddr, "0x%lx\n"); -cda_dev_mblk_attr(paddr, "0x%lx\n"); -cda_dev_mblk_attr(size, "0x%x\n"); -cda_dev_mblk_attr(req_size, "0x%x\n"); -cda_dev_mblk_attr(owner, "0x%p\n"); -cda_dev_mblk_attr(index, "%d\n"); -#pragma GCC diagnostic warning "-Wformat" - -static struct attribute *mblk_attrs[] = { - &mblk_vaddr_attr.attr, - &mblk_paddr_attr.attr, - &mblk_size_attr.attr, - &mblk_owner_attr.attr, - &mblk_req_size_attr.attr, - &mblk_index_attr.attr, - NULL, -}; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) -ATTRIBUTE_GROUPS(mblk); -#endif -static const struct sysfs_ops mblk_ops = { - .show = mblk_attr_show, -}; - -struct kobj_type mblk_type = { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) - .default_groups = mblk_groups, -#else - .default_attrs = mblk_attrs, -#endif - .sysfs_ops = &mblk_ops, - .release = mblk_release, -}; - -static ssize_t mblk_attr_show(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - struct cda_mblk *mblk = container_of(kobj, struct cda_mblk, kobj); - struct mblkitem_sysfs_entry *entry = - container_of(attr, struct mblkitem_sysfs_entry, attr); - - if (!entry->show) - return -EIO; - - return entry->show(mblk, buf); -} - -static void mblk_release(struct kobject *kobj) -{ - struct cda_mblk *mblk = container_of(kobj, struct cda_mblk, kobj); - kfree(mblk); -} - -#define to_memmap(obj) container_of(obj, struct cda_mmap, kobj) - -struct memmapitem_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct cda_mmap *, char *); - ssize_t (*store)(struct cda_mmap *, char *, size_t); -}; - -#define cda_dev_memmap_attr(_field, _fmt) \ - static ssize_t \ - memmap_##_field##_show(struct cda_mmap *memmap, char *buf) \ - { \ - return sprintf(buf, _fmt, memmap->_field); \ - } \ - static struct memmapitem_sysfs_entry memmap_##_field##_attr = \ - __ATTR(_field, S_IRUGO, memmap_##_field##_show, NULL); - -#pragma GCC diagnostic ignored "-Wformat" -cda_dev_memmap_attr(owner, "0x%p\n"); -cda_dev_memmap_attr(vaddr, "0x%lx\n"); -cda_dev_memmap_attr(size, "0x%x\n"); -cda_dev_memmap_attr(index, "%d\n"); -cda_dev_memmap_attr(blk_cnt, "%d\n"); - -static ssize_t -memmap_sglist_show(struct cda_mmap *memmap, char *buf) -{ - const int sg_list_item_size = 16 + 8 + 2; //"%016llx %08lx\n" - int res = 0; - int i = memmap->show_cnt; - memmap->show_cnt = 0; - buf[0] = '\0'; - for( ; i < memmap->blk_cnt; i++ ) { - if( (res + sg_list_item_size) >= (PAGE_SIZE - 1)) /* https://lwn.net/Articles/178634/ */{ - memmap->show_cnt = i; - //printk("Split SG list. Next read starts with item: %d\n", i); - break; - } - res += sprintf(&buf[res], "%016llx %08lx\n", memmap->sg_list[i].paddr, memmap->sg_list[i].size); - } - return res; -} - -static struct memmapitem_sysfs_entry memmap_sglist_attr = - __ATTR(sglist, S_IRUGO, memmap_sglist_show, NULL); - -#pragma GCC diagnostic warning "-Wformat" -static struct attribute *memmap_attrs[] = { - &memmap_owner_attr.attr, - &memmap_vaddr_attr.attr, - &memmap_size_attr.attr, - &memmap_index_attr.attr, - &memmap_blk_cnt_attr.attr, - &memmap_sglist_attr.attr, - NULL, -}; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) -ATTRIBUTE_GROUPS(memmap); -#endif - -static ssize_t memmap_attr_show(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - struct cda_mmap *memmap = to_memmap(kobj); - struct memmapitem_sysfs_entry *entry = - container_of(attr, struct memmapitem_sysfs_entry, attr); - - if (!entry->show) - return -EIO; - - return entry->show(memmap, buf); -} - -static void memmap_release(struct kobject *kobj) -{ - struct cda_mmap *memmap = to_memmap(kobj); - kfree(memmap); -} - -static const struct sysfs_ops memmap_ops = { - .show = memmap_attr_show, -}; - -struct kobj_type memmap_type = { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) - .default_groups = memmap_groups, -#else - .default_attrs = memmap_attrs, -#endif - .sysfs_ops = &memmap_ops, - .release = memmap_release, -}; - -static int mblk_mmap( struct file *file, - struct kobject *kobj, - struct bin_attribute *attr, - struct vm_area_struct *vma) -{ - struct cda_mblk *mblk = attr->private; - unsigned long requested = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - unsigned long pages = (unsigned long)mblk->req_size >> PAGE_SHIFT; - - if (vma->vm_pgoff + requested > pages) - return -EINVAL; - - if( dma_mmap_coherent( &mblk->dev->pcidev->dev, - vma, - mblk->vaddr, - mblk->paddr, - mblk->req_size) ) - { - dev_err(&mblk->dev->pcidev->dev, "DMA remapping failed"); - return -ENXIO; - } - return 0; -} - -void cda_hide_memmap(struct cda_mmap *memmap); -int cda_publish_memmap(struct cda_mmap *memmap); -void cda_hide_mblk(struct cda_mblk *mblk); -int cda_publish_mblk(struct cda_mblk *mblk); - -int cda_publish_mblk(struct cda_mblk *mblk) -{ - int ret; - struct bin_attribute *mmap_attr = &mblk->mmap_attr; - - ret = kobject_add( &mblk->kobj, mblk->dev->kobj_mems, - "%04d", mblk->index); - if (ret) - goto err_add; - - mmap_attr->mmap = mblk_mmap; - mmap_attr->attr.name = "mmap"; - mmap_attr->attr.mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - mmap_attr->size = mblk->req_size; - mmap_attr->private = mblk; - ret = sysfs_create_bin_file(&mblk->kobj, mmap_attr); - if (ret) - goto err_map_add; - - return 0; - -err_map_add: - kobject_del(&mblk->kobj); -err_add: - kobject_put(&mblk->kobj); - return ret; -} - - -void cda_hide_mblk(struct cda_mblk *mblk) -{ - sysfs_remove_bin_file(&mblk->kobj, &mblk->mmap_attr); - kobject_del(&mblk->kobj); -} - -int cda_publish_memmap(struct cda_mmap *memmap) -{ - int ret; - struct bin_attribute *mmap_attr = &memmap->mmap_attr; - - ret = kobject_add( &memmap->kobj, memmap->dev->kobj_mems, - "%04d", memmap->index); - if (ret) - goto err_add; - - mmap_attr->attr.name = "memmapobj"; - mmap_attr->attr.mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - mmap_attr->size = memmap->size; - mmap_attr->private = memmap; - ret = sysfs_create_bin_file(&memmap->kobj, mmap_attr); - if (ret) - goto err_map_add; - - return 0; - -err_map_add: - kobject_del(&memmap->kobj); -err_add: - kobject_put(&memmap->kobj); - return ret; -} - -void cda_hide_memmap(struct cda_mmap *memmap) -{ - sysfs_remove_bin_file(&memmap->kobj, &memmap->mmap_attr); - kobject_del(&memmap->kobj); -} - -int cda_alloc_mem(struct cda_dev *dev, void *owner, void __user *ureq) -{ - int ret = -ENOMEM; - int idx; - struct cda_mblk *mblk; - struct cda_alloc_mem req; - if (copy_from_user(&req, ureq, sizeof(req))) - return -EFAULT; - - mblk = kzalloc(sizeof(*mblk), in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - if (!mblk) { - dev_err(&dev->dev, "Can't alloc mblk\n"); - goto out; - } - INIT_LIST_HEAD(&mblk->list); - mblk->dev = dev; - kobject_init(&mblk->kobj, &mblk_type); - mblk->owner = owner; - mblk->size = req.size; - req.size = ALIGN(req.size, PAGE_SIZE); - - idr_preload(in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - spin_lock(&dev->mblk_sl); - ret = idr_alloc(&dev->mblk_idr, mblk, - 1L, 0, in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - spin_unlock(&dev->mblk_sl); - idr_preload_end(); - if (ret < 0) - goto err_idr; - mblk->index = req.index = idx = ret; - - mblk->vaddr = dma_alloc_coherent( - &dev->pcidev->dev, - req.size, - &mblk->paddr, - in_atomic() ? GFP_ATOMIC | GFP_KERNEL : GFP_KERNEL); - if (!mblk->vaddr) { - dev_err(&dev->dev, "Can't alloc DMA memory (size %u)", req.size); - ret = -1; - goto err_dma_alloc; - } - mblk->req_size = req.size; - - ret = cda_publish_mblk(mblk); - if (ret) { - dev_err(&dev->dev, "Can't publish mblk to sysfs: %d", ret); - goto err_publish; - } - - if(copy_to_user(ureq, &req, sizeof(req))) { - ret = -EFAULT; - goto err_copy_to_user; - } - - spin_lock(&dev->mblk_sl); - list_add(&mblk->list, &dev->mem_blocks); - spin_unlock(&dev->mblk_sl); - - return 0; - -err_copy_to_user: - cda_hide_mblk(mblk); -err_publish: - dma_free_coherent(&dev->pcidev->dev, mblk->req_size, - mblk->vaddr, mblk->paddr); -err_dma_alloc: - spin_lock(&dev->mblk_sl); - idr_remove(&dev->mblk_idr, idx); - spin_unlock(&dev->mblk_sl); -err_idr: - kobject_put(&mblk->kobj); -out: - return ret; -} - -static void cda_free_mem(struct cda_mblk *mblk) -{ - cda_hide_mblk(mblk); - dma_free_coherent(&mblk->dev->pcidev->dev, mblk->req_size, - mblk->vaddr, mblk->paddr); - kobject_put(&mblk->kobj); -} - -int cda_free_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq) -{ - int memidx; - struct cda_mblk *mblk; - if (copy_from_user(&memidx, (void __user *)ureq, sizeof(memidx))) { - return -EFAULT; - } - - spin_lock(&dev->mblk_sl); - mblk = idr_find(&dev->mblk_idr, memidx); - if (mblk && mblk->index == memidx) { - if( mblk->owner != owner ) { - dev_warn(&dev->dev, "Free mblk from another owner\n"); - idr_replace(&dev->mblk_idr, dev->dummy_blk, memidx); - } - list_del(&mblk->list); - } else if(mblk) { - dev_warn(&dev->dev, "Free mblk with index %d, required %d\n", mblk->index, memidx); - } - spin_unlock(&dev->mblk_sl); - if (!mblk) - return -ENOENT; - if (mblk->index) { - cda_free_mem(mblk); - spin_lock(&dev->mblk_sl); - idr_remove(&dev->mblk_idr, mblk->index); - spin_unlock(&dev->mblk_sl); - } - return 0; -} - -void cda_free_dev_mem(struct cda_dev *dev, void *owner) -{ - struct cda_mblk *mblk, *tmp; - LIST_HEAD(mblks); - - spin_lock(&dev->mblk_sl); - if( owner == NULL ){ - idr_destroy(&dev->mblk_idr); - list_replace_init(&dev->mem_blocks, &mblks); - } else { - list_for_each_entry_safe(mblk, tmp, &dev->mem_blocks, list) { - if( mblk->index > 0L && mblk->owner == owner ) { - list_move(&mblk->list, &mblks); - idr_replace(&dev->mblk_idr, dev->dummy_blk, mblk->index); - } - } - } - spin_unlock(&dev->mblk_sl); - list_for_each_entry_safe(mblk, tmp, &mblks, list) { - // Unmap blocks owned by specified owner or all if owner is NULL - cda_free_mem(mblk); - if( owner != NULL ){ - spin_lock(&dev->mblk_sl); - idr_remove(&dev->mblk_idr, mblk->index); - spin_unlock(&dev->mblk_sl); - } - } -} - -static void cda_release_map(struct cda_mmap *memmap) -{ - dma_unmap_sg(memmap->dev->pcidev == NULL ? NULL : &memmap->dev->pcidev->dev, memmap->sgt.sgl, memmap->sgt.orig_nents, DMA_BIDIRECTIONAL); - unpin_user_pages_dirty_lock(memmap->pages, memmap->blk_cnt, 1); - memmap->mapped_blk_cnt = 0; -} - -static int cda_perform_mapping( - struct cda_mmap *memmap) -{ - uint i; - int nents; - struct scatterlist *sg; - ulong len = memmap->size; - void __user *buf = memmap->vaddr; - struct cda_drv_sg_item *cda_sg_list = memmap->sg_list; - sg = memmap->sgt.sgl; - for (i = 0; i < memmap->sgt.orig_nents; i++, sg = sg_next(sg)) { - unsigned int offset = offset_in_page(buf); - unsigned int nbytes = - min_t(unsigned int, PAGE_SIZE - offset, len); - - sg_set_page(sg, memmap->pages[i], nbytes, offset); - - buf += nbytes; - len -= nbytes; - } - - nents = dma_map_sg(&memmap->dev->pcidev->dev, memmap->sgt.sgl, memmap->sgt.orig_nents, DMA_BIDIRECTIONAL); - if (!nents) { - dev_err(&memmap->dev->dev, "map sgl failed, sgt 0x%p.\n", &memmap->sgt); - return -EIO; - } - memmap->sgt.nents = nents; - - for (i = 0, sg = memmap->sgt.sgl; i < nents; i++, sg = sg_next(sg)) { - cda_sg_list[i].size = sg_dma_len(sg); - cda_sg_list[i].paddr = sg_dma_address(sg); - } - - memmap->mapped_blk_cnt = nents; - return 0; -} - -int cda_map_mem(struct cda_dev *dev, void *owner, void __user *ureq) -{ - int ret = -ENOMEM; - int idx; - int npages; - struct cda_mmap *memmap; - struct cda_map_mem req; - unsigned long offset; - void *req_vaddr; - - if (copy_from_user(&req, ureq, sizeof(req))) - return -EFAULT; - req_vaddr = (void __user *)req.vaddr; - offset = offset_in_page(req_vaddr); - npages = DIV_ROUND_UP(offset + req.size, PAGE_SIZE); - memmap = kzalloc(sizeof(*memmap) + npages * (sizeof(struct cda_drv_sg_item) + sizeof(struct page *)), - in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - if (!memmap) { - dev_err(&dev->dev, "Can't alloc memmap\n"); - goto out; - } - memmap->owner = owner; - memmap->sg_list = (struct cda_drv_sg_item *)((void *)memmap + sizeof(*memmap)); - memmap->pages = (struct page **)((void *)memmap + sizeof(*memmap) + npages * (sizeof(struct cda_drv_sg_item))); - - if (sg_alloc_table(&memmap->sgt, npages, - in_atomic() ? GFP_ATOMIC :GFP_KERNEL)) { - dev_err(&dev->dev, "Can't alloc sg table\n"); - goto out; - } - INIT_LIST_HEAD(&memmap->list); - memmap->dev = dev; - kobject_init(&memmap->kobj, &memmap_type); - - memmap->vaddr = req_vaddr; - memmap->size = req.size; - memmap->blk_cnt = npages; - idr_preload(in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - spin_lock(&dev->mblk_sl); - ret = idr_alloc(&dev->mblk_idr, memmap, - 1L, 0, in_atomic() ? GFP_ATOMIC : GFP_KERNEL); - spin_unlock(&dev->mblk_sl); - idr_preload_end(); - if (ret < 0) - goto err_idr; - memmap->index = req.index = idx = ret; - - ret = pin_user_pages_fast((ulong)req_vaddr, npages, - FOLL_WRITE, memmap->pages); - if ( ret < 0 ) { - dev_err(&dev->pcidev->dev, - "Pin user pages failed for addr=0x%p [ret=%d]\n", - req_vaddr, ret); - goto err_pin; - } - if (ret != npages) { - dev_err(&dev->pcidev->dev, - "Unable to pin all user pages for addr=0x%p\n", req_vaddr); - ret = -EFAULT; - goto err_pin; - } - - ret = cda_perform_mapping(memmap); - if ( ret ) { - dev_err(&dev->dev, "Can't map user memory for DMA (size %u)", req.size); - goto err_dma_alloc; - } - - ret = cda_publish_memmap(memmap); - if (ret) { - dev_err(&dev->dev, "Can't publish memmap to sysfs: %d", ret); - goto err_publish; - } - - if(copy_to_user(ureq, &req, sizeof(req))) { - ret = -EFAULT; - goto err_copy_to_user; - } - - spin_lock(&dev->mblk_sl); - list_add(&memmap->list, &dev->mem_maps); - spin_unlock(&dev->mblk_sl); - - dev_dbg(&dev->dev, "map vaddr %p, pages %d\n", memmap->vaddr, npages); - return 0; - -err_copy_to_user: - cda_hide_memmap(memmap); -err_publish: - cda_release_map(memmap); -err_dma_alloc: - unpin_user_pages_dirty_lock(memmap->pages, memmap->blk_cnt, 1); -err_pin: - spin_lock(&dev->mblk_sl); - idr_remove(&dev->mblk_idr, idx); - spin_unlock(&dev->mblk_sl); -err_idr: - kobject_put(&memmap->kobj); -out: - if( memmap ) { - if( memmap->pages ) { - kfree(memmap->pages); - } - kfree(memmap); - } - return ret; -} - -static void cda_free_map(struct cda_mmap *memmap) -{ - dev_dbg(&memmap->dev->dev, "unmap vaddr %p, pages %d\n", memmap->vaddr, memmap->blk_cnt); - cda_hide_memmap(memmap); - cda_release_map(memmap); - kobject_put(&memmap->kobj); -} - -int cda_unmap_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq) -{ - int memidx; - struct cda_mmap *memmap; - if (copy_from_user(&memidx, (void __user *)ureq, sizeof(memidx))) - return -EFAULT; - - spin_lock(&dev->mblk_sl); - memmap = idr_find(&dev->mblk_idr, memidx); - if (memmap && memmap->index == memidx) { - if( memmap->owner != owner ) - dev_warn(&dev->dev, "Unmap buffer by another user\n"); - idr_replace(&dev->mblk_idr, dev->dummy_blk, memidx); - list_del(&memmap->list); - } else if (memmap) - dev_warn(&dev->dev, "Unmap buffer with index %d, required %d\n", memmap->index, memidx); - spin_unlock(&dev->mblk_sl); - - if (!memmap) - return -ENOENT; // Somebody may already release this block in parallel - - if( memmap->index ) { - cda_free_map(memmap); - spin_lock(&dev->mblk_sl); - idr_remove(&dev->mblk_idr, memmap->index); - spin_unlock(&dev->mblk_sl); - } - return 0; -} - -void cda_unmmap_dev_mem(struct cda_dev *dev, void *owner) -{ - struct cda_mmap *memmap, *tmp; - LIST_HEAD(memmaps); - - spin_lock(&dev->mblk_sl); - if( owner == NULL ){ - list_replace_init(&dev->mem_maps, &memmaps); - } else { - list_for_each_entry_safe(memmap, tmp, &dev->mem_maps, list) { - if( memmap->index > 0L && memmap->owner == owner ) { - idr_replace(&dev->mblk_idr, dev->dummy_blk, memmap->index); - list_move(&memmap->list, &memmaps); - } - } - } - spin_unlock(&dev->mblk_sl); - list_for_each_entry_safe(memmap, tmp, &memmaps, list) { - // Unmap blocks owned by specified owner or all if owner is NULL - cda_free_map(memmap); - if( owner != NULL ){ - spin_lock(&dev->mblk_sl); - idr_remove(&dev->mblk_idr, memmap->index); - spin_unlock(&dev->mblk_sl); - } - } -} - -int cda_mems_create(struct cda_dev *cdadev) -{ - int ret = sysfs_create_group(&cdadev->dev.kobj, &cda_attr_grp); - if (ret) - goto err_group; - - cdadev->kobj_mems = kobject_create_and_add("mems", &cdadev->dev.kobj); - if (!cdadev->kobj_mems) - goto err_mems; - return 0; - -err_mems: - sysfs_remove_group(&cdadev->dev.kobj, &cda_attr_grp); -err_group: - dev_err(&cdadev->dev, "Couldn't create sysfs files: %d\n", ret); - return ret; -} - -void cda_mems_release(struct cda_dev *dev) -{ - //cda_release_bars(dev); - kobject_del(dev->kobj_mems); - kobject_put(dev->kobj_mems); - sysfs_remove_group(&dev->dev.kobj, &cda_attr_grp); -} +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2020 Egor Pomozov. +// +// Originally memalloc sequence was designed for simple driver +// in Aquantia Corp by Vadim Solomin +// Later was updated by QA team in Aquantia Corp. +// Later it was additionally modified by Egor Pomozov +// +// CDA linux driver memory request handler +// +// This program is free software; you can redistribute it and/or modify it +// under the terms and conditions of the GNU General Public License, +// version 2, as published by the Free Software Foundation. +// + +#include <linux/fs.h> +#include <linux/uaccess.h> +#include <linux/mm.h> +#include <linux/dma-mapping.h> + +#include "cdadrv.h" +#include "cdaioctl.h" + +#include <linux/version.h> + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0) +/** + * pin_user_pages_fast() - pin user pages in memory without taking locks + * + * For now, this is a placeholder function, until various call sites are + * converted to use the correct get_user_pages*() or pin_user_pages*() API. So, + * this is identical to get_user_pages_fast(). + * + * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It + * is NOT intended for Case 2 (RDMA: long-term pins). + */ +static int pin_user_pages_fast(unsigned long start, int nr_pages, + unsigned int gup_flags, struct page **pages) +{ + /* + * This is a placeholder, until the pin functionality is activated. + * Until then, just behave like the corresponding get_user_pages*() + * routine. + */ + return get_user_pages_fast(start, nr_pages, gup_flags, pages); +} + +/** + * unpin_user_page() - release a gup-pinned page + * @page: pointer to page to be released + * + * Pages that were pinned via pin_user_pages*() must be released via either + * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so + * that eventually such pages can be separately tracked and uniquely handled. In + * particular, interactions with RDMA and filesystems need special handling. + * + * unpin_user_page() and put_page() are not interchangeable, despite this early + * implementation that makes them look the same. unpin_user_page() calls must + * be perfectly matched up with pin*() calls. + */ +static inline void unpin_user_page(struct page *page) +{ + put_page(page); +} + +/** + * unpin_user_pages() - release an array of gup-pinned pages. + * @pages: array of pages to be marked dirty and released. + * @npages: number of pages in the @pages array. + * + * For each page in the @pages array, release the page using unpin_user_page(). + * + * Please see the unpin_user_page() documentation for details. + */ +static void unpin_user_pages(struct page **pages, unsigned long npages) +{ + unsigned long index; + + /* + * TODO: this can be optimized for huge pages: if a series of pages is + * physically contiguous and part of the same compound page, then a + * single operation to the head page should suffice. + */ + for (index = 0; index < npages; index++) + unpin_user_page(pages[index]); +} + +/** + * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages + * @pages: array of pages to be maybe marked dirty, and definitely released. + * @npages: number of pages in the @pages array. + * @make_dirty: whether to mark the pages dirty + * + * "gup-pinned page" refers to a page that has had one of the get_user_pages() + * variants called on that page. + * + * For each page in the @pages array, make that page (or its head page, if a + * compound page) dirty, if @make_dirty is true, and if the page was previously + * listed as clean. In any case, releases all pages using unpin_user_page(), + * possibly via unpin_user_pages(), for the non-dirty case. + * + * Please see the unpin_user_page() documentation for details. + * + * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is + * required, then the caller should a) verify that this is really correct, + * because _lock() is usually required, and b) hand code it: + * set_page_dirty_lock(), unpin_user_page(). + * + */ +static void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, + bool make_dirty) +{ + unsigned long index; + + /* + * TODO: this can be optimized for huge pages: if a series of pages is + * physically contiguous and part of the same compound page, then a + * single operation to the head page should suffice. + */ + + if (!make_dirty) { + unpin_user_pages(pages, npages); + return; + } + + for (index = 0; index < npages; index++) { + struct page *page = compound_head(pages[index]); + /* + * Checking PageDirty at this point may race with + * clear_page_dirty_for_io(), but that's OK. Two key + * cases: + * + * 1) This code sees the page as already dirty, so it + * skips the call to set_page_dirty(). That could happen + * because clear_page_dirty_for_io() called + * page_mkclean(), followed by set_page_dirty(). + * However, now the page is going to get written back, + * which meets the original intention of setting it + * dirty, so all is well: clear_page_dirty_for_io() goes + * on to call TestClearPageDirty(), and write the page + * back. + * + * 2) This code sees the page as clean, so it calls + * set_page_dirty(). The page stays dirty, despite being + * written back, so it gets written back again in the + * next writeback cycle. This is harmless. + */ + if (!PageDirty(page)) + set_page_dirty_lock(page); + unpin_user_page(page); + } +} + +#endif + +static ssize_t name_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct cda_dev *cdadev = container_of((dev), struct cda_dev, dev); + + return sprintf(buf, "cda%d\n", cdadev->minor); +} +static DEVICE_ATTR_RO(name); + +static struct attribute *cda_attrs[] = { + &dev_attr_name.attr, + NULL, +}; + +static struct attribute_group cda_attr_grp = { + .attrs = cda_attrs, +}; + +static ssize_t mblk_attr_show( + struct kobject *kobj, + struct attribute *attr, + char *buf); + +static void mblk_release(struct kobject *kobj); + +struct cda_mblk { + struct cda_dev *dev; + int index; + + struct kobject kobj; + uint32_t req_size; + void *vaddr; //kernel + uint32_t size; + dma_addr_t paddr; + void *owner; + struct list_head list; + struct bin_attribute mmap_attr; +}; + +struct cda_mmap { + struct cda_dev *dev; + int index; + + struct kobject kobj; + void *owner; + + void *vaddr; //original user + uint32_t size; //original user + uint32_t blk_cnt; + uint32_t mapped_blk_cnt; + uint32_t show_cnt; + struct sg_table sgt; + struct page **pages; + struct cda_drv_sg_item *sg_list; + struct list_head list; + struct bin_attribute mmap_attr; +}; + +struct mblkitem_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct cda_mblk *, char *); + ssize_t (*store)(struct cda_mblk *, char*, size_t); +}; + +#define cda_dev_mblk_attr(_field, _fmt) \ + static ssize_t \ + mblk_##_field##_show(struct cda_mblk *mblk, char *buf) \ + { \ + return sprintf(buf, _fmt, mblk->_field); \ + } \ + static struct mblkitem_sysfs_entry mblk_##_field##_attr = \ + __ATTR(_field, S_IRUGO, mblk_##_field##_show, NULL) + +#pragma GCC diagnostic ignored "-Wformat" +cda_dev_mblk_attr(vaddr, "0x%lx\n"); +cda_dev_mblk_attr(paddr, "0x%lx\n"); +cda_dev_mblk_attr(size, "0x%x\n"); +cda_dev_mblk_attr(req_size, "0x%x\n"); +cda_dev_mblk_attr(owner, "0x%p\n"); +cda_dev_mblk_attr(index, "%d\n"); +#pragma GCC diagnostic warning "-Wformat" + +static struct attribute *mblk_attrs[] = { + &mblk_vaddr_attr.attr, + &mblk_paddr_attr.attr, + &mblk_size_attr.attr, + &mblk_owner_attr.attr, + &mblk_req_size_attr.attr, + &mblk_index_attr.attr, + NULL, +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +ATTRIBUTE_GROUPS(mblk); +#endif +static const struct sysfs_ops mblk_ops = { + .show = mblk_attr_show, +}; + +static const struct kobj_type mblk_type = { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) + .default_groups = mblk_groups, +#else + .default_attrs = mblk_attrs, +#endif + .sysfs_ops = &mblk_ops, + .release = mblk_release, +}; + +static ssize_t mblk_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct cda_mblk *mblk = container_of(kobj, struct cda_mblk, kobj); + struct mblkitem_sysfs_entry *entry = + container_of(attr, struct mblkitem_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(mblk, buf); +} + +static void mblk_release(struct kobject *kobj) +{ + struct cda_mblk *mblk = container_of(kobj, struct cda_mblk, kobj); + + kfree(mblk); +} + +#define to_memmap(obj) container_of(obj, struct cda_mmap, kobj) + +struct memmapitem_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct cda_mmap *, char *); + ssize_t (*store)(struct cda_mmap *, char *, size_t); +}; + +#define cda_dev_memmap_attr(_field, _fmt) \ + static ssize_t \ + memmap_##_field##_show(struct cda_mmap *memmap, char *buf) \ + { \ + return sprintf(buf, _fmt, memmap->_field); \ + } \ + static struct memmapitem_sysfs_entry memmap_##_field##_attr = \ + __ATTR(_field, S_IRUGO, memmap_##_field##_show, NULL) + +#pragma GCC diagnostic ignored "-Wformat" +cda_dev_memmap_attr(owner, "0x%p\n"); +cda_dev_memmap_attr(vaddr, "0x%lx\n"); +cda_dev_memmap_attr(size, "0x%x\n"); +cda_dev_memmap_attr(index, "%d\n"); +cda_dev_memmap_attr(blk_cnt, "%d\n"); + +static ssize_t +memmap_sglist_show(struct cda_mmap *memmap, char *buf) +{ + const int sg_list_item_size = 16 + 8 + 2; //"%016llx %08lx\n" + int res = 0; + int i = memmap->show_cnt; + + memmap->show_cnt = 0; + buf[0] = '\0'; + for ( ; i < memmap->blk_cnt; i++) { + if ((res + sg_list_item_size) >= (PAGE_SIZE - 1)) /* https://lwn.net/Articles/178634/ */{ + memmap->show_cnt = i; + //printk("Split SG list. Next read starts with item: %d\n", i); + break; + } + res += sprintf(&buf[res], "%016llx %08lx\n", memmap->sg_list[i].paddr, memmap->sg_list[i].size); + } + return res; +} + +static struct memmapitem_sysfs_entry memmap_sglist_attr = + __ATTR(sglist, S_IRUGO, memmap_sglist_show, NULL); + +#pragma GCC diagnostic warning "-Wformat" +static struct attribute *memmap_attrs[] = { + &memmap_owner_attr.attr, + &memmap_vaddr_attr.attr, + &memmap_size_attr.attr, + &memmap_index_attr.attr, + &memmap_blk_cnt_attr.attr, + &memmap_sglist_attr.attr, + NULL, +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +ATTRIBUTE_GROUPS(memmap); +#endif + +static ssize_t memmap_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct cda_mmap *memmap = to_memmap(kobj); + struct memmapitem_sysfs_entry *entry = + container_of(attr, struct memmapitem_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(memmap, buf); +} + +static void memmap_release(struct kobject *kobj) +{ + struct cda_mmap *memmap = to_memmap(kobj); + + kfree(memmap); +} + +static const struct sysfs_ops memmap_ops = { + .show = memmap_attr_show, +}; + +static const struct kobj_type memmap_type = { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) + .default_groups = memmap_groups, +#else + .default_attrs = memmap_attrs, +#endif + .sysfs_ops = &memmap_ops, + .release = memmap_release, +}; + +static int mblk_mmap(struct file *file, + struct kobject *kobj, + struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + struct cda_mblk *mblk = attr->private; + unsigned long requested = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + unsigned long pages = (unsigned long)mblk->req_size >> PAGE_SHIFT; + + if (vma->vm_pgoff + requested > pages) + return -EINVAL; + + if (dma_mmap_coherent(&mblk->dev->pcidev->dev, + vma, + mblk->vaddr, + mblk->paddr, + mblk->req_size)) { + dev_err(&mblk->dev->pcidev->dev, "DMA remapping failed"); + return -ENXIO; + } + return 0; +} + +static int cda_publish_mblk(struct cda_mblk *mblk) +{ + int ret; + struct bin_attribute *mmap_attr = &mblk->mmap_attr; + + ret = kobject_add(&mblk->kobj, mblk->dev->kobj_mems, + "%04d", mblk->index); + if (ret) + goto err_add; + + mmap_attr->mmap = mblk_mmap; + mmap_attr->attr.name = "mmap"; + mmap_attr->attr.mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; + mmap_attr->size = mblk->req_size; + mmap_attr->private = mblk; + ret = sysfs_create_bin_file(&mblk->kobj, mmap_attr); + if (ret) + goto err_map_add; + + return 0; + +err_map_add: + kobject_del(&mblk->kobj); +err_add: + kobject_put(&mblk->kobj); + return ret; +} + + +static void cda_hide_mblk(struct cda_mblk *mblk) +{ + sysfs_remove_bin_file(&mblk->kobj, &mblk->mmap_attr); + kobject_del(&mblk->kobj); +} + +static int cda_publish_memmap(struct cda_mmap *memmap) +{ + int ret; + struct bin_attribute *mmap_attr = &memmap->mmap_attr; + + ret = kobject_add(&memmap->kobj, memmap->dev->kobj_mems, + "%04d", memmap->index); + if (ret) + goto err_add; + + mmap_attr->attr.name = "memmapobj"; + mmap_attr->attr.mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; + mmap_attr->size = memmap->size; + mmap_attr->private = memmap; + ret = sysfs_create_bin_file(&memmap->kobj, mmap_attr); + if (ret) + goto err_map_add; + + return 0; + +err_map_add: + kobject_del(&memmap->kobj); +err_add: + kobject_put(&memmap->kobj); + return ret; +} + +static void cda_hide_memmap(struct cda_mmap *memmap) +{ + sysfs_remove_bin_file(&memmap->kobj, &memmap->mmap_attr); + kobject_del(&memmap->kobj); +} + +int cda_alloc_mem(struct cda_dev *dev, void *owner, void __user *ureq) +{ + int ret = -ENOMEM; + int idx; + struct cda_mblk *mblk; + struct cda_alloc_mem req; + + if (copy_from_user(&req, ureq, sizeof(req))) + return -EFAULT; + + mblk = kzalloc(sizeof(*mblk), in_atomic() ? GFP_ATOMIC : GFP_KERNEL); + if (!mblk) + goto out; + INIT_LIST_HEAD(&mblk->list); + mblk->dev = dev; + kobject_init(&mblk->kobj, &mblk_type); + mblk->owner = owner; + mblk->size = req.size; + req.size = ALIGN(req.size, PAGE_SIZE); + + idr_preload(in_atomic() ? GFP_ATOMIC : GFP_KERNEL); + spin_lock(&dev->mblk_sl); + ret = idr_alloc(&dev->mblk_idr, mblk, + 1L, 0, in_atomic() ? GFP_ATOMIC : GFP_KERNEL); + spin_unlock(&dev->mblk_sl); + idr_preload_end(); + if (ret < 0) + goto err_idr; + mblk->index = req.index = idx = ret; + + mblk->vaddr = dma_alloc_coherent( + &dev->pcidev->dev, + req.size, + &mblk->paddr, + in_atomic() ? GFP_ATOMIC | GFP_KERNEL : GFP_KERNEL); + if (!mblk->vaddr) { + dev_err(&dev->dev, "Can't alloc DMA memory (size %u)", req.size); + ret = -1; + goto err_dma_alloc; + } + mblk->req_size = req.size; + + ret = cda_publish_mblk(mblk); + if (ret) { + dev_err(&dev->dev, "Can't publish mblk to sysfs: %d", ret); + goto err_publish; + } + + if (copy_to_user(ureq, &req, sizeof(req))) { + ret = -EFAULT; + goto err_copy_to_user; + } + + spin_lock(&dev->mblk_sl); + list_add(&mblk->list, &dev->mem_blocks); + spin_unlock(&dev->mblk_sl); + + return 0; + +err_copy_to_user: + cda_hide_mblk(mblk); +err_publish: + dma_free_coherent(&dev->pcidev->dev, mblk->req_size, + mblk->vaddr, mblk->paddr); +err_dma_alloc: + spin_lock(&dev->mblk_sl); + idr_remove(&dev->mblk_idr, idx); + spin_unlock(&dev->mblk_sl); +err_idr: + kobject_put(&mblk->kobj); +out: + return ret; +} + +static void cda_free_mem(struct cda_mblk *mblk) +{ + cda_hide_mblk(mblk); + dma_free_coherent(&mblk->dev->pcidev->dev, mblk->req_size, + mblk->vaddr, mblk->paddr); + kobject_put(&mblk->kobj); +} + +int cda_free_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq) +{ + int memidx; + struct cda_mblk *mblk; + + if (copy_from_user(&memidx, (void __user *)ureq, sizeof(memidx))) + return -EFAULT; + + spin_lock(&dev->mblk_sl); + mblk = idr_find(&dev->mblk_idr, memidx); + if (mblk && mblk->index == memidx) { + if (mblk->owner != owner) { + dev_warn(&dev->dev, "Free mblk from another owner\n"); + idr_replace(&dev->mblk_idr, dev->dummy_blk, memidx); + } + list_del(&mblk->list); + } else if (mblk) { + dev_warn(&dev->dev, "Free mblk with index %d, required %d\n", mblk->index, memidx); + } + spin_unlock(&dev->mblk_sl); + if (!mblk) + return -ENOENT; + if (mblk->index) { + cda_free_mem(mblk); + spin_lock(&dev->mblk_sl); + idr_remove(&dev->mblk_idr, mblk->index); + spin_unlock(&dev->mblk_sl); + } + return 0; +} + +void cda_free_dev_mem(struct cda_dev *dev, void *owner) +{ + struct cda_mblk *mblk, *tmp; + LIST_HEAD(mblks); + + spin_lock(&dev->mblk_sl); + if (owner == NULL) { + idr_destroy(&dev->mblk_idr); + list_replace_init(&dev->mem_blocks, &mblks); + } else { + list_for_each_entry_safe(mblk, tmp, &dev->mem_blocks, list) { + if (mblk->index > 0L && mblk->owner == owner) { + list_move(&mblk->list, &mblks); + idr_replace(&dev->mblk_idr, dev->dummy_blk, mblk->index); + } + } + } + spin_unlock(&dev->mblk_sl); + list_for_each_entry_safe(mblk, tmp, &mblks, list) { + // Unmap blocks owned by specified owner or all if owner is NULL + cda_free_mem(mblk); + if (owner != NULL) { + spin_lock(&dev->mblk_sl); + idr_remove(&dev->mblk_idr, mblk->index); + spin_unlock(&dev->mblk_sl); + } + } +} + +static void cda_release_map(struct cda_mmap *memmap) +{ + dma_unmap_sg(memmap->dev->pcidev == NULL ? NULL : &memmap->dev->pcidev->dev, memmap->sgt.sgl, memmap->sgt.orig_nents, DMA_BIDIRECTIONAL); + unpin_user_pages_dirty_lock(memmap->pages, memmap->blk_cnt, 1); + memmap->mapped_blk_cnt = 0; +} + +static int cda_perform_mapping( + struct cda_mmap *memmap) +{ + uint i; + int nents; + struct scatterlist *sg; + ulong len = memmap->size; + void __user *buf = memmap->vaddr; + struct cda_drv_sg_item *cda_sg_list = memmap->sg_list; + + sg = memmap->sgt.sgl; + for (i = 0; i < memmap->sgt.orig_nents; i++, sg = sg_next(sg)) { + unsigned int offset = offset_in_page(buf); + unsigned int nbytes = + min_t(unsigned int, PAGE_SIZE - offset, len); + + sg_set_page(sg, memmap->pages[i], nbytes, offset); + + buf += nbytes; + len -= nbytes; + } + + nents = dma_map_sg(&memmap->dev->pcidev->dev, memmap->sgt.sgl, memmap->sgt.orig_nents, DMA_BIDIRECTIONAL); + if (!nents) { + dev_err(&memmap->dev->dev, "map sgl failed, sgt 0x%p.\n", &memmap->sgt); + return -EIO; + } + memmap->sgt.nents = nents; + + for (i = 0, sg = memmap->sgt.sgl; i < nents; i++, sg = sg_next(sg)) { + cda_sg_list[i].size = sg_dma_len(sg); + cda_sg_list[i].paddr = sg_dma_address(sg); + } + + memmap->mapped_blk_cnt = nents; + return 0; +} + +int cda_map_mem(struct cda_dev *dev, void *owner, void __user *ureq) +{ + int ret = -ENOMEM; + int idx; + int npages; + struct cda_mmap *memmap; + struct cda_map_mem req; + unsigned long offset; + void *req_vaddr; + + if (copy_from_user(&req, ureq, sizeof(req))) + return -EFAULT; + req_vaddr = (void __user *)req.vaddr; + offset = offset_in_page(req_vaddr); + npages = DIV_ROUND_UP(offset + req.size, PAGE_SIZE); + memmap = kzalloc(sizeof(*memmap) + npages * (sizeof(struct cda_drv_sg_item) + sizeof(struct page *)), + in_atomic() ? GFP_ATOMIC : GFP_KERNEL); + if (!memmap) + goto out; + memmap->owner = owner; + memmap->sg_list = (struct cda_drv_sg_item *)((void *)memmap + sizeof(*memmap)); + memmap->pages = (struct page **)((void *)memmap + sizeof(*memmap) + npages * (sizeof(struct cda_drv_sg_item))); + + if (sg_alloc_table(&memmap->sgt, npages, + in_atomic() ? GFP_ATOMIC : GFP_KERNEL)) { + dev_err(&dev->dev, "Can't alloc sg table\n"); + goto out; + } + INIT_LIST_HEAD(&memmap->list); + memmap->dev = dev; + kobject_init(&memmap->kobj, &memmap_type); + + memmap->vaddr = req_vaddr; + memmap->size = req.size; + memmap->blk_cnt = npages; + idr_preload(in_atomic() ? GFP_ATOMIC : GFP_KERNEL); + spin_lock(&dev->mblk_sl); + ret = idr_alloc(&dev->mblk_idr, memmap, + 1L, 0, in_atomic() ? GFP_ATOMIC : GFP_KERNEL); + spin_unlock(&dev->mblk_sl); + idr_preload_end(); + if (ret < 0) + goto err_idr; + memmap->index = req.index = idx = ret; + + ret = pin_user_pages_fast((ulong)req_vaddr, npages, + FOLL_WRITE, memmap->pages); + if (ret < 0) { + dev_err(&dev->pcidev->dev, + "Pin user pages failed for addr=0x%p [ret=%d]\n", + req_vaddr, ret); + goto err_pin; + } + if (ret != npages) { + dev_err(&dev->pcidev->dev, + "Unable to pin all user pages for addr=0x%p\n", req_vaddr); + ret = -EFAULT; + goto err_pin; + } + + ret = cda_perform_mapping(memmap); + if (ret) { + dev_err(&dev->dev, "Can't map user memory for DMA (size %u)", req.size); + goto err_dma_alloc; + } + + ret = cda_publish_memmap(memmap); + if (ret) { + dev_err(&dev->dev, "Can't publish memmap to sysfs: %d", ret); + goto err_publish; + } + + if (copy_to_user(ureq, &req, sizeof(req))) { + ret = -EFAULT; + goto err_copy_to_user; + } + + spin_lock(&dev->mblk_sl); + list_add(&memmap->list, &dev->mem_maps); + spin_unlock(&dev->mblk_sl); + + dev_dbg(&dev->dev, "map vaddr %p, pages %d\n", memmap->vaddr, npages); + return 0; + +err_copy_to_user: + cda_hide_memmap(memmap); +err_publish: + cda_release_map(memmap); +err_dma_alloc: + unpin_user_pages_dirty_lock(memmap->pages, memmap->blk_cnt, 1); +err_pin: + spin_lock(&dev->mblk_sl); + idr_remove(&dev->mblk_idr, idx); + spin_unlock(&dev->mblk_sl); +err_idr: + kobject_put(&memmap->kobj); +out: + if (memmap) { + kfree(memmap->pages); + kfree(memmap); + } + return ret; +} + +static void cda_free_map(struct cda_mmap *memmap) +{ + dev_dbg(&memmap->dev->dev, "unmap vaddr %p, pages %d\n", memmap->vaddr, memmap->blk_cnt); + cda_hide_memmap(memmap); + cda_release_map(memmap); + kobject_put(&memmap->kobj); +} + +int cda_unmap_mem_by_idx(struct cda_dev *dev, void *owner, void __user *ureq) +{ + int memidx; + struct cda_mmap *memmap; + + if (copy_from_user(&memidx, (void __user *)ureq, sizeof(memidx))) + return -EFAULT; + + spin_lock(&dev->mblk_sl); + memmap = idr_find(&dev->mblk_idr, memidx); + if (memmap && memmap->index == memidx) { + if (memmap->owner != owner) + dev_warn(&dev->dev, "Unmap buffer by another user\n"); + idr_replace(&dev->mblk_idr, dev->dummy_blk, memidx); + list_del(&memmap->list); + } else if (memmap) + dev_warn(&dev->dev, "Unmap buffer with index %d, required %d\n", memmap->index, memidx); + spin_unlock(&dev->mblk_sl); + + if (!memmap) + return -ENOENT; // Somebody may already release this block in parallel + + if (memmap->index) { + cda_free_map(memmap); + spin_lock(&dev->mblk_sl); + idr_remove(&dev->mblk_idr, memmap->index); + spin_unlock(&dev->mblk_sl); + } + return 0; +} + +void cda_unmmap_dev_mem(struct cda_dev *dev, void *owner) +{ + struct cda_mmap *memmap, *tmp; + LIST_HEAD(memmaps); + + spin_lock(&dev->mblk_sl); + if (owner == NULL) { + list_replace_init(&dev->mem_maps, &memmaps); + } else { + list_for_each_entry_safe(memmap, tmp, &dev->mem_maps, list) { + if (memmap->index > 0L && memmap->owner == owner) { + idr_replace(&dev->mblk_idr, dev->dummy_blk, memmap->index); + list_move(&memmap->list, &memmaps); + } + } + } + spin_unlock(&dev->mblk_sl); + list_for_each_entry_safe(memmap, tmp, &memmaps, list) { + // Unmap blocks owned by specified owner or all if owner is NULL + cda_free_map(memmap); + if (owner != NULL) { + spin_lock(&dev->mblk_sl); + idr_remove(&dev->mblk_idr, memmap->index); + spin_unlock(&dev->mblk_sl); + } + } +} + +int cda_mems_create(struct cda_dev *cdadev) +{ + int ret; + + ret = sysfs_create_group(&cdadev->dev.kobj, &cda_attr_grp); + if (ret) + goto err_group; + + cdadev->kobj_mems = kobject_create_and_add("mems", &cdadev->dev.kobj); + if (!cdadev->kobj_mems) + goto err_mems; + return 0; + +err_mems: + sysfs_remove_group(&cdadev->dev.kobj, &cda_attr_grp); +err_group: + dev_err(&cdadev->dev, "Couldn't create sysfs files: %d\n", ret); + return ret; +} + +void cda_mems_release(struct cda_dev *dev) +{ + //cda_release_bars(dev); + kobject_del(dev->kobj_mems); + kobject_put(dev->kobj_mems); + sysfs_remove_group(&dev->dev.kobj, &cda_attr_grp); +} diff --git a/src/cdares.c b/src/cdares.c index 2a75df9..102dcfc 100644 --- a/src/cdares.c +++ b/src/cdares.c @@ -1,565 +1,578 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. -// -// CDA linux driver mem blocks/mem maps and interrupt request handler -// -// This program is free software; you can redistribute it and/or modify it -// under the terms and conditions of the GNU General Public License, -// version 2, as published by the Free Software Foundation. -// -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/pci.h> -#include <linux/interrupt.h> -#include <linux/sched.h> -#include <linux/delay.h> -#include <linux/uaccess.h> - -#include "cdadrv.h" -#include "cdaioctl.h" - -struct cda_vector { - volatile bool busy; - wait_queue_head_t wait; - atomic_t count; - unsigned irq; -}; - -struct cda_interrupts { - int num; - enum int_type type; - void *owner; - struct cda_vector *vecs; - struct msix_entry *msix_entries; -}; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) -struct cda_bar { - struct kobject kobj; - /* struct resource *res; */ - int index; - phys_addr_t paddr; - phys_addr_t len; - void *vaddr; - struct cda_dev *dev; - struct bin_attribute mmap_attr; -}; -#else -#if defined __has_attribute -# if __has_attribute (__fallthrough__) -# define fallthrough __attribute__((__fallthrough__)) -# endif -#else -# define fallthrough do {} while (0) /* fallthrough */ -#endif //has_attribute -#endif // LINUX_VERSION_CODE - -static int cda_alloc_msix(struct cda_dev *cdadev, uint32_t rvecs, struct cda_interrupts *ints) -{ - int i, ret; - struct msix_entry *entries; - entries = kcalloc(rvecs, sizeof(struct msix_entry), GFP_KERNEL); - if( !entries ) { - return -ENOMEM; - } - - for (i = 0; i < rvecs; i++) { - entries[i].entry = i; - entries[i].vector = 0; - } - - ret = pci_enable_msix_exact(cdadev->pcidev, entries, rvecs); - if( !ret ) { - ints->num = rvecs; - ints->msix_entries = entries; - ints->type = MSIX; - } else { - kfree(entries); - } - return ret; -} - -static irqreturn_t cda_isr(int irq, void *priv) -{ - struct cda_vector *vec = priv; - atomic_inc_return(&vec->count); - wake_up(&vec->wait); - return IRQ_HANDLED; -} - -int cda_init_interrupts(struct cda_dev *cdadev, void *owner, void __user *ureq) -{ - int ret = 0; - int nvecs, i; - struct cda_vector *vec; - struct cda_int_lock req; - struct cda_interrupts *ints; - - if( cdadev->ints ) { - dev_dbg(&cdadev->pcidev->dev, "Interrupts are already attached"); - return -EINVAL; // Already attached - } - if( copy_from_user(&req, (void __user *)ureq, sizeof(req)) ) - return -EFAULT; - - ints = kcalloc(1, sizeof(struct cda_interrupts), GFP_KERNEL); - if( !ints ) - return -ENOMEM; - - ints->owner = owner; - switch( req.inttype ) { - case MSIX: - ret = cda_alloc_msix(cdadev, req.vectors, ints); - if( !ret ) { - nvecs = req.vectors; - break; - } - if( ret == -ENOMEM ) { - kfree(ints); - return ret; - } - dev_warn(&cdadev->pcidev->dev, "No MSI-X vectors, try MSI. Error %x\n", ret); - fallthrough; - case MSI: -#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) - nvecs = pci_alloc_irq_vectors(cdadev->pcidev, 1, req.vectors, PCI_IRQ_MSI); -#else - nvecs = pci_alloc_irq_vectors_affinity(cdadev->pcidev, 1, req.vectors, PCI_IRQ_MSI, NULL); -#endif - if( nvecs > 0 ) { - ints->num = nvecs; - ints->type = MSI; - break; - } - dev_warn(&cdadev->pcidev->dev, "No MSI vectors, try legacy. Error %x\n", nvecs); - fallthrough; - case LEGACY_INTERRUPT: - ints->num = 1; - ints->type = LEGACY_INTERRUPT; - break; - } - - ints->vecs = kcalloc(ints->num, sizeof(struct cda_vector), GFP_KERNEL); - if( !ints->vecs ) { - ret = -ENOMEM; - goto err_alloc_vecs; - } - - for( i = 0; i < ints->num; i++ ) { - char name[10]; - vec = &ints->vecs[i]; - vec->irq = ints->type == MSIX ? - cdadev->ints->msix_entries[i].vector : - cdadev->pcidev->irq + i; - snprintf(name, sizeof(name), "cda%02d-%x", cdadev->minor, i); - init_waitqueue_head(&vec->wait); - atomic_set(&vec->count, 0); - ret = request_irq(vec->irq, cda_isr, ints->type == LEGACY_INTERRUPT ? IRQF_SHARED : 0, name, vec); - if( ret ) { - dev_err(&cdadev->pcidev->dev, "request_irq failed for vector %d: %d", i, ret); - break; - } - } - - // Return interrupt type and vector count to user - if( !ret ) { - req.inttype = ints->type; - req.vectors = ints->num; - if( copy_to_user(ureq, &req, sizeof(req)) ) - ret = -EFAULT; - } - - if( !ret ) { - cdadev->ints = ints; - return ret; - } - // Fail. Release - for( i -= 1; i >= 0; i-- ) { - struct cda_vector *vec = &ints->vecs[i]; - free_irq(vec->irq, vec); - } - kfree(ints->vecs); - -err_alloc_vecs: - pci_free_irq_vectors(cdadev->pcidev); - kfree(ints->msix_entries); - kfree(ints); - - return ret; -} - -int cda_free_irqs(struct cda_dev *cdadev, void *owner) -{ - int i; - struct cda_interrupts *ints; - if( cdadev->ints == NULL ) - return -EINVAL; - if( cdadev->ints->owner != owner ) { - dev_dbg(&cdadev->pcidev->dev, "Interrupts are not owned by %p", owner); - return -EINVAL; - } - mutex_lock(&cdadev->ilock); - ints = cdadev->ints; - cdadev->ints = NULL; - mutex_unlock(&cdadev->ilock); - if( ints && ints->num > 0 ) { - for( i = 0; i < ints->num; i++ ) { - struct cda_vector *vec = &ints->vecs[i]; - while( vec->busy ) { - wake_up(&vec->wait); - udelay(1); - } - free_irq(vec->irq, vec); - } - pci_free_irq_vectors(cdadev->pcidev); - kfree(ints->vecs); - kfree(ints->msix_entries); - kfree(ints); - } - return 0; -} - -int cda_req_int(struct cda_dev *cdadev, void *owner, void __user *ureq) -{ - struct cda_interrupts *ints; - struct cda_req_int req; - struct cda_vector *vec; - unsigned long timeout; - unsigned count; - - if (copy_from_user(&req, ureq, sizeof(req))) - return -EFAULT; - - if( cdadev->ints == NULL ) - return -EINVAL; - - if( cdadev->ints->owner != owner ) { - dev_err(&cdadev->pcidev->dev, "Interrupts are not owned by %p", owner); - return -EINVAL; - } - - mutex_lock(&cdadev->ilock); - ints = cdadev->ints; - if( !ints || (req.vector > ints->num) ) { - mutex_unlock(&cdadev->ilock); - return -EINVAL; - } - - vec = &ints->vecs[req.vector]; - if (req.reset) - atomic_set(&vec->count, 0); - - timeout = nsecs_to_jiffies(req.timeout); - count = atomic_xchg(&vec->count, 0); - if( !count ) - { - vec->busy = true; - mutex_unlock(&cdadev->ilock); - timeout = wait_event_interruptible_timeout(vec->wait, - (count = atomic_xchg(&vec->count, 0)), - timeout); - mutex_lock(&cdadev->ilock); - vec->busy = false; - } - mutex_unlock(&cdadev->ilock); - dev_dbg(&cdadev->pcidev->dev, "Interrupt vector %d timeout: %ld count %u reset %d\n", req.vector, timeout, count, req.reset); - return timeout > 0 ? 0 : timeout == 0 ? -ETIME : timeout; -} - -int cda_cancel_req(struct cda_dev *cdadev, void *owner) -{ - int i; - struct cda_interrupts *ints; - if( cdadev->ints == NULL ) - return -EINVAL; - - if( cdadev->ints->owner != owner ) { - dev_dbg(&cdadev->pcidev->dev, "Interrupts are not owned by %p", owner); - return -EINVAL; - } - - mutex_lock(&cdadev->ilock); - ints = cdadev->ints; - for (i = 0; ints && i < ints->num; i++) { - if( ints->vecs[i].busy ) - wake_up(&ints->vecs[i].wait); - } - mutex_unlock(&cdadev->ilock); - return 0; -} - -int cda_sem_aq(struct cda_dev *cdadev, void *owner, void __user *ureq) -{ - int res = 0; - struct cda_sem_aq req; - u64 cur_time; - if (copy_from_user(&req, ureq, sizeof(req))) - return -EFAULT; - - mutex_lock(&cdadev->ilock); - cur_time = ktime_get_ns(); - if( cdadev->semaphores[req.sem_id] < cur_time ) { - cdadev->semaphores[req.sem_id] = cur_time + req.time_ns > cur_time ? cur_time + req.time_ns : 0xFFFFFFFFFFFFFFFFULL; - cdadev->sem_owner[req.sem_id] = owner; - } else { - res = 1; - } - mutex_unlock(&cdadev->ilock); - return res; -} - -int cda_sem_rel(struct cda_dev *cdadev, void *owner, void __user *ureq) -{ - int res = 0; - int req_sem; - if (copy_from_user(&req_sem, ureq, sizeof(req_sem))) - return -EFAULT; - if( cdadev->sem_owner[req_sem] != owner ) { - dev_warn(&cdadev->pcidev->dev, "Semaphore %d is not owned by %p", req_sem, owner); - } else { - mutex_lock(&cdadev->ilock); - cdadev->semaphores[req_sem] = 0ULL; - cdadev->sem_owner[req_sem] = NULL; - mutex_unlock(&cdadev->ilock); - } - return res; -} - -void cda_sem_rel_by_owner(struct cda_dev *dev, void *owner) -{ - uint32_t i; - mutex_lock(&dev->ilock); - for( i = 0; i < CDA_MAX_DRV_SEMAPHORES; i++ ) { - if( dev->sem_owner[i] == owner ) { - dev->semaphores[i] = 0ULL; - dev->sem_owner[i] = NULL; - } - } - mutex_unlock(&dev->ilock); -} - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) -#define to_bar(obj) container_of((obj), struct cda_bar, kobj) -struct bar_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct cda_bar *, char *); - ssize_t (*store)(struct cda_bar *, char*, size_t); -}; - -#define cdadev_bar_attr(_field, _fmt) \ - static ssize_t \ - bar_##_field##_show(struct cda_bar *bar, char *buf) \ - { \ - return sprintf(buf, _fmt, bar->_field); \ - } \ - static struct bar_sysfs_entry bar_##_field##_attr = \ - __ATTR(_field, S_IRUGO, bar_##_field##_show, NULL); - -#pragma GCC diagnostic ignored "-Wformat" -cdadev_bar_attr(paddr, "0x%lx\n"); -cdadev_bar_attr(len, "0x%lx\n"); -cdadev_bar_attr(index, "%d\n"); -#pragma GCC diagnostic warning "-Wformat" - -static ssize_t bar_attr_show(struct kobject *kobj, struct attribute *attr, - char *buf) -{ - struct cda_bar *bar = to_bar(kobj); - struct bar_sysfs_entry *entry = - container_of(attr, struct bar_sysfs_entry, attr); - - if (!entry->show) - return -EIO; - - return entry->show(bar, buf); -} - -static const struct sysfs_ops bar_ops = { - .show = bar_attr_show, -}; - -static void bar_release(struct kobject *kobj) -{ - struct cda_bar *bar = to_bar(kobj); - kfree(bar); -} - -static struct attribute *bar_attrs[] = { - &bar_paddr_attr.attr, - &bar_len_attr.attr, - &bar_index_attr.attr, - NULL, -}; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) -ATTRIBUTE_GROUPS(bar); -#endif - -static struct kobj_type bar_type = { - .sysfs_ops = &bar_ops, - .release = bar_release, -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,18,0) - .default_groups = bar_groups, -#else - .default_attrs = bar_attrs, -#endif -}; - -// Secure enable support -static const struct vm_operations_struct pci_phys_vm_ops = { -#ifdef CONFIG_HAVE_IOREMAP_PROT - .access = generic_access_phys, -#endif -}; - -static int bar_mmap( struct file *file, - struct kobject *kobj, - struct bin_attribute *attr, - struct vm_area_struct *vma) -{ - struct cda_bar *bar = attr->private; - unsigned long requested = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - unsigned long pages = (unsigned long)bar->len >> PAGE_SHIFT; - unsigned long size; - - if (vma->vm_pgoff + requested > pages) - return -EINVAL; - - size = ((pci_resource_len(bar->dev->pcidev, bar->index) - 1) >> PAGE_SHIFT) + 1; - if (vma->vm_pgoff + vma_pages(vma) > size) - return -EINVAL; - - vma->vm_page_prot = pgprot_device(vma->vm_page_prot); - vma->vm_pgoff += (pci_resource_start(bar->dev->pcidev, bar->index) >> PAGE_SHIFT); - vma->vm_ops = &pci_phys_vm_ops; - - return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); -} - -int cda_open_bars(struct cda_dev *cdadev) -{ - int i; - int ret; - struct cda_bar *bar; - struct resource *res_child; - int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); - - ret = -EINVAL; - cdadev->kobj_bars = kobject_create_and_add("bars", &cdadev->dev.kobj); - if (!cdadev->kobj_bars) - goto err; - - for (i = 0; bars && i < PCI_ROM_RESOURCE; bars >>= 1, i++){ - struct bin_attribute *mmap_attr; - if (!(bars & 1)) - continue; - - if( !(pci_resource_flags(cdadev->pcidev, i) & IORESOURCE_MEM) ) - continue; - - ret = -ENOMEM; - bar = kzalloc(sizeof(*bar), GFP_KERNEL); - if (!bar) - goto err; - bar->index = i; - bar->paddr = pci_resource_start(cdadev->pcidev, i); - bar->len = pci_resource_len(cdadev->pcidev, i); - bar->vaddr = NULL; - bar->dev = cdadev; - cdadev->sysfs_bar[i] = bar; - kobject_init(&bar->kobj, &bar_type); - - if( (bar->vaddr = pci_iomap(cdadev->pcidev, i, bar->len)) == NULL ) - goto err; - ret = kobject_add(&bar->kobj, cdadev->kobj_bars, "mmio_bar%d", i); - if (ret) - goto err; - - mmap_attr = &bar->mmap_attr; - mmap_attr->mmap = bar_mmap; - mmap_attr->attr.name = "mmap"; - mmap_attr->attr.mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - mmap_attr->size = bar->len; - mmap_attr->private = bar; - - ret = sysfs_create_bin_file(&bar->kobj, mmap_attr); - if (ret) { - dev_err(&cdadev->dev, "Can't create kobject file for mmap"); - goto err; - } - - //Drop busy bit - res_child = cdadev->pcidev->resource[i].child; - - dev_info(&cdadev->dev, "Store resource %d flag: 0x%lx\n", i, res_child->flags); - cdadev->stored_flags[i] = res_child->flags; - if (IORESOURCE_BUSY & res_child->flags) { - res_child->flags &= ~IORESOURCE_BUSY; - } - } - return 0; - -err: - cda_release_bars(cdadev); - return ret; -} - -void cda_release_bars(struct cda_dev *cdadev) -{ - int i; - int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); - for( i = 0; i < PCI_ROM_RESOURCE; i++ ) { - struct cda_bar *bar = cdadev->sysfs_bar[i]; - if (!bar) - continue; - - cdadev->sysfs_bar[i] = NULL; - sysfs_remove_bin_file(&bar->kobj, &bar->mmap_attr); - kobject_del(&bar->kobj); - kobject_put(&bar->kobj); - - if( bars & (1 << i) ) { - cdadev->pcidev->resource[i].child->flags = cdadev->stored_flags[i]; - dev_info(&cdadev->dev, "Restore resource %d flag: %lx\n", i, cdadev->stored_flags[i]); - } - } - - kobject_del(cdadev->kobj_bars); - kobject_put(cdadev->kobj_bars); -} -#else // LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) -int cda_open_bars(struct cda_dev *cdadev) -{ - int i; - struct resource *res_child; - int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); - - for( i = 0; i < PCI_ROM_RESOURCE; i++ ) { - // Drop busy bit - if( bars & (1 << i) ) { - res_child = cdadev->pcidev->resource[i].child; - cdadev->stored_flags[i] = res_child->flags; - dev_info(&cdadev->dev, "Store resource %d flag: 0x%lx\n", i, res_child->flags); - if( IORESOURCE_BUSY & res_child->flags ) { - res_child->flags &= ~IORESOURCE_BUSY; - dev_dbg(&cdadev->dev, "Drop busy bit for resource %d", i); - } - } - } - return 0; -} - -void cda_release_bars(struct cda_dev *cdadev) -{ - int i; - int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); - for( i = 0; i < PCI_ROM_RESOURCE; i++ ) { - if( bars & (1 << i) ) { - cdadev->pcidev->resource[i].child->flags = cdadev->stored_flags[i]; - dev_info(&cdadev->dev, "Restore resource %d flag: %lx\n", i, cdadev->stored_flags[i]); - } - } -} -#endif +// SPDX-License-Identifier: GPL-2.0 +// Copyright(c) 2020 DeGirum Corp., Egor Pomozov. +// +// CDA linux driver mem blocks/mem maps and interrupt request handler +// +// This program is free software; you can redistribute it and/or modify it +// under the terms and conditions of the GNU General Public License, +// version 2, as published by the Free Software Foundation. +// +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/pci.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/uaccess.h> + +#include "cdadrv.h" +#include "cdaioctl.h" + +struct cda_vector { + volatile bool busy; + wait_queue_head_t wait; + atomic_t count; + unsigned int irq; +}; + +struct cda_interrupts { + int num; + enum int_type type; + void *owner; + struct cda_vector *vecs; + struct msix_entry *msix_entries; +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +struct cda_bar { + struct kobject kobj; + /* struct resource *res; */ + int index; + phys_addr_t paddr; + phys_addr_t len; + void *vaddr; + struct cda_dev *dev; + struct bin_attribute mmap_attr; +}; +#else +#if defined __has_attribute +# if __has_attribute(__fallthrough__) +# define fallthrough __attribute__((__fallthrough__)) +# endif +#else +# define fallthrough do {} while (0) /* fallthrough */ +#endif //has_attribute +#endif // LINUX_VERSION_CODE + +static int cda_alloc_msix(struct cda_dev *cdadev, uint32_t rvecs, struct cda_interrupts *ints) +{ + int i, ret; + struct msix_entry *entries; + + entries = kcalloc(rvecs, sizeof(struct msix_entry), GFP_KERNEL); + if (!entries) + return -ENOMEM; + + for (i = 0; i < rvecs; i++) { + entries[i].entry = i; + entries[i].vector = 0; + } + + ret = pci_enable_msix_exact(cdadev->pcidev, entries, rvecs); + if (!ret) { + ints->num = rvecs; + ints->msix_entries = entries; + ints->type = MSIX; + } else { + kfree(entries); + } + return ret; +} + +static irqreturn_t cda_isr(int irq, void *priv) +{ + struct cda_vector *vec = priv; + + atomic_inc_return(&vec->count); + wake_up(&vec->wait); + return IRQ_HANDLED; +} + +int cda_init_interrupts(struct cda_dev *cdadev, void *owner, void __user *ureq) +{ + int ret = 0; + int nvecs, i; + struct cda_vector *vec; + struct cda_int_lock req; + struct cda_interrupts *ints; + + if (cdadev->ints) { + dev_dbg(&cdadev->pcidev->dev, "Interrupts are already attached"); + return -EINVAL; // Already attached + } + if (copy_from_user(&req, (void __user *)ureq, sizeof(req))) + return -EFAULT; + + ints = kcalloc(1, sizeof(struct cda_interrupts), GFP_KERNEL); + if (!ints) + return -ENOMEM; + + ints->owner = owner; + switch (req.inttype) { + case MSIX: + ret = cda_alloc_msix(cdadev, req.vectors, ints); + if (!ret) { + nvecs = req.vectors; + break; + } + if (ret == -ENOMEM) { + kfree(ints); + return ret; + } + dev_warn(&cdadev->pcidev->dev, "No MSI-X vectors, try MSI. Error %x\n", ret); + fallthrough; + case MSI: +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + nvecs = pci_alloc_irq_vectors(cdadev->pcidev, 1, req.vectors, PCI_IRQ_MSI); +#else + nvecs = pci_alloc_irq_vectors_affinity(cdadev->pcidev, 1, req.vectors, PCI_IRQ_MSI, NULL); +#endif + if (nvecs > 0) { + ints->num = nvecs; + ints->type = MSI; + break; + } + dev_warn(&cdadev->pcidev->dev, "No MSI vectors, try legacy. Error %x\n", nvecs); + fallthrough; + case LEGACY_INTERRUPT: + ints->num = 1; + ints->type = LEGACY_INTERRUPT; + break; + } + + ints->vecs = kcalloc(ints->num, sizeof(struct cda_vector), GFP_KERNEL); + if (!ints->vecs) { + ret = -ENOMEM; + goto err_alloc_vecs; + } + + for (i = 0; i < ints->num; i++) { + char name[10]; + + vec = &ints->vecs[i]; + vec->irq = ints->type == MSIX ? + cdadev->ints->msix_entries[i].vector : + cdadev->pcidev->irq + i; + snprintf(name, sizeof(name), "cda%02d-%x", cdadev->minor, i); + init_waitqueue_head(&vec->wait); + atomic_set(&vec->count, 0); + ret = request_irq(vec->irq, cda_isr, ints->type == LEGACY_INTERRUPT ? IRQF_SHARED : 0, name, vec); + if (ret) { + dev_err(&cdadev->pcidev->dev, "request_irq failed for vector %d: %d", i, ret); + break; + } + } + + // Return interrupt type and vector count to user + if (!ret) { + req.inttype = ints->type; + req.vectors = ints->num; + if (copy_to_user(ureq, &req, sizeof(req))) + ret = -EFAULT; + } + + if (!ret) { + cdadev->ints = ints; + return ret; + } + // Fail. Release + for (i -= 1; i >= 0; i--) { + struct cda_vector *vec = &ints->vecs[i]; + + free_irq(vec->irq, vec); + } + kfree(ints->vecs); + +err_alloc_vecs: + pci_free_irq_vectors(cdadev->pcidev); + kfree(ints->msix_entries); + kfree(ints); + + return ret; +} + +int cda_free_irqs(struct cda_dev *cdadev, void *owner) +{ + int i; + struct cda_interrupts *ints; + + if (cdadev->ints == NULL) + return -EINVAL; + if (cdadev->ints->owner != owner) { + dev_dbg(&cdadev->pcidev->dev, "Interrupts are not owned by %p", owner); + return -EINVAL; + } + mutex_lock(&cdadev->ilock); + ints = cdadev->ints; + cdadev->ints = NULL; + mutex_unlock(&cdadev->ilock); + if (ints && ints->num > 0) { + for (i = 0; i < ints->num; i++) { + struct cda_vector *vec = &ints->vecs[i]; + + while (vec->busy) { + wake_up(&vec->wait); + udelay(1); + } + free_irq(vec->irq, vec); + } + pci_free_irq_vectors(cdadev->pcidev); + kfree(ints->vecs); + kfree(ints->msix_entries); + kfree(ints); + } + return 0; +} + +int cda_req_int(struct cda_dev *cdadev, void *owner, void __user *ureq) +{ + struct cda_interrupts *ints; + struct cda_req_int req; + struct cda_vector *vec; + unsigned long timeout; + unsigned int count; + + if (copy_from_user(&req, ureq, sizeof(req))) + return -EFAULT; + + if (cdadev->ints == NULL) + return -EINVAL; + + if (cdadev->ints->owner != owner) { + dev_err(&cdadev->pcidev->dev, "Interrupts are not owned by %p", owner); + return -EINVAL; + } + + mutex_lock(&cdadev->ilock); + ints = cdadev->ints; + if (!ints || (req.vector > ints->num)) { + mutex_unlock(&cdadev->ilock); + return -EINVAL; + } + + vec = &ints->vecs[req.vector]; + if (req.reset) + atomic_set(&vec->count, 0); + + timeout = nsecs_to_jiffies(req.timeout); + count = atomic_xchg(&vec->count, 0); + if (!count) { + vec->busy = true; + mutex_unlock(&cdadev->ilock); + timeout = wait_event_interruptible_timeout(vec->wait, + (count = atomic_xchg(&vec->count, 0)), + timeout); + mutex_lock(&cdadev->ilock); + vec->busy = false; + } + mutex_unlock(&cdadev->ilock); + dev_dbg(&cdadev->pcidev->dev, "Interrupt vector %d timeout: %ld count %u reset %d\n", req.vector, timeout, count, req.reset); + return timeout > 0 ? 0 : timeout == 0 ? -ETIME : timeout; +} + +int cda_cancel_req(struct cda_dev *cdadev, void *owner) +{ + int i; + struct cda_interrupts *ints; + + if (cdadev->ints == NULL) + return -EINVAL; + + if (cdadev->ints->owner != owner) { + dev_dbg(&cdadev->pcidev->dev, "Interrupts are not owned by %p", owner); + return -EINVAL; + } + + mutex_lock(&cdadev->ilock); + ints = cdadev->ints; + for (i = 0; ints && i < ints->num; i++) { + if (ints->vecs[i].busy) + wake_up(&ints->vecs[i].wait); + } + mutex_unlock(&cdadev->ilock); + return 0; +} + +int cda_sem_aq(struct cda_dev *cdadev, void *owner, void __user *ureq) +{ + int res = 0; + struct cda_sem_aq req; + u64 cur_time; + + if (copy_from_user(&req, ureq, sizeof(req))) + return -EFAULT; + + mutex_lock(&cdadev->ilock); + cur_time = ktime_get_ns(); + if (cdadev->semaphores[req.sem_id] < cur_time) { + cdadev->semaphores[req.sem_id] = cur_time + req.time_ns > cur_time ? cur_time + req.time_ns : 0xFFFFFFFFFFFFFFFFULL; + cdadev->sem_owner[req.sem_id] = owner; + } else { + res = 1; + } + mutex_unlock(&cdadev->ilock); + return res; +} + +int cda_sem_rel(struct cda_dev *cdadev, void *owner, void __user *ureq) +{ + int res = 0; + int req_sem; + + if (copy_from_user(&req_sem, ureq, sizeof(req_sem))) + return -EFAULT; + if (cdadev->sem_owner[req_sem] != owner) { + dev_warn(&cdadev->pcidev->dev, "Semaphore %d is not owned by %p", req_sem, owner); + } else { + mutex_lock(&cdadev->ilock); + cdadev->semaphores[req_sem] = 0ULL; + cdadev->sem_owner[req_sem] = NULL; + mutex_unlock(&cdadev->ilock); + } + return res; +} + +void cda_sem_rel_by_owner(struct cda_dev *dev, void *owner) +{ + uint32_t i; + + mutex_lock(&dev->ilock); + for (i = 0; i < CDA_MAX_DRV_SEMAPHORES; i++) { + if (dev->sem_owner[i] == owner) { + dev->semaphores[i] = 0ULL; + dev->sem_owner[i] = NULL; + } + } + mutex_unlock(&dev->ilock); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +#define to_bar(obj) container_of((obj), struct cda_bar, kobj) +struct bar_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct cda_bar *, char *); + ssize_t (*store)(struct cda_bar *, char*, size_t); +}; + +#define cdadev_bar_attr(_field, _fmt) \ + static ssize_t \ + bar_##_field##_show(struct cda_bar *bar, char *buf) \ + { \ + return sprintf(buf, _fmt, bar->_field); \ + } \ + static struct bar_sysfs_entry bar_##_field##_attr = \ + __ATTR(_field, S_IRUGO, bar_##_field##_show, NULL) + +#pragma GCC diagnostic ignored "-Wformat" +cdadev_bar_attr(paddr, "0x%lx\n"); +cdadev_bar_attr(len, "0x%lx\n"); +cdadev_bar_attr(index, "%d\n"); +#pragma GCC diagnostic warning "-Wformat" + +static ssize_t bar_attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct cda_bar *bar = to_bar(kobj); + struct bar_sysfs_entry *entry = + container_of(attr, struct bar_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(bar, buf); +} + +static const struct sysfs_ops bar_ops = { + .show = bar_attr_show, +}; + +static void bar_release(struct kobject *kobj) +{ + struct cda_bar *bar = to_bar(kobj); + + kfree(bar); +} + +static struct attribute *bar_attrs[] = { + &bar_paddr_attr.attr, + &bar_len_attr.attr, + &bar_index_attr.attr, + NULL, +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +ATTRIBUTE_GROUPS(bar); +#endif + +static const struct kobj_type bar_type = { + .sysfs_ops = &bar_ops, + .release = bar_release, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) + .default_groups = bar_groups, +#else + .default_attrs = bar_attrs, +#endif +}; + +// Secure enable support +static const struct vm_operations_struct pci_phys_vm_ops = { +#ifdef CONFIG_HAVE_IOREMAP_PROT + .access = generic_access_phys, +#endif +}; + +static int bar_mmap(struct file *file, + struct kobject *kobj, + struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + struct cda_bar *bar = attr->private; + unsigned long requested = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + unsigned long pages = (unsigned long)bar->len >> PAGE_SHIFT; + unsigned long size; + + if (vma->vm_pgoff + requested > pages) + return -EINVAL; + + size = ((pci_resource_len(bar->dev->pcidev, bar->index) - 1) >> PAGE_SHIFT) + 1; + if (vma->vm_pgoff + vma_pages(vma) > size) + return -EINVAL; + + vma->vm_page_prot = pgprot_device(vma->vm_page_prot); + vma->vm_pgoff += (pci_resource_start(bar->dev->pcidev, bar->index) >> PAGE_SHIFT); + vma->vm_ops = &pci_phys_vm_ops; + + return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); +} + +int cda_open_bars(struct cda_dev *cdadev) +{ + int i; + int ret; + struct cda_bar *bar; + struct resource *res_child; + int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); + + ret = -EINVAL; + cdadev->kobj_bars = kobject_create_and_add("bars", &cdadev->dev.kobj); + if (!cdadev->kobj_bars) + goto err; + + for (i = 0; bars && i < PCI_ROM_RESOURCE; bars >>= 1, i++) { + struct bin_attribute *mmap_attr; + + if (!(bars & 1)) + continue; + + if (!(pci_resource_flags(cdadev->pcidev, i) & IORESOURCE_MEM)) + continue; + + ret = -ENOMEM; + bar = kzalloc(sizeof(*bar), GFP_KERNEL); + if (!bar) + goto err; + bar->index = i; + bar->paddr = pci_resource_start(cdadev->pcidev, i); + bar->len = pci_resource_len(cdadev->pcidev, i); + bar->vaddr = NULL; + bar->dev = cdadev; + cdadev->sysfs_bar[i] = bar; + kobject_init(&bar->kobj, &bar_type); + + bar->vaddr = pci_iomap(cdadev->pcidev, i, bar->len); + if (!bar->vaddr) + goto err; + ret = kobject_add(&bar->kobj, cdadev->kobj_bars, "mmio_bar%d", i); + if (ret) + goto err; + + mmap_attr = &bar->mmap_attr; + mmap_attr->mmap = bar_mmap; + mmap_attr->attr.name = "mmap"; + mmap_attr->attr.mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; + mmap_attr->size = bar->len; + mmap_attr->private = bar; + + ret = sysfs_create_bin_file(&bar->kobj, mmap_attr); + if (ret) { + dev_err(&cdadev->dev, "Can't create kobject file for mmap"); + goto err; + } + + //Drop busy bit + res_child = cdadev->pcidev->resource[i].child; + + dev_info(&cdadev->dev, "Store resource %d flag: 0x%lx\n", i, res_child->flags); + cdadev->stored_flags[i] = res_child->flags; + if (IORESOURCE_BUSY & res_child->flags) + res_child->flags &= ~IORESOURCE_BUSY; + } + return 0; + +err: + cda_release_bars(cdadev); + return ret; +} + +void cda_release_bars(struct cda_dev *cdadev) +{ + int i; + int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); + + for (i = 0; i < PCI_ROM_RESOURCE; i++) { + struct cda_bar *bar = cdadev->sysfs_bar[i]; + + if (!bar) + continue; + + cdadev->sysfs_bar[i] = NULL; + sysfs_remove_bin_file(&bar->kobj, &bar->mmap_attr); + kobject_del(&bar->kobj); + kobject_put(&bar->kobj); + + if (bars & (1 << i)) { + cdadev->pcidev->resource[i].child->flags = cdadev->stored_flags[i]; + dev_info(&cdadev->dev, "Restore resource %d flag: %lx\n", i, cdadev->stored_flags[i]); + } + } + + kobject_del(cdadev->kobj_bars); + kobject_put(cdadev->kobj_bars); +} +#else // LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +int cda_open_bars(struct cda_dev *cdadev) +{ + int i; + struct resource *res_child; + int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); + + for (i = 0; i < PCI_ROM_RESOURCE; i++) { + // Drop busy bit + if (bars & (1 << i)) { + res_child = cdadev->pcidev->resource[i].child; + cdadev->stored_flags[i] = res_child->flags; + dev_info(&cdadev->dev, "Store resource %d flag: 0x%lx\n", i, res_child->flags); + if (IORESOURCE_BUSY & res_child->flags) { + res_child->flags &= ~IORESOURCE_BUSY; + dev_dbg(&cdadev->dev, "Drop busy bit for resource %d", i); + } + } + } + return 0; +} + +void cda_release_bars(struct cda_dev *cdadev) +{ + int i; + int bars = pci_select_bars(cdadev->pcidev, IORESOURCE_MEM); + + for (i = 0; i < PCI_ROM_RESOURCE; i++) { + if (bars & (1 << i)) { + cdadev->pcidev->resource[i].child->flags = cdadev->stored_flags[i]; + dev_info(&cdadev->dev, "Restore resource %d flag: %lx\n", i, cdadev->stored_flags[i]); + } + } +} +#endif
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
