forked from Freescale/linux-fslc
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
drm/panfrost: Add support for GPU heap allocations
The midgard/bifrost GPUs need to allocate GPU heap memory which is allocated on GPU page faults and not pinned in memory. The vendor driver calls this functionality GROW_ON_GPF. This implementation assumes that BOs allocated with the PANFROST_BO_NOEXEC flag are never mmapped or exported. Both of those may actually work, but I'm unsure if there's some interaction there. It would cause the whole object to be pinned in memory which would defeat the point of this. On faults, we map in 2MB at a time in order to utilize huge pages (if enabled). Currently, once we've mapped pages in, they are only unmapped if the BO is freed. Once we add shrinker support, we can unmap pages with the shrinker. Cc: Tomeu Vizoso <[email protected]> Cc: Boris Brezillon <[email protected]> Cc: Robin Murphy <[email protected]> Acked-by: Alyssa Rosenzweig <[email protected]> Reviewed-by: Steven Price <[email protected]> Signed-off-by: Rob Herring <[email protected]> Link: https://patchwork.freedesktop.org/patch/msgid/[email protected]
- Loading branch information
1 parent
b31bdd1
commit 187d292
Showing
6 changed files
with
172 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,13 +2,15 @@ | |
| /* Copyright 2019 Linaro, Ltd, Rob Herring <[email protected]> */ | ||
| #include <linux/bitfield.h> | ||
| #include <linux/delay.h> | ||
| #include <linux/dma-mapping.h> | ||
| #include <linux/interrupt.h> | ||
| #include <linux/io.h> | ||
| #include <linux/iopoll.h> | ||
| #include <linux/io-pgtable.h> | ||
| #include <linux/iommu.h> | ||
| #include <linux/platform_device.h> | ||
| #include <linux/pm_runtime.h> | ||
| #include <linux/shmem_fs.h> | ||
| #include <linux/sizes.h> | ||
|
|
||
| #include "panfrost_device.h" | ||
|
|
@@ -240,12 +242,12 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) | |
| size_t unmapped_page; | ||
| size_t pgsize = get_pgsize(iova, len - unmapped_len); | ||
|
|
||
| unmapped_page = ops->unmap(ops, iova, pgsize); | ||
| if (!unmapped_page) | ||
| break; | ||
|
|
||
| iova += unmapped_page; | ||
| unmapped_len += unmapped_page; | ||
| if (ops->iova_to_phys(ops, iova)) { | ||
| unmapped_page = ops->unmap(ops, iova, pgsize); | ||
| WARN_ON(unmapped_page != pgsize); | ||
| } | ||
| iova += pgsize; | ||
| unmapped_len += pgsize; | ||
| } | ||
|
|
||
| mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, | ||
|
|
@@ -281,6 +283,105 @@ static const struct iommu_gather_ops mmu_tlb_ops = { | |
| .tlb_sync = mmu_tlb_sync_context, | ||
| }; | ||
|
|
||
| static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr) | ||
| { | ||
| struct drm_mm_node *node; | ||
| u64 offset = addr >> PAGE_SHIFT; | ||
|
|
||
| drm_mm_for_each_node(node, &pfdev->mm) { | ||
| if (offset >= node->start && offset < (node->start + node->size)) | ||
| return node; | ||
| } | ||
| return NULL; | ||
| } | ||
|
|
||
| #define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE) | ||
|
|
||
| int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) | ||
| { | ||
| int ret, i; | ||
| struct drm_mm_node *node; | ||
| struct panfrost_gem_object *bo; | ||
| struct address_space *mapping; | ||
| pgoff_t page_offset; | ||
| struct sg_table *sgt; | ||
| struct page **pages; | ||
|
|
||
| node = addr_to_drm_mm_node(pfdev, as, addr); | ||
| if (!node) | ||
| return -ENOENT; | ||
|
|
||
| bo = drm_mm_node_to_panfrost_bo(node); | ||
| if (!bo->is_heap) { | ||
| dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)", | ||
| node->start << PAGE_SHIFT); | ||
| return -EINVAL; | ||
| } | ||
| /* Assume 2MB alignment and size multiple */ | ||
| addr &= ~((u64)SZ_2M - 1); | ||
| page_offset = addr >> PAGE_SHIFT; | ||
| page_offset -= node->start; | ||
|
|
||
| mutex_lock(&bo->base.pages_lock); | ||
|
|
||
| if (!bo->base.pages) { | ||
| bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, | ||
| sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); | ||
| if (!bo->sgts) | ||
| return -ENOMEM; | ||
|
|
||
| pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, | ||
| sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); | ||
| if (!pages) { | ||
| kfree(bo->sgts); | ||
| bo->sgts = NULL; | ||
| return -ENOMEM; | ||
| } | ||
| bo->base.pages = pages; | ||
| bo->base.pages_use_count = 1; | ||
| } else | ||
| pages = bo->base.pages; | ||
|
|
||
| mapping = bo->base.base.filp->f_mapping; | ||
| mapping_set_unevictable(mapping); | ||
|
|
||
| for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { | ||
| pages[i] = shmem_read_mapping_page(mapping, i); | ||
| if (IS_ERR(pages[i])) { | ||
| mutex_unlock(&bo->base.pages_lock); | ||
| ret = PTR_ERR(pages[i]); | ||
| goto err_pages; | ||
| } | ||
| } | ||
|
|
||
| mutex_unlock(&bo->base.pages_lock); | ||
|
|
||
| sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; | ||
| ret = sg_alloc_table_from_pages(sgt, pages + page_offset, | ||
| NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); | ||
| if (ret) | ||
| goto err_pages; | ||
|
|
||
| if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) { | ||
| ret = -EINVAL; | ||
| goto err_map; | ||
| } | ||
|
|
||
| mmu_map_sg(pfdev, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt); | ||
|
|
||
| bo->is_mapped = true; | ||
|
|
||
| dev_dbg(pfdev->dev, "mapped page fault @ %llx", addr); | ||
|
|
||
| return 0; | ||
|
|
||
| err_map: | ||
| sg_free_table(sgt); | ||
| err_pages: | ||
| drm_gem_shmem_put_pages(&bo->base); | ||
| return ret; | ||
| } | ||
|
|
||
| static const char *access_type_name(struct panfrost_device *pfdev, | ||
| u32 fault_status) | ||
| { | ||
|
|
@@ -317,9 +418,7 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) | |
| { | ||
| struct panfrost_device *pfdev = data; | ||
| u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT); | ||
| int i; | ||
|
|
||
| dev_err(pfdev->dev, "mmu irq status=%x\n", status); | ||
| int i, ret; | ||
|
|
||
| for (i = 0; status; i++) { | ||
| u32 mask = BIT(i) | BIT(i + 16); | ||
|
|
@@ -341,6 +440,18 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) | |
| access_type = (fault_status >> 8) & 0x3; | ||
| source_id = (fault_status >> 16); | ||
|
|
||
| /* Page fault only */ | ||
| if ((status & mask) == BIT(i)) { | ||
| WARN_ON(exception_type < 0xC1 || exception_type > 0xC4); | ||
|
|
||
| ret = panfrost_mmu_map_fault_addr(pfdev, i, addr); | ||
| if (!ret) { | ||
| mmu_write(pfdev, MMU_INT_CLEAR, BIT(i)); | ||
| status &= ~mask; | ||
| continue; | ||
| } | ||
| } | ||
|
|
||
| /* terminal fault, print info about the fault */ | ||
| dev_err(pfdev->dev, | ||
| "Unhandled Page fault in AS%d at VA 0x%016llX\n" | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters