diff --git a/drivers/staging/android/ion/Makefile b/drivers/staging/android/ion/Makefile index 5f4487b1a224..da386dbf3ffd 100644 --- a/drivers/staging/android/ion/Makefile +++ b/drivers/staging/android/ion/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_ION) += ion.o ion_heap.o +obj-$(CONFIG_ION) += ion.o ion_buffer.o ion_heap.o obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_system_heap.o ion_page_pool.o obj-$(CONFIG_ION_CMA_HEAP) += ion_cma_heap.o diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c index d5716fb7b155..ef7c59269d3a 100644 --- a/drivers/staging/android/ion/ion.c +++ b/drivers/staging/android/ion/ion.c @@ -26,153 +26,11 @@ #include #include -#include "ion.h" +#include "ion_private.h" static struct ion_device *internal_dev; static int heap_id; -/* this function should only be called while dev->lock is held */ -static void ion_buffer_add(struct ion_device *dev, - struct ion_buffer *buffer) -{ - struct rb_node **p = &dev->buffers.rb_node; - struct rb_node *parent = NULL; - struct ion_buffer *entry; - - while (*p) { - parent = *p; - entry = rb_entry(parent, struct ion_buffer, node); - - if (buffer < entry) { - p = &(*p)->rb_left; - } else if (buffer > entry) { - p = &(*p)->rb_right; - } else { - pr_err("%s: buffer already found.", __func__); - BUG(); - } - } - - rb_link_node(&buffer->node, parent, p); - rb_insert_color(&buffer->node, &dev->buffers); -} - -/* this function should only be called while dev->lock is held */ -static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, - struct ion_device *dev, - unsigned long len, - unsigned long flags) -{ - struct ion_buffer *buffer; - int ret; - - buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); - if (!buffer) - return ERR_PTR(-ENOMEM); - - buffer->heap = heap; - buffer->flags = flags; - buffer->dev = dev; - buffer->size = len; - - ret = heap->ops->allocate(heap, buffer, len, flags); - - if (ret) { - if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE)) - goto err2; - - ion_heap_freelist_drain(heap, 0); - ret = heap->ops->allocate(heap, buffer, len, flags); - if (ret) - goto err2; - } - - if (!buffer->sg_table) { - WARN_ONCE(1, "This heap needs to set the sgtable"); - ret = -EINVAL; - goto err1; - } - - spin_lock(&heap->stat_lock); - heap->num_of_buffers++; - heap->num_of_alloc_bytes += len; - if (heap->num_of_alloc_bytes > heap->alloc_bytes_wm) - heap->alloc_bytes_wm = heap->num_of_alloc_bytes; - spin_unlock(&heap->stat_lock); - - INIT_LIST_HEAD(&buffer->attachments); - mutex_init(&buffer->lock); - mutex_lock(&dev->buffer_lock); - ion_buffer_add(dev, buffer); - mutex_unlock(&dev->buffer_lock); - return buffer; - -err1: - heap->ops->free(buffer); -err2: - kfree(buffer); - return ERR_PTR(ret); -} - -void ion_buffer_destroy(struct ion_buffer *buffer) -{ - if (buffer->kmap_cnt > 0) { - pr_warn_once("%s: buffer still mapped in the kernel\n", - __func__); - buffer->heap->ops->unmap_kernel(buffer->heap, buffer); - } - buffer->heap->ops->free(buffer); - spin_lock(&buffer->heap->stat_lock); - buffer->heap->num_of_buffers--; - buffer->heap->num_of_alloc_bytes -= buffer->size; - spin_unlock(&buffer->heap->stat_lock); - - kfree(buffer); -} - -static void _ion_buffer_destroy(struct ion_buffer *buffer) -{ - struct ion_heap *heap = buffer->heap; - struct ion_device *dev = buffer->dev; - - mutex_lock(&dev->buffer_lock); - rb_erase(&buffer->node, &dev->buffers); - mutex_unlock(&dev->buffer_lock); - - if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) - ion_heap_freelist_add(heap, buffer); - else - ion_buffer_destroy(buffer); -} - -static void *ion_buffer_kmap_get(struct ion_buffer *buffer) -{ - void *vaddr; - - if (buffer->kmap_cnt) { - buffer->kmap_cnt++; - return buffer->vaddr; - } - vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer); - if (WARN_ONCE(!vaddr, - "heap->ops->map_kernel should return ERR_PTR on error")) - return ERR_PTR(-EINVAL); - if (IS_ERR(vaddr)) - return vaddr; - buffer->vaddr = vaddr; - buffer->kmap_cnt++; - return vaddr; -} - -static void ion_buffer_kmap_put(struct ion_buffer *buffer) -{ - buffer->kmap_cnt--; - if (!buffer->kmap_cnt) { - buffer->heap->ops->unmap_kernel(buffer->heap, buffer); - buffer->vaddr = NULL; - } -} - static struct sg_table *dup_sg_table(struct sg_table *table) { struct sg_table *new_table; @@ -307,7 +165,7 @@ static void ion_dma_buf_release(struct dma_buf *dmabuf) { struct ion_buffer *buffer = dmabuf->priv; - _ion_buffer_destroy(buffer); + ion_buffer_destroy(internal_dev, buffer); } static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) @@ -392,39 +250,14 @@ static const struct dma_buf_ops dma_buf_ops = { static struct dma_buf *ion_alloc_dmabuf(size_t len, unsigned int heap_id_mask, unsigned int flags) { - struct ion_device *dev = internal_dev; - struct ion_buffer *buffer = NULL; - struct ion_heap *heap; + struct ion_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); struct dma_buf *dmabuf; pr_debug("%s: len %zu heap_id_mask %u flags %x\n", __func__, len, heap_id_mask, flags); - /* - * traverse the list of heaps available in this system in priority - * order. If the heap type is supported by the client, and matches the - * request of the caller allocate from it. Repeat until allocate has - * succeeded or all heaps have been tried - */ - len = PAGE_ALIGN(len); - - if (!len) - return ERR_PTR(-EINVAL); - - down_read(&dev->lock); - plist_for_each_entry(heap, &dev->heaps, node) { - /* if the caller didn't specify this heap id */ - if (!((1 << heap->id) & heap_id_mask)) - continue; - buffer = ion_buffer_create(heap, dev, len, flags); - if (!IS_ERR(buffer)) - break; - } - up_read(&dev->lock); - - if (!buffer) - return ERR_PTR(-ENODEV); + buffer = ion_buffer_alloc(internal_dev, len, heap_id_mask, flags); if (IS_ERR(buffer)) return ERR_CAST(buffer); @@ -435,7 +268,7 @@ static struct dma_buf *ion_alloc_dmabuf(size_t len, unsigned int heap_id_mask, dmabuf = dma_buf_export(&exp_info); if (IS_ERR(dmabuf)) - _ion_buffer_destroy(buffer); + ion_buffer_destroy(internal_dev, buffer); return dmabuf; } diff --git a/drivers/staging/android/ion/ion.h b/drivers/staging/android/ion/ion.h index 7929acb992f1..ec20a9ea13f9 100644 --- a/drivers/staging/android/ion/ion.h +++ b/drivers/staging/android/ion/ion.h @@ -17,7 +17,6 @@ #include #include #include -#include #include "../uapi/ion.h" @@ -43,7 +42,6 @@ struct ion_buffer { struct rb_node node; struct list_head list; }; - struct ion_device *dev; struct ion_heap *heap; unsigned long flags; unsigned long private_flags; @@ -56,25 +54,6 @@ struct ion_buffer { struct list_head attachments; }; -void ion_buffer_destroy(struct ion_buffer *buffer); - -/** - * struct ion_device - the metadata of the ion device node - * @dev: the actual misc device - * @buffers: an rb tree of all the existing buffers - * @buffer_lock: lock protecting the tree of buffers - * @lock: rwsem protecting the tree of heaps and clients - */ -struct ion_device { - struct miscdevice dev; - struct rb_root buffers; - struct mutex buffer_lock; - struct rw_semaphore lock; - struct plist_head heaps; - struct dentry *debug_root; - int heap_cnt; -}; - /** * struct ion_heap_ops - ops to operate on a given heap * @allocate: allocate memory diff --git a/drivers/staging/android/ion/ion_buffer.c b/drivers/staging/android/ion/ion_buffer.c new file mode 100644 index 000000000000..8b8dd1e216d4 --- /dev/null +++ b/drivers/staging/android/ion/ion_buffer.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator - buffer interface + * + * Copyright (c) 2019, Google, Inc. + */ + +#include +#include + +#include "ion_private.h" + +/* this function should only be called while dev->lock is held */ +static void ion_buffer_add(struct ion_device *dev, + struct ion_buffer *buffer) +{ + struct rb_node **p = &dev->buffers.rb_node; + struct rb_node *parent = NULL; + struct ion_buffer *entry; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct ion_buffer, node); + + if (buffer < entry) { + p = &(*p)->rb_left; + } else if (buffer > entry) { + p = &(*p)->rb_right; + } else { + pr_err("%s: buffer already found.", __func__); + BUG(); + } + } + + rb_link_node(&buffer->node, parent, p); + rb_insert_color(&buffer->node, &dev->buffers); +} + +/* this function should only be called while dev->lock is held */ +static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, + struct ion_device *dev, + unsigned long len, + unsigned long flags) +{ + struct ion_buffer *buffer; + int ret; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + buffer->heap = heap; + buffer->flags = flags; + buffer->size = len; + + ret = heap->ops->allocate(heap, buffer, len, flags); + + if (ret) { + if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE)) + goto err2; + + ion_heap_freelist_drain(heap, 0); + ret = heap->ops->allocate(heap, buffer, len, flags); + if (ret) + goto err2; + } + + if (!buffer->sg_table) { + WARN_ONCE(1, "This heap needs to set the sgtable"); + ret = -EINVAL; + goto err1; + } + + spin_lock(&heap->stat_lock); + heap->num_of_buffers++; + heap->num_of_alloc_bytes += len; + if (heap->num_of_alloc_bytes > heap->alloc_bytes_wm) + heap->alloc_bytes_wm = heap->num_of_alloc_bytes; + spin_unlock(&heap->stat_lock); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + mutex_lock(&dev->buffer_lock); + ion_buffer_add(dev, buffer); + mutex_unlock(&dev->buffer_lock); + return buffer; + +err1: + heap->ops->free(buffer); +err2: + kfree(buffer); + return ERR_PTR(ret); +} + +struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len, + unsigned int heap_id_mask, + unsigned int flags) +{ + struct ion_buffer *buffer = NULL; + struct ion_heap *heap; + + if (!dev || !len) { + return ERR_PTR(-EINVAL); + } + + /* + * traverse the list of heaps available in this system in priority + * order. If the heap type is supported by the client, and matches the + * request of the caller allocate from it. Repeat until allocate has + * succeeded or all heaps have been tried + */ + len = PAGE_ALIGN(len); + if (!len) + return ERR_PTR(-EINVAL); + + down_read(&dev->lock); + plist_for_each_entry(heap, &dev->heaps, node) { + /* if the caller didn't specify this heap id */ + if (!((1 << heap->id) & heap_id_mask)) + continue; + buffer = ion_buffer_create(heap, dev, len, flags); + if (!IS_ERR(buffer)) + break; + } + up_read(&dev->lock); + + if (!buffer) + return ERR_PTR(-ENODEV); + + if (IS_ERR(buffer)) + return ERR_CAST(buffer); + + return buffer; +} + +void ion_buffer_release(struct ion_buffer *buffer) +{ + if (buffer->kmap_cnt > 0) { + pr_warn_once("%s: buffer still mapped in the kernel\n", + __func__); + buffer->heap->ops->unmap_kernel(buffer->heap, buffer); + } + buffer->heap->ops->free(buffer); + spin_lock(&buffer->heap->stat_lock); + buffer->heap->num_of_buffers--; + buffer->heap->num_of_alloc_bytes -= buffer->size; + spin_unlock(&buffer->heap->stat_lock); + + kfree(buffer); +} + +void ion_buffer_destroy(struct ion_device *dev, struct ion_buffer *buffer) +{ + struct ion_heap *heap; + + if (!dev || !buffer) { + pr_warn("%s: invalid argument\n", __func__); + return; + } + + heap = buffer->heap; + mutex_lock(&dev->buffer_lock); + rb_erase(&buffer->node, &dev->buffers); + mutex_unlock(&dev->buffer_lock); + + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + ion_heap_freelist_add(heap, buffer); + else + ion_buffer_release(buffer); +} + +void *ion_buffer_kmap_get(struct ion_buffer *buffer) +{ + void *vaddr; + + if (buffer->kmap_cnt) { + buffer->kmap_cnt++; + return buffer->vaddr; + } + vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer); + if (WARN_ONCE(!vaddr, + "heap->ops->map_kernel should return ERR_PTR on error")) + return ERR_PTR(-EINVAL); + if (IS_ERR(vaddr)) + return vaddr; + buffer->vaddr = vaddr; + buffer->kmap_cnt++; + return vaddr; +} + +void ion_buffer_kmap_put(struct ion_buffer *buffer) +{ + buffer->kmap_cnt--; + if (!buffer->kmap_cnt) { + buffer->heap->ops->unmap_kernel(buffer->heap, buffer); + buffer->vaddr = NULL; + } +} diff --git a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c index 473b465724f1..3b4f32b9cd2d 100644 --- a/drivers/staging/android/ion/ion_heap.c +++ b/drivers/staging/android/ion/ion_heap.c @@ -15,7 +15,7 @@ #include #include -#include "ion.h" +#include "ion_private.h" void *ion_heap_map_kernel(struct ion_heap *heap, struct ion_buffer *buffer) @@ -198,7 +198,7 @@ static size_t _ion_heap_freelist_drain(struct ion_heap *heap, size_t size, buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE; total_drained += buffer->size; spin_unlock(&heap->free_lock); - ion_buffer_destroy(buffer); + ion_buffer_release(buffer); spin_lock(&heap->free_lock); } spin_unlock(&heap->free_lock); @@ -236,7 +236,7 @@ static int ion_heap_deferred_free(void *data) list_del(&buffer->list); heap->free_list_size -= buffer->size; spin_unlock(&heap->free_lock); - ion_buffer_destroy(buffer); + ion_buffer_release(buffer); } return 0; diff --git a/drivers/staging/android/ion/ion_private.h b/drivers/staging/android/ion/ion_private.h new file mode 100644 index 000000000000..e80692c885b4 --- /dev/null +++ b/drivers/staging/android/ion/ion_private.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Memory Allocator - Internal header + * + * Copyright (C) 2019 Google, Inc. + */ + +#ifndef _ION_PRIVATE_H +#define _ION_PRIVATE_H + +#include +#include +#include +#include +#include +#include +#include + +#include "ion.h" + +/** + * struct ion_device - the metadata of the ion device node + * @dev: the actual misc device + * @buffers: an rb tree of all the existing buffers + * @buffer_lock: lock protecting the tree of buffers + * @lock: rwsem protecting the tree of heaps and clients + */ +struct ion_device { + struct miscdevice dev; + struct rb_root buffers; + struct mutex buffer_lock; + struct rw_semaphore lock; + struct plist_head heaps; + struct dentry *debug_root; + int heap_cnt; +}; + +/* ion_buffer manipulators */ +extern struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len, + unsigned int heap_id_mask, + unsigned int flags); +extern void ion_buffer_release(struct ion_buffer *buffer); +extern void ion_buffer_destroy(struct ion_device *dev, struct ion_buffer *buffer); +extern void *ion_buffer_kmap_get(struct ion_buffer *buffer); +extern void ion_buffer_kmap_put(struct ion_buffer *buffer); + +#endif /* _ION_PRIVATE_H */