diff --git a/fs/read_write.c b/fs/read_write.c index 334252c60fe9..b479ad8b230d 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -1085,7 +1085,7 @@ ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos, } EXPORT_SYMBOL(vfs_iter_write); -static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, +ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, unsigned long vlen, loff_t *pos, rwf_t flags) { struct iovec iovstack[UIO_FASTIOV]; diff --git a/fs/splice.c b/fs/splice.c index abb19d3114d8..ce75aec52274 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -342,6 +342,89 @@ const struct pipe_buf_operations nosteal_pipe_buf_ops = { }; EXPORT_SYMBOL(nosteal_pipe_buf_ops); +static ssize_t kernel_readv(struct file *file, const struct kvec *vec, + unsigned long vlen, loff_t offset) +{ + mm_segment_t old_fs; + loff_t pos = offset; + ssize_t res; + + old_fs = get_fs(); + set_fs(KERNEL_DS); + /* The cast to a user pointer is valid due to the set_fs() */ + res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); + set_fs(old_fs); + + return res; +} + +static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; + struct iov_iter to; + struct page **pages; + unsigned int nr_pages; + unsigned int mask; + size_t offset, base, copied = 0; + ssize_t res; + int i; + + if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) + return -EAGAIN; + + /* + * Try to keep page boundaries matching to source pagecache ones - + * it probably won't be much help, but... + */ + offset = *ppos & ~PAGE_MASK; + + iov_iter_pipe(&to, READ, pipe, len + offset); + + res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &base); + if (res <= 0) + return -ENOMEM; + + nr_pages = DIV_ROUND_UP(res + base, PAGE_SIZE); + + vec = __vec; + if (nr_pages > PIPE_DEF_BUFFERS) { + vec = kmalloc_array(nr_pages, sizeof(struct kvec), GFP_KERNEL); + if (unlikely(!vec)) { + res = -ENOMEM; + goto out; + } + } + + mask = pipe->ring_size - 1; + pipe->bufs[to.head & mask].offset = offset; + pipe->bufs[to.head & mask].len -= offset; + + for (i = 0; i < nr_pages; i++) { + size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); + vec[i].iov_base = page_address(pages[i]) + offset; + vec[i].iov_len = this_len; + len -= this_len; + offset = 0; + } + + res = kernel_readv(in, vec, nr_pages, *ppos); + if (res > 0) { + copied = res; + *ppos += res; + } + + if (vec != __vec) + kfree(vec); +out: + for (i = 0; i < nr_pages; i++) + put_page(pages[i]); + kvfree(pages); + iov_iter_advance(&to, copied); /* truncates and discards */ + return res; +} + /* * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' * using sendpage(). Return the number of bytes sent. @@ -725,6 +808,33 @@ done: EXPORT_SYMBOL(iter_file_splice_write); +static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, + struct splice_desc *sd) +{ + int ret; + void *data; + loff_t tmp = sd->pos; + + data = kmap(buf->page); + ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); + kunmap(buf->page); + + return ret; +} + +static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, + struct file *out, loff_t *ppos, + size_t len, unsigned int flags) +{ + ssize_t ret; + + ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); + if (ret > 0) + *ppos += ret; + + return ret; +} + /** * generic_splice_sendpage - splice data from a pipe to a socket * @pipe: pipe to splice from @@ -746,23 +856,15 @@ ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, EXPORT_SYMBOL(generic_splice_sendpage); -static int warn_unsupported(struct file *file, const char *op) -{ - pr_debug_ratelimited( - "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n", - op, file, current->pid, current->comm); - return -EINVAL; -} - /* * Attempt to initiate a splice from pipe to file. */ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, loff_t *ppos, size_t len, unsigned int flags) { - if (unlikely(!out->f_op->splice_write)) - return warn_unsupported(out, "write"); - return out->f_op->splice_write(pipe, out, ppos, len, flags); + if (out->f_op->splice_write) + return out->f_op->splice_write(pipe, out, ppos, len, flags); + return default_file_splice_write(pipe, out, ppos, len, flags); } /* @@ -784,9 +886,9 @@ static long do_splice_to(struct file *in, loff_t *ppos, if (unlikely(len > MAX_RW_COUNT)) len = MAX_RW_COUNT; - if (unlikely(!in->f_op->splice_read)) - return warn_unsupported(in, "read"); - return in->f_op->splice_read(in, ppos, pipe, len, flags); + if (in->f_op->splice_read) + return in->f_op->splice_read(in, ppos, pipe, len, flags); + return default_file_splice_read(in, ppos, pipe, len, flags); } /** diff --git a/include/linux/fs.h b/include/linux/fs.h index fc0cf45d0cde..aba30ade7ff7 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1907,6 +1907,8 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *); +extern ssize_t vfs_readv(struct file *, const struct iovec __user *, + unsigned long, loff_t *, rwf_t); extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *, loff_t, size_t, unsigned int); extern ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,