summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/io_uring.c25
-rw-r--r--io_uring/io_uring.h1
-rw-r--r--io_uring/kbuf.c2
-rw-r--r--io_uring/rsrc.c19
-rw-r--r--io_uring/sqpoll.c10
-rw-r--r--io_uring/uring_cmd.c7
6 files changed, 48 insertions, 16 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4199fbe6ce13..feb61d68dca6 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -533,6 +533,17 @@ static void io_queue_iowq(struct io_kiocb *req)
io_queue_linked_timeout(link);
}
+static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts)
+{
+ io_queue_iowq(req);
+}
+
+void io_req_queue_iowq(struct io_kiocb *req)
+{
+ req->io_task_work.func = io_req_queue_iowq_tw;
+ io_req_task_work_add(req);
+}
+
static __cold void io_queue_deferred(struct io_ring_ctx *ctx)
{
while (!list_empty(&ctx->defer_list)) {
@@ -3759,6 +3770,11 @@ SYSCALL_DEFINE2(io_uring_setup, u32, entries,
static int __init io_uring_init(void)
{
+ struct kmem_cache_args kmem_args = {
+ .useroffset = offsetof(struct io_kiocb, cmd.data),
+ .usersize = sizeof_field(struct io_kiocb, cmd.data),
+ };
+
#define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \
BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
BUILD_BUG_ON(sizeof_field(stype, ename) != esize); \
@@ -3843,12 +3859,9 @@ static int __init io_uring_init(void)
* range, and HARDENED_USERCOPY will complain if we haven't
* correctly annotated this range.
*/
- req_cachep = kmem_cache_create_usercopy("io_kiocb",
- sizeof(struct io_kiocb), 0,
- SLAB_HWCACHE_ALIGN | SLAB_PANIC |
- SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU,
- offsetof(struct io_kiocb, cmd.data),
- sizeof_field(struct io_kiocb, cmd.data), NULL);
+ req_cachep = kmem_cache_create("io_kiocb", sizeof(struct io_kiocb), &kmem_args,
+ SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
+ SLAB_TYPESAFE_BY_RCU);
io_buf_cachep = KMEM_CACHE(io_buffer,
SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT);
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 65078e641390..9d70b2cf7b1e 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -94,6 +94,7 @@ int io_uring_alloc_task_context(struct task_struct *task,
int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
int start, int end);
+void io_req_queue_iowq(struct io_kiocb *req);
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 1f503bcc9c9f..d407576ddfb7 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -129,7 +129,7 @@ static int io_provided_buffers_select(struct io_kiocb *req, size_t *len,
iov[0].iov_base = buf;
iov[0].iov_len = *len;
- return 0;
+ return 1;
}
static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 131bcdda577a..33a3d156a85b 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -397,10 +397,11 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
struct io_uring_rsrc_update2 *up,
unsigned int nr_args)
{
- struct iovec __user *uvec = u64_to_user_ptr(up->data);
u64 __user *tags = u64_to_user_ptr(up->tags);
struct iovec fast_iov, *iov;
struct page *last_hpage = NULL;
+ struct iovec __user *uvec;
+ u64 user_data = up->data;
__u32 done;
int i, err;
@@ -413,7 +414,8 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
struct io_mapped_ubuf *imu;
u64 tag = 0;
- iov = iovec_from_user(&uvec[done], 1, 1, &fast_iov, ctx->compat);
+ uvec = u64_to_user_ptr(user_data);
+ iov = iovec_from_user(uvec, 1, 1, &fast_iov, ctx->compat);
if (IS_ERR(iov)) {
err = PTR_ERR(iov);
break;
@@ -446,6 +448,10 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
ctx->user_bufs[i] = imu;
*io_get_tag_slot(ctx->buf_data, i) = tag;
+ if (ctx->compat)
+ user_data += sizeof(struct compat_iovec);
+ else
+ user_data += sizeof(struct iovec);
}
return done ? done : err;
}
@@ -1022,7 +1028,7 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
struct page *last_hpage = NULL;
struct io_rsrc_data *data;
struct iovec fast_iov, *iov = &fast_iov;
- const struct iovec __user *uvec = (struct iovec * __user) arg;
+ const struct iovec __user *uvec;
int i, ret;
BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
@@ -1045,7 +1051,8 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
if (arg) {
- iov = iovec_from_user(&uvec[i], 1, 1, &fast_iov, ctx->compat);
+ uvec = (struct iovec __user *) arg;
+ iov = iovec_from_user(uvec, 1, 1, &fast_iov, ctx->compat);
if (IS_ERR(iov)) {
ret = PTR_ERR(iov);
break;
@@ -1053,6 +1060,10 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
ret = io_buffer_validate(iov);
if (ret)
break;
+ if (ctx->compat)
+ arg += sizeof(struct compat_iovec);
+ else
+ arg += sizeof(struct iovec);
}
if (!iov->iov_base && *io_get_tag_slot(data, i)) {
diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c
index b800be719260..a26593979887 100644
--- a/io_uring/sqpoll.c
+++ b/io_uring/sqpoll.c
@@ -109,14 +109,14 @@ static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
struct fd f;
f = fdget(p->wq_fd);
- if (!f.file)
+ if (!fd_file(f))
return ERR_PTR(-ENXIO);
- if (!io_is_uring_fops(f.file)) {
+ if (!io_is_uring_fops(fd_file(f))) {
fdput(f);
return ERR_PTR(-EINVAL);
}
- ctx_attach = f.file->private_data;
+ ctx_attach = fd_file(f)->private_data;
sqd = ctx_attach->sq_data;
if (!sqd) {
fdput(f);
@@ -420,9 +420,9 @@ __cold int io_sq_offload_create(struct io_ring_ctx *ctx,
struct fd f;
f = fdget(p->wq_fd);
- if (!f.file)
+ if (!fd_file(f))
return -ENXIO;
- if (!io_is_uring_fops(f.file)) {
+ if (!io_is_uring_fops(fd_file(f))) {
fdput(f);
return -EINVAL;
}
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 8391c7c7c1ec..39c3c816ec78 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -277,6 +277,13 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
}
EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);
+void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
+{
+ struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
+
+ io_req_queue_iowq(req);
+}
+
static inline int io_uring_cmd_getsockopt(struct socket *sock,
struct io_uring_cmd *cmd,
unsigned int issue_flags)
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-23 22:59:27 +0000