diff options
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/futex.c | 11 | ||||
| -rw-r--r-- | io_uring/io_uring.c | 7 | ||||
| -rw-r--r-- | io_uring/io_uring.h | 1 | ||||
| -rw-r--r-- | io_uring/kbuf.c | 17 | ||||
| -rw-r--r-- | io_uring/kbuf.h | 3 | ||||
| -rw-r--r-- | io_uring/net.c | 4 | ||||
| -rw-r--r-- | io_uring/zcrx.c | 6 |
7 files changed, 37 insertions, 12 deletions
diff --git a/io_uring/futex.c b/io_uring/futex.c index fa374afbaa51..692462d50c8c 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -14,10 +14,7 @@ struct io_futex { struct file *file; - union { - u32 __user *uaddr; - struct futex_waitv __user *uwaitv; - }; + void __user *uaddr; unsigned long futex_val; unsigned long futex_mask; unsigned long futexv_owned; @@ -148,6 +145,8 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) !futex_validate_input(iof->futex_flags, iof->futex_mask)) return -EINVAL; + /* Mark as inflight, so file exit cancelation will find it */ + io_req_track_inflight(req); return 0; } @@ -186,13 +185,15 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (!futexv) return -ENOMEM; - ret = futex_parse_waitv(futexv, iof->uwaitv, iof->futex_nr, + ret = futex_parse_waitv(futexv, iof->uaddr, iof->futex_nr, io_futex_wakev_fn, req); if (ret) { kfree(futexv); return ret; } + /* Mark as inflight, so file exit cancelation will find it */ + io_req_track_inflight(req); iof->futexv_owned = 0; iof->futexv_unqueued = 0; req->flags |= REQ_F_ASYNC_DATA; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c7a9cecf528e..cf759c172083 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -408,7 +408,12 @@ static void io_clean_op(struct io_kiocb *req) req->flags &= ~IO_REQ_CLEAN_FLAGS; } -static inline void io_req_track_inflight(struct io_kiocb *req) +/* + * Mark the request as inflight, so that file cancelation will find it. + * Can be used if the file is an io_uring instance, or if the request itself + * relies on ->mm being alive for the duration of the request. + */ +inline void io_req_track_inflight(struct io_kiocb *req) { if (!(req->flags & REQ_F_INFLIGHT)) { req->flags |= REQ_F_INFLIGHT; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 0ea7a435d1de..d59c12277d58 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -83,6 +83,7 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags) bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags); void __io_commit_cqring_flush(struct io_ring_ctx *ctx); +void io_req_track_inflight(struct io_kiocb *req); struct file *io_file_get_normal(struct io_kiocb *req, int fd); struct file *io_file_get_fixed(struct io_kiocb *req, int fd, unsigned issue_flags); diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 8cce3ebd813f..2ea65f3cef72 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -108,6 +108,7 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags) buf = req->kbuf; bl = io_buffer_get_list(ctx, buf->bgid); list_add(&buf->list, &bl->buf_list); + bl->nbufs++; req->flags &= ~REQ_F_BUFFER_SELECTED; io_ring_submit_unlock(ctx, issue_flags); @@ -122,6 +123,7 @@ static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len, kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list); list_del(&kbuf->list); + bl->nbufs--; if (*len == 0 || *len > kbuf->len) *len = kbuf->len; if (list_empty(&bl->buf_list)) @@ -390,6 +392,7 @@ static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, for (i = 0; i < nbufs && !list_empty(&bl->buf_list); i++) { nxt = list_first_entry(&bl->buf_list, struct io_buffer, list); list_del(&nxt->list); + bl->nbufs--; kfree(nxt); cond_resched(); } @@ -491,14 +494,24 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, { struct io_buffer *buf; u64 addr = pbuf->addr; - int i, bid = pbuf->bid; + int ret = -ENOMEM, i, bid = pbuf->bid; for (i = 0; i < pbuf->nbufs; i++) { + /* + * Nonsensical to have more than sizeof(bid) buffers in a + * buffer list, as the application then has no way of knowing + * which duplicate bid refers to what buffer. + */ + if (bl->nbufs == USHRT_MAX) { + ret = -EOVERFLOW; + break; + } buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT); if (!buf) break; list_add_tail(&buf->list, &bl->buf_list); + bl->nbufs++; buf->addr = addr; buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); buf->bid = bid; @@ -508,7 +521,7 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, cond_resched(); } - return i ? 0 : -ENOMEM; + return i ? 0 : ret; } static int __io_manage_buffers_legacy(struct io_kiocb *req, diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index 4d2c209d1a41..5d83c7adc739 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -21,6 +21,9 @@ struct io_buffer_list { struct list_head buf_list; struct io_uring_buf_ring *buf_ring; }; + /* count of classic/legacy buffers in buffer list */ + int nbufs; + __u16 bgid; /* below is for ring provided buffers */ diff --git a/io_uring/net.c b/io_uring/net.c index d13f3e8f6c72..e16633fd6630 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -832,7 +832,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret, * If more is available AND it was a full transfer, retry and * append to this one */ - if (!sr->retry && kmsg->msg.msg_inq > 0 && this_ret > 0 && + if (!sr->retry && kmsg->msg.msg_inq > 1 && this_ret > 0 && !iov_iter_count(&kmsg->msg.msg_iter)) { req->cqe.flags = cflags & ~CQE_F_MASK; sr->len = kmsg->msg.msg_inq; @@ -1070,7 +1070,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg arg.mode |= KBUF_MODE_FREE; } - if (kmsg->msg.msg_inq > 0) + if (kmsg->msg.msg_inq > 1) arg.max_len = min_not_zero(sr->len, kmsg->msg.msg_inq); ret = io_buffers_peek(req, &arg); diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 1513431587a7..797247a34cb7 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -366,7 +366,8 @@ static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) static void io_zcrx_free_area(struct io_zcrx_area *area) { - io_zcrx_unmap_area(area->ifq, area); + if (area->ifq) + io_zcrx_unmap_area(area->ifq, area); io_release_area_mem(&area->mem); kvfree(area->freelist); @@ -631,12 +632,13 @@ ifq_free: void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) { struct io_zcrx_ifq *ifq; - unsigned long id; lockdep_assert_held(&ctx->uring_lock); while (1) { scoped_guard(mutex, &ctx->mmap_lock) { + unsigned long id = 0; + ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); if (ifq) xa_erase(&ctx->zcrx_ctxs, id); |