Lindenii Project Forge
Login

hare-aio

Asynchronous I/O event loops for Hare

Hi… I am well aware that this diff view is very suboptimal. It will be fixed when the refactored server comes along!

Commit info
ID
a86a73321c7d2436f0a3baa4616dd7e2a6a54050
Author
Eyal Sawady <ecs@d2evs.net>
Author date
Tue, 07 Dec 2021 11:12:09 +0000
Committer
Drew DeVault <sir@cmpwn.com>
Committer date
Wed, 08 Dec 2021 11:14:08 +0100
Actions
hare::parse::expr: update match binding syntax

And update the rest of the stdlib for the new syntax

Signed-off-by: Eyal Sawady <ecs@d2evs.net>
use errors;
use rt;

// Advances the completion queue by N items.
export fn cq_advance(ring: *io_uring, n: uint) void = {
	*ring.cq.khead = *ring.cq.khead + n;
};

// Call after processing a [[cqe]]. The cqe is returned to the pool and cannot
// be used by the application again.
export fn cqe_seen(ring: *io_uring, cqe: *cqe) void = cq_advance(ring, 1);

// Waits until a CQE is available, then returns it. The caller must pass the
// returned CQE to [[cqe_seen]] to advance the queue.
export fn wait(ring: *io_uring) (*cqe | error) = {
	match (get_cqe(ring, 0, 1)) {
	case err: error =>
	case let err: error =>
		return err;
	case cq: nullable *cqe =>
	case let cq: nullable *cqe =>
		assert(cq != null); // XXX: Correct?
		return cq: *cqe;
	};
};

// Peeks the next CQE from the queue and returns it, or null if none are
// pending. The caller must pass the returned CQE to [[cqe_seen]] to advance the
// queue.
export fn peek(ring: *io_uring) (nullable *cqe | error) = get_cqe(ring, 0, 0);

// Returns the result of a [[cqe]], or an error if unsuccessful.
export fn result(cqe: *cqe) (int | error) = {
	if (cqe.res >= 0) {
		return cqe.res;
	};
	switch (-cqe.res) {
	case rt::ENOBUFS =>
		return nobuffers;
	case =>
		return errors::errno(rt::wrap_errno(-cqe.res));
	};
};

// Gets the user data field of a [[cqe]]. See [[set_user]] for the corresponding
// SQE function.
export fn get_user(cqe: *cqe) nullable *void =
	cqe.user_data: uintptr: nullable *void;

// Returns the buffer ID used for this [[cqe]] in combination with
// [[set_buffer_select]]. Aborts the program if this CQE was not configured to
// use a buffer pool.
export fn get_buffer_id(cqe: *cqe) u16 = {
	// TODO: Handle ENOBUFS
	assert(cqe.flags & cqe_flags::F_BUFFER > 0,
		"get_buffer_id called for CQE without buffer");
	return (cqe.flags: u32 >> CQE_BUFFER_SHIFT): u16;
};

fn peek_cqe(ring: *io_uring) (nullable *cqe, uint) = {
	let head = *ring.cq.khead;
	let tail = *ring.cq.ktail;
	let mask = *ring.cq.kring_mask;
	let avail = tail - head;
	if (avail == 0) {
		return (null, 0);
	};
	return (&ring.cq.cqes[head & mask], avail);
};

fn get_cqe(
	ring: *io_uring,
	submit: uint,
	wait: uint,
) (nullable *cqe | error) = {
	let cq: nullable *cqe = null;
	for (cq == null) {
		let enter = false, overflow = false;
		let flags = enter_flags::NONE;

		// TODO: tuple destructuring
		let tup = peek_cqe(ring);
		let avail = tup.1;
		cq = tup.0;

		if (cq == null && wait == 0 && submit == 0) {
			if (!needs_flush(ring)) {
				return null;
			};
			overflow = true;
		};
		if (wait > avail || overflow) {
			flags |= enter_flags::GETEVENTS;
			enter = true;
		};
		if (submit > 0) {
			needs_enter(ring, &flags);
			enter = true;
		};
		if (!enter) {
			break;
		};

		match (rt::io_uring_enter(ring.fd,
			submit, wait, flags: uint, null)) {
		case err: rt::errno =>
		case let err: rt::errno =>
			return errors::errno(err);
		case n: uint =>
		case let n: uint =>
			submit -= n;
		};
	};
	return cq;
};
use errors;
use rt;

// TODO: Atomics

// Returns the next available [[sqe]] for this [[io_uring]], or null if the
// queue is full.
export fn get_sqe(ring: *io_uring) nullable *sqe = {
	const sq = &ring.sq;
	const head = *sq.khead, next = sq.sqe_tail + 1;
	if (next - head <= *sq.kring_entries) {
		let sqe = &sq.sqes[sq.sqe_tail & *sq.kring_mask];
		sq.sqe_tail = next;
		return sqe;
	};
	return null;
};

// Returns the next available [[sqe]] for this [[io_uring]], or aborts the
// program if the queue is full.
export fn must_get_sqe(ring: *io_uring) *sqe = {
	match (get_sqe(ring)) {
	case null =>
		abort("I/O queue full");
	case sq: *sqe =>
	case let sq: *sqe =>
		return sq;
	};
};

fn needs_enter(ring: *io_uring, flags: *enter_flags) bool = {
	if (ring.flags & setup_flags::IOPOLL != setup_flags::IOPOLL) {
		return true;
	};

	if (*ring.sq.kflags & sqring_flags::NEED_WAKEUP == sqring_flags::NEED_WAKEUP) {
		*flags |= enter_flags::SQ_WAKEUP;
		return true;
	};

	return false;
};

fn needs_flush(ring: *io_uring) bool =
	*ring.sq.kflags & sqring_flags::CQ_OVERFLOW == sqring_flags::CQ_OVERFLOW;

// Submits queued I/O asynchronously. Returns the number of submissions accepted
// by the kernel.
export fn submit(ring: *io_uring) (uint | error) =
	do_submit(ring, flush_sq(ring), 0u);

// Submits queued I/O asynchronously and blocks until at least "wait" events are
// complete. If setup_flags::IOPOLL was configured for this ring, the meaning of
// the "wait" parameter is different: a non-zero value will block until at least
// one event is completed.
//
// Returns the number of submissions accepted by the kernel.
export fn submit_wait(ring: *io_uring, wait: uint) (uint | error) =
	do_submit(ring, flush_sq(ring), wait);

fn flush_sq(ring: *io_uring) uint = {
	let sq = &ring.sq;
	let ktail = *sq.ktail;
	const mask = *sq.kring_mask;

	if (sq.sqe_head == sq.sqe_tail) {
		return ktail - *sq.khead;
	};

	for (let n = sq.sqe_tail - sq.sqe_head; n > 0; n -= 1u) {
		sq.array[ktail & mask] = sq.sqe_head & mask;
		ktail += 1u;
		sq.sqe_head += 1u;
	};

	*sq.ktail = ktail;
	return ktail - *sq.khead;
};

fn do_submit(
	ring: *io_uring,
	submitted: uint,
	wait: uint,
) (uint | error) = {
	let flags = enter_flags::GETEVENTS;
	if (needs_enter(ring, &flags) || wait != 0) {
		match (rt::io_uring_enter(ring.fd,
				submitted, wait, flags, null)) {
		case err: rt::errno =>
		case let err: rt::errno =>
			return errors::errno(err);
		case n: uint =>
		case let n: uint =>
			return n;
		};
	} else {
		return submitted;
	};
};
use errors;
use rt;
use types;

// Registers a set of fixed buffers with an [[io_uring]]. Note that you must
// call [[unregister_buffers]] before registering a new set of buffers (even if
// some of them have similar addresses to the old set). The buffers must be
// anonymous, non-file-backed memory (e.g. the kind returned by alloc or
// rt::mmap).
export fn register_buffers(ring: *io_uring, iov: []rt::iovec) (void | error) = {
	assert(len(iov) <= types::UINT_MAX);
	match (rt::io_uring_register(ring.fd, regop::REGISTER_BUFFERS,
			iov: *[*]rt::iovec, len(iov): uint)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Unregisters all fixed buffers associated with an [[io_uring]].
export fn unregister_buffers(ring: *io_uring) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::UNREGISTER_BUFFERS, null, 0)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Registers a set of file descriptors with an [[io_uring]]. The set of files
// may be sparse, meaning that some are set to -1, to be updated later using
// [[register_files_update]].
export fn register_files(ring: *io_uring, files: []int) (void | error) = {
	assert(len(files) <= types::UINT_MAX);
	match (rt::io_uring_register(ring.fd, regop::REGISTER_FILES,
			files: *[*]int, len(files): uint)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Applies a set of [[files_update]]s to the set of files registered with an
// [[io_uring]].
export fn register_files_update(
	ring: *io_uring,
	updates: []files_update,
) (void | error) = {
	assert(len(updates) <= types::UINT_MAX);
	match (rt::io_uring_register(ring.fd, regop::REGISTER_FILES_UPDATE,
			updates: *[*]files_update, len(updates): uint)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Unregisters all files associated with an [[io_uring]].
export fn unregister_files(ring: *io_uring) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::UNREGISTER_FILES, null, 0)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Registers an eventfd(2) with this [[io_uring]] to be notified of completion
// events.
export fn register_eventfd(ring: *io_uring, fd: int) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::REGISTER_EVENTFD, &fd, 1)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Similar to [[register_eventfd]], but only notifies of events which complet
// asyncronously.
export fn register_eventfd_async(ring: *io_uring, fd: int) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::REGISTER_EVENTFD_ASYNC, &fd, 1)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Unregisters the eventfd(2) associated with this [[io_uring]].
export fn unregister_eventfd(ring: *io_uring) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::UNREGISTER_EVENTFD, null, 0)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// XXX: This interface is pretty bad. It would be nice to improve on it
// a bit before making it part of the API.
//export fn register_probe(ring: *io_uring, out: *probe, nops: size) void = {
//	assert(nops * size(probe_op) <= types::UINT_MAX);
//	match (rt::io_uring_register(ring.fd, regop::REGISTER_PROBE,
//			out, (nops * size(probe)): uint)) {
//		rt::errno => abort("Unexpected io_uring REGISTER_PROBE error"),
//		void => void,
//	};
//};

// Registers the current process's credentials as a personality with an
// [[io_uring]], returning an ID. Set the personality field of an [[sqe]] to use
// that personality for an I/O submission.
export fn register_personality(ring: *io_uring) int = {
	match (rt::io_uring_register(ring.fd,
			regop::REGISTER_PERSONALITY, null, 0)) {
	case rt::errno =>
		abort("Unexpected io_uring REGISTER_PERSONALITY error");
	case i: int =>
	case let i: int =>
		return i;
	};
};

// Unregisters a personality previously configured with
// [[register_personality]].
export fn unregister_personality(ring: *io_uring, id: int) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::UNREGISTER_PERSONALITY, null, id: uint)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Enables submissions for an [[io_uring]] which was started in the disabled
// state via [[setup_flags::R_DISABLED]]. Future access to this io_uring is
// subject to any configured restrictions.
export fn register_enable_rings(ring: *io_uring) (void | error) = {
	match (rt::io_uring_register(ring.fd,
			regop::REGISTER_ENABLE_RINGS, null, 0)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};

// Registers a restriction for this [[io_uring]], limiting the kinds of future
// registrations and I/O submissions which are permitted for it. This is only
// accepted if the [[io_uring]] was set up in a disabled state via
// [[setup_flags::R_DISABLED]].
export fn register_restrictions(ring: *io_uring, res: []restriction) (void | error) = {
	assert(len(res) < types::UINT_MAX);
	match (rt::io_uring_register(ring.fd, regop::REGISTER_RESTRICTIONS,
			res: *[*]restriction, len(res): uint)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case int => void;
	};
};
use errors;
use rt;

// Sets up an io_uring. The params parameter must be initialized with the
// desired flags, sq_thread_cpu, and sq_thread_idle parameters; the remaining
// fields are initialized by the kernel.
export fn setup(entries: u32, params: *params) (io_uring | error) = {
	const fd = match (rt::io_uring_setup(entries, params)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case fd: int =>
	case let fd: int =>
		yield fd;
	};

	let uring = io_uring {
		sq = sq { ... },
		cq = cq { ... },
		fd = fd,
		flags = params.flags,
		features = params.features,
	};
	let sq = &uring.sq, cq = &uring.cq;

	sq.ring_sz = params.sq_off.array + params.sq_entries * size(uint);
	cq.ring_sz = params.cq_off.cqes + params.cq_entries * size(cqe);

	if (uring.features & features::SINGLE_MMAP == features::SINGLE_MMAP) {
		if (cq.ring_sz > sq.ring_sz) {
			sq.ring_sz = cq.ring_sz;
		};
		cq.ring_sz = sq.ring_sz;
	};

	sq.ring_ptr = match (rt::mmap(null,
			params.sq_off.array + entries * size(u32),
			rt::PROT_READ | rt::PROT_WRITE,
			rt::MAP_SHARED | rt::MAP_POPULATE,
			fd, OFF_SQ_RING)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case ptr: *void =>
	case let ptr: *void =>
		yield ptr;
	};

	cq.ring_ptr = if (uring.features & features::SINGLE_MMAP == features::SINGLE_MMAP) {
		yield sq.ring_ptr;
	} else match (rt::mmap(null, cq.ring_sz,
			rt::PROT_READ | rt::PROT_WRITE,
			rt::MAP_SHARED | rt::MAP_POPULATE,
			fd, OFF_CQ_RING)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case ptr: *void =>
	case let ptr: *void =>
		yield ptr;
	};

	const ring_ptr = sq.ring_ptr: uintptr;
	sq.khead = (ring_ptr + params.sq_off.head: uintptr): *uint;
	sq.ktail = (ring_ptr + params.sq_off.tail: uintptr): *uint;
	sq.kring_mask = (ring_ptr + params.sq_off.ring_mask: uintptr): *uint;
	sq.kring_entries = (ring_ptr + params.sq_off.ring_entries: uintptr): *uint;
	sq.kflags = (ring_ptr + params.sq_off.flags: uintptr): *sqring_flags;
	sq.kdropped = (ring_ptr + params.sq_off.dropped: uintptr): *uint;
	sq.array = (ring_ptr + params.sq_off.array: uintptr): *[*]uint;
	sq.sqes = match (rt::mmap(null,
			params.sq_entries * size(sqe),
			rt::PROT_READ | rt::PROT_WRITE,
			rt::MAP_SHARED | rt::MAP_POPULATE,
			fd, OFF_SQES)) {
	case err: rt::errno =>
	case let err: rt::errno =>
		return errors::errno(err);
	case ptr: *void =>
	case let ptr: *void =>
		yield ptr: *[*]sqe;
	};

	const ring_ptr = cq.ring_ptr: uintptr;
	cq.khead = (ring_ptr + params.cq_off.head: uintptr): *uint;
	cq.ktail = (ring_ptr + params.cq_off.tail: uintptr): *uint;
	cq.kring_mask = (ring_ptr + params.cq_off.ring_mask: uintptr): *uint;
	cq.kring_entries = (ring_ptr + params.cq_off.ring_entries: uintptr): *uint;
	cq.koverflow = (ring_ptr + params.cq_off.overflow: uintptr): *uint;
	cq.cqes = (ring_ptr + params.cq_off.cqes: uintptr): *[*]cqe;

	if (params.cq_off.flags != 0) {
		cq.kflags = (ring_ptr + params.cq_off.flags: uintptr): *cqring_flags;
	};

	return uring;
};

// Frees state associated with an [[io_uring]].
export fn finish(ring: *io_uring) void = {
	let sq = &ring.sq, cq = &ring.cq;
	rt::munmap(sq.ring_ptr, sq.ring_sz): void;
	if (cq.ring_ptr != null: *void && cq.ring_ptr != sq.ring_ptr) {
		rt::munmap(cq.ring_ptr, cq.ring_sz): void;
	};
	rt::close(ring.fd): void;
};
use errors;

// Returned when buffer pool use was configured for an [[sqe]], but there are no
// buffers available.
export type nobuffers = !void;

// All errors which may be returned by this module.
export type error = !(errors::error | nobuffers);

// Converts an [[error]] into a human-readable string.
export fn strerror(err: error) const str = {
	match (err) {
	case nobuffers =>
		return "Buffer pool exhausted";
	case err: errors::error =>
	case let err: errors::error =>
		return errors::strerror(err);
	};
};

// The maximum value for the first parameter of [[setup]].
export def MAX_ENTRIES: uint = 4096;

def CQE_BUFFER_SHIFT: u32 = 16;
def OFF_SQ_RING: u64 = 0;
def OFF_CQ_RING: u64 = 0x8000000;
def OFF_SQES: u64 = 0x10000000;

// An io_uring [[sqe]] operation.
export type op = enum u8 {
	NOP,
	READV,
	WRITEV,
	FSYNC,
	READ_FIXED,
	WRITE_FIXED,
	POLL_ADD,
	POLL_REMOVE,
	SYNC_FILE_RANGE,
	SENDMSG,
	RECVMSG,
	TIMEOUT,
	TIMEOUT_REMOVE,
	ACCEPT,
	ASYNC_CANCEL,
	LINK_TIMEOUT,
	CONNECT,
	FALLOCATE,
	OPENAT,
	CLOSE,
	FILES_UPDATE,
	STATX,
	READ,
	WRITE,
	FADVISE,
	MADVISE,
	SEND,
	RECV,
	OPENAT2,
	EPOLL_CTL,
	SPLICE,
	PROVIDE_BUFFERS,
	REMOVE_BUFFERS,
	TEE,
};

// Flags for an [[sqe]].
export type flags = enum u8 {
	NONE = 0,
	// Use fixed fileset
	FIXED_FILE = 1 << 0,
	// Issue after inflight IO
	IO_DRAIN = 1 << 1,
	// Links next sqe
	IO_LINK = 1 << 2,
	// Like LINK, but stronger
	IO_HARDLINK = 1 << 3,
	// Always go async
	ASYNC = 1 << 4,
	// Select buffer from sqe.buf_group
	BUFFER_SELECT = 1 << 5,
};

// Flags for an fsync operation.
export type fsync_flags = enum u32 {
	NONE = 0,
	DATASYNC = 1 << 0,
};

// Flags for a timeout operation.
export type timeout_flags = enum u32 {
	NONE = 0,
	// If set, the timeout will be "absolute", waiting until CLOCK_MONOTONIC
	// reaches the time defined by the timespec. If unset, it will be
	// interpted as a duration relative to the I/O submission.
	ABS = 1 << 0,
	// When combined with [[op::TIMEOUT_REMOVE]], causes the submission to
	// update the timer rather than remove it.
	UPDATE = 1 << 1,
};

// Flags for a splice operation.
export type splice_flags = enum u32 {
	NONE = 0,
	F_FD_IN_FIXED = 1 << 31,
};

// Flags for a [[cqe]].
export type cqe_flags = enum u32 {
	NONE = 0,
	F_BUFFER = 1 << 0,
	F_MORE = 1 << 1,
};

// A submission queue entry.
export type sqe = struct {
	opcode: op,
	flags: flags,
	ioprio: u16,
	fd: i32,
	union {
		off: u64,
		addr2: nullable *void,
	},
	union {
		addr: nullable *void,
		splice_off_in: u64,
	},
	length: u32,
	union {
		rw_flags: int,
		fsync_flags: fsync_flags,
		poll_events: u16,
		poll32_events: u32,
		sync_range_flags: u32,
		msg_flags: int,
		timeout_flags: timeout_flags,
		accept_flags: u32,
		cancel_flags: u32,
		open_flags: u32,
		statx_flags: u32,
		fadvise_advice: u32,
		splice_flags: splice_flags,
	},
	user_data: u64,
	// TODO: Remove the names on these embedded types
	// See https://todo.sr.ht/~sircmpwn/hare/493
	union {
		extras: struct {
			buffers: union {
				buf_index: u16,
				buf_group: u16,
			},
			personality: u16,
			splice_fd_in: i32,
		},
		pad2: [3]u64,
	},
};

// A completion queue entry.
export type cqe = struct {
	user_data: u64,
	res: i32,
	flags: cqe_flags,
};

// Filled with the offset for mmap(2)
export type sqring_offsets = struct {
	head: u32,
	tail: u32,
	ring_mask: u32,
	ring_entries: u32,
	flags: u32,
	dropped: u32,
	array: u32,
	resv1: u32,
	resv2: u64,
};

// Flags for the sq ring.
export type sqring_flags = enum u32 {
	NONE = 0,
	// Needs io_uring_enter wakeup
	NEED_WAKEUP = 1 << 0,
	// CQ ring is overflown
	CQ_OVERFLOW = 1 << 1,
};

// Filled with the offset for mmap(2)
export type cqring_offsets = struct {
	head: u32,
	tail: u32,
	ring_mask: u32,
	ring_entries: u32,
	overflow: u32,
	cqes: u32,
	flags: u32,
	resv1: u32,
	resv2: u64,
};

// Flags for the cq ring.
export type cqring_flags = enum u32 {
	NONE = 0,
	EVENTFD_DISABLED = 1 << 0,
};

// Flags for [[setup]].
export type setup_flags = enum u32 {
	NONE = 0,
	// io_context is polled
	IOPOLL = 1 << 0,
	// SQ poll thread
	SQPOLL = 1 << 1,
	// sq_thread_cpu is valid
	SQ_AFF = 1 << 2,
	// App defines CQ size
	CQSIZE = 1 << 3,
	// Clamp SQ/CQ ring sizes
	CLAMP = 1 << 4,
	// Attach to existing wq
	ATTACH_WQ = 1 << 5,
	// Start with ring disabled
	R_DISABLED = 1 << 6,
};

// Parameters for [[setup]]. Partially completed by the kernel.
export type params = struct {
	sq_entries: u32,
	cq_entries: u32,
	flags: setup_flags,
	sq_thread_cpu: u32,
	sq_thread_idle: u32,
	features: features,
	wq_fd: u32,
	resv: [3]u32,
	sq_off: sqring_offsets,
	cq_off: cqring_offsets,
};

// Features supported by the kernel.
export type features = enum u32 {
	NONE = 0,
	SINGLE_MMAP = 1 << 0,
	NODROP = 1 << 1,
	SUBMIT_STABLE = 1 << 2,
	RW_CUR_POS = 1 << 3,
	CUR_PERSONALITY = 1 << 4,
	FAST_POLL = 1 << 5,
	POLL_32BITS = 1 << 6,
};

// Flags for [[enter]].
export type enter_flags = enum uint {
	NONE = 0,
	GETEVENTS = 1 << 0,
	SQ_WAKEUP = 1 << 1,
	SQ_WAIT = 1 << 2,
};

// Operations for [[register]].
export type regop = enum uint {
	REGISTER_BUFFERS,
	UNREGISTER_BUFFERS,
	REGISTER_FILES,
	UNREGISTER_FILES,
	REGISTER_EVENTFD,
	UNREGISTER_EVENTFD,
	REGISTER_FILES_UPDATE,
	REGISTER_EVENTFD_ASYNC,
	REGISTER_PROBE,
	REGISTER_PERSONALITY,
	UNREGISTER_PERSONALITY,
	REGISTER_RESTRICTIONS,
	REGISTER_ENABLE_RINGS,
};

// Information for a REGISTER_FILES_UPDATE operation.
export type files_update = struct {
	offs: u32,
	resv: u32,
	fds: *int,
};

// Flags for a probe operation.
export type op_flags = enum u16 {
	NONE = 0,
	SUPPORTED = 1 << 0,
};

// REGISTER_PROBE operation details.
export type probe_op = struct {
	op: u8,
	resv: u8,
	flags: op_flags,
	resv2: u32,
};

// Summary of REGISTER_PROBE results.
export type probe = struct {
	last_op: u8,
	ops_len: u8,
	resv: u16,
	resv2: [3]u32,
	ops: [*]probe_op,
};

// Details for a REGISTER_RESTRICTIONS operation.
export type restriction = struct {
	opcode: resop,
	union {
		register_op: regop,
		sqe_op: op,
		flags: flags,
	},
	resv: u8,
	resv2: [3]u32,
};

// Opcode for a [[restriction]].
export type resop = enum u16 {
	NONE = 0,
	// Allow an io_uring_register(2) opcode
	REGISTER_OP = 0,
	// Allow an sqe opcode
	SQE_OP = 1,
	// Allow sqe flags
	SQE_FLAGS_ALLOWED = 2,
	// Require sqe flags (these flags must be set on each submission)
	SQE_FLAGS_REQUIRED = 3,
};

// State for an io_uring.
export type io_uring = struct {
	sq: sq,
	cq: cq,
	fd: int,
	flags: setup_flags,
	features: features,
};

// Submission queue state.
export type sq = struct {
	khead: *uint,
	ktail: *uint,
	kring_mask: *uint,
	kring_entries: *uint,
	kflags: *sqring_flags,
	kdropped: *uint,
	array: *[*]uint,
	sqes: *[*]sqe,
	sqe_head: uint,
	sqe_tail: uint,
	ring_sz: size,
	ring_ptr: *void,
};

// Completion queue state.
export type cq = struct {
	khead: *uint,
	ktail: *uint,
	kring_mask: *uint,
	kring_entries: *uint,
	kflags: *cqring_flags,
	koverflow: *uint,
	cqes: *[*]cqe,
	ring_sz: size,
	ring_ptr: *void,
};