Linux I/O Path: From write() to NVMe Doorbell

Representative path:

__x64_sys_write()
-> ksys_write()
-> vfs_write()
-> new_sync_write()
-> call_write_iter()

Key structs created or used here:

/* uapi/linux/uio.h */
struct iovec {
    void __user *iov_base;
    size_t       iov_len;
};

/* include/linux/uio.h */
struct iov_iter {
    unsigned int type;
    size_t       iov_offset;
    size_t       count;
    const struct iovec *iov;
};

/* include/linux/fs.h */
struct file {
    const struct file_operations *f_op;
    struct inode         *f_inode;
    struct address_space *f_mapping;
    loff_t                f_pos;
    unsigned int          f_flags;
};

struct kiocb {
    struct file *ki_filp;
    loff_t       ki_pos;
    int          ki_flags;
};

Field handoff:

During writeback, filesystem and iomap code build bio objects:

iomap_writepages()
-> bio_alloc_*()
-> bio_add_page()
-> submit_bio()

Trimmed shape:

/* include/linux/bio.h */
struct bio_vec {
    struct page *bv_page;
    unsigned int bv_len;
    unsigned int bv_offset;
};

struct bvec_iter {
    sector_t     bi_sector;
    unsigned int bi_size;
};

struct bio {
    struct block_device *bi_bdev;
    struct bio_vec      *bi_io_vec;
    unsigned short       bi_vcnt;
    struct bvec_iter     bi_iter;
    unsigned int         bi_opf;
    bio_end_io_t        *bi_end_io;
    void                *bi_private;
};

Fields that matter:

The NVMe driver maps the block request into an NVMe command and DMA description.

Important structs:

/* drivers/nvme/host/nvme.h */
struct nvme_ns {
    struct nvme_ctrl *ctrl;
    u32               ns_id;
    u8                lba_shift;
    struct gendisk   *disk;
};

struct nvme_iod {
    struct scatterlist *sg;
    int                 nents;
    dma_addr_t          first_dma;
    __le64             *dma_prps;
    int                 npages;
    size_t              length;
};

struct nvme_request {
    struct nvme_command cmd;
    struct nvme_iod    *iod;
    int                 status;
    u8                  retries;
};

NVMe command shape:

/* include/linux/nvme.h */
struct nvme_common_command {
    __u8   opcode;
    __u8   flags;
    __u16  command_id;
    __le32 nsid;
    __le64 mptr;
    __le64 prp1;
    __le64 prp2;
    __le32 cdw10;
    __le32 cdw11;
    __le32 cdw12;
    __le32 cdw13;
    __le32 cdw14;
    __le32 cdw15;
};

struct nvme_rw_command {
    __u8   opcode;
    __u8   flags;
    __u16  command_id;
    __le32 nsid;
    __le64 rsvd;
    __le64 slba;
    __le16 length;
    __le16 control;
    __le32 dsmgmt;
    __le32 reftag;
    __le16 apptag;
    __le16 appmask;
};

struct nvme_command {
    union {
        struct nvme_common_command common;
        struct nvme_rw_command     rw;
        /* other command formats */
    };
};

Key command fields:

The driver derives the NVMe LBA from the block sector:

nvme_lba = bio_sector >> (ns->lba_shift - 9)

Why subtract 9:

block layer sector = 512 bytes = 2^9
NVMe namespace LBA = 2^ns->lba_shift bytes

The controller executes the command and posts a completion queue entry.

struct nvme_completion {
    __le32 result;
    __u32  rsvd;
    __le16 sq_head;
    __le16 sq_id;
    __u16  command_id;
    __le16 status;
};

Completion path:

NVMe device writes CQE
-> interrupt or polling
-> nvme_irq() / nvme_poll()
-> nvme_process_cq()
-> blk_mq_complete_request(req)
-> bio_endio()
-> filesystem completion
-> kiocb completion or syscall return

Buffered writes usually complete the syscall earlier, after copying into the page cache. Writeback completion is still essential for durability and error reporting, but it is decoupled from the original write() return unless synchronous flags force waiting.

If the page cache already has uptodate folios:

filemap_read()
-> find folios in mapping->i_pages
-> copy_to_iter()
-> return to userspace

No bio, no request, no NVMe command.

Pointer chain:

kiocb
  |
  v
ki_filp: struct file
  |
  +-> f_op: struct file_operations
  |
  +-> f_inode: struct inode
  |     |
  |     +-> i_sb: struct super_block
  |     |     |
  |     |     +-> s_type: struct file_system_type
  |     |           |
  |     |           +-> name = "ext4", "xfs", "btrfs", ...
  |     |
  |     +-> i_mapping: page cache and a_ops
  |
  +-> f_mapping: usually inode->i_mapping

Representative filesystem shape:

ssize_t ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
    struct file *file = iocb->ki_filp;
    loff_t pos = iocb->ki_pos;

    if (iocb->ki_flags & IOCB_DIRECT)
        return iomap_dio_rw(iocb, from, &ext4_iomap_ops, ...);

    return generic_file_write_iter(iocb, from);
}

What the filesystem learns from kiocb:

Why file alone is not enough:

struct file:
persistent per-open state

struct kiocb:
transient per-operation state

Many threads or io_uring submissions can operate on the same struct file. Each operation needs its own kiocb.