Skip to content

Commit

Permalink
libuio refactor
Browse files Browse the repository at this point in the history
Signed-off-by: Eric Chan <[email protected]>
  • Loading branch information
erichchan999 committed Oct 31, 2024
1 parent 26f6bdb commit 8d3f665
Show file tree
Hide file tree
Showing 13 changed files with 399 additions and 235 deletions.
47 changes: 34 additions & 13 deletions examples/virtio/blk_driver_vmm.c
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,20 @@
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <stddef.h>
#include <stdint.h>
#include <microkit.h>
#include <blk_config.h>
#include <libvmm/arch/aarch64/fault.h>
#include <libvmm/arch/aarch64/linux.h>
#include <libvmm/guest.h>
#include <libvmm/virq.h>
#include <libvmm/uio/uio.h>
#include <libvmm/util/util.h>
#include <libvmm/virq.h>
#include <libvmm/virtio/virtio.h>
#include <libvmm/arch/aarch64/linux.h>
#include <libvmm/arch/aarch64/fault.h>
#include <microkit.h>
#include <sddf/serial/queue.h>
#include <serial_config.h>
#include <stddef.h>
#include <stdint.h>
#include <uio/blk.h>

#define GUEST_RAM_SIZE 0x6000000

Expand All @@ -39,6 +42,12 @@ extern char _guest_initrd_image_end[];
/* Microkit will set this variable to the start of the guest RAM memory region. */
uintptr_t guest_ram_vaddr;

/* Passing info from VMM to block uio driver */
driver_blk_vmm_info_passing_t *driver_blk_vmm_info_passing;
uintptr_t virt_blk_data;
uintptr_t client_vmm_1_blk_data;
uintptr_t client_vmm_2_blk_data;

/* sDDF block */
#define BLOCK_CH 1
#if defined(BOARD_odroidc4)
Expand All @@ -50,6 +59,11 @@ uintptr_t guest_ram_vaddr;
#define UIO_IRQ 50
#define UIO_CH 3

/* This global is kind of redundant, but for now it's needed to be passed
* through to the uio-vmm notify handler
*/
microkit_channel uio_ch = UIO_CH;

/* Serial */
#define SERIAL_VIRT_TX_CH 4
#define SERIAL_VIRT_RX_CH 5
Expand All @@ -66,11 +80,6 @@ char *serial_tx_data;

static struct virtio_console_device virtio_console;

void uio_ack(size_t vcpu_id, int irq, void *cookie)
{
microkit_notify(UIO_CH);
}

void init(void)
{
/* Initialise the VMM, the VCPU(s), and start the guest */
Expand Down Expand Up @@ -120,8 +129,20 @@ void init(void)
SERIAL_VIRT_TX_CH);
assert(success);

/* Register the UIO IRQ */
virq_register(GUEST_VCPU_ID, UIO_IRQ, uio_ack, NULL);
/* Register the block uio driver */
success = uio_register_driver(UIO_IRQ, &uio_ch, 0x80000000, 0x1000);
assert(success);

/* Populate vmm info passing to block uio driver */
driver_blk_vmm_info_passing->client_data_phys[0] = virt_blk_data;
driver_blk_vmm_info_passing->client_data_phys[1] = client_vmm_1_blk_data;
driver_blk_vmm_info_passing->client_data_phys[2] = client_vmm_2_blk_data;
driver_blk_vmm_info_passing->client_data_size[0] =
BLK_DATA_REGION_SIZE_DRIV;
driver_blk_vmm_info_passing->client_data_size[1] =
BLK_DATA_REGION_SIZE_CLI0;
driver_blk_vmm_info_passing->client_data_size[2] =
BLK_DATA_REGION_SIZE_CLI1;

#if defined(BOARD_odroidc4)
/* Register the SD card IRQ */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
0x00 0x80000000 0x00 0x1000
0x00 0x80200000 0x00 0x200000
0x00 0x80400000 0x00 0x200000
0x00 0x60600000 0x00 0x200000
0x00 0x80600000 0x00 0x200000
>;
interrupts = <0x00 18 0x04>;
// IRQ = 50
Expand All @@ -27,8 +27,10 @@
uio1 {
compatible = "generic-uio\0uio";
reg = <
0x00 0x60800000 0x00 0x200000
0x00 0x60a00000 0x00 0x200000
0x00 0x80800000 0x00 0x200000
0x00 0x80a00000 0x00 0x200000
0x00 0x80c00000 0x00 0x200000
0x00 0x80e00000 0x00 0x200000
>;
interrupts = <0x00 19 0x04>;
// IRQ = 51
Expand Down
35 changes: 21 additions & 14 deletions examples/virtio/board/odroidc4/virtio.system
Original file line number Diff line number Diff line change
Expand Up @@ -168,20 +168,21 @@
<memory_region name="driver_blk_storage_info" size="0x1000" page_size="0x1000"/>
<memory_region name="driver_blk_req_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="driver_blk_resp_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="driver_blk_data" size="0x200_000" page_size="0x200_000" phys_addr="0x60600000"/>
<memory_region name="driver_blk_data" size="0x200_000" page_size="0x200_000"/>

<memory_region name="client_vmm_1_blk_storage_info" size="0x1000" page_size="0x1000" />
<memory_region name="client_vmm_1_blk_req_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_1_blk_resp_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_1_blk_data" size="0x200_000" page_size="0x200_000" phys_addr="0x60800000"/>
<memory_region name="client_vmm_1_blk_data" size="0x200_000" page_size="0x200_000"/>

<memory_region name="client_vmm_2_blk_storage_info" size="0x1000" page_size="0x1000" />
<memory_region name="client_vmm_2_blk_req_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_2_blk_resp_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_2_blk_data" size="0x200_000" page_size="0x200_000" phys_addr="0x60a00000"/>
<memory_region name="client_vmm_2_blk_data" size="0x200_000" page_size="0x200_000"/>

<!-- UIO irq status page -->
<memory_region name="uio_irq_status" size="0x1000" page_size="0x1000" />
<!-- UIO metadata -->
<memory_region name="driver_blk_uio_irq_status" size="0x1000" page_size="0x1000" />
<memory_region name="driver_blk_vmm_info_passing" size="0x1000" page_size="0x1000" />

<protection_domain name="BLK_DRIVER_VMM" priority="200" budget="100" period="400">
<program_image path="blk_driver_vmm.elf" />
Expand All @@ -192,6 +193,11 @@
<map mr="driver_blk_vmm_serial_tx_data" vaddr="0x6_002_000" perms="rw" cached="true" setvar_vaddr="serial_tx_data" />
<map mr="driver_blk_vmm_serial_rx_data" vaddr="0x6_004_000" perms="rw" cached="true" setvar_vaddr="serial_rx_data" />

<map mr="driver_blk_vmm_info_passing" vaddr="0x80200000" perms="rw" cached="false" setvar_vaddr="driver_blk_vmm_info_passing" />
<setvar symbol="virt_blk_data" region_paddr="driver_blk_data" />
<setvar symbol="client_vmm_1_blk_data" region_paddr="client_vmm_1_blk_data" />
<setvar symbol="client_vmm_2_blk_data" region_paddr="client_vmm_2_blk_data" />

<virtual_machine name="blk_driver_linux" priority="200">
<vcpu id="0" />

Expand All @@ -206,16 +212,17 @@
<map mr="bus2-2" vaddr="0xff804000" perms="rw" cached="false" />
<map mr="bus3" vaddr="0xffd00000" perms="rw" cached="false" />

<!-- UIO metadata -->
<map mr="driver_blk_uio_irq_status" vaddr="0x79800000" perms="rw" cached="false" />
<map mr="driver_blk_vmm_info_passing" vaddr="0x80200000" perms="rw" cached="false" />

<!-- sDDF block -->
<map mr="driver_blk_storage_info" vaddr="0x80000000" perms="rw" cached="false" />
<map mr="driver_blk_req_queue" vaddr="0x80200000" perms="rw" cached="false" />
<map mr="driver_blk_resp_queue" vaddr="0x80400000" perms="rw" cached="false" />
<map mr="driver_blk_data" vaddr="0x60600000" perms="rw" cached="true" />
<map mr="client_vmm_1_blk_data" vaddr="0x60800000" perms="rw" cached="true" />
<map mr="client_vmm_2_blk_data" vaddr="0x60a00000" perms="rw" cached="true" />

<!-- UIO irq status -->
<map mr="uio_irq_status" vaddr="0x79800000" perms="rw" cached="true" />
<map mr="driver_blk_storage_info" vaddr="0x80400000" perms="rw" cached="false" />
<map mr="driver_blk_req_queue" vaddr="0x80600000" perms="rw" cached="false" />
<map mr="driver_blk_resp_queue" vaddr="0x80800000" perms="rw" cached="false" />
<map mr="driver_blk_data" vaddr="0x80a00000" perms="rw" cached="true" />
<map mr="client_vmm_1_blk_data" vaddr="0x80c00000" perms="rw" cached="true" />
<map mr="client_vmm_2_blk_data" vaddr="0x80e00000" perms="rw" cached="true" />
</virtual_machine>
<!-- SD Card -->
<irq irq="222" id="1"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
reg = <
0x00 0x79800000 0x00 0x1000
0x00 0x80000000 0x00 0x1000
0x00 0x80200000 0x00 0x200000
0x00 0x80200000 0x00 0X1000
0x00 0x80400000 0x00 0x200000
0x00 0x80600000 0x00 0x200000
>;
Expand All @@ -29,6 +29,8 @@
reg = <
0x00 0x80800000 0x00 0x200000
0x00 0x80a00000 0x00 0x200000
0x00 0x80c00000 0x00 0x200000
0x00 0x80e00000 0x00 0x200000
>;
interrupts = <0x00 19 0x04>;
// IRQ = 51
Expand Down
35 changes: 21 additions & 14 deletions examples/virtio/board/qemu_virt_aarch64/virtio.system
Original file line number Diff line number Diff line change
Expand Up @@ -167,20 +167,21 @@
<memory_region name="driver_blk_storage_info" size="0x1000" page_size="0x1000"/>
<memory_region name="driver_blk_req_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="driver_blk_resp_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="driver_blk_data" size="0x200_000" page_size="0x200_000" phys_addr="0x80600000"/>
<memory_region name="driver_blk_data" size="0x200_000" page_size="0x200_000"/>

<memory_region name="client_vmm_1_blk_storage_info" size="0x1000" page_size="0x1000" />
<memory_region name="client_vmm_1_blk_req_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_1_blk_resp_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_1_blk_data" size="0x200_000" page_size="0x200_000" phys_addr="0x80800000"/>
<memory_region name="client_vmm_1_blk_data" size="0x200_000" page_size="0x200_000"/>

<memory_region name="client_vmm_2_blk_storage_info" size="0x1000" page_size="0x1000" />
<memory_region name="client_vmm_2_blk_req_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_2_blk_resp_queue" size="0x200_000" page_size="0x200_000"/>
<memory_region name="client_vmm_2_blk_data" size="0x200_000" page_size="0x200_000" phys_addr="0x80a00000"/>
<memory_region name="client_vmm_2_blk_data" size="0x200_000" page_size="0x200_000"/>

<!-- UIO irq status page -->
<memory_region name="uio_irq_status" size="0x1000" page_size="0x1000" />
<!-- UIO metadata -->
<memory_region name="driver_blk_uio_irq_status" size="0x1000" page_size="0x1000" />
<memory_region name="driver_blk_vmm_info_passing" size="0x1000" page_size="0x1000" />

<protection_domain name="BLK_DRIVER_VMM" priority="200" budget="100" period="400">
<program_image path="blk_driver_vmm.elf" />
Expand All @@ -191,6 +192,11 @@
<map mr="driver_blk_vmm_serial_tx_data" vaddr="0x6_002_000" perms="rw" cached="true" setvar_vaddr="serial_tx_data" />
<map mr="driver_blk_vmm_serial_rx_data" vaddr="0x6_004_000" perms="rw" cached="true" setvar_vaddr="serial_rx_data" />

<map mr="driver_blk_vmm_info_passing" vaddr="0x80200000" perms="rw" cached="false" setvar_vaddr="driver_blk_vmm_info_passing" />
<setvar symbol="virt_blk_data" region_paddr="driver_blk_data" />
<setvar symbol="client_vmm_1_blk_data" region_paddr="client_vmm_1_blk_data" />
<setvar symbol="client_vmm_2_blk_data" region_paddr="client_vmm_2_blk_data" />

<virtual_machine name="blk_driver_linux" priority="200">
<vcpu id="0" />

Expand All @@ -199,16 +205,17 @@
<map mr="virtio_blk" vaddr="0xa000000" perms="rw" cached="false" />
<map mr="uart" vaddr="0x9000000" perms="rw" cached="false" />

<!-- UIO metadata -->
<map mr="driver_blk_uio_irq_status" vaddr="0x79800000" perms="rw" cached="false" />
<map mr="driver_blk_vmm_info_passing" vaddr="0x80200000" perms="rw" cached="false" />

<!-- sDDF block -->
<map mr="driver_blk_storage_info" vaddr="0x80000000" perms="rw" cached="false" />
<map mr="driver_blk_req_queue" vaddr="0x80200000" perms="rw" cached="false" />
<map mr="driver_blk_resp_queue" vaddr="0x80400000" perms="rw" cached="false" />
<map mr="driver_blk_data" vaddr="0x80600000" perms="rw" cached="true" />
<map mr="client_vmm_1_blk_data" vaddr="0x80800000" perms="rw" cached="true" />
<map mr="client_vmm_2_blk_data" vaddr="0x80a00000" perms="rw" cached="true" />

<!-- UIO irq status -->
<map mr="uio_irq_status" vaddr="0x79800000" perms="rw" cached="true" />
<map mr="driver_blk_storage_info" vaddr="0x80400000" perms="rw" cached="false" />
<map mr="driver_blk_req_queue" vaddr="0x80600000" perms="rw" cached="false" />
<map mr="driver_blk_resp_queue" vaddr="0x80800000" perms="rw" cached="false" />
<map mr="driver_blk_data" vaddr="0x80a00000" perms="rw" cached="true" />
<map mr="client_vmm_1_blk_data" vaddr="0x80c00000" perms="rw" cached="true" />
<map mr="client_vmm_2_blk_data" vaddr="0x80e00000" perms="rw" cached="true" />
</virtual_machine>
<!-- SD card IRQ -->
<irq irq="79" id="1" trigger="edge" />
Expand Down
1 change: 1 addition & 0 deletions examples/virtio/virtio.mk
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ CFLAGS := \
-I$(BOARD_DIR)/include \
-I$(SDDF)/include \
-I$(LIBVMM)/include \
-I$(LIBVMM)/tools/linux/include \
-I$(VIRTIO_EXAMPLE)/include \
-I$(SDDF)/$(LWIPDIR)/include \
-I$(SDDF)/$(LWIPDIR)/include/ipv4 \
Expand Down
18 changes: 18 additions & 0 deletions include/libvmm/uio/uio.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* Copyright 2024, UNSW
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once

#include <microkit.h>
#include <stdint.h>

/* This sets up the resources needed by a userspace driver. Includes registering
* the uio virq and vmm notify region. The uio driver will write into the notify
* region in order to transfer execution to the VMM for it to then notify other
* microkit components.
*/
bool uio_register_driver(int irq, microkit_channel *ch,
uintptr_t notify_region_base,
size_t notify_region_size);
46 changes: 46 additions & 0 deletions src/uio/uio.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@

/*
* Copyright 2024, UNSW
*
* SPDX-License-Identifier: BSD-2-Clause
*/

#include <libvmm/arch/aarch64/fault.h>
#include <libvmm/util/util.h>
#include <libvmm/virq.h>
#include <microkit.h>
#include <stddef.h>
#include <stdint.h>

bool uio_notify_fault_handle(size_t vcpu_id, size_t offset, size_t fsr,
seL4_UserContext *regs, void *data) {
microkit_channel ch = *(microkit_channel *)data;
if (fault_is_read(fsr)) {
LOG_VMM_ERR(
"Read into VMM notify region, but uio driver should never do that\n");
return false;
} else {
microkit_notify(ch);
}
return true;
}

void uio_ack(size_t vcpu_id, int irq, void *cookie) {
/* Do nothing for UIO ack */
}

bool uio_register_driver(int irq, microkit_channel *ch,
uintptr_t notify_region_base,
size_t notify_region_size) {
bool success = virq_register(GUEST_VCPU_ID, irq, uio_ack, NULL);
assert(success);
success = fault_register_vm_exception_handler(
notify_region_base, notify_region_size, &uio_notify_fault_handle, ch);
if (!success) {
LOG_VMM_ERR("Could not register uio virtual memory fault handler for "
"uio notify region [0x%lx..0x%lx)\n",
notify_region_base, notify_region_base + notify_region_size);
return false;
}
return true;
}
13 changes: 11 additions & 2 deletions tools/linux/include/uio/blk.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,16 @@
*/
#pragma once

#include <stddef.h>
#include <stdint.h>

int driver_init(void **maps, uintptr_t *maps_phys, int num_maps, int argc, char **argv);
void driver_notified();
#define MAX_BLK_CLIENTS 64

typedef struct driver_blk_vmm_info_passing {
uintptr_t client_data_phys[MAX_BLK_CLIENTS];
size_t client_data_size[MAX_BLK_CLIENTS];
} driver_blk_vmm_info_passing_t;

int driver_init(int uio_fd, void **maps, uintptr_t *maps_phys, int num_maps,
int argc, char **argv);
void driver_notified(int *events_fd, int num_events);
14 changes: 12 additions & 2 deletions tools/linux/include/uio/libuio.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,15 @@
*/
#pragma once

/* Notify the VMM */
void uio_notify();
/* Notify the VMM by writing into a registered
* memory region thereby invoking a hyp trap.
*/
void vmm_notify();

/* Writing 1 to the UIO device ACKs the IRQ and
* also re-enables the interrupt.
*/
void uio_irq_ack_and_enable();

/* Drivers can add their own events to the uio event loop */
void bind_fd_to_epoll(int fd);
Loading

0 comments on commit 8d3f665

Please sign in to comment.