refactor: set VirtIO interrupt during activation

The MMIO transport for VirtIO devices uses an `IrqTrigger` object as the
object that models the logic for sending interrupts from the device to
the guest. We create one such object for every VirtIO device when
creating it. The MMIO device manager associates this object with an IRQ
number and registers it with KVM.

This commit changes the timing of association of an `IrqTrigger` with a
VirtIO-mmio device. It only assigns such an object to the device during
its activation. We do this to prepare for supporting a PCI transport for
VirtIO devices. The cloud hypervisor implementation for these passes the
interrupt objects used by the device during activation, so we make this
change to have a uniform way to handle interrupts for both transports.
Functionally, nothing changes for MMIO devices, as before activation we
don't trigger any interrupts.

Signed-off-by: Babis Chalios <bchalios@amazon.es>

Author: bchalios

src/vmm/src/builder.rs

@@ -44,7 +44,7 @@ use crate::devices::virtio::block::device::Block;
 use crate::devices::virtio::device::VirtioDevice;
 use crate::devices::virtio::net::Net;
 use crate::devices::virtio::rng::Entropy;
-use crate::devices::virtio::transport::mmio::MmioTransport;
+use crate::devices::virtio::transport::mmio::{IrqTrigger, MmioTransport};
 use crate::devices::virtio::vsock::{Vsock, VsockUnixBackend};
 #[cfg(feature = "gdb")]
 use crate::gdb;
@@ -597,8 +597,14 @@ fn attach_virtio_device<T: 'static + VirtioDevice + MutEventSubscriber + Debug>(
 ) -> Result<(), MmioError> {
     event_manager.add_subscriber(device.clone());
 
+    let interrupt = Arc::new(IrqTrigger::new()?);
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    let device = MmioTransport::new(vmm.vm.guest_memory().clone(), device, is_vhost_user);
+    let device = MmioTransport::new(
+        vmm.vm.guest_memory().clone(),
+        interrupt,
+        device,
+        is_vhost_user,
+    );
     vmm.mmio_device_manager
         .register_mmio_virtio_for_boot(
             vmm.vm.fd(),

src/vmm/src/device_manager/mmio.rs

@@ -53,6 +53,8 @@ pub enum MmioError {
     InvalidDeviceType,
     /// {0}
     InternalDeviceError(String),
+    /// Could not create IRQ for MMIO device: {0}
+    CreateIrq(#[from] std::io::Error),
     /// Invalid MMIO IRQ configuration.
     InvalidIrqConfig,
     /// Failed to register IO event: {0}
@@ -205,7 +207,7 @@ impl MMIODeviceManager {
                 vm.register_ioevent(queue_evt, &io_addr, u32::try_from(i).unwrap())
                     .map_err(MmioError::RegisterIoEvent)?;
             }
-            vm.register_irqfd(&locked_device.interrupt_trigger().irq_evt, irq.get())
+            vm.register_irqfd(&mmio_device.interrupt.irq_evt, irq.get())
                 .map_err(MmioError::RegisterIrqFd)?;
         }
 
@@ -549,7 +551,8 @@ mod tests {
             cmdline: &mut kernel_cmdline::Cmdline,
             dev_id: &str,
         ) -> Result<u64, MmioError> {
-            let mmio_device = MmioTransport::new(guest_mem, device, false);
+            let interrupt = Arc::new(IrqTrigger::new().unwrap());
+            let mmio_device = MmioTransport::new(guest_mem, interrupt, device, false);
             let device_info = self.register_mmio_virtio_for_boot(
                 vm,
                 resource_allocator,
@@ -576,7 +579,7 @@ mod tests {
         dummy: u32,
         queues: Vec<Queue>,
         queue_evts: [EventFd; 1],
-        interrupt_trigger: IrqTrigger,
+        interrupt_trigger: Option<Arc<IrqTrigger>>,
     }
 
     impl DummyDevice {
@@ -585,7 +588,7 @@ mod tests {
                 dummy: 0,
                 queues: QUEUE_SIZES.iter().map(|&s| Queue::new(s)).collect(),
                 queue_evts: [EventFd::new(libc::EFD_NONBLOCK).expect("cannot create eventFD")],
-                interrupt_trigger: IrqTrigger::new().expect("cannot create eventFD"),
+                interrupt_trigger: None,
             }
         }
     }
@@ -617,8 +620,11 @@ mod tests {
             &self.queue_evts
         }
 
-        fn interrupt_trigger(&self) -> &IrqTrigger {
-            &self.interrupt_trigger
+        fn interrupt_trigger(&self) -> Arc<IrqTrigger> {
+            self.interrupt_trigger
+                .as_ref()
+                .expect("Device is not activated")
+                .clone()
         }
 
         fn ack_features_by_page(&mut self, page: u32, value: u32) {
@@ -636,7 +642,11 @@ mod tests {
             let _ = data;
         }
 
-        fn activate(&mut self, _: GuestMemoryMmap) -> Result<(), ActivateError> {
+        fn activate(
+            &mut self,
+            _: GuestMemoryMmap,
+            _: Arc<IrqTrigger>,
+        ) -> Result<(), ActivateError> {
             Ok(())
         }
 

src/vmm/src/device_manager/persist.rs

@@ -34,7 +34,7 @@ use crate::devices::virtio::rng::Entropy;
 use crate::devices::virtio::rng::persist::{
     EntropyConstructorArgs, EntropyPersistError as EntropyError, EntropyState,
 };
-use crate::devices::virtio::transport::mmio::MmioTransport;
+use crate::devices::virtio::transport::mmio::{IrqTrigger, MmioTransport};
 use crate::devices::virtio::vsock::persist::{
     VsockConstructorArgs, VsockState, VsockUdsConstructorArgs,
 };
@@ -473,11 +473,13 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                                   as_subscriber: Arc<Mutex<dyn MutEventSubscriber>>,
                                   id: &String,
                                   state: &MmioTransportState,
+                                  interrupt: Arc<IrqTrigger>,
                                   device_info: &MMIODeviceInfo,
                                   event_manager: &mut EventManager|
          -> Result<(), Self::Error> {
             let restore_args = MmioTransportConstructorArgs {
                 mem: mem.clone(),
+                interrupt,
                 device,
                 is_vhost_user,
             };
@@ -512,9 +514,12 @@ impl<'a> Persist<'a> for MMIODeviceManager {
         };
 
         if let Some(balloon_state) = &state.balloon_device {
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Balloon::restore(
                 BalloonConstructorArgs {
                     mem: mem.clone(),
+                    interrupt: interrupt.clone(),
                     restored_from_file: constructor_args.restored_from_file,
                 },
                 &balloon_state.device_state,
@@ -530,14 +535,20 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &balloon_state.device_id,
                 &balloon_state.transport_state,
+                interrupt,
                 &balloon_state.device_info,
                 constructor_args.event_manager,
             )?;
         }
 
         for block_state in &state.block_devices {
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Block::restore(
-                BlockConstructorArgs { mem: mem.clone() },
+                BlockConstructorArgs {
+                    mem: mem.clone(),
+                    interrupt: interrupt.clone(),
+                },
                 &block_state.device_state,
             )?));
 
@@ -551,6 +562,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &block_state.device_id,
                 &block_state.transport_state,
+                interrupt,
                 &block_state.device_info,
                 constructor_args.event_manager,
             )?;
@@ -573,9 +585,12 @@ impl<'a> Persist<'a> for MMIODeviceManager {
         }
 
         for net_state in &state.net_devices {
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Net::restore(
                 NetConstructorArgs {
                     mem: mem.clone(),
+                    interrupt: interrupt.clone(),
                     mmds: constructor_args
                         .vm_resources
                         .mmds
@@ -596,6 +611,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &net_state.device_id,
                 &net_state.transport_state,
+                interrupt,
                 &net_state.device_info,
                 constructor_args.event_manager,
             )?;
@@ -606,9 +622,12 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 cid: vsock_state.device_state.frontend.cid,
             };
             let backend = VsockUnixBackend::restore(ctor_args, &vsock_state.device_state.backend)?;
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Vsock::restore(
                 VsockConstructorArgs {
                     mem: mem.clone(),
+                    interrupt: interrupt.clone(),
                     backend,
                 },
                 &vsock_state.device_state.frontend,
@@ -624,13 +643,16 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &vsock_state.device_id,
                 &vsock_state.transport_state,
+                interrupt,
                 &vsock_state.device_info,
                 constructor_args.event_manager,
             )?;
         }
 
         if let Some(entropy_state) = &state.entropy_device {
-            let ctor_args = EntropyConstructorArgs::new(mem.clone());
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
+            let ctor_args = EntropyConstructorArgs::new(mem.clone(), interrupt.clone());
 
             let device = Arc::new(Mutex::new(Entropy::restore(
                 ctor_args,
@@ -647,6 +669,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &entropy_state.device_id,
                 &entropy_state.transport_state,
+                interrupt,
                 &entropy_state.device_info,
                 constructor_args.event_manager,
             )?;