runtime: implement distributed DMA

artiq/coredevice/dma.py

@@ -6,7 +6,7 @@
 """
 
 from artiq.language.core import syscall, kernel
-from artiq.language.types import TInt32, TInt64, TStr, TNone, TTuple
+from artiq.language.types import TInt32, TInt64, TStr, TNone, TTuple, TBool
 from artiq.coredevice.exceptions import DMAError
 
 from numpy import int64
@@ -17,7 +17,7 @@ def dma_record_start(name: TStr) -> TNone:
     raise NotImplementedError("syscall not simulated")
 
 @syscall
-def dma_record_stop(duration: TInt64) -> TNone:
+def dma_record_stop(duration: TInt64, enable_ddma: TBool) -> TNone:
     raise NotImplementedError("syscall not simulated")
 
 @syscall
@@ -47,6 +47,7 @@ class DMARecordContextManager:
     def __init__(self):
         self.name = ""
         self.saved_now_mu = int64(0)
+        self.enable_ddma = False
 
     @kernel
     def __enter__(self):
@@ -56,7 +57,7 @@ def __enter__(self):
 
     @kernel
     def __exit__(self, type, value, traceback):
-        dma_record_stop(now_mu()) # see above
+        dma_record_stop(now_mu(), self.enable_ddma) # see above
         at_mu(self.saved_now_mu)
 
 
@@ -74,12 +75,18 @@ def __init__(self, dmgr, core_device="core"):
         self.epoch    = 0
 
     @kernel
-    def record(self, name):
+    def record(self, name, enable_ddma=False):
         """Returns a context manager that will record a DMA trace called ``name``.
         Any previously recorded trace with the same name is overwritten.
-        The trace will persist across kernel switches."""
+        The trace will persist across kernel switches.
+        In DRTIO context, you can toggle distributed DMA with ``enable_ddma``.
+        Enabling it allows running DMA on satellites, rather than sending all
+        events from the master.
+        Disabling it may improve performance in some scenarios, 
+        e.g. when there are many small satellite buffers."""
         self.epoch += 1
         self.recorder.name = name
+        self.recorder.enable_ddma = enable_ddma
         return self.recorder
 
     @kernel

artiq/firmware/ksupport/lib.rs

@@ -269,7 +269,7 @@ extern fn dma_record_start(name: &CSlice<u8>) {
 }
 
 #[unwind(allowed)]
-extern fn dma_record_stop(duration: i64) {
+extern fn dma_record_stop(duration: i64, enable_ddma: bool) {
     unsafe {
         dma_record_flush();
 
@@ -285,7 +285,8 @@ extern fn dma_record_stop(duration: i64) {
 
         DMA_RECORDER.active = false;
         send(&DmaRecordStop {
-            duration: duration as u64
+            duration: duration as u64,
+            enable_ddma: enable_ddma
         });
     }
 }
@@ -370,7 +371,7 @@ extern fn dma_erase(name: &CSlice<u8>) {
 #[repr(C)]
 struct DmaTrace {
     duration: i64,
-    address:  i32,
+    address:  i32
 }
 
 #[unwind(allowed)]
@@ -404,6 +405,7 @@ extern fn dma_playback(timestamp: i64, ptr: i32) {
 
         csr::cri_con::selected_write(1);
         csr::rtio_dma::enable_write(1);
+        send(&DmaStartRemoteRequest { id: ptr as i32, timestamp: timestamp });
         while csr::rtio_dma::enable_read() != 0 {}
         csr::cri_con::selected_write(0);
 
@@ -424,6 +426,24 @@ extern fn dma_playback(timestamp: i64, ptr: i32) {
             }
         }
     }
+
+    send(&DmaAwaitRemoteRequest { id: ptr as i32 });
+    recv!(&DmaAwaitRemoteReply { timeout, error, channel, timestamp } => {
+        if timeout {
+            raise!("DMAError",
+                "Error running DMA on satellite device, timed out waiting for results");
+        }
+        if error & 1 != 0 {
+            raise!("RTIOUnderflow",
+                "RTIO underflow at channel {rtio_channel_info:0}, {1} mu",
+                channel as i64, timestamp as i64, 0);
+        }
+        if error & 2 != 0 {
+            raise!("RTIODestinationUnreachable",
+                "RTIO destination unreachable, output, at channel {rtio_channel_info:0}, {1} mu",
+                channel as i64, timestamp as i64, 0);
+        }
+    });
 }
 
 #[cfg(not(has_rtio_dma))]

artiq/firmware/libproto_artiq/drtioaux_proto.rs

@@ -14,8 +14,8 @@ impl<T> From<IoError<T>> for Error<T> {
     }
 }
 
-/* 512 (max size) - 4 (CRC) - 1 (packet ID) - 4 (trace ID) - 1 (last) - 2 (length) */
-const DMA_TRACE_MAX_SIZE: usize = 500;
+/* 512 (max size) - 4 (CRC) - 1 (packet ID) - 1 (destination) - 4 (trace ID) - 1 (last) - 2 (length) */
+pub const DMA_TRACE_MAX_SIZE: usize = 499;
 
 #[derive(PartialEq, Debug)]
 pub enum Packet {
@@ -58,13 +58,13 @@ pub enum Packet {
     SpiReadReply { succeeded: bool, data: u32 },
     SpiBasicReply { succeeded: bool },
 
-    DmaAddTraceRequest { id: u32, last: bool, length: u16, trace: [u8; DMA_TRACE_MAX_SIZE] },
+    DmaAddTraceRequest { destination: u8, id: u32, last: bool, length: u16, trace: [u8; DMA_TRACE_MAX_SIZE] },
     DmaAddTraceReply { succeeded: bool },
-    DmaRemoveTraceRequest { id: u32 },
+    DmaRemoveTraceRequest { destination: u8, id: u32 },
     DmaRemoveTraceReply { succeeded: bool },
-    DmaPlaybackRequest { id: u32, timestamp: u64 },
+    DmaPlaybackRequest { destination: u8, id: u32, timestamp: u64 },
     DmaPlaybackReply { succeeded: bool },
-    DmaPlaybackStatus { id: u32, error: u8, channel: u32, timestamp: u64 }
+    DmaPlaybackStatus { destination: u8, id: u32, error: u8, channel: u32, timestamp: u64 }
 
 }
 
@@ -198,12 +198,14 @@ impl Packet {
             },
 
             0xb0 => { 
+                let destination = reader.read_u8()?;
                 let id = reader.read_u32()?;
                 let last = reader.read_bool()?;
                 let length = reader.read_u16()?;
                 let mut trace: [u8; DMA_TRACE_MAX_SIZE] = [0; DMA_TRACE_MAX_SIZE];
                 reader.read_exact(&mut trace[0..length as usize])?;
                 Packet::DmaAddTraceRequest {
+                    destination: destination,
                     id: id,
                     last: last,
                     length: length as u16,
@@ -214,19 +216,22 @@ impl Packet {
                 succeeded: reader.read_bool()?
             },
             0xb2 => Packet::DmaRemoveTraceRequest {
+                destination: reader.read_u8()?,
                 id: reader.read_u32()?
             },
             0xb3 => Packet::DmaRemoveTraceReply {
                 succeeded: reader.read_bool()?
             },
             0xb4 => Packet::DmaPlaybackRequest {
+                destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 timestamp: reader.read_u64()?
             },
             0xb5 => Packet::DmaPlaybackReply {
                 succeeded: reader.read_bool()?
             },
             0xb6 => Packet::DmaPlaybackStatus {
+                destination: reader.read_u8()?,
                 id: reader.read_u32()?,
                 error: reader.read_u8()?,
                 channel: reader.read_u32()?,
@@ -392,8 +397,9 @@ impl Packet {
                 writer.write_bool(succeeded)?;
             },
 
-            Packet::DmaAddTraceRequest { id, last, trace, length } => {
+            Packet::DmaAddTraceRequest { destination, id, last, trace, length } => {
                 writer.write_u8(0xb0)?;
+                writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_bool(last)?;
                 // trace may be broken down to fit within drtio aux memory limit
@@ -405,25 +411,28 @@ impl Packet {
                 writer.write_u8(0xb1)?;
                 writer.write_bool(succeeded)?;
             },
-            Packet::DmaRemoveTraceRequest { id } => {
+            Packet::DmaRemoveTraceRequest { destination, id } => {
                 writer.write_u8(0xb2)?;
+                writer.write_u8(destination)?;
                 writer.write_u32(id)?;
             },
             Packet::DmaRemoveTraceReply { succeeded } => {
                 writer.write_u8(0xb3)?;
                 writer.write_bool(succeeded)?;
             },
-            Packet::DmaPlaybackRequest { id, timestamp } => {
+            Packet::DmaPlaybackRequest { destination, id, timestamp } => {
                 writer.write_u8(0xb4)?;
+                writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_u64(timestamp)?;
             },
             Packet::DmaPlaybackReply { succeeded } => {
                 writer.write_u8(0xb5)?;
                 writer.write_bool(succeeded)?;
             },
-            Packet::DmaPlaybackStatus { id, error, channel, timestamp } => {
+            Packet::DmaPlaybackStatus { destination, id, error, channel, timestamp } => {
                 writer.write_u8(0xb6)?;
+                writer.write_u8(destination)?;
                 writer.write_u32(id)?;
                 writer.write_u8(error)?;
                 writer.write_u32(channel)?;

artiq/firmware/libproto_artiq/kernel_proto.rs

@@ -20,7 +20,8 @@ pub enum Message<'a> {
     DmaRecordStart(&'a str),
     DmaRecordAppend(&'a [u8]),
     DmaRecordStop {
-        duration:  u64
+        duration:  u64,
+        enable_ddma: bool
     },
 
     DmaEraseRequest {
@@ -32,9 +33,24 @@ pub enum Message<'a> {
     },
     DmaRetrieveReply {
         trace:    Option<&'a [u8]>,
-        duration: u64
+        duration: u64,
     },
 
+    DmaStartRemoteRequest {
+        id: i32,
+        timestamp: i64,
+    },
+    DmaAwaitRemoteRequest {
+        id: i32
+    },
+    DmaAwaitRemoteReply {
+        timeout: bool,
+        error: u8,
+        channel: u32,
+        timestamp: u64
+    },
+
+
     RunFinished,
     RunException {
         exceptions: &'a [Option<eh::eh_artiq::Exception<'a>>],

artiq/firmware/runtime/main.rs

@@ -182,21 +182,24 @@ fn startup() {
     drtio_routing::interconnect_disable_all();
     let aux_mutex = sched::Mutex::new();
 
+    let ddma_mutex = sched::Mutex::new();
+
     let mut scheduler = sched::Scheduler::new(interface);
     let io = scheduler.io();
 
     if use_dhcp {
         io.spawn(4096, dhcp::dhcp_thread);
     }
 
-    rtio_mgt::startup(&io, &aux_mutex, &drtio_routing_table, &up_destinations);
+    rtio_mgt::startup(&io, &aux_mutex, &drtio_routing_table, &up_destinations, &ddma_mutex);
 
     io.spawn(4096, mgmt::thread);
     {
         let aux_mutex = aux_mutex.clone();
         let drtio_routing_table = drtio_routing_table.clone();
         let up_destinations = up_destinations.clone();
-        io.spawn(16384, move |io| { session::thread(io, &aux_mutex, &drtio_routing_table, &up_destinations) });
+        let ddma_mutex = ddma_mutex.clone();
+        io.spawn(16384, move |io| { session::thread(io, &aux_mutex, &drtio_routing_table, &up_destinations, &ddma_mutex) });
     }
     #[cfg(any(has_rtio_moninj, has_drtio))]
     {