Add support for FPGAs connected via the vfio-platform driver

.gitignore

@@ -22,6 +22,7 @@ compile_commands.json
 
 python/dist/
 python/.eggs/
+python/villas/node/villas_pb2.py
 
 # YouCompleteMe
 .ycm_extra_conf.py

common/include/villas/kernel/devices/device.hpp

@@ -0,0 +1,33 @@
+/* Interface for Linux/Unix devices.
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <optional>
+#include <villas/kernel/devices/driver.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class Device {
+public:
+  virtual ~Device(){};
+
+  virtual std::optional<std::unique_ptr<Driver>> driver() const = 0;
+  virtual std::optional<int> iommu_group() const = 0;
+  virtual std::string name() const = 0;
+  virtual std::filesystem::path override_path() const = 0;
+  virtual std::filesystem::path path() const = 0;
+  virtual void probe() const = 0;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/device_ip_matcher.hpp

@@ -0,0 +1,39 @@
+/* Devicematcher
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <villas/fpga/core.hpp>
+#include <villas/kernel/devices/ip_device.hpp>
+
+class DeviceIpMatcher {
+private:
+  std::vector<villas::kernel::devices::IpDevice> devices;
+  std::list<std::shared_ptr<villas::fpga::ip::Core>> ips;
+
+public:
+  DeviceIpMatcher(std::vector<villas::kernel::devices::IpDevice> devices,
+                  std::list<std::shared_ptr<villas::fpga::ip::Core>> ips)
+      : devices(devices), ips(ips) {}
+
+  std::vector<std::pair<std::shared_ptr<villas::fpga::ip::Core>,
+                        villas::kernel::devices::IpDevice>>
+  match() const {
+    std::vector<std::pair<std::shared_ptr<villas::fpga::ip::Core>,
+                          villas::kernel::devices::IpDevice>>
+        pairs;
+    for (auto device : devices) {
+      for (auto ip : ips) {
+        if (ip->getBaseaddr() == device.addr()) {
+          pairs.push_back(std::make_pair(ip, device));
+        }
+      }
+    }
+    return pairs;
+  }
+};

common/include/villas/kernel/devices/driver.hpp

@@ -0,0 +1,29 @@
+/* Interface for device drivers. OS/platform independend. 
+ * Implemented for Linux/Unix drivers in linux_driver.hpp
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class Device;
+
+class Driver {
+public:
+  virtual void attach(const Device &device) const = 0;
+  virtual void bind(const Device &device) const = 0;
+  virtual std::string name() const = 0;
+  virtual void override(const Device &device) const = 0;
+  virtual void unbind(const Device &device) const = 0;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/ip_device.hpp

@@ -0,0 +1,35 @@
+/* IpDevice: Linux/Unix device which represents an IP component of a FPGA.
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <villas/kernel/devices/platform_device.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class IpDevice : public PlatformDevice {
+public:
+  static IpDevice from(const std::filesystem::path unsafe_path);
+  static bool is_path_valid(const std::filesystem::path unsafe_path);
+
+private:
+  IpDevice() = delete;
+  IpDevice(const std::filesystem::path valid_path) //! Dont allow unvalidated paths
+      : PlatformDevice(valid_path){};
+
+public:
+  size_t addr() const;
+  std::string ip_name() const;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/ip_device_reader.hpp

@@ -0,0 +1,37 @@
+/* Devicetree reader for IpDevices
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <stdexcept>
+#include <vector>
+#include <filesystem>
+
+#include <villas/kernel/devices/ip_device.hpp>
+#include <villas/utils.hpp>
+
+class IpDeviceReader {
+public:
+  std::vector<villas::kernel::devices::IpDevice>
+  read(std::filesystem::path devices_directory) const {
+    std::vector<villas::kernel::devices::IpDevice> devices;
+
+    const std::vector<std::string> devicetree_names =
+        villas::utils::read_names_in_directory(devices_directory);
+    for (auto devicetree_name : devicetree_names) {
+      auto path_to_device =
+          devices_directory / std::filesystem::path(devicetree_name);
+      try {
+        auto device = villas::kernel::devices::IpDevice::from(path_to_device);
+        devices.push_back(device);
+      } catch (std::runtime_error &e) {
+      }
+    }
+    return devices;
+  }
+};

common/include/villas/kernel/devices/linux_driver.hpp

@@ -0,0 +1,52 @@
+/* Implementation of driver interface for Linux/Unix based operation system drivers.
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <villas/kernel/devices/driver.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class LinuxDriver : public Driver {
+private:
+  static constexpr char BIND_DEFAULT[] = "bind";
+  static constexpr char UNBIND_DEFAULT[] = "unbind";
+
+public:
+  const std::filesystem::path path;
+
+private:
+  const std::filesystem::path bind_path;
+  const std::filesystem::path unbind_path;
+
+public:
+  LinuxDriver(const std::filesystem::path path)
+      : LinuxDriver(path, path / std::filesystem::path(BIND_DEFAULT),
+                    path / std::filesystem::path(UNBIND_DEFAULT)){};
+
+  LinuxDriver(const std::filesystem::path path,
+              const std::filesystem::path bind_path,
+              const std::filesystem::path unbind_path)
+      : path(path), bind_path(bind_path), unbind_path(unbind_path){};
+
+public:
+  void attach(const Device &device) const override;
+  void bind(const Device &device) const override;
+  std::string name() const override;
+  void override(const Device &device) const override;
+  void unbind(const Device &device) const override;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/pci_device.hpp

@@ -15,6 +15,8 @@
 
 #include <villas/log.hpp>
 
+#include <villas/kernel/devices/device.hpp>
+
 namespace villas {
 namespace kernel {
 namespace devices {
@@ -58,7 +60,11 @@ struct Region {
   unsigned long long flags;
 };
 
-class PciDevice {
+class PciDevice : public Device {
+private:
+  static constexpr char PROBE_DEFAULT[] = "/sys/bus/pci/drivers_probe";
+  static constexpr char OVERRIDE_DEFAULT[] = "driver_override";
+
 public:
   PciDevice(Id i, Slot s) : id(i), slot(s), log(Log::get("kernel:pci")) {}
 
@@ -68,15 +74,17 @@ class PciDevice {
 
   bool operator==(const PciDevice &other);
 
-  // Get currently loaded driver for device
-  std::string getDriver() const;
+  // Implement device interface
+  std::optional<std::unique_ptr<Driver>> driver() const override;
+  std::optional<int> iommu_group() const override;
+  std::string name() const override;
+  std::filesystem::path override_path() const override;
+  std::filesystem::path path() const override;
+  void probe() const override;
 
   // Bind a new LKM to the PCI device
   bool attachDriver(const std::string &driver) const;
 
-  // Return the IOMMU group of this PCI device or -1 if the device is not in a group
-  int getIommuGroup() const;
-
   std::list<Region> getRegions() const;
 
   // Write 32-bit BAR value from to the PCI configuration space

common/include/villas/kernel/devices/platform_device.hpp

@@ -0,0 +1,51 @@
+/* Platform Device: Platform bus based Linux/Unix device.
+ *
+ * Author: Pascal Bauer <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <villas/kernel/devices/device.hpp>
+#include <villas/kernel/devices/driver.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class PlatformDevice : public Device {
+private:
+  static constexpr char PROBE_DEFAULT[] = "/sys/bus/platform/drivers_probe";
+  static constexpr char OVERRIDE_DEFAULT[] = "driver_override";
+
+private:
+  const std::filesystem::path m_path;
+  const std::filesystem::path m_probe_path;
+  const std::filesystem::path m_override_path;
+
+public:
+  PlatformDevice(const std::filesystem::path path)
+      : PlatformDevice(path, std::filesystem::path(PROBE_DEFAULT),
+                       path / std::filesystem::path(OVERRIDE_DEFAULT)){};
+
+  PlatformDevice(const std::filesystem::path path,
+                 const std::filesystem::path probe_path,
+                 const std::filesystem::path override_path)
+      : m_path(path), m_probe_path(probe_path),
+        m_override_path(override_path){};
+
+  // Implement device interface
+  std::optional<std::unique_ptr<Driver>> driver() const override;
+  std::optional<int> iommu_group() const override;
+  std::string name() const override;
+  std::filesystem::path override_path() const override;
+  std::filesystem::path path() const override;
+  void probe() const override;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/vfio_device.hpp

@@ -49,6 +49,8 @@ class Device {
   // Get the size of a device memory region
   size_t regionGetSize(size_t index);
 
+  int platformInterruptInit(int efds[32]);
+
   // Enable memory accesses and bus mastering for PCI device
   bool pciEnable();
 

common/include/villas/utils.hpp

@@ -11,6 +11,7 @@
 #include <list>
 #include <string>
 #include <vector>
+#include <filesystem>
 
 #include <cassert>
 #include <cstdint>
@@ -211,6 +212,9 @@ template <class... Ts> struct overloaded : Ts... {
 // explicit deduction guide (not needed as of C++20)
 template <class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
+void write_to_file(std::string data, const std::filesystem::path file);
+std::vector<std::string> read_names_in_directory(const std::filesystem::path &directory);
+
 namespace base64 {
 
 using byte = std::uint8_t;

common/lib/CMakeLists.txt

@@ -39,7 +39,10 @@ endif()
 
 if(CMAKE_SYSTEM_NAME STREQUAL Linux)
     target_sources(villas-common PRIVATE
+        kernel/devices/ip_device.cpp
+        kernel/devices/linux_driver.cpp
         kernel/devices/pci_device.cpp
+        kernel/devices/platform_device.cpp
         kernel/vfio_device.cpp
         kernel/vfio_group.cpp
         kernel/vfio_container.cpp