Merge branch 'master' into master

README.md

@@ -64,7 +64,7 @@ chmod +x /usr/local/bin/poseidon
 
 #### Faucet Configuration
 <img src="/docs/img/faucet.png" width="190" height="100">
-Poseidon requires at least Faucet version 1.8.6 or higher.
+NOTE: Poseidon requires at least Faucet version 1.9.46 or higher.
 
 Unless Poseidon and Faucet are running on the same host, Poseidon will connect to Faucet using SSH.  So you'll need to create an account that can SSH to the machine running Faucet and that has rights to modify the configuration file `faucet.yaml` (currently Poseidon expects it to be in the default `/etc/faucet/faucet.yaml` location and `dps` and `acls` must all be defined in `faucet.yaml` (not in `include`) for Poseidon to update the network posture correctly).
 

config/poseidon.config

@@ -8,6 +8,9 @@ controller_type = faucet
 controller_uri =
 automated_acls = False
 rules_file = /opt/poseidon/rules.yaml
+# A single string, being the interface name Poseidon will receive mirrored traffic on.
+# A single Poseidon instance can support just one collector_nic (to mirror multiple
+# switches centrally, use FAUCET stacking - see below).
 collector_nic = lo
 network_tap_ip = network_tap
 network_tap_port = 8080

poseidon/controllers/faucet/parser.py

@@ -10,6 +10,7 @@
 from poseidon.controllers.faucet.helpers import parse_rules
 from poseidon.controllers.faucet.helpers import yaml_in
 from poseidon.controllers.faucet.helpers import yaml_out
+from poseidon.volos.acls import Acl
 
 
 class Parser:
@@ -52,24 +53,26 @@ def _set_default_switch_conf(self, faucet_conf):
         if not faucet_conf:
             return faucet_conf
         dps = faucet_conf.get('dps', None)
+        acls = Acl()
         if dps is not None and self.mirror_ports:
+            acls.read(config_yaml=faucet_conf)
             root_stack_switch = [
                 switch for switch, switch_conf in dps.items()
                 if switch_conf.get('stack', {}).get('priority', None)]
             root_mirror_port = None
             if root_stack_switch:
                 root_stack_switch = root_stack_switch[0]
                 root_mirror_port = self.mirror_ports.get(root_stack_switch, None)
-                if 'acls' not in faucet_conf:
-                    faucet_conf['acls'] = {}
-                faucet_conf['acls'].update({
-                    self.tunnel_name: [
-                        # Safety rule to prevent looped packets being output to the loop.
-                        {'rule': {'vlan_vid': self.tunnel_vlan, 'actions': {'allow': 0}}},
-                        # Tunnel back to root.
-                        {'rule': {'actions': {'allow': 0, 'output': {'tunnel': {
-                            'type': 'vlan', 'tunnel_id': self.tunnel_vlan,
-                            'dp': root_stack_switch, 'port': root_mirror_port}}}}}]})
+                # Safety rule to prevent looped packets being output to the loop.
+                acls.add_rule(
+                    self.tunnel_name,
+                    {'rule': {'vlan_vid': self.tunnel_vlan, 'actions': {'allow': 0}}})
+                # Tunnel back to root.
+                acls.add_rule(
+                    self.tunnel_name,
+                    {'rule': {'actions': {'allow': 0, 'output': {'tunnel': {
+                        'type': 'vlan', 'tunnel_id': self.tunnel_vlan,
+                        'dp': root_stack_switch, 'port': root_mirror_port}}}}})
 
             for switch, mirror_port in self.mirror_ports.items():
                 if switch not in dps:
@@ -100,6 +103,8 @@ def _set_default_switch_conf(self, faucet_conf):
                             'coprocessor': {'strategy': 'vlan_vid'},
                         })
                 dps[switch] = switch_conf
+            # Merge ACL updates, if any back in)
+            faucet_conf['acls'] = acls.acls
         return faucet_conf
 
     @staticmethod

poseidon/helpers/actions.py

@@ -4,11 +4,10 @@
 @author: Charlie Lewis
 """
 from poseidon.helpers.collector import Collector
-from poseidon.volos.acls import Acl
-from poseidon.volos.coprocessor import Coprocessor
+from poseidon.volos.acls import VolosAcl
 
 
-class Actions(object):
+class Actions:
 
     def __init__(self, endpoint, sdnc):
         self.endpoint = endpoint
@@ -18,7 +17,6 @@ def shutdown_endpoint(self):
         ''' tell the controller to shutdown an endpoint '''
         if self.sdnc:
             self.sdnc.shutdown_endpoint()
-        return
 
     def mirror_endpoint(self):
         '''
@@ -27,9 +25,9 @@ def mirror_endpoint(self):
         '''
         status = False
         if self.sdnc:
-            if self.sdnc.mirror_mac(self.endpoint.endpoint_data['mac'], self.endpoint.endpoint_data['segment'], self.endpoint.endpoint_data['port']):
-                collector = Collector(self.endpoint,
-                                      self.endpoint.endpoint_data['segment'])
+            endpoint_data = self.endpoint.endpoint_data
+            if self.sdnc.mirror_mac(endpoint_data['mac'], endpoint_data['segment'], endpoint_data['port']):
+                collector = Collector(self.endpoint, endpoint_data['segment'])
                 if collector.nic:
                     status = collector.start_collector()
         else:
@@ -40,9 +38,9 @@ def unmirror_endpoint(self):
         ''' tell the controller to unmirror traffic '''
         status = False
         if self.sdnc:
-            if self.sdnc.unmirror_mac(self.endpoint.endpoint_data['mac'], self.endpoint.endpoint_data['segment'], self.endpoint.endpoint_data['port']):
-                collector = Collector(self.endpoint,
-                                      self.endpoint.endpoint_data['segment'])
+            endpoint_data = self.endpoint.endpoint_data
+            if self.sdnc.unmirror_mac(endpoint_data['mac'], endpoint_data['segment'], endpoint_data['port']):
+                collector = Collector(self.endpoint, endpoint_data['segment'])
                 if collector.nic:
                     status = collector.stop_collector()
         else:
@@ -55,17 +53,20 @@ def coprocess_endpoint(self):
         '''
         status = False
         if self.sdnc:
+            endpoint_data = self.endpoint.endpoint_data
             if self.sdnc.volos and self.sdnc.volos.enabled:
-                acl = Acl(self.endpoint, acl_dir=self.sdnc.volos.acl_dir, copro_vlans=[
-                          self.sdnc.volos.copro_vlan], copro_port=self.sdnc.volos.copro_port)
+                acl = VolosAcl(
+                    self.endpoint, acl_dir=self.sdnc.volos.acl_dir,
+                    copro_vlans=[self.sdnc.volos.copro_vlan], copro_port=self.sdnc.volos.copro_port)
                 endpoints = [self.endpoint]
                 force_apply_rules = [acl.acl_key]
                 coprocess_rules_files = [acl.acl_file]
-                port_list = self.sdnc.volos.get_port_list(self.endpoint.endpoint_data['mac'], ipv4=self.endpoint.endpoint_data.get(
-                    'ipv4', None), ipv6=self.endpoint.endpoint_data.get('ipv6', None))
+                port_list = self.sdnc.volos.get_port_list(
+                    endpoint_data['mac'], ipv4=endpoint_data.get('ipv4', None), ipv6=endpoint_data.get('ipv6', None))
                 if acl.ensure_acls_dir() and acl.write_acl_file(port_list):
                     status = self.sdnc.update_acls(
-                        rules_file=None, endpoints=endpoints, force_apply_rules=force_apply_rules, coprocess_rules_files=coprocess_rules_files)
+                        rules_file=None, endpoints=endpoints,
+                        force_apply_rules=force_apply_rules, coprocess_rules_files=coprocess_rules_files)
             else:
                 status = True
         return status
@@ -75,12 +76,14 @@ def uncoprocess_endpoint(self):
         status = False
         if self.sdnc:
             if self.sdnc.volos and self.sdnc.volos.enabled:
-                acl = Acl(self.endpoint, acl_dir=self.sdnc.volos.acl_dir, copro_vlans=[
-                          self.sdnc.volos.copro_vlan], copro_port=self.sdnc.volos.copro_port)
+                acl = VolosAcl(
+                    self.endpoint, acl_dir=self.sdnc.volos.acl_dir,
+                    copro_vlans=[self.sdnc.volos.copro_vlan], copro_port=self.sdnc.volos.copro_port)
                 endpoints = [self.endpoint]
                 force_remove_rules = [acl.acl_key]
                 if self.sdnc.update_acls(
-                        rules_file=None, endpoints=endpoints, force_remove_rules=force_remove_rules, coprocess_rules_files=None):
+                        rules_file=None, endpoints=endpoints,
+                        force_remove_rules=force_remove_rules, coprocess_rules_files=None):
                     status = acl.delete_acl_file()
         else:
             status = True
@@ -91,5 +94,6 @@ def update_acls(self, rules_file=None, endpoints=None, force_apply_rules=None, f
         status = False
         if self.sdnc:
             status = self.sdnc.update_acls(
-                rules_file=rules_file, endpoints=endpoints, force_apply_rules=force_apply_rules, force_remove_rules=force_remove_rules)
+                rules_file=rules_file, endpoints=endpoints,
+                force_apply_rules=force_apply_rules, force_remove_rules=force_remove_rules)
         return status

poseidon/volos/acls.py

@@ -3,94 +3,114 @@
 Created on 31 January 2020
 @author: Ryan Ashley
 """
-import json
+
 import logging
 import os
 from pathlib import Path
+from poseidon.controllers.faucet.helpers import yaml_in, yaml_out
+
+
+
+class Acl:
+
+    def __init__(self, acl_file=None):
+        self.acls = {}
+        self.acl_file = acl_file
+
+    def read(self, config_yaml=None):
+        try:
+            if config_yaml is None:
+                config_yaml = yaml_in(self.acl_file)
+            self.acls = config_yaml.get('acls', {})
+        except (FileNotFoundError, PermissionError):
+            return
+
+    def write(self):
+        try:
+            config = yaml_in(self.acl_file)
+            if 'acls' not in config:
+                config['acls'] = {}
+            config['acls'].update(self.acls)
+            yaml_out(self.acl_file, config)
+            return True
+        except (FileNotFoundError, PermissionError):
+            return False
+
+    def add_rule(self, name, rule):
+        if name not in self.acls:
+            self.acls[name] = []
+        self.acls[name].append(rule)
+
+
+class ExclusiveAcl(Acl):
 
-import yaml
+    def write(self):
+        try:
+            config = yaml_in(self.acl_file)
+            if 'acls' not in config:
+                config['acls'] = {}
+            config['acls'] = self.acls
+            yaml_out(self.acl_file, config)
+            return True
+        except (FileNotFoundError, PermissionError):
+            return False
 
 
-class Acl(object):
+class VolosAcl(ExclusiveAcl):
 
     def __init__(self, endpoint, acl_dir, copro_vlans=[2], copro_port=23):
+        self.mac = endpoint.endpoint_data['mac']
+        self.acl_key = f'volos_copro_{self.mac}'
+        self.acl_dir = acl_dir
+        acl_file = os.path.join(self.acl_dir, f'/%s.yaml' % self.acl_key)
+        super(VolosAcl, self).__init__(acl_file=acl_file)
         self.logger = logging.getLogger('coprocessor')
         self.endpoint = endpoint
         self.id = endpoint.name
-        self.mac = endpoint.endpoint_data['mac']
         self.copro_vlans = copro_vlans
         self.copro_port = copro_port
-        self.acl_dir = acl_dir
-        self.acl_key = f'volos_copro_{self.mac}'
-        self.acl_file = os.path.join(
-            self.acl_dir, f'/volos_copro_{self.mac}.yaml')
 
     def write_acl_file(self, port_list=[]):
-        acls = {}
-        acls[self.acl_key] = []
-        status = False
+        self.acls = {}
         for port in port_list:
-            if 'ipv4_addresses' in self.endpoint.metadata:
-                for ip in self.endpoint.metadata['ipv4_addresses']:
-                    rule = {'rule': {
-                        'dl_type': 0x800,
-                        'nw_proto': port['proto_id'],
-                        'ipv4_src': ip,
-                        'actions': {
-                            'output': {
-                                'ports': [self.copro_port],
-                                'vlan_vid': self.copro_vlans
-                            }
-                        },
-                    }}
-                    rule['rule'][port['proto'] + '_dst'] = port['port']
-                    acls[self.acl_key].append(rule)
-            if 'ipv6_addresses' in self.endpoint.metadata:
-                for ip in self.endpoint.metadata['ipv6_addresses']:
-                    rule = {'rule': {
-                        'dl_type': 0x86dd,
-                        'nw_proto': port['proto_id'],
-                        'ipv6_src': ip,
-                        'actions': {
-                            'output': {
-                                'ports': [self.copro_port],
-                                'vlan_vid': self.copro_vlans
-                            }
-                        },
-
-                    }}
-                    rule['rule'][port['proto'] + '_dst'] = port['port']
-                    acls[self.acl_key].append(rule)
-        acls[self.acl_key].append({'rule': {'actions': {'allow': 1}}})
-        try:
-            with open(self.acl_file, 'w') as acl_file:
-                status = yaml.dump({'acls': acls}, acl_file)
-        except Exception as e:  # pragma: no cover
+            for eth_type in (0x0800, 0x86dd):
+                ip_str = port['proto']
+                addresses = self.endpoint.metadata.get('%s_addresses' % ip_str, None)
+                if addresses:
+                    for ip in addresses:
+                        rule = {'rule': {
+                            'dl_type': eth_type,
+                            'nw_proto': port['proto_id'],
+                            '%s_src' % ip_str: ip,
+                            'actions': {
+                                'output': {
+                                    'ports': [self.copro_port],
+                                    'vlan_vid': self.copro_vlans}}}}
+                    rule['rule']['%s_dst' % ip_str] = port['port']
+                    self.add_rule(self.acl_key, rule)
+        self.add_rule(self.acl_key, {'rule': {'actions': {'allow': 1}}})
+        status = self.write()
+        if not status:
             self.logger.error(
                 'Volos ACL file:{0} could not be written. Coprocessing may not work as expected'.format(self.acl_file))
-            status = False
         return status
 
     def delete_acl_file(self):
-        status = False
         try:
             if os.path.exists(self.acl_file):
                 os.remove(self.acl_file)
-                status = True
+                return True
         except Exception as e:  # pragma: no cover
             self.logger.error(
                 'Volos ACL file:{0} could not be deleted. Coprocessing may not work as expected'.format(self.acl_file))
-        return status
+        return False
 
     def ensure_acls_dir(self):
-        status = False
         try:
             if not os.path.exists(self.acl_dir):
                 Path(self.acl_dir).mkdir(parents=True, exist_ok=True)
-                status = True
+                return True
         except Exception as e:  # pragma: no cover
             self.logger.error(
                 'Volos ACL directory:{0} could not be created. Coprocessing may not work as expected'.format(self.acl_file))
-            status = False
-
-        return status
+        return False

requirements.txt

@@ -5,7 +5,7 @@ pika==1.1.0
 prometheus_client==0.8.0
 pyyaml==5.3.1
 redis==3.5.3
-requests==2.23.0
+requests==2.24.0
 schedule==0.6.0
 scp==0.13.2
 texttable==1.6.2

tests/test_parser.py

@@ -115,6 +115,11 @@ def test_stack_default_config():
                 stack:
                     dp: s1
                     port: 4
+acls:
+    existing_acl:
+        - rule:
+            actions:
+                allow: 1
 """
     new_switch_conf_str = """
 dps:
@@ -151,6 +156,10 @@ def test_stack_default_config():
                     dp: s1
                     port: 4
 acls:
+    existing_acl:
+        - rule:
+            actions:
+                allow: 1
     poseidon_tunnel:
         - rule:
             vlan_vid: 999

tests/test_volos.py

@@ -4,7 +4,7 @@
 
 from poseidon.helpers.config import Config
 from poseidon.helpers.endpoint import endpoint_factory
-from poseidon.volos.acls import Acl
+from poseidon.volos.acls import VolosAcl
 from poseidon.volos.coprocessor import Coprocessor
 from poseidon.volos.volos import Volos
 
@@ -25,7 +25,7 @@ def test_Acl():
     controller = Config().get_config()
     endpoint = endpoint_factory('foo')
     endpoint.endpoint_data = {'mac': '00:00:00:00:00:00'}
-    a = Acl(endpoint, controller['acl_dir'])
+    a = VolosAcl(endpoint, controller['acl_dir'])
 
 
 def test_Coprocessor():