Add route addition/deletion lifecycle for ControllerStatic

CapybaraNetty.py

@@ -23,7 +23,7 @@ class CapybaraNetty(metaclass=MetaCapybaraNetty):
     def __init__(self):
         self.config = {"external_targets_for_optimizations": [], "pinger_interval": 10}
         self.setup_config()
-        logging.basicConfig(stream=sys.stdout, format="%(levelname)-8s:%(name)-24s.%(funcName)-40s:%(message)-s")
+        logging.basicConfig(stream=sys.stdout, format="%(levelname)-8s:%(name)-40s.%(funcName)-40s:%(message)-s")
         logging.getLogger(CapybaraNetty.__name__).setLevel(self.config["log_level"])
 
     def setup_config(self):
@@ -57,7 +57,3 @@ class CapybaraNetty(metaclass=MetaCapybaraNetty):
     @staticmethod
     def log_format_ip(ip):
         return f"{ip:<16s}"
-
-    @staticmethod
-    def log_format_name(name):
-        return f"{name:<20s}"

ControllerStatic.py

@@ -21,7 +21,9 @@ class ControllerStatic(CapybaraNetty):
         if len(self.pinger_data) > 0:
             G = self.get_latency_graph()
             self.find_viable_paths(G)
-            await self.add_viable_external_routes(G)
+            current_best_routes = await self.get_viable_external_routes(G)
+            await self.install_viable_external_routes(current_best_routes)
+
             # print(G)
         # G_simple = await self.get_host_latency_overview()
 
@@ -80,11 +82,12 @@ class ControllerStatic(CapybaraNetty):
                         self.__logger.info(f"VIABLE OPTIMIZATION: {optimized_ms:<3d}ms {'->'.join(path[::-1])}")
         return G
 
-    async def add_viable_external_routes(self, G):
+    async def get_viable_external_routes(self, G):
+        current_best_routes = {}
         for node in G.nodes():
             if G.nodes[node]["type"] == "router" and "improved_path_info" in G.nodes[node]:
                 current_router = G.nodes[node]["obj"]
-                routes = []
+                current_router_improved_routes = []
                 for path_obj in G.nodes[node]["improved_path_info"]:
                     idx = path_obj["path"].index(node)
                     path_len = len(path_obj["path"])
@@ -99,7 +102,7 @@ class ControllerStatic(CapybaraNetty):
                             exit_gateway_ip = ipaddress.ip_interface(
                                 next_router_interfaces[inbound_interface_name]["ipAddresses"][0]["address"]
                             ).ip
-                            routes.append(
+                            current_router_improved_routes.append(
                                 {
                                     "prefix": f"{path_obj['target']}/32",
                                     "gateway": exit_gateway_ip,
@@ -109,8 +112,24 @@ class ControllerStatic(CapybaraNetty):
 
                     except IndexError:
                         self.__logger.debug(f"Skipping route for {path_obj['target']} due to IndexError")
-
-                await current_router.add_routes(routes)
+                # Store the new route objects for best paths
+                current_best_routes.update({current_router: current_router_improved_routes})
+
+        return current_best_routes
+
+    async def install_viable_external_routes(self, current_best_routes):
+        # Compare existing routes and the ones which are CURRENT best routes
+        # If there are any existing routes which are not used
+        for router, new_routes in current_best_routes.items():
+            # Convert dict to list of routes
+            current_managed_routes = list(router.managed_routes.values())
+            routes_to_be_deleted = [old_route for old_route in current_managed_routes if old_route not in new_routes]
+            routes_to_be_added = [new_route for new_route in new_routes if new_route not in current_managed_routes]
+            self.__logger.debug(f"{router.log_name()} has {len(routes_to_be_deleted)} DEL ROUTE actions")
+            self.__logger.debug(f"{router.log_name()} has {len(routes_to_be_added)} ADD ROUTE actions")
+            # todo async
+            await router.del_routes(routes_to_be_deleted)
+            await router.add_routes(routes_to_be_added)
 
     @staticmethod
     def get_improvement_obj(target, path, optimization_ms):

Router.py

@@ -70,7 +70,7 @@ class Router(CapybaraNetty):
             self.default_interface_name = default_route["nexthops"][0]["interfaceName"]
             return self.default_interface_name
         except StopIteration:
-            self.__logger.error(f"{self.log_format_name(self.name)} has no default route")
+            self.__logger.error(f"{self.log_name()} has no default route")
             raise
 
     def get_default_interface_ip(self):
@@ -91,15 +91,20 @@ class Router(CapybaraNetty):
         if self.debug_assert_routes:
             await self.assert_route_state()
 
+    async def del_routes(self, routes):
+        for route in routes:
+            self.del_managed_route_info(route)
+        stdout, stderr = await self.vty.del_routes(routes)
+        if self.debug_assert_routes:
+            await self.assert_route_state(must_not_exist=routes)
+
     async def del_existing_interfaces(self):
         command = f"sudo ip link delete group {self.interface_group_id}"
         try:
             await self.ssh_exec(command)
         except asyncssh.process.ProcessError as e:
             if "No such device" in e.stderr:
-                self.__logger.warning(
-                    f"{self.log_format_name(self.name)} There were no interfaces in group {self.interface_group_id}"
-                )
+                self.__logger.warning(f"{self.log_name()} There were no interfaces in group {self.interface_group_id}")
             else:
                 raise
         self.interfaces = []
@@ -114,6 +119,9 @@ class Router(CapybaraNetty):
     def add_managed_route_info(self, route):
         self.managed_routes.update({route["prefix"]: route})
 
+    def del_managed_route_info(self, route):
+        del self.managed_routes[route["prefix"]]
+
     def del_managed_routes_info(self):
         self.managed_routes = {}
 
@@ -148,19 +156,33 @@ class Router(CapybaraNetty):
         self.interfaces = await self.vty.get_interfaces()
         return self.interfaces
 
-    # Confirms that the routes we have created actually exist
-    async def assert_route_state(self):
+    # By default checks if our managed routes - exist on the router
+    # if must_not_exist is passed - will check for the on-existence of passed routes
+    async def assert_route_state(self, must_not_exist=None):
+        if must_not_exist is None:
+            must_not_exist = []
         await self.get_routes()
         for k, v in self.managed_routes.items():
             try:
-                self.__logger.debug(f"{self.log_format_name(self.name)} Asserting existence of {v['prefix']}")
+                self.__logger.debug(f"{self.log_name()} Asserting existence of {v['prefix']}")
                 assert any(route["prefix"] == v["prefix"] for route in self.routes)
             except AssertionError:
-                self.__logger.error(f"{self.log_format_name(self.name)} Route {v['prefix']} is not setup")
+                self.__logger.error(f"{self.log_name()} Route {v['prefix']} is not setup")
+                raise
+
+        for v in must_not_exist:
+            try:
+                self.__logger.debug(f"{self.log_name()} Asserting nonexistence of {v['prefix']}")
+                assert not any(route["prefix"] == v["prefix"] for route in self.routes)
+            except AssertionError:
+                self.__logger.error(f"{self.log_name()} Route {v['prefix']} is setup, but shouldn't be")
                 raise
 
     async def ssh_exec(self, command, **kwargs):
         return await super(Router, self).ssh_exec(self.ip, self.ssh_user, command=command)
 
     def __str__(self):
-        return f"{self.name} @ {self.ip} with {len(self.managed_interfaces)} interfaces"
+        return f"{self.log_name()} @ {self.ip} with {len(self.managed_interfaces)} interfaces"
+
+    def log_name(self):
+        return f"{self.name:<20s}"

Vty.py

@@ -14,8 +14,11 @@ class Vty(CapybaraNetty):
     def ssh_is_alive(self, **kwargs):
         return super(Vty, self).ssh_is_alive(self.host_ip, self.host_user)
 
-    def build_vty_exec(self, commands: list):
-        base = "sudo vtysh -c 'configure term'"
+    def build_vty_exec(self, commands: list, echo_commands=False):
+        additional_vtysh_flags = ""
+        if echo_commands:
+            additional_vtysh_flags += " -E "
+        base = f"sudo vtysh {additional_vtysh_flags} -c 'configure term'"
         exec_commands = []
         for command in commands:
             self.__logger.info(f"{self.log_format_ip(self.host_ip)} CMD: {command}")
@@ -47,6 +50,16 @@ class Vty(CapybaraNetty):
                 self.build_vty_exec(commands),
             )
 
+    async def del_routes(self, routes: list):
+        commands = []
+        if len(routes) > 0:
+            for route in routes:
+                commands.append(f"no {self.generate_route_config_cmd(**route)} tag 2")
+            return await self.ssh_exec(
+                self.build_vty_exec(commands, echo_commands=True),
+            )
+        return None, None
+
     async def show_route(self, prefix):
         stdout, stderr = await self.ssh_exec(self.build_vty_exec([f"do show ip route {prefix}"]))
         return stdout
@@ -91,7 +104,7 @@ class Vty(CapybaraNetty):
         for route in route_cmds:
             no_route_cmds.append(f"no {route}")
         await self.ssh_exec(
-            self.build_vty_exec(no_route_cmds),
+            self.build_vty_exec(no_route_cmds, echo_commands=True),
         )
 
     async def get_interfaces(self):