Commit e90e597b authored by Alberts S's avatar Alberts S
Browse files

Handle dead routers more gracefully; Keep ssh_exec DRY

parent 160cc531
import asyncio
import logging
import sys
import asyncssh
logging.basicConfig(stream=sys.stdout, format="%(levelname)-8s:%(name)-24s.%(funcName)-40s:%(message)-s")
logging.getLogger("CapybaraNetty").setLevel(logging.INFO)
......@@ -20,3 +23,18 @@ class MetaCapybaraNetty(type):
class CapybaraNetty(metaclass=MetaCapybaraNetty):
def __init__(self):
pass
@staticmethod
async def ssh_exec(ip, username, command):
async with asyncssh.connect(host=ip, username=username, known_hosts=None, connect_timeout=10) as conn:
result = await conn.run(command)
return result.stdout.strip("\n").split("\n"), result.stderr.strip("\n").split("\n")
@staticmethod
async def ssh_is_alive(ip, username):
try:
async with asyncssh.connect(host=ip, username=username, known_hosts=None, connect_timeout=3) as conn:
await conn.run("true")
return True
except (asyncssh.misc.PermissionDenied, asyncio.exceptions.TimeoutError) as e:
return False
......@@ -35,11 +35,11 @@ class Controller(CapybaraNetty):
async def main(self):
router_coroutines = []
for host in self.hosts:
router_coroutines.append(
Router.run(
ip=host["public_ipv4"], ssh_user=self.config["default_router_ssh_user"], name=host["hostname"]
)
router = Router(
ip=host["public_ipv4"], ssh_user=self.config["default_router_ssh_user"], name=host["hostname"]
)
router_coroutines.append(router.main())
self.routers = await asyncio.gather(*router_coroutines)
for router in self.routers:
......
......@@ -16,10 +16,8 @@ class Pinger(CapybaraNetty):
self.ping_addresses = ping_addresses
self.latest_results = list()
async def ssh_exec(self, command):
async with asyncssh.connect(host=self.router_ip, username=self.router_ssh_user, known_hosts=None) as conn:
result = await conn.run(command)
return result.stdout.strip("\n").split("\n"), result.stderr.strip("\n").split("\n")
async def ssh_exec(self, command, **kwargs):
return await super(Pinger, self).ssh_exec(self.router_ip, self.router_ssh_user, command=command)
def set_ping_addresses(self, ping_addresses):
self.ping_addresses = ping_addresses
......@@ -27,20 +25,33 @@ class Pinger(CapybaraNetty):
@staticmethod
def parse_fping_line(s):
[host, right] = s.split(" : ")
[packets, time] = right.split(", ")
[_, packets] = packets.split(" = ")
[sent, received, loss] = packets.split("/")
[_, time] = time.split(" = ")
[minimum, average, maximum] = time.split("/")
return {
"minimum": float(minimum),
"average": float(average),
"maximum": float(maximum.rstrip("\n")),
"host": host.strip(),
"sent": int(sent),
"received": int(received),
"loss": int(loss.rstrip("%")),
}
try:
[packets, time] = right.split(", ")
[_, packets] = packets.split(" = ")
[sent, received, loss] = packets.split("/")
[_, time] = time.split(" = ")
[minimum, average, maximum] = time.split("/")
return {
"minimum": float(minimum),
"average": float(average),
"maximum": float(maximum.rstrip("\n")),
"host": host.strip(),
"sent": int(sent),
"received": int(received),
"loss": int(loss.rstrip("%")),
}
except ValueError as e:
# This happens when host is down
# '16.171.xxx.xxx : xmt/rcv/%loss = 10/0/100%'
packets = right
[_, packets] = packets.split(" = ")
[sent, received, loss] = packets.split("/")
return {
"host": host.strip(),
"sent": int(sent),
"received": int(received),
"loss": int(loss.rstrip("%")),
}
async def get_ping(self):
if len(self.ping_addresses) > 0:
......@@ -53,10 +64,16 @@ class Pinger(CapybaraNetty):
ping_results = []
for line in stderr: # First line is empty
parsed_line = self.parse_fping_line(line)
ping_results.append(parsed_line)
self.__logger.debug(
f"PING: {self.router_ip:<16s} => {parsed_line['host']:<16s} {int(parsed_line['average']):<5d}"
)
if parsed_line["loss"] < 100:
ping_results.append(parsed_line)
self.__logger.debug(
f"PING: {self.router_ip:<16s} => {parsed_line['host']:<16s} {int(parsed_line['average']):<5d}"
)
else:
self.__logger.debug(
f"PING: DOWN {self.router_ip:<16s} => {parsed_line['host']:<16s} {int(parsed_line['loss']):<5d}% packet loss"
)
self.latest_results = ping_results
return ping_results
else:
......@@ -66,5 +83,8 @@ class Pinger(CapybaraNetty):
async def run_daemon(self, interval_seconds):
while True:
await self.get_ping()
try:
await self.get_ping()
except (asyncssh.misc.PermissionDenied, asyncio.exceptions.TimeoutError) as e:
self.__logger.warning(f"Pinger {self.router_ip:<16s} is unable to connect via SSH")
await asyncio.sleep(interval_seconds)
......@@ -19,13 +19,19 @@ class Router(CapybaraNetty):
await self.main()
return self
def is_alive(self):
return self.vty.ssh_is_alive()
async def main(self):
if not await self.is_alive():
self.__logger.warning(f"{self.ip} skipped due to SSH connection failure")
return self
await self.get_routes()
await self.get_config_routes()
self.get_default_interface()
self.get_default_gateway()
await self.del_all_config_routes()
pass
return self
async def get_routes(self):
self.routes = await self.vty.get_routes()
......
......@@ -11,10 +11,8 @@ class Vty(CapybaraNetty):
self.host_ip = host_ip
self.host_user = host_user
async def ssh_exec(self, command):
async with asyncssh.connect(host=self.host_ip, username=self.host_user, known_hosts=None) as conn:
result = await conn.run(command)
return result.stdout.strip("\n").split("\n"), result.stderr.strip("\n").split("\n")
def ssh_is_alive(self, **kwargs):
return super(Vty, self).ssh_is_alive(self.host_ip, self.host_user)
def build_vty_exec(self, commands: list):
base = "sudo vtysh -c 'configure term'"
......@@ -25,7 +23,9 @@ class Vty(CapybaraNetty):
return base + "".join(exec_commands)
async def get_routes(self):
route_json, stderr = await self.ssh_exec(self.build_vty_exec(["do show ip route json"]))
route_json, stderr = await self.ssh_exec(
self.build_vty_exec(["do show ip route json"]),
)
# It returns list of strings which combined makes up JSON - need to join it up
routes = json.loads("".join(route_json))
# Routes now has dict of lists, where d[prefix][0] is one of the possible routes for prefix, change this so
......@@ -42,7 +42,9 @@ class Vty(CapybaraNetty):
commands = []
for route in routes:
commands.append(self.generate_route_config_cmd(**route))
await self.ssh_exec(self.build_vty_exec(commands))
await self.ssh_exec(
self.build_vty_exec(commands),
)
def generate_route_config_cmd(self, prefix, gateway, interface, distance=1, **kwargs):
additional_cmd_args = ""
......@@ -68,7 +70,9 @@ class Vty(CapybaraNetty):
return False
return True
routes_config, stderr = await self.ssh_exec(self.build_vty_exec(["do show running-config | include ip route"]))
routes_config, stderr = await self.ssh_exec(
self.build_vty_exec(["do show running-config | include ip route"]),
)
routes = []
for route in routes_config:
# Remove newlines at end
......@@ -81,4 +85,9 @@ class Vty(CapybaraNetty):
no_route_cmds = []
for route in route_cmds:
no_route_cmds.append(f"no {route}")
await self.ssh_exec(self.build_vty_exec(no_route_cmds))
await self.ssh_exec(
self.build_vty_exec(no_route_cmds),
)
async def ssh_exec(self, command, **kwargs):
return await super(Vty, self).ssh_exec(self.host_ip, self.host_user, command=command)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment