netbox-scanner/nbs/__init__.py

import configparser
import logging
import pynetbox.models.dcim
import requests
import docker
import ipaddress
from collections import deque

import requests
from pynetbox import api


class NetBoxScanner(object):

    def __init__(self, address, token, tls_verify, tag, cleanup):
        if (tls_verify == 'no'):
            session = requests.Session()
            session.verify = False
            self.netbox = api(address, token)
            self.netbox.http_session = session
            self.tag = tag
            self.cleanup = cleanup
            self.stats = {
                'unchanged': 0,
                'created': 0,
                'updated': 0,
                'deleted': 0,
                'errors': 0
            }
        else:
            self.netbox = api(address, token)
            self.tag = tag
            self.cleanup = cleanup
            self.stats = {
                'unchanged': 0,
                'created': 0,
                'updated': 0,
                'deleted': 0,
                'errors': 0
            }

    def sync_host(self, host):
        '''Syncs a single host to NetBox

        host: a tuple like ('10.0.0.1','Gateway')
        returns: True if syncing is good or False for errors
        '''
        try:
            nbhost = self.netbox.ipam.ip_addresses.get(address=host[0])
        except ValueError as e:
            logging.error(e)
            logging.error(f'possibly duplicated: {host[0]}/32')
            self.stats['errors'] += 1
            return False

        if nbhost:
            tag_update = False
            for tag in nbhost.tags:
                if (self.tag == tag.name):
                    tag_update = True
                    break
            if tag_update:
                if (host[1] != nbhost.description):
                    aux = nbhost.description
                    nbhost.description = host[1]
                    nbhost.save()
                    logging.info(
                        f'updated: {host[0]}/32 "{aux}" -> "{host[1]}"')
                    self.stats['updated'] += 1
                else:
                    logging.info(f'unchanged: {host[0]}/32 "{host[1]}"')
                    self.stats['unchanged'] += 1
            else:
                logging.info(f'no-tag(unchanged): {host[0]}/32 "{host[1]}"')
                self.stats['unchanged'] += 1
        else:
            self.netbox.ipam.ip_addresses.create(
                address=host[0],
                tags=[{"name": self.tag}],
                # dns_name=host[1],
                description=host[1]
            )
            logging.info(f'created: {host[0]}/32 "{host[1]}"')
            self.stats['created'] += 1

        return True

    def garbage_collector(self, hosts):
        '''Removes records from NetBox not found in last sync'''
        nbhosts = self.netbox.ipam.ip_addresses.filter(tag=self.tag)
        for nbhost in nbhosts:
            nbh = str(nbhost).split('/')[0]
            if not any(nbh == addr[0] for addr in hosts):
                nbhost.delete()
                logging.info(f'deleted: {nbhost}')
                self.stats['deleted'] += 1

    def sync(self, hosts):
        '''Syncs hosts to NetBox
        hosts: list of tuples like [(addr,description),...]
        '''
        for s in self.stats:
            self.stats[s] = 0

        logging.info('started: {} hosts'.format(len(hosts)))
        for host in hosts:
            self.sync_host(host)

        if self.cleanup:
            self.garbage_collector(hosts)

        logging.info('finished: .{} +{} ~{} -{} !{}'.format(
            self.stats['unchanged'],
            self.stats['created'],
            self.stats['updated'],
            self.stats['deleted'],
            self.stats['errors']
        ))

        return True

    def init_docker(self, dockerDef: configparser.SectionProxy):
        ctype = dockerDef.get('cluster_type')
        clusterType = self.netbox.virtualization.cluster_types.get(name=ctype)
        if clusterType is None:
            self.netbox.virtualization.cluster_types.create(name=ctype,slug=ctype)

    def sync_docker(self, dockerConf: dict[str, str], dockerDef: configparser.SectionProxy):

        try:
            deviceName = dockerConf['device']
            devices = self.netbox.dcim.devices.filter(name=deviceName)
            if len(devices) == 0:
                logging.error(f'No devices matched name {deviceName}')
            device = next(devices)
            site = device.site
            clusterName = f'Docker {device['name']}'
            cluster = self.netbox.virtualization.clusters.get(name=clusterName)
            if cluster is None:
                logging.info(f'No Cluster exists for device {deviceName}, creating...')
                clusterType = self.netbox.virtualization.cluster_types.get(name=dockerDef.get('cluster_type'))
                clusterParams = {
                    'name': clusterName,
                    'type': clusterType['id'],
                    'status': 'active'
                }
                if site is not None:
                    clusterParams['site'] = site.id
                cluster = self.netbox.virtualization.clusters.create(**clusterParams)
                self.netbox.dcim.devices.update([
                    {
                        'id': device.id, 'cluster': cluster.id
                    }
                ])

            client = docker.DockerClient(base_url=dockerConf['host'])
            networks = client.networks.list()
            containers = client.containers.list()

            networkData = {}

            for container in containers:

                logging.info(f'Processing Container: {container.name}')

                vmName = 'Docker Standalone'
                # is standalone or compose?
                composed = 'com.docker.compose.config-hash' in container.labels
                composeProject = None
                if composed:
                    composeProject = container.labels['com.docker.compose.project']
                    vmName = f'Docker Compose {composeProject}'
                vm = self.netbox.virtualization.virtual_machines.get(name=vmName,cluster_id=cluster.id)

                if vm is None:
                    vm = self.netbox.virtualization.virtual_machines.create(
                        name=vmName,
                        status="active",
                        cluster=cluster.id,
                        device=device.id,
                        site=site.id
                    )
                    logging.info(f'Created missing VM for docker compose project {vmName} with ID {vm.id}')
                    # if composeProject is None:
                    #     # it's a bridge
                    #     if 'bridge' not in networkData:
                    #         net = self.docker_upsert_network(device, nw)
                    #         networkData['bridge'] = net

                containerNetworks = []
                containerNetwork = None
                ips = []
                hasExternalIp = False
                ns = container.attrs.get('NetworkSettings')
                if ns is not None:
                    for networkName in ns['Networks']:
                        if networkName not in networkData:
                            for nw in networks:
                                if nw.name == networkName:
                                    net = self.docker_upsert_network(device, nw)
                                    networkData[networkName] = net
                                    containerNetwork = net
                                    break
                        else:
                            containerNetwork = networkData[networkName]
                        containerNetworks.append({ 'netNetwork': containerNetwork, 'netContainer': ns['Networks'][networkName] })
                        if 'external' in containerNetwork:
                            hasExternalIp = True

                        for nets in containerNetworks:

                            interfaceName = f'compose_{composeProject}' if composed and nets['netNetwork']['name'] != 'bridge' else 'bridge'
                            intParams = {
                                'virtual_machine_id': vm.id,
                                'name': interfaceName
                            }
                            if nets['netNetwork']['vrf'] is not None:
                                intParams['vrf_id'] = nets['netNetwork']['vrf'].id
                            interface = self.netbox.virtualization.interfaces.get(**intParams)
                            if interface is None:
                                del intParams['virtual_machine_id']
                                intParams['virtual_machine'] = vm.id
                                del intParams['vrf_id']
                                if nets['netNetwork']['vrf'] is not None:
                                    intParams['vrf'] = nets['netNetwork']['vrf'].id
                                interface = self.netbox.virtualization.interfaces.create(**intParams)
                                logging.info(f'Created missing Virtual Interface {interfaceName} for VM {vm.id}')

                            containerIp = nets['netContainer']['IPAddress']
                            if containerIp == '' and nets['netNetwork']['name'] == 'host':
                                if nets['netNetwork']['external'] is None:
                                    logging.info(f'Cant process because on host network but no external ip determined!')
                                    continue
                                containerIp = nets['netNetwork']['external'].split('/')[0]
                            ipLookupParams = {
                                'address': f'{containerIp}/32',
                                #'vminterface_id': vm.id
                            }
                            if nets['netNetwork']['vrf'] is not None:
                                ipLookupParams['vrf_id'] = nets['netNetwork']['vrf'].id
                                ipLookupParams['vminterface_id'] = vm.id
                            ip = self.netbox.ipam.ip_addresses.get(**ipLookupParams)
                            if ip is None:
                                ipCreateParams = {
                                    'address': f'{containerIp}/32',
                                }
                                if nets['netNetwork']['vrf'] is not None:
                                    ipCreateParams['assigned_object_type'] = 'virtualization.vminterface'
                                    ipCreateParams['assigned_object_id'] = vm.id
                                    ipCreateParams['vrf'] = nets['netNetwork']['vrf'].id
                                    # address=f'{containerIp}/32',vrf=nets['netNetwork']['vrf'].id,assigned_object_type='virtualization.vminterface',assigned_object_id=interface.id
                                ip = self.netbox.ipam.ip_addresses.create(**ipCreateParams)
                                logging.info(f'Created missing IP {containerIp} on {interfaceName} interface')
                            ips.append(ip)

                tcp = False
                ports = []
                ipIds = []

                for containerPortDesc in container.ports:
                    if 'tcp' in containerPortDesc:
                        tcp = True
                    portList = container.ports[containerPortDesc]
                    if portList is not None:
                        ports.append(portList[0]['HostPort'])

                for ip in ips:
                    ipIds.append(ip.id)

                serviceName = container.name
                service = self.netbox.ipam.services.get(name=serviceName,virtual_machine_id=vm.id)
                if service is None:
                    serviceDescription = None
                    if 'org.opencontainers.image.title' in container.labels:
                        serviceDescription = container.labels['org.opencontainers.image.title']
                    if 'org.opencontainers.image.description' in container.labels:
                        if serviceDescription is not None:
                            serviceDescription = f'{serviceDescription} - {container.labels['org.opencontainers.image.description']}'
                        else:
                            serviceDescription = container.labels['org.opencontainers.image.description']

                    serviceParams = {
                        'name': serviceName,
                        'virtual_machine': vm.id,
                        'ipaddresses': ipIds
                    }
                    if serviceDescription is not None:
                        # https://stackoverflow.com/a/2872519/1469797
                        serviceParams['description'] = (serviceDescription[:50] + '..') if len(serviceDescription) > 50 else serviceDescription
                    if len(ports) > 0:
                        serviceParams['ports'] = ports
                        serviceParams['protocol'] = 'tcp' if tcp else 'udp'
                    else:
                        serviceParams['ports'] = [1]
                        serviceParams['protocol'] = 'tcp'

                    # if serviceDescription is not None:
                    #     service = self.netbox.ipam.services.create(name=serviceName,virtual_machine=vm.id,description=serviceDescription,ipaddresses=ipIds,ports=ports,protocol='tcp' if tcp else 'udp')
                    # else:
                    #     service = self.netbox.ipam.services.create(name=serviceName,virtual_machine=vm.id,ipaddresses=ipIds,ports=ports,protocol='tcp' if tcp else 'udp')
                    service = self.netbox.ipam.services.create(**serviceParams)
                    logging.info(f'Created missing service {service} on VM {vm.id}')
                else:

                    serviceUpdateParams = {
                        'ipaddresses': ipIds,
                        'id': service.id
                    }
                    if len(ports) > 0:
                        serviceUpdateParams['ports'] = ports
                        serviceUpdateParams['protocol'] = 'tcp' if tcp else 'udp'

                    # update addresses and ports
                    self.netbox.ipam.services.update([
                        serviceUpdateParams
                    ])
                    logging.info(f'Update addresses and ports for service {service} on VM {vm.id}')

        except ValueError as e:
            logging.error(e)
            return False

    # def docker_upsert_vm(self, cluster, device: pynetbox.models.dcim.Devices):

    def docker_upsert_network(self,device: pynetbox.models.dcim.Devices, d_network: docker.client.NetworkCollection.model):
        primaryIp = None
        if device.primary_ip4 is not None:
            primaryIp = device.primary_ip4.address

        networkName = d_network.name
        driver = d_network.attrs.get('Driver')
        if networkName == 'host' or networkName == 'none' or driver == 'host':
            return {
                'name': networkName,
                'vrf': None,
                'range': None,
                'internal': primaryIp,
                'external': primaryIp
            }
        # deal with this later
        if driver != 'bridge':
            return {
                'name': networkName,
                'vrf': None,
                'range': None,
                'internal': None,
                'external': None
            }

        vrf = self.netbox.ipam.vrfs.get(name=f'Docker on {device['name']}')
        if vrf is None:
            vrf = self.netbox.ipam.vrfs.create(name=f'Docker on {device['name']}', rd=f'docker-{device['name']}')
            logging.info(f'Created missing VRF for docker on {device['name']} with ID {vrf.id}')

        range = self.netbox.ipam.ip_ranges.get(description=networkName,vrf_id=vrf.id)
        if range is None:
            subnet = d_network.attrs.get('IPAM')['Config'][0]['Subnet']
            subnet_range = self.get_subnet_range(subnet)
            range = self.netbox.ipam.ip_ranges.create(
                description=networkName,
                vrf=vrf.id,
                start_address=f'{subnet_range[0].exploded}/32',
                end_address=f'{subnet_range[1].exploded}/32'
            )
            logging.info(f'Created missing IP Range for docker network {networkName} {range.start_address} => {range.end_address} in VRF {vrf.id}')

        return {
            'name': networkName,
            'vrf': vrf,
            'range': range,
            'internal': None,
            'external': primaryIp
        }
        # subnet = d_network.attrs.get('IPAM')['Config'][0]['Subnet']

    def get_subnet_range(self, subnet):
        ip_range = ipaddress.ip_network(subnet)
        hosts = ip_range.hosts()
        firstHost = next(ip_range.hosts())

        # https://stackoverflow.com/a/48232574/1469797
        dd = deque(hosts, maxlen=1)
        lastHost = dd.pop()
        return [firstHost, lastHost]