From 8bfaf2d7d5d9ee54a01855cc3d09b654972a84a7 Mon Sep 17 00:00:00 2001 From: guanana Date: Sun, 13 Dec 2020 04:45:07 +0000 Subject: [PATCH 1/5] netbox-scanner.conf: Add networks path netbox-scanner.py: Add dir_config option Read networks file networks.txt: Move to expected location and added some examples nmap.py: Use of nmap library to simplify nmap scan requirements.txt: Update requirements.txt with nmap library and update other libraries to latest compatible version --- nbs/nmap.py | 47 ++++++++++++++++++++++++--------------------- netbox-scanner.conf | 1 + netbox-scanner.py | 7 ++++++- networks.txt | 5 +++++ requirements.txt | 11 ++++++----- 5 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 networks.txt diff --git a/nbs/nmap.py b/nbs/nmap.py index e1615bb..168a77b 100644 --- a/nbs/nmap.py +++ b/nbs/nmap.py @@ -1,30 +1,33 @@ -import os -import xml.etree.ElementTree as ET - +import nmap3 class Nmap(object): - def __init__(self, path, unknown): + def __init__(self, unknown, networks): self.unknown = unknown - self.path = path + self.networks = networks self.hosts = list() + self.scan_results = {} + + def scan(self): + nmap = nmap3.NmapHostDiscovery() # instantiate nmap object + for item in self.networks: + temp_scan_result = nmap.nmap_no_portscan(item.replace('\n', '')) + self.scan_results = {**self.scan_results, **temp_scan_result} + return self.scan_results def run(self): - for f in os.listdir(self.path): - if not f.endswith('.xml'): - continue - abspath = os.path.join(self.path, f) - tree = ET.parse(abspath) - root = tree.getroot() + scan_result = self.scan() + scan_result.pop("stats") + scan_result.pop("runtime") + for k,v in scan_result.items(): + try: + self.hosts.append(( + k, + v['hostname'][0]['name'] + )) + except (IndexError, KeyError): + self.hosts.append(( + k, + self.unknown + )) - for host in root.findall('host'): - try: - self.hosts.append(( - host.find('address').attrib['addr'], - host.find('hostnames').find('hostname').attrib['name'] - )) - except AttributeError: - self.hosts.append(( - host.find('address').attrib['addr'], - self.unknown - )) diff --git a/netbox-scanner.conf b/netbox-scanner.conf index e36825f..820693f 100644 --- a/netbox-scanner.conf +++ b/netbox-scanner.conf @@ -8,6 +8,7 @@ tls_verify = no [NMAP] path = ./ +networks = networks.txt unknown = autodiscovered:netbox-scanner tag = nmap cleanup = no diff --git a/netbox-scanner.py b/netbox-scanner.py index 1815c07..8134e71 100644 --- a/netbox-scanner.py +++ b/netbox-scanner.py @@ -24,12 +24,15 @@ if argument == 'prime': local_config = expanduser('~/.netbox-scanner.conf') global_config = '/opt/netbox/netbox-scanner.conf' +dir_config = './netbox-scanner.conf' config = ConfigParser() if isfile(local_config): config.read(local_config) elif isfile(global_config): config.read(global_config) +elif isfile(dir_config): + config.read(dir_config) else: raise FileNotFoundError('Configuration file was not found.') @@ -66,9 +69,11 @@ logging.getLogger().addHandler(logging.StreamHandler()) # useful if you have tls_verify set to no disable_warnings(InsecureRequestWarning) +with open(nmap['networks'], 'r') as file: + networks = file.readlines() def cmd_nmap(s): # nmap handler - h = Nmap(nmap['path'], nmap['unknown']) + h = Nmap(nmap['unknown'], networks) h.run() s.sync(h.hosts) diff --git a/networks.txt b/networks.txt new file mode 100644 index 0000000..b40af15 --- /dev/null +++ b/networks.txt @@ -0,0 +1,5 @@ +192.168.2.0/24 +192.168.3.0/24 +192.168.4.0/24 +192.168.5.0/24 +192.168.15.0/24 diff --git a/requirements.txt b/requirements.txt index c4120ab..f5dd287 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -certifi==2020.4.5.1 +certifi==2020.12.5 chardet==3.0.4 -idna==2.9 -pynetbox==4.3.1 -requests==2.23.0 +idna==2.10 +pynetbox==5.1.0 +requests==2.25.0 six==1.15.0 -urllib3==1.25.9 +urllib3==1.26.2 +python3-nmap==1.4.9 From 5e23996d41dc326d8011523cff01bbe7ae30f9bf Mon Sep 17 00:00:00 2001 From: guanana Date: Sun, 13 Dec 2020 20:07:59 +0000 Subject: [PATCH 2/5] Update README.md with new instructions --- README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3d57a05..de2c764 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ After installation, use the `netbox-scanner.conf` file as an example to create y ## Quick Start 0. Clone the repo and install the dependencies as shown above. -1. Move the `netbox-scanner.conf` file to your Netbox directory (`/opt/netbox`) and fill out the variables according to your setup. Don't forget to change the path to match where you put this repo under `[NMAP].path`. -2. Go to the `samples` subdirectory of this repo and execute `./nmap-scan.sh` to get a first look at the behavior of this project. +1. Use the `netbox-scanner.conf` file that you find in the project directory or create your own under (`/opt/netbox/netbox-scanner.conf` or `~/.netbox-scanner.conf`) and fill out the variables according to your setup. +2. Fill the networks you want to scan (one per line) under `networks.txt` and then run `python netbox-scanner.py nmap` to get a first look at the behavior of this project. ## Basics netbox-scanner reads a user-defined source to discover IP addresses and descriptions, and insert them into NetBox. To control what was previously inserted, netbox-scanner adds tags to each record, so it will know that that item can be handled. In order to guarantee the integrity of manual inputs, records without such tags will not be updated or removed. It is important to note that if netbox-scanner cannot define the description for a given host, then it will insert the string defined in the `unknown` parameter. Users can change those names at their own will. -For NetBox access, this script uses [pynetbox](https://github.com/digitalocean/pynetbox) --worth saying that was tested under NetBox v2.6.7. +For NetBox access, this script uses [pynetbox](https://github.com/digitalocean/pynetbox) --worth saying that was tested under NetBox v2.9.11. ### Garbage Collection If the user marked the `cleanup` option to `yes`, then netbox-scanner will run a garbage collector after the synchronization finishes. Basically, it will get all IP addresses recorded in NetBox under the same tag. Then, both lists will be compared: the one just retrieved from NetBox and the list that was synced. Hosts in the first list that don't appear in the second list will be deleted. @@ -40,9 +40,9 @@ If the user marked the `cleanup` option to `yes`, then netbox-scanner will run a ## Configuration Users can interact with netbox-scanner by command line and configuration file. The latter is pretty simple and straight forward: the only parameter accepted is the module you want to use. -The configuration file (`netbox-scanner.conf`) is where netbox-scanner looks for details such as authentication data and path to files. This file can be stored on the user's home directory or on `/opt/netbox`, but if you choose the first option, it must be a hidden file --`.netbox-scanner.conf`. +The configuration file (`netbox-scanner.conf`) is where netbox-scanner looks for details such as authentication data and path to files. This file can be stored on the user's home directory, on `/opt/netbox` or under the project root directory, but if you choose the first option, it must be a hidden file --`.netbox-scanner.conf`. -> Remember that netbox-scanner will always look for this file at home directory, then at `/opt/netbox`, in this order. The first occurrence will be considered. +> Remember that netbox-scanner will always look for this file at home directory, then at `/opt/netbox`, and finally in the project root dir, in this order. The first occurrence will be considered. ## Modules @@ -54,10 +54,8 @@ Since version 2.0, netbox-scanner is based on modules. This way, this program i ## Nmap Module -Performing the scans is beyond netbox-scanner features, so you must run [Nmap](https://nmap.org/) and save the output as an XML file using the `-oX` parameter. Since this file can grow really fast, you can scan each network and save it as a single XML file. You just have to assure that all files are under the same directory before running the script --see `samples/nmap.sh` for an example. - -To properly setup this module, you must inform the path to the directory where the XML files reside, define a tag to insert to discovered hosts, and decide if clean up will take place. Tested on Nmap v7.80. - +Performing the scans is incorporated now under netbox-scanner features, so you don't need to run [Nmap](https://nmap.org/) yourself. +In this version the scan doesn't need root privileges since it only checks for hosts up. ## Prime Module This script accesses [Prime](https://www.cisco.com/c/en/us/products/cloud-systems-management/prime-infrastructure/index.html) through RESTful API and all routines are implemented here. Users only have to point to Prime's API, which looks like `https://prime.domain/webacs/api/v4/`, inform valid credentials allowed to use the API, and fill the other variables, just like in Nmap. From 14335e49609a8bb61e051f1012e9bd202bff35b9 Mon Sep 17 00:00:00 2001 From: guanana Date: Sun, 13 Dec 2020 20:25:14 +0000 Subject: [PATCH 3/5] Update unittest for nmap module --- tests/test_nmap.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_nmap.py b/tests/test_nmap.py index 3a1e4fd..9a6659e 100644 --- a/tests/test_nmap.py +++ b/tests/test_nmap.py @@ -1,16 +1,17 @@ import unittest -from os import environ + from nbs.nmap import Nmap class TestRequest(unittest.TestCase): def test_api(self): - path = environ.get('NMAP_PATH') - nmap = Nmap(path, 'test') + nmap = Nmap("test", ["127.0.0.1/32"]) self.assertIsInstance(nmap, Nmap) nmap.run() self.assertIsInstance(nmap.hosts, list) + self.assertEqual(nmap.hosts[0][0], "127.0.0.1") + self.assertEqual(nmap.hosts[0][1], "localhost") if __name__ == '__main__': From afecb475a5f311ba1b0f97a74a8ba09d547ef8c1 Mon Sep 17 00:00:00 2001 From: guanana Date: Tue, 15 Dec 2020 01:27:34 +0000 Subject: [PATCH 4/5] __init__.py: Improve tag checking (With the previous version of check wasn't working, don't know if it was dependant on old Netbox version) nmap.py: Add extra function to improve DNS resolution since nmap is not always consistent --- nbs/__init__.py | 11 ++++++++--- nbs/nmap.py | 27 +++++++++++++++++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/nbs/__init__.py b/nbs/__init__.py index ee79cc3..a3ee322 100644 --- a/nbs/__init__.py +++ b/nbs/__init__.py @@ -1,6 +1,6 @@ import logging -import requests +import requests from pynetbox import api @@ -47,7 +47,12 @@ class NetBoxScanner(object): return False if nbhost: - if (self.tag in nbhost.tags): + tag_update = False + for tag in nbhost.tags: + if (self.tag == tag.name): + tag_update = True + break + if tag_update: if (host[1] != nbhost.description): aux = nbhost.description nbhost.description = host[1] @@ -59,7 +64,7 @@ class NetBoxScanner(object): logging.info(f'unchanged: {host[0]}/32 "{host[1]}"') self.stats['unchanged'] += 1 else: - logging.info(f'unchanged: {host[0]}/32 "{host[1]}"') + logging.info(f'no-tag(unchanged): {host[0]}/32 "{host[1]}"') self.stats['unchanged'] += 1 else: self.netbox.ipam.ip_addresses.create( diff --git a/nbs/nmap.py b/nbs/nmap.py index 168a77b..2850530 100644 --- a/nbs/nmap.py +++ b/nbs/nmap.py @@ -1,5 +1,8 @@ +import socket + import nmap3 + class Nmap(object): def __init__(self, unknown, networks): @@ -9,17 +12,30 @@ class Nmap(object): self.scan_results = {} def scan(self): - nmap = nmap3.NmapHostDiscovery() # instantiate nmap object + nmap = nmap3.NmapHostDiscovery() # instantiate nmap object for item in self.networks: temp_scan_result = nmap.nmap_no_portscan(item.replace('\n', '')) self.scan_results = {**self.scan_results, **temp_scan_result} + self.scan_results.pop("stats") + self.scan_results.pop("runtime") return self.scan_results + def dns_resolution(self): + # Try to improve DNS resolution since NMAP is not consistent + for ip, v in self.scan_results.items(): + try: + name, arpa, ip = socket.gethostbyaddr(ip) + try: + v["hostname"][0]["name"] + except (TypeError, IndexError): + v.update({"hostname": {"name": name, "type": 'PTR'}}) + except socket.herror: + pass + def run(self): - scan_result = self.scan() - scan_result.pop("stats") - scan_result.pop("runtime") - for k,v in scan_result.items(): + self.scan() + self.dns_resolution() + for k,v in self.scan().items(): try: self.hosts.append(( k, @@ -30,4 +46,3 @@ class Nmap(object): k, self.unknown )) - From e5ae3f73b542c49a298606996145608fea876d3f Mon Sep 17 00:00:00 2001 From: guanana Date: Tue, 15 Dec 2020 02:49:11 +0000 Subject: [PATCH 5/5] Remove extra function since no need (nmap after all has an option to force always DNS resolution) --- nbs/nmap.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/nbs/nmap.py b/nbs/nmap.py index 2850530..5360e2d 100644 --- a/nbs/nmap.py +++ b/nbs/nmap.py @@ -1,8 +1,5 @@ -import socket - import nmap3 - class Nmap(object): def __init__(self, unknown, networks): @@ -14,27 +11,14 @@ class Nmap(object): def scan(self): nmap = nmap3.NmapHostDiscovery() # instantiate nmap object for item in self.networks: - temp_scan_result = nmap.nmap_no_portscan(item.replace('\n', '')) + temp_scan_result = nmap.nmap_no_portscan(item.replace('\n', ''), args="-R --system-dns") self.scan_results = {**self.scan_results, **temp_scan_result} self.scan_results.pop("stats") self.scan_results.pop("runtime") return self.scan_results - def dns_resolution(self): - # Try to improve DNS resolution since NMAP is not consistent - for ip, v in self.scan_results.items(): - try: - name, arpa, ip = socket.gethostbyaddr(ip) - try: - v["hostname"][0]["name"] - except (TypeError, IndexError): - v.update({"hostname": {"name": name, "type": 'PTR'}}) - except socket.herror: - pass - def run(self): self.scan() - self.dns_resolution() for k,v in self.scan().items(): try: self.hosts.append((