Merge branch 'bile0026-master'

merging code improvements and v2.9 support

.gitignore

@@ -106,3 +106,9 @@ venv.bak/
 # trash
 .vscode/
 
+scans/*
+logs/*
+
+test_api.py
+
+samples/custom-nmap-scan.sh

nbs/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import requests
 
 from pynetbox import api
 
@@ -6,16 +7,31 @@ from pynetbox import api
 class NetBoxScanner(object):
 
     def __init__(self, address, token, tls_verify, tag, cleanup):
-        self.netbox = api(address, token, ssl_verify=tls_verify)
+        if (tls_verify == 'no'):
-        self.tag = tag
+            session = requests.Session()
-        self.cleanup = cleanup
+            session.verify = False
-        self.stats = {
+            self.netbox = api(address, token)
-            'unchanged': 0,
+            self.netbox.http_session = session
-            'created': 0,
+            self.tag = tag
-            'updated': 0,
+            self.cleanup = cleanup
-            'deleted': 0,
+            self.stats = {
-            'errors': 0
+                'unchanged': 0,
-        }
+                'created': 0,
+                'updated': 0,
+                'deleted': 0,
+                'errors': 0
+            }
+        else:
+            self.netbox = api(address, token)
+            self.tag = tag
+            self.cleanup = cleanup
+            self.stats = {
+                'unchanged': 0,
+                'created': 0,
+                'updated': 0,
+                'deleted': 0,
+                'errors': 0
+            }
 
     def sync_host(self, host):
         '''Syncs a single host to NetBox
@@ -36,18 +52,20 @@ class NetBoxScanner(object):
                     aux = nbhost.description
                     nbhost.description = host[1]
                     nbhost.save()
-                    logging.info(f'updated: {host[0]}/32 "{aux}" -> "{host[1]}"')
+                    logging.info(
+                        f'updated: {host[0]}/32 "{aux}" -> "{host[1]}"')
                     self.stats['updated'] += 1
                 else:
                     logging.info(f'unchanged: {host[0]}/32 "{host[1]}"')
                     self.stats['unchanged'] += 1
             else:
-                    logging.info(f'unchanged: {host[0]}/32 "{host[1]}"')
+                logging.info(f'unchanged: {host[0]}/32 "{host[1]}"')
-                    self.stats['unchanged'] += 1
+                self.stats['unchanged'] += 1
         else:
             self.netbox.ipam.ip_addresses.create(
                 address=host[0],
-                # tags=[self.tag],
+                tags=[{"name": self.tag}],
+                # dns_name=host[1],
                 description=host[1]
             )
             logging.info(f'created: {host[0]}/32 "{host[1]}"')

nbs/nmap.py

@@ -28,4 +28,3 @@ class Nmap(object):
                         host.find('address').attrib['addr'],
                         self.unknown
                     ))
-        

netbox-scanner.conf

@@ -1,29 +1,29 @@
 [NETBOX]
-address    = https://netbox.domain
+address    = <server>
-token      = 
+token      = <token>
+logs       = logs/
+# use lowercase no if you want to skip ssl verification.
+# any other value will verify the server ssl certificate.
 tls_verify = no
-logs       = .
 
 [NMAP]
-path     = /opt/netbox-scanner/samples/nmap
+path     = ./
 unknown  = autodiscovered:netbox-scanner
 tag      = nmap
-cleanup  = yes
+cleanup  = no
 
 [NETXMS]
 address    = https://netxms.domain
-username   = 
+username   =
 password   =
-tls_verify = no
 unknown    = autodiscovered:netbox-scanner
 tag        = netxms
 cleanup    = yes
 
 [PRIME]
 address    = https://prime.domain/webacs/api/v4
-username   = 
+username   =
-password   = 
+password   =
-tls_verify = no
 unknown    = autodiscovered:netbox-scanner
 tag        = prime
 cleanup    = yes

netbox-scanner.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import logging
+import sys
 
 from configparser import ConfigParser
 from argparse import ArgumentParser
@@ -10,9 +11,15 @@ from urllib3 import disable_warnings
 from urllib3.exceptions import InsecureRequestWarning
 
 from nbs import NetBoxScanner
-from nbs.nmap import Nmap
+
-from nbs.netxms import NetXMS
+argument = str(sys.argv[1])
-from nbs.prime import Prime
+
+if argument == 'nmap':
+    from nbs.nmap import Nmap
+if argument == 'netxms':
+    from nbs.netxms import NetXMS
+if argument == 'prime':
+    from nbs.prime import Prime
 
 
 local_config = expanduser('~/.netbox-scanner.conf')
@@ -27,16 +34,22 @@ else:
     raise FileNotFoundError('Configuration file was not found.')
 
 netbox = config['NETBOX']
-nmap = config['NMAP']
+if argument == 'nmap':
-netxms = config['NETXMS']
+    nmap = config['NMAP']
-prime = config['PRIME']
+if argument == 'netxms':
+    netxms = config['NETXMS']
+if argument == 'prime':
+    prime = config['PRIME']
 
 parser = ArgumentParser(description='netbox-scanner')
 subparsers = parser.add_subparsers(title='Commands', dest='command')
 subparsers.required = True
-argsp = subparsers.add_parser('nmap', help='Nmap module')
+if argument == 'nmap':
-argsp = subparsers.add_parser('netxms', help='NetXMS module')
+    argsp = subparsers.add_parser('nmap', help='Nmap module')
-argsp = subparsers.add_parser('prime', help='Cisco Prime module')
+if argument == 'netxms':
+    argsp = subparsers.add_parser('netxms', help='NetXMS module')
+if argument == 'prime':
+    argsp = subparsers.add_parser('prime', help='Cisco Prime module')
 args = parser.parse_args()
 
 logfile = '{}/netbox-scanner-{}.log'.format(
@@ -44,12 +57,13 @@ logfile = '{}/netbox-scanner-{}.log'.format(
     datetime.now().isoformat()
 )
 logging.basicConfig(
-    filename=logfile, 
+    filename=logfile,
-    level=logging.INFO, 
+    level=logging.INFO,
     format='%(asctime)s\tnetbox-scanner\t%(levelname)s\t%(message)s'
 )
 logging.getLogger().addHandler(logging.StreamHandler())
 
+# useful if you have tls_verify set to no
 disable_warnings(InsecureRequestWarning)
 
 
@@ -58,6 +72,7 @@ def cmd_nmap(s):  # nmap handler
     h.run()
     s.sync(h.hosts)
 
+
 def cmd_netxms(s):  # netxms handler
     h = NetXMS(
         netxms['address'],
@@ -69,12 +84,13 @@ def cmd_netxms(s):  # netxms handler
     h.run()
     s.sync(h.hosts)
 
+
 def cmd_prime(s):  # prime handler
     h = Prime(
         prime['address'],
         prime['username'],
         prime['password'],
-        prime.getboolean('tls_verify'), 
+        prime.getboolean('tls_verify'),
         prime['unknown']
     )
     h.run()  # set access_point=True to process APs
@@ -85,8 +101,8 @@ if __name__ == '__main__':
     scanner = NetBoxScanner(
         netbox['address'],
         netbox['token'],
-        netbox.getboolean('tls_verify'), 
+        netbox['tls_verify'],
-        nmap['tag'], 
+        nmap['tag'],
         nmap.getboolean('cleanup')
     )
 

samples/networks.txt

@@ -0,0 +1,5 @@
+192.168.0.0/24
+192.168.1.0/24
+192.168.2.0/24
+192.168.3.0/24
+172.16.0.0/24

samples/nmap-1.xml

@@ -1,72 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet href="file:///usr/local/bin/../share/nmap/nmap.xsl" type="text/xsl"?>
-<!-- Nmap 5.59BETA3 scan initiated Fri Sep  9 18:33:41 2011 as:
-     nmap -T4 -A -p 1-1000 -oX - scanme.nmap.org -->
-<nmaprun scanner="nmap" args="nmap -T4 -A -p 1-1000 -oX - scanme.nmap.org" start="1315618421"
-         startstr="Fri Sep  9 18:33:41 2011" version="5.59BETA3" xmloutputversion="1.03">
- <scaninfo type="syn" protocol="tcp" numservices="1000" services="1-1000"/>
- <verbose level="0"/>
- <debugging level="0"/>
- <host starttime="1315618421" endtime="1315618434">
-  <status state="up" reason="echo-reply"/>
-  <address addr="74.207.244.221" addrtype="ipv4"/>
-  <hostnames>
-   <hostname name="scanme.nmap.org" type="user"/>
-   <hostname name="li86-221.members.linode.com" type="PTR"/>
-  </hostnames>
-  <ports>
-   <extraports state="closed" count="997">
-    <extrareasons reason="resets" count="997"/>
-   </extraports>
-   <port protocol="tcp" portid="22">
-    <state state="open" reason="syn-ack" reason_ttl="53"/>
-    <service name="ssh" product="OpenSSH" version="5.3p1 Debian 3ubuntu7"
-             extrainfo="protocol 2.0" ostype="Linux" method="probed" conf="10">
-     <cpe>cpe:/a:openbsd:openssh:5.3p1</cpe>
-     <cpe>cpe:/o:linux:kernel</cpe>
-    </service>
-    <script id="ssh-hostkey"
-            output="1024 8d:60:f1:7c:ca:b7:3d:0a:d6:67:54:9d:69:d9:b9:dd (DSA)&#xa;
-                    2048 79:f8:09:ac:d4:e2:32:42:10:49:d3:bd:20:82:85:ec (RSA)"/>
-   </port>
-   <port protocol="tcp" portid="80">
-    <state state="open" reason="syn-ack" reason_ttl="53"/>
-    <service name="http" product="Apache httpd" version="2.2.14"
-             extrainfo="(Ubuntu)" method="probed" conf="10">
-     <cpe>cpe:/a:apache:http_server:2.2.14</cpe>
-    </service>
-    <script id="http-title" output="Go ahead and ScanMe!"/>
-   </port>
-  </ports>
-  <os>
-   <portused state="open" proto="tcp" portid="22"/>
-   <portused state="closed" proto="tcp" portid="1"/>
-   <portused state="closed" proto="udp" portid="31289"/>
-   <osclass type="general purpose" vendor="Linux" osfamily="Linux"
-            osgen="2.6.X" accuracy="100">
-    <cpe>cpe:/o:linux:linux_kernel:2.6.39</cpe>
-   </osclass>
-   <osmatch name="Linux 2.6.39" accuracy="100" line="39278"/>
-  </os>
-  <uptime seconds="23450" lastboot="Fri Sep  9 12:03:04 2011"/>
-  <distance value="11"/>
-  <tcpsequence index="199" difficulty="Good luck!"
-               values="49018209,48C3EBED,495A2E7F,493EF30C,48ED43B3,495A9B0C"/>
-  <ipidsequence class="All zeros" values="0,0,0,0,0,0"/>
-  <tcptssequence class="1000HZ"
-                 values="165CC09,165CC6E,165CCD2,165CD36,165CD9A,165CE48"/>
-  <trace port="256" proto="tcp">
-   <!-- Several hop elements removed for brevity -->
-   <hop ttl="9" ipaddr="72.52.92.109" rtt="15.69" host="10gigabitethernet1-1.core1.fmt1.he.net"/>
-   <hop ttl="10" ipaddr="64.62.250.6" rtt="12.06" host="linode-llc.10gigabitethernet2-3.core1.fmt1.he.net"/>
-   <hop ttl="11" ipaddr="74.207.244.221" rtt="16.55" host="li86-221.members.linode.com"/>
-  </trace>
-  <times srtt="26517" rttvar="19989" to="106473"/>
- </host>
- <runstats>
-  <finished time="1315618434" timestr="Fri Sep  9 18:33:54 2011" elapsed="13.66"
-            summary="Nmap done at Fri Sep  9 18:33:54 2011; 1 IP address (1 host up)
-                     scanned in 13.66 seconds" exit="success"/>
-  <hosts up="1" down="0" total="1"/>
- </runstats>
-</nmaprun>

samples/nmap-scan.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# This is just an example. 
+# This is just an example.
 #
 # Since scanning many networks can produce huge XML files,
 # the idea is to create one XML file per network, then
@@ -10,20 +10,37 @@
 # want to experiment, feel free to use the `-iL` option of
 # Nmap, passing a list of all networks and hosts to be
 # scanned.
-# 
+#
+# If you have a large number of networks, use the mapfile option.
+# In order to use mapfile, populate your networks, one per line,
+# in a file called networks.txt.
+#
+# If you have a small number of networks, comment out the mapfile
+# lines, and uncomment the "small array" line.
+#
 # For the purpose of this example, assume that netbox-scanner
 # is configured to use the same directory of this script
 # to look for XML files.
 ##
 
-NETWORKS="10.1.2.3/24 10.2.3.4/32 192.168.0.0/19"
+# mapfile
-TODAY="$(date +%d%m%yT%H%M%S%Z)"
+declare -a NETWORKS
+mapfile -t NETWORKS < samples/networks.txt
 
-for net in $NETWORKS; do
+# small array
-  rawNet="${net:0:-3}"
+#NETWORKS="192.168.3.0/24 192.168.252.0/24"
-  sudo nmap -T4 -O -F --host-timeout 30s -oX nmap-"$rawNet".xml "$net"
+
+
+TODAY="$(date +%d.%m.%yT%H:%M:%S%Z)"
+
+for net in "${NETWORKS[@]}"; do
+    NETNAME=$(echo $net | tr -s '/' '-')
+    # requires sudo
+    nmap "$net" -T4 -O -F --host-timeout 30s -oX nmap-"$NETNAME".xml
+    # does not require sudo
+    #nmap "$net" -T4 -sn --host-timeout 30s -oX nmap-"$NETNAME".xml
 done
 
-python ../netbox-scanner.py nmap
+python3 netbox-scanner.py nmap
-tar -czvf nmap-"$TODAY".tar.gz *.xml
+tar -czvf scans/nmap-"$TODAY".tar.gz *.xml
 rm -rf *.xml