From 4dde860c54ddee986a2a92418df7173e9a3f0b9a Mon Sep 17 00:00:00 2001 From: dade Date: Wed, 31 May 2017 14:44:00 -0700 Subject: [PATCH 1/9] Added host updating --- nmapdb.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/nmapdb.py b/nmapdb.py index 83a7f8c..97e1c32 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -33,9 +33,16 @@ def usage(name): print " (-f) --frequency list most frequent open ports from specified DB" print " (-n) --nodb do not perform any DB operations (i.e. dry run)" print " (-V) --version output version number and exit" + print "\nexample: ./nmapdb.py -v -c nmapdb.sql -d myscan.db scan_data.xml" return +def dict_factory(cursor, row): + d = {} + for idx,col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + def main(argv, environ): global vflag nodb_flag = false @@ -208,8 +215,40 @@ def main(argv, environ): (ip, mac, hostname, protocol, os_name, os_family, os_accuracy, os_gen, timestamp, state, mac_vendor, whois_str)) except sqlite.IntegrityError, msg: - print "%s: warning: %s: table hosts: ip: %s\n" % (argv[0], msg, ip) - continue + cursor.execute("SELECT * FROM hosts WHERE ip = '%s'" % ip ) + db = dict_factory(cursor, cursor.fetchone()) + if not ( db['ip'] == ip and db['mac'] == mac and db['hostname'] == hostname and db['protocol'] == protocol and db['os_name'] == os_name + and db['os_family'] == os_family and db['os_accuracy'] == int(os_accuracy) and db['os_gen'] == os_gen and db['state'] == state and + db['mac_vendor'] == mac_vendor and db['whois'] == whois_str): + # So we already have an entry. If theres no new information we continue to ports + # If there's a bunch of new entries we'll ask the user what to do + print "[hosts] %s entry exists" % ip + print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" + print "[hosts] Name: Old --> New" + print("[hosts] ip: "+db['ip']+" --> %s" % ip) + print("[hosts] mac: "+db['mac']+" --> %s" % mac) + print("[hosts] hostname: "+db['hostname']+" --> %s" % hostname) + print("[hosts] protocol: "+db['protocol']+" --> %s" % protocol) + print("[hosts] os_name: "+db['os_name']+" --> %s" % os_name) + print("[hosts] os_family: "+db['os_family']+" --> %s" % os_family) + print("[hosts] os_accuracy: "+str(db['os_accuracy'])+" --> %s" % os_accuracy) + print("[hosts] os_gen: "+db['os_gen']+" --> %s" % os_gen) + print("[hosts] timestamp: "+str(db['last_update'])+" --> %s" % timestamp) + print("[hosts] state: "+db['state']+" --> %s" % state) + print("[hosts] mac_vendor: "+db['mac_vendor']+" --> %s" % mac_vendor) + print("[hosts] whois: "+db['whois']+" --> %s" % whois_str) + print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" + print "[hosts] Update entry? y/n" + user_input = sys.stdin.readline().strip()[:1] + if user_input == 'y': + myprint("[hosts] updating %s entry" % ip) + sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', whois='%s' WHERE ip = '%s'" % + (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,whois_str, ip )) + cursor.execute(sql) + + else: + myprint("[hosts] Skipping %s entry" % ip) + continue except: print "%s: unknown exception during insert into table hosts\n" % (argv[0]) continue From 9b560b60b5f3a1c6523dd4d3ab64345902bbf7e2 Mon Sep 17 00:00:00 2001 From: dade Date: Wed, 31 May 2017 15:41:20 -0700 Subject: [PATCH 2/9] Added ports update functionality --- nmapdb.py | 81 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/nmapdb.py b/nmapdb.py index 97e1c32..ac44b41 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -217,26 +217,31 @@ def main(argv, environ): except sqlite.IntegrityError, msg: cursor.execute("SELECT * FROM hosts WHERE ip = '%s'" % ip ) db = dict_factory(cursor, cursor.fetchone()) - if not ( db['ip'] == ip and db['mac'] == mac and db['hostname'] == hostname and db['protocol'] == protocol and db['os_name'] == os_name - and db['os_family'] == os_family and db['os_accuracy'] == int(os_accuracy) and db['os_gen'] == os_gen and db['state'] == state and - db['mac_vendor'] == mac_vendor and db['whois'] == whois_str): + if not ( db['mac'] == mac + and db['hostname'] == hostname + and db['protocol'] == protocol + and db['os_name'] == os_name + and db['os_family'] == os_family + and db['os_gen'] == os_gen + and db['state'] == state + and db['mac_vendor'] == mac_vendor + and db['whois'] == whois_str): # So we already have an entry. If theres no new information we continue to ports # If there's a bunch of new entries we'll ask the user what to do print "[hosts] %s entry exists" % ip print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" - print "[hosts] Name: Old --> New" - print("[hosts] ip: "+db['ip']+" --> %s" % ip) - print("[hosts] mac: "+db['mac']+" --> %s" % mac) - print("[hosts] hostname: "+db['hostname']+" --> %s" % hostname) - print("[hosts] protocol: "+db['protocol']+" --> %s" % protocol) - print("[hosts] os_name: "+db['os_name']+" --> %s" % os_name) - print("[hosts] os_family: "+db['os_family']+" --> %s" % os_family) - print("[hosts] os_accuracy: "+str(db['os_accuracy'])+" --> %s" % os_accuracy) - print("[hosts] os_gen: "+db['os_gen']+" --> %s" % os_gen) - print("[hosts] timestamp: "+str(db['last_update'])+" --> %s" % timestamp) - print("[hosts] state: "+db['state']+" --> %s" % state) - print("[hosts] mac_vendor: "+db['mac_vendor']+" --> %s" % mac_vendor) - print("[hosts] whois: "+db['whois']+" --> %s" % whois_str) + print "[hosts] Name: 'Old' --> 'New'" + print("[hosts] mac: '"+db['mac']+"' --> '%s'" % mac) + print("[hosts] hostname: '"+db['hostname']+"' --> '%s'" % hostname) + print("[hosts] protocol: '"+db['protocol']+"' --> '%s'" % protocol) + print("[hosts] os_name: '"+db['os_name']+"' --> '%s'" % os_name) + print("[hosts] os_family: '"+db['os_family']+"' --> '%s'" % os_family) + print("[hosts] os_accuracy: '"+str(db['os_accuracy'])+"' --> '%s'" % os_accuracy) + print("[hosts] os_gen: '"+db['os_gen']+"' --> '%s'" % os_gen) + print("[hosts] timestamp: '"+str(db['last_update'])+"' --> '%s'" % timestamp) + print("[hosts] state: '"+db['state']+"' --> '%s'" % state) + print("[hosts] mac_vendor: '"+db['mac_vendor']+" --> '%s'" % mac_vendor) + print("[hosts] whois: '"+db['whois']+"' --> '%s'" % whois_str) print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" print "[hosts] Update entry? y/n" user_input = sys.stdin.readline().strip()[:1] @@ -295,30 +300,52 @@ def main(argv, environ): if script_id != "" and script_output != "": info_str += "%s: %s\n" % (script_id, script_output) - myprint("\t------------------------------------------------") + myprint("------------------------------------------------") - myprint("\t[ports] ip:\t\t%s" % (ip)) - myprint("\t[ports] port:\t\t%s" % (pn)) - myprint("\t[ports] protocol:\t%s" % (protocol)) - myprint("\t[ports] name:\t\t%s" % (port_name)) - myprint("\t[ports] state:\t\t%s" % (state)) - myprint("\t[ports] service:\t%s" % (service_str)) + myprint("[ports] ip:\t\t%s" % (ip)) + myprint("[ports] port:\t\t%s" % (pn)) + myprint("[ports] protocol:\t%s" % (protocol)) + myprint("[ports] name:\t\t%s" % (port_name)) + myprint("[ports] state:\t\t%s" % (state)) + myprint("[ports] service:\t%s" % (service_str)) if info_str != "": - myprint("\t[ports] info:\n") + myprint("[ports] info:\n") myprint("%s\n" % (info_str)) if nodb_flag == false: try: cursor.execute("INSERT INTO ports VALUES (?, ?, ?, ?, ?, ?, ?)", (ip, pn, protocol, port_name, state, service_str, info_str)) except sqlite.IntegrityError, msg: - print "%s: warning: %s: table ports: ip: %s\n" % (argv[0], msg, ip) - continue + cursor.execute("SELECT * FROM ports WHERE ip = '%s' AND port = '%s' and protocol ='%s'" % (ip, pn, protocol) ) + db = dict_factory(cursor, cursor.fetchone()) + + if info_str != "" and db['info'].find(info_str) <= 0: + new_info = db['info'] + "\n" + info_str + myprint("[ports] Appending info %s" % info_str) + cursor.execute("UPDATE ports SET info=? WHERE ip = ? AND port = ? and protocol =?" , (new_info, ip, pn, protocol)) + + + if db['name'] != port_name or db['service'] != service_str or db['state'] != state: + print '[ports] %s:%s %s exists' % (ip, pn, protocol) + print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" + print "[ports] Name: 'Old' --> 'New'" + print("[ports] ip: '"+db['ip']+"' --> '%s'" % ip) + print("[ports] port: '"+str(db['port'])+"' --> '%s'" % pn) + print("[ports] protocol: '"+db['protocol']+"' --> '%s'" % protocol) + print("[ports] name: '"+db['name']+"' --> '%s'" % port_name) + print("[ports] state: '"+db['state']+"' --> '%s'" % state) + print("[ports] service: '"+db['service']+"' --> '%s'" % service_str) + print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" + print "[ports] Update entry? y/n" + user_input = sys.stdin.readline().strip()[:1] + cursor.execute("UPDATE ports SET name=?, state=?, service=? WHERE ip = ? AND port = ? and protocol = ?", + (port_name, state, service_str, ip, pn, protocol)) + except: print "%s: unknown exception during insert into table ports\n" % (argv[0]) continue - myprint("\t------------------------------------------------") myprint("================================================================") From 7fc300128e8627e83ca6718598e5a3e8f0a58d80 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 21:51:31 -0700 Subject: [PATCH 3/9] removing files --- LICENSE | 29 --------------------------- TODO | 4 ---- nmapdb.sql | 57 ------------------------------------------------------ 3 files changed, 90 deletions(-) delete mode 100644 LICENSE delete mode 100644 TODO delete mode 100644 nmapdb.sql diff --git a/LICENSE b/LICENSE deleted file mode 100644 index c410b2b..0000000 --- a/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ - nmapdb - Parse nmap's XML output files and insert them into an SQLite database - - Copyright (c) 2012 Patroklos Argyroudis - Copyright (c) 2012 Census, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The names of the authors and copyright holders may not be used to - endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/TODO b/TODO deleted file mode 100644 index 0bbb33f..0000000 --- a/TODO +++ /dev/null @@ -1,4 +0,0 @@ -* Helpful SQL queries on a database (e.g. open port frequency) - -* Add the MySQL support submitted by @antonat - diff --git a/nmapdb.sql b/nmapdb.sql deleted file mode 100644 index 99ae040..0000000 --- a/nmapdb.sql +++ /dev/null @@ -1,57 +0,0 @@ -/* - * nmapdb - Parse nmap's XML output files and insert them into an SQLite database - * Copyright (c) 2012 Patroklos Argyroudis - */ - -CREATE TABLE IF NOT EXISTS hosts ( - ip VARCHAR(16) PRIMARY KEY NOT NULL, - mac VARCHAR(18), - hostname VARCHAR(129), - protocol VARCHAR(5) DEFAULT 'ipv4', - os_name TEXT, - os_family TEXT, - os_accuracy INTEGER, - os_gen TEXT, - last_update TIMESTAMP, - state VARCHAR(8) DEFAULT 'down', - mac_vendor TEXT, - whois TEXT -); - -CREATE TABLE IF NOT EXISTS ports ( - ip VARCHAR(16) NOT NULL, - port INTEGER NOT NULL, - protocol VARCHAR(4) NOT NULL, - name VARCHAR(33), - state VARCHAR(33) DEFAULT 'closed', - service TEXT, - info TEXT, - PRIMARY KEY (ip, port, protocol), - CONSTRAINT fk_ports_hosts FOREIGN KEY (ip) REFERENCES hosts(ip) ON DELETE CASCADE -); - -CREATE TRIGGER IF NOT EXISTS fki_ports_hosts_ip -BEFORE INSERT ON ports -FOR EACH ROW BEGIN - SELECT CASE - WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) - THEN RAISE(ABORT, 'insert on table "ports" violates foreign key constraint "fk_ports_hosts"') - END; -END; - -CREATE TRIGGER IF NOT EXISTS fku_ports_hosts_ip -BEFORE UPDATE ON ports -FOR EACH ROW BEGIN - SELECT CASE - WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) - THEN RAISE(ABORT, 'update on table "ports" violates foreign key constraint "fk_ports_hosts"') - END; -END; - -CREATE TRIGGER IF NOT EXISTS fkd_ports_hosts_ip -BEFORE DELETE ON hosts -FOR EACH ROW BEGIN - DELETE from ports WHERE ip = OLD.ip; -END; - -/* EOF */ From ac3f1e21a7d3ab49ae2a4351fcea4fee6e2445f0 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 21:53:00 -0700 Subject: [PATCH 4/9] Completly rewrote the script to use libnmap and add support for color and being able to add updates --- README | 168 ++++++++++----- nmapdb.py | 633 +++++++++++++++++++++++++++--------------------------- 2 files changed, 438 insertions(+), 363 deletions(-) diff --git a/README b/README index 567fda2..e4b8508 100644 --- a/README +++ b/README @@ -1,63 +1,135 @@ -nmapdb parses nmap's XML output files and inserts them into an SQLite database. +# NmapDB -I coded this a while back (mid 2009) and have been using it since. Some -people I have shared nmapdb with have found it useful, so I am releasing it -publicly. +nmapdb parses nmap's XML output files and inserts them into an SQLite database. -Example usage: +## Example usage: +``` $ sudo nmap -A -oX scanme.xml scanme.nmap.org Starting Nmap ... $ ls scanme.xml scanme.xml -$ ./nmapdb.py -h -usage: ./nmapdb.py [options] -options: - (-h) --help this message - (-v) --verbose verbose output - (-c) --create specify input SQL file to create SQLite DB - (-d) --database specify output SQLite DB file - (-f) --frequency list most frequent open ports from specified DB - (-n) --nodb do not perform any DB operations (i.e. dry run) - (-V) --version output version number and exit - -Use -c to create a database from the schema on the first run: - -$ ./nmapdb.py -c nmapdb.sql -d myscan.db scanme.xml -$ file myscan.db -myscan.db: SQLite 3.x database -$ sqlite3 myscan.db -SQLite version 3.7.7 ... +``` +### Running: + +``` +$ ./nmapdb.py +usage: nmapdb.py [-h] [--debug] [-d SCANDB] nmap_xml +nmapdb.py: error: too few arguments + +$ ./nmapdb.py -h +usage: nmapdb.py [-h] [--debug] [-d SCANDB] nmap_xml + +Nmap XML file to SQLite database. Default file nmap.db will be used if none is +supplied. + +positional arguments: + nmap_xml Nmap XML file you wish to parse + +optional arguments: + -h, --help show this help message and exit + --debug Print verbose information + -d SCANDB, --database SCANDB + Filename to use for database. If file doesn't exist it + will be created. Default is 'nmap.db.' + +``` +**nmapdb** will create a database from internal schema if the tables hosts and ports doesn't exist in your sqlite database. If no database file is supplied it will use the default file `nmap.db`. + +So long as you are ok with the defaults the program can be used with: `./nmapdb.py output.xml`. + +``` +$ ./nmapdb.py scanme.xml +[+] Start >> Using DB file: nmap.db and XML file scanme.xml + +$ file nmap.db +nmap.db: SQLite 3.x database, last written using SQLite version 3014001 + +$ sqlite3 nmap.db +SQLite version 3.14.1 2016-08-11 18:53:32 +Enter ".help" for usage hints. sqlite> select * from hosts; -74.207.244.221||scanme.nmap.org|ipv4|Linux 2.6.18|Linux|85|2.6.X|1316681984|up| +45.33.32.156||scanme.nmap.org|ipv4|DD-WRT v24-sp2 (Linux 2.4.37)|Linux|100|2.4.X|1496637122|up|| sqlite> select * from ports; -74.207.244.221|22|tcp|ssh|open| -74.207.244.221|80|tcp|http|open| - -Subsequent scans can be entered into the same database: - -$ ./nmapdb.py -d myscan.db bar.xml foo.xml host1.xml host2.xml \ - host3.xml host4.xml meh.xml (or simply *.xml) -$ sqlite3 myscan.db -SQLite version 3.7.7 ... -sqlite> select * from ports where ports.port='22'; -aa.bb.244.221|22|tcp|ssh|open| -204.cc.ddd.250|22|tcp|ssh|open| -bbb.242.aa.180|22|tcp|ssh|open| -aa.bb.121.21|22|tcp|ssh|open| -sqlite> select * from ports where ports.port='23'; -192.168.1.254|23|tcp|telnet|open| -sqlite> select * from hosts inner join ports on hosts.ip=ports.ip where hosts.ip='192.168.1.254' and ports.state='open'; -192.168.1.254|00:00:C5:CF:86:30|modem|ipv4||||||up|Farallon Computing/netopia|192.168.1.254|23|tcp|telnet|open| -192.168.1.254|00:00:C5:CF:86:30|modem|ipv4||||||up|Farallon Computing/netopia|192.168.1.254|80|tcp|http|open| -sqlite> select * from hosts inner join ports on hosts.ip=ports.ip where hosts.os_name like '%bsd%' and ports.port=22; -aa.bb.91.25||foo.bar.org|ipv4|FreeBSD 7.0-STABLE|FreeBSD|95|7.X|1231841556|up||aa.bb.91.25|22|tcp|ssh|open| +45.33.32.156|22|tcp|ssh|open|product: OpenSSH version: 6.6.1p1 Ubuntu 2ubuntu2.8 extrainfo: Ubuntu Linux; protocol 2.0 ostype: Linux +45.33.32.156|80|tcp|http|open|product: Apache httpd version: 2.4.7 extrainfo: (Ubuntu)|[45.33.32.156:80] http-favicon: +[45.33.32.156:80] +[45.33.32.156:80] http-title: +[45.33.32.156:80] Go ahead and ScanMe! +[45.33.32.156:80] Elements: +[45.33.32.156:80] title: Go ahead and ScanMe! +[45.33.32.156:80] +45.33.32.156|9929|tcp|nping-echo|open|product: Nping echo| +45.33.32.156|31337|tcp|tcpwrapped|open|| +sqlite> +``` -Feel free to fork, submit patches, whatever. +### Debug + +Turning on debug with `--debug` will show you a lot more information: + +``` +$ ./nmapdb.py --debug scanme.xml +[+] Start >> Debug Enabled +[+] Start >> Using DB file: nmap.db and XML file scanme.xml +[+] Start >> Successfully connected to SQLite DB "nmap.db" +[+] Start >> Database already exists. Continuing. +[+] Parser >> Nmap Results: Nmap done at Mon Jun 5 00:32:02 2017; 1 IP address (1 host up) scanned in 16.90 seconds +[+] [host] >> ip: 45.33.32.156 +[+] [host] >> mac: +[+] [host] >> hostname: scanme.nmap.org +[+] [host] >> protocol: ipv4 +[+] [host] >> os_name: DD-WRT v24-sp2 (Linux 2.4.37) +[+] [host] >> os_family: Linux +[+] [host] >> os_accuracy: 100 +[+] [host] >> os_gen: 2.4.X +[+] [host] >> last_update: 1496637122 +[+] [host] >> state: up +[+] [host] >> mac_vendor: +[+] [host] >> info: +[+] [host] >> Inserting 45.33.32.156 in to database +[+] [host] >> Entry alread exists for 45.33.32.156 in database attempting update +[+] [port] >> Found script output for ssh-hostkey +[+] [port] >> ---------------------- Start 22 ---------------------- +[+] [port] >> ip: 45.33.32.156 +[+] [port] >> port: 22 +[+] [port] >> protocol: tcp +[+] [port] >> name: ssh +[+] [port] >> state: open +[+] [port] >> service: product: OpenSSH version: 6.6.1p1 Ubuntu 2ubuntu2.8 extrainfo: Ubuntu Linux; protocol 2.0 ostype: Linux +``` -Thanks to antonat and thomas for providing feedback. +Subsequent scans can be entered into the same database. New script output will be appended to existing script output. If there are conflicts between what is in the database and the XML file you will be prompted to update: + +``` +$ ./nmapdb.py --debug newscan.xml +[+] [port] >> ---------------------- Start 445 ---------------------- +[+] [port] >> ip: 10.11.1.128 +[+] [port] >> port: 445 +[+] [port] >> protocol: tcp +[+] [port] >> name: microsoft-ds +[+] [port] >> state: open +[+] [port] >> service: product: Windows 2000 microsoft-ds +[+] [port] >> info: +[+] [host] >> Attempting to inserting 10.11.1.128:445 in to database +[+] [port] >> Entry alread exists for 10.11.1.128:445 in database attempting update +[+] [host] >> Could not update automatically - Manual update required +[!] [!!!!] >> ------------------- MANUAL UPDATE REQUIRED! ----------------------- +[+] [port] >> 10.11.1.128:445 tcp exists +[+] [port] >> Name: 'Old' --> 'New' +[+] [port] >> ip: '10.11.1.128' --> '10.11.1.128' +[+] [port] >> port: '445' --> '445' +[+] [port] >> protocol: 'tcp' --> 'tcp' +[+] [port] >> name: 'microsoft-ds' --> 'microsoft-ds' +[+] [port] >> state: 'open' --> 'open' +[+] [port] >> service: 'product: Microsoft Windows 2000 microsoft-ds ostype: Windows 2000' --> 'product: Windows 2000 microsoft-ds' +[+] [port] >> Update entry? y/n +``` + + +Feel free to fork, submit patches, whatever. -argp, Mon Apr 30 14:49:21 EEST 2012 +Huge thanks to Patroklos Argyroudis for the original nmapdb.py which was the catalyst for this script. diff --git a/nmapdb.py b/nmapdb.py index ac44b41..9a032b0 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -1,42 +1,110 @@ #!/usr/bin/env python # # nmapdb - Parse nmap's XML output files and insert them into an SQLite database -# Copyright (c) 2012 Patroklos Argyroudis +# Original nmapdb.py by Patroklos Argyroudis +# Updated/rewritten by Phil Young aka Soldier of FORTRAN +# - Completly re-wrote script: +# - Added support for updating entries in the database +# - Added host script support +# - Added script output appending (script output will not overwrite previous output) +# - Removed booleans, replaced with python built-in import sys import os -import getopt -import xml.dom.minidom +import argparse from pysqlite2 import dbapi2 as sqlite - -VERSION = "1.2" -DEFAULT_DATABASE = "./nmapdb.db" - -true = 1 -false = 0 -vflag = false - -def myprint(msg): - global vflag - if vflag == true: - print msg - +from libnmap.parser import NmapParser + +class c: + BLUE = '\033[94m' + DARKBLUE = '\033[0;34m' + PURPLE = '\033[95m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + WHITE = '\033[1;37m' + ENDC = '\033[0m' + DARKGREY = '\033[1;30m' + + + def disable(self): + self.BLUE = '' + self.GREEN = '' + self.YELLOW = '' + self.DARKBLUE = '' + self.PURPLE = '' + self.WHITE= '' + self.RED = '' + self.ENDC = '' + +verbose = False + +def debug(area ='',msg=''): + global verbose + if verbose: + print '%s[+]%s %s %s>> %s%s%s' % (c.WHITE,c.GREEN,area,c.WHITE,c.YELLOW,msg,c.ENDC) return -def usage(name): - print "usage: %s [options] " % name - print "options:" - print " (-h) --help this message" - print " (-v) --verbose verbose output" - print " (-c) --create specify input SQL file to create SQLite DB" - print " (-d) --database specify output SQLite DB file" - print " (-f) --frequency list most frequent open ports from specified DB" - print " (-n) --nodb do not perform any DB operations (i.e. dry run)" - print " (-V) --version output version number and exit" - print "\nexample: ./nmapdb.py -v -c nmapdb.sql -d myscan.db scan_data.xml" +def mesg(area ='',msg=''): + print '%s[+]%s %s %s>> %s%s%s' % (c.WHITE,c.GREEN,area,c.WHITE,c.YELLOW,msg,c.ENDC) + return +def err(area ='',msg=''): + print '%s[!]%s %s %s>> %s%s%s' % (c.RED,c.RED,area,c.RED,c.RED,msg,c.ENDC) return +def sql_struct(): + return '''CREATE TABLE IF NOT EXISTS hosts ( + ip VARCHAR(16) PRIMARY KEY NOT NULL, + mac VARCHAR(18), + hostname VARCHAR(129), + protocol VARCHAR(5) DEFAULT 'ipv4', + os_name TEXT, + os_family TEXT, + os_accuracy INTEGER, + os_gen TEXT, + last_update TIMESTAMP, + state VARCHAR(8) DEFAULT 'down', + mac_vendor TEXT, + info TEXT +); + +CREATE TABLE IF NOT EXISTS ports ( + ip VARCHAR(16) NOT NULL, + port INTEGER NOT NULL, + protocol VARCHAR(4) NOT NULL, + name VARCHAR(33), + state VARCHAR(33) DEFAULT 'closed', + service TEXT, + info TEXT, + PRIMARY KEY (ip, port, protocol), + CONSTRAINT fk_ports_hosts FOREIGN KEY (ip) REFERENCES hosts(ip) ON DELETE CASCADE +); + +CREATE TRIGGER IF NOT EXISTS fki_ports_hosts_ip +BEFORE INSERT ON ports +FOR EACH ROW BEGIN + SELECT CASE + WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) + THEN RAISE(ABORT, 'insert on table "ports" violates foreign key constraint "fk_ports_hosts"') + END; +END; + +CREATE TRIGGER IF NOT EXISTS fku_ports_hosts_ip +BEFORE UPDATE ON ports +FOR EACH ROW BEGIN + SELECT CASE + WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) + THEN RAISE(ABORT, 'update on table "ports" violates foreign key constraint "fk_ports_hosts"') + END; +END; + +CREATE TRIGGER IF NOT EXISTS fkd_ports_hosts_ip +BEFORE DELETE ON hosts +FOR EACH ROW BEGIN + DELETE from ports WHERE ip = OLD.ip; +END;''' + def dict_factory(cursor, row): d = {} for idx,col in enumerate(cursor.description): @@ -44,316 +112,251 @@ def dict_factory(cursor, row): return d def main(argv, environ): - global vflag - nodb_flag = false - freq_flag = false - db_path = DEFAULT_DATABASE - sql_file = "" - argc = len(argv) - - if argc == 1: - usage(argv[0]) - sys.exit(0) - - try: - alist, args = getopt.getopt(argv[1:], "hvd:c:f:nV", - ["help", "verbose", "database=", "create=", "frequency=", - "nodb", "version"]) - except getopt.GetoptError, msg: - print "%s: %s\n" % (argv[0], msg) - usage(argv[0]); - sys.exit(1) - - for(field, val) in alist: - if field in ("-h", "--help"): - usage(argv[0]) - sys.exit(0) - if field in ("-v", "--verbose"): - vflag = true - if field in ("-d", "--database"): - db_path = val - if field in ("-c", "--create"): - sql_file = val - if field in ("-f", "--frequency"): - freq_flag = true - db_path = val - if field in ("-n", "--nodb"): - nodb_flag = true - if field in ("-V", "--version"): - print "nmapdb v%s by Patroklos Argyroudis " % (VERSION) - print "parse nmap's XML output files and insert them into an SQLite database" - sys.exit(0) - - if freq_flag == false: - if len(args[0]) == 0: - usage(argv[0]) + global verbose + #start argument parser + parser = argparse.ArgumentParser(description='Nmap XML file to SQLite database. Default file nmap.db will be used if none is supplied.') + parser.add_argument('--debug',help='Print verbose information',default=False,dest='debug',action='store_true') + parser.add_argument('-d','--database',help='Filename to use for database. If file doesn\'t exist it will be created. Default is \'nmap.db.\'',dest='scandb', default='nmap.db') + parser.add_argument('nmap_xml',help='Nmap XML file you wish to parse') + args = parser.parse_args() + + if args.debug: + verbose = True + debug('Start ','Debug Enabled') + + mesg('Start ','Using DB file: %s and XML file %s' % (args.scandb,args.nmap_xml)) + conn = sqlite.connect(args.scandb) + cursor = conn.cursor() + debug("Start ","Successfully connected to SQLite DB \"%s\"" % (args.scandb)) + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + if (u'hosts',) not in cursor.fetchall(): + debug('Start ', 'Database does not exist. Creating...Done') + try: + cursor.executescript(sql_struct()) + except sqlite.ProgrammingError, msg: + err("Start ","%s: error: %s\n" % (argv[0], msg)) sys.exit(1) + else: + debug('Start ', 'Database already exists. Continuing.') - if nodb_flag == false: - if db_path == DEFAULT_DATABASE: - print "%s: no output SQLite DB file specified, using \"%s\"\n" % (argv[0], db_path) - - conn = sqlite.connect(db_path) - cursor = conn.cursor() - - myprint("%s: successfully connected to SQLite DB \"%s\"\n" % (argv[0], db_path)) - - # helpful queries on the database - if freq_flag == true: - freq_sql = "select count(port) as frequency,port as fport from ports where ports.state='open' group by port having count(fport) > 1000" - - cursor.execute(freq_sql) - print "Frequency|Port" + try: + nmap_report = NmapParser.parse_fromfile(args.nmap_xml) + debug("Parser","Nmap Results: {0}".format(nmap_report.summary)) + except IOError: + err("Parser"," %s: error: file \"%s\" doesn't exist" % (argv[0], args.nmap_xml)) + sys.exit(-1) + except: + err("Parser"," %s: error: file \"%s\" Issue parsing Nmap XML" % (argv[0], args.nmap_xml)) + sys.exit(-1) + + for host in nmap_report.hosts: + ip = host.address + mac = host.mac + if not host.ipv6: + protocol = 'ipv4' + else: + protocol = 'ipv6' + + if len(host.hostnames) > 0: #I know this isn't great but its fine for now + hostname = host.hostnames[0] + else: + hostname = '' + if host.os_fingerprinted: + os_name = host.os.osmatches[0].name + os_family = host.os.osmatches[0].osclasses[0].osfamily + os_accuracy = host.os.osmatches[0].accuracy + os_gen = host.os.osmatches[0].osclasses[0].osgen + else: + os_name = '' + os_family = '' + os_accuracy = '' + os_gen = '' + + timestamp = host.endtime + state = host.status + mac_vendor = host.vendor + + # Some script store results in host scripts, we'll store them in one long string + host_script = "" + for script_out in host.scripts_results: + debug("[host]","Found script output for %s" % script_out['id']) + host_script += script_out['id']+":\n"+script_out['output']+'\n' + if len(script_out['elements']) > 0: host_script += ' Elements:\n' + for elem in script_out['elements']: + if type(script_out['elements'][elem]) is not dict: + host_script += ' '+str(elem)+": "+ str(script_out['elements'][elem])+'\n' + else: + host_script += ' '+elem+':\n' + for item in script_out['elements'][elem]: + if item is not None and script_out['elements'][elem][item] is not None: + host_script += ' '+item+': '+script_out['elements'][elem][item].strip()+'\n' + host_script += '\n' - for row in cursor: - print(row) - - sys.exit(0) + info_str = '' + for line in host_script.splitlines(): + info_str += '['+ip+'] '+line+'\n' - if nodb_flag == false: - if sql_file != "": - sql_string = open(sql_file, "r").read() - try: - cursor.executescript(sql_string) - except sqlite.ProgrammingError, msg: - print "%s: error: %s\n" % (argv[0], msg) - sys.exit(1) - - myprint("%s: SQLite DB created using SQL file \"%s\"\n" % (argv[0], sql_file)) - - for fname in args: + debug("[host]","ip: %s" % (ip)) + debug("[host]","mac: %s" % (mac)) + debug("[host]","hostname: %s" % (hostname)) + debug("[host]","protocol: %s" % (protocol)) + debug("[host]","os_name: %s" % (os_name)) + debug("[host]","os_family: %s" % (os_family)) + debug("[host]","os_accuracy: %s" % (os_accuracy)) + debug("[host]","os_gen: %s" % (os_gen)) + debug("[host]","last_update: %s" % (timestamp)) + debug("[host]","state: %s" % (state)) + debug("[host]","mac_vendor: %s" % (mac_vendor)) + debug("[host]","info: %s" % info_str) + debug("[host]","Inserting %s in to database" % ip) try: - doc = xml.dom.minidom.parse(fname) - except IOError: - print "%s: error: file \"%s\" doesn't exist\n" % (argv[0], fname) - continue - except xml.parsers.expat.ExpatError: - print "%s: error: file \"%s\" doesn't seem to be XML\n" % (argv[0], fname) - continue - - for host in doc.getElementsByTagName("host"): - try: - address = host.getElementsByTagName("address")[0] - ip = address.getAttribute("addr") - protocol = address.getAttribute("addrtype") - except: - # move to the next host since the IP is our primary key - continue + cursor.execute("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (ip, mac, hostname, protocol, os_name, os_family, os_accuracy, + os_gen, timestamp, state, mac_vendor, info_str)) + debug("[host]","Inserted %s in to database" % ip) - try: - mac_address = host.getElementsByTagName("address")[1] - mac = mac_address.getAttribute("addr") - mac_vendor = mac_address.getAttribute("vendor") - except: - mac = "" - mac_vendor = "" - - try: - hname = host.getElementsByTagName("hostname")[0] - hostname = hname.getAttribute("name") - except: - hostname = "" + except sqlite.IntegrityError, msg: - try: - status = host.getElementsByTagName("status")[0] - state = status.getAttribute("state") - except: - state = "" + debug("[host]","Entry alread exists for %s in database attempting update" % ip) + cursor.execute("SELECT * FROM hosts WHERE ip = '%s'" % ip ) + db = dict_factory(cursor, cursor.fetchone()) + + # We automatically append new script output without asking the user + if host_script != "" and db['info'].find(info_str) < 0: + new_info = db['info'] + "\n" + info_str + debug("[host]"," Appending script output") + cursor.execute("UPDATE hosts SET info=? WHERE ip = ?" , (new_info, ip)) + + # If the host already exsits lets check with the user if we wan't to update + if ( db['mac'] != mac + and db['hostname'] != hostname and db['hostname'] != '' + and db['protocol'] != protocol + and db['os_name'] != os_name and db['os_name'] != '' + and db['os_family'] != os_family and db['os_family'] != '' + and db['os_gen'] != os_gen and db['os_gen'] != '' + and db['state'] != state + and db['mac_vendor']!= mac_vendor + and db['info'] != host_script): + # So we already have an entry. If theres no new information we continue to ports + # If there's a bunch of new entries we'll ask the user what to do + debug("[host]","Could not update automatically - Manual update required") + mesg("[host]","%s entry exists" % ip) + mesg("[host]","Name: 'Old' --> 'New'") + mesg("[host]","mac: '"+db['mac']+"' --> '%s'" % mac) + mesg("[host]","hostname: '"+db['hostname']+"' --> '%s'" % hostname) + mesg("[host]","protocol: '"+db['protocol']+"' --> '%s'" % protocol) + mesg("[host]","os_name: '"+db['os_name']+"' --> '%s'" % os_name) + mesg("[host]","os_family: '"+db['os_family']+"' --> '%s'" % os_family) + mesg("[host]","os_accuracy: '"+str(db['os_accuracy'])+"' --> '%s'" % os_accuracy) + mesg("[host]","os_gen: '"+db['os_gen']+"' --> '%s'" % os_gen) + mesg("[host]","timestamp: '"+str(db['last_update'])+"' --> '%s'" % timestamp) + mesg("[host]","state: '"+db['state']+"' --> '%s'" % state) + mesg("[host]","mac_vendor: '"+db['mac_vendor']+" --> '%s'" % mac_vendor) + mesg("[host]","info: '"+db['info']+"' --> '%s'" % host_script) + mesg("[host]","Update entry? y/n") + user_input = sys.stdin.readline().strip()[:1] + if user_input == 'y': + debug("[hosts]","Updating %s entry" % ip) + sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', info='%s' WHERE ip = '%s'" % + (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) + cursor.execute(sql) - try: - os_el = host.getElementsByTagName("os")[0] - os_match = os_el.getElementsByTagName("osmatch")[0] - os_name = os_match.getAttribute("name") - os_accuracy = os_match.getAttribute("accuracy") - os_class = os_el.getElementsByTagName("osclass")[0] - os_family = os_class.getAttribute("osfamily") - os_gen = os_class.getAttribute("osgen") - except: - os_name = "" - os_accuracy = "" - os_family = "" - os_gen = "" + else: + debug("[hosts]","Skipping %s entry" % ip) + except: + print "%s: unknown exception during insert into table hosts\n" % (argv[0]) + continue - try: - timestamp = host.getAttribute("endtime") - except: - timestamp = "" + ports = host.get_open_ports() + + for port in ports: + svc = host.get_service(port[0]) + service_str = svc.banner + port_name = svc.service + pn = str(svc.port) + protocol = svc.protocol + state = svc.state + + # Service Scripts - we'll store them in one long string + svc_script = "" + for script_out in svc.scripts_results: + debug("[port]","Found script output for %s" % script_out['id']) + svc_script += script_out['id']+":\n"+script_out['output']+'\n' + if len(script_out['elements']) > 0: svc_script += ' Elements:\n' + for elem in script_out['elements']: + if not elem: + continue + if (type(script_out['elements'][elem]) is not dict + and type(script_out['elements'][elem]) is not list): + svc_script += ' '+str(elem)+": "+ script_out['elements'][elem]+'\n' + elif type(script_out['elements'][elem]) is dict: + svc_script += ' '+elem+':\n' + for item in script_out['elements'][elem]: + if item is not None and script_out['elements'][elem][item] is not None: + svc_script += ' '+item+': '+script_out['elements'][elem][item].strip()+'\n' + + svc_script += '\n' + #print svc_script + info_str = '' + for line in svc_script.splitlines(): + info_str += "[%s:%s] %s\n" % (ip, pn, line) + + debug("[port]","---------------------- Start "+pn+" ----------------------") + debug("[port]","ip: %s" % (ip)) + debug("[port]","port: %s" % (pn)) + debug("[port]","protocol: %s" % (protocol)) + debug("[port]","name: %s" % (port_name)) + debug("[port]","state: %s" % (state)) + debug("[port]","service: %s" % (service_str)) + debug("[port]","info: %s" % (info_str)) + debug("[host]","Attempting to inserting %s:%s in to database" % (ip,pn)) try: - hostscript = host.getElementsByTagName("hostscript")[0] - script = hostscript.getElementsByTagName("script")[0] - id = script.getAttribute("id") - - if id == "whois": - whois_str = script.getAttribute("output") - else: - whois_str = "" + cursor.execute("INSERT INTO ports VALUES (?, ?, ?, ?, ?, ?, ?)", (ip, pn, protocol, port_name, state, service_str, info_str)) + except sqlite.IntegrityError, msg: + debug("[port]","Entry alread exists for %s:%s in database attempting update" % (ip,pn)) + + cursor.execute("SELECT * FROM ports WHERE ip = '%s' AND port = '%s' and protocol ='%s'" % (ip, pn, protocol) ) + db = dict_factory(cursor, cursor.fetchone()) + + # We automatically append new script output without asking the user + if info_str != "" and db['info'].find(info_str) < 0: + new_info = db['info'] + "\n" + info_str + debug("[ports]","Appending script output %s" % info_str) + cursor.execute("UPDATE ports SET info=? WHERE ip = ? AND port = ? and protocol =?" , (new_info, ip, pn, protocol)) + - except: - whois_str = "" - - myprint("================================================================") - - myprint("[hosts] ip:\t\t%s" % (ip)) - myprint("[hosts] mac:\t\t%s" % (mac)) - myprint("[hosts] hostname:\t%s" % (hostname)) - myprint("[hosts] protocol:\t%s" % (protocol)) - myprint("[hosts] os_name:\t%s" % (os_name)) - myprint("[hosts] os_family:\t%s" % (os_family)) - myprint("[hosts] os_accuracy:\t%s" % (os_accuracy)) - myprint("[hosts] os_gen:\t\t%s" % (os_gen)) - myprint("[hosts] last_update:\t%s" % (timestamp)) - myprint("[hosts] state:\t\t%s" % (state)) - myprint("[hosts] mac_vendor:\t%s" % (mac_vendor)) - myprint("[hosts] whois:\n") - myprint("%s\n" % (whois_str)) - - if nodb_flag == false: - try: - cursor.execute("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (ip, mac, hostname, protocol, os_name, os_family, os_accuracy, - os_gen, timestamp, state, mac_vendor, whois_str)) - except sqlite.IntegrityError, msg: - cursor.execute("SELECT * FROM hosts WHERE ip = '%s'" % ip ) - db = dict_factory(cursor, cursor.fetchone()) - if not ( db['mac'] == mac - and db['hostname'] == hostname - and db['protocol'] == protocol - and db['os_name'] == os_name - and db['os_family'] == os_family - and db['os_gen'] == os_gen - and db['state'] == state - and db['mac_vendor'] == mac_vendor - and db['whois'] == whois_str): - # So we already have an entry. If theres no new information we continue to ports - # If there's a bunch of new entries we'll ask the user what to do - print "[hosts] %s entry exists" % ip - print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" - print "[hosts] Name: 'Old' --> 'New'" - print("[hosts] mac: '"+db['mac']+"' --> '%s'" % mac) - print("[hosts] hostname: '"+db['hostname']+"' --> '%s'" % hostname) - print("[hosts] protocol: '"+db['protocol']+"' --> '%s'" % protocol) - print("[hosts] os_name: '"+db['os_name']+"' --> '%s'" % os_name) - print("[hosts] os_family: '"+db['os_family']+"' --> '%s'" % os_family) - print("[hosts] os_accuracy: '"+str(db['os_accuracy'])+"' --> '%s'" % os_accuracy) - print("[hosts] os_gen: '"+db['os_gen']+"' --> '%s'" % os_gen) - print("[hosts] timestamp: '"+str(db['last_update'])+"' --> '%s'" % timestamp) - print("[hosts] state: '"+db['state']+"' --> '%s'" % state) - print("[hosts] mac_vendor: '"+db['mac_vendor']+" --> '%s'" % mac_vendor) - print("[hosts] whois: '"+db['whois']+"' --> '%s'" % whois_str) - print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" - print "[hosts] Update entry? y/n" - user_input = sys.stdin.readline().strip()[:1] - if user_input == 'y': - myprint("[hosts] updating %s entry" % ip) - sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', whois='%s' WHERE ip = '%s'" % - (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,whois_str, ip )) - cursor.execute(sql) - - else: - myprint("[hosts] Skipping %s entry" % ip) - continue - except: - print "%s: unknown exception during insert into table hosts\n" % (argv[0]) - continue + if (db['name'] != port_name or db['service'] != service_str or db['state'] != state) and service_str != '': + debug("[host]","Could not update automatically - Manual update required") + err("[!!!!]","------------------- MANUAL UPDATE REQUIRED! -----------------------") + mesg("[port]","%s:%s %s exists" % (ip, pn, protocol)) + mesg("[port]","Name: 'Old' --> 'New'") + mesg("[port]","ip: '"+db['ip']+"' --> '%s'" % ip) + mesg("[port]","port: '"+str(db['port'])+"' --> '%s'" % pn) + mesg("[port]","protocol: '"+db['protocol']+"' --> '%s'" % protocol) + mesg("[port]","name: '"+db['name']+"' --> '%s'" % port_name) + mesg("[port]","state: '"+db['state']+"' --> '%s'" % state) + mesg("[port]","service: '"+db['service']+"' --> '%s'" % service_str) + mesg("[port]","Update entry? y/n") + user_input = sys.stdin.readline().strip()[:1] + if user_input == 'y': + cursor.execute("UPDATE ports SET name=?, state=?, service=? WHERE ip = ? AND port = ? and protocol = ?", + (port_name, state, service_str, ip, pn, protocol)) + else: debug('[port]',' skipping entry') - try: - ports = host.getElementsByTagName("ports")[0] - ports = ports.getElementsByTagName("port") except: - print "%s: host %s has no open ports\n" % (argv[0], ip) + print "%s: unknown exception during insert into table ports\n" % (argv[0]) continue + debug("[port]","---------------------- End "+pn+" ----------------------") - for port in ports: - pn = port.getAttribute("portid") - protocol = port.getAttribute("protocol") - state_el = port.getElementsByTagName("state")[0] - state = state_el.getAttribute("state") - - try: - service = port.getElementsByTagName("service")[0] - port_name = service.getAttribute("name") - product_descr = service.getAttribute("product") - product_ver = service.getAttribute("version") - product_extra = service.getAttribute("extrainfo") - except: - service = "" - port_name = "" - product_descr = "" - product_ver = "" - product_extra = "" - - service_str = "%s %s %s" % (product_descr, product_ver, product_extra) - - info_str = "" - - for i in (0, 1): - try: - script = port.getElementsByTagName("script")[i] - script_id = script.getAttribute("id") - script_output = script.getAttribute("output") - except: - script_id = "" - script_output = "" - - if script_id != "" and script_output != "": - info_str += "%s: %s\n" % (script_id, script_output) - - myprint("------------------------------------------------") - - myprint("[ports] ip:\t\t%s" % (ip)) - myprint("[ports] port:\t\t%s" % (pn)) - myprint("[ports] protocol:\t%s" % (protocol)) - myprint("[ports] name:\t\t%s" % (port_name)) - myprint("[ports] state:\t\t%s" % (state)) - myprint("[ports] service:\t%s" % (service_str)) - - if info_str != "": - myprint("[ports] info:\n") - myprint("%s\n" % (info_str)) - - if nodb_flag == false: - try: - cursor.execute("INSERT INTO ports VALUES (?, ?, ?, ?, ?, ?, ?)", (ip, pn, protocol, port_name, state, service_str, info_str)) - except sqlite.IntegrityError, msg: - cursor.execute("SELECT * FROM ports WHERE ip = '%s' AND port = '%s' and protocol ='%s'" % (ip, pn, protocol) ) - db = dict_factory(cursor, cursor.fetchone()) - - if info_str != "" and db['info'].find(info_str) <= 0: - new_info = db['info'] + "\n" + info_str - myprint("[ports] Appending info %s" % info_str) - cursor.execute("UPDATE ports SET info=? WHERE ip = ? AND port = ? and protocol =?" , (new_info, ip, pn, protocol)) - - - if db['name'] != port_name or db['service'] != service_str or db['state'] != state: - print '[ports] %s:%s %s exists' % (ip, pn, protocol) - print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" - print "[ports] Name: 'Old' --> 'New'" - print("[ports] ip: '"+db['ip']+"' --> '%s'" % ip) - print("[ports] port: '"+str(db['port'])+"' --> '%s'" % pn) - print("[ports] protocol: '"+db['protocol']+"' --> '%s'" % protocol) - print("[ports] name: '"+db['name']+"' --> '%s'" % port_name) - print("[ports] state: '"+db['state']+"' --> '%s'" % state) - print("[ports] service: '"+db['service']+"' --> '%s'" % service_str) - print "=-=-=-=-=-=-=-=-=-=-=-=-=-=-" - print "[ports] Update entry? y/n" - user_input = sys.stdin.readline().strip()[:1] - cursor.execute("UPDATE ports SET name=?, state=?, service=? WHERE ip = ? AND port = ? and protocol = ?", - (port_name, state, service_str, ip, pn, protocol)) - - except: - print "%s: unknown exception during insert into table ports\n" % (argv[0]) - continue + debug("[host]", "====================== End "+ip+" ======================") - myprint("================================================================") - if nodb_flag == false: - conn.commit() + conn.commit() if __name__ == "__main__": main(sys.argv, os.environ) sys.exit(0) -# EOF From db0a3b0a8d5422f9e920645465e7bdb3c318d183 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 21:54:03 -0700 Subject: [PATCH 5/9] README.md --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From a0f3cb8c9e55fa5f5314bf6ba8f7233bdc2cf196 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 22:14:09 -0700 Subject: [PATCH 6/9] added --force-update for Derek --- nmapdb.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/nmapdb.py b/nmapdb.py index 9a032b0..cf875b2 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -116,6 +116,7 @@ def main(argv, environ): #start argument parser parser = argparse.ArgumentParser(description='Nmap XML file to SQLite database. Default file nmap.db will be used if none is supplied.') parser.add_argument('--debug',help='Print verbose information',default=False,dest='debug',action='store_true') + parser.add_argument('--force-update',help='Overwrite previous information.',default=False,dest='force',action='store_true') parser.add_argument('-d','--database',help='Filename to use for database. If file doesn\'t exist it will be created. Default is \'nmap.db.\'',dest='scandb', default='nmap.db') parser.add_argument('nmap_xml',help='Nmap XML file you wish to parse') args = parser.parse_args() @@ -228,6 +229,12 @@ def main(argv, environ): debug("[host]"," Appending script output") cursor.execute("UPDATE hosts SET info=? WHERE ip = ?" , (new_info, ip)) + if args.force: + debug("[host]","--force-update enabled - overwritting entry!") + sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', info='%s' WHERE ip = '%s'" % + (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) + cursor.execute(sql) + # If the host already exsits lets check with the user if we wan't to update if ( db['mac'] != mac and db['hostname'] != hostname and db['hostname'] != '' @@ -237,7 +244,7 @@ def main(argv, environ): and db['os_gen'] != os_gen and db['os_gen'] != '' and db['state'] != state and db['mac_vendor']!= mac_vendor - and db['info'] != host_script): + and db['info'] != host_script) and not args.force: # So we already have an entry. If theres no new information we continue to ports # If there's a bunch of new entries we'll ask the user what to do debug("[host]","Could not update automatically - Manual update required") @@ -257,7 +264,7 @@ def main(argv, environ): mesg("[host]","Update entry? y/n") user_input = sys.stdin.readline().strip()[:1] if user_input == 'y': - debug("[hosts]","Updating %s entry" % ip) + debug("[host]","Updating %s entry" % ip) sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', info='%s' WHERE ip = '%s'" % (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) cursor.execute(sql) @@ -323,12 +330,16 @@ def main(argv, environ): # We automatically append new script output without asking the user if info_str != "" and db['info'].find(info_str) < 0: new_info = db['info'] + "\n" + info_str - debug("[ports]","Appending script output %s" % info_str) + debug("[port]","Appending script output %s" % info_str) cursor.execute("UPDATE ports SET info=? WHERE ip = ? AND port = ? and protocol =?" , (new_info, ip, pn, protocol)) + if args.force: + debug("[port]","--force-update enabled - overwritting entry!") + cursor.execute("UPDATE ports SET name=?, state=?, service=? WHERE ip = ? AND port = ? and protocol = ?", + (port_name, state, service_str, ip, pn, protocol)) - if (db['name'] != port_name or db['service'] != service_str or db['state'] != state) and service_str != '': - debug("[host]","Could not update automatically - Manual update required") + if (not args.force) and (db['name'] != port_name or db['service'] != service_str or db['state'] != state) and service_str != '': + debug("[port]","Could not update automatically - Manual update required") err("[!!!!]","------------------- MANUAL UPDATE REQUIRED! -----------------------") mesg("[port]","%s:%s %s exists" % (ip, pn, protocol)) mesg("[port]","Name: 'Old' --> 'New'") From 86ddd04ad6b406e97034395a24f8c5264b874f61 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 22:17:06 -0700 Subject: [PATCH 7/9] fixed stupid typo --- nmapdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmapdb.py b/nmapdb.py index cf875b2..b09d114 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -317,7 +317,7 @@ def main(argv, environ): debug("[port]","state: %s" % (state)) debug("[port]","service: %s" % (service_str)) debug("[port]","info: %s" % (info_str)) - debug("[host]","Attempting to inserting %s:%s in to database" % (ip,pn)) + debug("[host]","Attempting to insert %s:%s in to database" % (ip,pn)) try: cursor.execute("INSERT INTO ports VALUES (?, ?, ?, ?, ?, ?, ?)", (ip, pn, protocol, port_name, state, service_str, info_str)) From 401c6a3f6a0bcca6ffa9726d329cff6995bed565 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 22:27:32 -0700 Subject: [PATCH 8/9] fix in sql statement when dealing with script output --- nmapdb.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nmapdb.py b/nmapdb.py index b09d114..1b48745 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -231,9 +231,8 @@ def main(argv, environ): if args.force: debug("[host]","--force-update enabled - overwritting entry!") - sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', info='%s' WHERE ip = '%s'" % - (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) - cursor.execute(sql) + cursor.execute("UPDATE hosts SET mac=?, hostname=?, protocol=?, os_name=?, os_family=?, os_accuracy=?, os_gen=?, last_update=?, state=?, mac_vendor=?, info=? WHERE ip = ?", (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) + # If the host already exsits lets check with the user if we wan't to update if ( db['mac'] != mac @@ -265,10 +264,7 @@ def main(argv, environ): user_input = sys.stdin.readline().strip()[:1] if user_input == 'y': debug("[host]","Updating %s entry" % ip) - sql = ("UPDATE hosts SET mac='%s', hostname='%s', protocol='%s', os_name='%s', os_family='%s', os_accuracy='%s', os_gen='%s', last_update='%s', state='%s', mac_vendor='%s', info='%s' WHERE ip = '%s'" % - (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) - cursor.execute(sql) - + cursor.execute("UPDATE hosts SET mac=?, hostname=?, protocol=?, os_name=?, os_family=?, os_accuracy=?, os_gen=?, last_update=?, state=?, mac_vendor=?, info=? WHERE ip = ?", (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) else: debug("[hosts]","Skipping %s entry" % ip) except: From 0a38dbfca9698c3f99473a1c1191d546bd218725 Mon Sep 17 00:00:00 2001 From: Dade Murphy Date: Sun, 4 Jun 2017 22:31:45 -0700 Subject: [PATCH 9/9] updated documentation with new features --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4b8508..571a4e5 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ usage: nmapdb.py [-h] [--debug] [-d SCANDB] nmap_xml nmapdb.py: error: too few arguments $ ./nmapdb.py -h -usage: nmapdb.py [-h] [--debug] [-d SCANDB] nmap_xml +usage: nmapdb.py [-h] [--debug] [--force-update] [-d SCANDB] nmap_xml Nmap XML file to SQLite database. Default file nmap.db will be used if none is supplied. @@ -31,6 +31,7 @@ positional arguments: optional arguments: -h, --help show this help message and exit --debug Print verbose information + --force-update Overwrite previous information. -d SCANDB, --database SCANDB Filename to use for database. If file doesn't exist it will be created. Default is 'nmap.db.' @@ -101,6 +102,8 @@ $ ./nmapdb.py --debug scanme.xml [+] [port] >> service: product: OpenSSH version: 6.6.1p1 Ubuntu 2ubuntu2.8 extrainfo: Ubuntu Linux; protocol 2.0 ostype: Linux ``` +### Updates + Subsequent scans can be entered into the same database. New script output will be appended to existing script output. If there are conflicts between what is in the database and the XML file you will be prompted to update: ```