diff --git a/LICENSE b/LICENSE deleted file mode 100644 index c410b2b..0000000 --- a/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ - nmapdb - Parse nmap's XML output files and insert them into an SQLite database - - Copyright (c) 2012 Patroklos Argyroudis - Copyright (c) 2012 Census, Inc. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The names of the authors and copyright holders may not be used to - endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README b/README deleted file mode 100644 index 567fda2..0000000 --- a/README +++ /dev/null @@ -1,63 +0,0 @@ -nmapdb parses nmap's XML output files and inserts them into an SQLite database. - -I coded this a while back (mid 2009) and have been using it since. Some -people I have shared nmapdb with have found it useful, so I am releasing it -publicly. - -Example usage: - -$ sudo nmap -A -oX scanme.xml scanme.nmap.org - -Starting Nmap ... - -$ ls scanme.xml -scanme.xml -$ ./nmapdb.py -h -usage: ./nmapdb.py [options] -options: - (-h) --help this message - (-v) --verbose verbose output - (-c) --create specify input SQL file to create SQLite DB - (-d) --database specify output SQLite DB file - (-f) --frequency list most frequent open ports from specified DB - (-n) --nodb do not perform any DB operations (i.e. dry run) - (-V) --version output version number and exit - -Use -c to create a database from the schema on the first run: - -$ ./nmapdb.py -c nmapdb.sql -d myscan.db scanme.xml -$ file myscan.db -myscan.db: SQLite 3.x database -$ sqlite3 myscan.db -SQLite version 3.7.7 ... -sqlite> select * from hosts; -74.207.244.221||scanme.nmap.org|ipv4|Linux 2.6.18|Linux|85|2.6.X|1316681984|up| -sqlite> select * from ports; -74.207.244.221|22|tcp|ssh|open| -74.207.244.221|80|tcp|http|open| - -Subsequent scans can be entered into the same database: - -$ ./nmapdb.py -d myscan.db bar.xml foo.xml host1.xml host2.xml \ - host3.xml host4.xml meh.xml (or simply *.xml) -$ sqlite3 myscan.db -SQLite version 3.7.7 ... -sqlite> select * from ports where ports.port='22'; -aa.bb.244.221|22|tcp|ssh|open| -204.cc.ddd.250|22|tcp|ssh|open| -bbb.242.aa.180|22|tcp|ssh|open| -aa.bb.121.21|22|tcp|ssh|open| -sqlite> select * from ports where ports.port='23'; -192.168.1.254|23|tcp|telnet|open| -sqlite> select * from hosts inner join ports on hosts.ip=ports.ip where hosts.ip='192.168.1.254' and ports.state='open'; -192.168.1.254|00:00:C5:CF:86:30|modem|ipv4||||||up|Farallon Computing/netopia|192.168.1.254|23|tcp|telnet|open| -192.168.1.254|00:00:C5:CF:86:30|modem|ipv4||||||up|Farallon Computing/netopia|192.168.1.254|80|tcp|http|open| -sqlite> select * from hosts inner join ports on hosts.ip=ports.ip where hosts.os_name like '%bsd%' and ports.port=22; -aa.bb.91.25||foo.bar.org|ipv4|FreeBSD 7.0-STABLE|FreeBSD|95|7.X|1231841556|up||aa.bb.91.25|22|tcp|ssh|open| - -Feel free to fork, submit patches, whatever. - -Thanks to antonat and thomas for providing feedback. - -argp, Mon Apr 30 14:49:21 EEST 2012 - diff --git a/README.md b/README.md new file mode 100644 index 0000000..571a4e5 --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# NmapDB + +nmapdb parses nmap's XML output files and inserts them into an SQLite database. + +## Example usage: + +``` +$ sudo nmap -A -oX scanme.xml scanme.nmap.org + +Starting Nmap ... + +$ ls scanme.xml +scanme.xml +``` +### Running: + +``` +$ ./nmapdb.py +usage: nmapdb.py [-h] [--debug] [-d SCANDB] nmap_xml +nmapdb.py: error: too few arguments + +$ ./nmapdb.py -h +usage: nmapdb.py [-h] [--debug] [--force-update] [-d SCANDB] nmap_xml + +Nmap XML file to SQLite database. Default file nmap.db will be used if none is +supplied. + +positional arguments: + nmap_xml Nmap XML file you wish to parse + +optional arguments: + -h, --help show this help message and exit + --debug Print verbose information + --force-update Overwrite previous information. + -d SCANDB, --database SCANDB + Filename to use for database. If file doesn't exist it + will be created. Default is 'nmap.db.' + +``` +**nmapdb** will create a database from internal schema if the tables hosts and ports doesn't exist in your sqlite database. If no database file is supplied it will use the default file `nmap.db`. + +So long as you are ok with the defaults the program can be used with: `./nmapdb.py output.xml`. + +``` +$ ./nmapdb.py scanme.xml +[+] Start >> Using DB file: nmap.db and XML file scanme.xml + +$ file nmap.db +nmap.db: SQLite 3.x database, last written using SQLite version 3014001 + +$ sqlite3 nmap.db +SQLite version 3.14.1 2016-08-11 18:53:32 +Enter ".help" for usage hints. +sqlite> select * from hosts; +45.33.32.156||scanme.nmap.org|ipv4|DD-WRT v24-sp2 (Linux 2.4.37)|Linux|100|2.4.X|1496637122|up|| +sqlite> select * from ports; +45.33.32.156|22|tcp|ssh|open|product: OpenSSH version: 6.6.1p1 Ubuntu 2ubuntu2.8 extrainfo: Ubuntu Linux; protocol 2.0 ostype: Linux +45.33.32.156|80|tcp|http|open|product: Apache httpd version: 2.4.7 extrainfo: (Ubuntu)|[45.33.32.156:80] http-favicon: +[45.33.32.156:80] +[45.33.32.156:80] http-title: +[45.33.32.156:80] Go ahead and ScanMe! +[45.33.32.156:80] Elements: +[45.33.32.156:80] title: Go ahead and ScanMe! +[45.33.32.156:80] +45.33.32.156|9929|tcp|nping-echo|open|product: Nping echo| +45.33.32.156|31337|tcp|tcpwrapped|open|| +sqlite> +``` + +### Debug + +Turning on debug with `--debug` will show you a lot more information: + +``` +$ ./nmapdb.py --debug scanme.xml +[+] Start >> Debug Enabled +[+] Start >> Using DB file: nmap.db and XML file scanme.xml +[+] Start >> Successfully connected to SQLite DB "nmap.db" +[+] Start >> Database already exists. Continuing. +[+] Parser >> Nmap Results: Nmap done at Mon Jun 5 00:32:02 2017; 1 IP address (1 host up) scanned in 16.90 seconds +[+] [host] >> ip: 45.33.32.156 +[+] [host] >> mac: +[+] [host] >> hostname: scanme.nmap.org +[+] [host] >> protocol: ipv4 +[+] [host] >> os_name: DD-WRT v24-sp2 (Linux 2.4.37) +[+] [host] >> os_family: Linux +[+] [host] >> os_accuracy: 100 +[+] [host] >> os_gen: 2.4.X +[+] [host] >> last_update: 1496637122 +[+] [host] >> state: up +[+] [host] >> mac_vendor: +[+] [host] >> info: +[+] [host] >> Inserting 45.33.32.156 in to database +[+] [host] >> Entry alread exists for 45.33.32.156 in database attempting update +[+] [port] >> Found script output for ssh-hostkey +[+] [port] >> ---------------------- Start 22 ---------------------- +[+] [port] >> ip: 45.33.32.156 +[+] [port] >> port: 22 +[+] [port] >> protocol: tcp +[+] [port] >> name: ssh +[+] [port] >> state: open +[+] [port] >> service: product: OpenSSH version: 6.6.1p1 Ubuntu 2ubuntu2.8 extrainfo: Ubuntu Linux; protocol 2.0 ostype: Linux +``` + +### Updates + +Subsequent scans can be entered into the same database. New script output will be appended to existing script output. If there are conflicts between what is in the database and the XML file you will be prompted to update: + +``` +$ ./nmapdb.py --debug newscan.xml +[+] [port] >> ---------------------- Start 445 ---------------------- +[+] [port] >> ip: 10.11.1.128 +[+] [port] >> port: 445 +[+] [port] >> protocol: tcp +[+] [port] >> name: microsoft-ds +[+] [port] >> state: open +[+] [port] >> service: product: Windows 2000 microsoft-ds +[+] [port] >> info: +[+] [host] >> Attempting to inserting 10.11.1.128:445 in to database +[+] [port] >> Entry alread exists for 10.11.1.128:445 in database attempting update +[+] [host] >> Could not update automatically - Manual update required +[!] [!!!!] >> ------------------- MANUAL UPDATE REQUIRED! ----------------------- +[+] [port] >> 10.11.1.128:445 tcp exists +[+] [port] >> Name: 'Old' --> 'New' +[+] [port] >> ip: '10.11.1.128' --> '10.11.1.128' +[+] [port] >> port: '445' --> '445' +[+] [port] >> protocol: 'tcp' --> 'tcp' +[+] [port] >> name: 'microsoft-ds' --> 'microsoft-ds' +[+] [port] >> state: 'open' --> 'open' +[+] [port] >> service: 'product: Microsoft Windows 2000 microsoft-ds ostype: Windows 2000' --> 'product: Windows 2000 microsoft-ds' +[+] [port] >> Update entry? y/n +``` + + +Feel free to fork, submit patches, whatever. + +Huge thanks to Patroklos Argyroudis for the original nmapdb.py which was the catalyst for this script. + diff --git a/TODO b/TODO deleted file mode 100644 index 0bbb33f..0000000 --- a/TODO +++ /dev/null @@ -1,4 +0,0 @@ -* Helpful SQL queries on a database (e.g. open port frequency) - -* Add the MySQL support submitted by @antonat - diff --git a/nmapdb.py b/nmapdb.py index 83a7f8c..1b48745 100755 --- a/nmapdb.py +++ b/nmapdb.py @@ -1,293 +1,369 @@ #!/usr/bin/env python # # nmapdb - Parse nmap's XML output files and insert them into an SQLite database -# Copyright (c) 2012 Patroklos Argyroudis +# Original nmapdb.py by Patroklos Argyroudis +# Updated/rewritten by Phil Young aka Soldier of FORTRAN +# - Completly re-wrote script: +# - Added support for updating entries in the database +# - Added host script support +# - Added script output appending (script output will not overwrite previous output) +# - Removed booleans, replaced with python built-in import sys import os -import getopt -import xml.dom.minidom +import argparse from pysqlite2 import dbapi2 as sqlite - -VERSION = "1.2" -DEFAULT_DATABASE = "./nmapdb.db" - -true = 1 -false = 0 -vflag = false - -def myprint(msg): - global vflag - if vflag == true: - print msg - +from libnmap.parser import NmapParser + +class c: + BLUE = '\033[94m' + DARKBLUE = '\033[0;34m' + PURPLE = '\033[95m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + WHITE = '\033[1;37m' + ENDC = '\033[0m' + DARKGREY = '\033[1;30m' + + + def disable(self): + self.BLUE = '' + self.GREEN = '' + self.YELLOW = '' + self.DARKBLUE = '' + self.PURPLE = '' + self.WHITE= '' + self.RED = '' + self.ENDC = '' + +verbose = False + +def debug(area ='',msg=''): + global verbose + if verbose: + print '%s[+]%s %s %s>> %s%s%s' % (c.WHITE,c.GREEN,area,c.WHITE,c.YELLOW,msg,c.ENDC) return -def usage(name): - print "usage: %s [options] " % name - print "options:" - print " (-h) --help this message" - print " (-v) --verbose verbose output" - print " (-c) --create specify input SQL file to create SQLite DB" - print " (-d) --database specify output SQLite DB file" - print " (-f) --frequency list most frequent open ports from specified DB" - print " (-n) --nodb do not perform any DB operations (i.e. dry run)" - print " (-V) --version output version number and exit" +def mesg(area ='',msg=''): + print '%s[+]%s %s %s>> %s%s%s' % (c.WHITE,c.GREEN,area,c.WHITE,c.YELLOW,msg,c.ENDC) + return +def err(area ='',msg=''): + print '%s[!]%s %s %s>> %s%s%s' % (c.RED,c.RED,area,c.RED,c.RED,msg,c.ENDC) return +def sql_struct(): + return '''CREATE TABLE IF NOT EXISTS hosts ( + ip VARCHAR(16) PRIMARY KEY NOT NULL, + mac VARCHAR(18), + hostname VARCHAR(129), + protocol VARCHAR(5) DEFAULT 'ipv4', + os_name TEXT, + os_family TEXT, + os_accuracy INTEGER, + os_gen TEXT, + last_update TIMESTAMP, + state VARCHAR(8) DEFAULT 'down', + mac_vendor TEXT, + info TEXT +); + +CREATE TABLE IF NOT EXISTS ports ( + ip VARCHAR(16) NOT NULL, + port INTEGER NOT NULL, + protocol VARCHAR(4) NOT NULL, + name VARCHAR(33), + state VARCHAR(33) DEFAULT 'closed', + service TEXT, + info TEXT, + PRIMARY KEY (ip, port, protocol), + CONSTRAINT fk_ports_hosts FOREIGN KEY (ip) REFERENCES hosts(ip) ON DELETE CASCADE +); + +CREATE TRIGGER IF NOT EXISTS fki_ports_hosts_ip +BEFORE INSERT ON ports +FOR EACH ROW BEGIN + SELECT CASE + WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) + THEN RAISE(ABORT, 'insert on table "ports" violates foreign key constraint "fk_ports_hosts"') + END; +END; + +CREATE TRIGGER IF NOT EXISTS fku_ports_hosts_ip +BEFORE UPDATE ON ports +FOR EACH ROW BEGIN + SELECT CASE + WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) + THEN RAISE(ABORT, 'update on table "ports" violates foreign key constraint "fk_ports_hosts"') + END; +END; + +CREATE TRIGGER IF NOT EXISTS fkd_ports_hosts_ip +BEFORE DELETE ON hosts +FOR EACH ROW BEGIN + DELETE from ports WHERE ip = OLD.ip; +END;''' + +def dict_factory(cursor, row): + d = {} + for idx,col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + def main(argv, environ): - global vflag - nodb_flag = false - freq_flag = false - db_path = DEFAULT_DATABASE - sql_file = "" - argc = len(argv) - - if argc == 1: - usage(argv[0]) - sys.exit(0) - - try: - alist, args = getopt.getopt(argv[1:], "hvd:c:f:nV", - ["help", "verbose", "database=", "create=", "frequency=", - "nodb", "version"]) - except getopt.GetoptError, msg: - print "%s: %s\n" % (argv[0], msg) - usage(argv[0]); - sys.exit(1) - - for(field, val) in alist: - if field in ("-h", "--help"): - usage(argv[0]) - sys.exit(0) - if field in ("-v", "--verbose"): - vflag = true - if field in ("-d", "--database"): - db_path = val - if field in ("-c", "--create"): - sql_file = val - if field in ("-f", "--frequency"): - freq_flag = true - db_path = val - if field in ("-n", "--nodb"): - nodb_flag = true - if field in ("-V", "--version"): - print "nmapdb v%s by Patroklos Argyroudis " % (VERSION) - print "parse nmap's XML output files and insert them into an SQLite database" - sys.exit(0) - - if freq_flag == false: - if len(args[0]) == 0: - usage(argv[0]) + global verbose + #start argument parser + parser = argparse.ArgumentParser(description='Nmap XML file to SQLite database. Default file nmap.db will be used if none is supplied.') + parser.add_argument('--debug',help='Print verbose information',default=False,dest='debug',action='store_true') + parser.add_argument('--force-update',help='Overwrite previous information.',default=False,dest='force',action='store_true') + parser.add_argument('-d','--database',help='Filename to use for database. If file doesn\'t exist it will be created. Default is \'nmap.db.\'',dest='scandb', default='nmap.db') + parser.add_argument('nmap_xml',help='Nmap XML file you wish to parse') + args = parser.parse_args() + + if args.debug: + verbose = True + debug('Start ','Debug Enabled') + + mesg('Start ','Using DB file: %s and XML file %s' % (args.scandb,args.nmap_xml)) + conn = sqlite.connect(args.scandb) + cursor = conn.cursor() + debug("Start ","Successfully connected to SQLite DB \"%s\"" % (args.scandb)) + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + if (u'hosts',) not in cursor.fetchall(): + debug('Start ', 'Database does not exist. Creating...Done') + try: + cursor.executescript(sql_struct()) + except sqlite.ProgrammingError, msg: + err("Start ","%s: error: %s\n" % (argv[0], msg)) sys.exit(1) + else: + debug('Start ', 'Database already exists. Continuing.') - if nodb_flag == false: - if db_path == DEFAULT_DATABASE: - print "%s: no output SQLite DB file specified, using \"%s\"\n" % (argv[0], db_path) - - conn = sqlite.connect(db_path) - cursor = conn.cursor() - - myprint("%s: successfully connected to SQLite DB \"%s\"\n" % (argv[0], db_path)) - - # helpful queries on the database - if freq_flag == true: - freq_sql = "select count(port) as frequency,port as fport from ports where ports.state='open' group by port having count(fport) > 1000" - - cursor.execute(freq_sql) - print "Frequency|Port" + try: + nmap_report = NmapParser.parse_fromfile(args.nmap_xml) + debug("Parser","Nmap Results: {0}".format(nmap_report.summary)) + except IOError: + err("Parser"," %s: error: file \"%s\" doesn't exist" % (argv[0], args.nmap_xml)) + sys.exit(-1) + except: + err("Parser"," %s: error: file \"%s\" Issue parsing Nmap XML" % (argv[0], args.nmap_xml)) + sys.exit(-1) + + for host in nmap_report.hosts: + ip = host.address + mac = host.mac + if not host.ipv6: + protocol = 'ipv4' + else: + protocol = 'ipv6' + + if len(host.hostnames) > 0: #I know this isn't great but its fine for now + hostname = host.hostnames[0] + else: + hostname = '' + if host.os_fingerprinted: + os_name = host.os.osmatches[0].name + os_family = host.os.osmatches[0].osclasses[0].osfamily + os_accuracy = host.os.osmatches[0].accuracy + os_gen = host.os.osmatches[0].osclasses[0].osgen + else: + os_name = '' + os_family = '' + os_accuracy = '' + os_gen = '' + + timestamp = host.endtime + state = host.status + mac_vendor = host.vendor + + # Some script store results in host scripts, we'll store them in one long string + host_script = "" + for script_out in host.scripts_results: + debug("[host]","Found script output for %s" % script_out['id']) + host_script += script_out['id']+":\n"+script_out['output']+'\n' + if len(script_out['elements']) > 0: host_script += ' Elements:\n' + for elem in script_out['elements']: + if type(script_out['elements'][elem]) is not dict: + host_script += ' '+str(elem)+": "+ str(script_out['elements'][elem])+'\n' + else: + host_script += ' '+elem+':\n' + for item in script_out['elements'][elem]: + if item is not None and script_out['elements'][elem][item] is not None: + host_script += ' '+item+': '+script_out['elements'][elem][item].strip()+'\n' + host_script += '\n' - for row in cursor: - print(row) - - sys.exit(0) + info_str = '' + for line in host_script.splitlines(): + info_str += '['+ip+'] '+line+'\n' - if nodb_flag == false: - if sql_file != "": - sql_string = open(sql_file, "r").read() - try: - cursor.executescript(sql_string) - except sqlite.ProgrammingError, msg: - print "%s: error: %s\n" % (argv[0], msg) - sys.exit(1) - - myprint("%s: SQLite DB created using SQL file \"%s\"\n" % (argv[0], sql_file)) - - for fname in args: + debug("[host]","ip: %s" % (ip)) + debug("[host]","mac: %s" % (mac)) + debug("[host]","hostname: %s" % (hostname)) + debug("[host]","protocol: %s" % (protocol)) + debug("[host]","os_name: %s" % (os_name)) + debug("[host]","os_family: %s" % (os_family)) + debug("[host]","os_accuracy: %s" % (os_accuracy)) + debug("[host]","os_gen: %s" % (os_gen)) + debug("[host]","last_update: %s" % (timestamp)) + debug("[host]","state: %s" % (state)) + debug("[host]","mac_vendor: %s" % (mac_vendor)) + debug("[host]","info: %s" % info_str) + debug("[host]","Inserting %s in to database" % ip) try: - doc = xml.dom.minidom.parse(fname) - except IOError: - print "%s: error: file \"%s\" doesn't exist\n" % (argv[0], fname) - continue - except xml.parsers.expat.ExpatError: - print "%s: error: file \"%s\" doesn't seem to be XML\n" % (argv[0], fname) - continue + cursor.execute("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (ip, mac, hostname, protocol, os_name, os_family, os_accuracy, + os_gen, timestamp, state, mac_vendor, info_str)) + debug("[host]","Inserted %s in to database" % ip) - for host in doc.getElementsByTagName("host"): - try: - address = host.getElementsByTagName("address")[0] - ip = address.getAttribute("addr") - protocol = address.getAttribute("addrtype") - except: - # move to the next host since the IP is our primary key - continue - - try: - mac_address = host.getElementsByTagName("address")[1] - mac = mac_address.getAttribute("addr") - mac_vendor = mac_address.getAttribute("vendor") - except: - mac = "" - mac_vendor = "" + except sqlite.IntegrityError, msg: - try: - hname = host.getElementsByTagName("hostname")[0] - hostname = hname.getAttribute("name") - except: - hostname = "" - - try: - status = host.getElementsByTagName("status")[0] - state = status.getAttribute("state") - except: - state = "" - - try: - os_el = host.getElementsByTagName("os")[0] - os_match = os_el.getElementsByTagName("osmatch")[0] - os_name = os_match.getAttribute("name") - os_accuracy = os_match.getAttribute("accuracy") - os_class = os_el.getElementsByTagName("osclass")[0] - os_family = os_class.getAttribute("osfamily") - os_gen = os_class.getAttribute("osgen") - except: - os_name = "" - os_accuracy = "" - os_family = "" - os_gen = "" - - try: - timestamp = host.getAttribute("endtime") - except: - timestamp = "" - - try: - hostscript = host.getElementsByTagName("hostscript")[0] - script = hostscript.getElementsByTagName("script")[0] - id = script.getAttribute("id") - - if id == "whois": - whois_str = script.getAttribute("output") + debug("[host]","Entry alread exists for %s in database attempting update" % ip) + cursor.execute("SELECT * FROM hosts WHERE ip = '%s'" % ip ) + db = dict_factory(cursor, cursor.fetchone()) + + # We automatically append new script output without asking the user + if host_script != "" and db['info'].find(info_str) < 0: + new_info = db['info'] + "\n" + info_str + debug("[host]"," Appending script output") + cursor.execute("UPDATE hosts SET info=? WHERE ip = ?" , (new_info, ip)) + + if args.force: + debug("[host]","--force-update enabled - overwritting entry!") + cursor.execute("UPDATE hosts SET mac=?, hostname=?, protocol=?, os_name=?, os_family=?, os_accuracy=?, os_gen=?, last_update=?, state=?, mac_vendor=?, info=? WHERE ip = ?", (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) + + + # If the host already exsits lets check with the user if we wan't to update + if ( db['mac'] != mac + and db['hostname'] != hostname and db['hostname'] != '' + and db['protocol'] != protocol + and db['os_name'] != os_name and db['os_name'] != '' + and db['os_family'] != os_family and db['os_family'] != '' + and db['os_gen'] != os_gen and db['os_gen'] != '' + and db['state'] != state + and db['mac_vendor']!= mac_vendor + and db['info'] != host_script) and not args.force: + # So we already have an entry. If theres no new information we continue to ports + # If there's a bunch of new entries we'll ask the user what to do + debug("[host]","Could not update automatically - Manual update required") + mesg("[host]","%s entry exists" % ip) + mesg("[host]","Name: 'Old' --> 'New'") + mesg("[host]","mac: '"+db['mac']+"' --> '%s'" % mac) + mesg("[host]","hostname: '"+db['hostname']+"' --> '%s'" % hostname) + mesg("[host]","protocol: '"+db['protocol']+"' --> '%s'" % protocol) + mesg("[host]","os_name: '"+db['os_name']+"' --> '%s'" % os_name) + mesg("[host]","os_family: '"+db['os_family']+"' --> '%s'" % os_family) + mesg("[host]","os_accuracy: '"+str(db['os_accuracy'])+"' --> '%s'" % os_accuracy) + mesg("[host]","os_gen: '"+db['os_gen']+"' --> '%s'" % os_gen) + mesg("[host]","timestamp: '"+str(db['last_update'])+"' --> '%s'" % timestamp) + mesg("[host]","state: '"+db['state']+"' --> '%s'" % state) + mesg("[host]","mac_vendor: '"+db['mac_vendor']+" --> '%s'" % mac_vendor) + mesg("[host]","info: '"+db['info']+"' --> '%s'" % host_script) + mesg("[host]","Update entry? y/n") + user_input = sys.stdin.readline().strip()[:1] + if user_input == 'y': + debug("[host]","Updating %s entry" % ip) + cursor.execute("UPDATE hosts SET mac=?, hostname=?, protocol=?, os_name=?, os_family=?, os_accuracy=?, os_gen=?, last_update=?, state=?, mac_vendor=?, info=? WHERE ip = ?", (mac,hostname,protocol,os_name,os_family,os_accuracy,os_gen,timestamp,state,mac_vendor,host_script, ip )) else: - whois_str = "" + debug("[hosts]","Skipping %s entry" % ip) + except: + print "%s: unknown exception during insert into table hosts\n" % (argv[0]) + continue - except: - whois_str = "" - - myprint("================================================================") - - myprint("[hosts] ip:\t\t%s" % (ip)) - myprint("[hosts] mac:\t\t%s" % (mac)) - myprint("[hosts] hostname:\t%s" % (hostname)) - myprint("[hosts] protocol:\t%s" % (protocol)) - myprint("[hosts] os_name:\t%s" % (os_name)) - myprint("[hosts] os_family:\t%s" % (os_family)) - myprint("[hosts] os_accuracy:\t%s" % (os_accuracy)) - myprint("[hosts] os_gen:\t\t%s" % (os_gen)) - myprint("[hosts] last_update:\t%s" % (timestamp)) - myprint("[hosts] state:\t\t%s" % (state)) - myprint("[hosts] mac_vendor:\t%s" % (mac_vendor)) - myprint("[hosts] whois:\n") - myprint("%s\n" % (whois_str)) - - if nodb_flag == false: - try: - cursor.execute("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (ip, mac, hostname, protocol, os_name, os_family, os_accuracy, - os_gen, timestamp, state, mac_vendor, whois_str)) - except sqlite.IntegrityError, msg: - print "%s: warning: %s: table hosts: ip: %s\n" % (argv[0], msg, ip) - continue - except: - print "%s: unknown exception during insert into table hosts\n" % (argv[0]) - continue + ports = host.get_open_ports() + + for port in ports: + svc = host.get_service(port[0]) + service_str = svc.banner + port_name = svc.service + pn = str(svc.port) + protocol = svc.protocol + state = svc.state + + # Service Scripts - we'll store them in one long string + svc_script = "" + for script_out in svc.scripts_results: + debug("[port]","Found script output for %s" % script_out['id']) + svc_script += script_out['id']+":\n"+script_out['output']+'\n' + if len(script_out['elements']) > 0: svc_script += ' Elements:\n' + for elem in script_out['elements']: + if not elem: + continue + if (type(script_out['elements'][elem]) is not dict + and type(script_out['elements'][elem]) is not list): + svc_script += ' '+str(elem)+": "+ script_out['elements'][elem]+'\n' + elif type(script_out['elements'][elem]) is dict: + svc_script += ' '+elem+':\n' + for item in script_out['elements'][elem]: + if item is not None and script_out['elements'][elem][item] is not None: + svc_script += ' '+item+': '+script_out['elements'][elem][item].strip()+'\n' + + svc_script += '\n' + #print svc_script + info_str = '' + for line in svc_script.splitlines(): + info_str += "[%s:%s] %s\n" % (ip, pn, line) + + debug("[port]","---------------------- Start "+pn+" ----------------------") + debug("[port]","ip: %s" % (ip)) + debug("[port]","port: %s" % (pn)) + debug("[port]","protocol: %s" % (protocol)) + debug("[port]","name: %s" % (port_name)) + debug("[port]","state: %s" % (state)) + debug("[port]","service: %s" % (service_str)) + debug("[port]","info: %s" % (info_str)) + debug("[host]","Attempting to insert %s:%s in to database" % (ip,pn)) try: - ports = host.getElementsByTagName("ports")[0] - ports = ports.getElementsByTagName("port") + cursor.execute("INSERT INTO ports VALUES (?, ?, ?, ?, ?, ?, ?)", (ip, pn, protocol, port_name, state, service_str, info_str)) + except sqlite.IntegrityError, msg: + debug("[port]","Entry alread exists for %s:%s in database attempting update" % (ip,pn)) + + cursor.execute("SELECT * FROM ports WHERE ip = '%s' AND port = '%s' and protocol ='%s'" % (ip, pn, protocol) ) + db = dict_factory(cursor, cursor.fetchone()) + + # We automatically append new script output without asking the user + if info_str != "" and db['info'].find(info_str) < 0: + new_info = db['info'] + "\n" + info_str + debug("[port]","Appending script output %s" % info_str) + cursor.execute("UPDATE ports SET info=? WHERE ip = ? AND port = ? and protocol =?" , (new_info, ip, pn, protocol)) + + if args.force: + debug("[port]","--force-update enabled - overwritting entry!") + cursor.execute("UPDATE ports SET name=?, state=?, service=? WHERE ip = ? AND port = ? and protocol = ?", + (port_name, state, service_str, ip, pn, protocol)) + + if (not args.force) and (db['name'] != port_name or db['service'] != service_str or db['state'] != state) and service_str != '': + debug("[port]","Could not update automatically - Manual update required") + err("[!!!!]","------------------- MANUAL UPDATE REQUIRED! -----------------------") + mesg("[port]","%s:%s %s exists" % (ip, pn, protocol)) + mesg("[port]","Name: 'Old' --> 'New'") + mesg("[port]","ip: '"+db['ip']+"' --> '%s'" % ip) + mesg("[port]","port: '"+str(db['port'])+"' --> '%s'" % pn) + mesg("[port]","protocol: '"+db['protocol']+"' --> '%s'" % protocol) + mesg("[port]","name: '"+db['name']+"' --> '%s'" % port_name) + mesg("[port]","state: '"+db['state']+"' --> '%s'" % state) + mesg("[port]","service: '"+db['service']+"' --> '%s'" % service_str) + mesg("[port]","Update entry? y/n") + user_input = sys.stdin.readline().strip()[:1] + if user_input == 'y': + cursor.execute("UPDATE ports SET name=?, state=?, service=? WHERE ip = ? AND port = ? and protocol = ?", + (port_name, state, service_str, ip, pn, protocol)) + else: debug('[port]',' skipping entry') + except: - print "%s: host %s has no open ports\n" % (argv[0], ip) + print "%s: unknown exception during insert into table ports\n" % (argv[0]) continue + debug("[port]","---------------------- End "+pn+" ----------------------") - for port in ports: - pn = port.getAttribute("portid") - protocol = port.getAttribute("protocol") - state_el = port.getElementsByTagName("state")[0] - state = state_el.getAttribute("state") - - try: - service = port.getElementsByTagName("service")[0] - port_name = service.getAttribute("name") - product_descr = service.getAttribute("product") - product_ver = service.getAttribute("version") - product_extra = service.getAttribute("extrainfo") - except: - service = "" - port_name = "" - product_descr = "" - product_ver = "" - product_extra = "" - - service_str = "%s %s %s" % (product_descr, product_ver, product_extra) - - info_str = "" - - for i in (0, 1): - try: - script = port.getElementsByTagName("script")[i] - script_id = script.getAttribute("id") - script_output = script.getAttribute("output") - except: - script_id = "" - script_output = "" - - if script_id != "" and script_output != "": - info_str += "%s: %s\n" % (script_id, script_output) - - myprint("\t------------------------------------------------") - - myprint("\t[ports] ip:\t\t%s" % (ip)) - myprint("\t[ports] port:\t\t%s" % (pn)) - myprint("\t[ports] protocol:\t%s" % (protocol)) - myprint("\t[ports] name:\t\t%s" % (port_name)) - myprint("\t[ports] state:\t\t%s" % (state)) - myprint("\t[ports] service:\t%s" % (service_str)) - - if info_str != "": - myprint("\t[ports] info:\n") - myprint("%s\n" % (info_str)) - - if nodb_flag == false: - try: - cursor.execute("INSERT INTO ports VALUES (?, ?, ?, ?, ?, ?, ?)", (ip, pn, protocol, port_name, state, service_str, info_str)) - except sqlite.IntegrityError, msg: - print "%s: warning: %s: table ports: ip: %s\n" % (argv[0], msg, ip) - continue - except: - print "%s: unknown exception during insert into table ports\n" % (argv[0]) - continue - myprint("\t------------------------------------------------") + debug("[host]", "====================== End "+ip+" ======================") - myprint("================================================================") - if nodb_flag == false: - conn.commit() + conn.commit() if __name__ == "__main__": main(sys.argv, os.environ) sys.exit(0) -# EOF diff --git a/nmapdb.sql b/nmapdb.sql deleted file mode 100644 index 99ae040..0000000 --- a/nmapdb.sql +++ /dev/null @@ -1,57 +0,0 @@ -/* - * nmapdb - Parse nmap's XML output files and insert them into an SQLite database - * Copyright (c) 2012 Patroklos Argyroudis - */ - -CREATE TABLE IF NOT EXISTS hosts ( - ip VARCHAR(16) PRIMARY KEY NOT NULL, - mac VARCHAR(18), - hostname VARCHAR(129), - protocol VARCHAR(5) DEFAULT 'ipv4', - os_name TEXT, - os_family TEXT, - os_accuracy INTEGER, - os_gen TEXT, - last_update TIMESTAMP, - state VARCHAR(8) DEFAULT 'down', - mac_vendor TEXT, - whois TEXT -); - -CREATE TABLE IF NOT EXISTS ports ( - ip VARCHAR(16) NOT NULL, - port INTEGER NOT NULL, - protocol VARCHAR(4) NOT NULL, - name VARCHAR(33), - state VARCHAR(33) DEFAULT 'closed', - service TEXT, - info TEXT, - PRIMARY KEY (ip, port, protocol), - CONSTRAINT fk_ports_hosts FOREIGN KEY (ip) REFERENCES hosts(ip) ON DELETE CASCADE -); - -CREATE TRIGGER IF NOT EXISTS fki_ports_hosts_ip -BEFORE INSERT ON ports -FOR EACH ROW BEGIN - SELECT CASE - WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) - THEN RAISE(ABORT, 'insert on table "ports" violates foreign key constraint "fk_ports_hosts"') - END; -END; - -CREATE TRIGGER IF NOT EXISTS fku_ports_hosts_ip -BEFORE UPDATE ON ports -FOR EACH ROW BEGIN - SELECT CASE - WHEN ((SELECT ip FROM hosts WHERE ip = NEW.ip) IS NULL) - THEN RAISE(ABORT, 'update on table "ports" violates foreign key constraint "fk_ports_hosts"') - END; -END; - -CREATE TRIGGER IF NOT EXISTS fkd_ports_hosts_ip -BEFORE DELETE ON hosts -FOR EACH ROW BEGIN - DELETE from ports WHERE ip = OLD.ip; -END; - -/* EOF */