pydog4apache/pydog4apache/core/main.py

#!/usr/bin/env python 
# -*- coding: utf-8 -*-"
"""
PyDog4Apache - 2015 - by psy (epsylon@riseup.net)

You should have received a copy of the GNU General Public License along
with PyDog4Apache; if not, write to the Free Software Foundation, Inc., 51
Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""
from options import PyDog4ApacheOptions
from update import Updater
import os, traceback, sys, re, gzip, datetime, string, stat

try:
    from ipwhois import IPWhois
except:
    print "\n[Warning] - Error importing: ipwhois lib. \n\n On Debian based systems:\n\n $ sudo apt-get install python-pip && sudo pip install ipwhois\n"
    print "[Source] - Pypi-ipwhois: https://pypi.python.org/pypi/ipwhois/\n"
    sys.exit(2)

DEBUG = 0

class PyDog4Apache(object):
    def __init__(self):
        self.visitants = {} # visitants
        self.visitants_ips = [] # used to not repeat checks

    def set_options(self, options):
        self.options = options

    def create_options(self, args=None):
        self.optionParser = PyDog4ApacheOptions()
        self.options = self.optionParser.get_options(args)
        if not self.options:
            return False
        return self.options

    def banner(self):
        print '='*75, "\n"
        print " ____        ____              _  _     _                     _          "
        print "|  _ \ _   _|  _ \  ___   __ _| || |   / \   _ __   __ _  ___| |__   ___ "
        print "| |_) | | | | | | |/ _ \ / _` | || |_ / _ \ | '_ \ / _` |/ __| '_ \ / _ |"
        print "|  __/| |_| | |_| | (_) | (_| |__   _/ ___ \| |_) | (_| | (__| | | |  __/"
        print "|_|    \__, |____/ \___/ \__, |  |_|/_/   \_\ .__/ \__,_|\___|_| |_|\___|"
        print "       |___/             |___/              |_|                          "
        print self.optionParser.description, "\n"
        print '='*75

    def try_running(self, func, error, args=None):
        options = self.options
        args = args or []
        try:
            return func(*args)
        except Exception as e:
            print(error, "error")
            if DEBUG:
                traceback.print_exc()

    def check_root(self): # check root permissions
        if not os.geteuid()==0:
            print "[Error] - Some of your 'sources' need more rights to be accessed."
            sys.exit("\n[Info] - Try to launch it as root (ex: 'sudo ./pydog4apache')\n")

    def is_readable(self, folder): # check if logs are readable without root permissions
        try:
            st = os.stat(folder)
            root = st.st_uid
        except:
            root = None 
        return root

    def check_access(self, logs): # check if logs required root
        for folder in logs:
            root = self.is_readable(folder)
            if root == None: # wrong folder
                print "[Error] - This source:", folder, "is not valid!. Passing..."
            if root is 0: # root needed
                check_perms = self.try_running(self.check_root, "\nInternal error checking root permissions.")

    def extract_logs(self): # extract logs from folder (ex: 'sources.txt')
        try:
            f = open('sources.txt')
            logs = f.readlines()
            logs = [ log.replace('\n','') for log in logs ]
            f.close()
            if not logs:
                print "\n[Error] - Imposible to retrieve 'sources' from file.\n"
                return
            else:
                return logs
        except:
            if os.path.exists('sources.txt') == True:
                print '\n[Error] - Cannot open:', 'sources.txt', "\n"
                sys.exit(2)
            else:
                print '\n[Error] - Cannot found:', 'sources.txt', "\n"
                sys.exit(2)

    def extract_whois(self, ip): # extract whois description
        try:
            if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip): # only IPs
                w = IPWhois(ip, timeout=5) # timeout 5
                res = w.lookup_whois(retry_count=2) # legacy whois / retries 2
                descr = res["nets"][0]['description']
                if self.options.verbose:
                    print"[Verbose] - Resolving:", ip
            else:
                descr = None
        except:
            descr = None
        return descr

    def is_valid_ip(self, ip): # extract visitors IP / perform whois
        if not re.match(r'^127\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip) or re.match(r'^10\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip) or re.match(r'^192.168\.\d{1,3}\.\d{1,3}', ip) or re.match(r'^172.(1[6-9]|2[0-9]|3[0-1]).[0-9]{1,3}.[0-9]{1,3}', ip) or ip.startswith('localhost'): # non LAN IP
            if ip not in self.visitants_ips: # create a list with non repeated IPs
                if ip[0].isdigit(): # parse if IP starts with a number
                    self.visitants_ips.append(ip)
                    ip_found = True
                    return ip_found

    def check_visitants(self, descr): # check visitors list
        try:
            for v in self.keys:
                if v.lower() in descr.lower(): # visitant found!
                    key = v.lower()
                    return key
                else:
                    pass
        except:
            if os.path.exists('keywords.txt') == True:
                return
            else:
                print '\n[Error] - Cannot found:', 'keywords.txt', "\n"
                return

    def run(self, opts=None):
        if opts:
            options = self.create_options(opts)
            self.set_options(options)
        options = self.options
        self.banner()
        if options.update:
            try:
                print("\n[Info] - Trying to update automatically to the latest stable version.\n")
                Updater()
            except:
                print("\nSomething was wrong!. You should clone PyDog2Apache manually with:\n")
                print("$ git clone https://github.com/epsylon/pydog2apache\n")
            sys.exit(2)
        print "\n[Info] - Sending dogs to sniff logs... Please wait!\n"
        print "-"*22
        logs = self.try_running(self.extract_logs, "\nInternal error extracting logs.")
        access = self.check_access(logs)
        f = open('keywords.txt')
        self.keys = f.readlines()
        self.keys = [ self.key.replace('\n','') for self.key in self.keys ]
        f.close()
        if not self.keys:
            print "\n[Error] - Imposible to retrieve 'visitants' from file.\n"
            return
        for folder in logs:
            if not folder.endswith('/'):
                folder = folder + "/"
            try:
                listing = os.listdir(folder)
                for log in listing:
                    if self.options.verbose:
                        print "[Verbose] - Analyzing:", folder+log
                    if log.endswith('.gz'): # also read on compressed logs
                        with gzip.open(folder+log, 'rb') as f:
                            dog_sniff = f.readlines()
                    else:
                        try:
                            f = open(folder+log)
                        except:
                            return
                        dog_sniff = f.readlines()
                    sep = '-'
                    sep2= '['
                    for record in dog_sniff:
                        ip = record.split(sep, 1)[0]
                        ip = ''.join(ip.split())
                        ip_found = self.is_valid_ip(ip)
                        if ip_found is True: # extract info
                            date = [record.split(']')[0] for p in record.split('[') if ']' in p]
                            for d in date:
                                date_visit = d.split(sep2, 1)[1]
                            descr = self.extract_whois(ip)
                            if descr is not None:
                                key = self.check_visitants(descr)
                                if key:
                                    self.visitants[str(ip)] = "[" + str(key.upper()) + "]" + "|" + str(str(descr) + "|" + str(date_visit) + "|" + str(folder+log))
                    f.close()
            except:
                pass
        if self.options.verbose:
            print "-"*22
        if self.options.file: # export results to file
            namefile = str(self.options.file)
            self.report = open(namefile, 'w')
            self.report.write("# Apache web logs sneaker - GPLv3 - by psy\n")
            self.report.write("# Project: https://github.com/epsylon/pydog4apache - 03c8.net\n")
            self.report.write("# Reported at: " + str(datetime.datetime.now()) + "\n\n")
        for key,val in self.visitants.items():
            print("{} -> {}".format(key, val))
            print "-"*12
            if self.options.file: 
                self.report.write(key + " -> " + val + "\n")
                self.report.write("-"*12 + "\n")
        if not self.visitants.items():
            print "[Info] - Not any 'keyword' found on your 'visitants'.\n"
            if self.options.file:
                self.report.write("Not any 'keyword' found on your 'visitants'.\n")
        if self.options.file: # close file containter
            self.report.close()
        if self.options.emails: # send results via email
            import smtplib, socket
            from email.mime.text import MIMEText
            self.notify = open('tempmail', 'w')
            self.notify.write("# Apache web logs sneaker - GPLv3 - by psy\n")
            self.notify.write("# Project: https://github.com/epsylon/pydog4apache - 03c8.net\n")
            self.notify.write("# Reported at: " + str(datetime.datetime.now()) + "\n\n")
            for key,val in self.visitants.items():
                self.notify.write(key + " -> " + val + "\n")
                self.notify.write("-"*12 + "\n")
            if not self.visitants.items():
                self.notify.write("Not any 'keyword' found on your 'visitants'.\n")
            self.notify.close()
            self.notify = open('tempmail', 'rb')
            msg = MIMEText(self.notify.read())
            self.notify.close()
            doggy = 'pydog4apache@'+str(socket.gethostname())
            herd = self.options.emails
            herd = herd.split(',')
            msg['Subject'] = '[pydog4apache] You have interesting visitants...'
            msg['From'] = doggy
            msg['To'] = ", ".join(herd)
            msg.preamble = 'You have a report with visitants...'
            try:
                s = smtplib.SMTP('localhost')
                s.sendmail(doggy, herd, msg.as_string())
                s.quit()
            except:
                print "[Error] - Not any SMTP server running on: '"+str(socket.gethostname())+"'. Aborting email report...\n"
            os.remove('tempmail') # remove temporal email container

if __name__ == "__main__":
    app = PyDog4Apache()
    options = app.create_options()
    if options:
        app.set_options(options)
        app.run()