python实时分析用户登录

通过nginx日志

#!/usr/local/bin/python3
# coding:utf-8

# ====================================================
# Author: chang - EMail:changbo@hmg100.com
# Last modified: 2017-5-8
# Filename: nginxanalysis.py
# Description: real time analysis nginx log,base time, os, thread
# blog:http://www.cnblogs.com/changbo
# ====================================================

"""
需求：每隔1分钟读取nginx日志文件
notice:
模拟日志切割过程中初始化脚本参数
cp access.log access2017xxxx.log  && echo > access.log && echo '0'> offset.txt
"""


import time
import os
import re
import pymysql
from threading import Thread
# from django.db import connection

keypage = ['/sys/get_user.do', '/hmcsapiV1.0.7/user/login']
engdate = {'Jan': '1', 'Feb': '2', 'Mar': '3', 'Apr': '4', 'May': '5', 'Jun': '6', 'Jul': '7', 'Aug': '8', 'Sept': '9',
           'Oct': '10', 'Nov': '11', 'Dec': '12'}


def dateformat(nginxdate):
    day = (nginxdate.split('[')[1]).split('/')[0]
    month = engdate[(nginxdate.split('[')[1]).split('/')[1]]
    year = (nginxdate.split('[')[1]).split('/')[2]

    return year + '-' + month + '-' + day


# write log offset
def writeoffset(number):
    with open('offset.txt', 'w+') as f3:
        f3.write(number)
        f3.flush()


# get log offset
def getoffset():
    with open('offset.txt') as f2:
        offset = f2.readline()
        return offset

db = pymysql.connect("xxxxxx", "xxxx", "xxxx", "xxxx")
cursor = db.cursor()
# cleantable = 'TRUNCATE abnormal'

listtime = []
listuser = []


def analysisdb():
    while True:
        time.sleep(60)
        try:
            sql3 = 'SELECT user,time FROM userlogin'
            cursor.execute(sql3)
            results = cursor.fetchall()
            for row in results:
                listuser.append(row[0])
                listtime.append(row[1])

            # 统计1分钟内用户登录次数
            sql1 = "SELECT count(*) from userlogin where time='%s' and user='%s'" % (listtime[0], listuser[0])
            # 如果不满足条件则删除该条记录
            sql2 = "DELETE from userlogin where time='%s' and user='%s'" % (listtime[0], listuser[0])

            print(listtime[0], listuser[0])
            cursor.execute(sql3)
            cursor.execute(sql1)
            datad = cursor.fetchone()
            print(datad)
            if datad[0] < 3:
                cursor.execute(sql2)
                db.commit()
                print('-----delete success -------')
            del listtime[0]
            del listuser[0]
        except Exception as e:
            time.sleep(60)
            print(e)



def handleline(logline):
    susptmp = logline.split(" ")
    if len(susptmp) > 2:
        if susptmp[6] == keypage[0]:
            del susptmp[1:3]
            del susptmp[7:]
            del susptmp[2:6]
            if len(susptmp) > 2:
                ip = susptmp[0]
                time1 = ((susptmp[1].split(':', 1))[1])[0:5]
                date = dateformat((susptmp[1].split(':', 1))[0])
                _, _, user, _, passd, _ = re.split(r'[&="]', susptmp[2])

                # print(ip + '---', time1 + '---', date + '---', user + '---', passd + '---')
                sql = "INSERT INTO userlogin(ip, time, user, passd, date) VALUES('%s', '%s', '%s', '%s', '%s')" % (ip, time1, user, '*****', date)
                try:
                    cursor.execute(sql)
                    db.commit()
                    print('Insert success!')
                except Exception as e:
                    print(e)


# online analysis log
def analysislog():
    with open('access.log') as f1:
        while True:
            # get offset
            lastoffset = getoffset()
            # jump the Specify log line
            f1.seek(int(lastoffset))
            # 获取该行偏移量
            where = f1.tell()
            line = f1.readline()
            writeoffset(str(where))
            if not line:
                time.sleep(10)
                f1.seek(where)
            else:
                # 处理该行，并获取改行的偏移量且写入文件
                handleline(line)
                nowoffset = f1.tell()
                writeoffset(str(nowoffset))

if __name__ == '__main__':
    if not os.path.exists('offset.txt'):
        with open("offset.txt", 'w') as f:
            f.write('0')

    t1 = Thread(target=analysislog)
    t2 = Thread(target=analysisdb)
    t1.start()
    t2.start()

END!