增加日志回扫功能

Server/log.py

@@ -1,5 +1,6 @@
 import json
 import time
+import operator
 
 import process
 import rule
@@ -193,3 +194,101 @@ def process_log(host, json_log, raw_log):
         if item.risk_score >= config.MAX_THREAT_SCORE:
             item.print_process()
     """
+
+
+def process_raw_log(raw_logs: list) -> list:
+    return_data = []
+    process_chain_list = []
+
+    raw_logs.sort(key=operator.attrgetter("timestamp"))
+
+    def _get_process_chain(pid, host: str) -> process.ProcessChain:
+        for iter in process_chain_list:
+            chain_item: process.ProcessChain = iter
+            if chain_item.host != host:
+                continue
+            process_item = chain_item.find_process_by_pid(pid)
+            if process_item is not None:
+                return chain_item
+        return None
+
+    for log in raw_logs:
+        log: sql.raw_process_log = log
+        pid = log.pid
+        ppid = log.ppid
+        path = log.path
+        params = log.commandline
+        user = log.user
+        hash = log.hash
+        create_time = log.timestamp
+        host = log.host
+        current_process:process.Process = None
+        if path in process.skip_process_path :
+            continue
+        if log.action.lower() == "processcreate":
+
+            chain = _get_process_chain(pid, host)
+            if chain is not None:
+                parent_process = chain.find_process_by_pid(ppid)
+            else:
+                parent_process = None
+
+            if chain is None:
+                # build a process chain
+                current_process = process.Process(
+                    pid, ppid, path, params, create_time, hash, user, host
+                )
+                chain = process.create_chain(current_process)
+                process_chain_list.append(chain)
+            else:
+                current_process = process.Process(
+                    pid, ppid, path, params, create_time, hash, user, host
+                )
+                chain.add_process(current_process, ppid)
+        elif log.action.lower() == "processterminal":
+            chain = _get_process_chain(pid, host)
+            if chain is not None:
+                current_process = chain.find_process_by_pid(pid)
+                current_process.active = False
+                current_process.chain.terminate_count += 1
+                if (
+                    current_process.chain.terminate_count
+                    >= current_process.chain.active_count
+                ):
+                    current_process.chain.active = False
+            else:
+                # 不在指定时段内被创建的进程的结束事件
+                continue
+        else:
+            chain = _get_process_chain(pid, host)
+            if chain is None:
+                continue
+            current_process = chain.find_process_by_pid(pid)
+            if current_process is None:
+                continue
+
+        # if current_process is None :
+        #     breakpoint()
+        start_process = current_process.chain.root_process
+        start_process_info = {
+            "path": start_process.path,
+            "hash": start_process.md5,
+            "params": start_process.params,
+            "user": start_process.user,
+            "create_time": start_process.time,
+        }
+        return_data.append(
+            {
+                "host": current_process.host,
+                "chain_hash": current_process.chain.hash,
+                "hit_rule": log.hit,
+                "time": log.timestamp,
+                "type": log.type,
+                "risk_score": log.score,
+                "id": log.id,
+                "is_end": current_process.chain.active == False,
+                "start_process": start_process_info,
+            }
+        )
+
+    return return_data

Server/sql.py

@@ -10,6 +10,7 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import delete
+import sqlalchemy
 import json
 
 g_engine = None
@@ -166,17 +167,19 @@ def push_process_raw(
     return result
 
 
-def select_create_process_raw_log_by_time(start, end):
+def select_process_raw_log_by_time(start: int, end: int):
     global g_rawdata_table
     sql_session = sessionmaker(bind=g_engine)
+    # 用g_rawdata_table 不行, utf8编码问题？
     raw_log = (
         sql_session()
-        .query(g_rawdata_table)
+        .query(raw_process_log)
         .filter(
-            raw_process_log.timestamp >= start,
-            raw_process_log.timestamp < end,
-            raw_process_log.action == "processcreate",
+            sqlalchemy.and_(
+                raw_process_log.timestamp >= start, raw_process_log.timestamp < end
+            )
         )
+        .all()
     )
 
     sql_session().close()

Server/webserver.py

@@ -8,144 +8,157 @@
 from flask import Flask, render_template, request
 import plugin
 import logging
-app = Flask(__name__,
-            template_folder="./templates",
-            static_folder="./templates",
-            static_url_path="")
-app.jinja_env.variable_start_string = '{.<'
-app.jinja_env.variable_end_string = '>.}'
 
+app = Flask(
+    __name__,
+    template_folder="./templates",
+    static_folder="./templates",
+    static_url_path="",
+)
+app.jinja_env.variable_start_string = "{.<"
+app.jinja_env.variable_end_string = ">.}"
 
-@app.route('/')
+
+@app.route("/")
 def root():
     if request.remote_addr not in config.ALLOW_ACCESS_IP:
         return "Access Denied"
     return render_template("index.html")
 
 
-@app.route('/static/<path:path>')
+@app.route("/static/<path:path>")
 def on_vue_static(path):
     if request.remote_addr not in config.ALLOW_ACCESS_IP:
         return "Access Denied"
     return app.send_static_file("./" + path)
 
 
-@app.route('/plugin/<path:path>')
+@app.route("/plugin/<path:path>")
 def on_plugin_access(path):
     if request.remote_addr not in config.ALLOW_ACCESS_IP:
         return "Access Denied"
     return plugin.dispath_html_draw(path)
 
 
-@app.route('/api/v1/get/plugin_menu')
+@app.route("/api/v1/get/plugin_menu")
 def plugin_menu():
     if request.remote_addr not in config.ALLOW_ACCESS_IP:
         return "Access Denied"
-    return {'data': {'menu': plugin.dispath_html_menu()}}
+    return {"data": {"menu": plugin.dispath_html_menu()}}
 
 
-@app.route('/api/v1/get/threat_statistics', methods=['GET'])
+@app.route("/api/v1/get/threat_statistics", methods=["GET"])
 def threat_statistics():
     if request.remote_addr not in config.ALLOW_ACCESS_IP:
         return "Access Denied"
     # sqlite的count啥的还不如自己查出来自己统计
     threat_datas = sql.query_all_threat_log(-1)
-    return_data = {
-        'all': len(threat_datas),
-        'confirm': 0,
-        'ingore': 0,
-        'working': 0
-    }
+    return_data = {"all": len(threat_datas), "confirm": 0, "ingore": 0, "working": 0}
     for iter in threat_datas:
         if iter[9] == 1:
-            return_data['confirm'] += 1
+            return_data["confirm"] += 1
         elif iter[9] == 2:
-            return_data['ingore'] += 1
+            return_data["ingore"] += 1
         if iter[7] == 0:
-            return_data['working'] += 1
-    return {'data': return_data}
+            return_data["working"] += 1
+    return {"data": return_data}
 
 
-@app.route('/api/v1/get/process_chain/handle', methods=['GET'])
+@app.route("/api/v1/get/process_chain/handle", methods=["GET"])
 def handle_chain_data():
-    id = request.args.get('id')
-    handletype = request.args.get('handletype')
-    if request.remote_addr not in config.ALLOW_ACCESS_IP or (id is None or handletype is None):
+    id = request.args.get("id")
+    handletype = request.args.get("handletype")
+    if request.remote_addr not in config.ALLOW_ACCESS_IP or (
+        id is None or handletype is None
+    ):
         return "Access Denied"
     sql.handle_threat_log(id, handletype)
-    return {'data': {'success': 1}}
+    return {"data": {"success": 1}}
 
 
-@app.route('/api/v1/get/process_chain/delete', methods=['GET'])
+@app.route("/api/v1/get/process_chain/delete", methods=["GET"])
 def delete_chain_data():
-    id = request.args.get('id')
+    id = request.args.get("id")
     if request.remote_addr not in config.ALLOW_ACCESS_IP or id is None:
         return "Access Denied"
     sql.delete_threat(id)
-    return {'data': {'success': 1}}
+    return {"data": {"success": 1}}
 
 
-@app.route('/api/v1/get/process_chain/pull', methods=['GET'])
+@app.route("/api/v1/get/process_chain/pull", methods=["GET"])
 def pull_chain_data():
     if request.remote_addr not in config.ALLOW_ACCESS_IP:
         return "Access Denied"
-    id = request.args.get('id')
+    id = request.args.get("id")
     return_data = {}
     if id is not None:
         threat_data = sql.query_one_threat(id)
         return_data = {
-            'host': threat_data[1],
-            'chain_hash': threat_data[2],
-            'type': threat_data[3],
-            'risk_score': threat_data[4],
-            'hit_rule': json.loads(threat_data[5]),
-            'chain': json.loads(threat_data[6]),
-            'is_end': threat_data[7]
+            "host": threat_data[1],
+            "chain_hash": threat_data[2],
+            "type": threat_data[3],
+            "risk_score": threat_data[4],
+            "hit_rule": json.loads(threat_data[5]),
+            "chain": json.loads(threat_data[6]),
+            "is_end": threat_data[7],
         }
-    return {'data': return_data}
+    return {"data": return_data}
 
 
-@app.route('/api/v1/get/process_chain/all')
+@app.route("/api/v1/get/process_chain/all")
 def process_chain():
     # -1全部 0未处理的 1处理的 2忽略的
-    query_type = request.args.get('query_type')
+    query_type = request.args.get("query_type")
     if request.remote_addr not in config.ALLOW_ACCESS_IP or query_type is None:
         return "Access Denied"
     threat_datas = sql.query_all_threat_log(query_type)
     return_data = []
     for iter in threat_datas:
-        return_data.append({
-            'host': iter[0],
-            'chain_hash': iter[1],
-            'hit_rule': json.loads(iter[2]),
-            'time': iter[3],
-            'type': iter[4],
-            'risk_score': iter[5],
-            'id': iter[6],
-            'is_end': iter[7],
-            'start_process': json.loads(iter[8]),
-        })
-    return {'data': return_data}
-
-
-@app.route('/api/v1/process', methods=['POST'])
+        return_data.append(
+            {
+                "host": iter[0],
+                "chain_hash": iter[1],
+                "hit_rule": json.loads(iter[2]),
+                "time": iter[3],
+                "type": iter[4],
+                "risk_score": iter[5],
+                "id": iter[6],
+                "is_end": iter[7],
+                "start_process": json.loads(iter[8]),
+            }
+        )
+    return {"data": return_data}
+
+
+@app.route("/api/v1/process", methods=["POST"])
 def process():
-    if request.method == 'POST':
+    if request.method == "POST":
         # print(request.data)
         body_data = request.data.decode()
         # 转小写
         host = request.remote_addr
         log.process_log(host, json.loads(body_data.lower()), body_data)
 
-    return {'status': 'success'}
+    return {"status": "success"}
+
+
+@app.route("/api/v1/log_hunt", methods=["POST"])
+def log_rescan():
+    if request.remote_addr not in config.ALLOW_ACCESS_IP:
+        return "Access Denied"
+    start_time = request.args.get("start_time")
+    end_time = request.args.get("end_time")
+    raw_logs = sql.select_process_raw_log_by_time(int(start_time), int(end_time))
+    threat_data = log.process_raw_log(raw_logs)
+    return {"data": threat_data}
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     plugin.reload_plugs()
     sql.init()
     rule.init_rule()
 
     # 如果你觉得日志太多了,去掉这个注释...
-    flask_log = logging.getLogger('werkzeug')
+    flask_log = logging.getLogger("werkzeug")
     flask_log.setLevel(logging.ERROR)
     app.run(debug=True, host="0.0.0.0")