mtools——mongodb日志分析工具
项目地址
为什么要用它？
今天应急过程中，客户mongodb数据库被删了，原因是mongo未授权，需要溯源攻击者IP，数据库是什么时候被删的，以什么权限被删的，哪个用户
部署
可以通过pip3进行安装，当然这也是最简单的办法
sudo pip3 install mtools
sudo pip3 install mtools[all]
/* 需要python3.6或更高版本 */
通过git部署
git clone https://github.com/rueckstiess/mtools
cd mtools
sudo pip3 install -r requirements.txt
sudo python setup.py install
过程中我用到的命令：
mloginfo logs --connections //统计ip连接
source: logs
host: *:27017
start: 2018 Jul 31 13:40:30.757
end: 2020 May 31 04:36:13.772
date format: iso8601-local
timezone: UTC +0800
length: 1845952
binary: mongod
version: 4.0.0
storage: wiredTiger
CONNECTIONS
total opened: 55893
total closed: 55404
no unique IPs: 1109
socket exceptions: 52
112.124.*.70 opened: 18537 closed: 18516
127.0.0.1 opened: 5140 closed: 4722
106.13.*.238 opened: 1250 closed: 1250
mlogfilter logs --from May --slow 1000 --operation remove --json
----
logs：日志名称
--from：日期
--slow：慢查询
--operation：操作类型
--json：以json输出
然后你就会看到一堆json
{
"line_str": "2020-05-10T11:39:39.065+0800 I WRITE [conn9] remove bookcircle.msgQueue command: { q: { receiver: \"5cca30682f29113ecdcc9230\", deviceId: \"imei860219041784179\", type: 21 }, limit: 0 } planSummary: IXSCAN { receiver: 1 } keysExamined:546 docsExamined:546 ndeleted:0 numYields:38 locks:{ Global: { acquireCount: { r: 39, w: 39 } }, Database: { acquireCount: { w: 39 } }, Collection: { acquireCount: { w: 39 } } } 1549ms",
"split_tokens": ["2020-05-10T11:39:39.065+0800", "I", "WRITE", "[conn9]", "remove", "bookcircle.msgQueue", "command:", "{", "q:", "{", "receiver:", "\"5cca30682f29113ecdcc9230\",", "deviceId:", "\"imei860219041784179\",", "type:", "21", "},", "limit:", "0", "}", "planSummary:", "IXSCAN", "{", "receiver:", "1", "}", "keysExamined:546", "docsExamined:546", "ndeleted:0", "numYields:38", "locks:{", "Global:", "{", "acquireCount:", "{", "r:", "39,", "w:", "39", "}", "},", "Database:", "{", "acquireCount:", "{", "w:", "39", "}", "},", "Collection:", "{", "acquireCount:", "{", "w:", "39", "}", "}", "}", "1549ms"],
"datetime": "2020-05-10T11:39:39.065000+08:00",
"operation": "remove",
"thread": "conn9",
"namespace": "bookcircle.msgQueue",
"nscanned": 546,
"ndeleted": 0,
"duration": 1549,
"numYields": 38
} {
"line_str": "2020-05-10T14:23:33.371+0800 I WRITE [conn134] remove bookcircle.msgQueue command: { q: { receiver: \"5cc2bbc02f291109cec898af\", deviceId: \"idc35e1c8f-55ff-4ff3-86f6-2c6e767dc016\", type: 21 }, limit: 0 } planSummary: IXSCAN { receiver: 1 } keysExamined:219 docsExamined:219 ndeleted:0 numYields:26 locks:{ Global: { acquireCount: { r: 27, w: 27 } }, Database: { acquireCount: { w: 27 } }, Collection: { acquireCount: { w: 27 } } } 1126ms",
"split_tokens": ["2020-05-10T14:23:33.371+0800", "I", "WRITE", "[conn134]", "remove", "bookcircle.msgQueue", "command:", "{", "q:", "{", "receiver:", "\"5cc2bbc02f291109cec898af\",", "deviceId:", "\"idc35e1c8f-55ff-4ff3-86f6-2c6e767dc016\",", "type:", "21", "},", "limit:", "0", "}", "planSummary:", "IXSCAN", "{", "receiver:", "1", "}", "keysExamined:219", "docsExamined:219", "ndeleted:0", "numYields:26", "locks:{", "Global:", "{", "acquireCount:", "{", "r:", "27,", "w:", "27", "}", "},", "Database:", "{", "acquireCount:", "{", "w:", "27", "}", "},", "Collection:", "{", "acquireCount:", "{", "w:", "27", "}", "}", "}", "1126ms"],
"datetime": "2020-05-10T14:23:33.371000+08:00",
"operation": "remove",
"thread": "conn134",
"namespace": "bookcircle.msgQueue",
"nscanned": 219,
"ndeleted": 0,
"duration": 1126,
"numYields": 26
} {
"line_str": "2020-05-24T10:18:24.305+0800 I WRITE [conn133] remove bookcircle.msgQueue command: { q: { receiver: \"5cca30682f29113ecdcc9230\", deviceId: \"imei860219041784179\", type: 21 }, limit: 0 } planSummary: IXSCAN { receiver: 1 } keysExamined:554 docsExamined:554 ndeleted:0 numYields:20 locks:{ Global: { acquireCount: { r: 21, w: 21 } }, Database: { acquireCount: { w: 21 } }, Collection: { acquireCount: { w: 21 } } } 1036ms",
"split_tokens": ["2020-05-24T10:18:24.305+0800", "I", "WRITE", "[conn133]", "remove", "bookcircle.msgQueue", "command:", "{", "q:", "{", "receiver:", "\"5cca30682f29113ecdcc9230\",", "deviceId:", "\"imei860219041784179\",", "type:", "21", "},", "limit:", "0", "}", "planSummary:", "IXSCAN", "{", "receiver:", "1", "}", "keysExamined:554", "docsExamined:554", "ndeleted:0", "numYields:20", "locks:{", "Global:", "{", "acquireCount:", "{", "r:", "21,", "w:", "21", "}", "},", "Database:", "{", "acquireCount:", "{", "w:", "21", "}", "},", "Collection:", "{", "acquireCount:", "{", "w:", "21", "}", "}", "}", "1036ms"],
"datetime": "2020-05-24T10:18:24.305000+08:00",
"operation": "remove",
"thread": "conn133",
"namespace": "bookcircle.msgQueue",
"nscanned": 554,
"ndeleted": 0,
"duration": 1036,
"numYields": 20
} {
"line_str": "2020-05-27T11:45:22.932+0800 I WRITE [conn133] remove bookcircle.msgQueue command: { q: { receiver: \"5cca30682f29113ecdcc9230\", deviceId: \"imei860219041784179\", type: 21 }, limit: 0 } planSummary: IXSCAN { receiver: 1 } keysExamined:556 docsExamined:556 ndeleted:0 numYields:22 locks:{ Global: { acquireCount: { r: 23, w: 23 } }, Database: { acquireCount: { w: 23 } }, Collection: { acquireCount: { w: 23 } } } 1147ms",
"split_tokens": ["2020-05-27T11:45:22.932+0800", "I", "WRITE", "[conn133]", "remove", "bookcircle.msgQueue", "command:", "{", "q:", "{", "receiver:", "\"5cca30682f29113ecdcc9230\",", "deviceId:", "\"imei860219041784179\",", "type:", "21", "},", "limit:", "0", "}", "planSummary:", "IXSCAN", "{", "receiver:", "1", "}", "keysExamined:556", "docsExamined:556", "ndeleted:0", "numYields:22", "locks:{", "Global:", "{", "acquireCount:", "{", "r:", "23,", "w:", "23", "}", "},", "Database:", "{", "acquireCount:", "{", "w:", "23", "}", "},", "Collection:", "{", "acquireCount:", "{", "w:", "23", "}", "}", "}", "1147ms"],
"datetime": "2020-05-27T11:45:22.932000+08:00",
"operation": "remove",
"thread": "conn133",
"namespace": "bookcircle.msgQueue",
"nscanned": 556,
"ndeleted": 0,
"duration": 1147,
"numYields": 22
} {
"line_str": "2020-05-29T08:56:47.903+0800 I WRITE [conn149] remove bookcircle.msgQueue command: { q: { receiver: \"5c9efb9b2f2911028436af98\", deviceId: \"imei860219041784179\", type: 21 }, limit: 0 } planSummary: IXSCAN { receiver: 1 } keysExamined:646 docsExamined:646 ndeleted:0 numYields:25 locks:{ Global: { acquireCount: { r: 26, w: 26 } }, Database: { acquireCount: { w: 26 } }, Collection: { acquireCount: { w: 26 } } } 2300ms",
"split_tokens": ["2020-05-29T08:56:47.903+0800", "I", "WRITE", "[conn149]", "remove", "bookcircle.msgQueue", "command:", "{", "q:", "{", "receiver:", "\"5c9efb9b2f2911028436af98\",", "deviceId:", "\"imei860219041784179\",", "type:", "21", "},", "limit:", "0", "}", "planSummary:", "IXSCAN", "{", "receiver:", "1", "}", "keysExamined:646", "docsExamined:646", "ndeleted:0", "numYields:25", "locks:{", "Global:", "{", "acquireCount:", "{", "r:", "26,", "w:", "26", "}", "},", "Database:", "{", "acquireCount:", "{", "w:", "26", "}", "},", "Collection:", "{", "acquireCount:", "{", "w:", "26", "}", "}", "}", "2300ms"],
"datetime": "2020-05-29T08:56:47.903000+08:00",
"operation": "remove",
"thread": "conn149",
"namespace": "bookcircle.msgQueue",
"nscanned": 646,
"ndeleted": 0,
"duration": 2300,
"numYields": 25
} {
"line_str": "2020-05-30T09:23:31.244+0800 I WRITE [conn154] remove bookcircle.msgQueue command: { q: { receiver: \"5cca30682f29113ecdcc9230\", deviceId: \"imei860219041784179\", type: 21 }, limit: 0 } planSummary: IXSCAN { receiver: 1 } keysExamined:557 docsExamined:557 ndeleted:0 numYields:33 locks:{ Global: { acquireCount: { r: 34, w: 34 } }, Database: { acquireCount: { w: 34 } }, Collection: { acquireCount: { w: 34 } } } 1939ms",
"split_tokens": ["2020-05-30T09:23:31.244+0800", "I", "WRITE", "[conn154]", "remove", "bookcircle.msgQueue", "command:", "{", "q:", "{", "receiver:", "\"5cca30682f29113ecdcc9230\",", "deviceId:", "\"imei860219041784179\",", "type:", "21", "},", "limit:", "0", "}", "planSummary:", "IXSCAN", "{", "receiver:", "1", "}", "keysExamined:557", "docsExamined:557", "ndeleted:0", "numYields:33", "locks:{", "Global:", "{", "acquireCount:", "{", "r:", "34,", "w:", "34", "}", "},", "Database:", "{", "acquireCount:", "{", "w:", "34", "}", "},", "Collection:", "{", "acquireCount:", "{", "w:", "34", "}", "}", "}", "1939ms"],
"datetime": "2020-05-30T09:23:31.244000+08:00",
"operation": "remove",
"thread": "conn154",
"namespace": "bookcircle.msgQueue",
"nscanned": 557,
"ndeleted": 0,
"duration": 1939,
"numYields": 33
}
根据关键字过滤日志
mlogfilter mongod.log --word assert warning error
根据匹配条件查询日志
mlogfilter mongod.log --pattern '{"_id": 1, "host": 1, "ns": 1}
这工具其实很强大，还有很多操作你们可以去研究看看，windows还有个logparse，一个轻量强大日志分析工具其实在应急过程中很常用