Work with MongoDB logs to troubleshoot problems using Mtools

mtools.txt

# Grab MongoDB logs from the server
1. SSH into the server and become root user
2. Go to /data folder
3. Go to the application folder
4. Sort the logs by date (ls -lrt)
5. Copy the bottom of the logs to the /tmp folder (cp logs.log /tmp)
6. Exit from root user and go to /tmp folder
7. Change ownership to current user (sudo chown $USER:$USER logs.log)
8. Compress the logs file (sudo gzip logs.log)
9. Download/Upload to bucket:
	In case download:
		Click on right top gear icon and select the download option and enter /tmp/logs.log.gz and hit download.
	
	In case upload to bucket:
		# Login into your account
		gcloud auth login
		gcloud auth list

		# Upload downloaded file to GCP bucket
		gsutil cp ./<downloaded backup file> gs://<storage bucket>
		OR
		# Upload directory to GCP bucket
		gsutil cp -r dir gs://<storage bucket>

# RUN mtools from Windows machine:
	
	# Prerequisites:
		Python
			You need to have Python 3.6.x, 3.7.x, or 3.8.x installed in order to use mtools
			pip3 install mtools
			pip3 install mtools[all]
			


# Filter the logs based on time and execution time
	Syntax: 
		mlogfilter "pathto\mongod.log" --from [24hr time] --to [24hr time] --slow [number in milliseconds] > filtered-query-logs.log 2>&1

	In my case: 
		mlogfilter "C:\Program Files\MongoDB\Server\3.6\bin\data\mongodb.log" --from 05:00 --to 06:00 --slow 1000 > 2021-12-01T11-11-39.log 2>&1

# Filter the logs based on slowness of the queres execution
	mlogfilter mongodb.log --from 21:45 --to 22:00 --slow 1000 > slow-queries.log 2>&1

	In my case: 
		mlogfilter "C:\Program Files\MongoDB\Server\3.6\bin\data\mongodb.log" --from 21:45 --to 22:00 --slow 1000 > slow-queries.log 2>&1


# Plot graphs
	Syntax: 
		mplotqueries "pathto\mongod.log" --group operation
	In my case: 
		mplotqueries "C:\Program Files\MongoDB\Server\3.6\bin\data\mongodb.log" --group operation


# operation groups
	mlogfilter mongod.log --operation update --namespace test.users | mplotqueries --type histogram --group "_id: ObjectId\('([^']+)'\)"


# long-running oplog.rs operations as duration lines
grep "oplog.rs" mongod.log | mplotqueries --type durline --overlay
mplotqueries mongod.log --group operation


# plots the inserts per hour (3600 seconds) as a histogram plot
mlogfilter mongod.log --operation insert | mplotqueries --type histogram --bucketsize 3600

# all admin commands that have been slower than 1 second.
mlogfilter mongod.log --namespace admin.\$cmd --slow 1000

# trace a single connection to the database
mlogfilter mongod.log --thread conn1234

# From today to last 2hr
mlogfilter mongod.log --from today --to +2hours

# To get the collection scan collections 
mlogfilter mongod.log --planSummary=COLLSCAN

# To get the slow queries more than 1000ms 
mlogfilter mongod.log --slow 1000

# To get the errors,warnings 
mlogfilter mongod.log  --word assert warning errors

# Go through the log file and find all queries (including queries from updates) and collect some statistics for each query pattern. 
mloginfo --queries logs.mongo/mongod.log

# sort the results
mloginfo mongod.log --queries --sort count
mloginfo mongod.log --queries --sort sum 

# Go through the log file and find all server restarts
mloginfo mongod.log --restarts

# general information about opened and closed connections in the log file
mloginfo mongod.log --connections 

# information about every detected replica set state change
mloginfo mongod.log --rsstate

# overview of the log file of what kind of lines appear in the file
mloginfo mongod.log --distinct


# creates a self-contained html file that shows an interactive visualization
mlogvis mongod.log --out mongo.html

# Filter log entries based on a specific namespace (collection or database):
mlogfilter --namespace mydatabase.mycollection mongod.log

# Filter log entries within a specific time range:
mlogfilter --start "2019-01-01T00:00:00" --end "2019-01-02T00:00:00" mongod.log

# Filter log entries based on a specific operation, e.g., queries:
mlogfilter --operation query mongod.log

# Filter log entries based on a specific query pattern:
mlogfilter --queryPattern "find({ 'status': 'active' })" mongod.log

# Filter log entries based on query duration:
mlogfilter --duration 100 mongod.log

# Filter log entries from a specific host:
mlogfilter --host myhostname mongod.log

# Generate test data for MongoDB based on a specified schema.
mgenerate schema.json --num 1000 | mongoimport --uri "mongodb://localhost:27017/mydatabase" --collection mycollection

# Merge multiple log files into a single file for consolidated analysis.
mlogmerge /path/to/logfile1 /path/to/logfile2 -o /path/to/merged.log

# Generate a plot to visualize replication lag in a MongoDB replica set.
mplotreplication /path/to/mongod.log

# Display slow operations from the log file, helping to identify performance bottlenecks.
mlogslow /path/to/mongod.log

# Parse and display log entries in a more readable format.
mlogparse /path/to/mongod.log

# Launch an interactive web-based log visualization tool.
mlogvis /path/to/mongod.log

# Filter log entries based on a specific pattern, like a specific namespace or operation.
mlogfilter /path/to/mongod.log --ns mydatabase.mycollection --op update

# Extract distinct IP addresses from the log file.
mlogdistinct --field remote mongod.log

# Display basic information about a MongoDB log file.
mloginfo mongod.log

# Show all log entries with level "WRITE".
mlogfilter --level WRITE mongod.log

# Display all slow operations with a threshold of 100 milliseconds.
mlogslow --threshold 100 mongod.log

# Show unique client IPs in the log.
mlogdistinct --field client mongod.log

# Extract all lines related to a specific query pattern.
mlogextract --query "{find: 'mycollection'}" mongod.log

# Validate the syntax of log files.
mlogvalidate mongod.log

# Generate an HTML report from log data.
mloginfo2html mongod.log > report.html

# Filter MongoDB log files based on various criteria.
mlogfilter mongod.log --query '{ "severity" : "I" }'




https://github.com/rueckstiess/mtools







mgenerate:
	
	# Generate test data for MongoDB based on a specified schema.
	mgenerate schema.json --num 1000 | mongoimport --uri "mongodb://localhost:27017/mydatabase" --collection mycollection

mlog commands:
	
	mlogvalidate:
		
		# Validate the syntax of log files.
		mlogvalidate mongod.log
	
	mloginfo:
		
		# sort the results
		mloginfo mongod.log --queries --sort count
		mloginfo mongod.log --queries --sort sum
		
		# Go through the log file and find all server restarts
		mloginfo mongod.log --restarts


			C:\Windows\System32>mloginfo -h
			usage: mloginfo [-h] [--version] [--no-progressbar] [--debug] [--verbose]
			                [--clients] [--connections] [--connstats] [--cursors]
			                [--distinct] [--queries]
			                [--sort {namespace,pattern,count,min,max,mean,95%,sum}]
			                [--rounding {0,1,2,3,4}] [--restarts] [--rsinfo] [--rsstate]
			                [--sharding] [--errors] [--migrations] [--storagestats]
			                [--transactions] [--tsort {duration}]
			                [logfile [logfile ...]]

	mlogmerge:
	mlogfilter:
	mlogextract:
	mlogdistinct:
	mlogslow:
	mlogparse:
mplot commands:
	mplotreplication:
	mplotqueries:
		grep -v "writebacklisten" mongod.log | mplotqueries
		mlogfilter mongod.log --namespace "admin.\$cmd" --exclude | mplotqueries
		mplotqueries mongod.log --group operation
		grep "serverStatus was very slow" mongod.log | mplotqueries --type event
		mplotqueries `ls mongod_*` --type range
		mplotqueries mongod.log --group operation --overlay
		mplotqueries mongod.log --type rsstate



Check connectivity: 
	nc -zv mongodb_server_ip 27017
	sudo netstat -tulpn