Magika Benchmark

.gitignore

python
file-examples

README.md

A Simple Magika Benchmark

Setup

From scratch run:

make clean-all init analyze

This takes a while, so I've committed the data from my machine

Run

make
sqlite3 < analyze.sql
file_avg              python_avg         python_x_times_slower_single_cli
--------------------  -----------------  --------------------------------
0.000874874856301821  0.179884610224334  205.611818568799
file_avg            python_avg     python_x_times_slower_bulk_cli
------------------  -------------  ------------------------------
0.0229037048478788  0.71230794506  31.1001189454277

analyze.sql

create table test_single(command TEXT, mean NUM, stddev NUM, median NUM, user NUM, system NUM, min NUM, max NUM, parameter_F TEXT);
create table test_bulk(command TEXT, mean NUM, stddev NUM, median NUM, user NUM, system NUM, min NUM, max NUM);
.mode csv
.import test-single.csv test_single
.import test-bulk.csv test_bulk

.mode column
with file as (select avg(mean) avg_mean from test_single where command like 'file%'),
python as (select avg(mean) avg_mean from test_single where command like 'PYTHON%')
select file.avg_mean as file_avg, python.avg_mean as python_avg, (python.avg_mean / file.avg_mean) as python_x_times_slower_single_cli from file, python;

with file as (select avg(mean) avg_mean from test_bulk where command like 'file%'),
python as (select avg(mean) avg_mean from test_bulk where command like 'PYTHON%')
select file.avg_mean as file_avg, python.avg_mean as python_avg, (python.avg_mean / file.avg_mean) as python_x_times_slower_bulk_cli from file, python;

Makefile

SHELL := /bin/bash
file_list := $(shell find ./file-examples/ -type f -not -path './file-examples/.git/*')
comma_delimited_file_list := $(shell printf "%s" "${file_list}" | tr ' ' ',')

.PHONY: analyze
analyze: test-bulk.csv test-single.csv
	sqlite3 < analyze.sql

test-bulk.csv:
	hyperfine -w 20 --export-csv test-bulk.csv 'PYTHONPATH="$$PWD/python" ./python/bin/magika ${file_list}' 'file ${file_list}'

test-single.csv:
	hyperfine -w 20 --export-csv test-single.csv -L 'F' "${comma_delimited_file_list}" 'file {F}' 'PYTHONPATH="$$PWD/python" ./python/bin/magika {F}'

.PHONY: init
init:
	git clone git@github.com:moshen/file-examples-fork.git file-examples
	pip3 install magika -t python

.PHONY: clean-csv
clean-csv:
	rm test-single.csv test-bulk.csv

.PHONY: clean-all
clean-all: clean-csv
	rm -rf python file-examples

test-bulk.csv

test-single.csv