Skip to content

Instantly share code, notes, and snippets.

@ChiChou
Last active May 26, 2020 03:37
Show Gist options
  • Save ChiChou/48472eaf5a0921489b5fb29b315f8bdc to your computer and use it in GitHub Desktop.
Save ChiChou/48472eaf5a0921489b5fb29b315f8bdc to your computer and use it in GitHub Desktop.
Recursively scan all executables in a folder and generate IDA databases in parallel

Recursively scan all executables (PE, ELF and MachO!) in a folder and generate IDA databases in parallel

Usage

node batch.js [path]

const fs = require('fs').promises;
const path = require('path');
const os = require('os');
const { spawn } = require('child_process');
const { EventEmitter } = require('events');
async function* walk(dir) {
const ignored = new Set('id0,id1,nam,log,til,idb,i64,js,py'.split(','));
const files = await fs.readdir(dir);
for (file of files) {
const ext = path.extname(file).toLowerCase();
if (ignored.has(ext)) {
continue;
}
const joint = path.join(dir, file);
try {
const stat = await fs.stat(joint);
if (stat.isDirectory()) {
yield* walk(joint);
} else {
yield joint;
}
} catch(_) {
continue;
}
}
}
async function check64bit(filename) {
const FAT_MAGIC = 0xcafebabe,
FAT_CIGAM = 0xbebafeca,
MH_MAGIC = 0xfeedface,
MH_CIGAM = 0xcefaedfe,
MH_MAGIC_64 = 0xfeedfacf,
MH_CIGAM_64 = 0xcffaedfe;
const IMAGE_FILE_MACHINE_I386 = 0x14c,
IMAGE_FILE_MACHINE_AMD64 = 0x8664,
IMAGE_FILE_MACHINE_ARM = 0x1c0,
IMAGE_FILE_MACHINE_ARM64 = 0xaa64;
const handle = await fs.open(filename);
const buffer = Buffer.allocUnsafe(4);
try {
const { bytesRead } = await handle.read(buffer, 0, 4);
if (bytesRead < 4) throw new RangeError('invalid file size');
const magic = buffer.readUInt32LE();
if (new Set([FAT_MAGIC, FAT_CIGAM, MH_MAGIC_64, MH_CIGAM_64]).has(magic)) {
return 64;
} else if (new Set([MH_MAGIC, MH_CIGAM]).has(magic)) {
return 32;
} else if (buffer.slice(0, 2).compare(Buffer.from('MZ')) === 0) {
await handle.read(buffer, 0, 2, 0x3c);
const offset = buffer.readUInt16LE();
await handle.read(buffer, 0, 4, offset);
if (buffer.compare(Buffer.from('PE\x00\x00')) !== 0) throw new Error('invalid COFF magic');
await handle.read(buffer, 0, 2, offset + 4);
// little endian only
const cpuMagic = buffer.readUInt16LE();
if ([IMAGE_FILE_MACHINE_I386, IMAGE_FILE_MACHINE_ARM].includes(cpuMagic)) return 64;
if ([IMAGE_FILE_MACHINE_AMD64, IMAGE_FILE_MACHINE_ARM64].includes(cpuMagic)) return 32;
throw new Error('Unknown CPU');
} else if (buffer.compare(Buffer.from('\x7fELF')) === 0) {
await handle.read(buffer, 0, 1, 4);
const val = buffer.readUInt8();
if ([1, 2].includes(val)) return val * 32;
throw new Error('invalid EI_CLASS');
} else {
throw new Error('Unknown executable format');
}
} finally {
await handle.close();
}
}
class Parallel extends EventEmitter {
constructor(count) {
super();
this.max = count || os.cpus().length;
this.pool = new Set();
process.on('SIGINT', () => {
this.removeAllListeners();
this.pool.forEach(p => p.kill());
process.exit();
});
}
spawn(cmd, filename) {
const name = path.basename(filename);
const py = path.join(__dirname, 'idc.py');
const child = spawn(cmd, ['-c', '-A', `-S${py}`, `-L${filename}.log`, `-o${filename}`, filename]);
this.pool.add(child);
child.on('close', (code) => {
this.pool.delete(child);
this.emit('finish', { name, code });
});
child.on('error', () => {
});
return child;
}
async consume(name) {
let bits
try {
bits = await check64bit(name);
} catch(e) {
return;
}
const cmd = bits == 64 ? 'idat64': 'idat';
const ext = bits == 64 ? 'i64' : 'idb';
const st = await fs.stat(`${name}.${ext}`).catch(e => {});
if (st) return;
if (this.pool.size >= this.max) {
await new Promise(resolve => this.once('finish', resolve)).catch(e => console.error(e));
}
try {
this.spawn(cmd, name);
} catch(e) {
console.log(e);
}
}
}
async function main(cwd) {
const job = new Parallel();
for await (let name of walk(cwd)) {
await job.consume(name);
}
}
main(process.argv[2] || '.');
import idc
def main():
# todo: add your analyzer
idc.auto_wait()
idc.qexit(0)
if __name__ == "__main__":
main()
@ChiChou
Copy link
Author

ChiChou commented Apr 24, 2020

Forgot to mention that you need to idat and idat64 to $PATH

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment