An example how to normalise scan outputs and send to ES

SCAN_TO_ES.md

Index per scan

elasticdump \
    --input="./output.json" \
    --output="http://127.0.0.1:9200" \
    --output-index="subfinder" \
    --limit=10000 \
    --type=data \
    --transform="doc._source=Object.assign({},doc)"

find_and_send_to_es.py

import os
import json

# run subfinder

cmd = "subfinder -d {} -oJ -silent -o {}".format("test.com", "output.json")

os.system(cmd)

with open("output.json", "r+") as f:
    data = json.load(f)

final_data = {}
final_data["subdomains"] = data
with open("mod_output.json", "w+") as f:
    json.dump(final_data, f)


cmd = "elasticdump  --input \"{}\"  --output=http://127.0.0.1:9200/ --output-index={} --type=data --transform=\"doc._source=Object.assign({},doc)\"".format("mod_output.json", "subfinder")
os.system(cmd)