Import the Destiny2 manifest into a MongoDB installation

manifestToMongoDB.py

# python -m pip install pymongo
from pymongo import MongoClient
import urllib.request
import sqlite3, os, zipfile, json

# %% Settings
#################

# Language can be: en, fr, es, es-mx, de, it, ja, pt-br, ru, pl, ko, zh-cht, zh-chs
MANIFEST_LANGUAGE = "en"

MONGODB_USERNAME = "user"
MONGODB_PASSWORD = "password"
MONGODB_PORT = 27017
MONGODB_HOST = "localhost"

# Code.
# I recommend not to change anything below this line
#################

BUNGIE_BASE = "https://bungie.net/"
BUNGIE_API_BASE = "https://bungie.net/Platform/"

# %% Step1: Find the correct manifest to download
manifestPaths = json.loads(urllib.request.urlopen(BUNGIE_API_BASE + "/Destiny2/Manifest/").read())["Response"]
manifestPath = manifestPaths["mobileWorldContentPaths"][MANIFEST_LANGUAGE]
print("Selected manifest url:", manifestPath)

# %% Step 2, download and unzip the manifest

if not os.path.exists("./tmp"):
    os.mkdir("./tmp")

zipped_manifest = "./tmp/manifest.zip"
with open(zipped_manifest, "wb") as manifestFile:
    manifestFile.write(urllib.request.urlopen(BUNGIE_BASE + manifestPath).read())
print("Successfully downloaded the manifest into '" + zipped_manifest + "'")

# %%
with zipfile.ZipFile(zipped_manifest, 'r') as zip_ref:
    zip_ref.extractall("./tmp/")
print("Successfully unzipped the manifest")

# %% Step 3: Open the SQLite connection

con = sqlite3.connect("./tmp/" + os.path.basename(manifestPath))
cur = con.cursor()

sql_query_selectAllTables = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
cur.execute(sql_query_selectAllTables)

manifestTables = [t[0] for t in cur.fetchall() if t[0].endswith("Definition")]

# %% Step 4: Import!
# mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
MONGODB_URL = "mongodb://{0}:{1}@{2}:{3}".format(MONGODB_USERNAME, MONGODB_PASSWORD, MONGODB_HOST, MONGODB_PORT)

client = MongoClient(MONGODB_URL)

db = client.get_database("d2manifest_" + MANIFEST_LANGUAGE)

for tableToFill in manifestTables:
    tableName = tableToFill[7:-10]
    print("Start with " + tableName)
    collection = db.get_collection(tableName)
    collection.drop()

    sql_query_selectJson = "select json from " + tableToFill + ";"
    cur.execute(sql_query_selectJson)
    jsonContent = [json.loads(j[0]) for j in cur.fetchall()]
    if len(jsonContent) == 0:
        print("#### WARNING: no content found. Ignoring ####")
    else:
        collection.insert_many(jsonContent)
        print("Done with " + tableName)


con.close()
client.close()
print()
print("DONE!")