Experimental listings.csv reformat

reformat.py

import os

VERSION = 1
EPOCH = 1700000000
# id,station_id,commodity_id,supply,supply_bracket,buy_price,sell_price,demand,demand_bracket,collected_at
# 1,128000000,128049152,0,-1,0,81368,901,-1,1715284476

# Trade Dangerous Data Listing Format:
# Trade-Dangerous Data Listing, based on jsonl.
# line 1: dict describing the listing
#           - epoch is used to reduce the size of timestamps,
#           - item_ids is a list of actual item_ids, the listings refere to
#            positions in this sorted list.
# lines 2+ are either a dict or a list:
#         dict: starts a new station block. s=station_id relative to previous, t=modified timestamp, n=number of entries,
#         list: [0] = which item in item_ids this is,
#               [1] = 0 or [demand price, demand units, demand level]
#               [2] = 0 or [supply price, supply units, supply level]
#         level: 0 = unknown, 1 = low, 2 = med, 3 = high


listings_csv = "listings.csv"
if not os.path.exists(listings_csv):
    listings_csv = "eddb/" + listings_csv
    if not os.path.exists(listings_csv):
        listings_csv = "data/" + listings_csv
item_csv = "Item.csv"
if not os.path.exists(item_csv):
    item_csv = "data/" + item_csv

i = 0
with open(listings_csv, "rb") as infh:
    infh.readline()
    first_station = int(infh.readline().split(b',')[1])

item_map = {}
with open(item_csv, "rb") as infh:
    infh.readline()
    for line in infh:
        item_id = line.split(b',', 2)[0]
        local_id = len(item_map)
        item_map[item_id] = local_id

item_id_list = list(int(i) for i in item_map.keys())
item_offset_list = list(item_id_list)
for i in range(1, len(item_id_list)):
    item_offset_list[i] -= item_id_list[i - 1]

items = ",".join(f"{int(i)}" for i in item_offset_list)


def save_station(outfh, station, previous, modified, lines, items):
    int_station = int(station)
    rel_station = int_station - previous
    outfh.write(f'{{"s":{rel_station},"t":{modified-EPOCH},"n":{items}}}\n'.encode())
    outfh.write(station_lines)
    return int_station


with open(listings_csv, "rb") as infh, open("Listing.tdml", "wb") as outfh:

    outfh.write(
        f'{{"version":{VERSION},'
        f'"epoch":{EPOCH},'
        f'"items_ids":[{items}],'
        '"doc":"https://github.com/eyeonus/Trade-Dangerous"'
        '}\n'.encode()
    )

    in_lines = iter(infh)
    next(in_lines)

    last_station = 0
    cur_station = None
    station_lines = b""
    max_modified = 0
    line_count = None

    for in_line in in_lines:
        (_, station_label, item_label, s_units, s_level, s_price, d_price,
         d_units, d_level, timestamp) = in_line.rstrip().split(b',')
        if station_label != cur_station:  # Station Changed
            if cur_station:
                last_station = save_station(outfh, cur_station, last_station, max_modified, station_lines, line_count)
            cur_station = station_label
            max_modified = EPOCH
            station_lines = b""
            line_count = 0

        line_count += 1

        station_lines += f"[{item_map[item_label]},".encode()
        if s_price == b"0":
            station_lines += b"0,"
        else:
            price, units, level = int(s_price), int(s_units), int(s_level) + 1
            station_lines += f"[{price},{units},{level}],".encode()
        if d_price == b"0":
            station_lines += b"0]\n"
        else:
            price, units, level = int(d_price), int(d_units), int(d_level) + 1
            station_lines += f"[{price},{units},{level}]]\n".encode()
        max_modified = max(max_modified, int(timestamp))

    if cur_station:
        last_station = save_station(outfh, cur_station, last_station, max_modified, station_lines, line_count)