necget.py

#!/usr/bin/env python3
import json
import sys

import bs4
import requests

# this is public domain

# A00100C is pc-9800, untested on other groups
# sample usage:
#     $ necget ids A00100C > ids.json
#     $ necget urls ids.json > urls
#     $ for i in $(grep -v '^#' urls); do curl -LJO "$i"; done


BASE = "http://121ware.com/psp/PA121/NECS_SUPPORT_SITE/CRM/s/WEBLIB_NECS_DID"
SUBLIST_URL = BASE + ".NECS_MOD_CATEGORY.FieldFormula.IScript_VDown_Series_SubList"
MOD_URL     = BASE + ".PRODUCT_ID.FieldFormula.IScript_VDown_Id_Mod"
DETAILS_URL = "http://121ware.com/psp/PA121/121ETC2/CRM/s/WEBLIB_NECS_DID.PRODUCT_ID.FieldFormula.IScript_VDown_Details"


def group_id_links(group_id):
	r = requests.post(SUBLIST_URL, params={"EX_GROUP_ID": group_id})
	soup = bs4.BeautifulSoup(r.text, "html.parser")

	for i in soup.select("select[name=EX_GROUP_ID_LINK] > option"):
		if i.get("value"):
			yield i['value']


def prod_ids(group_id_link):
	params = {
		"EX_GROUP_ID_LINK": group_id_link,
		"EX_GROUP_ID": group_id_link[:-3],
	}

	r = requests.post(SUBLIST_URL, params=params)
	soup = bs4.BeautifulSoup(r.text, "html.parser")

	for i in soup.select("table table[width=624] > tr > td[bgcolor=#E5E5E5] > a"):
		yield i.text


def mod_ids(prod_id):
	r = requests.get(MOD_URL, params={"prodId": prod_id})
	soup = bs4.BeautifulSoup(r.text, "html.parser")

	for i in soup.select("td[nowrap] + td[bgcolor=#E5E5E5]"):
		yield i.text


def file_urls(mod_id):
	r = requests.get(DETAILS_URL, params={"modId": mod_id})
	soup = bs4.BeautifulSoup(r.text, "html.parser")

	for i in soup.select("tr[bgcolor=#E6E6E8] > td a"):
		if i.get("href"):
			yield i['href']


def get_ids(gid):
	ids = {}

	print("make yourself a coffee or something", file=sys.stderr)
	for g in group_id_links(gid):
		for p in prod_ids(g):
			ids[p] = []
			for m in mod_ids(p):
				ids[p].append(m)
			print("got all ids for prod {}".format(p), file=sys.stderr)
		print("got all ids for group {}".format(g), file=sys.stderr)

	return ids


def get_urls(ids):
	for i in ids:
		print("# {}".format(i))
		for url in file_urls(i):
			print(url)


if __name__ == "__main__":
	if len(sys.argv) < 3:
		print("usage: {} mode".format(sys.argv[0]), file=sys.stderr)
		exit(1)

	if sys.argv[1] == "ids":
		ids = get_ids(sys.argv[2])
		print(json.dumps(ids))
	elif sys.argv[1] == "urls":
		with open(sys.argv[2]) as f:
			ids = {i for k, v in json.load(f).items() for i in v}
			get_urls(ids)