NccRange.java

package fin.meta;

import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.TimeZone;
import java.util.zip.GZIPInputStream;
import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.Elements;

public class NccRange {

	public static void main(String[] args) throws Exception {
		SimpleDateFormat sdf = new SimpleDateFormat("dd.MM.yyyy");
		sdf.setTimeZone(TimeZone.getTimeZone("Europe/Moscow"));
		SimpleDateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
		df.setTimeZone(TimeZone.getTimeZone("Europe/Moscow"));
		
		String date = args == null || args.length == 0 ? null : args[0];
		if (date == null) {
			// tomorrow date
			date = sdf.format(System.currentTimeMillis());
		}
		System.out.println("# request date " + date + " : current time " + df.format(System.currentTimeMillis()) + " MSK");
		System.out.println("security,asset_1,asset_2,board,settlementDate,maxPrice,minPrice,changeDate");
		query(date, 1, 100);
	}

	public static void query(String date, int page, int pageSize) throws IOException {
		String uri = "https://www.nationalclearingcentre.com/iss/fondPriceRanges/?pageNumber=" + page + "&xls=false&type=0&secId=&date=" + date + "&date1=" + date + "&date2=" + date;
		if (pageSize > 50) {
			uri += "&pageSize="+pageSize;
		}
		//System.out.println(uri);
		URL url = new URL(uri);
		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
		conn.setRequestMethod("GET");
		conn.setRequestProperty("Content-Language", "en-US");
		conn.setRequestProperty("Host", "www.nationalclearingcentre.com");
		conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:84.0) Gecko/20100101 Firefox/84.0");
		conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
		conn.setRequestProperty("Accept-Encoding", "zip, deflate, br");
		conn.setRequestProperty("DNT", "1");
		conn.setRequestProperty("Pragma", "no-cache");
		conn.setRequestProperty("Cache-Control", "no-cache");
		conn.setUseCaches(false);

		int code = conn.getResponseCode();
		if (code == HttpURLConnection.HTTP_OK) {
			InputStreamReader reader;
			if ("gzip".equals(conn.getContentEncoding()) || conn.getContentType().contains("application/gzip")) {
				reader = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), StandardCharsets.UTF_8);
			} else {
				reader = new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8);
			}

			BufferedReader brx = new BufferedReader(reader);

			String line = brx.readLine();
			String resp = line;
			while (line != null) {
				resp += "\n" + line;
				line = brx.readLine();
				// System.out.println(line);
			}
			brx.close();

			Document doc = Jsoup.parse(resp);
			
			int securities = 0;
			Elements items = doc.select(".dinam-table > tbody > tr");
			for (Element item : items) {
				Elements tds = item.select("> td");
				if (tds.isEmpty()) {
					continue;
				}
				securities++;
				String security = tds.get(0).text();
				String asset_1 = tds.get(1).text();
				String asset_2 = tds.get(2).text();
				String cls = tds.get(3).text();
				String settlementDate = tds.get(4).text();
				String maxPrice = tds.get(5).text();
				String minPrice = tds.get(6).text();
				String changeDate = tds.get(7).text();
				System.out.println(security + "," + asset_1 + "," + asset_2 + "," + cls + "," + settlementDate + "," + maxPrice + "," + minPrice + "," + changeDate);

			}
			//System.out.println("Done: " + securities);
			Elements pages = doc.select(".pagination > li");
			String lastPage = pages.last().text();
			if (lastPage.equals("»")) {
				//System.out.println("CONTINUE " + (page+1));
				query(date, page+1, pageSize);
			} else {
				//System.out.println("STOP at! " + page);
			}
		} else {
			throw new IOException(code + " : " + conn.getResponseMessage() + " : " + uri);
		}
	}
}