기상청 실시간 지진 감시 페이지 크롤러.

Pews.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Threading;
using System.Net.Http;
using System.Net;
using System.IO;
using System.Web;

namespace CSTest
{
    class Program
    {
        const int HeadLength = 4;
        const int MaxEqkStrLen = 60;
        const int MaxEqkInfoLen = 120;
        static string[] AreaNames = { "서울", "부산", "대구", "인천", "광주", "대전", "울산", "세종", "경기", "강원", "충북", "충남", "전북", "전남", "경북", "경남", "제주" };

        static bool StationUpdate = true;

        static void Main(string[] args)
        {
            using (var sw = new StreamWriter(new FileStream("log.txt", FileMode.Append)))
            {
                sw.WriteLine(DateTime.UtcNow);
            }


            string prevBinTime = string.Empty;
            double tide = 1000;
            DateTime nextSyncTime = DateTime.MinValue;

            while (true)
            {
                try
                {
                    string binTime = DateTime.UtcNow.AddMilliseconds(-tide).ToString("yyyyMMddHHmmss");
                    if (prevBinTime == binTime)
                    {
                        continue;
                    }
                    prevBinTime = binTime;

                    Console.WriteLine(binTime);

                    string url = $"https://www.weather.go.kr/pews/data/{binTime}";


                    byte[] bytes = null;

                    using (var client = new WebClient())
                    {
                        client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");

                        bytes = client.DownloadData(url + ".b");


                        // 시간 동기화.
                        if (DateTime.UtcNow >= nextSyncTime)
                        {
                            nextSyncTime = DateTime.UtcNow + TimeSpan.FromSeconds(10.0);

                            string stStr = client.ResponseHeaders.Get("ST");
                            if (!string.IsNullOrWhiteSpace(stStr)
                                && double.TryParse(stStr, out double serverTime))
                            {
                                tide = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - serverTime * 1000 + 1000;
                                Console.WriteLine("Sync: {0}", tide);
                            }
                        }
                    }


                    if (bytes != null && bytes.Length > MaxEqkStrLen)
                    {
                        var headerBuff = new StringBuilder();
                        for (int i = 0; i < HeadLength; ++i)
                        {
                            headerBuff.Append(ByteToBinStr(bytes[i]));
                        }
                        string header = headerBuff.ToString();

                        var bodyBuff = new StringBuilder(ByteToBinStr(bytes[0]));
                        for (int i = HeadLength; i < bytes.Length; ++i)
                        {
                            bodyBuff.Append(ByteToBinStr(bytes[i]));
                        }
                        string body = bodyBuff.ToString();


                        StationUpdate = (StationUpdate || (header[0] == '1'));

                        int phase = 0;
                        if (header[1] == '0')
                        {
                            phase = 1;
                        }
                        else if (header[1] == '1' && header[2] == '0')
                        {
                            phase = 2;
                        }
                        else if (header[2] == '1')
                        {
                            phase = 3;
                        }


                        if (phase > 1)
                        {
                            using (var client = new WebClient())
                            {
                                client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");

                                if (!Directory.Exists("bin"))
                                {
                                    Directory.CreateDirectory("bin");
                                }

                                client.DownloadFile(url + ".b", $"bin/{binTime}.b");
                            }

                            var infoBytes = bytes.Skip(bytes.Length - MaxEqkStrLen).ToArray();
                            HandleEqk(body, infoBytes);
                        }


                        if (StationUpdate)
                        {
                            byte[] stnBytes = null;

                            using (var client = new WebClient())
                            {
                                client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");

                                stnBytes = client.DownloadData(url + ".s");
                            }

                            if (!Directory.Exists("bin"))
                            {
                                Directory.CreateDirectory("bin");
                            }
                            File.WriteAllBytes($"bin/{binTime}.s", stnBytes);

                            bodyBuff = new StringBuilder();
                            for (int i = 0; i < stnBytes.Length; ++i)
                            {
                                bodyBuff.Append(ByteToBinStr(stnBytes[i]));
                            }

                            HandleStn(bodyBuff.ToString(), body);
                        }
                        else
                        {
                            var mmiData = ParseMmi(body);

                            if (mmiData.Max() >= 2)
                            {
                                Console.WriteLine("관측소 현재 최대, 최소 진도: {0}, {1}", mmiData.Max(), mmiData.Min());

                                mmiData.RemoveAll((v) => v <= 1);
                                if (mmiData.Count > 0)
                                {
                                    Console.WriteLine("진도 목록: {0}", string.Join(", ", mmiData));
                                }

                                if (!Directory.Exists("bin"))
                                {
                                    Directory.CreateDirectory("bin");
                                }
                                File.WriteAllBytes($"bin/{binTime}.b", bytes);
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message);
                    Console.WriteLine(e.StackTrace);

                    using (var sw = new StreamWriter(new FileStream("log.txt", FileMode.Append)))
                    {
                        sw.WriteLine(DateTime.UtcNow);
                        sw.WriteLine(e.Message);
                        sw.WriteLine(e.StackTrace);
                    }
                }


                Thread.Sleep(100);
            }
        }

        static string ByteToBinStr(byte val)
        {
            return Convert.ToString(val, 2).PadLeft(8, '0');
        }

        static void HandleEqk(string body, byte[] infoBytes)
        {
            string data = body.Substring(body.Length - (MaxEqkStrLen * 8 + MaxEqkInfoLen));
            string eqkStr = WebUtility.UrlDecode(Encoding.UTF8.GetString(infoBytes));

            double origLat = 30 + (double)Convert.ToInt32(data.Substring(0, 10), 2) / 100;
            double origLon = 124 + (double)Convert.ToInt32(data.Substring(10, 10), 2) / 100;
            double eqkMag = (double)Convert.ToInt32(data.Substring(20, 7), 2) / 10;
            double eqkDep = (double)Convert.ToInt32(data.Substring(27, 10), 2) / 10;
            int eqkTime = Convert.ToInt32(data.Substring(37, 32), 2);
            string eqkId = "20" + Convert.ToInt32(data.Substring(69, 26), 2);
            int eqkMax = Convert.ToInt32(data.Substring(95, 4), 2);
            string eqkMaxAreaStr = data.Substring(99, 17);
            var eqkMaxArea = new List<string>();
            if (eqkMaxAreaStr != new string('1', eqkMaxAreaStr.Length))
            {
                for (int i = 0; i < eqkMaxAreaStr.Length; ++i)
                {
                    if (eqkMaxAreaStr[i] == '1')
                    {
                        eqkMaxArea.Add(AreaNames[i]);
                    }
                }
            }

            Console.WriteLine("위도: {0}", origLat);
            Console.WriteLine("경도: {0}", origLon);
            Console.WriteLine("규모: {0}", eqkMag);
            Console.WriteLine("깊이: {0}", eqkDep);
            Console.WriteLine("시각(UTC): {0}", DateTimeOffset.FromUnixTimeSeconds(eqkTime).AddHours(9.0));
            Console.WriteLine("ID: {0}", eqkId);
            Console.WriteLine("문구: {0}", eqkStr);
            Console.WriteLine("진도: {0}", eqkMax);
            Console.WriteLine("지역: {0}", string.Join(", ", eqkMaxArea));
        }

        static void HandleStn(string stnBody, string binBody)
        {
            var stnLat = new List<double>();
            var stnLon = new List<double>();

            for (int i = 0; i + 20 <= stnBody.Length; i += 20)
            {
                stnLat.Add(30 + (double)Convert.ToInt32(stnBody.Substring(i, 10), 2) / 100);
                stnLon.Add(120 + (double)Convert.ToInt32(stnBody.Substring(i + 10, 10), 2) / 100);
            }

            Console.WriteLine("관측소 수: {0}", stnLat.Count);

            if (stnLat.Count < 99)
            {
                // 재시도.
                return;
            }

            var mmiData = ParseMmi(binBody);

            Console.WriteLine("관측소 현재 최대, 최소 진도: {0}, {1}", mmiData.Max(), mmiData.Min());

            // mmiData[i]가 stnLat[i], stnLon[i]에 위치한 관측소의 진도 정보임.
            // 단, 목록의 수가 같지 않을 수 있음을 주의. (보통 mmiData의 수가 더 많음.)

            mmiData.RemoveAll((v) => v <= 1);
            if (mmiData.Count > 0)
            {
                Console.WriteLine("진도 목록: {0}", string.Join(", ", mmiData));
            }

            if (stnLat.Count >= 99)
            {
                // 얻은 관측소 수가 충분하여 다음 업데이트 대기.
                StationUpdate = false;
            }
        }

        static List<int> ParseMmi(string data)
        {
            var mmiData = new List<int>();

            string mmiBody = data.Split(new[] { "11111111" }, StringSplitOptions.None).First();
            for (int i = 8; i < mmiBody.Length; i += 4)
            {
                mmiData.Add(Convert.ToInt32(mmiBody.Substring(i, 4), 2));
            }

            return mmiData;
        }
    }
}

https://www.weather.go.kr/pews/pews.html