Last active
August 23, 2022 14:56
-
-
Save kingsj0405/efbf0c879f37d79230caee4279c6d101 to your computer and use it in GitHub Desktop.
연세대학교 관련 스크립트
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// 신청인원 보고서 | |
// 신청내역 나오는 화면에서 콘솔(F12) 열어서 | |
var data_my_mil; | |
var finished1 = false; | |
var finished2 = false; | |
function call_data(){ | |
call_data2(); | |
jQuery('table').fadeTo('fast',0.5); | |
finished1 = false; | |
jQuery.ajax({ | |
method : "POST", | |
url : "https://ysweb.yonsei.ac.kr/sugang.Ysv?action=getCurri", | |
headers : {"submissionid" : "sbm_getWish"}, | |
data : JSON.stringify({curriInfo:{code:"getWish",srhKey:"",lang:"ko"}}), | |
dataType: 'json', | |
contentType: "application/json", | |
success : function(data){ | |
summary_all(data,0); | |
}, | |
complete : function(){ | |
finished1 = true; | |
if(finished1 && finished2) | |
jQuery('table').fadeTo('fast',1); | |
} | |
}); | |
} | |
function call_data2(){ | |
jQuery('table').fadeTo('fast',0.5); | |
finished2 = false; | |
jQuery.ajax({ | |
method : "POST", | |
url : "https://ysweb.yonsei.ac.kr/sugang.Ysv?action=getCurri", | |
headers : {"submissionid" : "sbm_getMileage"}, | |
data : JSON.stringify({curriInfo:{code:"getMileage",srhKey:"",lang:"ko"}}), | |
dataType: 'json', | |
contentType: "application/json", | |
success : function(data){ | |
summary_all(data,1); | |
}, | |
complete : function(){ | |
finished2 = true; | |
if(finished1 && finished2) | |
jQuery('table').fadeTo('fast',1); | |
} | |
}); | |
} | |
function summary_all(data,num){ | |
var table_jq = jQuery(jQuery('table tbody')[num]); | |
if(num == 0) | |
table_jq.empty().append('<tr id="dummy"><td colspan="10" style="height:5px;"></td></tr>'); | |
else | |
table_jq.empty(); | |
for(var index in data.data){ | |
var is_applied = (typeof data_my_mil[data.data[index].HAKBBSBB] != 'undefined'); | |
var rate = (data.data[index].WAITCNT/data.data[index].MAX1).toFixed(2); | |
var background_color = 'white'; | |
var txt_color = 'black'; | |
if(rate >= 2 ){ | |
background_color = '#D60036'; | |
txt_color = 'white'; | |
}else if(rate >= 1.5){ | |
background_color = '#FFA2B2'; | |
}else if(rate >= 1){ | |
background_color = '#FFD03E'; | |
}else{ | |
background_color = '#10cd7d'; | |
} | |
var tr_string = '<tr style="background-color:'+background_color+';color:'+txt_color+';" class="grid_body_row">'+ | |
'<td class="gridBodyDefault">'+data.data[index].KNA+'</td>'+ | |
'<td class="gridBodyDefault">'+data.data[index].PROF+'</td>'+ | |
'<td class="gridBodyDefault">'+data.data[index].TIME+'</td>'+ | |
'<td class="gridBodyDefault">'+(is_applied ? data_my_mil[data.data[index].HAKBBSBB] : '')+'</td>'+ | |
'<td class="gridBodyDefault"><strong>'+rate+'</strong></td>'+ | |
'<td class="gridBodyDefault"><strong>'+data.data[index].WAITCNT+'</strong></td>'+ | |
'<td class="gridBodyDefault">'+data.data[index].MAX1+'</td>'+ | |
'<td class="gridBodyDefault">'+data.data[index].FIX_HAKYOUN+'</td>'+ | |
'<td class="gridBodyDefault">'+data.data[index].MAX_MAJOR+'</td>'+ | |
'<td class="gridBodyDefault">'+data.data[index].WISHCNT+'</td>'+ | |
'</tr>'; | |
table_jq.append(tr_string); | |
if(is_applied && num==0){ | |
jQuery('#dummy').before(tr_string); | |
} | |
} | |
if(num == 1) | |
table_jq.append('<tr><td colspan="10" style="text-align:center"><input type="button" value="Reload" onclick="javascript:call_data();" style="padding:10px;"/></td></tr>'); | |
} | |
var trs = jQuery('#gridMileage_body_table tbody tr:not(.w2grid_hidedRow)'); | |
if(typeof data_my_mil == 'undefined'){ | |
data_my_mil = []; | |
for(var index = 0 ; index < trs.length ; index++){ | |
var this_tr = jQuery(jQuery('#gridMileage_body_table tbody tr:not(.w2grid_hidedRow)')[index]); | |
var id = jQuery(this_tr.find('td')[0]).find('span').clone().children().remove().end().text().trim(); | |
var mil = jQuery(this_tr.find('td')[13]).find('span').clone().children().remove().end().text().trim(); | |
data_my_mil[id] = mil; | |
} | |
} | |
jQuery(document.body).html('').append('<table id="gridWish_body_table" class="gridHeaderTableDefault" style="width: 1201px;margin:0 auto;"></table>') | |
.append('<table id="gridWish_body_table2" class="gridHeaderTableDefault" style="width: 1201px;margin:0 auto;"></table>'); | |
jQuery('table').append('<thead id="gridWish_head_table" class="gridHeaderTableDefault"><tr><th>과목명</th><th>교수</th><th>시간</th><th>내 마일리지</th><th>경쟁률</th><th>신청인원</th><th>정원</th><th>정원(학년)</th><th>정원(전공)</th><th>희망인원</th></tr></thead>') | |
.append('<tbody id="gridWish_body_tbody"></tbody>'); | |
call_data(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
This code use selenium and PhantomJS. | |
You have to install PhantomJS on following link. | |
http://phantomjs.org/download.html | |
""" | |
import json | |
import re | |
from selenium import webdriver | |
from selenium.common.exceptions import StaleElementReferenceException | |
from selenium.webdriver.common.by import By | |
from selenium.webdriver.support import expected_conditions as EC | |
from selenium.webdriver.support.ui import Select | |
from selenium.webdriver.support.ui import WebDriverWait | |
from constants import IGNORE_YEAR, IGNORE_SEMESTER | |
def crawl(): | |
driver = webdriver.PhantomJS() | |
driver.get("http://ysweb.yonsei.ac.kr:8888/curri120601/curri_new.jsp#top") | |
year = Select(driver.find_element_by_id('HY')) | |
semester = Select(driver.find_element_by_id('HG')) | |
college = Select(driver.find_element_by_id('OCODE1')) | |
department = Select(driver.find_element_by_id('S2')) | |
for y in reversed(list(filter(lambda y: y.text not in IGNORE_YEAR, year.options))): | |
year.select_by_visible_text(y.text) | |
semester = Select(driver.find_element_by_id('HG')) | |
for s in reversed(list(filter(lambda s: (y.text, s.text) not in IGNORE_SEMESTER, semester.options))): | |
semester.select_by_visible_text(s.text) | |
college = Select(driver.find_element_by_id('OCODE1')) | |
meta_data = { | |
'cnt_of_subject': 0, | |
} | |
data = list() | |
for c in college.options: | |
college.select_by_visible_text(c.text) | |
department = Select(driver.find_element_by_id('S2')) | |
for d in department.options: | |
print('|'.join([y.text, s.text, c.text, d.text])) | |
meta_data_key = '_'.join(['cnt', c.text, d.text]) | |
meta_data[meta_data_key] = 0 | |
department.select_by_visible_text(d.text) | |
driver.execute_script("searchGb('search',1);") | |
wait = WebDriverWait(driver, 3) | |
wait.until(EC.visibility_of_all_elements_located( | |
(By.ID, "row0jqxgrid"))) | |
while True: | |
for i in range(15): | |
row_id = 'row' + str(i) + 'jqxgrid' | |
row = driver.find_element_by_id(row_id) | |
cells = row.find_elements_by_css_selector( | |
'.jqx-grid-cell') | |
empty_cell_cnt = 0 | |
no_data = False | |
row_data = [ | |
y.text, | |
s.text, | |
c.text, | |
d.text | |
] | |
for cell in cells: | |
if not cell or not cell.text: | |
empty_cell_cnt += 1 | |
if 'No data to display' in cell.text: | |
no_data = True | |
row_data.append(cell.text.strip()) | |
if empty_cell_cnt >= 18 or no_data: | |
break | |
meta_data['cnt_of_subject'] += 1 | |
meta_data[meta_data_key] += 1 | |
data.append(row_data) | |
pager = driver.find_element_by_id("pager") | |
state = re.findall('\d+', pager.text) | |
if state[1] == state[2]: | |
break | |
buttons = driver.find_elements_by_css_selector( | |
'.jqx-button') | |
try: | |
buttons[1].click() | |
except StaleElementReferenceException: | |
break | |
print('all_cnt: ', meta_data['cnt_of_subject'], | |
'current_department: ', meta_data[meta_data_key]) | |
with open('yonsei_lecture-' + y.text + '-' + s.text + '.json', 'w') as outfile: | |
json.dump({'metadata': meta_data, 'data': data}, outfile) | |
def post_process(y, s): | |
json_data = [] | |
new_data = [] | |
infile_path = 'yonsei_lecture-' + str(y) + '-' + str(s) + '.json' | |
outfile_path = 'yonsei_lecture-' + str(y) + '-' + str(s) + '_fixed.json' | |
with open(infile_path, 'r') as infile: | |
json_data = json.load(infile) | |
for row_data in json_data: | |
empty_cell_cnt = 0 | |
for cell_data in row_data: | |
if not cell_data: | |
empty_cell_cnt += 1 | |
if empty_cell_cnt >= 18: | |
continue | |
new_data.append(row_data) | |
with open(outfile_path, 'w') as outfile: | |
json.dump(new_data, outfile) | |
crawl() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# -*- coding: utf-8 -*- | |
import json | |
import requests | |
from bs4 import BeautifulSoup | |
''' all data is string ''' | |
''' if you don't want to save string, you have to change code ''' | |
class Subject(): | |
def __init__(self, hyhg, domain, hakno, bb, sbb): | |
self.hyhg = hyhg | |
self.domain = domain | |
self.hakno = hakno | |
self.bb = bb | |
self.sbb = sbb | |
self.person_list = [] | |
'''' Ex) major_info : 0 (N) major maximum(2-major include) ''' | |
def set_info(self, name, credit, professor, time, place, maximum, applied_num, major_info, maximum1, maximum2, | |
maximum3, maximum4, exchange_possible): | |
self.name = name | |
self.credit = credit | |
self.professor = professor | |
self.time = time | |
self.place = place | |
self.maximum = maximum | |
self.applied_num = applied_num | |
self.major_info = major_info | |
self.maximum1 = maximum1 | |
self.maximum2 = maximum2 | |
self.maximum3 = maximum3 | |
self.maximum4 = maximum4 | |
self.sechange_possible = exchange_possible | |
def add_person(self, rank, mileage, major, num_subject, graduate, first_apply, option1, option2, grade, success): | |
self.person_list.append( | |
Person(rank, mileage, major, num_subject, graduate, first_apply, option1, option2, grade, success)) | |
return | |
class Person(): | |
''' Ex) major : N (N) major or multi major ''' | |
''' (this subject has fixed major student) ''' | |
''' option1 : 총 이수 학점 / 졸업 이수 학점 ''' | |
''' option2 : 직전학기 이수학점 / 학기당 수강학점 ''' | |
def __init__(self, rank, mileage, major, num_subject, graduate, first_apply, option1, option2, grade, success): | |
self.rank = rank | |
self.mileage = mileage | |
self.major = major | |
self.num_subject = num_subject | |
self.graduate = graduate | |
self.first_apply = first_apply | |
self.option1 = option1 | |
self.option2 = option2 | |
self.grade = grade | |
self.success = success | |
def mileage_crawl_2015_2(hyhg, domain, hakno, bb, sbb): | |
subject = Subject(hyhg, domain, hakno, bb, sbb) | |
url = 'http://ysweb.yonsei.ac.kr:8888/curri120601/curri_pop_mileage_20152.jsp' | |
data = {'yshs_hyhg': hyhg, | |
'yshs_domain': domain, | |
'yshs_hakno': hakno, | |
'yshs_bb': bb, | |
'yshs_sbb': sbb} | |
req = requests.post(url, data=data) | |
''' if cannot find all list, switch to html.parser ''' | |
''' = BeautifulSoup(req.text, "html.parser") ''' | |
soup = BeautifulSoup(req.text, 'lxml') | |
table_list = soup.find_all('table') | |
try: | |
subject_info = table_list[1].find_all('tr')[3].find_all('td') | |
subject.set_info(subject_info[1].text, | |
subject_info[2].text, | |
subject_info[3].text, | |
subject_info[4].text, | |
subject_info[5].text, | |
subject_info[6].text, | |
subject_info[7].text, | |
subject_info[8].text, | |
subject_info[9].text, | |
subject_info[10].text, | |
subject_info[11].text, | |
subject_info[12].text, | |
subject_info[13].text) | |
person_info = table_list[2].find_all('tr')[2:] | |
for person_iter in person_info: | |
info = person_iter.find_all('td') | |
subject.add_person(info[0].text, | |
info[1].text, | |
info[2].text, | |
info[3].text, | |
info[4].text, | |
info[5].text, | |
info[6].text, | |
info[7].text, | |
info[8].text, | |
info[9].text) | |
return subject | |
except IndexError as e: | |
''' this subject do not have mileage info ''' | |
return -1 | |
def mileage_crawl_2016_1_to_2017_2(hyhg, domain, hakno, bb, sbb): | |
subject = Subject(hyhg, domain, hakno, bb, sbb) | |
url = 'http://ysweb.yonsei.ac.kr:8888/curri120601/curri_pop_mileage_result01.jsp' | |
data = {'yshs_hyhg': hyhg, | |
'yshs_domain': domain, | |
'yshs_hakno': hakno, | |
'yshs_bb': bb, | |
'yshs_sbb': sbb} | |
req = requests.post(url, data=data) | |
''' if lxml cannot find all list, switch to html.parser ''' | |
''' soup = BeautifulSoup(req.text, 'html.parser') ''' | |
soup = BeautifulSoup(req.text, 'lxml') | |
table_list = soup.find_all('table') | |
try: | |
subject_info = table_list[1].find_all('tr')[3].find_all('td') | |
subject.set_info(subject_info[1].text, | |
subject_info[2].text, | |
subject_info[3].text, | |
subject_info[4].text, | |
subject_info[5].text, | |
subject_info[6].text, | |
subject_info[7].text, | |
subject_info[8].text, | |
subject_info[9].text, | |
subject_info[10].text, | |
subject_info[11].text, | |
subject_info[12].text, | |
subject_info[13].text) | |
person_info = table_list[2].find_all('tr')[2:] | |
for person_iter in person_info: | |
info = person_iter.find_all('td') | |
subject.add_person(info[0].text, | |
info[1].text, | |
info[2].text, | |
info[3].text, | |
info[4].text, | |
info[5].text, | |
info[6].text, | |
info[7].text, | |
info[8].text, | |
info[9].text) | |
return subject | |
except IndexError as e: | |
''' this subject do not have mileage info ''' | |
return -1 | |
def make_output(ys, input_name, output_name, func): | |
''' file read ''' | |
''' need year, semester, domain(?), hakno, bb, sbb ''' | |
with open(input_name, 'r', encoding='utf-8') as f: | |
json_data = json.load(f) | |
data = json_data['data'] | |
result = [] | |
for iter_data in data: | |
print(iter_data) | |
if iter_data[8] == 'No data to display': | |
continue | |
if iter_data[10] == 'TEST000-01-00': | |
continue | |
iter_data[10] = iter_data[10].strip() | |
domain = "H1" | |
hak = iter_data[10].split('-') | |
hakno = hak[0] | |
bb = hak[1] | |
sbb = hak[2][:2] | |
subject = [] | |
subject.append(func(ys, domain, hakno, bb, sbb)) | |
for sub in subject: | |
if sub == -1: | |
print(iter_data) | |
print(iter_data[12] + 'do not have mileage data') | |
continue | |
iter_data[0] = sub.hyhg[:4] | |
iter_data[1] = sub.hyhg[-1] + '학기' | |
iter_data[17] = sub.professor | |
iter_data[18] = sub.time | |
iter_data[19] = sub.place | |
mileage_list = [] | |
for per in sub.person_list: | |
mileage_list.append( | |
[per.rank, per.mileage, per.major, per.num_subject, per.graduate, per.first_apply, per.option1, | |
per.option2, per.grade, per.success]) | |
iter_data.append(mileage_list) | |
iter_data.append(sub.maximum) | |
iter_data.append(sub.applied_num) | |
iter_data.append(sub.major_info) | |
iter_data.append(sub.maximum1) | |
iter_data.append(sub.maximum2) | |
iter_data.append(sub.maximum3) | |
iter_data.append(sub.maximum4) | |
result.append(iter_data) | |
with open(output_name, 'w', encoding='utf-8') as output_file: | |
json.dump(result, output_file, ensure_ascii=False) | |
if __name__ == "__main__": | |
# make_output('20152', 'yonsei_lecture-2015-2학기.json', 'yonsei_mileage-2015-2학기.json', mileage_crawl_2015_2) | |
# make_output('20161', 'yonsei_lecture-2016-1학기.json', 'yonsei_mileage-2016-1학기.json', mileage_crawl_2016_1_to_2017_2) | |
# make_output('20162', 'yonsei_lecture-2016-2학기.json', 'yonsei_mileage-2016-2학기.json', mileage_crawl_2016_1_to_2017_2) | |
# make_output('20171', 'yonsei_lecture-2017-1학기.json', 'yonsei_mileage-2017-1학기.json', mileage_crawl_2016_1_to_2017_2) | |
# make_output('20172', 'yonsei_lecture-2017-2학기.json', 'yonsei_mileage-2017-2학기.json', mileage_crawl_2016_1_to_2017_2) | |
make_output('20181', 'yonsei_lecture-2018-1학기.json', | |
'yonsei_mileage-2018-1학기.json', mileage_crawl_2016_1_to_2017_2) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// 4점, '딱히 없습니다' 일괄 평가 | |
// 강의평가할 과목 선택 후 콘솔(F12) 열어서 복붙엔터 | |
// 수정하고 싶은 값 있는 경우 confirm 취소하고 수정 후 맨 뒤로 가서 완료 버튼 선택 | |
$("input[value='4']").click(); | |
$("textarea").val("딱히 없습니다."); | |
$("td.al.w2tb_noTH > input").click(); | |
$("span#btnSave_span").click(); |
@haruleekim ㅋㄷㅋㄷ
성지순례
What is the link "https://ysweb.yonsei.ac.kr/sugang.Ysv?action=getCurri"??
코딩을 잘 모르는 사람입니다. 현재는 waitcnt가 전부 0으로 뜨는데 볼 방법이 있을까요?
@he450 @MarigoldJ @The-Polar-Bear @sungle3737
Thank you for your interest.
These scripts is wrriten for other project, Doughsam(도우샘), Supporters of Yonsei Course Registration.
The service was designed to provide the summary of mileage history as a pretty graph.
Unfortunately the service is not maintained anymore...
So malfunction for current status is possible.
And that kind of usecase is cheating.
I just wonder how you guys find this place.
Is there any answer...?
양키가되어버린것인가
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
이거머임 충격적이네