dom_parser.py

import websocket
# pip install websocket-client
import json
try:
    import thread
except ImportError:
    import _thread as thread
import time
import requests
# pip install requests
 
mm = {
    'id': 0,
    'method': 'Log.enable'
}

mm2 = {
    'id': 1,
    'method': 'Network.enable'
}

method_id = 10
requset_dict = dict()
response_dict = dict()
request_id_dict = dict()
response_body_dict = dict()

# http://localhost:9222/json -> webSocketDebuggerUrl
websocket_url = 'ws://localhost:9222/devtools/page/3F16DFD0A8C59E8B860643E970CB2C1B'


def on_message(ws, message):
    ee = json.loads(message)
    # td element select request
    if 'id' in ee and ee['id'] == 3:
        doc_id = ee['result']['root']['nodeId']
        mm4 = {
            'method': 'DOM.querySelectorAll',
            'params': {
                'nodeId' : doc_id,
                'selector' : 'td'
            }
        }
        print(mm4)
        send_msg(mm4)
    # get td node request
    if 'id' in ee and 'result' in ee and 'nodeIds' in ee['result']:
        print(message)
        ids = ee['result']['nodeIds']
        for ii in ids:
            mm = {
                'method': 'DOM.describeNode',
                'params': {
                    'nodeId' : ii,
                    'depth' : -1
                }
            }
            send_msg(mm)

    #get text in td
    if 'id' in ee and 'result' in ee and 'node' in ee['result'] \
       and 'children' in ee['result']['node']:
        print(ee)
        for cc in ee['result']['node']['children']:
            if cc['nodeName'] == '#text':
                print(cc['nodeValue'])
        
        
    
    # save log(need fix)
    if 'method' in ee and ee['method'] == 'Log.entryAdded':
        eee = ee['params']['entry']
        print('[%s] %s' % (eee['level'], eee['text']))
    # save response
    elif 'method' in ee and ee['method'] == 'Network.responseReceived':
        id = ee['params']['requestId']
        response_dict[id] = ee['params']['response']
        get_response_body(ws, id)
    # save request 
    elif 'method' in ee and ee['method'] == 'Network.requestWillBeSent':
        id = ee['params']['requestId']
        ee['params']['request']['id'] = id
        requset_dict[id] = ee['params']['request']
    # integrate request, response, body
    elif 'id' in ee and 'body' in ee['result']:
        m_id = ee['id']
        request_id = request_id_dict[m_id]
        body = ee['result']['body']
        response_dict[request_id]['body'] = body

        """
        print('--------------------request--------------------')
        print(requset_dict[request_id])
        print('--------------------response--------------------')
        print(response_dict[request_id])
        """
    

def send_msg(dd):
    global method_id
    method_id += 1
    dd['id'] = method_id
    ws.send(json.dumps(dd))
    return dd['id']

    

    
        
def get_response_body(ws, request_id):
    get_response_body_method = {
        'method': 'Network.getResponseBody',
        'params': {
            'requestId' : request_id
        }
    }
    send_msg(get_response_body_method)
    request_id_dict[method_id] = request_id


def on_error(ws, error):
    print(error)
 

def on_close(ws):
    print("### closed ###")

    
def on_open(ws):
    def run(*args):
        ws.send(json.dumps(mm))
        time.sleep(1)
        ws.send(json.dumps(mm2))
        mm3 = {
            'id': 3,
            'method': 'DOM.getDocument'
        }
        ws.send(json.dumps(mm3))

        
        time.sleep(1)
     
    thread.start_new_thread(run, ())
 
 
if __name__ == "__main__":
    
    r = requests.get('http://localhost:9222/json')
    dd = r.json()

    websocket_url = dd[0]['webSocketDebuggerUrl']
    
    websocket.enableTrace(True)
    ws = websocket.WebSocketApp(websocket_url,
                              on_message = on_message,
                              on_error = on_error,
                              on_close = on_close)
    ws.on_open = on_open
    ws.run_forever()