anki_jisho_sentence_scrape.py

import sys
import requests, bs4
import json
from collections import namedtuple

JishoSentence = namedtuple('JishoSentence', ['jp', 'en'])
JishoLookup = namedtuple('JishoLookup', ['json', 'sentences'])

def ac_text(action, params):
    json_ins = None
    if isinstance(params, dict):
        json_ins = ', "params": ' + json.dumps(params)
    elif params == None:
        json_ins = ''
    else:
        json_ins = ', "params": {' + params + '}'

    return '{{ "action": "{}", "version": 6{} }} '.format(action, json_ins)

def ac_request(action, params, to_json=True):
    """
    Sends a request to AnkiConnect.

    Returns:
        The result of the request, as a dictionary.
    """

    res = ac_text(action, params)

    res = requests.post('http://localhost:8765', res)
    if not res.status_code == 200:
        print('Error connecting to AnkiConnect: {}'.format(res.status_code), file=sys.stderr)
        exit(1)

    return json.loads(res.text) if to_json else res

request_queue = []

def ac_batch(action, params):
    global request_queue
    request_queue.append(ac_text(action, params))

def ac_flush():
    global request_queue
    ac_request("multi", {"actions": request_queue})
    request_queue.clear()

def jisho_fetch(term):
    """ 
    Fetch information from Jisho for a specific word. 
    """

    # First get its dictionary entry.
    jisho = json.loads(requests.get('https://jisho.org/api/v1/search/words?keyword={}'.format(term)).text)
    if len(jisho['data']) > 0:
        jisho = jisho['data'][0]
    else:
        return None

    # We scrape the webpage because the Jisho API doesn't give sentences yet.
    r = requests.get('https://jisho.org/search/{} %23sentences'.format(term))

    if not r.status_code == 200:
        print('Error fetching Jisho results: {}'.format(r.status_code), file=sys.stderr)
        exit(1)

    soup = bs4.BeautifulSoup(r.text, 'html.parser')

    sentences = []

    for sentence_content in soup.find_all('div', {'class': 'sentence_content'}):
        sentence_jp = sentence_content.find('ul', {'class': 'japanese_sentence'})
        for f in sentence_jp.find_all('span', {'class': 'furigana'}):
            # Strip furigana from sentence.
            f.decompose()

        sentence_en = sentence_content.find('div', {'class': 'english_sentence'}).find('span', {'class': 'english'})

        sentences.append(JishoSentence(sentence_jp.text.strip(), sentence_en.text.strip()))

    return JishoLookup(jisho, sentences)

if (len(sys.argv) > 2 and sys.argv[1] == '-d'):
    if len(sys.argv) > 2:
        print('--- JSON ---')
        print(json.dumps(ac_request(sys.argv[2], None if len(sys.argv) < 4 else sys.argv[3])))
else:
    notesInDeck = ac_request('findNotes', {'query': 'deck:"JP::KKLC Vocab"'})

    notes_info = ac_request('notesInfo', { 'notes': notesInDeck['result'] })
    print(notes_info)
    
    for n in notes_info['result']:
        n = n['fields']
        jisho = jisho_fetch(n['Word']['value'].split('(')[0])
        req_add = {
            "note": {
                "deckName": "KKLCNew",
                "modelName": "KKLCNewNote",
                "fields": {
                    "word": n['Word']['value'],
                    "reading": n['Reading']['value'],
                    "meaning": n['Meaning']['value'],
                    "kanji_index": n['Kanji index']['value']
                },
                "tags": [ ]
            }
        }
        
        if not jisho == None:
            req_add['note']['fields']['sentence_jp-1'] = jisho.sentences[0].jp if len(jisho.sentences) > 1 else ""
            req_add['note']['fields']['sentence_en-1'] = jisho.sentences[0].en if len(jisho.sentences) > 1 else ""

        for i in range(1, 5 + 1):
            idx_sentence_jp = 'sentence_jp-{}'.format(i)
            idx_sentence_en = 'sentence_en-{}'.format(i)
            if jisho == None or len(jisho.sentences) < i:
                req_add['note']['fields'][idx_sentence_jp] = ""
                req_add['note']['fields'][idx_sentence_en] = ""
            else:
                req_add['note']['fields'][idx_sentence_jp] = jisho.sentences[i - 1].jp
                req_add['note']['fields'][idx_sentence_en] = jisho.sentences[i - 1].en
                
        print(req_add)

        ac_request('addNote', req_add)
        
# vim: set ft=python