msguner

"""Helper module for logging.

Example Usage:

from log import get_logger, get_out_dir_of_logger

LOG = get_logger(__name__)
LOG.info('Logging to %s' % get_out_dir_of_logger(LOG))
"""

msguner

#
# This small example shows you how to access JS-based requests via Selenium
# Like this, one can access raw data for scraping, 
# for example on many JS-intensive/React-based websites
#

from time import sleep

from selenium import webdriver
from selenium.webdriver import DesiredCapabilities

msguner

# see rkengler.com for related blog post
# https://www.rkengler.com/how-to-capture-network-traffic-when-scraping-with-selenium-and-python/

import json
import pprint

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

capabilities = DesiredCapabilities.CHROME

msguner

import re

def replace_json_placeholders(json, values):
  # find all placeholders
  placeholders = re.findall('<[\w ]+>', json)
  # clear_placeholders = list(map(lambda x: x.replace('<', '').replace('>', ''), placeholders))

  assert len(placeholders) == len(values), "Please enter the values of all placeholders."

  # replaces all placeholders with values

msguner

def get_keys(dl, key_list):
    if isinstance(dl, dict):
        for k, v in dl.items():
            key_list.append(k)
            if isinstance(v, list) or isinstance(v, dict):
                get_keys(v, key_list)
    if isinstance(dl, list):
        for i in dl:
            if isinstance(i, list) or isinstance(i, dict):
                get_keys(i, key_list)

msguner

def transformJsonKeys(obj, func=(lambda x: x)):
    if isinstance(obj, list):
        return [transformJsonKeys(element, func) for element in obj]
    elif isinstance(obj, dict):
        return {func(key): transformJsonKeys(value, func) for key, value in obj.items()}
    else:
        return obj


dic = {'k1': 1, 'k2': 2, 'k3': [{'k31': 31, 'k2': 32, 'k33': 33}]}