lowweihong

from websocket import create_connection
import json
import pprint

# Initialize the headers needed for the websocket connection
headers = json.dumps({
    'Date': 'Sun, 27 Aug 2017 09:42:15 GMT',
    'Connection': 'upgrade',
    'Host': 'www.kolumbus.no',
    'Origin': 'https://www.kolumbus.no',

lowweihong

import requests

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
           "Origin": "https://www2.sgx.com",
           "Referer": "https://www2.sgx.com/securities/securities-prices"}

# Start downloading stocks info from sgx
req = requests.get("https://api.sgx.com/securities/v1.1?excludetypes=bonds&params=nc,adjusted-vwap,b,bv,p,c,change_vs_pc,change_vs_pc_percentage,cx,cn,dp,dpc,du,ed,fn,h,iiv,iopv,lt,l,o,p_,pv,ptd,s,sv,trading_time,v_,v,vl,vwap,vwap-currency",
                   headers=HEADERS)

lowweihong

import gspread
from oauth2client.service_account import ServiceAccountCredentials
import pandas as pd
import requests
from bs4 import BeautifulSoup 

# Access credentials for google sheet and access the google sheet
scope = ['https://spreadsheets.google.com/feeds',
         'https://www.googleapis.com/auth/drive']

lowweihong

import gspread
from oauth2client.service_account import ServiceAccountCredentials
import pandas as pd
import requests
from bs4 import BeautifulSoup 

# Access credentials for google sheet and access the google sheet
scope = ['https://spreadsheets.google.com/feeds',
         'https://www.googleapis.com/auth/drive']

lowweihong

from pyspider.libs.base_handler import *


class Handler(BaseHandler):
    crawl_config = {
    }
    
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
               "Host": "www.similarweb.com",
               "Upgrade-Insecure-Requests": "1",

lowweihong

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
import requests
import pandas as pd
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

def get_pdf(url, folder, filename):
    """

lowweihong

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
import requests
import pandas as pd
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

# Downloading today data from SGX using SGX API
today = datetime.now()

lowweihong

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
import requests
import pandas as pd
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

gauth = GoogleAuth()
# Try to load saved client credentials

lowweihong

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
import requests
import pandas as pd
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

def create_folder(drive, folder_name, parent_folder_id):
    """ 

lowweihong

from bs4 import BeautifulSoup
import wget
soup = BeautifulSoup(r.text, 'lxml')

for node_link in soup.select('tr > td:nth-child(2) > a:nth-child(3)'):
   url = node_link['href']
   if is_downloadable(url):
        wget.download(url,
                      './data/' + url.split('&file=')[-1].split('&format')[0] + '.mid')