#Common Commands
- Upgrade all packages with pip
sudo pip freeze --local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 sudo pip install -U
- Split and Concat files in Shell
split -l 10000 new_data_lower.txt
sudo defaults write /Library/Preferences/com.apple.windowserver DisplayResolutionEnabled -bool YES; | |
sudo defaults delete /Library/Preferences/com.apple.windowserver DisplayResolutionDisabled; | |
// by the way, you need to logout and log back in for this to take effect. Or at least that's what | |
// Quartz Debug says. Who knows, maybe it's lying? | |
// P.S. Go to [Apple menu --> System Preferences --> Displays --> Display --> Scaled] after logging | |
// back in, and you'll see a bunch of "HiDPI" resolutions in the list to choose from. |
# input list from csv files | |
file = open('input_file_name.csv', 'r') | |
data = csv.reader(file) | |
urls = list(data) | |
url = [] | |
for i in urls: | |
url.append(i[0]) | |
start_urls = url[:] | |
file.close() |
BOT_NAME = 'so_users' | |
SPIDER_MODULES = ['so_users.spiders'] | |
NEWSPIDER_MODULE = 'so_users.spiders' | |
DEFAULT_ITEM_CLASS = 'so_users.items.SoUsersItem' | |
ITEM_PIPELINES = [ | |
# 'goodread.pipelines.RequiredFieldsPipeline', | |
# # 'dirbot.pipelines.FilterWordsPipeline', | |
'so_users.pipelines.MySQLStorePipeline', |
CREATE TABLE user ( | |
user_id INT Primary Key, | |
userXXXXX | |
updated datetime | |
) DEFAULT CHARSET=utf8; | |
CREATE TABLE follower ( | |
user_id INT, -- References user. | |
follower_id INT, -- References user. | |
PRIMARY KEY (user_id, follower_id), |
from scrapy.item import Item, Field | |
from scrapy.contrib.loader import XPathItemLoader | |
from scrapy.contrib.loader.processor import TakeFirst | |
class XXXXX(Item): | |
item1 = Field() | |
item2 = Field() | |
item3 = Field() |
DROP TABLE IF EXISTS table_name; | |
CREATE TABLE user ( | |
var1 varchar(155) PRIMARY KEY, | |
var2 int , | |
var3 varchar(155), | |
var4 varchar(155), | |
var5 varchar(155), | |
updated datetime | |
) DEFAULT CHARSET=utf8; |
class bcolors: | |
HEADER = '\033[95m' | |
OKBLUE = '\033[94m' | |
OKGREEN = '\033[92m' | |
WARNING = '\033[93m' | |
FAIL = '\033[91m' | |
ENDC = '\033[0m' | |
BOLD = '\033[1m' | |
UNDERLINE = '\033[4m' |