Sho Shimauchi shiumachi

## cosine_similarities.py
import pandas as pd
import numpy as np
import MeCab
from collections import Counter
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

m = MeCab.Tagger("-Ochasen")
m2 = MeCab.Tagger("-Owakati")

## get_socks_proxy_command.sh
PROFILE=your_profile
INSTANCE_NAME=your_instance_name
SSH_KEYPATH=your_ssh_key_path
PUBLIC_HOSTNAME=`aws --profile ${PROFILE} ec2 describe-instances | jq -r ".Reservations[] | select(.Instances[0].Tags[].Value == \"${INSTANCE_NAME}\") | .Instances[0] | .PublicDnsName"`

echo "establish SOCKS proxy"
echo "ssh -i ${SSH_KEYPATH} -D 8157 -q ec2-user@${PUBLIC_HOSTNAME}"

## get_instance_hostname_and_ipaddress.sh
PROFILE=your_profile
INSTANCE_NAME=your_instance_name
aws --profile ${PROFILE} ec2 describe-instances | jq -r ".Reservations[] | select(.Instances[0].Tags[].Value == \"${INSTANCE_NAME}\") | .Instances[0] | {PrivateDnsName: .PrivateDnsName, PrivateIpAddress: .PrivateIpAddress, PublicDnsName: .PublicDnsName, PublicIpAddress: .PublicIpAddress}"

## dirs_compressor.py
# dirs_compressor.py
#
# Usage:
# $ python dirs_compressor.py targed_dir
#

import sys
import os
import os.path
import logging

## tips_python_web_crawler.md

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                shiumachi
                / tips_python_web_crawler.md
            
            
              Last active
              January 4, 2017 01:17
            
              
                PythonでのWebクローラ作成時に学んだことメモ
              
          
    例外

try:
except (ConnectionError, ChunkedEncodingError, TooManyRedirects, NewConnectionError) as e:
                logging.warn("Skip URL {} Reason: {}".format(url, e))

SQLite


## hive_many_columns.py
COLS=100000

with open("/tmp/create_table_with_many_columns.hql", "w") as f:

    f.write("CREATE TABLE many_cols_tbl (\n")

    for i in xrange(COLS):
        f.write("id%d INT,\n" % (i,))

    f.write("last_id INT);\n")

## create_ansible_directory_layout.sh
#!/bin/bash
# Create Ansible directory layaout based on Ansible Documentation http://docs.ansible.com/ansible/playbooks_best_practices.html
#

# inventory file for production servers
touch production

# inventory file for staging environment
touch staging

## sentence_generator.py
#! /usr/bin/env python3

import random
from wsgiref import simple_server

誰 = ['太郎', '二郎', '花子']
どこ = ['東京', '大阪', '名古屋']
どうした = ['泳いだ', '走った', '仕事した']

def pick(l):

## hipchatdump2csv.py
# coding=utf-8
#
# hipchatdump2csv.py
#
# convert hipchat message logs from json to csv.
# csv format:
#     from.name, from.user_id, date, message
#
# usage: python convert.py
#

## match_literal_text.py
import re

# ref: Regular Expression Cookbook p.26

text = """!\"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
aaa
bbb
"""
	import pandas as pd
	import numpy as np
	import MeCab
	from collections import Counter
	from sklearn.feature_extraction.text import CountVectorizer
	from sklearn.feature_extraction.text import TfidfVectorizer

	m = MeCab.Tagger("-Ochasen")
	m2 = MeCab.Tagger("-Owakati")
	PROFILE=your_profile
	INSTANCE_NAME=your_instance_name
	SSH_KEYPATH=your_ssh_key_path
	PUBLIC_HOSTNAME=`aws --profile ${PROFILE} ec2 describe-instances \| jq -r ".Reservations[] \| select(.Instances[0].Tags[].Value == \"${INSTANCE_NAME}\") \| .Instances[0] \| .PublicDnsName"`

	echo "establish SOCKS proxy"
	echo "ssh -i ${SSH_KEYPATH} -D 8157 -q ec2-user@${PUBLIC_HOSTNAME}"
	PROFILE=your_profile
	INSTANCE_NAME=your_instance_name
	aws --profile ${PROFILE} ec2 describe-instances \| jq -r ".Reservations[] \| select(.Instances[0].Tags[].Value == \"${INSTANCE_NAME}\") \| .Instances[0] \| {PrivateDnsName: .PrivateDnsName, PrivateIpAddress: .PrivateIpAddress, PublicDnsName: .PublicDnsName, PublicIpAddress: .PublicIpAddress}"
	# dirs_compressor.py
	#
	# Usage:
	# $ python dirs_compressor.py targed_dir
	#

	import sys
	import os
	import os.path
	import logging
	COLS=100000

	with open("/tmp/create_table_with_many_columns.hql", "w") as f:

	f.write("CREATE TABLE many_cols_tbl (\n")

	for i in xrange(COLS):
	f.write("id%d INT,\n" % (i,))

	f.write("last_id INT);\n")
	#!/bin/bash
	# Create Ansible directory layaout based on Ansible Documentation http://docs.ansible.com/ansible/playbooks_best_practices.html
	#

	# inventory file for production servers
	touch production

	# inventory file for staging environment
	touch staging
	#! /usr/bin/env python3

	import random
	from wsgiref import simple_server

	誰 = ['太郎', '二郎', '花子']
	どこ = ['東京', '大阪', '名古屋']
	どうした = ['泳いだ', '走った', '仕事した']

	def pick(l):
	# coding=utf-8
	#
	# hipchatdump2csv.py
	#
	# convert hipchat message logs from json to csv.
	# csv format:
	# from.name, from.user_id, date, message
	#
	# usage: python convert.py
	#
	import re

	# ref: Regular Expression Cookbook p.26

	text = """!\"#$%&'()*+,-./:;<=>?@[\]^_`{\|}~
	aaa
	bbb
	"""