python3, selenium, tmux, japanese fonts and timezone, github, bitbucket. aws linux ec2インスタンス起動時用

clone_all.sh

mkdir -p umihico_gist
git clone git@github.com:umihico/aws.git ./aws
git clone git@github.com:umihico/calc_effective_keymap.git ./calc_effective_keymap
git clone git@bitbucket.org:umihico/dj_webserver.git ./dj_webserver
git clone git@gist.github.com:96b18f481ebc41c8df6e5c54b19ccf78.git ./google_search_backup
git clone git@github.com:umihico/ipo_autoapplier.git ./ipo_autoapplier
git clone git@github.com:umihico/passpacker.git ./passpacker
git clone git@bitbucket.org:umihico/crowdworks.git ./crowdworks
git clone git@bitbucket.org:umihico/private.git ./private
git clone git@bitbucket.org:umihico/scrap_ochan.git ./scrap_ochan
git clone git@github.com:umihico/stenizer.git ./stenizer
git clone git@bitbucket.org:umihico/trading_sanic.git ./trading
git clone git@github.com:umihico/umihico_commons.git ./umihico_commons
git clone git@gist.github.com:f859b0840ca7fc39121cd7a15c63c99b.git ./umihico_gist/aws_cli
git clone git@gist.github.com:5c6718309a9b9e84f7d49863eee019cc.git ./umihico_gist/bitcoin_boyd
git clone git@gist.github.com:784ad69c5c79560e9928f265535989f5.git ./umihico_gist/detect_text
git clone git@gist.github.com:b0aca9c916fa59ea5d253cac6b794ba8.git ./umihico_gist/download_github
git clone git@gist.github.com:4175ce953d550d1115e3a21984bb5bff.git ./umihico_gist/gen_detected_text_pic
git clone git@gist.github.com:279c87001046928811f0341b06203e4d.git ./umihico_gist/get_address_regex
git clone git@gist.github.com:0507ff16c3b733d54ca86effc4c808d6.git ./umihico_gist/get_all_company_urls_wiki
git clone git@gist.github.com:269965e5069ba8884e1ce9b612f0c8a0.git ./umihico_gist/get_amazon_product
git clone git@gist.github.com:5bfaaf53cde33239ff6e3887f7e218e2.git ./umihico_gist/get_anonymous_proxy
git clone git@gist.github.com:43fd5a383c984448f446b0f33610b029.git ./umihico_gist/get_google_image_search_result
git clone git@gist.github.com:107d38fa06ef01810d3059c587ad039c.git ./umihico_gist/get_info_wikipedia
git clone git@gist.github.com:1557a74430b9d238c6f99570e766aa9d.git ./umihico_gist/get_investment_urls_wikipedia
git clone git@gist.github.com:06c9beed6275a165a134944d3d5b1f3f.git ./umihico_gist/get_nomura_shikiho
git clone git@gist.github.com:0d1cd0f20a269f6ccdfad15d284b322a.git ./umihico_gist/get_related_keywords
git clone git@gist.github.com:fc5e17881390c6a3595eeb5acf3130bd.git ./umihico_gist/get_stock_info_yahoo
git clone git@gist.github.com:dac0641e56d2f7cf905f02de36de12a6.git ./umihico_gist/get_stock_metadata
git clone git@gist.github.com:60168c5992a7588339304bcd2fcf51b6.git ./umihico_gist/get_yahoo_investment_chats
git clone git@gist.github.com:c47531bb241623316fd4a25d4e4a988a.git ./umihico_gist/get_yahoo_investment_chats_context
git clone git@gist.github.com:033931bc2832b0049aacd17e497d2e44.git ./umihico_gist/get_yahoo_investment_chat_all
git clone git@gist.github.com:06509f220fd3732f5712e19eb3b34f79.git ./umihico_gist/get_yauc_closed_item
git clone git@gist.github.com:bc6adcd0f66486af609bf4a91229b5be.git ./umihico_gist/get_yauc_closed_list
git clone git@gist.github.com:96b18f481ebc41c8df6e5c54b19ccf78.git ./umihico_gist/google_search
git clone git@gist.github.com:7271eb501564c5a42bdeaa2b3046f733.git ./umihico_gist/neyo
git clone git@gist.github.com:cdd4c5f779f861583c3b3027263e6e53.git ./umihico_gist/requests_threading
git clone git@gist.github.com:56e5c00211a24a9d3a763e1833225927.git ./umihico_gist/scrap_google_image_search
git clone git@gist.github.com:bc8783f40636fed2074ba3af014a697c.git ./umihico_gist/scrap_kindle_highlights
git clone git@github.com:0806b81a2c5143ee3c0ae843f7c676fc.git ./umihico_gist/setup_aws
git clone git@bitbucket.org:umihico/umihico_private.git ./umihico_private
git clone git@github.com:umihico/webscraping_commons.git ./webscraping_commons
git clone git@github.com:umihico/xpath_detecter.git ./xpath_detecter
git pull git@github.com:umihico/aws.git ./aws
git pull git@github.com:umihico/calc_effective_keymap.git ./calc_effective_keymap
git pull git@bitbucket.org:umihico/dj_webserver.git ./dj_webserver
git pull git@gist.github.com:96b18f481ebc41c8df6e5c54b19ccf78.git ./google_search_backup
git pull git@github.com:umihico/ipo_autoapplier.git ./ipo_autoapplier
git pull git@github.com:umihico/passpacker.git ./passpacker
git pull git@bitbucket.org:umihico/private.git ./private
git pull git@bitbucket.org:umihico/scrap_ochan.git ./scrap_ochan
git pull git@github.com:umihico/stenizer.git ./stenizer
git pull git@bitbucket.org:umihico/trading_sanic.git ./trading
git pull git@github.com:umihico/umihico_commons.git ./umihico_commons
git pull git@gist.github.com:f859b0840ca7fc39121cd7a15c63c99b.git ./umihico_gist/aws_cli
git pull git@gist.github.com:5c6718309a9b9e84f7d49863eee019cc.git ./umihico_gist/bitcoin_boyd
git pull git@gist.github.com:784ad69c5c79560e9928f265535989f5.git ./umihico_gist/detect_text
git pull git@gist.github.com:b0aca9c916fa59ea5d253cac6b794ba8.git ./umihico_gist/download_github
git pull git@gist.github.com:4175ce953d550d1115e3a21984bb5bff.git ./umihico_gist/gen_detected_text_pic
git pull git@gist.github.com:279c87001046928811f0341b06203e4d.git ./umihico_gist/get_address_regex
git pull git@gist.github.com:0507ff16c3b733d54ca86effc4c808d6.git ./umihico_gist/get_all_company_urls_wiki
git pull git@gist.github.com:269965e5069ba8884e1ce9b612f0c8a0.git ./umihico_gist/get_amazon_product
git pull git@gist.github.com:5bfaaf53cde33239ff6e3887f7e218e2.git ./umihico_gist/get_anonymous_proxy
git pull git@gist.github.com:43fd5a383c984448f446b0f33610b029.git ./umihico_gist/get_google_image_search_result
git pull git@gist.github.com:107d38fa06ef01810d3059c587ad039c.git ./umihico_gist/get_info_wikipedia
git pull git@gist.github.com:1557a74430b9d238c6f99570e766aa9d.git ./umihico_gist/get_investment_urls_wikipedia
git pull git@gist.github.com:06c9beed6275a165a134944d3d5b1f3f.git ./umihico_gist/get_nomura_shikiho
git pull git@gist.github.com:0d1cd0f20a269f6ccdfad15d284b322a.git ./umihico_gist/get_related_keywords
git pull git@gist.github.com:fc5e17881390c6a3595eeb5acf3130bd.git ./umihico_gist/get_stock_info_yahoo
git pull git@gist.github.com:dac0641e56d2f7cf905f02de36de12a6.git ./umihico_gist/get_stock_metadata
git pull git@gist.github.com:60168c5992a7588339304bcd2fcf51b6.git ./umihico_gist/get_yahoo_investment_chats
git pull git@gist.github.com:c47531bb241623316fd4a25d4e4a988a.git ./umihico_gist/get_yahoo_investment_chats_context
git pull git@gist.github.com:033931bc2832b0049aacd17e497d2e44.git ./umihico_gist/get_yahoo_investment_chat_all
git pull git@gist.github.com:06509f220fd3732f5712e19eb3b34f79.git ./umihico_gist/get_yauc_closed_item
git pull git@gist.github.com:bc6adcd0f66486af609bf4a91229b5be.git ./umihico_gist/get_yauc_closed_list
git pull git@gist.github.com:96b18f481ebc41c8df6e5c54b19ccf78.git ./umihico_gist/google_search
git pull git@gist.github.com:7271eb501564c5a42bdeaa2b3046f733.git ./umihico_gist/neyo
git pull git@gist.github.com:cdd4c5f779f861583c3b3027263e6e53.git ./umihico_gist/requests_threading
git pull git@gist.github.com:56e5c00211a24a9d3a763e1833225927.git ./umihico_gist/scrap_google_image_search
git pull git@gist.github.com:bc8783f40636fed2074ba3af014a697c.git ./umihico_gist/scrap_kindle_highlights
git pull git@github.com:0806b81a2c5143ee3c0ae843f7c676fc.git ./umihico_gist/setup_aws
git pull git@bitbucket.org:umihico/umihico_private.git ./umihico_private
git pull git@github.com:umihico/webscraping_commons.git ./webscraping_commons
git pull git@github.com:umihico/xpath_detecter.git ./xpath_detecter

python_requirements.txt

altgraph
asn1crypto
autopep8
beautifulsoup4
bs4
certifi
cffi
chardet
crypto
cryptography
dill
et-xmlfile
ez-setup
feedparser
flake8
flake8-pep257
future
google
hacking
idna
jaconv
jdcal
llvmlite
lxml
macholib
mccabe
multiprocess
Naked
numba
numpy
oauthlib
olefile
openpyxl
packaging
passlib
pathos
pbr
pefile
pep257
pep8
Pillow
pip-review
pox
ppft
psutil
pubnub
pudb
pycodestyle
pycparser
pycryptodome
pycryptodomex
pyflakes
Pygments
PyInstaller
PyJWT
pyparsing
pyperclip
pyreadline
PySocks
pytesseract
pyttsx
PyYAML
requests
requests-oauthlib
selenium
shellescape
six
tabulate
tendo
tqdm
tweepy
urllib3
urwid
websocket-client
websocket-server
xlrd
XlsxWriter
xlwt
youtube-python
zenhan
gspread
oauth2client
sanic
tinydb

setup_apache.sh

#!/bin/bash -eu
yum -y install httpd
chkconfig httpd on
service httpd start

setup_django.sh

#!/bin/bash -eu
export PYTHONPATH="/home/ec2-user/github"
cd /home/ec2-user/github
git clone git@bitbucket.org:umihico/dj_webserver.git
cd ./dj_webserver/mysite
python manage.py runserver

setup_font.sh

#!/bin/bash -eu
mkdir -p /home/ec2-user/downloads/fonts
cd /home/ec2-user/downloads/fonts
wget https://noto-website-2.storage.googleapis.com/pkgs/Noto-hinted.zip
unzip Noto-hinted.zip
mkdir -p /usr/share/fonts/opentype/noto
cp *otf *ttf /usr/share/fonts/opentype/noto
fc-cache -f -v # optional
rm -rf /home/ec2-user/downloads/fonts

setup_git_service.sh

#!/bin/bash -eu
cp /home/ec2-user/setup_aws/zip.zip /root/.ssh/
cd /root/.ssh/
unzip -P $git_password zip.zip
chmod 400 ./id_rsa
chmod 400 ./id_rsa.pub
eval `ssh-agent`
ssh-add ./id_rsa
git config --global --unset user.email
git config --global --unset user.name
git config --global --add user.email umihico_dummy@users.noreply.github.com
git config --global --add user.name umihico
echo StrictHostKeyChecking no>>~/.ssh/config
chmod 600 ~/.ssh/config
ssh -T git@github.com
ssh -T git@bitbucket.org
mkdir -p /home/ec2-user/github
cd /home/ec2-user/github
source /home/ec2-user/setup_aws/clone_all.sh

setup_proxy.sh

#!/bin/bash -eu
yum -y install squid
yum -y install httpd-tools
htpasswd -b -c /etc/squid/passwd umihicoumihico umihicoumihico
chmod o+r /etc/squid/passwd
cd /etc/squid/
sed -i "s/http_access deny all/http_access allow all/" squid.conf
sed -i "s/http_port 3128/http_port 8604/" squid.conf
sed -i '1ihttp_access allow ncsa_users' squid.conf
sed -i '1iacl ncsa_users proxy_auth REQUIRED' squid.conf
sed -i '1iauth_param basic program /usr/lib64/squid/basic_ncsa_auth /etc/squid/passwd' squid.conf
echo "visible_hostname unknown">>squid.conf
echo "forwarded_for off">>squid.conf
echo "request_header_access X-FORWARDED-FOR deny all">>squid.conf
echo "request_header_access Via deny all">>squid.conf
echo "request_header_access Cache-Control deny all">>squid.conf
service squid start
chkconfig squid on

setup_python.sh

#!/bin/bash -eu
git clone https://github.com/yyuu/pyenv.git /usr/bin/.pyenv
cd /usr/bin/.pyenv
mkdir shims
mkdir versions
chown -R ec2-user:ec2-user /usr/bin/.pyenv
echo 'export PYENV_ROOT="/usr/bin/.pyenv"
if [ -d "${PYENV_ROOT}" ]; then
export PATH=${PYENV_ROOT}/bin:$PATH
eval "$(pyenv init -)"
fi' >> /root/.bashrc
source /root/.bashrc
pyenv install --list
pyenv install 3.6.5
python -V
pyenv global 3.6.5
python -V
pip install --upgrade pip
cd /home/ec2-user/setup_aws
pip install -r python_requirements.txt
export PYTHONPATH="/home/ec2-user/github"

setup_selenium.sh

#!/bin/bash -eu
cd /home/ec2-user/
mkdir -p downloads
cd downloads/
curl https://intoli.com/install-google-chrome.sh | bash
wget https://chromedriver.storage.googleapis.com/2.38/chromedriver_linux64.zip
unzip chromedriver_linux64.zip
# mv chromedriver /usr/local/bin/
mv chromedriver /usr/bin/.pyenv/bin
echo '[CentOS-base]
name=CentOS-6 - Base
mirrorlist=http://mirrorlist.centos.org/?release=6&arch=x86_64&repo=os
gpgcheck=1
gpgkey=http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-6

#released updates
[CentOS-updates]
name=CentOS-6 - Updates
mirrorlist=http://mirrorlist.centos.org/?release=6&arch=x86_64&repo=updates
gpgcheck=1
gpgkey=http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-6

#additional packages that may be useful
[CentOS-extras]
name=CentOS-6 - Extras
mirrorlist=http://mirrorlist.centos.org/?release=6&arch=x86_64&repo=extras
gpgcheck=1
gpgkey=http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-6' > /etc/yum.repos.d/centos.repo
rpm --import http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-6
yum -y install GConf2


pip install selenium

setup_timezone.sh

cp /usr/share/zoneinfo/Japan ./
mv Japan localtime
mv ./localtime /etc/localtime

setup_tor.sh

#!/bin/bash -eu
cd /home/ec2-user/
mkdir -p downloads
cd /home/ec2-user/downloads
wget https://www.torproject.org/dist/tor-0.3.3.9.tar.gz
tar xf tor-0.3.3.9.tar.gz
cd /home/ec2-user/downloads/tor-0.3.3.9
yum -y install libevent libevent-devel
./configure
make
make install
cp /usr/local/etc/tor/torrc.sample /usr/local/etc/tor/torrc
# cd /usr/local/etc/tor

zip.zip