mylamour

import csv
import json
import os
from urlparse import urlparse

csvfile = open('url.csv', 'r')
jsonfile = open('test.json', 'w')

fieldnames = ("@id","sourceUrl")
reader = csv.DictReader( csvfile, fieldnames)

mylamour

import luigi
import psycopg2

class QueryBackToTmp(luigi.Task):
    def run(self):
            conn_string = "host='ec2-54-zzz-xxx-yyy.cn-north-1.compute.amazonaws.com.cn' " \
				     "dbname='musicbrainz' " \
				     "user='postgres' " \
				     "password='password'"

mylamour

import psycopg2
import psycopg2.extras
import sys
import json
import uuid

def main():
	conn_string = "host='ec2-54-xxx-yyy-zzz.cn-north-1.compute.amazonaws.com.cn' " \
				  "dbname='musicbrainz' " \
				  "user='postgres' " \

mylamour

##!/bin/bash  

step=1

for (( i = 0; i < 60; i=(i+step) )); do
   # ls ~/tmp/javfor.me/ | wc -l
   # ls ~/tmp/javfor.me/ | grep -E "html\."
        ls -R ~/jav/ | wc -l
        
        sleep $step

mylamour

• use crontab -e to define a event every minuter,due to the script was exahust 1 min , so it's mean this log can be ouput every second
• and you can see what you want , just change the listEverySecond.sh , you can easyliy get file info , and something else

1. just insert this string : */1 * * * * /bin/bash ~/listEverySecond.sh >> cat.log
2. just make a test, in your dest dir : wget -m -p -c http://your.test.domain.name
3. log view (In a new terminal window) : tail -f cat.log

mylamour

#!/bin/sh

for((i=0;i<100;i++))

do

     sleep 0.1

     echo $i | dialog --title 'Copy' --gauge 'Backp file from postgresql!' 10 70 0

mylamour

#这个是真奇怪，从pg里拿到的数据，怎么转换也不行，最后只能用这种办法修改的，
# s.decode("UTF-8").encode("GBK")  之前爬虫时可以用，但是现在这个场景不适合。

import json

with open('/home/ubuntu/origin/musicgroup.json') as origin, open('/home/ubuntu/json/musicgroup.json','w') as dest:
    for i in origin.readlines():
        t = json.loads(i)
        dest.write(json.dumps(t, ensure_ascii=False).encode('utf8') + '\n')           
    

mylamour

python:

• python2 -m SimpleHTTPServer
• python3 -m http.server
• twistd -n web -p 8000 --path .

ruby:

• ruby -rwebrick -e "WEBrick::HTTPServer.new(:Port => 8888, :DocumentRoot => Dir.pwd).start"
• ruby -run -ehttpd . -p8000

mylamour

• name="UniProt+Fastaheader"

`/^>[^\|]*\|([^\|]*)\|.*OS=([^=]*).*GN=([^ ]*).*$/g`

Matches UniProt accessionnumber, genename and organism in a UniProt fasta header

• name="E-mail+validator+for+International+Domain"

mylamour

$ git config --global --unset https.proxy
$ git config --global http.proxy	

$ python -m pip --proxy http://127.0.0.1:1080 install --upgrade pip

$ npm config set proxy http://127.0.0.1:1080
$ npm config set https-proxy http://127.0.0.1:1080