| package main | |
| import ( | |
| "bytes" | |
| "fmt" | |
| "net" | |
| "runtime" | |
| "sync" | |
| "sync/atomic" | |
| "time" |
| package main | |
| import ( | |
| "bytes" | |
| "fmt" | |
| "github.com/miekg/dns" | |
| "io/ioutil" | |
| "log" | |
| "net/http" | |
| ) |
| wget -q http://s3.amazonaws.com/alexa-static/top-1m.csv.zip;unzip top-1m.csv.zip; awk -F ',' '{print $2}' top-1m.csv|head -1000 > top-1000.txt; rm top-1m.csv* |
This guide sets up a non-clustered Nutch crawler, which stores its data via HBase. We will not learn how to setup Hadoop et al., but just the bare minimum to crawl and index websites on a single machine.
| #!/usr/bin/python3 | |
| # coding=utf-8 | |
| import re | |
| SSURT_RE = r""" | |
| \A | |
| (?P<scheme> [a-zA-Z] [a-zA-Z0-9+.-]* : )? | |
| (?P<authority> | |
| (?P<slashes> /* ) |
| # 并发查询所有上游DNS服务器 | |
| all-servers | |
| # 本地DNS缓存最小有效期 | |
| min-cache-ttl=3600 | |
| # 本地DNS缓存数目 | |
| cache-size=65535 | |
| # 上游DNS服务器, 可设置多个 | |
| server=8.8.4.4 | |
| #server=8.8.8.8 | |
| server=114.114.114.114 |
| def get_count(q): | |
| count_q = q.statement.with_only_columns([func.count()]).order_by(None) | |
| count = q.session.execute(count_q).scalar() | |
| return count | |
| q = session.query(TestModel).filter(...).order_by(...) | |
| # Slow: SELECT COUNT(*) FROM (SELECT ... FROM TestModel WHERE ...) ... | |
| print q.count() |
| #!/usr/bin/env python | |
| # coding: utf-8 | |
| import os | |
| import sys | |
| from PIL import Image, ImageChops | |
| templates = [] |
| /* | |
| Navicat Premium Data Transfer | |
| Source Server : localhost | |
| Source Server Type : MySQL | |
| Source Server Version : 50542 | |
| Source Host : localhost | |
| Source Database : rule | |
| Target Server Type : MySQL |
