Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
#string should not around ""
#regex no need escape \
[DEFAULT]
ArrayStringSplitter: ,
GoProfEnabled: false
SplitByUrlParameter: p,pn,pid
[cluster]
name: Gopa
[fetch]
shard: 1-10
[Global]
MaxGoRoutine: 2
StoreWebPageTogether: true
[Switch]
ParseUrlsFromSavedFileLog: true
LoadTemplatedFetchJob: false
LoadPendingFetchJobs: true
LoadRuledFetchJob: true
ParseUrlsFromPreviousSavedPage: false
HttpEnabled: true
[BloomFilter]
ItemSize: 100000
[CrawlerRule]
#Name: GopaTask1 #货源
Name: GopaTask2 #车源
#------ExtractLinksFromPage-----------
SkipPageParsePattern: .*?\\.((js)|(css)|(rar)|(gz)|(zip)|(exe)|(bmp)|(jpeg)|(gif)|(png)|(jpg)|(apk))\\b
LinkUrlExtractRegex: <a href="(/\d+/\d+.html)" onclick="ClickCheline #车源
#LinkUrlExtractRegex: <a target="_blank" rel="nofollow" href="(/\d+/\d+.html)" > #货源
LinkUrlExtractRegexGroupIndex: 1
FetchDelayThreshold: 800 #800ms
[RuledFetch]
#UrlTemplate: http://www.chinawutong.com/103.html?pid={id}&f=&t=&lx=&hk=&hl= #货源
UrlTemplate: http://www.chinawutong.com/102.html?pid={id} #车源
From:0
To:550
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.