donfanning

var phantom = require('phantom');

var Crawler = require("simplecrawler");
var mycrawler = Crawler.crawl("http://www.example.com/");
mycrawler.maxDepth = 3;
mycrawler.interval = 500;
mycrawler.addFetchCondition(function(parsedURL) {
    if (parsedURL.path.match(/\.(css|jpg|pdf|docx|js|png|ico)/i)) {
        // console.log("Ignored ",parsedURL.path);
        return false;

donfanning

require_relative "xpath_crawler"
require_relative "parser"

module ShareCrawler
  class << self
    def get(crawler)
      xpath_crawler = XPathCrawler.new(crawler["url"])
      parsed = { "warning" => 0 }
      parsed["value"] = Parser.parse_value(xpath_crawler.parse(crawler["xpath_value"]))
      parsed["date"] = Parser.parse_date(xpath_crawler.parse(crawler["xpath_date"]))

donfanning

# Webスクレイピングのサンプル(るりまサーチを例にして)
require 'open-uri' # ダウンロード用のライブラリ
require 'nokogiri' # パース用のライブラリ

Encoding.default_external = "UTF-8" # 内部のエンコーディングをUTF-8にしておく
keyword = "include" # 検索キーワード

# 検索用URLを作成
url = "https://docs.ruby-lang.org/ja/search/query:#{keyword}/"
# ダウンロード処理(charsetに対象サイトのエンコーディングが入る)

donfanning

// Create Docker
// Run Docker 
// Grab Twitter
// save to firebase 
// send link 
// shut it down 

const TwitterCrawler = require('twitter-crawler');
const fs = require('fs');
const log = require('winston');

donfanning

var Crawler = require("simplecrawler");
var Url = require("url");

var target = "https://graph.facebook.com/ledzeppelin/feed?access_token=1597581200507009%7Ce749be55ea86249f92ae56b081c37b38&fields=from%2Cmessage%2Ccreated_time%2Ctype%2Clink%2Ccomments.summary(true)%2Clikes.summary(true)%2Cshares&since=2016-07-11&until=2016-07-14&limit=10";
var url = Url.parse(target);

var crawler = new Crawler(url.host);
crawler.initialPath = url.path;
crawler.initialPort = 443;
crawler.initialProtocol = "https";

donfanning

#!/usr/bin/env ruby

require 'uri'
require 'nokogiri'
require 'mechanize'
require 'logger'

trap('INT') { @crawler.report; exit }

class Crawler

donfanning

var cheerio = require('cheerio');
var Crawler = require('simplecrawler');

var initialTopic = 'SpaceX';
var blacklist = ["#", "/w/", "/static/", "/api/", "/beacon/", "File:",
                 "Wikipedia:", "Template:", "MediaWiki:", "Help:", "Special:",
                 "Category:", "Portal:", "Main_Page", "Talk:", "User:",
                 "User_talk:", "Template_talk:", "Module:"]; //useless special cases from wikipedia

var url = '/wiki/' + initialTopic;

donfanning

var cheerio = require('cheerio');
var Crawler = require('simplecrawler');

var initialTopic = 'SpaceX';
var blacklist = ["#", "/w/", "/static/", "/api/", "/beacon/", "File:",
                 "Wikipedia:", "Template:", "MediaWiki:", "Help:", "Special:",
                 "Category:", "Portal:", "Main_Page", "Talk:", "User:",
                 "User_talk:", "Template_talk:", "Module:"]; //useless special cases from wikipedia

var url = '/wiki/' + initialTopic;

donfanning

#!/usr/bin/ruby

require 'capybara'
require 'capybara/dsl'
require 'capybara/poltergeist'
require 'nokogiri'
require 'open-uri'


Capybara.configure do |config|

donfanning

This crawler gets all posts of a given Facebook group plus all events from a set of given Facebook pages.

Things todo:

• crawl multiple Facebook groups