This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class CreateCrawledUrls < ActiveRecord::Migration | |
def change | |
create_table :crawled_urls do |t| | |
t.string :url_scheme, null: false | |
t.text :host, null: false | |
t.text :path, null: false | |
t.text :fragment | |
t.text :query_strings | |
t.datetime :last_parsed_at | |
t.datetime :last_check_at |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class NegativeExpression < ActiveRecord::Base | |
# Scope que verifica se existe alguma | |
# expressão negativa para o dominio (host) informado | |
scope :expressions_for, -> (host) { | |
where("lower(?) ~* ANY(domains)", host) | |
} | |
# Scope que faz um regex match no campo expressions | |
# verificando se o path informado retornando | |
# todas as expressões que passou no match |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'rails_helper' | |
RSpec.describe NegativeExpression, :type => :model do | |
describe ".expressions_for" do | |
subject { NegativeExpression.expressions_for 'http://example.com' } | |
context "should get expressions" do | |
context "when have a domain expression" do | |
before do |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Arquivo de migração gerado pelo: rails g model NegativeExpression | |
class CreateNegativeExpressions < ActiveRecord::Migration | |
def change | |
create_table :negative_expressions do |t| | |
t.text :domains, array: true, null: false | |
t.text :expressions, array: true, null: false | |
t.timestamps null: false | |
end | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module Crawler | |
class Web | |
attr_reader :valid_urls | |
# Metodo de classe que já coleta e retorna um array unico | |
# com todos os links percorridos dentro do site | |
def self.collect_links_from url | |
_ = new url | |
_.collect_links! | |
_.valid_urls.uniq |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "rails_helper" | |
RSpec.describe Crawler::Web, :type => :service do | |
describe ".collect_links_from" do | |
let(:collected_links) { | |
[ | |
'http://www.example.com/page_link_1.html', | |
'http://www.example.com/page_link_2.html', | |
'http://www.example.com/page_link_3.html', | |
'http://www.example.com/page_link_4.html', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# user system total real | |
# MetaInspector: 1.150000 0.040000 1.190000 (1.241086) | |
# Nokogiri #xpath: 0.010000 0.000000 0.010000 (0.011218) | |
# Nokogiri #css 0.010000 0.000000 0.010000 (0.010822) | |
# Nokogiri #search 0.010000 0.000000 0.010000 (0.010091) | |
require 'open-uri' | |
namespace :benchmarks do | |
desc 'MetaInspector vs Nokogiri' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Use Gists to store code you would like to remember later on | |
console.log(window); // log the "window" object to the console |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#config/initializers/interceptor.rb | |
require "#{Rails.root}/lib/development_mail_interceptor" | |
ActionMailer::Base.register_interceptor(DevelopmentMailInterceptor) if Rails.env.development? | |
#lib/development_mail_interceptor.rb | |
class DevelopmentMailInterceptor | |
def self.delivering_email(message) | |
message.subject = "[#{message.to}] #{message.subject}" | |
message.to = `git config --global user.email`.strip | |
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class ResquedDeviseMailer < Devise::Mailer | |
include Resque::Mailer | |
private | |
# Configure default email options | |
def devise_mail(record, action) | |
record = hack_record(record) | |
@scope_name = Devise::Mapping.find_scope!(record) |