Skip to content

Instantly share code, notes, and snippets.

@HunnyJummani
HunnyJummani / infinite_scrolling_web_scraper.rb
Created December 23, 2019 13:37
Infinite scrolling web scraping in Kimurai - Ruby on Rails
require 'kimurai'
class WebScrapper < Kimurai::Base
@name = "web_scrapper_spider"
@engine = :selenium_chrome
@start_urls = ["https://metaruby.com/"]
@config = {
user_agent: "Chrome/68.0.3440.84"
}
@HunnyJummani
HunnyJummani / static_web_scraper.rb
Created December 23, 2019 13:40
Static Web Scraping- Kimurai ( Ruby on Rails )
class WebScrapper < Kimurai::Base
@name = "web_scrapper_spider"
@engine = :mechanize
@start_urls = ["https://metaruby.com/"]
@config = {
user_agent: "Chrome/68.0.3440.84"
}
def parse(response, url:, data: { })
blogs = []
# Install chromium-browser and firefox
sudo apt install -q -y chromium-browser firefox
# ChromeDriver:
cd /tmp && wget https://chromedriver.storage.googleapis.com/2.44/chromedriver_linux64.zip
sudo unzip chromedriver_linux64.zip -d /usr/local/bin
rm -f chromedriver_linux64.zip
# Firefox Drivers:
# id => unique identifier for a channel so use UUID as recommended by GOOGLE CALENDAR API
# type => set to web_hook
# address => webhook callback URL to receive notifications.
# callback_url => POST request to receive notifications in the web app.
# token(optional) => arbitrary string value used as channel token.
channel = Google::Apis::CalendarV3::Channel.new(
id: uuid,
type: 'web_hook',
address: callback_url(host: YOUR_HOST),
token: { user_id: user_id }.to_json
# id => unique identifier for channel ie UUID
# resource_id => unique indentifier for a resource to be watched.
calendar.stop_channel(
Google::Apis::CalendarV3::Channel.new(
id: channel_id,
resource_id: resource_id
)
)
# client => Any Google API client
# refresh_token as token in params is mandatory.
uri = URI('https://accounts.google.com/o/oauth2/revoke')
params = { token: client.refresh_token }
uri.query = URI.encode_www_form(params)
response = Net::HTTP.get(uri)
# will get a {} if access is revoked and errors if any.
result = JSON.parse(response)
require 'send_otp'
class Otp
# You can configure it in credentials/secrets as per your requirement and initialize both here.
AUTH_KEY = MSG_91_AUTH_KEY_HERE
SENDER_ID = MSG_91_SENDER_ID_HERE
attr_accessor :phone_no, :sendotp
def initialize(phone_no)
class ApplicationController < ActionController::Base
before_action :configure_permitted_parameters, if: :devise_controller?
protected
def configure_permitted_parameters
added_attrs = [:username, :email, :password, :password_confirmation, :remember_me]
devise_parameter_sanitizer.permit :sign_up, keys: added_attrs
devise_parameter_sanitizer.permit :account_update, keys: added_attrs
end
# Here Devise model is your User model.
class DeviseModel < ActiveRecord::Base
attr_accessor :login
def login
@login || self.username || self.email
end
def email_required?
#Example: Pure-Object-Oriented
## returns String as class
"Hello".class
## returns Integer as class
12.class
#Example: Dynamically typed
## Print 1 - 5 on terminal.