Add background service for news.dpi.dev

how-to-setup-background-job.md

This is how we created news.dpi.dev, a feed of all DPI trainees blog posts on dev.to. We used sidekiq for background processing, but you could easily use another gem like good_job.

app/controllers/news_controller.rb

class NewsController < ApplicationController
  layout "news"
  skip_before_action :authenticate_user!
  before_action { authorize(:news) }

  def index
    @q = policy_scope(DevtoArticle).page(params[:page]).ransack(params[:q])
    @articles = @q.result.includes(:author).default_order
  end
end

app/models/devto_article.rb

# == Schema Information
#
# Table name: devto_articles
#
#  id           :uuid             not null, primary key
#  description  :text
#  published_at :datetime
#  title        :string
#  url          :string
#  created_at   :datetime         not null
#  updated_at   :datetime         not null
#  author_id    :uuid             not null
#  devto_id     :integer
#
# Indexes
#
#  index_devto_articles_on_author_id  (author_id)
#
# Foreign Keys
#
#  fk_rails_...  (author_id => users.id)
#
class DevtoArticle < ApplicationRecord
  include Ransackable

  belongs_to :author, class_name: "User", foreign_key: "author_id"

  scope :default_order, -> { order(published_at: :desc) }
end

app/models/user.rb

class User < ApplicationRecord
  ...
  include Blogable
  ...
end

app/models/concerns/user/blogable/rb

module User::Blogable
  extend ActiveSupport::Concern

  included do
    scope :bloggers, -> { where.not(devto_username: nil)}
    has_many :devto_articles, class_name: "DevtoArticle", foreign_key: "author_id"
    before_save { self.devto_username = devto_username.downcase if devto_username.present? }
  end

  class_methods do
    def fetch_devto_articles
      User.bloggers.each do |u|
        u.fetch_devto_articles
        sleep(1) # hack fix for 429 status code
      end
    end
  end

  def fetch_devto_articles
    return unless devto_username.present?

    DevtoService.fetch_articles({username: devto_username})&.each do |article|
      devto_article = devto_articles.find_or_initialize_by(devto_id: article["id"])
      devto_article.title = article["title"]
      devto_article.description = article["description"]
      devto_article.url = article["url"]
      devto_article.published_at = article["published_timestamp"]
      devto_article.save
    end
  end
end

app/services/devto_service.rb

require 'http'
require 'json'

class DevtoService
  BASE_URL = "https://dev.to/api"

  def self.fetch_articles(params)
    articles_url = "#{BASE_URL}/articles?#{params.to_query}"
    response = HTTP.headers('Accept': 'application/vnd.forem.api-v1+json').get(articles_url)

    if response.status.success?
      body = response.body.to_s
      parsed_response = JSON.parse(body)
      return parsed_response
    else
      puts "Failed to fetch articles for username #{params.dig(:username)}. Response code: #{response.status}"
      return nil
    end
  end
end

Setting up the cron job will be different depending on which background processor you choose to use (sidekiq, goodjob, etc.)

config/schedule.yml

fetch_devto_articles:
  cron: "*/30 * * * *" # execute every 30 minutes
  class: "FetchDevtoArticlesJob"
  queue: news

app/jobs/fetch_devto_articles_job.rb

class FetchDevtoArticlesJob < ApplicationJob
  queue_as :news

  def perform
    User.fetch_devto_articles
  end
end