Skip to content

Instantly share code, notes, and snippets.

@jnunemaker
Last active September 30, 2022 13:10
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jnunemaker/8353d0dad45666743c95e3bd83f55bd2 to your computer and use it in GitHub Desktop.
Save jnunemaker/8353d0dad45666743c95e3bd83f55bd2 to your computer and use it in GitHub Desktop.
diff --git a/Gemfile b/Gemfile
index e3a71e1..34ba75c 100644
--- a/Gemfile
+++ b/Gemfile
@@ -107,3 +107,4 @@ gem "pundit"
gem "rack-canonical-host"
gem "ruby-readability"
gem "rinku", require: "rails_rinku"
+gem "virtus"
diff --git a/app/controllers/articles_controller.rb b/app/controllers/articles_controller.rb
index 92e0108..f5b7d45 100644
--- a/app/controllers/articles_controller.rb
+++ b/app/controllers/articles_controller.rb
@@ -9,7 +9,17 @@ def show
def click
@article = Article.find(params[:id])
- # TODO: Store the click in the database
+
+ ErrorReporter.rescue_and_report do
+ # TODO: Batch these in memory at some point and insert in batches.
+ Click.create({
+ article_id: @article.id,
+ user_id: current_user&.id,
+ ip: request.remote_ip,
+ created_at: Time.zone.now,
+ })
+ end
+
redirect_to @article.url, allow_other_host: true
end
end
diff --git a/app/jobs/click_hour_aggregate_job.rb b/app/jobs/click_hour_aggregate_job.rb
new file mode 100644
index 0000000..5a92aa8
--- /dev/null
+++ b/app/jobs/click_hour_aggregate_job.rb
@@ -0,0 +1,11 @@
+class ClickHourAggregateJob < ApplicationJob
+ queue_as :aggregates
+
+ def perform
+ now = Time.now.utc
+ # TODO: Store last successful aggregation and aggregate from that point
+ # instead of past few hours. Then use the last aggregation time to determine
+ # whether to use live or aggregate data in the click count for articles.
+ ClickHourAggregate.rollup(from: now - 2.hours, to: now)
+ end
+end
diff --git a/app/jobs/clicks_maintenance_job.rb b/app/jobs/clicks_maintenance_job.rb
new file mode 100644
index 0000000..379b476
--- /dev/null
+++ b/app/jobs/clicks_maintenance_job.rb
@@ -0,0 +1,13 @@
+class ClicksMaintenanceJob < ApplicationJob
+ queue_as :maintenance
+
+ def perform
+ ErrorReporter.rescue_and_report do
+ Click.partition.premake(3)
+ end
+
+ ErrorReporter.rescue_and_report do
+ Click.partition.retain(14)
+ end
+ end
+end
diff --git a/app/models/article.rb b/app/models/article.rb
index 1d91d06..d3c3b81 100644
--- a/app/models/article.rb
+++ b/app/models/article.rb
@@ -1,4 +1,7 @@
class Article < ApplicationRecord
+ # The time at which we started tracking and aggregating clicks.
+ AGGREGATION_START_TIME = Time.utc(2022, 9, 26, 11).freeze
+
belongs_to :site
scope :by_published_at, -> { order(published_at: :desc) }
@@ -6,4 +9,51 @@ class Article < ApplicationRecord
validates :title, presence: true
validates :url, presence: true, uniqueness: true, format: { with: URI::regexp(%w{http https}) }
validates :published_at, presence: true
+
+ # The aggregated data for this article (click count and latest timestamp).
+ def aggregated_data
+ return @aggregated_data if defined?(@aggregated_data)
+
+ current_hour = Time.now.utc.beginning_of_hour
+ sql = SQL.new <<~SQL.squish, article_id: id, current_hour: current_hour
+ SELECT
+ sum(count) as count,
+ max(ts) as max_ts
+ FROM #{ClickHourAggregate.table_name}
+ WHERE
+ article_id = :article_id AND
+ ts < :current_hour
+ SQL
+
+ row = sql.hash_results.first || {}
+ max_ts = row["max_ts"]
+
+ @aggregated_data = {
+ count: row["count"] || 0,
+ max_ts: max_ts,
+ }
+ end
+
+ # The count of clicks that have been aggregated.
+ def aggregated_count
+ aggregated_data.fetch(:count)
+ end
+
+ # The maximum timestamp of any aggregated data.
+ def aggregated_max_ts
+ aggregated_data.fetch(:max_ts)
+ end
+
+ # Sums up counts that have not been aggregated yet.
+ def live_count
+ return @live_count if defined?(@live_count)
+
+ start = (aggregated_max_ts ? aggregated_max_ts + 1.hour : AGGREGATION_START_TIME)
+ @live_count = Click.count(article_id: id, created_at: (start..Time.now.utc))
+ end
+
+ # Sums up aggregated counts and live counts that have not been aggregated yet.
+ def click_count
+ aggregated_count + live_count
+ end
end
diff --git a/app/models/click.rb b/app/models/click.rb
new file mode 100644
index 0000000..bbf01e1
--- /dev/null
+++ b/app/models/click.rb
@@ -0,0 +1,182 @@
+class Click
+ class Row
+ include Virtus.value_object
+
+ attribute :user_id, Integer
+ attribute :article_id, Integer
+ attribute :ip, String
+ attribute :created_at, DateTime
+
+ delegate :site, to: :article
+
+ def article
+ return if article_id.blank?
+ return @article if defined?(@article)
+
+ @article = Article.find_by_id(article_id)
+ end
+
+ def user
+ return if user_id.blank?
+ return @user if defined?(@user)
+
+ @user = User.find_by_id(user_id)
+ end
+ end
+
+ # Name of the parent table that the partitions inherit from.
+ def self.table_name
+ "clicks".freeze
+ end
+
+ # Build a partition instance for a given time.
+ def self.partition(time = Time.now.utc)
+ PartitionByDay.new(table_name, time)
+ end
+
+ # Public: Create one or more adapter request logs from an Array of Hashes.
+ #
+ # Note: This method assumes the partition is already created.
+ # Use partition(time).create if it does not exist.
+ # In production, jobs should automatically create new partitions.
+ #
+ # Returns Result.
+ def self.create_many(*rows)
+ Result.new {
+ insert_rows = rows.flatten.map do |row|
+ [
+ row[:user_id].presence || SQL::NULL,
+ row[:article_id],
+ row[:ip],
+ row.fetch(:created_at).utc,
+ ]
+ end
+
+ SQL.run <<~SQL.squish, rows: SQL::ROWS(insert_rows)
+ INSERT INTO #{table_name} (user_id, article_id, ip, created_at)
+ VALUES :rows
+ SQL
+
+ nil
+ }
+ end
+
+ # Public: Create an adapter request log from a Hash.
+ #
+ # Note: This method assumes the partition is already created.
+ # Use partition(time).create if it does not exist.
+ # In production, jobs should automatically create new partitions.
+ #
+ # Returns Result.
+ def self.create(attributes = {})
+ create_many([attributes])
+ end
+
+ # TODO: See how to use pagy for this pagination instead of custom.
+ #
+ # Public: Paginate adapter request logs.
+ #
+ # page - The Integer page (default: 1).
+ # per_page - The Integer per_page (default: 20).
+ # site_id - The Integer site_id to filter returned logs for.
+ # user_id - The Integer user_id to filter returned logs for.
+ #
+ # Returns PaginateResponse.
+ def self.paginate(page: 1, per_page: 20, site_id: nil, user_id: nil, article_id: nil, created_at: nil)
+ page ||= 1
+ per_page ||= 20
+ page = page.to_i
+ per_page = per_page.to_i
+
+ raise ArgumentError, "page must be >= 1 (was #{page})" unless page >= 1
+ raise ArgumentError, "per_page must be >= 1 (was #{per_page})" unless per_page >= 1
+
+ limit = per_page + 1
+ offset = (page - 1) * per_page
+
+ sql = build_sql(
+ select: "#{table_name}.*",
+ site_id: site_id,
+ user_id: user_id,
+ article_id: article_id,
+ created_at: created_at
+ )
+ sql.add "ORDER BY created_at DESC"
+ sql.add "LIMIT :limit OFFSET :offset", limit: limit, offset: offset
+
+ PaginateResponse.new({
+ page: page,
+ per_page: per_page,
+ has_next_page: sql.hash_results.slice!(per_page, 1).present?,
+ rows: sql.hash_results.map { |row| Row.new(row) },
+ })
+ end
+
+ # Private: Count the number of adapter request logs. Only use this in tests.
+ #
+ # Returns Integer number of logs.
+ def self.count(article_id: nil, site_id: nil, user_id: nil, created_at: nil)
+ build_sql(
+ select: "COUNT(*)",
+ article_id: article_id,
+ site_id: site_id,
+ user_id: user_id,
+ created_at: created_at
+ ).value
+ end
+
+ # Private: Return the last adapter request log row. Only use this in tests.
+ #
+ # Returns a Row if found else nil.
+ def self.last
+ rows = SQL.hash_results <<~SQL.squish
+ SELECT * FROM #{table_name} ORDER BY created_at DESC LIMIT 1
+ SQL
+
+ if rows.size == 1
+ Row.new(rows[0])
+ else
+ nil
+ end
+ end
+
+ # Private: Build a SQL query for clicks that can filter based on article,
+ # site and user.
+ def self.build_sql(select: "*", article_id: nil, site_id: nil, user_id: nil, created_at: nil)
+ sql = SQL.new("SELECT #{select} FROM #{table_name}")
+ sql.add "INNER JOIN articles a ON a.id = #{table_name}.article_id INNER JOIN sites s ON s.id = a.site_id" if site_id.present?
+ sql.add "INNER JOIN users u ON u.id = #{table_name}.user_id" if user_id.present?
+
+ fragments = []
+ binds = {}
+
+ if user_id.present?
+ fragments << "u.id = :user_id"
+ binds[:user_id] = user_id
+ end
+
+ if site_id.present?
+ fragments << "s.id = :site_id"
+ binds[:site_id] = site_id
+ end
+
+ if article_id.present?
+ fragments << "#{table_name}.article_id = :article_id"
+ binds[:article_id] = article_id
+ end
+
+ if created_at.present?
+ fragments << "#{table_name}.created_at >= :from AND #{table_name}.created_at <= :to"
+ binds[:from] = created_at.first
+ binds[:to] = created_at.last
+ end
+
+ if fragments.any?
+ sql.add "WHERE"
+ sql.add fragments.join(" AND "), binds
+ end
+
+ sql
+ end
+ class << self; private :build_sql; end
+end
diff --git a/app/models/click_hour_aggregate.rb b/app/models/click_hour_aggregate.rb
new file mode 100644
index 0000000..f1509a6
--- /dev/null
+++ b/app/models/click_hour_aggregate.rb
@@ -0,0 +1,32 @@
+class ClickHourAggregate < ApplicationRecord
+ def self.rollup(from:, to:)
+ raise ArgumentError, "from is required" if from.blank?
+ raise ArgumentError, "to is required" if to.blank?
+ raise ArgumentError, "from must be less than to" unless from < to
+
+ binds = {
+ from: from,
+ to: to,
+ insert_table: SQL::LITERAL(table_name),
+ select_table: SQL::LITERAL(Click.table_name),
+ }
+
+ SQL.run <<~SQL.squish, binds
+ INSERT INTO :insert_table (article_id, ts, count)
+ SELECT
+ article_id,
+ date_trunc('hour', created_at) AS ts,
+ count(*) as count
+ FROM :select_table
+ WHERE
+ article_id IS NOT NULL AND
+ created_at BETWEEN :from AND :to
+ GROUP BY 1, 2
+ ON CONFLICT (article_id, ts)
+ DO UPDATE SET
+ count = EXCLUDED.count
+ SQL
+ end
+
+ belongs_to :article
+end
diff --git a/app/models/paginate_response.rb b/app/models/paginate_response.rb
new file mode 100644
index 0000000..63ecd6f
--- /dev/null
+++ b/app/models/paginate_response.rb
@@ -0,0 +1,54 @@
+class PaginateResponse
+ include Enumerable
+ include Virtus.value_object
+
+ attribute :page, Integer
+ attribute :per_page, Integer
+ attribute :has_next_page, Boolean
+ attribute :rows, Array
+
+ def next_page
+ has_next_page? ? page + 1 : nil
+ end
+
+ def next_page?
+ has_next_page?
+ end
+
+ def prev_page
+ prev_page? ? page - 1 : nil
+ end
+
+ def prev_page?
+ page > 1
+ end
+
+ def length
+ rows.length
+ end
+ alias size length
+
+ def empty?
+ rows.empty?
+ end
+
+ def to_ary
+ rows
+ end
+
+ def [](idx)
+ rows[idx]
+ end
+
+ def last(n = nil)
+ n ? rows.last(n) : rows.last
+ end
+
+ def each(&block)
+ if block_given?
+ rows.each(&block)
+ else
+ rows.to_enum { @rows.size }
+ end
+ end
+end
diff --git a/app/models/partition_by_day.rb b/app/models/partition_by_day.rb
new file mode 100644
index 0000000..27d6c31
--- /dev/null
+++ b/app/models/partition_by_day.rb
@@ -0,0 +1,254 @@
+# https://www.postgresql.org/docs/13/ddl-partitioning.html#DDL-PARTITIONING-DECLARATIVE
+class PartitionByDay
+ class Row
+ include Virtus.model
+
+ attribute :name, String
+ attribute :expression, String
+ end
+
+ def self.validate_table(table:)
+ raise ArgumentError, "table cannot be blank" if table.blank?
+ raise ArgumentError, "table must be a String" unless table.is_a?(String)
+
+ table
+ end
+
+ def self.validate_name(table:, name:)
+ validate_table(table: table)
+
+ raise ArgumentError, "name must be a String" unless name.is_a?(String)
+ unless name.starts_with?(table)
+ raise ArgumentError, "name (#{name}) must start with table (#{table})"
+ end
+ unless name =~ /_\d{4}_\d{2}_\d{2}$/
+ raise ArgumentError, "name must end with yyyy_mm_dd but does not (#{name})"
+ end
+
+ name
+ end
+
+ def self.validate_from(from:)
+ raise ArgumentError, "from must not be nil" if from.nil?
+
+ from
+ end
+
+ def self.validate_to(to:)
+ raise ArgumentError, "to must not be nil" if to.nil?
+
+ to
+ end
+
+ def self.validate_number(number:)
+ raise ArgumentError, "number must not be nil" if number.nil?
+ unless number >= 2
+ raise ArgumentError, "number should be at least 2 or whats the point"
+ end
+
+ number
+ end
+
+ # Fetch all partitions for a given table.
+ def self.all(table:)
+ validate_table(table: table)
+
+ rows = SQL.hash_results <<-SQL.squish, table: table
+ SELECT pg_class.relname AS name,
+ pg_get_expr(pg_class.relpartbound, pg_class.oid, true) AS expression
+ FROM pg_class base_tb
+ JOIN pg_inherits ON pg_inherits.inhparent = base_tb.oid
+ JOIN pg_class ON pg_class.oid = pg_inherits.inhrelid
+ WHERE base_tb.oid = :table::regclass;
+ SQL
+
+ rows.map { |row| Row.new(row) }
+ end
+
+ # Generate a partition name based on table and from time.
+ #
+ # table - The String name of the source table.
+ # from - The Time of the new partition.
+ #
+ # Returns String partition name.
+ def self.name(table:, from:)
+ validate_table(table: table)
+ validate_from(from: from)
+
+ "#{table}_%d_%02d_%02d" % [from.year, from.month, from.day]
+ end
+
+ # Create new partition for provided table.
+ #
+ # table - The String name of the source table.
+ # name - The String name of the new partition.
+ # from - The Time to start the range of the partition.
+ # to - The Time to end the range of the partition.
+ #
+ # Returns nothing.
+ # Raises if anything goes wrong.
+ def self.create(table:, name:, from:, to:)
+ validate_name(table: table, name: name)
+ validate_from(from: from)
+ validate_to(to: to)
+
+ binds = {
+ table: SQL::LITERAL(table),
+ name: SQL::LITERAL(name),
+ from: from,
+ to: to,
+ }
+
+ SQL.run <<~SQL.squish, binds
+ CREATE TABLE IF NOT EXISTS :name
+ PARTITION OF :table FOR VALUES FROM (:from) TO (:to)
+ SQL
+
+ nil
+ end
+
+ # Premake several partitions from a given time. Also tries to create a
+ # partition for the from time so sometimes you ask for 3 partitions but get 4
+ # if the partition does not exist for the provided time.
+ #
+ # table - The String name of the source table.
+ # from - The Time to start premaking partitions from.
+ # number - The Integer number of partitions to create.
+ #
+ # Returns nothing.
+ # Raises if anything goes wrong.
+ def self.premake(table:, from: Time.now.utc, number: 3)
+ validate_table(table: table)
+ validate_from(from: from)
+ validate_number(number: number)
+
+ start = from.to_date
+ stop = start + number
+
+ (start..stop).each do |date|
+ new(table, date).create
+ end
+
+ nil
+ end
+
+ # Retain a given number of partitions and detch + drop the rest.
+ #
+ # table - The String name of the source table.
+ # from - The Time to determine retention from.
+ # number - The Integer number of partitions to older than from time.
+ #
+ # Returns nothing.
+ # Raises if anything goes wrong.
+ def self.retain(table:, from: Time.now.utc, number: 14)
+ validate_table(table: table)
+ validate_from(from: from)
+ validate_number(number: number)
+
+ date = from.to_date - number
+ binds = {
+ relname_pattern: "#{table}_%",
+ max_relname: name(table: table, from: date),
+ }
+ prunable = SQL.values <<~SQL.squish, binds
+ SELECT relname
+ FROM pg_class c
+ JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE nspname = 'public' AND
+ relname LIKE :relname_pattern AND
+ relkind = 'r' AND
+ relname <= :max_relname
+ ORDER BY relname
+ SQL
+
+ prunable.each { |name|
+ detach(table: table, name: name)
+ drop(table: table, name: name)
+ }
+
+ nil
+ end
+
+ # Drops a partition table.
+ #
+ # table - The String name of the source table.
+ # name - The String name of the partition.
+ #
+ # Returns nothing.
+ # Raises if anything goes wrong.
+ def self.drop(table:, name:)
+ validate_name(table: table, name: name)
+
+ SQL.run <<~SQL.squish, name: SQL::LITERAL(name)
+ DROP TABLE IF EXISTS :name
+ SQL
+
+ nil
+ end
+
+ # Detaches a partition from a table. Once detached you can do whatever with it
+ # and it won't show up in query results.
+ #
+ # table - The String name of the source table.
+ # name - The String name of the partition.
+ #
+ # Returns nothing.
+ # Raises if anything goes wrong.
+ def self.detach(table:, name:)
+ validate_name(table: table, name: name)
+
+ SQL.run <<~SQL.squish, table: SQL::LITERAL(table), name: SQL::LITERAL(name)
+ ALTER TABLE IF EXISTS :table DETACH PARTITION :name;
+ SQL
+
+ nil
+ end
+
+ def self.exists?(name)
+ raise ArgumentError, "name can't be blank" if name.blank?
+
+ ActiveRecord::Base.connection.table_exists?(name)
+ end
+ class << self; alias exist? exists?; end
+
+ attr_reader :from, :to, :table, :name
+
+ def initialize(table, from)
+ self.class.validate_table(table: table)
+ self.class.validate_from(from: from)
+
+ @from = from.to_time.utc.beginning_of_day
+ @to = @from + 1.day
+ @table = table
+ @name = self.class.name(table: @table, from: @from)
+ end
+
+ def create
+ self.class.create(table: @table, name: @name, from: @from, to: @to)
+ end
+
+ def premake(number)
+ self.class.premake(table: @table, from: @from, number: number)
+ end
+
+ def retain(number)
+ self.class.retain(table: @table, from: @from, number: number)
+ end
+
+ def detach
+ self.class.detach(table: @table, name: @name)
+ end
+
+ def drop
+ self.class.drop(table: @table, name: @name)
+ end
+
+ def exists?
+ self.class.exists?(@name)
+ end
+ alias :exist? :exists?
+
+ def all
+ self.class.all(table: @table)
+ end
+end
diff --git a/app/views/articles/index.html.erb b/app/views/articles/index.html.erb
index 15d7ea1..b0207c5 100644
--- a/app/views/articles/index.html.erb
+++ b/app/views/articles/index.html.erb
@@ -13,6 +13,7 @@
<div class="small text-muted">
<%= article.site.name %>
<span title="<%= article.published_at %>"><%= time_ago_in_words article.published_at %> ago</span>
+ &bull; <%= pluralize article.click_count, "view" %>
</div>
</div>
<%- end -%>
diff --git a/config/application.rb b/config/application.rb
index 70686ae..f53c3c9 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -24,6 +24,9 @@ class Application < Rails::Application
config.active_job.queue_adapter = :good_job
+ # Because we use partitioned tables and ruby/rails schema doesn't support that.
+ config.active_record.schema_format = :sql
+
config.generators.assets = false
config.generators.helper = false
config.generators.jbuilder = false
@@ -35,10 +38,15 @@ class Application < Rails::Application
config.good_job.enable_cron = true
config.good_job.cron = {
- frequent_task: {
- cron: "@hourly",
+ refresh_sites: {
+ cron: "1 * * * *",
class: "RefreshSitesJob",
},
+
+ click_hour_aggregates: {
+ cron: "5 0-23 * * *",
+ class: "ClickHourAggregateJob",
+ },
}
end
end
diff --git a/db/migrate/20220926122307_create_partitioned_clicks.rb b/db/migrate/20220926122307_create_partitioned_clicks.rb
new file mode 100644
index 0000000..145df80
--- /dev/null
+++ b/db/migrate/20220926122307_create_partitioned_clicks.rb
@@ -0,0 +1,20 @@
+class CreatePartitionedClicks < ActiveRecord::Migration[7.0]
+ def up
+ execute <<~SQL
+ CREATE TABLE clicks (
+ user_id integer,
+ article_id integer NOT NULL,
+ ip inet NOT NULL,
+ created_at timestamp without time zone NOT NULL
+ ) PARTITION BY RANGE (created_at);
+ SQL
+
+ add_index :clicks, :user_id, where: "user_id IS NOT NULL"
+ add_index :clicks, :article_id
+ add_index :clicks, :created_at, order: {created_at: :desc}
+ end
+
+ def down
+ drop_table :clicks
+ end
+end
diff --git a/db/migrate/20220926134853_create_click_hour_aggregates.rb b/db/migrate/20220926134853_create_click_hour_aggregates.rb
new file mode 100644
index 0000000..1ae96f8
--- /dev/null
+++ b/db/migrate/20220926134853_create_click_hour_aggregates.rb
@@ -0,0 +1,10 @@
+class CreateClickHourAggregates < ActiveRecord::Migration[7.0]
+ def change
+ create_table :click_hour_aggregates do |t|
+ t.references :article, null: false
+ t.integer :count, null: false
+ t.datetime :ts, null: false
+ t.index [:article_id, :ts], unique: true
+ end
+ end
+end
diff --git a/test/controllers/articles_controller_test.rb b/test/controllers/articles_controller_test.rb
index a17acf9..8fe769b 100644
--- a/test/controllers/articles_controller_test.rb
+++ b/test/controllers/articles_controller_test.rb
@@ -19,8 +19,24 @@ class ArticlesControllerTest < ActionDispatch::IntegrationTest
end
test "should get click" do
+ freeze_time
+ Click.partition(Time.zone.now).create
article = articles(:rare_unusual_patek_calatrava)
- get click_article_path(article)
+ assert_difference 'Click.count(article_id: article.id)' do
+ get click_article_path(article)
+ end
+ assert_redirected_to article.url
+ end
+
+ test "should get click with signed in user" do
+ freeze_time
+ Click.partition(Time.zone.now).create
+ user = users(:john)
+ article = articles(:rare_unusual_patek_calatrava)
+ sign_in user
+ assert_difference 'Click.count(article_id: article.id, user_id: user.id)' do
+ get click_article_path(article)
+ end
assert_redirected_to article.url
end
end
diff --git a/test/fixtures/click_hour_aggregates.yml b/test/fixtures/click_hour_aggregates.yml
new file mode 100644
index 0000000..1f0df1d
--- /dev/null
+++ b/test/fixtures/click_hour_aggregates.yml
@@ -0,0 +1,11 @@
+# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+# This model initially had no columns defined. If you add columns to the
+# model remove the "{}" from the fixture names and add the columns immediately
+# below each fixture, per the syntax in the comments below
+#
+# one: {}
+# column: value
+#
+# two: {}
+# column: value
diff --git a/test/jobs/click_hour_aggregate_job_test.rb b/test/jobs/click_hour_aggregate_job_test.rb
new file mode 100644
index 0000000..5558cc6
--- /dev/null
+++ b/test/jobs/click_hour_aggregate_job_test.rb
@@ -0,0 +1,7 @@
+require "test_helper"
+
+class ClickHourAggregateJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/models/article_test.rb b/test/models/article_test.rb
index 9c9ba95..75ffd3f 100644
--- a/test/models/article_test.rb
+++ b/test/models/article_test.rb
@@ -1,7 +1,42 @@
require "test_helper"
class ArticleTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ test "click_count for article with clicks" do
+ freeze_time
+
+ article = articles(:rare_unusual_patek_calatrava)
+ current_hour = Time.zone.now.beginning_of_hour.utc
+ two_hours_ago = current_hour - 2.hours
+ three_hours_ago = current_hour - 3.hours
+ live_time = current_hour == Time.zone.now ? current_hour : current_hour + 1.second
+
+ ClickHourAggregate.create!(article: article, count: 2, ts: current_hour)
+ ClickHourAggregate.create!(article: article, count: 2, ts: two_hours_ago)
+ ClickHourAggregate.create!(article: article, count: 6, ts: three_hours_ago)
+
+ [
+ current_hour,
+ two_hours_ago,
+ three_hours_ago,
+ live_time,
+ ].each do |time|
+ Click.partition(time).create
+ end
+
+ Click.create_many([
+ {article_id: article.id, created_at: two_hours_ago, ip: "127.0.0.1"}, # shouldn't count
+ {article_id: article.id, created_at: two_hours_ago, ip: "127.0.0.1"}, # shouldn't count
+ {article_id: article.id, created_at: three_hours_ago, ip: "127.0.0.1"}, # shouldn't count
+ {article_id: article.id, created_at: live_time, ip: "127.0.0.1"},
+ {article_id: article.id, created_at: live_time, ip: "127.0.0.1"},
+ {article_id: article.id, created_at: live_time, ip: "127.0.0.1"},
+ ]).value!
+
+ assert_equal 11, article.click_count
+ end
+
+ test "click_count for article without clicks" do
+ article = articles(:rare_unusual_patek_calatrava)
+ assert_equal 0, article.click_count
+ end
end
diff --git a/test/models/click_hour_aggregate_test.rb b/test/models/click_hour_aggregate_test.rb
new file mode 100644
index 0000000..ac3c9b6
--- /dev/null
+++ b/test/models/click_hour_aggregate_test.rb
@@ -0,0 +1,51 @@
+require "test_helper"
+
+class ClickHourAggregateTest < ActiveSupport::TestCase
+ setup do
+ freeze_time
+ Click.partition.create
+ @end_of_day = Time.now.utc.end_of_day
+ end
+
+ test "rollup" do
+ Click.create_many([
+ {
+ article_id: 1,
+ ip: "127.0.0.1",
+ created_at: @end_of_day,
+ },
+ {
+ article_id: 1,
+ ip: "127.0.0.1",
+ created_at: @end_of_day - 1.hour,
+ },
+ {
+ article_id: 2,
+ ip: "127.0.0.1",
+ created_at: @end_of_day - 1.hour,
+ },
+ {
+ article_id: 3,
+ ip: "127.0.0.1",
+ created_at: @end_of_day - 2.hours,
+ },
+ ])
+
+ assert_difference 'ClickHourAggregate.count', 4 do
+ ClickHourAggregate.rollup(from: @end_of_day - 2.days, to: @end_of_day)
+ end
+
+ hours = ClickHourAggregate.where(article_id: 1)
+ assert_equal 2, hours.size
+ assert_equal 1, hours[0].count
+ assert_equal 1, hours[1].count
+
+ hours = ClickHourAggregate.where(article_id: 2)
+ assert_equal 1, hours.size
+ assert_equal 1, hours[0].count
+
+ hours = ClickHourAggregate.where(article_id: 3)
+ assert_equal 1, hours.size
+ assert_equal 1, hours[0].count
+ end
+end
diff --git a/test/models/click_test.rb b/test/models/click_test.rb
new file mode 100644
index 0000000..496e51b
--- /dev/null
+++ b/test/models/click_test.rb
@@ -0,0 +1,95 @@
+require "test_helper"
+
+class ClickTest < ActiveSupport::TestCase
+ setup do
+ freeze_time
+ @attributes = valid_attributes
+ Click.partition(@attributes[:created_at]).create
+ end
+
+ test ".create" do
+ Click.create(@attributes)
+
+ log = Click.last
+ refute_nil log
+ assert_equal users(:john).id, log["user_id"]
+ assert_equal articles(:rare_unusual_patek_calatrava).id, log["article_id"]
+ assert_equal "127.0.0.1", log["ip"]
+ assert_equal Time.zone.now, log["created_at"]
+ end
+
+ test ".create without user_id" do
+ Click.create(@attributes.except(:user_id))
+
+ log = Click.last
+ refute_nil log
+ assert_nil log["user_id"]
+ assert_equal articles(:rare_unusual_patek_calatrava).id, log["article_id"]
+ assert_equal "127.0.0.1", log["ip"]
+ assert_equal Time.zone.now, log["created_at"]
+ end
+
+ test ".create_many with args" do
+ Click.create_many(@attributes, @attributes, @attributes)
+ assert_equal 3, Click.count
+ end
+
+ test ".create_many with array" do
+ Click.create_many([@attributes, @attributes, @attributes])
+ assert_equal 3, Click.count
+ end
+
+ test ".create_many fails when partition doesn't exist" do
+ Click.partition(@attributes[:created_at]).drop
+ result = Click.create_many(@attributes)
+ refute_predicate result, :ok?
+ assert_instance_of ActiveRecord::StatementInvalid, result.error
+ assert_instance_of PG::CheckViolation, result.error.cause
+ end
+
+ test ".count" do
+ Click.create(@attributes)
+ assert_equal 1, Click.count
+ assert_equal 1, Click.count(
+ article_id: articles(:rare_unusual_patek_calatrava).id,
+ user_id: users(:john).id
+ )
+ assert_equal 1, Click.count(site_id: articles(:rare_unusual_patek_calatrava).site_id)
+ assert_equal 1, Click.count(created_at: Time.zone.now.beginning_of_day..Time.zone.now.end_of_day)
+ end
+
+ test ".paginate" do
+ Click.create(@attributes)
+ assert_equal 1, Click.paginate.size
+ assert_equal 1, Click.paginate(
+ article_id: articles(:rare_unusual_patek_calatrava).id,
+ user_id: users(:john).id
+ ).size
+ assert_equal 0, Click.paginate(
+ article_id: articles(:rare_unusual_patek_calatrava).id,
+ user_id: users(:kris).id
+ ).size
+ assert_equal 0, Click.paginate(article_id: 0).size
+ assert_equal 1, Click.paginate(site_id: articles(:rare_unusual_patek_calatrava).site_id).size
+ assert_equal 0, Click.paginate(page: 2).size
+
+ Click.create(@attributes)
+ assert_equal 1, Click.paginate(
+ article_id: articles(:rare_unusual_patek_calatrava).id,
+ user_id: users(:john).id,
+ per_page: 1,
+ page: 2
+ ).size
+ end
+
+ private
+
+ def valid_attributes(attributes = {})
+ {
+ user_id: users(:john).id,
+ article_id: articles(:rare_unusual_patek_calatrava).id,
+ ip: "127.0.0.1",
+ created_at: Time.zone.now,
+ }.merge(attributes)
+ end
+end
diff --git a/test/models/partition_by_day_test.rb b/test/models/partition_by_day_test.rb
new file mode 100644
index 0000000..0b59cf4
--- /dev/null
+++ b/test/models/partition_by_day_test.rb
@@ -0,0 +1,150 @@
+require 'test_helper'
+
+class PartitionByDayTest < ActiveSupport::TestCase
+ def table_name
+ "partitioned_fake_table"
+ end
+
+ setup do
+ SQL.run <<~SQL.squish
+ CREATE TABLE #{table_name} (
+ created_at timestamp without time zone NOT NULL
+ ) PARTITION BY RANGE (created_at);
+ SQL
+ end
+
+ test ".all and #all" do
+ time = Time.utc(2021, 10, 12, 12, 3, 3)
+ partition = PartitionByDay.new(table_name, time)
+ partition.create
+
+ partitions = PartitionByDay.all(table: table_name).map(&:name)
+ assert_includes partitions, partition.name
+ assert_includes partition.all.map(&:name), partition.name
+ end
+
+ test "#name" do
+ {
+ Time.utc(2021, 9, 9, 12, 3, 3) => "#{table_name}_2021_09_09",
+ Time.utc(2021, 10, 9, 12, 3, 3) => "#{table_name}_2021_10_09",
+ Time.utc(2021, 10, 12, 12, 3, 3) => "#{table_name}_2021_10_12",
+ }.each do |time, name|
+ assert_equal name, PartitionByDay.new(table_name, time).name
+ assert_equal name, PartitionByDay.new(table_name, time).name
+ end
+ end
+
+ test "#from" do
+ {
+ Time.utc(2021, 9, 9, 12, 3, 3) => Time.utc(2021, 9, 9, 0, 0, 0),
+ Time.utc(2021, 10, 12, 12, 3, 3) => Time.utc(2021, 10, 12, 0, 0, 0),
+ }.each do |time, name|
+ assert_equal name, PartitionByDay.new(table_name, time).from
+ assert_equal name, PartitionByDay.new(table_name, time).from
+ end
+ end
+
+ test "#to" do
+ {
+ Time.utc(2021, 9, 9, 12, 3, 3) => Time.utc(2021, 9, 10, 0, 0, 0),
+ Time.utc(2021, 10, 12, 12, 3, 3) => Time.utc(2021, 10, 13, 0, 0, 0),
+ }.each do |time, name|
+ assert_equal name, PartitionByDay.new(table_name, time).to
+ assert_equal name, PartitionByDay.new(table_name, time).to
+ end
+ end
+
+ test "#create" do
+ time = Time.utc(2021, 10, 12, 12, 3, 3)
+ partition = PartitionByDay.new(table_name, time)
+ refute_includes ActiveRecord::Base.connection.tables, partition.name
+ partition.create
+ assert_includes ActiveRecord::Base.connection.tables, partition.name
+
+ # verify the partition is all created right
+ partitions = PartitionByDay.all(table: table_name)
+ assert_equal 1, partitions.size
+ assert_equal "#{table_name}_2021_10_12", partitions[0].name
+ assert_equal "FOR VALUES FROM ('2021-10-12 00:00:00') TO ('2021-10-13 00:00:00')",
+ partitions[0].expression
+ end
+
+ test "#exist?" do
+ time = Time.utc(2021, 10, 12, 12, 3, 3)
+ partition = PartitionByDay.new(table_name, time)
+ refute_predicate partition, :exist?
+ partition.create
+ assert_predicate partition, :exist?
+ end
+
+ test "#exists?" do
+ time = Time.utc(2021, 10, 12, 12, 3, 3)
+ partition = PartitionByDay.new(table_name, time)
+ refute_predicate partition, :exists?
+ partition.create
+ assert_predicate partition, :exists?
+ end
+
+ test "#detach" do
+ time = Time.utc(2021, 10, 12, 12, 3, 3)
+ partition = PartitionByDay.new(table_name, time)
+ partition.create
+ partition.detach
+ partitions = PartitionByDay.all(table: table_name).map(&:name)
+ refute_includes partitions, partition.name
+ end
+
+ test "#drop" do
+ time = Time.utc(2021, 10, 12, 12, 3, 3)
+ partition = PartitionByDay.new(table_name, time)
+ partition.create
+ partition.drop
+ partitions = PartitionByDay.all(table: table_name).map(&:name)
+ refute_includes partitions, partition.name
+ end
+
+ test "#premake" do
+ time = Time.utc(2021, 9, 9)
+ partition = PartitionByDay.new(table_name, time)
+ partition.create
+
+ ActiveRecord::Base.uncached do
+ partition.premake(3)
+ assert_equal 4, PartitionByDay.all(table: table_name).size
+
+ partition.premake(5)
+ assert_equal 6, PartitionByDay.all(table: table_name).size
+ end
+ end
+
+ test "#retain" do
+ hi = Time.utc(2021, 9, 30)
+ low = hi - 20.days
+
+ (low.to_date..hi.to_date).each do |date|
+ PartitionByDay.new(table_name, date.to_time).create
+ end
+
+ ActiveRecord::Base.uncached do
+ assert_equal 21, PartitionByDay.all(table: table_name).size
+ PartitionByDay.new(table_name, hi).retain(14)
+ assert_equal 14, PartitionByDay.all(table: table_name).size
+ end
+ end
+
+ private
+
+ def valid_attributes(attributes = {})
+ {
+ status: 200,
+ method: "GET",
+ path: "/adapter/features",
+ ip: "127.0.0.1",
+ token_id: tokens(:john_api).id,
+ created_at: Time.zone.now,
+ headers: {
+ "content_type" => "application/json",
+ },
+ }.merge(attributes)
+ end
+end
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 4fe687a..c03a9c4 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -23,3 +23,7 @@ class ActiveSupport::TestCase
class ActionDispatch::IntegrationTest
include Devise::Test::IntegrationHelpers
end
+
+Click.partition.all.map(&:name).each do |name|
+ PartitionByDay.drop(table: Click.table_name, name: name)
+end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment