Skip to content

Instantly share code, notes, and snippets.

@jtraulle

jtraulle/kunena3.rb

Last active August 23, 2019 11:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jtraulle/a8ed0244a2a77498f29ac32fd2bc0f65 to your computer and use it in GitHub Desktop.
Save jtraulle/a8ed0244a2a77498f29ac32fd2bc0f65 to your computer and use it in GitHub Desktop.
Download ZIP
Customized kunena3.rb Discourse import script for Dolibarr french forum migration
Raw
kunena3.rb
# frozen_string_literal: true
require "mysql2"
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
class ImportScripts::Kunena < ImportScripts::Base
DB_HOST ||= ENV['DB_HOST'] || "172.17.0.1"
DB_NAME ||= ENV['DB_NAME'] || "kunena"
DB_USER ||= ENV['DB_USER'] || "root"
DB_PW ||= ENV['DB_PW'] || "root"
KUNENA_PREFIX ||= ENV['KUNENA_PREFIX'] || "gvrsi_"
IMAGE_PREFIX ||= ENV['IMAGE_PREFIX'] || "https://www.dolibarr.fr/media/kunena/attachments"
PARENT_FIELD ||= ENV['PARENT_FIELD'] || "parent_id"
DISCOURSE_BASEURL ||= ENV['DISCOURSE_BASEURL'] || "http://localhost"
ATTACHMENTS_LOCAL_FOLDER ||= ENV['ATTACHMENTS_LOCAL_FOLDER'] || "/kunena_attachments"
BATCH_SIZE ||= 1000
def initialize
super
@users = {}
@client = Mysql2::Client.new(
host: DB_HOST,
username: DB_USER,
password: DB_PW,
database: DB_NAME
)
end
def execute
import_users
import_categories
import_posts
import_likes
end
def import_users
puts "", "Importing users..."
last_user_id = -1
unified_users_query = <<-SQL
SELECT
id,
username,
email,
password,
registerDate,
signature,
location,
websiteurl,
birthdate,
avatar,
moderator,
1 AS ordering
FROM #{KUNENA_PREFIX}users
INNER JOIN #{KUNENA_PREFIX}kunena_users ON #{KUNENA_PREFIX}kunena_users.userid = #{KUNENA_PREFIX}users.id
UNION
SELECT
userid,
name,
SUBSTRING_INDEX(GROUP_CONCAT(email), ',',1) AS email,
NULL AS password,
FROM_UNIXTIME(MIN(time)) AS registerDate,
NULL AS signature,
NULL AS location,
NULL AS website,
NULL AS birthdate,
NULL AS avatar,
0 AS moderator,
2 AS ordering
FROM #{KUNENA_PREFIX}kunena_messages
WHERE userid NOT IN(SELECT id FROM #{KUNENA_PREFIX}users)
AND email IS NOT NULL AND email <> ''
GROUP BY userid, name
UNION
SELECT
userid,
name,
NULL AS email,
NULL AS password,
FROM_UNIXTIME(MIN(time)) AS registerDate,
NULL AS signature,
NULL AS location,
NULL AS website,
NULL AS birthdate,
NULL AS avatar,
0 AS moderator,
3 AS ordering
FROM #{KUNENA_PREFIX}kunena_messages
WHERE userid NOT IN(SELECT id FROM #{KUNENA_PREFIX}users)
AND (email IS NULL OR email = '')
AND userid NOT IN(
SELECT DISTINCT userid
FROM #{KUNENA_PREFIX}kunena_messages
WHERE userid NOT IN(SELECT id FROM #{KUNENA_PREFIX}users)
AND email IS NOT NULL AND email <> ''
GROUP BY userid
)
GROUP BY userid, name
ORDER BY ordering, id
SQL
# We populate this array on each user creation to be able to check if we have duplicates emails
emails = []
total = @client.query("
SELECT COUNT(*) count
FROM
(
#{unified_users_query}
) count;
").first['count']
batches(BATCH_SIZE) do |offset|
users = @client.query("
#{unified_users_query}
LIMIT #{BATCH_SIZE}
OFFSET #{offset};
", cache_rows: false).to_a
break if users.empty?
last_user_id = users[-1]["id"]
user_ids = users.map { |u| u["id"] }
create_users(users, total: total, offset: offset) do |u|
# If the current email has been already used for a previous user
# we discard the email to be able to create the account and keep the previous username
if u["email"].presence
if emails.include?(u["email"])
u["email"] = fake_email
else
emails << u["email"]
end
end
{
id: u["id"],
moderator: (u["moderator"].to_i == 1),
created_at: u["registerDate"],
email: u["email"].presence || fake_email,
username: u["username"],
website: u["websiteurl"],
location: u["location"],
date_of_birth: u["birthdate"],
bio_raw: u["signature"],
suspended_at: u["banned"].present? ? Time.zone.now : nil,
suspended_till: u["banned"].present? ? 100.years.from_now : nil,
post_create_action: proc do |user|
if u["avatar"].present?
UserAvatar.import_url_for_user("https://www.dolibarr.fr/media/kunena/avatars/" + u["avatar"], user) rescue nil
end
user.custom_fields['import_pass'] = u["password"]
user.save
end
}
end
end
end
def import_categories
puts '', "Importing categories..."
categories = [ {
"id": 1,
"name": (+"Annonces & actualités"),
"description": (+"Vous trouverez dans cette catégorie des annonces et actualités en lien avec le projet Dolibarr : nouvelles versions, évènements, mise à jour des outils communautaire du projet, etc."),
"position": 1,
"parent_category_id": 0
}, {
"id": 2,
"name": (+"Installer mon Dolibarr"),
"description": (+"Obtenir de l’aide sur l’installation de Dolibarr quelque soit votre système (GNU/Linux, macOS, Windows, système NAS), votre méthode d’installation (depuis les sources, DoliWamp, DoliDeb, DoliRpm) et votre environnement (serveur local, mutualisé, dédié, cloud)."),
"position": 2,
"parent_category_id": 0
}, {
"id": 3,
"name": (+"Utiliser mon Dolibarr"),
"description": (+"Obtenir de l’aide sur l’utilisation des modules fournis avec Dolibarr, suggérer des nouvelles fonctionnalités ou signaler des anomalies."),
"position": 3,
"parent_category_id": 0
}, {
"id": 4,
"name": (+"Modules GRC/GRF"),
"description": (+"Tiers, Contrats/Abonnements, Interventions, Commandes, Expéditions, Tickets"),
"position": 4,
"parent_category_id": 3
}, {
"id": 5,
"name": (+"Modules GRH"),
"description": (+"Utilisateurs, Adhérents, Congés, Notes de frais"),
"position": 5,
"parent_category_id": 3
}, {
"id": 6,
"name": (+"Modules PM"),
"description": (+"Produits, Services"),
"position": 6,
"parent_category_id": 3
}, {
"id": 7,
"name": (+"Modules financiers"),
"description": (+"Facturation, Banque/Caisse, Dons, Comptabilité, Comptabilité avancée"),
"position": 7,
"parent_category_id": 3
}, {
"id": 8,
"name": (+"Autres modules natifs"),
"description": (+"Projets/Travail collaboratif, GED, Outils multi-modules, etc."),
"position": 8,
"parent_category_id": 3
}, {
"id": 9,
"name": (+"Modules externes du DoliStore"),
"description": (+"Obtenir de l’aide sur l’utilisation des modules disponibles sur la platforme DoliStore."),
"position": 9,
"parent_category_id": 3
}, {
"id": 10,
"name": (+"Retours d'expérience"),
"description": (+"Partager un retour d'expérience sur la mise en place de l'ERP/CRM Dolibarr au sein de votre entreprise."),
"position": 10,
"parent_category_id": 3
}, {
"id": 11,
"name": (+"Maintenir mon Dolibarr"),
"description": (+"Obtenir de l’aide sur la mise à jour de Dolibarr vers une version plus récente, la migration d’une instance Dolibarr vers un nouveau serveur, les opérations de sauvegarde et de restauration (fichiers et base de données) et plus généralement, tout ce qui touche à la configuration et à la maintenance système de la machine hébergeant votre instance Dolibarr (cron, permissions, etc.)."),
"position": 11,
"parent_category_id": 0
}, {
"id": 12,
"name": (+"Développer pour Dolibarr"),
"description": (+"Obtenir de l’aide concernant le développement de modules spécifiques ou thèmes pour Dolibarr mais également sur la façon d’utiliser les modules API REST et SOAP pour interconnecter Dolibarr avec une application externe."),
"position": 12,
"parent_category_id": 0
}, {
"id": 13,
"name": (+"Discuter entre Dolibarriens"),
"description": (+"Cet espace de discussion vous permet d’échanger avec les autres utilisateurs de Dolibarr sur des sujets sans lien avec Dolibarr."),
"position": 13,
"parent_category_id": 0
} ]
created = 0
total = categories.count
categories.each do |c|
h = { id: c[:id], name: c[:name], description: c[:description], position: c[:position].to_i }
if c[:parent_category_id].to_i > 0
h[:parent_category_id] = category_id_from_imported_category_id(c[:parent_category_id])
end
new_category = create_category(h, h[:id])
created_category(new_category)
created += 1
print_status(created, total, get_start_time("categories"))
end
end
def import_posts
puts '', "Retreiving attachments info from kunena_attachments table..."
attachments_records = @client.query("
SELECT id, mesid, folder, filename
FROM #{KUNENA_PREFIX}kunena_attachments;
", cache_rows: false).to_a
attachments = Hash.new
attachments_records.each do |a|
unless attachments[a['mesid']].is_a?(Array)
attachments[a['mesid']] = []
end
attachments[a['mesid']] << {"id" => a['id'], "folder" => a['folder'], "filename" => a['filename']}
end
puts '', "Importing topics and posts..."
mapping_categories = {
"14" => category_id_from_imported_category_id(1),
"3" => category_id_from_imported_category_id(2),
"8" => category_id_from_imported_category_id(11),
"12" => category_id_from_imported_category_id(3),
"5" => category_id_from_imported_category_id(3),
"527" => category_id_from_imported_category_id(3),
"529" => category_id_from_imported_category_id(3),
"614" => category_id_from_imported_category_id(3),
"15" => category_id_from_imported_category_id(10),
"11" => category_id_from_imported_category_id(3),
"514" => category_id_from_imported_category_id(3),
"604" => 3,
"531" => category_id_from_imported_category_id(9),
"511" => category_id_from_imported_category_id(12),
"510" => category_id_from_imported_category_id(4),
"505" => category_id_from_imported_category_id(9),
"507" => category_id_from_imported_category_id(8),
"509" => category_id_from_imported_category_id(9),
"526" => category_id_from_imported_category_id(9),
"543" => category_id_from_imported_category_id(5),
"545" => category_id_from_imported_category_id(4),
"546" => category_id_from_imported_category_id(4),
"547" => category_id_from_imported_category_id(7),
"548" => category_id_from_imported_category_id(6),
"549" => category_id_from_imported_category_id(8),
"551" => category_id_from_imported_category_id(3),
"553" => category_id_from_imported_category_id(8),
"554" => category_id_from_imported_category_id(3),
"556" => category_id_from_imported_category_id(11),
"557" => category_id_from_imported_category_id(12),
"558" => category_id_from_imported_category_id(8),
"585" => category_id_from_imported_category_id(2),
"586" => category_id_from_imported_category_id(2),
"587" => category_id_from_imported_category_id(2),
"607" => category_id_from_imported_category_id(2),
"608" => category_id_from_imported_category_id(2),
"537" => category_id_from_imported_category_id(3),
"615" => category_id_from_imported_category_id(3),
"616" => category_id_from_imported_category_id(3),
"618" => category_id_from_imported_category_id(3),
"515" => category_id_from_imported_category_id(3),
"516" => category_id_from_imported_category_id(3),
"524" => category_id_from_imported_category_id(3),
"528" => category_id_from_imported_category_id(3),
"522" => category_id_from_imported_category_id(3),
"609" => category_id_from_imported_category_id(12),
"610" => category_id_from_imported_category_id(12),
"611" => category_id_from_imported_category_id(12),
"532" => category_id_from_imported_category_id(9),
"534" => category_id_from_imported_category_id(9),
"536" => category_id_from_imported_category_id(8),
"605" => category_id_from_imported_category_id(8),
"606" => category_id_from_imported_category_id(8),
"613" => category_id_from_imported_category_id(9),
"544" => category_id_from_imported_category_id(5),
"550" => category_id_from_imported_category_id(5),
"579" => category_id_from_imported_category_id(5),
"598" => category_id_from_imported_category_id(5),
"580" => category_id_from_imported_category_id(4),
"589" => category_id_from_imported_category_id(4),
"592" => category_id_from_imported_category_id(4),
"594" => category_id_from_imported_category_id(4),
"595" => category_id_from_imported_category_id(4),
"596" => category_id_from_imported_category_id(4),
"597" => category_id_from_imported_category_id(4),
"600" => category_id_from_imported_category_id(4),
"581" => category_id_from_imported_category_id(4),
"582" => category_id_from_imported_category_id(4),
"572" => category_id_from_imported_category_id(7),
"573" => category_id_from_imported_category_id(7),
"574" => category_id_from_imported_category_id(7),
"575" => category_id_from_imported_category_id(7),
"576" => category_id_from_imported_category_id(7),
"577" => category_id_from_imported_category_id(7),
"588" => category_id_from_imported_category_id(7),
"602" => category_id_from_imported_category_id(7),
"568" => category_id_from_imported_category_id(6),
"569" => category_id_from_imported_category_id(6),
"570" => category_id_from_imported_category_id(6),
"571" => category_id_from_imported_category_id(6),
"593" => category_id_from_imported_category_id(6),
"552" => category_id_from_imported_category_id(3),
"591" => category_id_from_imported_category_id(3),
"561" => category_id_from_imported_category_id(8),
"567" => category_id_from_imported_category_id(3),
"590" => category_id_from_imported_category_id(8),
"603" => category_id_from_imported_category_id(8),
"555" => category_id_from_imported_category_id(7),
"559" => category_id_from_imported_category_id(11),
"578" => category_id_from_imported_category_id(7),
"584" => category_id_from_imported_category_id(8),
"560" => category_id_from_imported_category_id(8),
"601" => category_id_from_imported_category_id(8),
}
#SELECT id FROM #{KUNENA_PREFIX}kunena_topics
#21428,28012,28637,40680,42506,56540,56838,59279,59433,61168,62661,62681,62730,62844,62846,64351,64371,64380,64475,64599,64629,64778
total_count = @client.query("
SELECT COUNT(*) count
FROM #{KUNENA_PREFIX}kunena_messages m
WHERE thread IN(
SELECT id FROM #{KUNENA_PREFIX}kunena_topics
);
").first['count']
batches(BATCH_SIZE) do |offset|
results = @client.query("
SELECT m.id id,
m.thread thread,
m.parent parent,
m.catid catid,
m.userid userid,
s.icon_id = 8 AS solved,
s.ordering AS pinned,
s.locked,
s.subject subject,
m.time time,
t.message message
FROM #{KUNENA_PREFIX}kunena_messages m
INNER JOIN #{KUNENA_PREFIX}kunena_messages_text t ON m.id = t.mesid
INNER JOIN #{KUNENA_PREFIX}kunena_topics s ON m.thread = s.id
WHERE thread IN(
SELECT id FROM #{KUNENA_PREFIX}kunena_topics
)
ORDER BY m.id
LIMIT #{BATCH_SIZE}
OFFSET #{offset};
", cache_rows: false)
break if results.size < 1
next if all_records_exist? :posts, results.map { |p| p['id'].to_i }
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
post = {}
post[:id] = m['id']
post[:user_id] = user_id_from_imported_user_id(m['userid']) || -1
# Break line after [code] only for lines that starts by [code] and followed by anything before a line break
m["message"] = m["message"].gsub(/(\[code\])(.+)[^\n]+/, "\\1\n\\2")
# Before making any alteration to the post text, we need to remove [code][/code]
# this is because we do not want to alter code blocks
codeblocks = []
codeblocks_to_remove = m["message"].scan(/(\[code\].+?\[\/code\])/m)
codeblock_number = 0
for codeblock in codeblocks_to_remove
codeblock_value = codeblock[0]
codeblocks << codeblock_value
m["message"] = m["message"].gsub(codeblock_value, "[!--###kunena-codeblock-#{codeblock_number}###--]")
codeblock_number += 1
end
# Escape some commonly used chars to not be wrongly parsed as Markdown
m["message"] = m["message"].gsub(/>/, "\\>")
m["message"] = m["message"].gsub(/\* /, "\\\\* ")
m["message"] = m["message"].gsub(/- /, "\\- ")
m["message"] = m["message"].gsub(/\+ /, "\\\\+ ")
# Convert some emojis shortcuts to preserve custom emojis previously used
m["message"] = m["message"].gsub(/:\)/, ":happy:")
m["message"] = m["message"].gsub(/:\(/, ":unhappy:")
m["message"] = m["message"].gsub(/:P/, ":tongue:")
m["message"] = m["message"].gsub(/:S/, ":confused:")
m["message"] = m["message"].gsub(/B\)/, ":sunglasses:")
# We keep a list of ids of attachments and pathsof legacy attachment to be able to add
# non inserted attachment on the bottom of the post
legacy_attachments_path = m["message"].scan(/\[(?:(?:img)|(?:file)).*?\](?:https?:\/\/)?www\.dolibarr\.(?:(?:fr)|(?:org))\/media\/kunena\/attachments(\/legacy\/(?:(?:images)|(?:files))\/.+?)\[\/(?:(?:img)|(?:file)).*?\]/).flatten
attachments_ids = m["message"].scan(/\[attachment=([0-9]+)\].+?\[\/attachment\]/).flatten
# Replace [attachment] BBCode tag by native Discourse attachment
user_id = m['userid']
attachments_to_alter = m["message"].scan(/(\[attachment=[0-9]+\](.+?)\[\/attachment\])/)
for attachment in attachments_to_alter
original_attachment_to_replace = attachment[0]
filename = attachment[1]
path = "#{ATTACHMENTS_LOCAL_FOLDER}/#{user_id}/#{filename}"
if File.exists?(path)
upload = create_upload(post[:user_id], path, filename)
if upload.present? && upload.persisted?
upload_markdown = html_for_upload(upload, filename)
m["message"] = m["message"].gsub(original_attachment_to_replace, "\n\n#{upload_markdown}")
end
else
m["message"] = m["message"].gsub(original_attachment_to_replace, "> **La pièce jointe #{filename} est absente ou indisponible**")
end
end
# Replace legacy attachment using [img] and [file] BBCode tags by native Discourse attachment
attachments_to_alter = m["message"].scan(/(\[(?:(?:img)|(?:file)).*?\](?:https?:\/\/)?www\.dolibarr\.(?:(?:fr)|(?:org))\/media\/kunena\/attachments(\/legacy\/(?:(?:images)|(?:files))\/(.+?))\[\/(?:(?:img)|(?:file)).*?\])/)
for attachment in attachments_to_alter
original_attachment_to_replace = attachment[0]
path = attachment[1]
filename = attachment[2]
path = "#{ATTACHMENTS_LOCAL_FOLDER}#{path}"
if File.exists?(path)
upload = create_upload(post[:user_id], path, filename)
if upload.present? && upload.persisted?
upload_markdown = html_for_upload(upload, filename)
m["message"] = m["message"].gsub(original_attachment_to_replace, "\n\n#{upload_markdown}")
end
else
m["message"] = m["message"].gsub(original_attachment_to_replace, "\n> **La pièce jointe #{filename} est absente ou indisponible**")
end
end
# Print out unattached attachment at the bottom of the post
if attachments.has_key? m["id"]
attachments_to_add = "\n\n > Pièces jointes :\n"
for attachment in attachments[m["id"]]
filename = attachment["filename"]
path = attachment["folder"].gsub("media/kunena/attachments", "")
file_already_in_post_text = (attachments_ids.include?(attachment["id"].to_s) or legacy_attachments_path.include?("#{path}/#{filename}"))
path = "#{ATTACHMENTS_LOCAL_FOLDER}#{path}/#{filename}"
unless file_already_in_post_text
if File.exists?(path)
upload = create_upload(post[:user_id], path, filename)
if upload.present? && upload.persisted?
upload_markdown = attachment_html(upload, filename)
attachments_to_add += "> * #{upload_markdown}\n"
end
end
end
end
unless attachments_to_add == "\n\n > Pièces jointes :\n"
m["message"] += attachments_to_add
end
end
# Deal with quote block (add a line break after [quote] BBCode tag if there is none)
m["message"] = m["message"].gsub(/\[quote([^\]]*)\]/, "\n\[quote\\1\]\n")
m["message"] = m["message"].gsub(/(.*)\[\/quote\]/, "\\1\n\[/quote\]\n")
# Replacing old posts reference in quote blocks by new ones
posts_references_to_alter = m["message"].scan(/(\[quote="(.*)" post=(\d{1,6}))/)
for result in posts_references_to_alter
origin_pattern_to_replace = result[0]
origin_post_user = result[1]
origin_post_id = result[2]
referenced_topic = topic_lookup_from_imported_post_id(origin_post_id)
if referenced_topic
topic_id = referenced_topic[:topic_id]
post_number = referenced_topic[:post_number]
m["message"] = m["message"].gsub(origin_pattern_to_replace, "\[quote=\"" + origin_post_user + ", post:" + post_number.to_s + ", topic:" + topic_id.to_s + "\"")
end
end
# Removing remaining posts references in quote blocks for posts that does not exists anymore
m["message"] = m["message"].gsub(/(\[quote[^ ]*) post=\d{1,6}(\])/, "\\1\\2")
# Replacing old posts & topics forum URLs by new ones
posts_url_to_alter = m["message"].scan(/((?:https?:\/\/)?www\.dolibarr\.fr\/forum\/[^\/]+\/(\d[^-]+)-[^# \n]+(?:#(\d+))?)/)
for result in posts_url_to_alter
origin_url = result[0]
origin_topic_id = result[1]
origin_post_id = result[2]
referenced_topic = {}
if origin_post_id.nil?
origin_post_id = @client.query("SELECT id FROM #{KUNENA_PREFIX}kunena_messages WHERE thread = #{origin_topic_id} ORDER BY id LIMIT 1;").first['id'] rescue nil
end
unless origin_post_id.nil?
referenced_topic = topic_lookup_from_imported_post_id(origin_post_id)
if referenced_topic
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}" + referenced_topic[:url] )
else
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-topic-no-longer-exists" )
end
else
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-topic-no-longer-exists" )
end
end
# Replacing old forum categories URL by new ones
categories_url_to_alter = m["message"].scan(/((?:https?:\/\/)?www\.dolibarr\.fr\/forum\/([^\/ \n\]]+))[ \]]/)
for result in categories_url_to_alter
origin_url = result[0]
origin_category_slug = result[1]
origin_category_id = @client.query("SELECT id FROM #{KUNENA_PREFIX}kunena_categories WHERE alias = '#{origin_category_slug}' LIMIT 1;").first['id'] rescue nil
unless origin_category_id.nil?
category_id = mapping_categories[origin_category_id.to_s]
category = Category.find_by(id: category_id) rescue nil
unless category.nil?
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/c/" + category.slug )
else
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-category-no-longuer-exists" )
end
else
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-category-no-longuer-exists" )
end
end
# Replacing old profile URL by new ones
profile_url_to_alter = m["message"].scan(/((?:https?:\/\/)?www\.dolibarr\.fr\/forum\/profile\/userid-(\d+))/)
for result in profile_url_to_alter
origin_url = result[0]
origin_user_id = result[1]
user = find_user_by_import_id(origin_user_id) rescue nil
unless user.nil?
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/u/" + user.username )
else
m["message"] = m["message"].gsub(origin_url, "#{DISCOURSE_BASEURL}/#this-user-no-longuer-exists" )
end
end
# Converting [table] definitions into Markdown syntax
tables_to_alter = m["message"].scan(/(\[table\](.+?)\[\/table\])/m)
for table in tables_to_alter
original_table_to_replace = table[0]
markdown_converted_table = ""
lines = original_table_to_replace.scan(/\[tr\](.+?)\[\/tr\]/m)
line_number = 1
for line in lines
line_value = line[0]
columns = line_value.scan(/\[td\](.+?)\[\/td\]/m)
total_columns = columns.count
column_number = 1
for column in columns
column_value = column[0]
if column_number == total_columns
markdown_converted_table += "| " + column_value + " |\n"
if line_number == 1
for i in 1..total_columns
markdown_converted_table += "| - "
end
markdown_converted_table += "|\n"
end
else
markdown_converted_table += "| " + column_value.gsub("|", "\|") + " "
end
column_number += 1
end
line_number += 1
end
m["message"] = m["message"].gsub(original_table_to_replace, markdown_converted_table )
end
# Break line after [ul] only for lines that starts by [ul] and followed by [li] tag on the same line
m["message"] = m["message"].gsub(/(^\[ul\])( *\[li\].+$)/m, "\\1\n\\2")
# Replace [strike][/strike] BBCode tag by <s></s> HTML equivalent for markdown parser
m["message"] = m["message"].gsub(/\[strike\]([^\[]*)\[\/strike\]/, "<s>\\1</s>")
# Break line before/after ---------- (horizontal line)
m["message"] = m["message"].gsub(/(-{5,200})[^ \S]/m, "\n\\1\n")
# We restore the previously removed [code] blocks after all replacements has been made
codeblocks_to_insert = m["message"].scan(/(\[!--###kunena-codeblock-(\d+)###--\])/m)
for codeblock in codeblocks_to_insert
codeblock_marker_to_replace = codeblock[0]
codeblock_number_to_replace = codeblock[1]
m["message"] = m["message"].gsub(codeblock_marker_to_replace, codeblocks[codeblock_number_to_replace.to_i])
codeblock_number += 1
end
post[:raw] = m["message"]
post[:created_at] = Time.zone.at(m['time'])
if m['parent'] == 0
if mapping_categories.key?(m['catid'].to_s)
post[:category] = mapping_categories[m['catid'].to_s]
else
puts "Catégorie " + m['catid'].to_s + " non trouvée dans le mapping."
post[:category] = 1
end
post[:title] = m['subject']
else
parent = topic_lookup_from_imported_post_id(m['parent'])
if parent
post[:topic_id] = parent[:topic_id]
post[:reply_to_post_number] = parent[:post_number] if parent[:post_number] > 1
else
# If we got here, this is probably because a user used the Reply button at the bottom of a topic's post
# and not the reply button of the topic itselft. In that case, we fetch the post id of the first post
# of the topic and use it instead
thread_id = m['thread']
first_topic_post = @client.query("
SELECT id, parent FROM #{KUNENA_PREFIX}kunena_messages
WHERE thread = #{thread_id}
ORDER BY id LIMIT 1;
")
parent_post_id = first_topic_post.first['id']
parent = topic_lookup_from_imported_post_id(parent_post_id)
if parent
post[:topic_id] = parent[:topic_id]
post[:reply_to_post_number] = parent[:post_number] if parent[:post_number] > 1
else
# If the first post of the topic has been deleted, we use the first remaining available (default Kunena behaviour)
if first_topic_post.first['parent'] != 0
if mapping_categories.key?(m['catid'].to_s)
post[:category] = mapping_categories[m['catid'].to_s]
else
puts "SC Catégorie " + m['catid'].to_s + " non trouvée dans le mapping."
post[:category] = 1
end
post[:title] = m['subject']
else
puts "Parent post #{parent_post_id} doesn't exist. Skipping #{m["id"]}: #{m["subject"][0..40]}"
skip = true
end
end
end
end
if m["pinned"].to_i == 1 and m['parent'].to_i == 0
post[:pinned_until] = DateTime.now.next_year(1000).to_time
post[:pinned_at] = Time.zone.at(m['time'])
end
post[:post_create_action] = proc do |action_post|
topic_id = action_post.topic.id
Jobs.enqueue_at(topic.pinned_until, :unpin_topic, topic_id: topic_id) if action_post.topic.pinned_until
title_contain_solved_pattern = m['subject'] =~ /(?:Resolu)|(?:Résolu)|(?:Solved)/i
if (m['solved'].to_i == 1 or title_contain_solved_pattern) and m['parent'].to_i == 0
time = Time.zone.at(m['time'])
DB.exec <<-SQL
INSERT INTO topic_custom_fields (name, value, topic_id, created_at, updated_at)
VALUES ('accepted_answer_post_id', -1, #{topic_id}, '#{time}', '#{time}')
SQL
end
if m['locked'].to_i == 1
DB.exec <<-SQL
UPDATE topics SET closed = true WHERE id = #{topic_id}
SQL
end
end
skip ? nil : post
end
end
end
def import_likes
puts "", "Importing post likes..."
total_count = @client.query("SELECT COUNT(*) count FROM #{KUNENA_PREFIX}kunena_thankyou;").first['count']
count = 0
batches(BATCH_SIZE) do |offset|
likes = @client.query("
SELECT postid, userid
FROM #{KUNENA_PREFIX}kunena_thankyou
LIMIT #{BATCH_SIZE}
OFFSET #{offset};
", cache_rows: false).to_a
break if likes.empty?
likes.each do |result|
print_status(count += 1, total_count, get_start_time("import_likes"))
next unless user_id = user_id_from_imported_user_id(result["userid"])
next unless post_id = post_id_from_imported_post_id(result["postid"])
next unless user = User.find_by(id: user_id)
next unless post = Post.find_by(id: post_id)
PostActionCreator.like(user, post) rescue nil
end
end
end
end
ImportScripts::Kunena.new.perform
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment