Skip to content

Instantly share code, notes, and snippets.

@manuc66
Created June 13, 2023 21:33
Show Gist options
  • Save manuc66/a21cf9de87f040b505a7a13b7d5ef5cf to your computer and use it in GitHub Desktop.
Save manuc66/a21cf9de87f040b505a7a13b7d5ef5cf to your computer and use it in GitHub Desktop.
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using Newtonsoft.Json;
using SQLite;
namespace SignalMediaExporter
{
public class Program
{
private static readonly string SignalDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config/Signal");
public static void Main(string[] args)
{
var config = new
{
config = "./config.json",
maxMessages = 0,
outputDir = "./media",
signalDir = SignalDir,
sqlcipher = new { cipher_compatibility = 4 }
};
var arguments = ParseArguments(args);
config = MergeConfigs(config, arguments);
var key = GetKey(config);
var messages = GetMessages(config, key);
var stats = new Dictionary<string, int>();
using (var report = Progress(verbose: config.verbose, stats, messages.Count))
{
foreach (var msg in messages)
{
var msgStats = SaveAttachments(config, msg.Item1, msg.Item2);
if (msgStats != null)
{
foreach (var pair in msgStats)
{
if (!stats.ContainsKey(pair.Key))
stats[pair.Key] = 0;
stats[pair.Key] += pair.Value;
}
}
report();
}
}
if (stats.Count == 0)
{
Console.WriteLine("No media messages found.");
Environment.Exit(-1);
}
}
private static object ParseArguments(string[] args)
{
var arguments = new
{
config = (string)null,
outputDir = (string)null,
signalDir = (string)null,
includeExpiringMessages = false,
includeAttachments = (string)null,
verbose = false,
maxMessages = (int?)null
};
// Parse arguments and populate the 'arguments' object
return arguments;
}
private static T MergeConfigs<T>(T defaultConfig, T customConfig)
{
var jsonSettings = new JsonSerializerSettings
{
ObjectCreationHandling = ObjectCreationHandling.Replace,
NullValueHandling = NullValueHandling.Ignore
};
var defaultJson = JsonConvert.SerializeObject(defaultConfig, Formatting.Indented, jsonSettings);
var customJson = JsonConvert.SerializeObject(customConfig, Formatting.Indented, jsonSettings);
var mergedConfig = JsonConvert.DeserializeObject<T>(defaultJson);
JsonConvert.PopulateObject(customJson, mergedConfig);
return mergedConfig;
}
private static string GetKey(dynamic config)
{
var configPath = Path.Combine(config.signalDir, "config.json");
var signalConfig = JsonConvert.DeserializeObject(File.ReadAllText(configPath));
var key = signalConfig.key;
Console.WriteLine($"Read sqlcipher key: 0x{key.Substring(0, 8)}...");
return key;
}
private static List<Tuple<string, dynamic>> GetMessages(dynamic config, string key)
{
Console.WriteLine("Connecting to sql/db.sqlite, reading messages...");
var dbPath = Path.Combine(config.signalDir, "sql/db.sqlite");
var connection = new SQLiteConnection(dbPath);
connection.SetKey(Encoding.UTF8.GetBytes(key));
foreach (var setting in config.sqlcipher)
{
var query = $"PRAGMA {setting.Key} = {setting.Value}";
connection.Execute(query);
}
var numberQuery = "SELECT json FROM items WHERE id = ?";
var numberJson = connection.ExecuteScalar<string>(numberQuery, "number_id");
var numberId = JsonConvert.DeserializeObject(numberJson);
var ownNumber = numberId.value.Split('.')[0];
var deviceId = numberId.value.Split('.')[1];
Console.WriteLine($"Own number: {ownNumber}, device ID: {deviceId}");
var include = config.includeAttachments ?? "visual";
var includeQuery = "";
if (include == "visual")
includeQuery = "hasVisualMediaAttachments > 0";
else if (include == "file")
includeQuery = "hasFileAttachments > 0";
else if (include == "all")
includeQuery = "hasAttachments > 0";
else
throw new ArgumentException($"Invalid value '{include}' for 'includeAttachments' in config");
var expiringQuery = config.includeExpiringMessages ? "" : "expires_at is null";
var limitQuery = config.maxMessages > 0 ? $"LIMIT {config.maxMessages}" : "";
var messagesQuery = $"SELECT id, json FROM messages WHERE {includeQuery} AND {expiringQuery} ORDER BY sent_at {limitQuery}";
return connection.Query<(string, string)>(messagesQuery)
.Select(row => Tuple.Create(row.Item1, JsonConvert.DeserializeObject(row.Item2))).ToList();
}
private static Dictionary<string, int> SaveAttachments(dynamic config, string msgId, dynamic msg)
{
var stats = new Dictionary<string, int>();
try
{
var sent = DateTimeOffset.FromUnixTimeMilliseconds(msg.sent_at).DateTime;
var sender = msg.source;
if (config.map != null && config.map.ContainsKey(sender))
sender = config.map[sender];
for (var idx = 0; idx < msg.attachments.Length; idx++)
{
var attachment = msg.attachments[idx];
if (!attachment.contentType.ToLower().StartsWith("image/") &&
!attachment.contentType.ToLower().StartsWith("video/") &&
!attachment.contentType.ToLower().StartsWith("audio/"))
continue;
var ext = GetFileExtension(attachment);
var name = new List<string> { "signal", sent.ToString("yyyy-MM-dd-HHmmss") };
if (msg.attachments.Length > 1)
name.Add(idx.ToString());
var fileName = $"{string.Join("-", name)}.{ext}";
if (attachment.pending || string.IsNullOrEmpty(attachment.path))
{
Console.WriteLine($"Skipping {sender}/{fileName} (media file not downloaded)");
continue;
}
var atPath = attachment.path.Replace("\\", "/");
var src = Path.Combine(config.signalDir, "attachments.noindex", atPath);
var dst = Path.Combine(config.outputDir, sender, fileName);
if (!File.Exists(src))
{
Console.WriteLine($"Skipping {sender}/{fileName} (media file not found)");
continue;
}
stats["attachments"] = stats.TryGetValue("attachments", out var attachments) ? attachments + 1 : 1;
stats["attachments_size"] = stats.TryGetValue("attachments_size", out var attachmentsSize) ? attachmentsSize + new FileInfo(src).Length : new FileInfo(src).Length;
var quickHash = HashFileQuick(src);
if (config.hashes != null && config.hashes.ContainsKey(quickHash))
{
if (HashFileSha256(src) == config.hashes[quickHash].First(hash => hash == HashFileSha256(hash)))
{
Console.WriteLine($"Skipping {sender}/{fileName} (already saved an identical file)");
continue;
}
}
if (File.Exists(dst))
{
Console.WriteLine($"Skipping {sender}/{fileName} (file exists)");
config.hashes[quickHash] = config.hashes.TryGetValue(quickHash, out var hashList) ? hashList : new List<string>();
config.hashes[quickHash].Add(src);
continue;
}
Directory.CreateDirectory(Path.GetDirectoryName(dst));
File.Copy(src, dst);
try
{
File.SetLastWriteTimeUtc(dst, sent);
}
catch (Exception)
{
// Ignore PermissionError
}
var size = new FileInfo(dst).Length;
Console.WriteLine($"Saved {dst} [{size / 1024.0:F1} KiB]");
stats["saved_attachments"] = stats.TryGetValue("saved_attachments", out var savedAttachments) ? savedAttachments + 1 : 1;
stats["saved_attachments_size"] = stats.TryGetValue("saved_attachments_size", out var savedAttachmentsSize) ? savedAttachmentsSize + size : size;
config.hashes[quickHash] = config.hashes.TryGetValue(quickHash, out var hashes) ? hashes : new List<string>();
config.hashes[quickHash].Add(src);
}
}
catch (Exception e)
{
Console.WriteLine($"Skipping {msgId} ({e.Message})");
}
return stats;
}
private static string GetFileExtension(dynamic attachment)
{
var ext = attachment.contentType.ToLower().Split('/')[1];
if (ext.Contains(";"))
ext = ext.Split(';')[0];
return ext;
}
private static string SanitizePhoneNumber(string phoneNumber)
{
return Regex.Replace(phoneNumber, @"[^+\d]", "");
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment