Skip to content

Instantly share code, notes, and snippets.

@null-dev
Created July 2, 2021 09:12
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save null-dev/b7e25331b4613f8b1ab1132ed7f1d464 to your computer and use it in GitHub Desktop.
Save null-dev/b7e25331b4613f8b1ab1132ed7f1d464 to your computer and use it in GitHub Desktop.
Import likes from Soundcloud into NewPipe.
[package]
name = "NewPipeSoundcloudImport"
version = "0.1.0"
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
rusqlite = "0.25.3"
reqwest = "0.11.4"
scraper = "0.12.0"
tokio = { version = "1.7.1", features = ["full"] }
anyhow = "1.0.41"
serde_json = "1.0.64"
chrono = "0.4.19"
extern crate reqwest;
extern crate tokio;
extern crate anyhow;
extern crate serde_json;
extern crate chrono;
extern crate rusqlite;
use std::env::{args, args_os};
use reqwest::{Client, Response};
use anyhow::{anyhow, Result, Error};
use scraper::{Html, Selector};
use reqwest::header::{HeaderMap, HeaderValue};
use chrono::DateTime;
use rusqlite::{Connection, params, OptionalExtension};
use std::path::Path;
async fn get_user_id(client: &Client, user: &str) -> Result<String> {
let url = format!("https://soundcloud.com/{}", user);
let doc = client.get(url)
.send()
.await?
.text()
.await?;
let parsed_doc = Html::parse_fragment(&doc);
let selector = Selector::parse("meta[property='al:ios:url']").unwrap();
for element in parsed_doc.select(&selector) {
let after = element.value().attr("content").unwrap().to_owned();
let index = after.rfind(":").unwrap();
return Ok(after[index + 1 ..].to_owned());
}
return Err(anyhow!("Could not find user ID!"));
}
#[derive(Debug)]
struct Track {
url: String,
title: String,
duration: i32,
uploader: String,
thumb: String,
views: i64,
upload_date: String,
upload_date_long: i64
}
async fn get_tracks(client: &Client, user_id: &str, client_id: &str) -> Result<Vec<Track>> {
let mut url = format!(
"https://api-v2.soundcloud.com/users/{}/track_likes?client_id={}&limit=200&offset=0&linked_partitioning=1",
user_id,
client_id
);
let mut vec = Vec::new();
loop {
let doc = client.get(url)
.send()
.await?
.text()
.await?;
let json: serde_json::Value = serde_json::from_str(&doc)?;
if let serde_json::Value::Array(content) = json.get("collection").unwrap() {
for item in content {
if let serde_json::Value::Object(obj) = item {
if let serde_json::Value::Object(track) = obj.get("track").unwrap() {
let user = track.get("user").unwrap().as_object().unwrap();
let created_at = track.get("created_at").unwrap().as_str().unwrap().to_owned();
let parsed = DateTime::parse_from_rfc3339(&created_at).unwrap();
let track = Track {
url: track.get("permalink_url").unwrap().as_str().unwrap().to_owned(),
title: track.get("title").unwrap().as_str().unwrap().to_owned(),
duration: (track.get("full_duration").unwrap().as_i64().unwrap() / 1000) as i32,
uploader: user.get("username").unwrap().as_str().unwrap().to_owned(),
thumb: track.get("artwork_url").unwrap().as_str().or_else(|| user.get("avatar_url").unwrap().as_str()).unwrap().to_owned(),
views: track.get("playback_count").unwrap().as_i64().unwrap_or(0),
upload_date: created_at,
upload_date_long: parsed.timestamp_millis()
};
vec.push(track);
} else {
panic!("Invalid track object!");
}
} else {
panic!("Invalid like object!");
}
}
}
url = if let serde_json::Value::String(new_url) = json.get("next_href").unwrap() {
format!("{}&client_id={}", new_url, client_id)
} else {
break
}
}
return Ok(vec);
}
#[tokio::main]
async fn main() {
let args: Vec<String> = args().collect();
let client_id = &args[1];
let username = &args[2];
let db = &args[3];
let client = Client::builder()
.user_agent("Mozilla/5.0 (X11; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0")
.build()
.unwrap();
let user_id = get_user_id(&client, username).await.unwrap();
let conn = Connection::open(Path::new(db)).unwrap();
let mut prev_uid: i64 = conn.query_row("SELECT uid FROM streams ORDER BY uid DESC LIMIT 1;", [], |r| r.get(0)).unwrap();
let tracks = get_tracks(&client, &user_id, client_id).await.unwrap();
let mut idx = 0;
for track in tracks {
let old_row: Option<i64> = conn.query_row("SELECT uid FROM streams WHERE url = ?1 LIMIT 1;", params![track.url], |r| r.get(0)).optional().unwrap();
let new_uid = if let Some(some) = old_row {
println!("Inserting (old): {}", idx);
some
} else {
println!("Inserting (new): {}", idx);
prev_uid = prev_uid + 1;
conn.execute(
"INSERT INTO streams (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date, is_upload_date_approximation) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
params![prev_uid, 1, track.url, track.title, "AUDIO_STREAM", track.duration, track.uploader, track.thumb, track.views, track.upload_date, track.upload_date_long, 0]
).unwrap();
prev_uid
};
conn.execute(
"INSERT INTO playlist_stream_join (playlist_id, stream_id, join_index) VALUES (?1, ?2, ?3)",
params![2, new_uid, idx]
).unwrap();
idx = idx + 1;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment