Skip to content

Instantly share code, notes, and snippets.

@yuumi3

yuumi3/index.ts Secret

Created Feb 7, 2020
Embed
What would you like to do?
ImpExpWebArchives
import firebase from 'firebase/app'
import "firebase/auth"
import "firebase/firestore"
import "firebase/storage"
import fs from "fs"
import request from "request"
import config from "./firebaseConfigs"
import BLANK_PNG from "./blank_png"
(global as any).XMLHttpRequest = require("xhr2") // https://github.com/firebase/firebase-js-sdk/issues/349
interface Artcle {
title: string
category: string
created_at: firebase.firestore.Timestamp
thumb: firebase.firestore.Blob
pdf?: string
url: string
}
type UnixTime = number
type Base64String = string
interface ArtcleJson {
title: string
category: string
created_at: UnixTime
thumb: Base64String
pdf?: string
url: string
}
const articleToJson = (article: Artcle) : ArtcleJson => (
{
title: article.title,
category: article.category,
created_at: article.created_at.seconds,
thumb: article.thumb ? article.thumb.toBase64() : "",
pdf: article.pdf,
url: article.url
}
)
const articleFromJson = (json: ArtcleJson) : Artcle => {
let article:Artcle = {
title: json.title,
category: json.category,
created_at: new firebase.firestore.Timestamp(json.created_at, 0),
thumb: firebase.firestore.Blob.fromBase64String(json.thumb ? json.thumb : BLANK_PNG),
url: json.url
}
if (json.pdf) article.pdf = json.pdf
return article
}
const exportArticles = async (): Promise<ArtcleJson[]> => {
const db = firebase.firestore()
const querySnapshot = await db.collection("articles").orderBy("created_at", "desc").get()
return querySnapshot.docs.map(e => articleToJson(e.data() as Artcle))
}
const importArticles = async (artcles: Artcle[]): Promise<void> => {
const db = firebase.firestore()
await Promise.all(artcles.map(async(artcle) => await db.collection("articles").add(artcle)))
}
const downloadFile = async (url: string, filename: string): Promise<void> => {
new Promise((resolve, reject) => {
request.get(url)
.on('error', (err) => reject(err))
.on('response', (res) => {
if (res.statusCode != 200) { reject('HTTP error = ' + res.statusCode) }
})
.pipe(fs.createWriteStream(filename))
.on('close', () => {
console.log(" downloaded: " + filename)
resolve()
})
.on('error', (err) => reject(err))
})
}
const getDownloadURLs = async (pdfPaths: string[]): Promise<string[]> => {
const storage = firebase.storage()
const urls: string[] = await Promise.all(pdfPaths.map(async (pdf) => (
await storage.ref().child(pdf).getDownloadURL()
)))
return urls
}
const downloadPDFs = async (urls: string[], paths: string[]): Promise<void> => {
await Promise.all(urls.map(async(url, ix) =>
await downloadFile(url, paths[ix]))
}
const authFirebase = async (config: any): Promise<void> => {
const email = process.env.FIREBASE_LOGIN
const password = process.env.FIREBASE_PASSWORD
if (!email || !password) {
console.log("Please set FIREBASE_LOGIN and FIREBASE_PASSWORD")
process.exit(0)
}
const app = firebase.initializeApp(config)
try {
const auth = app.auth()
await auth.signInWithEmailAndPassword(email, password)
} catch (err) {
console.log('** Error', err)
}
}
const exportArchive = async (artcleFile: string, pdfDir: string): Promise<void> => {
try {
await authFirebase(config.exp)
console.log('-- export articles')
const artcles = await exportArticles()
await fs.promises.writeFile(artcleFile, JSON.stringify(artcles, undefined, 2))
console.log('-- get PDF download URLs')
const pdfPaths = (artcles.map(artcle => artcle.pdf).filter(pdf => !!pdf)) as string[]
console.log(pdfPaths.length)
const urls = await getDownloadURLs(pdfPaths)
const paths = pdfPaths.map(path => pdfDir + path.replace(/^pdf\//, "/"))
console.log('-- download PDFs')
if (!fs.existsSync(pdfDir)) {
fs.mkdirSync (pdfDir)
}
await downloadPDFs(urls, paths)
} catch (err) {
console.log('** Error', err)
}
}
const uploadPDF = async (fileName: string, filePath: string): Promise<void> => {
console.log('- upload ' + fileName)
const buffer = await fs.promises.readFile(filePath)
const storage = firebase.storage()
const pdfRef = storage.ref().child("pdf/" + fileName)
await pdfRef.put(Uint8Array.from(buffer), {contentType: 'application/pdf'})
}
const uploaAllPDFs = async (pdfDir: string): Promise<void> => {
const files = await fs.promises.readdir(pdfDir)
await Promise.all(files.map(async(file) => uploadPDF(file, pdfDir + "/" + file)))
}
const importArchive = async (artcleFile: string, pdfDir: string): Promise<void> => {
try {
await authFirebase(config.imp)
console.log('-- import articles')
const jsonText = await fs.promises.readFile(artcleFile, 'utf-8')
const artclesJson = JSON.parse(jsonText) as ArtcleJson[]
const articles = artclesJson.map(json => articleFromJson(json))
await importArticles(articles)
console.log('-- import PDFs')
await uploaAllPDFs(pdfDir)
console.log('end')
} catch (err) {
console.log('** Error', err)
}
}
if (process.argv.length < 5) {
console.log("Usage: node exp|imp ARTICLES_EXPORT_FILE PDF_DIRECTORY")
} else if (process.argv[2] == 'imp') {
importArchive(process.argv[3], process.argv[4])
} else {
exportArchive(process.argv[3], process.argv[4])
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment