ImpExpWebArchives

index.ts

import firebase from 'firebase/app'
import "firebase/auth"
import "firebase/firestore"
import "firebase/storage"
import fs from "fs"
import request from "request"
import config from "./firebaseConfigs"
import BLANK_PNG from "./blank_png"

(global as any).XMLHttpRequest = require("xhr2") // https://github.com/firebase/firebase-js-sdk/issues/349

interface Artcle {
  title: string
  category: string
  created_at: firebase.firestore.Timestamp
  thumb: firebase.firestore.Blob
  pdf?: string
  url: string
}

type UnixTime = number
type Base64String = string
interface ArtcleJson {
  title: string
  category: string
  created_at: UnixTime
  thumb: Base64String
  pdf?: string
  url: string
}


const articleToJson = (article: Artcle) : ArtcleJson => (
  {
    title: article.title,
    category: article.category,
    created_at: article.created_at.seconds,
    thumb: article.thumb ? article.thumb.toBase64() : "",
    pdf: article.pdf,
    url: article.url
  }
)
const articleFromJson = (json: ArtcleJson) : Artcle => {
  let article:Artcle = {
    title: json.title,
    category: json.category,
    created_at: new firebase.firestore.Timestamp(json.created_at, 0),
    thumb: firebase.firestore.Blob.fromBase64String(json.thumb ? json.thumb : BLANK_PNG),
    url: json.url
  }
  if (json.pdf) article.pdf = json.pdf

  return article
}

const exportArticles = async (): Promise<ArtcleJson[]> => {
  const db = firebase.firestore()
  const querySnapshot = await db.collection("articles").orderBy("created_at", "desc").get()
  return querySnapshot.docs.map(e => articleToJson(e.data() as Artcle))
}

const importArticles = async (artcles: Artcle[]): Promise<void> => {
  const db = firebase.firestore()
  await Promise.all(artcles.map(async(artcle) => await db.collection("articles").add(artcle)))
}

const downloadFile = async (url: string, filename: string): Promise<void> => {
  new Promise((resolve, reject) => {
    request.get(url)
    .on('error', (err) => reject(err))
    .on('response', (res) => {
      if (res.statusCode != 200)  { reject('HTTP error = ' + res.statusCode) }
    })
    .pipe(fs.createWriteStream(filename))
      .on('close', () => {
        console.log(" downloaded: " + filename)
        resolve()
      })
      .on('error', (err) => reject(err))
  })
}

const getDownloadURLs = async (pdfPaths: string[]): Promise<string[]> => {
  const storage = firebase.storage()

  const urls: string[] = await Promise.all(pdfPaths.map(async (pdf) => (
     await storage.ref().child(pdf).getDownloadURL()
  )))

  return urls
}

const downloadPDFs = async (urls: string[], paths: string[]): Promise<void> => {
  await Promise.all(urls.map(async(url, ix) =>
    await downloadFile(url, paths[ix]))
}

const authFirebase = async (config: any): Promise<void> => {
  const email = process.env.FIREBASE_LOGIN
  const password = process.env.FIREBASE_PASSWORD
  if (!email || !password) {
    console.log("Please set FIREBASE_LOGIN and FIREBASE_PASSWORD")
    process.exit(0)
  }
  const app = firebase.initializeApp(config)
  try {
    const auth = app.auth()
    await auth.signInWithEmailAndPassword(email, password)
  } catch (err) {
    console.log('** Error', err)
  }
}

const exportArchive = async (artcleFile: string, pdfDir: string): Promise<void> => {
  try {
    await authFirebase(config.exp)
    console.log('-- export articles')
    const artcles = await exportArticles()
    await fs.promises.writeFile(artcleFile, JSON.stringify(artcles, undefined, 2))

    console.log('-- get PDF download URLs')
    const pdfPaths = (artcles.map(artcle => artcle.pdf).filter(pdf => !!pdf)) as string[]
    console.log(pdfPaths.length)
    const urls = await getDownloadURLs(pdfPaths)
    const paths = pdfPaths.map(path => pdfDir + path.replace(/^pdf\//, "/"))

    console.log('-- download PDFs')
    if (!fs.existsSync(pdfDir)) {
      fs.mkdirSync (pdfDir)
    }
    await downloadPDFs(urls, paths)

  } catch (err) {
    console.log('** Error', err)
  }
}

const uploadPDF = async (fileName: string, filePath: string): Promise<void> => {
  console.log('- upload ' + fileName)
  const buffer = await fs.promises.readFile(filePath)

  const storage = firebase.storage()
  const pdfRef = storage.ref().child("pdf/" + fileName)
  await pdfRef.put(Uint8Array.from(buffer), {contentType: 'application/pdf'})
}

const uploaAllPDFs = async (pdfDir: string): Promise<void> => {
  const files = await fs.promises.readdir(pdfDir)
  await Promise.all(files.map(async(file) => uploadPDF(file, pdfDir + "/" + file)))
}

const importArchive = async (artcleFile: string, pdfDir: string): Promise<void> => {
  try {
    await authFirebase(config.imp)
    console.log('-- import articles')
    const jsonText = await fs.promises.readFile(artcleFile, 'utf-8')
    const artclesJson = JSON.parse(jsonText) as ArtcleJson[]
    const articles = artclesJson.map(json => articleFromJson(json))
    await importArticles(articles)
    console.log('-- import PDFs')
    await uploaAllPDFs(pdfDir)
    console.log('end')
  } catch (err) {
    console.log('** Error', err)
  }
}

if (process.argv.length < 5) {
  console.log("Usage: node exp|imp ARTICLES_EXPORT_FILE PDF_DIRECTORY")
} else if (process.argv[2] == 'imp') {
  importArchive(process.argv[3], process.argv[4])
} else {
  exportArchive(process.argv[3], process.argv[4])
}