VaccineMY101-API-behind-the-scene

centre.json

[
    {
        "st": "Johor",
        "dist": "Kota Tinggi",
        "ppvc": "KK Bandar Penawar",
        "cd": "johor",
        "lat": "1.55332",
        "lon": "104.23373",
        "linky": ""
    },
    {
        "st": "Johor",
        "dist": "Mersing",
        "ppvc": "KK Tenggaroh 2",
        "cd": "johor",
        "lat": "2.08402",
        "lon": "103.94636",
        "linky": ""
    },
    {
        "st": "Johor",
        "dist": "Batu Pahat",
        "ppvc": "Dewan Jubli Intan",
        "cd": "johor",
        "lat": "1.84940",
        "lon": "102.93260",
        "linky": "https://goo.gl/maps/xvqBLFLtxGoEnJXD9"
    },
    ...
]

cron_crawl_vaccination_data_functions.sh

#!/bin/bash
curl --request POST \
  --url https://my-clodfunctions-api.cloudfunctions.net/vaccinemy101api/crawl \
  --header 'Authorization: Bearer <MY-PRIVATE-API-KEY>' \
  --header 'Content-Type: application/json'

crontab.txt

*/15 * * * * /home/techrino/cron_crawl_vaccination_data_functions.sh > /home/techrino/crawl_vaccination_data_functions.txt

import.js

const os = require('os')
const path = require('path')
const fs = require('fs')
module.exports = async (object, bucket, db, log) => {
	log(object.contentType)
	log(object.bucket)
	log(object.name)
	log(os.tmpdir())
	let filename = path.basename(object.name)
	const tempFilePath = path.join(os.tmpdir(),filename )
	if (object.contentType === "application/json" && object.name.startsWith("centre")) {
		log(`Download to temp file path ${tempFilePath}`)
		try {
			await deleteCollection(db, 'vaccination_centre')
			await bucket.file(object.name).download({ destination: tempFilePath, validation: false })
			let records = require(tempFilePath)
			const batch = db.batch();
			let stateRecorded = {};
			
			let parent = null;
			records.forEach((record, index) => {
				parent = db.collection(`vaccination_centre`).doc(record.cd)
				if (stateRecorded[record.cd] === undefined) {
					batch.set(parent, { state: record.cd, state_fullname: record.st})
					stateRecorded[record.cd] = 1;
				} else {
					stateRecorded[record.cd] += 1;
				}
				let location = parent.collection('locations')
				batch.set(location.doc(), record)
			})
			commitResponse = await batch.commit();

			updateBatch = db.batch();
			Object.keys(stateRecorded).forEach(stateMeta => {
				record = db.collection(`vaccination_centre`).doc(stateMeta)
				updateBatch.update(record, {count: stateRecorded[stateMeta]})
			})

			await updateBatch.commit();

			log(`Completed  ${commitResponse.length} vaccination centre data import at ${Date()}`)
			return fs.unlinkSync(tempFilePath);
		} catch (error) {
			log(error)
		}
		
	}
}

index.js

exports.autoImportVaccinationCentre = functions.runWith(runtimeOpts).region('asia-southeast2').storage.object().onFinalize(async (object) => {
  if (object.name == "centre.json" ) {
     await autoImportVaccinationCentre(object, bucket, db, log)
  }
});

sync.js

const axios = require('axios');

module.exports =  (req, res, functions, db, log) => {
	if (req.headers.authorization === `Bearer ${functions.config().server.cronjob_key}`) {
		return handleCrawl(req, res, functions, db, log)
	}
	return res.status(401).json({result: "unauthorized"})
}


async function handleCrawl(req, res, functions, db, log)  {
	let currentUnixTime = Math.floor(Date.now() / 1000)

	let params = {};
	params[currentUnixTime] = null;

	log(`Incoming Request from ${req.ip}`)
	// crawl news
	try {
		let statisticResponse = await axios.get("https://covidbucketbbc.s3-ap-southeast-1.amazonaws.com/heatdata.json", {
			params,
			headers: {
				"User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36 Edg/91.0.864.41"
			}
		});
	
		await deleteCollection(db, "statistics")
		
		const statisticBatch = db.batch();
		statisticResponse.data.data.forEach((record) => {
			let _record = {
				nme: record['nme'] ?? "",
				id: record['nme'] ? record['nme'].toLowerCase().split(" ").join("") : "",
				regtotal: Number(record['regtotal'] ?? 0),
				pop_18: Number(record['pop_18'] ?? 0),
				vakdose1: Number(record['vakdose1'] ?? 0),
				vakdose2: Number(record['vakdose2'] ?? 0),
				vakdosecomplete: Number(record['vakdosecomplete'] ?? 0)
			}
			statisticBatch.set(db.collection('statistics').doc(), _record)
		})
		let statisticCommitResponse = await statisticBatch.commit();
	
		await db.collection('meta').doc('statistics').set({lastUpdated: statisticResponse.data.updated})
		let result = { result: "done", message: `${statisticCommitResponse.length} state statistic records imported` }
		log(result)
		return res.status(200).json(result)
	} catch (error) {
		log(error)
		return res.status(500).json({result: error.toString()})
	}

}
async function deleteCollection(db, collectionPath) {
  const collectionRef = db.collection(collectionPath);
  const query = collectionRef.orderBy('__name__');

  return new Promise((resolve, reject) => {
    deleteQueryBatch(db, query, resolve).catch(reject);
  });
}

async function deleteQueryBatch(db, query, resolve) {
  const snapshot = await query.get();

  const batchSize = snapshot.size;
  if (batchSize === 0) {
    // When there are no documents left, we are done
    resolve();
    return;
  }

  // Delete documents in a batch
  const batch = db.batch();
  snapshot.docs.forEach((doc) => {
    batch.delete(doc.ref);
  });
  await batch.commit();

  // Recurse on the next process tick, to avoid
  // exploding the stack.
  process.nextTick(() => {
    deleteQueryBatch(db, query, resolve);
  });
}