Skip to content

Instantly share code, notes, and snippets.

@uchilaka
Last active October 5, 2020 13:28
Show Gist options
  • Save uchilaka/8ff612171536788175d7b2b27addf83c to your computer and use it in GitHub Desktop.
Save uchilaka/8ff612171536788175d7b2b27addf83c to your computer and use it in GitHub Desktop.
A Google Cloud Function that acts on new Storage File objects, extracts EXIF data using ImageMagick and parses out text content using the Google Cloud Vision API
module.exports = {
parser: "babel-eslint",
parserOptions: {
ecmaVersion: 6,
sourceType: 'module',
ecmaFeatures: {
modules: true,
//experimentalObjectRestSpread: true
}
},
env: {
node: true
},
extends: [
'eslint:recommended',
],
rules: {
'no-console': 0,
//'no-extra-semi': ['error', 'always'],
semi: ['warn', 'never']
}
}
#!/bin/bash
# this is an example of the code you will need after setting up firebase-tools via npm or yarn on your dev machine.
# this piece of code updates the configuration for your function in the cloud, so you don't have to ship secrets with
# your repo
firebase functions:config:set fbse.api_key="S0meKeyFromYourGCPConsole" \
fbse.auth_domain="example-gcp-project.firebaseapp.com" \
fbse.database_url="https://example-gcp-project.firebaseio.com" \
fbse.project_id="example-gcp-project" \
fbse.storage_bucket="example-gcp-project.appspot.com" \
fbse.messaging_sender_id="9999999" \
gc_vision_api.key="example-gcp-project-api-key-with-vision-api-enabled"
// {app-root}/functions/index.js
// GCP Function directory structure:
// .
// ├── README.md
// ├── bin
// │ └── deploy.sh
// ├── firebase.json
// ├── functions
// │ ├── index.js
// │ ├── node_modules
// │ ├── package.json
// │ └── yarn.lock
// ├── package.json
// ├── private
// │ └── config-setup.sh
// └── yarn.lock
const functions = require('firebase-functions')
// // Create and Deploy Your First Cloud Functions
// // https://firebase.google.com/docs/functions/write-firebase-functions
//
// exports.helloWorld = functions.https.onRequest((request, response) => {
// response.send("Hello from Firebase!");
// });
const os = require('os')
const fs = require('fs')
const crypto = require('crypto')
const path = require('path')
const requestPromise = require('request-promise')
const gcs = require('@google-cloud/storage')()
const spawn = require('child-process-promise').spawn
const admin = require('firebase-admin')
// pull firebase configurations
const { fbse, gc_vision_api } = functions.config()
// initialize firebase
admin.initializeApp({
apiKey: fbse.api_key,
databaseURL: fbse.database_url,
authDomain: fbse.auth_domain,
projectId: fbse.project_id,
storageBucket: fbse.storage_bucket,
messagingSenderId: fbse.messaging_sender_id
})
/**
* @description - When an image is uploaded in the Storage bucket, the information and metadata of the image
* (the output of ImageMagick's `identity -verbose`) is saved in the Realtime Database
*/
exports.extractImageMetadata = functions.storage.object().onFinalize((object) => {
const filePath = object.name
// Create random filename with extension of an uploaded file
const randomFileName = crypto.randomBytes(20).toString('hex') + path.extname(filePath)
const tempLocalFile = path.join(os.tmpdir(), randomFileName)
// Exit if this is triggered on a file that is not an image
if (!object.contentType.startsWith('image/')) {
console.log(`File ${filePath} is not an image. Aborting processing.`)
return null
}
// Found an image!
let metadata
// Download file from bucket
const bucket = gcs.bucket(object.bucket)
return bucket.file(filePath)
.download({ destination: tempLocalFile })
.then(() => {
// Get Metadata from image.
return spawn('identify', ['-verbose', tempLocalFile], { capture: ['stdout', 'stderr'] })
})
.then(result => {
// Save metadata
metadata = imageMagickOutputToObject(result.stdout)
const safeKey = makeKeyFirebaseCompatible(filePath)
console.log(`FBSE compatible data key: ${safeKey}`)
// get file base64 data
const body = {
requests: []
}
body.requests.push({
image: {
content: fs.readFileSync(tempLocalFile, 'base64')
},
features: [
{ type: "DOCUMENT_TEXT_DETECTION" }
]
})
// send request
return requestPromise({
method: 'POST',
uri: 'https://vision.googleapis.com/v1/images:annotate',
qs: {
key: gc_vision_api.key
},
body,
json: true
})
.then(respo => {
let OCRData = null
if (respo && respo.responses) {
OCRData = respo.responses[0]
}
console.log('OCR Data -> ', OCRData)
return admin.database().ref(safeKey).update({ OCRData, magikExif: metadata })
})
.catch(err => {
console.log('OCR Request error', err)
return admin.database().ref(safeKey).update({ magikExif: metadata })
})
})
.then(() => {
console.log(`Wrote to: ${filePath} data: `, metadata)
return fs.unlinkSync(tempLocalFile)
})
.then(() => {
console.log('Cleanup successful!')
})
})
/**
* Convert the output of ImageMagick's `identify -verbose` command to a JavaScript Object.
*/
function imageMagickOutputToObject(output) {
let previousLineIndent = 0;
const lines = output.match(/[^\r\n]+/g);
lines.shift(); // Remove First line
lines.forEach((line, index) => {
const currentIdent = line.search(/\S/);
line = line.trim();
if (line.endsWith(':')) {
lines[index] = makeKeyFirebaseCompatible(`"${line.replace(':', '":{')}`);
} else {
const split = line.replace('"', '\\"').split(': ');
split[0] = makeKeyFirebaseCompatible(split[0]);
lines[index] = `"${split.join('":"')}",`;
}
if (currentIdent < previousLineIndent) {
lines[index - 1] = lines[index - 1].substring(0, lines[index - 1].length - 1);
lines[index] = new Array(1 + (previousLineIndent - currentIdent) / 2).join('}') + ',' + lines[index];
}
previousLineIndent = currentIdent;
});
output = lines.join('');
output = '{' + output.substring(0, output.length - 1) + '}'; // remove trailing comma.
output = JSON.parse(output);
console.log('Metadata extracted from image', output);
return output;
}
/**
* @description - Makes sure the given string does not contain characters that can't be used as Firebase
* Realtime Database keys such as '.' and replaces them by '*'.
*/
function makeKeyFirebaseCompatible(key) {
return key.replace(/\./g, '*')
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment