Last active
October 5, 2020 13:28
-
-
Save uchilaka/8ff612171536788175d7b2b27addf83c to your computer and use it in GitHub Desktop.
A Google Cloud Function that acts on new Storage File objects, extracts EXIF data using ImageMagick and parses out text content using the Google Cloud Vision API
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module.exports = { | |
parser: "babel-eslint", | |
parserOptions: { | |
ecmaVersion: 6, | |
sourceType: 'module', | |
ecmaFeatures: { | |
modules: true, | |
//experimentalObjectRestSpread: true | |
} | |
}, | |
env: { | |
node: true | |
}, | |
extends: [ | |
'eslint:recommended', | |
], | |
rules: { | |
'no-console': 0, | |
//'no-extra-semi': ['error', 'always'], | |
semi: ['warn', 'never'] | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# this is an example of the code you will need after setting up firebase-tools via npm or yarn on your dev machine. | |
# this piece of code updates the configuration for your function in the cloud, so you don't have to ship secrets with | |
# your repo | |
firebase functions:config:set fbse.api_key="S0meKeyFromYourGCPConsole" \ | |
fbse.auth_domain="example-gcp-project.firebaseapp.com" \ | |
fbse.database_url="https://example-gcp-project.firebaseio.com" \ | |
fbse.project_id="example-gcp-project" \ | |
fbse.storage_bucket="example-gcp-project.appspot.com" \ | |
fbse.messaging_sender_id="9999999" \ | |
gc_vision_api.key="example-gcp-project-api-key-with-vision-api-enabled" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// {app-root}/functions/index.js | |
// GCP Function directory structure: | |
// . | |
// ├── README.md | |
// ├── bin | |
// │ └── deploy.sh | |
// ├── firebase.json | |
// ├── functions | |
// │ ├── index.js | |
// │ ├── node_modules | |
// │ ├── package.json | |
// │ └── yarn.lock | |
// ├── package.json | |
// ├── private | |
// │ └── config-setup.sh | |
// └── yarn.lock | |
const functions = require('firebase-functions') | |
// // Create and Deploy Your First Cloud Functions | |
// // https://firebase.google.com/docs/functions/write-firebase-functions | |
// | |
// exports.helloWorld = functions.https.onRequest((request, response) => { | |
// response.send("Hello from Firebase!"); | |
// }); | |
const os = require('os') | |
const fs = require('fs') | |
const crypto = require('crypto') | |
const path = require('path') | |
const requestPromise = require('request-promise') | |
const gcs = require('@google-cloud/storage')() | |
const spawn = require('child-process-promise').spawn | |
const admin = require('firebase-admin') | |
// pull firebase configurations | |
const { fbse, gc_vision_api } = functions.config() | |
// initialize firebase | |
admin.initializeApp({ | |
apiKey: fbse.api_key, | |
databaseURL: fbse.database_url, | |
authDomain: fbse.auth_domain, | |
projectId: fbse.project_id, | |
storageBucket: fbse.storage_bucket, | |
messagingSenderId: fbse.messaging_sender_id | |
}) | |
/** | |
* @description - When an image is uploaded in the Storage bucket, the information and metadata of the image | |
* (the output of ImageMagick's `identity -verbose`) is saved in the Realtime Database | |
*/ | |
exports.extractImageMetadata = functions.storage.object().onFinalize((object) => { | |
const filePath = object.name | |
// Create random filename with extension of an uploaded file | |
const randomFileName = crypto.randomBytes(20).toString('hex') + path.extname(filePath) | |
const tempLocalFile = path.join(os.tmpdir(), randomFileName) | |
// Exit if this is triggered on a file that is not an image | |
if (!object.contentType.startsWith('image/')) { | |
console.log(`File ${filePath} is not an image. Aborting processing.`) | |
return null | |
} | |
// Found an image! | |
let metadata | |
// Download file from bucket | |
const bucket = gcs.bucket(object.bucket) | |
return bucket.file(filePath) | |
.download({ destination: tempLocalFile }) | |
.then(() => { | |
// Get Metadata from image. | |
return spawn('identify', ['-verbose', tempLocalFile], { capture: ['stdout', 'stderr'] }) | |
}) | |
.then(result => { | |
// Save metadata | |
metadata = imageMagickOutputToObject(result.stdout) | |
const safeKey = makeKeyFirebaseCompatible(filePath) | |
console.log(`FBSE compatible data key: ${safeKey}`) | |
// get file base64 data | |
const body = { | |
requests: [] | |
} | |
body.requests.push({ | |
image: { | |
content: fs.readFileSync(tempLocalFile, 'base64') | |
}, | |
features: [ | |
{ type: "DOCUMENT_TEXT_DETECTION" } | |
] | |
}) | |
// send request | |
return requestPromise({ | |
method: 'POST', | |
uri: 'https://vision.googleapis.com/v1/images:annotate', | |
qs: { | |
key: gc_vision_api.key | |
}, | |
body, | |
json: true | |
}) | |
.then(respo => { | |
let OCRData = null | |
if (respo && respo.responses) { | |
OCRData = respo.responses[0] | |
} | |
console.log('OCR Data -> ', OCRData) | |
return admin.database().ref(safeKey).update({ OCRData, magikExif: metadata }) | |
}) | |
.catch(err => { | |
console.log('OCR Request error', err) | |
return admin.database().ref(safeKey).update({ magikExif: metadata }) | |
}) | |
}) | |
.then(() => { | |
console.log(`Wrote to: ${filePath} data: `, metadata) | |
return fs.unlinkSync(tempLocalFile) | |
}) | |
.then(() => { | |
console.log('Cleanup successful!') | |
}) | |
}) | |
/** | |
* Convert the output of ImageMagick's `identify -verbose` command to a JavaScript Object. | |
*/ | |
function imageMagickOutputToObject(output) { | |
let previousLineIndent = 0; | |
const lines = output.match(/[^\r\n]+/g); | |
lines.shift(); // Remove First line | |
lines.forEach((line, index) => { | |
const currentIdent = line.search(/\S/); | |
line = line.trim(); | |
if (line.endsWith(':')) { | |
lines[index] = makeKeyFirebaseCompatible(`"${line.replace(':', '":{')}`); | |
} else { | |
const split = line.replace('"', '\\"').split(': '); | |
split[0] = makeKeyFirebaseCompatible(split[0]); | |
lines[index] = `"${split.join('":"')}",`; | |
} | |
if (currentIdent < previousLineIndent) { | |
lines[index - 1] = lines[index - 1].substring(0, lines[index - 1].length - 1); | |
lines[index] = new Array(1 + (previousLineIndent - currentIdent) / 2).join('}') + ',' + lines[index]; | |
} | |
previousLineIndent = currentIdent; | |
}); | |
output = lines.join(''); | |
output = '{' + output.substring(0, output.length - 1) + '}'; // remove trailing comma. | |
output = JSON.parse(output); | |
console.log('Metadata extracted from image', output); | |
return output; | |
} | |
/** | |
* @description - Makes sure the given string does not contain characters that can't be used as Firebase | |
* Realtime Database keys such as '.' and replaces them by '*'. | |
*/ | |
function makeKeyFirebaseCompatible(key) { | |
return key.replace(/\./g, '*') | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment