Created
September 24, 2020 14:58
-
-
Save pepetox/a0fa238d84820f139ba821fa5a3c4fac to your computer and use it in GitHub Desktop.
Workshop OpenSpace 2020 ML - Step 2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
//gcloud functions deploy processData --runtime nodejs10 --trigger-resource mysandoxbucket --trigger-event google.storage.object.finalize --project sandbox-project | |
//set the vars | |
const datasetId = 'openspace2020'; | |
const tableId = 'imageData'; | |
//load the dependences | |
const {BigQuery} = require('@google-cloud/bigquery'); | |
const bigquery = new BigQuery(); | |
const vision = require('@google-cloud/vision'); | |
const client = new vision.ImageAnnotatorClient(); | |
exports.processData = async (file, context) => { | |
//Ask for labels of image | |
const [resultLabels] = await client.labelDetection(`gs://${file.bucket}/${file.name}`); | |
const labels = resultLabels.labelAnnotations; | |
console.log(`labels for ${file.name}:`); | |
let labelsDetected = [] | |
labels.forEach(label => labelsDetected.push(label.description)); | |
//Ask for face detection | |
const [resultFaces] = await client.faceDetection(`gs://${file.bucket}/${file.name}`); | |
const faces = resultFaces.faceAnnotations; | |
//ask safe content | |
const [resultSafe] = await client.safeSearchDetection(`gs://${file.bucket}/${file.name}`); | |
const detections = resultSafe.safeSearchAnnotation; | |
//OCR | |
const [resultOCR] = await client.documentTextDetection(`gs://${file.bucket}/${file.name}`); | |
const fullTextAnnotation = resultOCR.fullTextAnnotation; | |
//insert in bigquery | |
try { | |
let picText = "" | |
if(fullTextAnnotation && fullTextAnnotation.text){ | |
picText= JSON.stringify(fullTextAnnotation.text); | |
} | |
const rows = [ | |
{ | |
name: file.name, | |
link: `https://storage.cloud.google.com/${file.bucket}/${file.name}`, | |
labels: labelsDetected, | |
faces: faces.length, | |
adult: detections.adult, | |
medical: detections.medical, | |
spoof: detections.spoof, | |
violence: detections.violence, | |
racy: detections.racy, | |
text: picText | |
} | |
]; | |
await bigquery.dataset(datasetId).table(tableId).insert(rows).catch(e => console.error(JSON.stringify(e))); | |
console.log(`Inserted ${rows.length} rows`); | |
console.log("finish") | |
} catch (error) { | |
console.log("Can´t process the file. ¿Are you sure is a picture?") | |
} | |
}; | |
//code for generate the table´s schema on BigQuery | |
let bigquery_label = [ | |
{ | |
"name": "name", | |
"type": "STRING" | |
}, | |
{ | |
"name": "labels", | |
"type": "STRING", | |
"mode": "REPEATED" | |
}, | |
{ | |
"name": "link", | |
"type": "STRING" | |
}, | |
{ | |
"name": "faces", | |
"type": "INTEGER" | |
}, | |
{ | |
"name": "adult", | |
"type": "STRING" | |
}, | |
{ | |
"name": "medical", | |
"type": "STRING" | |
}, | |
{ | |
"name": "spoof", | |
"type": "STRING" | |
}, | |
{ | |
"name": "violence", | |
"type": "STRING" | |
}, | |
{ | |
"name": "racy", | |
"type": "STRING" | |
}, | |
{ | |
"name": "text", | |
"type": "STRING" | |
} | |
] | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "taller-openspace", | |
"version": "1.0.0", | |
"description": "", | |
"main": "index.js", | |
"scripts": { | |
"test": "echo \"Error: no test specified\" && exit 1" | |
}, | |
"author": "", | |
"license": "ISC", | |
"dependencies": { | |
"@google-cloud/bigquery": "^5.2.0", | |
"@google-cloud/vision": "^2.1.1" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment