Skip to content

Instantly share code, notes, and snippets.

@ebendutoit
Last active September 25, 2020 13:16
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ebendutoit/1717e5eba2f55ab23544153d2ef098a8 to your computer and use it in GitHub Desktop.
Save ebendutoit/1717e5eba2f55ab23544153d2ef098a8 to your computer and use it in GitHub Desktop.
Tranforming BigQuery query json results into a flattened version that's easier to use
convertBQToMySQLResults(schema, rows) {
var resultRows = []
function recurse (schemaCur, rowsCur, colName) {
if (Array.isArray(schemaCur) && !Array.isArray(result[colName])) {
for(var i=0, l=schemaCur.length; i<l; i++) {
if (colName === "")
recurse(schemaCur[i], rowsCur.f[i], colName + schemaCur[i].name)
else
recurse(schemaCur[i], rowsCur.f[i], colName + "." + schemaCur[i].name)
}
}
if (schemaCur.type && schemaCur.type === "RECORD") {
if (schemaCur.mode !== "REPEATED") {
var valIndex = 0
for (var p in schemaCur.fields) {
if (rowsCur.v === null) {
recurse(schemaCur.fields[p], rowsCur, colName + "." + schemaCur.fields[p].name)
} else {
recurse(schemaCur.fields[p], rowsCur.v.f[valIndex], colName + "." + schemaCur.fields[p].name)
}
valIndex++
}
}
if (schemaCur.mode === "REPEATED") {
result[colName] = []
for (var x in rowsCur.v) {
recurse(schemaCur.fields, rowsCur.v[x], colName)
}
}
} else {
if (schemaCur.mode === "REPEATED") {
if (rowsCur.v !== null) {
result[colName] = rowsCur.v.map( (value, index) => { return value.v })
} else {
result[colName] = [ null ]
}
} else if (Array.isArray(result[colName])) {
let nextRow = {}
for (var j in schemaCur) {
nextRow[colName + "." + schemaCur[j].name] = Array.isArray(rowsCur.v.f[j].v) ? rowsCur.v.f[j].v.map( (value, index) => { return value.v }) : rowsCur.v.f[j].v
}
result[colName].push(nextRow)
} else {
if (colName !== "")
result[colName] = rowsCur.v
}
}
}
for (var r=0, rowsCount=rows.length; r<rowsCount; r++) {
var result = {};
recurse(schema, rows[r], "")
resultRows.push(result)
}
return resultRows
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment