Skip to content

Instantly share code, notes, and snippets.

@jdanbrown
Last active April 4, 2017 01:17
Show Gist options
  • Save jdanbrown/c08c60d388b16d6052b4545ab791a9af to your computer and use it in GitHub Desktop.
Save jdanbrown/c08c60d388b16d6052b4545ab791a9af to your computer and use it in GitHub Desktop.
bq: unnest array vs. union select
# TODO(db): Figure out if either of these is better than the other
# - Both give "query resources exceeded" on what_is_junk with all of the homes/addresses fields
def template_unnest_array(table, fields):
df = bq.Query(with_od_homes('''
select * from unnest([
%(array_elems)s
])
order by i
''' % dict(
array_elems=',\n'.join([
'''
struct(
%(i)s as i,
...
)
''' % dict(
i=i,
...
)
for i, ... in enumerate(...)
]),
))).to_dataframe()
df = df.drop('i', axis=1)
return df
def template_union_select(table, fields):
df = bq.Query(with_od_homes('''
%(selects)s
order by i
''' % dict(
selects='\nunion all\n'.join([
'''
select
%(i)s as i,
...
from %(table)s
''' % dict(
i=i,
...
)
for i, ... in enumerate(...)
])
))).to_dataframe()
df = df.drop('i', axis=1)
return df
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment