Skip to content

Instantly share code, notes, and snippets.

@Duranjit
Created February 6, 2025 09:09
Show Gist options
  • Save Duranjit/856d4574238a72e42c3736bb65325376 to your computer and use it in GitHub Desktop.
Save Duranjit/856d4574238a72e42c3736bb65325376 to your computer and use it in GitHub Desktop.
Can I push data into BigQuery without using ValueSaver? I attempted to insert data directly, but it failed because the data type contains interface{}. How can I properly insert interface{} data into BigQuery without using ValueSaver? Any guidance or best practices would be appreciated.
func (b *BigQuery) PushToBigQuery(ctx context.Context, data interface{}, dataset, tableName string) error {
if dataset == "" || tableName == "" {
b.logger.Printf("Dataset or table name missing.")
return fmt.Errorf("dataset or table name is missing")
}
b.logger.Printf("Preparing to push data to BigQuery table: %s.%s", "[dataset]", "[tableName]")
// Build the BigQuery inserter
inserter := b.client.Dataset("[dataset]").Table("[tableName]").Inserter()
// Convert the incoming data to a slice of ValueSavers dynamically
items, err := createValueSavers(data)
if err != nil {
b.logger.Printf("Failed to create ValueSavers: %v", err)
return fmt.Errorf("failed to create ValueSavers: %v", err)
}
const batchSize = 500
// Insert in batches
for i := 0; i < len(items); i += batchSize {
end := i + batchSize
if end > len(items) {
end = len(items)
}
batch := items[i:end]
b.logger.Printf("Uploading batch %d to %d (out of %d items)", i, end, len(items))
if err := inserter.Put(ctx, batch); err != nil {
b.logger.Printf("Failed to upload batch %d to %d: %v", i, end, err)
return fmt.Errorf("failed to upload data to BigQuery table '%s': %v", "[tableName]", err)
}
}
b.logger.Printf("Data successfully pushed to BigQuery table: %s.%s", "[dataset]", "[tableName]")
return nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment