Created
February 6, 2025 09:09
-
-
Save Duranjit/856d4574238a72e42c3736bb65325376 to your computer and use it in GitHub Desktop.
Can I push data into BigQuery without using ValueSaver? I attempted to insert data directly, but it failed because the data type contains interface{}. How can I properly insert interface{} data into BigQuery without using ValueSaver? Any guidance or best practices would be appreciated.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
func (b *BigQuery) PushToBigQuery(ctx context.Context, data interface{}, dataset, tableName string) error { | |
if dataset == "" || tableName == "" { | |
b.logger.Printf("Dataset or table name missing.") | |
return fmt.Errorf("dataset or table name is missing") | |
} | |
b.logger.Printf("Preparing to push data to BigQuery table: %s.%s", "[dataset]", "[tableName]") | |
// Build the BigQuery inserter | |
inserter := b.client.Dataset("[dataset]").Table("[tableName]").Inserter() | |
// Convert the incoming data to a slice of ValueSavers dynamically | |
items, err := createValueSavers(data) | |
if err != nil { | |
b.logger.Printf("Failed to create ValueSavers: %v", err) | |
return fmt.Errorf("failed to create ValueSavers: %v", err) | |
} | |
const batchSize = 500 | |
// Insert in batches | |
for i := 0; i < len(items); i += batchSize { | |
end := i + batchSize | |
if end > len(items) { | |
end = len(items) | |
} | |
batch := items[i:end] | |
b.logger.Printf("Uploading batch %d to %d (out of %d items)", i, end, len(items)) | |
if err := inserter.Put(ctx, batch); err != nil { | |
b.logger.Printf("Failed to upload batch %d to %d: %v", i, end, err) | |
return fmt.Errorf("failed to upload data to BigQuery table '%s': %v", "[tableName]", err) | |
} | |
} | |
b.logger.Printf("Data successfully pushed to BigQuery table: %s.%s", "[dataset]", "[tableName]") | |
return nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment