Skip to content

Instantly share code, notes, and snippets.

@Highstaker
Created May 20, 2019 17:41
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Highstaker/6976175d4c1e877c9b721aa1016ed665 to your computer and use it in GitHub Desktop.
Save Highstaker/6976175d4c1e877c9b721aa1016ed665 to your computer and use it in GitHub Desktop.
Just some experiments and attempts to abstract MySQL quering boilerplate away.
package main
import (
"database/sql"
"errors"
"fmt"
"log"
"math/rand"
"runtime"
"strings"
"sync"
"time"
_ "github.com/go-sql-driver/mysql"
)
const dbAddrress = "root:12345@(172.17.0.3:3306)"
const maxDBConnections = 100
//global manager of connections
var mainDB *sql.DB
func initDatabase() {
//creates the manager that handles connections by itself. No need to close it
lMainDB, err := sql.Open("mysql", dbAddrress+"/map")
if err != nil {
log.Fatalln("Could not create map database pool:", err.Error())
} else {
mainDB = lMainDB
//setting maximum number of database connections
mainDB.SetMaxOpenConns(maxDBConnections)
//needed to circumvent "unexpected EOF" and "invalid connection" errors as shown in https://github.com/go-sql-driver/mysql/issues/674
//maybe it should be one second?
//also, consider db.SetMaxIdleConns(0) if needed
mainDB.SetConnMaxLifetime(time.Second * 60)
}
}
var Error = fmt.Println
var Warning = fmt.Println
type dummyError struct{}
func (d dummyError) Error() string {
return "Dummy"
}
var preparedStatements = map[string]*sql.Stmt{}
var preparedStatementsMutex = sync.RWMutex{}
func getCreateStatement(db *sql.DB, query string) (*sql.Stmt, error) {
preparedStatementsMutex.RLock()
var stmt *sql.Stmt
var present bool
if stmt, present = preparedStatements[query]; present {
preparedStatementsMutex.RUnlock()
fmt.Println("statement exists for", query) //Debug
return stmt, nil
}
//If not present, let's create.
fmt.Println("Creating statement for", query) //Debug
preparedStatementsMutex.RUnlock()
preparedStatementsMutex.Lock()
defer preparedStatementsMutex.Unlock()
stmt, err := db.Prepare(query)
if err != nil {
return nil, err
}
preparedStatements[query] = stmt
return stmt, nil
}
func QueryDBMultiple(db *sql.DB, query string, params []interface{}, storages ...[]interface{}) (counter int, err error) {
err = dummyError{}
var stmt *sql.Stmt
for err != nil {
stmt, err = getCreateStatement(db, query)
if err != nil {
Error("Could not build query statement", query, ":", err.Error())
time.Sleep(time.Second * 1)
continue
}
rows, err := stmt.Query(params...)
if err != nil {
// query error
Error("Could not execute query", query, ":", err.Error())
time.Sleep(time.Second * 1)
continue
}
var lenStorages int
if len(storages) > 0 {
lenStorages = len(storages[0])
} else {
break
}
for rows.Next() {
if counter >= lenStorages {
break
}
resultsAmount := len(storages)
dest := make([]interface{}, resultsAmount)
for i := 0; i < resultsAmount; i++ {
dest[i] = &(storages[i][counter])
}
if err = rows.Scan(dest...); err != nil {
Error("Could not read query results:", query, ":", err.Error())
rows.Close()
time.Sleep(time.Second * 1)
continue
}
counter++
}
if err := rows.Err(); err != nil {
Error("Could not scan rows fully:", query, ":", err.Error())
rows.Close()
time.Sleep(time.Second * 1)
continue
}
rows.Close()
}
return counter, nil
}
func queryDBSingular(db *sql.DB, query string, params []interface{}, storages ...interface{}) error {
var err error = dummyError{}
var stmt *sql.Stmt
for err != nil {
stmt, err = getCreateStatement(db, query)
if err != nil {
Error("Could not build query statement", query, ":", err.Error())
time.Sleep(time.Second * 1)
continue
}
// fmt.Println("Quering", query) //debug
row := stmt.QueryRow(params...)
// fmt.Println("Scanning", query) //debug
err = row.Scan(storages...)
// fmt.Println("Scanning error", err) //debug
if err != nil {
if err == sql.ErrNoRows {
return sql.ErrNoRows
}
Error("Could not execute query!", err.Error())
time.Sleep(time.Second * 1)
continue
}
}
return nil
}
func execDB(db *sql.DB, query string, params []interface{}) (affectedRows int64, err error) {
err = dummyError{}
var stmt *sql.Stmt
for err != nil {
stmt, err = getCreateStatement(db, query)
if err != nil {
Error("Could not build query statement", query, ":", err.Error())
time.Sleep(time.Second * 1)
continue
}
var result sql.Result
result, err = stmt.Exec(params...)
if err != nil {
Error("Could not execute query", query, ":", err.Error())
time.Sleep(time.Second * 1)
continue
}
var erro error
affectedRows, erro = result.RowsAffected()
if erro != nil {
//Not ever ydatabase supports this.
//Not a problematic error. Will show a message but will not terminate execution.
Warning("Could not get amount of affected rows:", erro.Error())
}
}
return
}
func QueryDB(db *sql.DB, query string, params []interface{}, storages ...interface{}) (numericResult int64, err error) {
if strings.HasPrefix(strings.ToUpper(strings.Trim(query, " \n\r")), "SELECT") {
storagesAmount := len(storages)
if storagesAmount == 0 {
return 0, errors.New("storage variables are required for SELECT statement")
}
// https://stackoverflow.com/q/40343471/2052138
switch x := storages[0].(type) {
case []interface{}:
//The provided storages are slices.
//We will try fetching several rows.
firstStorageLen := len(x)
lStorages := make([][]interface{}, storagesAmount)
for i, storage := range storages {
if ss, ok := storage.([]interface{}); ok {
if len(ss) != firstStorageLen {
return 0, errors.New("storages are not of the same size")
}
lStorages[i] = ss
} else {
return 0, errors.New("not all storages are slices, even though the first one is")
}
}
// if sss, ok := storages.([]interface{}); ok {
returnedRows, erro := QueryDBMultiple(db, query, params, lStorages...)
numericResult, err = int64(returnedRows), erro
// }
case interface{}:
//The provided storages are not slices.
//Therefore we will get only one row.
erro := queryDBSingular(db, query, params, storages...)
if erro == nil {
numericResult = 1
} else if erro == sql.ErrNoRows {
err = nil
numericResult = 0
} else {
err = erro
}
default:
err = fmt.Errorf("unsupported type: %T", x)
}
} else {
numericResult, err = execDB(db, query, params)
}
return
}
const (
dbQueryStringInserter = "INSERT INTO pootis VALUES (?,?)"
dbQueryStringSelector = "SELECT arg1, arg2 from pootis where arg1=?"
dbQueryStringUpdater = "UPDATE pootis SET arg2=? WHERE arg1=42"
)
func randomUint8() uint8 {
return uint8(rand.Int() % 255)
}
func inserter(done chan<- bool) {
for i := 0; i < 5; i++ {
////////
//INSERT
////////
params := make([]interface{}, 2)
params[0], params[1] = randomUint8(), randomUint8()
// fmt.Println("Inserting", mainDB.Stats()) //debug
rowsInserted, err := QueryDB(mainDB, dbQueryStringInserter, params)
if err == nil {
fmt.Println("Successfully inserted", rowsInserted, "rows!")
} else {
Error("Error on insertion!", err.Error())
}
fmt.Println()
///////////////
//SELECT single
///////////////
params = make([]interface{}, 1)
params[0] = 0
var result1, result2 uint8
fmt.Println("Selecting single") //debug
rowsObtained, err := QueryDB(mainDB, dbQueryStringSelector, params, &result1, &result2)
if err == nil {
fmt.Println("Successfully obtained", rowsObtained, "rows!")
if rowsObtained > 0 {
fmt.Println(params, result1, result2)
}
} else {
Error("Error on insertion!", err.Error())
}
fmt.Println()
/////////////////
//SELECT multiple
/////////////////
const bufferSize = 10
params = make([]interface{}, 1)
params[0] = 0
sresult1 := make([]interface{}, bufferSize)
sresult2 := make([]interface{}, bufferSize)
// fmt.Println("Selecting multiple", mainDB.Stats()) //debug
rowsAmount, _ := QueryDB(mainDB, dbQueryStringSelector, params, sresult1, sresult2)
sresult1 = sresult1[:rowsAmount]
sresult2 = sresult2[:rowsAmount]
fmt.Println(params, sresult1, sresult2)
// fmt.Printf("%T\n", sresult1[0])
// data := make([]uint8, len(sresult2))
// for i, el := range sresult2 {
// if s, ok := el.(int64); ok {
// data[i] = uint8(s)
// }
// }
// fmt.Println("data", data, data[0]+data[1])
fmt.Println()
////////
//UPDATE
////////
params = make([]interface{}, 1)
params[0] = randomUint8()
// fmt.Println("Updating", mainDB.Stats()) //debug
rowsInserted, err = QueryDB(mainDB, dbQueryStringUpdater, params)
if err == nil {
fmt.Println("Successfully updated", rowsInserted, "rows!")
} else {
Error("Error on insertion!", err.Error())
}
fmt.Println()
fmt.Println("////////////////////////////////////////////////////")
fmt.Println()
}
done <- true
}
func main() {
fmt.Println("Version:", runtime.Version())
initDatabase()
nRoutines := maxDBConnections * 2
nRoutines = 1
ok := make(chan bool, nRoutines)
for i := 0; i < nRoutines; i++ {
go inserter(ok)
}
for i := 0; i < nRoutines; i++ {
<-ok
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment