Skip to content

Instantly share code, notes, and snippets.

@dasl-
Last active July 12, 2023 16:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dasl-/29f0ee159a97d8f18e67286b54406bf6 to your computer and use it in GitHub Desktop.
Save dasl-/29f0ee159a97d8f18e67286b54406bf6 to your computer and use it in GitHub Desktop.
diff --git a/go/vt/vtgate/vindexes/sqlite_lookup.go b/go/vt/vtgate/vindexes/sqlite_lookup.go
index 870fb8be32..ff6df7f31b 100644
--- a/go/vt/vtgate/vindexes/sqlite_lookup.go
+++ b/go/vt/vtgate/vindexes/sqlite_lookup.go
@@ -29,106 +29,115 @@ import (
"vitess.io/vitess/go/stats"
"vitess.io/vitess/go/vt/key"
_ "github.com/mattn/go-sqlite3" // sqlite driver
)
const (
// Timing metric keys
connectTimingKey = "Connect"
queryTimingKey = "Query"
)
var (
_ SingleColumn = (*SqliteLookupUnique)(nil)
_ Lookup = (*SqliteLookupUnique)(nil)
+ initSqliteDbMutex sync.RWMutex
// Metrics
timings = stats.NewTimings("SqliteLookupTimings", "Sqlite lookup timings", "operation")
)
func init() {
Register("etsy_sqlite_lookup_unique", NewSqliteLookupUnique)
}
// SqliteLookupUnique defines a vindex that uses a sqlite lookup table.
// The table is expected to define the id column as unique. It's
// Unique and a Lookup.
type SqliteLookupUnique struct {
name string
db *sql.DB
path string
table string
from string
to string
cacheSize string
preparedSelect *sql.Stmt
+ initFinished boolean
}
// NewSqliteLookupUnique creates a SqliteLookupUnique vindex.
// The supplied map has the following required fields:
//
// path: path to the backing sqlite file.
// table: name of the backing table.
// from: the column in the table that has the 'from' value of the lookup vindex.
// to: the 'to' column name of the table that contains the keyrange or keyspace id.
//
// The following fields are optional:
//
// cache_size: changes the maximum number of disk pages sqlite holds in memory at once
//
// As writes cannot be distributed across all vtgate machines, and writes must obtain
// locks on the sqlite db, SQLite lookups are always read only
func NewSqliteLookupUnique(name string, m map[string]string) (Vindex, error) {
slu := &SqliteLookupUnique{name: name}
slu.path = m["path"]
slu.table = m["table"]
slu.from = m["from"]
slu.to = m["to"]
if cacheSize, ok := m["cache_size"]; ok {
slu.cacheSize = cacheSize
}
return slu, nil
}
func (slu *SqliteLookupUnique) initSqliteDb() error {
+ initSqliteDbMutex.Lock()
+ defer initSqliteDbMutex.Unlock()
+ if slu.initFinished == true {
+ return nil // another thread raced and already finished initializing the DB
+ }
+
var err error
// Options defined here: https://github.com/mattn/go-sqlite3#connection-string
dbDSN := "file:" + slu.path + "?mode=ro&_query_only=true&immutable=true"
if len(slu.cacheSize) > 0 {
dbDSN += "&_cache_size=" + slu.cacheSize
}
connectTime := time.Now()
db, err := sql.Open("sqlite3", dbDSN)
if err != nil {
return err
}
if db != nil {
db.SetMaxOpenConns(0) // no maximum
db.SetMaxIdleConns(100)
db.SetConnMaxIdleTime(10 * time.Minute)
db.SetConnMaxLifetime(time.Hour)
}
slu.db = db
timings.Record(connectTimingKey, connectTime)
stmt, err := slu.db.Prepare(fmt.Sprintf("select %s, %s from %s where %s = ?", slu.from, slu.to, slu.table, slu.from))
if err != nil {
return err
}
slu.preparedSelect = stmt
+ slu.initFinished = true
return nil
}
// String returns the name of the vindex.
func (slu *SqliteLookupUnique) String() string {
return slu.name
}
// Cost returns the cost of this vindex as 5.
func (slu *SqliteLookupUnique) Cost() int {
return 5
}
// IsUnique returns true since the Vindex is unique.
@@ -180,31 +189,31 @@ func (slu *SqliteLookupUnique) Verify(ctx context.Context, vcursor VCursor, ids
out = append(out, bytes.Equal(ksid, ksids[i]))
} else {
out = append(out, ok)
}
}
return out, err
}
func (slu *SqliteLookupUnique) retrieveIDKsidMap(ids []sqltypes.Value) (resultMap map[string][]byte, err error) {
// Query sqlite database
var query string
var results *sql.Rows
queryStart := time.Now()
- if slu.db == nil {
+ if slu.initFinished == false {
err = slu.initSqliteDb()
if err != nil {
return nil, err
}
}
if len(ids) == 1 { // use prepared statement
results, err = slu.preparedSelect.Query(ids[0].ToString())
if err != nil {
return nil, err
}
} else { // do not use prepared statement
query = fmt.Sprintf("select %s, %s from %s where %s in (", slu.from, slu.to, slu.table, slu.from)
for _, id := range ids {
query += id.ToString() + ", "
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment