Skip to content

Instantly share code, notes, and snippets.

View jordanlewis's full-sized avatar
👀
large data banking

Jordan Lewis jordanlewis

👀
large data banking
View GitHub Profile
@jordanlewis
jordanlewis / gist:4e4fa1ff34e09ebbcfd658268ece773d
Created October 11, 2021 22:29
Example userfile workflow
-- Create database
create database mydb;
use mydb;
create table foo (i int primary key);
insert into foo select * from generate_series(1, 100);
create database backups;
-- Backup:
use backups;
/Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb/structured.pb.go:6039:3: loop copies large value each iteration
6039 for _, msg := range m.NewIndexes {
/Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb/structured.pb.go:6283:3: loop copies large value each iteration
6283 for _, msg := range m.Indexes {
/Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb/structured.pb.go:7646:3: loop copies large value each iteration
7646 for _, e := range m.NewIndexes {
/Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb/structured.pb.go:7781:3: loop copies large value each iteration
7781 for _, e := range m.Indexes {
/Users/jordan/go/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc/structured.go:3200:5: loop copies large value each iteration
3200 for i, idx := range desc.Indexes {
proposal for syntax/method for doing type templating:
all functions that need to do type-specific manipulation (let’s say, equals for the purpose of this example) will get a template argument T. T will be a struct TypeInfo (naming tbd) that contains the types.T information, as well as //execgen:inline’d methods that provide each overload, as well as utility methods.
still working on syntax for the “go type templates” - aka, how do we specify that a template needs int and []int in the case of int64 columns, but []byte and coldata.Bytes in the case of bytes columns, so ignore that for now.
for example:
// execgen:template<typ>
func genericEqualityFunc(colA []_GOTYPE, colB []_GOTYPE, idx int, typ *TypeInfo) bool {
a := typ.UnsafeGet(colA, idx)
b := typ.UnsafeGet(colB, idx)
eq := typ.Equals(a, b)
return eq
func newSingleDistinct(
input colexecbase.Operator, distinctColIdx int, outputCol []bool, t *types.T,
) (colexecbase.Operator, error) {
switch typeconv.TypeFamilyToCanonicalTypeFamily(t.Family()) {
case types.BoolFamily:
switch t.Width() {
case -1:
default:
return &distinctBoolOp{
OneInputNode: NewOneInputNode(input),
@jordanlewis
jordanlewis / gist:36b2cf775198886c6a65ac91de1911d0
Created June 5, 2020 21:42
distinct.eg.go after execgen:inline
// Code generated by execgen; DO NOT EDIT.
// Copyright 2018 The Cockroach Authors.
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
package colexec
import re
if "stacktrace" in input_data:
s = input_data["stacktrace"]
lines = s.split("\n")
l = ""
for line in lines:
if line.startswith("stacktrace: "):
l = line.lstrip("stacktrace: ")
break
#!/usr/bin/python
field_to_len = [
("PUBSCNUM",8),
("REG",10),
("RACE1V",2),
("SEX",1),
("AGE_DX",3),
("YR_BRTH", 4),
("SEQ_NUM", 2),
query T
EXPLAIN (VEC) SELECT count(*) FROM kv NATURAL INNER HASH JOIN kv kv2
----
├── Node 1
│ └── *distsqlrun.materializer
│ └── *exec.orderedAggregator
│ └── *exec.oneShotOp
│ └── exec.fnOp
│ └── *exec.UnorderedSynchronizer
root@127.0.0.1:63435/defaultdb> explain(opt,verbose) SELECT * FROM c WHERE (SELECT min(ship) FROM o WHERE o.c_id=c.c_id) IN (SELECT ship FROM o WHERE o.c_id=c.c_id);
text
+-------------------------------------------------------------------------------------------------
Left expr has relational output (1,2)
Left bound cols: (1)
Creating indexed vars for cols [0 1 2 5 3]
panic: invalid var index 2 (columns: 2) [recovered]
panic: panic while executing 1 statements: SELECT * FROM _ WHERE (SELECT min(_) FROM _ WHERE _._ = _._) IN (SELECT _ FROM _ WHERE _._ = _._); caused by invalid var index 2 (columns: 2)
goroutine 145 [running]:
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).closeWrapper(0xc00185a000, 0x6bf4360, 0xc000879d00, 0x6139180, 0xc0017a9f50)
/Users/jordan/repo/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:721 +0x36d
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn.func1(0xc00185a000, 0x6bf4360, 0xc000879d00)