Skip to content

Commit bb43161

Browse files
committed
sync2: add sqlstore
The `sqlstore` package provides simple sequence-based interface to the tables being synchronized. It is used by the FPTree data structure as the database layer, and doesn't do range fingerprinting by itself. `SyncedTable` and `SyncedTableSnapshot` provide methods that wrap the necessary SQL operations. `sql/expr` package was added to facilitate SQL generation.
1 parent ef30f47 commit bb43161

16 files changed

+2221
-0
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ require (
3535
github.com/prometheus/client_model v0.6.1
3636
github.com/prometheus/common v0.60.0
3737
github.com/quic-go/quic-go v0.48.1
38+
github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd
3839
github.com/rs/cors v1.11.1
3940
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
4041
github.com/seehuhn/mt19937 v1.0.0

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,8 @@ github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+
176176
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
177177
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
178178
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
179+
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
180+
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
179181
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
180182
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
181183
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -577,6 +579,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG
577579
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
578580
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
579581
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
582+
github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk=
583+
github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg=
580584
github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
581585
github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
582586
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=

sql/database.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"strings"
1212
"sync"
1313
"sync/atomic"
14+
"testing"
1415
"time"
1516

1617
sqlite "github.com/go-llsqlite/crawshaw"
@@ -236,6 +237,15 @@ func InMemory(opts ...Opt) *sqliteDatabase {
236237
return db
237238
}
238239

240+
// InMemoryTest returns an in-mem database for testing and ensures database is closed during `tb.Cleanup`.
241+
func InMemoryTest(tb testing.TB, opts ...Opt) *sqliteDatabase {
242+
// When using empty DB schema, we don't want to check for schema drift due to
243+
// "PRAGMA user_version = 0;" in the initial schema retrieved from the DB.
244+
db := InMemory(append(opts, WithNoCheckSchemaDrift())...)
245+
tb.Cleanup(func() { db.Close() })
246+
return db
247+
}
248+
239249
// Open database with options.
240250
//
241251
// Database is opened in WAL mode and pragma synchronous=normal.

sql/expr/expr.go

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
// Package expr proviedes a simple SQL expression parser and builder.
2+
// It wraps the rqlite/sql package and provides a more convenient API that contains only
3+
// what's needed for the go-spacemesh codebase.
4+
package expr
5+
6+
import (
7+
"strings"
8+
9+
rsql "github.com/rqlite/sql"
10+
)
11+
12+
// SQL operations.
13+
const (
14+
NE = rsql.NE // !=
15+
EQ = rsql.EQ // =
16+
LE = rsql.LE // <=
17+
LT = rsql.LT // <
18+
GT = rsql.GT // >
19+
GE = rsql.GE // >=
20+
BITAND = rsql.BITAND // &
21+
BITOR = rsql.BITOR // |
22+
BITNOT = rsql.BITNOT // !
23+
LSHIFT = rsql.LSHIFT // <<
24+
RSHIFT = rsql.RSHIFT // >>
25+
PLUS = rsql.PLUS // +
26+
MINUS = rsql.MINUS // -
27+
STAR = rsql.STAR // *
28+
SLASH = rsql.SLASH // /
29+
REM = rsql.REM // %
30+
CONCAT = rsql.CONCAT // ||
31+
DOT = rsql.DOT // .
32+
AND = rsql.AND
33+
OR = rsql.OR
34+
NOT = rsql.NOT
35+
)
36+
37+
// Expr represents a parsed SQL expression.
38+
type Expr = rsql.Expr
39+
40+
// Statement represents a parsed SQL statement.
41+
type Statement = rsql.Statement
42+
43+
// MustParse parses an SQL expression and panics if there's an error.
44+
func MustParse(s string) rsql.Expr {
45+
expr, err := rsql.ParseExprString(s)
46+
if err != nil {
47+
panic("error parsing SQL expression: " + err.Error())
48+
}
49+
return expr
50+
}
51+
52+
// MustParseStatement parses an SQL statement and panics if there's an error.
53+
func MustParseStatement(s string) rsql.Statement {
54+
st, err := rsql.NewParser(strings.NewReader(s)).ParseStatement()
55+
if err != nil {
56+
panic("error parsing SQL statement: " + err.Error())
57+
}
58+
return st
59+
}
60+
61+
// MaybeAnd joins together several SQL expressions with AND, ignoring any nil exprs.
62+
// If no non-nil expressions are passed, nil is returned.
63+
// If a single non-nil expression is passed, that single expression is returned.
64+
// Otherwise, the expressions are joined together with ANDs:
65+
// a AND b AND c AND d.
66+
func MaybeAnd(exprs ...Expr) Expr {
67+
var r Expr
68+
for _, expr := range exprs {
69+
switch {
70+
case expr == nil:
71+
case r == nil:
72+
r = expr
73+
default:
74+
r = Op(r, AND, expr)
75+
}
76+
}
77+
return r
78+
}
79+
80+
// Ident constructs SQL identifier expression for the identifier with the specified name.
81+
func Ident(name string) *rsql.Ident {
82+
return &rsql.Ident{Name: name}
83+
}
84+
85+
// Number constructs a number literal.
86+
func Number(value string) *rsql.NumberLit {
87+
return &rsql.NumberLit{Value: value}
88+
}
89+
90+
// TableSource constructs a Source clause for SELECT statement that corresponds to
91+
// selecting from a single table with the specified name.
92+
func TableSource(name string) rsql.Source {
93+
return &rsql.QualifiedTableName{Name: Ident(name)}
94+
}
95+
96+
// Op constructs a binary expression such as x + y or x < y.
97+
func Op(x Expr, op rsql.Token, y Expr) Expr {
98+
return &rsql.BinaryExpr{
99+
X: x,
100+
Op: op,
101+
Y: y,
102+
}
103+
}
104+
105+
// Bind constructs the unnamed bind expression (?).
106+
func Bind() Expr {
107+
return &rsql.BindExpr{Name: "?"}
108+
}
109+
110+
// Between constructs BETWEEN expression: x BETWEEN a AND b.
111+
func Between(x, a, b Expr) Expr {
112+
return Op(x, rsql.BETWEEN, &rsql.Range{X: a, Y: b})
113+
}
114+
115+
// Call constructs a call expression with specified arguments such as max(x).
116+
func Call(name string, args ...Expr) Expr {
117+
return &rsql.Call{Name: Ident(name), Args: args}
118+
}
119+
120+
// CountStar returns a COUNT(*) expression.
121+
func CountStar() Expr {
122+
return &rsql.Call{Name: Ident("count"), Star: rsql.Pos{Offset: 1}}
123+
}
124+
125+
// Asc constructs an ascending ORDER BY term.
126+
func Asc(expr Expr) *rsql.OrderingTerm {
127+
return &rsql.OrderingTerm{X: expr}
128+
}
129+
130+
// Desc constructs a descedning ORDER BY term.
131+
func Desc(expr Expr) *rsql.OrderingTerm {
132+
return &rsql.OrderingTerm{X: expr, Desc: rsql.Pos{Offset: 1}}
133+
}
134+
135+
// SelectBuilder is used to construct a SELECT statement.
136+
type SelectBuilder struct {
137+
st *rsql.SelectStatement
138+
}
139+
140+
// Select returns a SELECT statement builder.
141+
func Select(columns ...any) SelectBuilder {
142+
sb := SelectBuilder{st: &rsql.SelectStatement{}}
143+
return sb.Columns(columns...)
144+
}
145+
146+
// SelectBasedOn returns a SELECT statement builder based on the specified SELECT statement.
147+
// The statement must be parseable, otherwise SelectBasedOn panics.
148+
// The builder methods can be used to alter the statement.
149+
func SelectBasedOn(st Statement) SelectBuilder {
150+
st = rsql.CloneStatement(st)
151+
return SelectBuilder{st: st.(*rsql.SelectStatement)}
152+
}
153+
154+
// Get returns the underlying SELECT statement.
155+
func (sb SelectBuilder) Get() *rsql.SelectStatement {
156+
return sb.st
157+
}
158+
159+
// String returns the underlying SELECT statement as a string.
160+
func (sb SelectBuilder) String() string {
161+
return sb.st.String()
162+
}
163+
164+
// Columns sets columns in the SELECT statement.
165+
func (sb SelectBuilder) Columns(columns ...any) SelectBuilder {
166+
sb.st.Columns = make([]*rsql.ResultColumn, len(columns))
167+
for n, column := range columns {
168+
switch c := column.(type) {
169+
case *rsql.ResultColumn:
170+
sb.st.Columns[n] = c
171+
case Expr:
172+
sb.st.Columns[n] = &rsql.ResultColumn{Expr: c}
173+
default:
174+
panic("unexpected column type")
175+
}
176+
}
177+
return sb
178+
}
179+
180+
// From adds FROM clause to the SELECT statement.
181+
func (sb SelectBuilder) From(s rsql.Source) SelectBuilder {
182+
sb.st.Source = s
183+
return sb
184+
}
185+
186+
// From adds WHERE clause to the SELECT statement.
187+
func (sb SelectBuilder) Where(s Expr) SelectBuilder {
188+
sb.st.WhereExpr = s
189+
return sb
190+
}
191+
192+
// From adds ORDER BY clause to the SELECT statement.
193+
func (sb SelectBuilder) OrderBy(terms ...*rsql.OrderingTerm) SelectBuilder {
194+
sb.st.OrderingTerms = terms
195+
return sb
196+
}
197+
198+
// From adds LIMIT clause to the SELECT statement.
199+
func (sb SelectBuilder) Limit(limit Expr) SelectBuilder {
200+
sb.st.LimitExpr = limit
201+
return sb
202+
}
203+
204+
// ColumnExpr returns nth column expression from the SELECT statement.
205+
func ColumnExpr(st Statement, n int) Expr {
206+
return st.(*rsql.SelectStatement).Columns[n].Expr
207+
}
208+
209+
// WhereExpr returns WHERE expression from the SELECT statement.
210+
func WhereExpr(st Statement) Expr {
211+
return st.(*rsql.SelectStatement).WhereExpr
212+
}

0 commit comments

Comments
 (0)