go-sqlsmith: support hints in select/update/delete statement (#146)

Signed-off-by: Illyrix <Illyrix@outlook.com>

Author: Illyrix

cmd/abtest/main.go

@@ -44,6 +44,7 @@ func main() {
 	pocketConfig.Options.Path = fixture.Context.ABTestConfig.LogPath
 	pocketConfig.Options.Concurrency = fixture.Context.ABTestConfig.Concurrency
 	pocketConfig.Options.GeneralLog = fixture.Context.ABTestConfig.GeneralLog
+	pocketConfig.Options.EnableHint = fixture.Context.EnableHint
 	suit := util.Suit{
 		Config:      &cfg,
 		Provisioner: cluster.NewK8sProvisioner(),

cmd/cdc-pocket/main.go

@@ -44,6 +44,7 @@ func main() {
 	pocketConfig.Options.Concurrency = fixture.Context.ABTestConfig.Concurrency
 	pocketConfig.Options.GeneralLog = fixture.Context.ABTestConfig.GeneralLog
 	pocketConfig.Options.SyncTimeout.Duration = fixture.Context.BinlogConfig.SyncTimeout
+	pocketConfig.Options.EnableHint = fixture.Context.EnableHint
 	suit := util.Suit{
 		Config:      &cfg,
 		Provisioner: cluster.NewK8sProvisioner(),

cmd/pocket/main.go

@@ -44,6 +44,7 @@ func main() {
 	pocketConfig.Options.Concurrency = fixture.Context.ABTestConfig.Concurrency
 	pocketConfig.Options.GeneralLog = fixture.Context.ABTestConfig.GeneralLog
 	pocketConfig.Options.SyncTimeout.Duration = fixture.Context.BinlogConfig.SyncTimeout
+	pocketConfig.Options.EnableHint = fixture.Context.EnableHint
 	suit := util.Suit{
 		Config:      &cfg,
 		Provisioner: cluster.NewK8sProvisioner(),

cmd/tiflash-pocket/main.go

@@ -41,6 +41,7 @@ func main() {
 	pocketConfig := config.Init()
 	pocketConfig.Options.Serialize = false
 	pocketConfig.Options.Path = fixture.Context.TiFlashConfig.LogPath
+	pocketConfig.Options.EnableHint = fixture.Context.EnableHint
 	suit := util.Suit{
 		Config:      &cfg,
 		Provisioner: cluster.NewK8sProvisioner(),

go.mod

@@ -15,18 +15,18 @@ require (
 	github.com/pingcap/chaos-mesh v0.0.0-20200221071630-a3e79a893072
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
 	github.com/pingcap/go-tpc v0.0.0-20200229030315-98ee0f8f09d3
-	github.com/pingcap/kvproto v0.0.0-20200102065152-5d51d93be892
-	github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9
+	github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5
+	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
 
-	github.com/pingcap/parser v0.0.0-20200109073933-a9496438d77d
+	github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87
 	github.com/pingcap/tidb v2.1.0-beta+incompatible
 	github.com/pingcap/tidb-operator v1.1.0-beta.1.0.20200326133238-0fe67bf5e069
 	github.com/rogpeppe/fastuuid v1.2.0
 
 	github.com/satori/go.uuid v1.2.0
 	github.com/stretchr/testify v1.5.1
 	github.com/uber-go/atomic v1.5.0 // indirect
-	golang.org/x/net v0.0.0-20191112182307-2180aed22343
+	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 // indirect
 	k8s.io/api v0.0.0
@@ -40,7 +40,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
 
 replace github.com/pingcap/pd => github.com/pingcap/pd v1.1.0-beta.0.20200106144140-f5a7aa985497
 
-replace github.com/pingcap/tidb => github.com/pingcap/tidb v1.1.0-beta.0.20200110034112-1b34cc234e82
+replace github.com/pingcap/tidb => github.com/pingcap/tidb v0.0.0-20200317142013-5268094afe05
 
 replace github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33
 

go.sum

@@ -24,7 +24,7 @@ import (
 // GenerateFuncCallExpr generate random builtin chain
 func GenerateFuncCallExpr(table *types.Table, args int, stable bool) ast.ExprNode {
 	if args == 0 && util.Rd(2) == 0 {
-		return ast.NewValueExpr(util.GenerateRandDataItem())
+		return ast.NewValueExpr(util.GenerateRandDataItem(), "", "")
 	}
 
 	funcCallExpr := ast.FuncCallExpr{}

pkg/go-sqlsmith/builtin/function.go

@@ -0,0 +1,159 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package builtin
+
+import (
+	"log"
+	"math/rand"
+
+	"github.com/pingcap/parser/ast"
+	"github.com/pingcap/parser/model"
+
+	"github.com/pingcap/tipocket/pkg/go-sqlsmith/types"
+	"github.com/pingcap/tipocket/pkg/go-sqlsmith/util"
+)
+
+type hintClass struct {
+	name     string
+	minArg   int
+	maxArg   int
+	constArg bool
+	mysql    bool
+	stable   bool
+}
+
+var hintKeywords = []*hintClass{
+	// with no args
+	{"hash_agg", 0, 0, false, false, false},
+	{"stream_agg", 0, 0, false, false, false},
+	{"agg_to_cop", 0, 0, false, false, false},
+	{"read_consistent_replica", 0, 0, false, false, false},
+	{"no_index_merge", 0, 0, false, false, false},
+
+	// with bool (TRUE or FALSE)
+	{"use_toja", 1, 1, false, false, false},
+	{"enable_plan_cache", 1, 1, false, false, false},
+	{"use_cascades", 1, 1, false, false, false},
+
+	// these have been renamed
+	// {"tidb_hj", 2, 3, false, false, true},
+	// {"tidb_smj", 2, 3, false, false, true},
+	// {"tidb_inlj", 2, 3, false, false, true},
+	// with 2 or more args
+	{"hash_join", 1, -1, false, true, false},
+	{"merge_join", 1, -1, false, false, false},
+	{"inl_join", 1, -1, false, false, false},
+
+	// with int (byte)
+	{"memory_quota", 1, 1, false, false, false},
+	// with int (ms)
+	{"max_execution_time", 1, 1, false, false, false},
+}
+
+var indexHintKeywords = []*hintClass{
+	// with table name and at least one idx name
+	{"use_index", 2, -1, false, false, false},
+	{"ignore_index", 2, -1, false, false, false},
+	{"use_index_merge", 2, -1, false, false, false},
+}
+
+// these will not be generated for some reason
+var disabledHintKeywords = []*hintClass{
+	{"qb_name", 0, 0, false, false, false},
+
+	// not released?
+	{"time_range", 2, -1, false, false, false},
+	// storage type with tablename: TIKV[t1]
+	{"read_from_storage", 2, -1, false, false, false},
+	// not released?
+	{"query_type", 1, 1, false, false, false},
+
+	{"inl_hash_join", 1, -1, false, false, false},
+	{"inl_merge_join", 1, -1, false, false, false},
+}
+
+func GenerateHintExpr(table *types.Table) (h *ast.TableOptimizerHint) {
+	enabledKeywords := hintKeywords
+	if len(table.Indexes) > 0 {
+		enabledKeywords = append(enabledKeywords, indexHintKeywords...)
+	}
+	h = new(ast.TableOptimizerHint)
+	hintKeyword := enabledKeywords[util.Rd(len(enabledKeywords))]
+	h.HintName = model.NewCIStr(hintKeyword.name)
+
+	if hintKeyword.maxArg == 0 {
+		return
+	}
+
+	if hintKeyword.maxArg == 1 {
+		switch hintKeyword.name {
+		case "use_toja", "enable_plan_cache", "use_cascades":
+			h.HintData = util.RdBool()
+		case "memory_quota":
+			h.HintData = int64(util.RdRange(30720000, 40960000))
+		case "max_execution_time":
+			h.HintData = uint64(util.RdRange(500, 1500))
+		default:
+			log.Fatalf("unreachable hintKeyword.name:%s", hintKeyword.name)
+		}
+		return
+	}
+
+	shuffledTables := make([]ast.HintTable, 0)
+	for _, t := range table.InnerTableList {
+		shuffledTables = append(shuffledTables, ast.HintTable{
+			TableName: model.NewCIStr(t.Table),
+		})
+	}
+	rand.Shuffle(len(shuffledTables), func(i, j int) {
+		shuffledTables[i], shuffledTables[j] = shuffledTables[j], shuffledTables[i]
+	})
+
+	shuffledIndexes := make([]model.CIStr, 0)
+	for _, idx := range table.Indexes {
+		if idx != "" {
+			shuffledIndexes = append(shuffledIndexes, model.NewCIStr(idx))
+		}
+	}
+	rand.Shuffle(len(shuffledIndexes), func(i, j int) {
+		shuffledIndexes[i], shuffledIndexes[j] = shuffledIndexes[j], shuffledIndexes[i]
+	})
+
+	switch hintKeyword.name {
+	case "hash_join", "merge_join", "inl_join", "inl_hash_join", "inl_merge_join":
+		if len(shuffledTables) < 2 {
+			h = nil
+			return
+		}
+
+		n := util.MinInt(util.Rd(4)+2, len(shuffledTables)) // avoid case n < 2
+		for ; n > 0; n-- {
+			h.Tables = append(h.Tables, shuffledTables[n-1])
+		}
+	case "use_index", "ignore_index", "use_index_merge":
+		// if no table nor index return empty
+		if len(shuffledTables) == 0 || len(shuffledIndexes) == 0 {
+			h = nil
+			return
+		}
+		h.Tables = append(h.Tables, shuffledTables[util.Rd(len(shuffledTables))])
+		n := util.MinInt(util.Rd(4)+1, len(shuffledIndexes)) // avoid case n == 0
+		for ; n > 0; n-- {
+			h.Indexes = append(h.Indexes, shuffledIndexes[n-1])
+		}
+	default:
+		log.Fatalf("unreachable hintKeyword.name:%s", hintKeyword.name)
+	}
+	return
+}

pkg/go-sqlsmith/builtin/hint.go

@@ -41,6 +41,7 @@ func (s *SQLSmith) selectStmt(depth int) ast.Node {
 		selectStmtNode.Where = s.binaryOperationExpr(util.Rd(depth), 1)
 	}
 
+	selectStmtNode.TableHints = s.tableHintsExpr()
 	selectStmtNode.From = s.tableRefsClause(depth)
 
 	return &selectStmtNode
@@ -66,9 +67,11 @@ func (s *SQLSmith) updateStmt() ast.Node {
 	if whereRand < 8 {
 		updateStmtNode.Where = s.binaryOperationExpr(whereRand, 0)
 	} else {
-		updateStmtNode.Where = ast.NewValueExpr(1)
+		updateStmtNode.Where = ast.NewValueExpr(1, "", "")
 	}
 
+	updateStmtNode.TableHints = s.tableHintsExpr()
+
 	return &updateStmtNode
 }
 
@@ -94,12 +97,37 @@ func (s *SQLSmith) deleteStmt() ast.Node {
 	if whereRand < 8 {
 		deleteStmtNode.Where = s.binaryOperationExpr(whereRand, 0)
 	} else {
-		deleteStmtNode.Where = ast.NewValueExpr(1)
+		deleteStmtNode.Where = ast.NewValueExpr(1, "", "")
 	}
 
+	deleteStmtNode.TableHints = s.tableHintsExpr()
+
 	return &deleteStmtNode
 }
 
+func (s *SQLSmith) tableHintsExpr() (hints []*ast.TableOptimizerHint) {
+	if !s.Hint() {
+		return
+	}
+	length := 0
+	switch n := util.Rd(7)*util.Rd(7) - 17; {
+	case n <= 0:
+		length = 0
+	case n < 4:
+		length = 1
+	case n < 9:
+		length = 2
+	case n < 14:
+		length = 3
+	default:
+		length = 4
+	}
+	for i := 0; i < length; i++ {
+		hints = append(hints, &ast.TableOptimizerHint{})
+	}
+	return
+}
+
 func (s *SQLSmith) tableRefsClause(depth int) *ast.TableRefsClause {
 	tableRefsClause := ast.TableRefsClause{
 		TableRefs: &ast.Join{
@@ -177,7 +205,7 @@ func (s *SQLSmith) binaryOperationExpr(depth, complex int) ast.ExprNode {
 				node.Op = opcode.EQ
 			}
 			node.L = &ast.ColumnNameExpr{}
-			node.R = ast.NewValueExpr(1)
+			node.R = ast.NewValueExpr(1, "", "")
 		}
 	}
 	return &node
@@ -189,7 +217,7 @@ func (s *SQLSmith) patternInExpr() *ast.PatternInExpr {
 	// case *ast.SubqueryExpr:
 	// 	// may need refine after fully support of ResultSetNode interface
 	// 	node.Query.(*ast.SelectStmt).Limit = &ast.Limit {
-	// 		Count: ast.NewValueExpr(1),
+	// 		Count: ast.NewValueExpr(1, "", ""),
 	// 	}
 	// }
 
@@ -215,7 +243,7 @@ func (s *SQLSmith) exprNode(cons bool) ast.ExprNode {
 	default:
 		// hope there is an empty value type
 		if cons {
-			return ast.NewValueExpr(1)
+			return ast.NewValueExpr(1, "", "")
 		}
 		return &ast.ColumnNameExpr{}
 	}

pkg/go-sqlsmith/dml_ast.go

@@ -38,6 +38,7 @@ type SQLSmith struct {
 	currDB        string
 	debug         bool
 	stable        bool
+	hint          bool
 }
 
 // New create SQLSmith instance
@@ -87,6 +88,14 @@ func (s *SQLSmith) SetStable(stable bool) {
 	s.stable = stable
 }
 
+func (s *SQLSmith) Hint() bool {
+	return s.hint
+}
+
+func (s *SQLSmith) SetHint(hint bool) {
+	s.hint = hint
+}
+
 // Walk will walk the tree and fillin tables and columns data
 func (s *SQLSmith) Walk(tree ast.Node) (string, string, error) {
 	node, table, err := stateflow.New(s.GetDB(s.currDB), s.stable).WalkTree(tree)