Skip to content

Commit 3cc0f31

Browse files
msaf1980msaf1980
committed
rollup: split rules for plain/tagged metrics (for perfomance)
1 parent bbb3a62 commit 3cc0f31

File tree

10 files changed

+908
-56
lines changed

10 files changed

+908
-56
lines changed

‎config/config.go

+20-19
Original file line numberDiff line numberDiff line change
@@ -129,24 +129,25 @@ var knownDataTableContext = map[string]bool{
129129

130130
// DataTable configs
131131
type DataTable struct {
132-
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
133-
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
134-
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
135-
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
136-
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
137-
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
138-
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
139-
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
140-
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
141-
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
142-
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
143-
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
144-
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
145-
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
146-
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
147-
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
148-
ContextMap map[string]bool `toml:"-" json:"-"`
149-
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
132+
Table string `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
133+
Reverse bool `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
134+
MaxAge time.Duration `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
135+
MinAge time.Duration `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
136+
MaxInterval time.Duration `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
137+
MinInterval time.Duration `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
138+
TargetMatchAny string `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
139+
TargetMatchAll string `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
140+
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
141+
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
142+
RollupConf string `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
143+
RollupAutoTable string `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
144+
RollupDefaultPrecision uint32 `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
145+
RollupDefaultFunction string `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
146+
RollupUseReverted bool `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
147+
RollupRuleTypeAutodetect bool `toml:"rollup-rule-type-autodetect" json:"rollup-rule-type-autodetect"`
148+
Context []string `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
149+
ContextMap map[string]bool `toml:"-" json:"-"`
150+
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
150151
}
151152

152153
// Debug config
@@ -453,7 +454,7 @@ func (c *Config) ProcessDataTables() (err error) {
453454
} else if c.DataTable[i].RollupConf == "none" {
454455
c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf)
455456
} else {
456-
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf)
457+
c.DataTable[i].Rollup, err = rollup.NewXMLFile(c.DataTable[i].RollupConf, rdp, rdf, c.DataTable[i].RollupRuleTypeAutodetect)
457458
}
458459

459460
if err != nil {

‎helper/rollup/compact.go

+27-2
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,12 @@ compact form of rollup rules for tests
1212
regexp;function;age:precision,age:precision,...
1313
*/
1414

15-
func parseCompact(body string) (*Rules, error) {
15+
func parseCompact(body string, auto bool) (*Rules, error) {
1616
lines := strings.Split(body, "\n")
1717
patterns := make([]Pattern, 0)
1818

1919
for _, line := range lines {
20+
var ruleType RuleType
2021
if strings.TrimSpace(line) == "" {
2122
continue
2223
}
@@ -29,6 +30,30 @@ func parseCompact(body string) (*Rules, error) {
2930
return nil, fmt.Errorf("can't parse line: %#v", line)
3031
}
3132
regexp := strings.TrimSpace(line[:p1])
33+
if len(regexp) > 8 && regexp[0] == '<' && regexp[1] == '!' && regexp[7] == '>' {
34+
typeStr := regexp[1:7]
35+
switch typeStr {
36+
case "!ALL_T":
37+
ruleType = RuleAll
38+
case "!PLAIN":
39+
ruleType = RulePlain
40+
case "!TAG_R":
41+
ruleType = RuleTaggedRegex
42+
//case "!TAG_T":
43+
// ruleType = RuleTagged
44+
default:
45+
return nil, fmt.Errorf("not realised rule type for line: %#v", line)
46+
}
47+
regexp = regexp[8:]
48+
} else {
49+
if ruleType == RuleAuto {
50+
if auto && len(regexp) > 0 {
51+
ruleType = AutoDetectRuleType(regexp)
52+
} else {
53+
ruleType = RuleAll
54+
}
55+
}
56+
}
3257
function := strings.TrimSpace(line[p1+1 : p2])
3358
retention := make([]Retention, 0)
3459

@@ -55,7 +80,7 @@ func parseCompact(body string) (*Rules, error) {
5580
}
5681
}
5782

58-
patterns = append(patterns, Pattern{Regexp: regexp, Function: function, Retention: retention})
83+
patterns = append(patterns, Pattern{RuleType: ruleType, Regexp: regexp, Function: function, Retention: retention})
5984
}
6085

6186
return (&Rules{Pattern: patterns}).compile()

‎helper/rollup/compact_test.go

+179-4
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,186 @@ import (
99
func TestParseCompact(t *testing.T) {
1010
config := `
1111
click_cost;any;0:3600,86400:60
12-
;max;0:60,3600:300,86400:3600`
12+
<!PLAIN>\.max$;max;0:3600,86400:60
13+
<!TAG_R>\.max\?;max;0:3600
14+
\.min$;min;0:3600,86400:60
15+
\.min\?;min;0:3600
16+
env=cloud;avg;0:3600
17+
;avg;0:60,3600:300,86400:3600`
1318

1419
expected, _ := (&Rules{
1520
Pattern: []Pattern{
16-
Pattern{Regexp: "click_cost", Function: "any", Retention: []Retention{
21+
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
1722
Retention{Age: 0, Precision: 3600},
1823
Retention{Age: 86400, Precision: 60},
1924
}},
20-
Pattern{Regexp: "", Function: "max", Retention: []Retention{
25+
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
26+
Retention{Age: 0, Precision: 3600},
27+
Retention{Age: 86400, Precision: 60},
28+
}},
29+
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
30+
Retention{Age: 0, Precision: 3600},
31+
}},
32+
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
33+
Retention{Age: 0, Precision: 3600},
34+
Retention{Age: 86400, Precision: 60},
35+
}},
36+
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
37+
Retention{Age: 0, Precision: 3600},
38+
}},
39+
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
40+
Retention{Age: 0, Precision: 3600},
41+
}},
42+
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
43+
Retention{Age: 0, Precision: 60},
44+
Retention{Age: 3600, Precision: 300},
45+
Retention{Age: 86400, Precision: 3600},
46+
}},
47+
},
48+
}).compile()
49+
50+
expectedPlain, _ := (&Rules{
51+
Pattern: []Pattern{
52+
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
53+
Retention{Age: 0, Precision: 3600},
54+
Retention{Age: 86400, Precision: 60},
55+
}},
56+
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
57+
Retention{Age: 0, Precision: 3600},
58+
Retention{Age: 86400, Precision: 60},
59+
}},
60+
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
61+
Retention{Age: 0, Precision: 3600},
62+
Retention{Age: 86400, Precision: 60},
63+
}},
64+
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
65+
Retention{Age: 0, Precision: 3600},
66+
}},
67+
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
68+
Retention{Age: 0, Precision: 3600},
69+
}},
70+
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
71+
Retention{Age: 0, Precision: 60},
72+
Retention{Age: 3600, Precision: 300},
73+
Retention{Age: 86400, Precision: 3600},
74+
}},
75+
},
76+
}).compile()
77+
78+
expectedTagged, _ := (&Rules{
79+
Pattern: []Pattern{
80+
Pattern{RuleType: RuleAll, Regexp: "click_cost", Function: "any", Retention: []Retention{
81+
Retention{Age: 0, Precision: 3600},
82+
Retention{Age: 86400, Precision: 60},
83+
}},
84+
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
85+
Retention{Age: 0, Precision: 3600},
86+
}},
87+
Pattern{RuleType: RuleAll, Regexp: `\.min$`, Function: "min", Retention: []Retention{
88+
Retention{Age: 0, Precision: 3600},
89+
Retention{Age: 86400, Precision: 60},
90+
}},
91+
Pattern{RuleType: RuleAll, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
92+
Retention{Age: 0, Precision: 3600},
93+
}},
94+
Pattern{RuleType: RuleAll, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
95+
Retention{Age: 0, Precision: 3600},
96+
}},
97+
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
98+
Retention{Age: 0, Precision: 60},
99+
Retention{Age: 3600, Precision: 300},
100+
Retention{Age: 86400, Precision: 3600},
101+
}},
102+
},
103+
}).compile()
104+
105+
assert := assert.New(t)
106+
r, err := parseCompact(config, false)
107+
assert.NoError(err)
108+
assert.Equal(expected, r)
109+
110+
assert.Equal(len(expected.patternPlain), 6)
111+
assert.Equal(expectedPlain.Pattern, r.patternPlain)
112+
113+
assert.Equal(len(expected.patternTagged), 6)
114+
assert.Equal(expectedTagged.Pattern, r.patternTagged)
115+
}
116+
117+
func TestParseCompactAutoDetect(t *testing.T) {
118+
config := `
119+
click_cost;any;0:3600,86400:60
120+
<!PLAIN>\.max$;max;0:3600,86400:60
121+
<!TAG_R>\.max\?;max;0:3600
122+
\.min$;min;0:3600,86400:60
123+
\.min\?;min;0:3600
124+
env=cloud;avg;0:3600
125+
;avg;0:60,3600:300,86400:3600`
126+
127+
expected, _ := (&Rules{
128+
Pattern: []Pattern{
129+
Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{
130+
Retention{Age: 0, Precision: 3600},
131+
Retention{Age: 86400, Precision: 60},
132+
}},
133+
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
134+
Retention{Age: 0, Precision: 3600},
135+
Retention{Age: 86400, Precision: 60},
136+
}},
137+
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
138+
Retention{Age: 0, Precision: 3600},
139+
}},
140+
Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{
141+
Retention{Age: 0, Precision: 3600},
142+
Retention{Age: 86400, Precision: 60},
143+
}},
144+
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
145+
Retention{Age: 0, Precision: 3600},
146+
}},
147+
Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
148+
Retention{Age: 0, Precision: 3600},
149+
}},
150+
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
151+
Retention{Age: 0, Precision: 60},
152+
Retention{Age: 3600, Precision: 300},
153+
Retention{Age: 86400, Precision: 3600},
154+
}},
155+
},
156+
}).compile()
157+
158+
expectedPlain, _ := (&Rules{
159+
Pattern: []Pattern{
160+
Pattern{RuleType: RulePlain, Regexp: "click_cost", Function: "any", Retention: []Retention{
161+
Retention{Age: 0, Precision: 3600},
162+
Retention{Age: 86400, Precision: 60},
163+
}},
164+
Pattern{RuleType: RulePlain, Regexp: `\.max$`, Function: "max", Retention: []Retention{
165+
Retention{Age: 0, Precision: 3600},
166+
Retention{Age: 86400, Precision: 60},
167+
}},
168+
Pattern{RuleType: RulePlain, Regexp: `\.min$`, Function: "min", Retention: []Retention{
169+
Retention{Age: 0, Precision: 3600},
170+
Retention{Age: 86400, Precision: 60},
171+
}},
172+
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
173+
Retention{Age: 0, Precision: 60},
174+
Retention{Age: 3600, Precision: 300},
175+
Retention{Age: 86400, Precision: 3600},
176+
}},
177+
},
178+
}).compile()
179+
180+
expectedTagged, _ := (&Rules{
181+
Pattern: []Pattern{
182+
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.max\?`, Function: "max", Retention: []Retention{
183+
Retention{Age: 0, Precision: 3600},
184+
}},
185+
Pattern{RuleType: RuleTaggedRegex, Regexp: `\.min\?`, Function: "min", Retention: []Retention{
186+
Retention{Age: 0, Precision: 3600},
187+
}},
188+
Pattern{RuleType: RuleTaggedRegex, Regexp: `env=cloud`, Function: "avg", Retention: []Retention{
189+
Retention{Age: 0, Precision: 3600},
190+
}},
191+
Pattern{RuleType: RuleAll, Regexp: "", Function: "avg", Retention: []Retention{
21192
Retention{Age: 0, Precision: 60},
22193
Retention{Age: 3600, Precision: 300},
23194
Retention{Age: 86400, Precision: 3600},
@@ -26,7 +197,11 @@ func TestParseCompact(t *testing.T) {
26197
}).compile()
27198

28199
assert := assert.New(t)
29-
r, err := parseCompact(config)
200+
r, err := parseCompact(config, true)
30201
assert.NoError(err)
31202
assert.Equal(expected, r)
203+
204+
assert.Equal(expectedPlain.Pattern, r.patternPlain)
205+
206+
assert.Equal(expectedTagged.Pattern, r.patternTagged)
32207
}

‎helper/rollup/remote.go

+11-5
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,12 @@ import (
1414
)
1515

1616
type rollupRulesResponseRecord struct {
17-
Regexp string `json:"regexp"`
18-
Function string `json:"function"`
19-
Age string `json:"age"`
20-
Precision string `json:"precision"`
21-
IsDefault int `json:"is_default"`
17+
RuleType RuleType `json:"type"`
18+
Regexp string `json:"regexp"`
19+
Function string `json:"function"`
20+
Age string `json:"age"`
21+
Precision string `json:"precision"`
22+
IsDefault int `json:"is_default"`
2223
}
2324
type rollupRulesResponse struct {
2425
Data []rollupRulesResponseRecord `json:"data"`
@@ -74,7 +75,11 @@ func parseJson(body []byte) (*Rules, error) {
7475
}
7576
} else {
7677
if last() == nil || last().Regexp != d.Regexp || last().Function != d.Function {
78+
if d.RuleType == RuleAuto {
79+
d.RuleType = RuleAll // for remote rules - no auto-detect rule type
80+
}
7781
r.Pattern = append(r.Pattern, Pattern{
82+
RuleType: d.RuleType,
7883
Retention: make([]Retention, 0),
7984
Regexp: d.Regexp,
8085
Function: d.Function,
@@ -92,6 +97,7 @@ func parseJson(body []byte) (*Rules, error) {
9297

9398
if defaultFunction != "" || len(defaultRetention) != 0 {
9499
r.Pattern = append(r.Pattern, Pattern{
100+
RuleType: RuleAll,
95101
Regexp: "",
96102
Function: defaultFunction,
97103
Retention: defaultRetention,

‎helper/rollup/remote_test.go

+21-3
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,25 @@ func TestParseJson(t *testing.T) {
8787
"precision": "0",
8888
"is_default": 0
8989
},
90+
{
91+
"type": "plain",
92+
"regexp": "\\.min$",
93+
"function": "min",
94+
"age": "0",
95+
"precision": "3600",
96+
"is_default": 0
97+
},
98+
{
99+
"type": "tagged_regex",
100+
"regexp": "\\.min\\?",
101+
"function": "min",
102+
"age": "0",
103+
"precision": "3600",
104+
"is_default": 0
105+
},
90106
{
91107
"regexp": "",
92-
"function": "max",
108+
"function": "avg",
93109
"age": "0",
94110
"precision": "60",
95111
"is_default": 1
@@ -112,11 +128,13 @@ func TestParseJson(t *testing.T) {
112128
total$;sum;
113129
min$;min;
114130
max$;max;
115-
;max;0:60
131+
<!PLAIN>\.min$;min;0:3600
132+
<!TAG_R>\.min\?;min;0:3600
133+
;avg;0:60
116134
`
117135

118136
assert := assert.New(t)
119-
expected, err := parseCompact(compact)
137+
expected, err := parseCompact(compact, false)
120138
assert.NoError(err)
121139

122140
r, err := parseJson([]byte(response))

0 commit comments

Comments
 (0)