Adds backtick for the quoted string token lexer. (#2095)

* Fixes unit in stats request log. (#2093) * Fix unit conversion in stats log. 🤦 Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fix the tests because you know ! you know ? Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> wip Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add backtick for the quoted string token lexer. This allows to pass unescaped string to Loki. For example we now support this : `` {name="cassandra"} |~ `error=\w+` `` `` {name!~`mysql-\d+`} `` As you can see no escape is required when using ` quoted strings. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes newlines. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add back missing test. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
grafana · May 19, 2020 · f9a2c43 · f9a2c43
1 parent c81be66
commit f9a2c43
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 17 deletions.
diff --git a/docs/logql.md b/docs/logql.md
@@ -19,6 +19,9 @@ the labels passed to the log stream selector will affect the relative
 performance of the query's execution. The filter expression is then used to do a
 distributed `grep` over the aggregated logs from the matching log streams.
 
+> To avoid escaping special characters you can use the `` ` ``(back-tick) instead of `"` when quoting strings.
+For example `` `\w+` `` is the same as `"\\w+"`, this is specially useful when writing regular expression which can contains many backslash that require escaping.
+
 ### Log Stream Selector
 
 The log stream selector determines which log streams should be included in your
@@ -51,6 +54,7 @@ Examples:
 
 - `{name=~"mysql.+"}`
 - `{name!~"mysql.+"}`
+- `` {name!~`mysql-\d+`} ``
 
 The same rules that apply for [Prometheus Label
 Selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#instant-vector-selectors)
@@ -64,6 +68,7 @@ regex:
 
 - `{job="mysql"} |= "error"`
 - `{name="kafka"} |~ "tsdb-ops.*io:2003"`
+- `` {name="cassandra"} |~  `error=\w+` ``
 - `{instance=~"kafka-[23]",name="kafka"} != kafka.server:type=ReplicaManager`
 
 In the previous examples, `|=`, `|~`, and `!=` act as **filter operators** and

diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go
@@ -66,7 +66,7 @@ func (l *lexer) Lex(lval *exprSymType) int {
 		lval.str = l.TokenText()
 		return NUMBER
 
-	case scanner.String:
+	case scanner.String, scanner.RawString:
 		var err error
 		lval.str, err = strconv.Unquote(l.TokenText())
 		if err != nil {

diff --git a/pkg/logql/lex_test.go b/pkg/logql/lex_test.go
@@ -14,6 +14,8 @@ func TestLex(t *testing.T) {
 		expected []int
 	}{
 		{`{foo="bar"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
+		{"{foo=\"bar\"} |~  `\\w+`", []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING}},
+		{`{foo="bar"} |~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE_MATCH, STRING}},
 		{`{ foo = "bar" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
 		{`{ foo != "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NEQ, STRING, CLOSE_BRACE}},
 		{`{ foo =~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, RE, STRING, CLOSE_BRACE}},

diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go
@@ -20,6 +20,25 @@ func TestParse(t *testing.T) {
 		exp Expr
 		err error
 	}{
+		{
+			// raw string
+			in: "count_over_time({foo=~`bar\\w+`}[12h] |~ `error\\`)",
+			exp: &rangeAggregationExpr{
+				operation: "count_over_time",
+				left: &logRange{
+					left: &filterExpr{
+						ty:    labels.MatchRegexp,
+						match: "error\\",
+						left: &matchersExpr{
+							matchers: []*labels.Matcher{
+								mustNewMatcher(labels.MatchRegexp, "foo", "bar\\w+"),
+							},
+						},
+					},
+					interval: 12 * time.Hour,
+				},
+			},
+		},
 		{
 			// test [12h] before filter expr
 			in: `count_over_time({foo="bar"}[12h] |= "error")`,
@@ -658,10 +677,10 @@ func TestParse(t *testing.T) {
 		},
 		{
 			in: `
-			sum(count_over_time({foo="bar"}[5m])) by (foo) ^
-			sum(count_over_time({foo="bar"}[5m])) by (foo) /
-			sum(count_over_time({foo="bar"}[5m])) by (foo)
-			`,
+					sum(count_over_time({foo="bar"}[5m])) by (foo) ^
+					sum(count_over_time({foo="bar"}[5m])) by (foo) /
+					sum(count_over_time({foo="bar"}[5m])) by (foo)
+					`,
 			exp: mustNewBinOpExpr(
 				OpTypeDiv,
 				mustNewBinOpExpr(
@@ -720,10 +739,10 @@ func TestParse(t *testing.T) {
 		{
 			// operator precedence before left associativity
 			in: `
-			sum(count_over_time({foo="bar"}[5m])) by (foo) +
-			sum(count_over_time({foo="bar"}[5m])) by (foo) /
-			sum(count_over_time({foo="bar"}[5m])) by (foo)
-			`,
+					sum(count_over_time({foo="bar"}[5m])) by (foo) +
+					sum(count_over_time({foo="bar"}[5m])) by (foo) /
+					sum(count_over_time({foo="bar"}[5m])) by (foo)
+					`,
 			exp: mustNewBinOpExpr(
 				OpTypeAdd,
 				mustNewVectorAggregationExpr(newRangeAggregationExpr(
@@ -781,10 +800,10 @@ func TestParse(t *testing.T) {
 		},
 		{
 			in: `sum by (job) (
-					count_over_time({namespace="tns"} |= "level=error"[5m])
-				/
-					count_over_time({namespace="tns"}[5m])
-				)`,
+							count_over_time({namespace="tns"} |= "level=error"[5m])
+						/
+							count_over_time({namespace="tns"}[5m])
+						)`,
 			exp: mustNewVectorAggregationExpr(
 				mustNewBinOpExpr(OpTypeDiv,
 					newRangeAggregationExpr(
@@ -812,10 +831,10 @@ func TestParse(t *testing.T) {
 		},
 		{
 			in: `sum by (job) (
-					count_over_time({namespace="tns"} |= "level=error"[5m])
-				/
-					count_over_time({namespace="tns"}[5m])
-				) * 100`,
+							count_over_time({namespace="tns"} |= "level=error"[5m])
+						/
+							count_over_time({namespace="tns"}[5m])
+						) * 100`,
 			exp: mustNewBinOpExpr(OpTypeMul, mustNewVectorAggregationExpr(
 				mustNewBinOpExpr(OpTypeDiv,
 					newRangeAggregationExpr(