Merge pull request #2 from djluck/types

Adding stronger validation of PromQL expressions, principally type checking which will allow consumers to validate the type an expression produces. 

This has meant a lot of improvements:
- Aggregate operator and function signatures are now defined in the library and parameter counts and types can be checked.
- Binary expressions are now parsed with the (correct) associativeness: left associativeness.
- Binary operators now have precedence associated with them and binary expressions are grouped according to this
- Every expression can determine it's return type
- Expression types preceeding offset expressions are now validated more carefully
- Positions of parsed expressions from the source input are now available in all AST nodes

README.md

@@ -87,20 +87,29 @@ Superpower.ParseException: Syntax error (line 1, column 21): Unexpected left bra
    at Superpower.Tokenizer`1.Tokenize(String source)
    at PromQL.Parser.Parser.ParseExpression(String input, Tokenizer tokenizer)
    at UserQuery.Main() in C:\Users\james.luck\AppData\Local\Temp\LINQPad6\_aryncqeb\dwfjew\LINQPadQuery:line [[
-```
+``` 
 
-**Only syntax is validated, semantic validation is not applied**. For example, the following expressions 
-are considered valid by this library:
 
+### Type checking
+Syntax is not only validated but expressions can also be typed checked via `CheckType`, e.g:
+```
+Parser.ParseExpression("1 + sum_over_time(some_metric[1h])").CheckType().Dump();
+```
+Returns:
+```
+ValueType.Vector
 ```
-# Invalid number of function arguments
-time(1, 2, 3, 4)
-
-# Invalid number and types of aggregation function arguments
-sum(1.0, "a string")
 
-# Binary operations not defined for strings
-"a" + "b"
+Expressions that violate the PromQL type system will throw an exception, e.g:
+```
+Parser.ParseExpression("'a' + 'b'").CheckType();
+```
+Throws:
+```
+Unexpected type 'string' was provided, expected one of 'scalar', 'instant vector': 0 (line 1, column 1)
+   at PromQL.Parser.TypeChecker.ExpectTypes(Expr expr, ValueType[] expected)
+   at PromQL.Parser.TypeChecker.CheckType(Expr expr)
+   at UserQuery.Main(), line 1
 ```
 
 ### Modifying PromQL expressions
@@ -161,7 +170,7 @@ This library aims to parse 99% of the PromQL language and has [extensive test co
 - Hexadecimal/ octal string escaping
 - Hexadecimal number representation 
 - [The `@` modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier)
-- Operator associativity (all operators are parsed with the same associativity)
+- Exponential operator associativity (is parsed with left associativity)
 
 If any of these features are important to you, please create an issue or submit a pull request. 
 

src/PromQL.Parser/Ast.cs

@@ -2,6 +2,8 @@
 using System.Collections.Immutable;
 using System.Text;
 using ExhaustiveMatching;
+using Superpower.Model;
+using Superpower.Parsers;
 
 namespace PromQL.Parser.Ast
 {
@@ -11,6 +13,7 @@ namespace PromQL.Parser.Ast
     public interface IPromQlNode
     {
         void Accept(IVisitor visitor);
+        TextSpan? Span { get; }
     }
 
     /// <summary>
@@ -29,34 +32,39 @@ public interface IPromQlNode
         typeof(UnaryExpr),
         typeof(VectorSelector)
     )]
-    public interface Expr : IPromQlNode {}
+    public interface Expr : IPromQlNode
+    {
+        ValueType Type { get; }
+    }
 
     /// <summary>
     /// Represents an aggregation operation on a Vector.
     /// </summary>
-    /// <param name="OperatorName">The used aggregation operation.</param>
+    /// <param name="Operator">The used aggregation operation.</param>
     /// <param name="Expr">The Vector expression over which is aggregated.</param>
     /// <param name="Param">Parameter used by some aggregators.</param>
     /// <param name="GroupingLabels">The labels by which to group the Vector.</param>
     /// <param name="Without"> Whether to drop the given labels rather than keep them.</param>
-    public record AggregateExpr(string OperatorName, Expr Expr, Expr? Param,
-        ImmutableArray<string> GroupingLabels, bool Without) : Expr
+    public record AggregateExpr(AggregateOperator Operator, Expr Expr, Expr? Param,
+        ImmutableArray<string> GroupingLabels, bool Without, TextSpan? Span = null) : Expr
     {
-        public AggregateExpr(string operatorName, Expr expr)
-            : this (operatorName, expr, null, ImmutableArray<string>.Empty, false)
+        public AggregateExpr(AggregateOperator @operator, Expr expr)
+            : this (@operator, expr, null, ImmutableArray<string>.Empty, false)
         {
         }
 
-        public AggregateExpr(string operatorName, Expr expr, Expr param, bool without = false, params string[] groupingLabels)
-            : this (operatorName, expr, param, groupingLabels.ToImmutableArray(), without)
+        public AggregateExpr(AggregateOperator @operator, Expr expr, Expr param, bool without = false, params string[] groupingLabels)
+            : this (@operator, expr, param, groupingLabels.ToImmutableArray(), without)
         {
         }
 
-        public string OperatorName { get; set;} = OperatorName;
+        public AggregateOperator Operator { get; set;} = Operator;
         public Expr Expr { get; set;} = Expr;
         public Expr? Param { get; set;} = Param;
         public ImmutableArray<string> GroupingLabels { get; set;} = GroupingLabels;
         public bool Without { get; set;} = Without;
+
+        public ValueType Type => ValueType.Vector;
 
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
@@ -69,13 +77,24 @@ public AggregateExpr(string operatorName, Expr expr, Expr param, bool without =
     /// <param name="Operator">The operation of the expression</param>
     /// <param name="VectorMatching">The matching behavior for the operation to be applied if both operands are Vectors.</param>
     public record BinaryExpr(Expr LeftHandSide, Expr RightHandSide, Operators.Binary Operator,
-        VectorMatching? VectorMatching = null) : Expr
+        VectorMatching? VectorMatching = null, TextSpan? Span = null) : Expr
     {
         public Expr LeftHandSide { get; set; } = LeftHandSide;
         public Expr RightHandSide { get; set; } = RightHandSide;
         public Operators.Binary Operator { get; set; } = Operator;
         public VectorMatching? VectorMatching { get; set; } = VectorMatching;
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+
+        public ValueType Type
+        {
+            get
+            {
+                if (RightHandSide.Type == ValueType.Scalar && LeftHandSide.Type == ValueType.Scalar)
+                    return ValueType.Scalar;
+
+                return ValueType.Vector;
+            }
+        }
     }
 
     /// <summary>
@@ -87,7 +106,7 @@ public record BinaryExpr(Expr LeftHandSide, Expr RightHandSide, Operators.Binary
     /// <param name="Include">Contains additional labels that should be included in the result from the side with the lower cardinality.</param>
     /// <param name="ReturnBool">If a comparison operator, return 0/1 rather than filtering.</param>
     public record VectorMatching(Operators.VectorMatchCardinality MatchCardinality, ImmutableArray<string> MatchingLabels,
-        bool On, ImmutableArray<string> Include, bool ReturnBool) : IPromQlNode
+        bool On, ImmutableArray<string> Include, bool ReturnBool, TextSpan? Span = null) : IPromQlNode
     {
         public static Operators.VectorMatchCardinality DefaultMatchCardinality { get; } = Operators.VectorMatchCardinality.OneToOne;
 
@@ -104,91 +123,104 @@ public VectorMatching(bool returnBool) : this (DefaultMatchCardinality, Immutabl
         public bool On { get; set; } = On;
         public ImmutableArray<string> Include { get; set; } = Include;
         public bool ReturnBool { get; set; } = ReturnBool;
-        
+
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     };
 
     /// <summary>
     /// A function call.
     /// </summary>
-    /// <param name="Identifier">The function that was called.</param>
+    /// <param name="Function">The function that was called.</param>
     /// <param name="Args">Arguments used in the call.</param>
-    public record FunctionCall(string Identifier, ImmutableArray<Expr> Args) : Expr
+    public record FunctionCall(Function Function, ImmutableArray<Expr> Args, TextSpan? Span = null) : Expr
     {
-        public FunctionCall(string identifier, params Expr[] args) 
-            : this (identifier, args.ToImmutableArray())
+        public FunctionCall(Function function, params Expr[] args) 
+            : this (function, args.ToImmutableArray())
         {
         }
 
-        public string Identifier { get; set; } = Identifier;
+        public Function Function { get; set; } = Function;
         public ImmutableArray<Expr> Args { get; set; } = Args;
-
-        public void Accept(IVisitor visitor) => visitor.Visit(this);
 
+        public ValueType Type => Function.ReturnType;
+
+        public void Accept(IVisitor visitor) => visitor.Visit(this);
+
         protected virtual bool PrintMembers(StringBuilder builder)
         {
-            builder.AppendLine($"{nameof(Identifier)} = {Identifier}, ");
+            builder.AppendLine($"{nameof(Function)} = {Function.Name}, ");
             builder.Append($"{nameof(Args)} = ");
             Args.PrintArray(builder);
-            
+
             return true;
         }
     }
 
-    public record ParenExpression(Expr Expr) : Expr
+    public record ParenExpression(Expr Expr, TextSpan? Span = null) : Expr
     {
         public Expr Expr { get; set; } = Expr;
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+        public ValueType Type => Expr.Type;
     }
 
-    public record OffsetExpr(Expr Expr, Duration Duration) : Expr
+    public record OffsetExpr(Expr Expr, Duration Duration, TextSpan? Span = null) : Expr
     {
         public Expr Expr { get; set; } = Expr;
         public Duration Duration { get; set; } = Duration;
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+        public ValueType Type => Expr.Type;
     }
 
-    public record MatrixSelector(VectorSelector Vector, Duration Duration) : Expr
+    public record MatrixSelector(VectorSelector Vector, Duration Duration, TextSpan? Span = null) : Expr
     {
         public VectorSelector Vector { get; set; } =Vector;
         public Duration Duration { get; set; } = Duration;
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+        public ValueType Type => ValueType.Matrix;
     }
 
-    public record UnaryExpr(Operators.Unary Operator, Expr Expr) : Expr
+    public record UnaryExpr(Operators.Unary Operator, Expr Expr, TextSpan? Span = null) : Expr
     {
         public Operators.Unary Operator { get; set; } = Operator;
         public Expr Expr { get; set; } = Expr;
 
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+        public ValueType Type => Expr.Type;
     }
 
     public record VectorSelector : Expr
     {
-        public VectorSelector(MetricIdentifier metricIdentifier)
+        public VectorSelector(MetricIdentifier metricIdentifier, TextSpan? span = null)
         {
             MetricIdentifier = metricIdentifier;
+            Span = span;
         }
 
-        public VectorSelector(LabelMatchers labelMatchers)
+        public VectorSelector(LabelMatchers labelMatchers, TextSpan? span = null)
         {
             LabelMatchers = labelMatchers;
+            Span = span;
         }
 
-        public VectorSelector(MetricIdentifier metricIdentifier, LabelMatchers labelMatchers)
+        public VectorSelector(MetricIdentifier metricIdentifier, LabelMatchers labelMatchers, TextSpan? span = null)
         {
+
             MetricIdentifier = metricIdentifier;
             LabelMatchers = labelMatchers;
+            Span = span;
         }
 
         public MetricIdentifier? MetricIdentifier { get; set; }
         public LabelMatchers? LabelMatchers { get; set; }
+        public TextSpan? Span { get; }
+        public ValueType Type => ValueType.Vector;
+
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
 
-    public record LabelMatchers(ImmutableArray<LabelMatcher> Matchers) : IPromQlNode
+    public record LabelMatchers(ImmutableArray<LabelMatcher> Matchers, TextSpan? Span = null) : IPromQlNode
     {
-        protected virtual bool PrintMembers(StringBuilder builder)
+        protected virtual bool PrintMembers(System.Text.StringBuilder builder)
         {
             builder.Append($"{nameof(Matchers)} = ");
             Matchers.PrintArray(builder);
@@ -201,43 +233,46 @@ protected virtual bool PrintMembers(StringBuilder builder)
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
 
-    public record LabelMatcher(string LabelName, Operators.LabelMatch Operator, StringLiteral Value) : IPromQlNode
+    public record LabelMatcher(string LabelName, Operators.LabelMatch Operator, StringLiteral Value, TextSpan? Span = null) : IPromQlNode
     {
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
 
-    public record MetricIdentifier(string Value) : IPromQlNode
+    public record MetricIdentifier(string Value, TextSpan? Span = null) : IPromQlNode
     {
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
 
-    public record NumberLiteral(double Value) : Expr
+    public record NumberLiteral(double Value, TextSpan? Span = null) : Expr
     {
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+        public ValueType Type => ValueType.Scalar;
     }
 
-    public record Duration(TimeSpan Value) : IPromQlNode 
+    public record Duration(TimeSpan Value, TextSpan? Span = null) : IPromQlNode 
     {
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
 
-    public record StringLiteral(char Quote, string Value) : Expr
+    public record StringLiteral(char Quote, string Value, TextSpan? Span = null) : Expr
     {
         public void Accept(IVisitor visitor) => visitor.Visit(this);
+        public ValueType Type => ValueType.String;
     }
 
-    public record SubqueryExpr(Expr Expr, Duration Range, Duration? Step = null) : Expr
+    public record SubqueryExpr(Expr Expr, Duration Range, Duration? Step = null, TextSpan? Span = null) : Expr
     {
         public Expr Expr { get; set; } = Expr;
         public Duration Range { get; set; } = Range;
         public Duration? Step { get; set; } = Step;
+        public ValueType Type => ValueType.Matrix;
 
         public void Accept(IVisitor visitor) => visitor.Visit(this);
     }
 
     internal static class Extensions
     {
-        internal static void PrintArray<T>(this ImmutableArray<T> arr, StringBuilder sb)
+        internal static void PrintArray<T>(this ImmutableArray<T> arr, System.Text.StringBuilder sb)
             where T : notnull
         {
             sb.Append("[ ");