Add integration tests plus scripting

Fix datetime formatting

x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec

@@ -131,3 +131,103 @@ interval
 ---------------
 +0 02:43:59.163  
 ;
+
+testDatePlusInterval
+SELECT MONTH(birth_date) AS m, MONTH(birth_date + INTERVAL '1-2' YEAR TO MONTH) AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+       m       |       f       
+---------------+---------------
+null           |null           
+2              |4              
+4              |6              
+5              |7              
+6              |8     
+;
+
+testDatePlusMixInterval
+SELECT birth_date, birth_date + INTERVAL '1-2' YEAR TO MONTH AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+     birth_date:ts  |         f:ts         
+--------------------+--------------------
+null                |null                
+1952-02-27T00:00:00Z|1953-04-27T00:00:00Z
+1952-04-19T00:00:00Z|1953-06-19T00:00:00Z
+1952-05-15T00:00:00Z|1953-07-15T00:00:00Z
+1952-06-13T00:00:00Z|1953-08-13T00:00:00Z
+;
+
+
+testDateMinusInterval
+SELECT YEAR(birth_date) AS y, YEAR(birth_date - INTERVAL 1 YEAR) AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+       y       |       f       
+---------------+---------------
+null           |null           
+1952           |1951           
+1952           |1951           
+1952           |1951           
+1952           |1951  
+;
+
+testDatePlusMixInterval
+SELECT birth_date, birth_date + INTERVAL '1-2' YEAR TO MONTH AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+     birth_date:ts  |         f:ts          
+--------------------+--------------------
+null                |null                
+1952-02-27T00:00:00Z|1953-04-27T00:00:00Z
+1952-04-19T00:00:00Z|1953-06-19T00:00:00Z
+1952-05-15T00:00:00Z|1953-07-15T00:00:00Z
+1952-06-13T00:00:00Z|1953-08-13T00:00:00Z
+;
+
+
+testDateAndMultipleIntervals
+SELECT birth_date, birth_date - INTERVAL 1 YEAR  + INTERVAL '2-3' YEAR TO MONTH AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+     birth_date:ts  |         f:ts          
+--------------------+--------------------
+null                |null                
+1952-02-27T00:00:00Z|1953-05-27T00:00:00Z
+1952-04-19T00:00:00Z|1953-07-19T00:00:00Z
+1952-05-15T00:00:00Z|1953-08-15T00:00:00Z
+1952-06-13T00:00:00Z|1953-09-13T00:00:00Z
+;
+
+
+testDatePlusIntervalWhereClause
+SELECT birth_date, YEAR(birth_date + INTERVAL 1 YEAR) AS f FROM test_emp WHERE YEAR(birth_date + INTERVAL 1 YEAR) > 1 GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+     birth_date:ts  |         f:i          
+--------------------+--------------------
+1952-02-27T00:00:00Z|1953           
+1952-04-19T00:00:00Z|1953           
+1952-05-15T00:00:00Z|1953           
+1952-06-13T00:00:00Z|1953           
+1952-07-08T00:00:00Z|1953   
+;
+
+testDateMinusIntervalOrder
+SELECT birth_date, MONTH(birth_date - INTERVAL 1 YEAR) AS f FROM test_emp GROUP BY birth_date ORDER BY MONTH(birth_date - INTERVAL 1 YEAR) ASC LIMIT 5;
+
+     birth_date:ts  |         f:i          
+--------------------+--------------------
+null                |null           
+1952-02-27T00:00:00Z|2              
+1952-04-19T00:00:00Z|4              
+1952-05-15T00:00:00Z|5              
+1952-06-13T00:00:00Z|6   
+;
+
+// see https://github.com/elastic/elasticsearch/issues/35745
+testDatePlusIntervalHavingClause-Ignore
+SELECT birth_date, MAX(hire_date) - INTERVAL 1 YEAR AS f FROM test_emp GROUP BY birth_date ORDER BY birth_date ASC LIMIT 5;
+
+     birth_date:ts  |         f:ts          
+--------------------+--------------------
+1952-02-27T00:00:00Z|1953           
+1952-04-19T00:00:00Z|1953           
+1952-05-15T00:00:00Z|1953           
+1952-06-13T00:00:00Z|1953           
+1952-07-08T00:00:00Z|1953   
+;

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/CliFormatter.java

@@ -9,13 +9,11 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xpack.sql.proto.ColumnInfo;
+import org.elasticsearch.xpack.sql.proto.StringUtils;
 
-import org.elasticsearch.xpack.sql.proto.DateUtils;
 import java.io.IOException;
-import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Objects;
 
 /**
  * Formats {@link SqlQueryResponse} for the CLI. {@linkplain Writeable} so
@@ -47,7 +45,7 @@ public CliFormatter(List<ColumnInfo> columns, List<List<Object>> rows) {
             for (int i = 0; i < width.length; i++) {
                 // TODO are we sure toString is correct here? What about dates that come back as longs.
                 // Tracked by https://github.com/elastic/x-pack-elasticsearch/issues/3081
-                width[i] = Math.max(width[i], toString(row.get(i)).length());
+                width[i] = Math.max(width[i], StringUtils.toString(row.get(i)).length());
             }
         }
     }
@@ -120,7 +118,8 @@ private String formatWithoutHeader(StringBuilder sb, List<List<Object>> rows) {
                 }
                 // TODO are we sure toString is correct here? What about dates that come back as longs.
                 // Tracked by https://github.com/elastic/x-pack-elasticsearch/issues/3081
-                String string = toString(row.get(i));                if (string.length() <= width[i]) {
+                String string = StringUtils.toString(row.get(i));
+                if (string.length() <= width[i]) {
                     // Pad
                     sb.append(string);
                     int padding = width[i] - string.length();
@@ -138,14 +137,6 @@ private String formatWithoutHeader(StringBuilder sb, List<List<Object>> rows) {
         return sb.toString();
     }
 
-    private static String toString(Object object) {
-        if (object instanceof ZonedDateTime) {
-            return DateUtils.toString((ZonedDateTime) object);
-        } else {
-            return Objects.toString(object);
-        }
-    }
-
     /**
      * Pick a good estimate of the buffer size needed to contain the rows.
      */

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java

@@ -13,8 +13,8 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.sql.proto.ColumnInfo;
-import org.elasticsearch.xpack.sql.proto.DateUtils;
 import org.elasticsearch.xpack.sql.proto.Mode;
+import org.elasticsearch.xpack.sql.proto.StringUtils;
 
 import java.io.IOException;
 import java.time.ZonedDateTime;
@@ -176,7 +176,7 @@ public static XContentBuilder value(XContentBuilder builder, Mode mode, Object v
             }
             // otherwise use the ISO format
             else {
-                builder.value(DateUtils.toString(zdt));
+                builder.value(StringUtils.toString(zdt));
             }
         } else {
             builder.value(value);

x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/StringUtils.java

@@ -9,10 +9,33 @@
 import java.sql.Timestamp;
 import java.time.Duration;
 import java.time.Period;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
-public class StringUtils {
+import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
+import static java.time.temporal.ChronoField.HOUR_OF_DAY;
+import static java.time.temporal.ChronoField.MILLI_OF_SECOND;
+import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+
+public final class StringUtils {
+
+    private static final DateTimeFormatter ISO_WITH_MILLIS = new DateTimeFormatterBuilder()
+            .parseCaseInsensitive()
+            .append(ISO_LOCAL_DATE)
+            .appendLiteral('T')
+            .appendValue(HOUR_OF_DAY, 2)
+            .appendLiteral(':')
+            .appendValue(MINUTE_OF_HOUR, 2)
+            .appendLiteral(':')
+            .appendValue(SECOND_OF_MINUTE, 2)
+            .appendFraction(MILLI_OF_SECOND, 3, 3, true)
+            .appendOffsetId()
+            .toFormatter(Locale.ROOT);
 
     private static final int SECONDS_PER_MINUTE = 60;
     private static final int SECONDS_PER_HOUR = SECONDS_PER_MINUTE * 60;
@@ -30,13 +53,22 @@ public static String toString(Object value) {
             return ts.toInstant().toString();
         }
 
+        if (value instanceof ZonedDateTime) {
+            return ((ZonedDateTime) value).format(ISO_WITH_MILLIS);
+        }
+
         // handle intervals
         // YEAR/MONTH/YEAR TO MONTH -> YEAR TO MONTH
         if (value instanceof Period) {
             // +yyy-mm - 7 chars
             StringBuilder sb = new StringBuilder(7);
             Period p = (Period) value;
-            sb.append(p.isNegative() ? "-" : "+");
+            if (p.isNegative()) {
+                sb.append("-");
+                p = p.negated();
+            } else {
+                sb.append("+");
+            }
             sb.append(p.getYears());
             sb.append("-");
             sb.append(p.getMonths());
@@ -48,7 +80,12 @@ public static String toString(Object value) {
             // +ddd hh:mm:ss.mmmmmmmmm - 23 chars
             StringBuilder sb = new StringBuilder(23);
             Duration d = (Duration) value;
-            sb.append(d.isNegative() ? "-" : "+");
+            if (d.isNegative()) {
+                sb.append("-");
+                d = d.negated();
+            } else {
+                sb.append("+");
+            }
 
             long durationInSec = d.getSeconds();
 

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java

@@ -49,10 +49,6 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe
         this.planner = new Planner();
     }
 
-    public NamedWriteableRegistry writableRegistry() {
-        return writableRegistry;
-    }
-
     private SqlSession newSession(Configuration cfg) {
         return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, optimizer, planner);
     }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
 
 import org.elasticsearch.xpack.sql.expression.Expression;
-import org.elasticsearch.xpack.sql.expression.FieldAttribute;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor.DateTimeExtractor;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
 import org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder;
@@ -43,18 +42,19 @@ public static Integer dateTimeChrono(ZonedDateTime dateTime, String tzId, String
     private static Integer dateTimeChrono(ZonedDateTime dateTime, ChronoField field) {
         return Integer.valueOf(dateTime.get(field));
     }
-
+    
     @Override
-    public ScriptTemplate scriptWithField(FieldAttribute field) {
+    public ScriptTemplate asScript() {
         ParamsBuilder params = paramsBuilder();
 
-        String template = null;
-        template = formatTemplate("{sql}.dateTimeChrono(doc[{}].value, {}, {})");
-        params.variable(field.name())
+        ScriptTemplate script = super.asScript();
+        String template = formatTemplate("{sql}.dateTimeChrono(" + script.template() + ", {}, {})");
+        params.script(script.params())
               .variable(timeZone().getID())
               .variable(extractor.chronoField().name());
 
         return new ScriptTemplate(template, params.build(), dataType());
+
     }
 
     @Override