From cebc7ffad9bb67e2ea00dd8c82668a2db2c8c4f0 Mon Sep 17 00:00:00 2001 From: b41sh Date: Sat, 30 Mar 2019 05:34:37 -0500 Subject: [PATCH] types: refine the parsing logic of INTERVAL to correct DATE_ADD/DATE_SUB (#9874) --- expression/integration_test.go | 10 +- types/time.go | 316 ++++++++++----------------------- types/time_test.go | 18 +- 3 files changed, 119 insertions(+), 225 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 8f4a4718fe204..004c439de561b 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -1828,11 +1828,11 @@ func (s *testIntegrationSuite) TestTimeBuiltin(c *C) { {"\"2011-11-11\"", "\"abc1000\"", "MICROSECOND", "", ""}, {"\"20111111 10:10:10\"", "\"1\"", "DAY", "", ""}, - {"\"2011-11-11\"", "\"10\"", "SECOND_MICROSECOND", "", ""}, - {"\"2011-11-11\"", "\"10.0000\"", "MINUTE_MICROSECOND", "", ""}, - {"\"2011-11-11\"", "\"10:10:10\"", "MINUTE_MICROSECOND", "", ""}, + {"\"2011-11-11\"", "\"10\"", "SECOND_MICROSECOND", "2011-11-11 00:00:00.100000", "2011-11-10 23:59:59.900000"}, + {"\"2011-11-11\"", "\"10.0000\"", "MINUTE_MICROSECOND", "2011-11-11 00:00:10", "2011-11-10 23:59:50"}, + {"\"2011-11-11\"", "\"10:10:10\"", "MINUTE_MICROSECOND", "2011-11-11 00:10:10.100000", "2011-11-10 23:49:49.900000"}, - {"cast(\"2011-11-11\" as datetime)", "\"10:10:10\"", "MINUTE_MICROSECOND", "", ""}, + {"cast(\"2011-11-11\" as datetime)", "\"10:10:10\"", "MINUTE_MICROSECOND", "2011-11-11 00:10:10.100000", "2011-11-10 23:49:49.900000"}, {"cast(\"2011-11-11 00:00:00\" as datetime)", "1", "DAY", "2011-11-12 00:00:00", "2011-11-10 00:00:00"}, {"cast(\"2011-11-11 00:00:00\" as datetime)", "10", "HOUR", "2011-11-11 10:00:00", "2011-11-10 14:00:00"}, {"cast(\"2011-11-11 00:00:00\" as datetime)", "10", "MINUTE", "2011-11-11 00:10:00", "2011-11-10 23:50:00"}, @@ -1843,7 +1843,7 @@ func (s *testIntegrationSuite) TestTimeBuiltin(c *C) { {"cast(\"2011-11-11 00:00:00\" as datetime)", "\"10\"", "MINUTE", "2011-11-11 00:10:00", "2011-11-10 23:50:00"}, {"cast(\"2011-11-11 00:00:00\" as datetime)", "\"10\"", "SECOND", "2011-11-11 00:00:10", "2011-11-10 23:59:50"}, - {"cast(\"2011-11-11\" as date)", "\"10:10:10\"", "MINUTE_MICROSECOND", "", ""}, + {"cast(\"2011-11-11\" as date)", "\"10:10:10\"", "MINUTE_MICROSECOND", "2011-11-11 00:10:10.100000", "2011-11-10 23:49:49.900000"}, {"cast(\"2011-11-11 00:00:00\" as date)", "1", "DAY", "2011-11-12", "2011-11-10"}, {"cast(\"2011-11-11 00:00:00\" as date)", "10", "HOUR", "2011-11-11 10:00:00", "2011-11-10 14:00:00"}, {"cast(\"2011-11-11 00:00:00\" as date)", "10", "MINUTE", "2011-11-11 00:10:00", "2011-11-10 23:50:00"}, diff --git a/types/time.go b/types/time.go index 99a8821282ddb..8b98bd38b9774 100644 --- a/types/time.go +++ b/types/time.go @@ -79,6 +79,50 @@ const ( TimeMaxValueSeconds = TimeMaxHour*3600 + TimeMaxMinute*60 + TimeMaxSecond ) +const ( + // YearIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + YearIndex = 0 + iota + // MonthIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + MonthIndex + // DayIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + DayIndex + // HourIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + HourIndex + // MinuteIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + MinuteIndex + // SecondIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + SecondIndex + // MicrosecondIndex is index of 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + MicrosecondIndex +) + +const ( + // YearMonthMaxCnt is max parameters count 'YEARS-MONTHS' expr Format allowed + YearMonthMaxCnt = 2 + // DayHourMaxCnt is max parameters count 'DAYS HOURS' expr Format allowed + DayHourMaxCnt = 2 + // DayMinuteMaxCnt is max parameters count 'DAYS HOURS:MINUTES' expr Format allowed + DayMinuteMaxCnt = 3 + // DaySecondMaxCnt is max parameters count 'DAYS HOURS:MINUTES:SECONDS' expr Format allowed + DaySecondMaxCnt = 4 + // DayMicrosecondMaxCnt is max parameters count 'DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format allowed + DayMicrosecondMaxCnt = 5 + // HourMinuteMaxCnt is max parameters count 'HOURS:MINUTES' expr Format allowed + HourMinuteMaxCnt = 2 + // HourSecondMaxCnt is max parameters count 'HOURS:MINUTES:SECONDS' expr Format allowed + HourSecondMaxCnt = 3 + // HourMicrosecondMaxCnt is max parameters count 'HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format allowed + HourMicrosecondMaxCnt = 4 + // MinuteSecondMaxCnt is max parameters count 'MINUTES:SECONDS' expr Format allowed + MinuteSecondMaxCnt = 2 + // MinuteMicrosecondMaxCnt is max parameters count 'MINUTES:SECONDS.MICROSECONDS' expr Format allowed + MinuteMicrosecondMaxCnt = 3 + // SecondMicrosecondMaxCnt is max parameters count 'SECONDS.MICROSECONDS' expr Format allowed + SecondMicrosecondMaxCnt = 2 + // TimeValueCnt is parameters count 'YEARS-MONTHS DAYS HOURS:MINUTES:SECONDS.MICROSECONDS' expr Format + TimeValueCnt = 7 +) + // Zero values for different types. var ( // ZeroDuration is the zero value for Duration type. @@ -1574,234 +1618,67 @@ func extractSingleTimeValue(unit string, format string) (int64, int64, int64, fl return 0, 0, 0, 0, errors.Errorf("invalid singel timeunit - %s", unit) } -// extractSecondMicrosecond extracts second and microsecond from a string and its format is `SS.FFFFFF`. -func extractSecondMicrosecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, ".") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - seconds, err := strconv.ParseFloat(fields[0], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - microseconds, err := strconv.ParseFloat(alignFrac(fields[1], MaxFsp), 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - return 0, 0, 0, seconds*float64(gotime.Second) + microseconds*float64(gotime.Microsecond), nil -} - -// extractMinuteMicrosecond extracts minutes and microsecond from a string and its format is `MM:SS.FFFFFF`. -func extractMinuteMicrosecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, ":") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - minutes, err := strconv.ParseFloat(fields[0], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - _, _, _, value, err := extractSecondMicrosecond(fields[1]) - if err != nil { - return 0, 0, 0, 0, errors.Trace(err) - } - - return 0, 0, 0, minutes*float64(gotime.Minute) + value, nil -} - -// extractMinuteSecond extracts minutes and second from a string and its format is `MM:SS`. -func extractMinuteSecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, ":") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - minutes, err := strconv.ParseFloat(fields[0], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - seconds, err := strconv.ParseFloat(fields[1], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - return 0, 0, 0, minutes*float64(gotime.Minute) + seconds*float64(gotime.Second), nil -} - -// extractHourMicrosecond extracts hour and microsecond from a string and its format is `HH:MM:SS.FFFFFF`. -func extractHourMicrosecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, ":") - if len(fields) != 3 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - hours, err := strconv.ParseFloat(fields[0], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - minutes, err := strconv.ParseFloat(fields[1], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - _, _, _, value, err := extractSecondMicrosecond(fields[2]) - if err != nil { - return 0, 0, 0, 0, errors.Trace(err) - } - - return 0, 0, 0, hours*float64(gotime.Hour) + minutes*float64(gotime.Minute) + value, nil -} - -// extractHourSecond extracts hour and second from a string and its format is `HH:MM:SS`. -func extractHourSecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, ":") - if len(fields) != 3 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - hours, err := strconv.ParseFloat(fields[0], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - minutes, err := strconv.ParseFloat(fields[1], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - seconds, err := strconv.ParseFloat(fields[2], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - return 0, 0, 0, hours*float64(gotime.Hour) + minutes*float64(gotime.Minute) + seconds*float64(gotime.Second), nil -} - -// extractHourMinute extracts hour and minute from a string and its format is `HH:MM`. -func extractHourMinute(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, ":") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - hours, err := strconv.ParseFloat(fields[0], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - minutes, err := strconv.ParseFloat(fields[1], 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - return 0, 0, 0, hours*float64(gotime.Hour) + minutes*float64(gotime.Minute), nil -} - -// extractDayMicrosecond extracts day and microsecond from a string and its format is `DD HH:MM:SS.FFFFFF`. -func extractDayMicrosecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, " ") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - days, err := strconv.ParseInt(fields[0], 10, 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) +// extractTimeValue extracts years, months, days, microseconds from a string +// MySQL permits any punctuation delimiter in the expr format. +// See https://dev.mysql.com/doc/refman/8.0/en/expressions.html#temporal-intervals +func extractTimeValue(format string, index, cnt int) (int64, int64, int64, float64, error) { + neg := false + originalFmt := format + format = strings.TrimSpace(format) + if len(format) > 0 && format[0] == '-' { + neg = true + format = format[1:] } - - _, _, _, value, err := extractHourMicrosecond(fields[1]) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + fields := make([]string, TimeValueCnt) + for i := range fields { + fields[i] = "0" } - - return 0, 0, days, value, nil -} - -// extractDaySecond extracts day and hour from a string and its format is `DD HH:MM:SS`. -func extractDaySecond(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, " ") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + matches := numericRegex.FindAllString(format, -1) + if len(matches) > cnt { + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - - days, err := strconv.ParseInt(fields[0], 10, 64) - if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + for i := range matches { + if neg { + fields[index] = "-" + matches[len(matches)-1-i] + } else { + fields[index] = matches[len(matches)-1-i] + } + index-- } - _, _, _, value, err := extractHourSecond(fields[1]) + years, err := strconv.ParseInt(fields[YearIndex], 10, 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - - return 0, 0, days, value, nil -} - -// extractDayMinute extracts day and minute from a string and its format is `DD HH:MM`. -func extractDayMinute(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, " ") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - days, err := strconv.ParseInt(fields[0], 10, 64) + months, err := strconv.ParseInt(fields[MonthIndex], 10, 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - - _, _, _, value, err := extractHourMinute(fields[1]) + days, err := strconv.ParseInt(fields[DayIndex], 10, 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - return 0, 0, days, value, nil -} - -// extractDayHour extracts day and hour from a string and its format is `DD HH`. -func extractDayHour(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, " ") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - days, err := strconv.ParseInt(fields[0], 10, 64) + hours, err := strconv.ParseFloat(fields[HourIndex], 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - - hours, err := strconv.ParseFloat(fields[1], 64) + minutes, err := strconv.ParseFloat(fields[MinuteIndex], 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - - return 0, 0, days, hours * float64(gotime.Hour), nil -} - -// extractYearMonth extracts year and month from a string and its format is `YYYY-MM`. -func extractYearMonth(format string) (int64, int64, int64, float64, error) { - fields := strings.Split(format, "-") - if len(fields) != 2 { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) - } - - years, err := strconv.ParseInt(fields[0], 10, 64) + seconds, err := strconv.ParseFloat(fields[SecondIndex], 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } - - months, err := strconv.ParseInt(fields[1], 10, 64) + microseconds, err := strconv.ParseFloat(alignFrac(fields[MicrosecondIndex], MaxFsp), 64) if err != nil { - return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(format) + return 0, 0, 0, 0, ErrIncorrectDatetimeValue.GenWithStackByArgs(originalFmt) } + durations := hours*float64(gotime.Hour) + minutes*float64(gotime.Minute) + + seconds*float64(gotime.Second) + microseconds*float64(gotime.Microsecond) - return years, months, 0, 0, nil + return years, months, days, durations, nil } // ExtractTimeValue extracts time value from time unit and format. @@ -1810,27 +1687,27 @@ func ExtractTimeValue(unit string, format string) (int64, int64, int64, float64, case "MICROSECOND", "SECOND", "MINUTE", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR": return extractSingleTimeValue(unit, format) case "SECOND_MICROSECOND": - return extractSecondMicrosecond(format) + return extractTimeValue(format, MicrosecondIndex, SecondMicrosecondMaxCnt) case "MINUTE_MICROSECOND": - return extractMinuteMicrosecond(format) + return extractTimeValue(format, MicrosecondIndex, MinuteMicrosecondMaxCnt) case "MINUTE_SECOND": - return extractMinuteSecond(format) + return extractTimeValue(format, SecondIndex, MinuteSecondMaxCnt) case "HOUR_MICROSECOND": - return extractHourMicrosecond(format) + return extractTimeValue(format, MicrosecondIndex, HourMicrosecondMaxCnt) case "HOUR_SECOND": - return extractHourSecond(format) + return extractTimeValue(format, SecondIndex, HourSecondMaxCnt) case "HOUR_MINUTE": - return extractHourMinute(format) + return extractTimeValue(format, MinuteIndex, HourMinuteMaxCnt) case "DAY_MICROSECOND": - return extractDayMicrosecond(format) + return extractTimeValue(format, MicrosecondIndex, DayMicrosecondMaxCnt) case "DAY_SECOND": - return extractDaySecond(format) + return extractTimeValue(format, SecondIndex, DaySecondMaxCnt) case "DAY_MINUTE": - return extractDayMinute(format) + return extractTimeValue(format, MinuteIndex, DayMinuteMaxCnt) case "DAY_HOUR": - return extractDayHour(format) + return extractTimeValue(format, HourIndex, DayHourMaxCnt) case "YEAR_MONTH": - return extractYearMonth(format) + return extractTimeValue(format, MonthIndex, YearMonthMaxCnt) default: return 0, 0, 0, 0, errors.Errorf("invalid singel timeunit - %s", unit) } @@ -2391,6 +2268,9 @@ var twoDigitRegex = regexp.MustCompile("^[1-9][0-9]?") // oneToSixDigitRegex: it was just for [0, 999999] var oneToSixDigitRegex = regexp.MustCompile("^[0-9]{0,6}") +// numericRegex: it was for any numeric characters +var numericRegex = regexp.MustCompile("[0-9]+") + // parseTwoNumeric is used for pattens 0..31 0..24 0..60 and so on. // It returns the parsed int, and remain data after parse. func parseTwoNumeric(input string) (int, string) { diff --git a/types/time_test.go b/types/time_test.go index 2ad3396cef89a..177c6ff6b211e 100644 --- a/types/time_test.go +++ b/types/time_test.go @@ -1201,15 +1201,29 @@ func (s *testTimeSuite) TestExtractTimeValue(c *C) { {"04", "MONTH", 0, 04, 0, 0}, {"1", "QUARTER", 0, 1 * 3, 0, 0}, {"2019", "YEAR", 2019, 0, 0, 0}, + {"10567890", "SECOND_MICROSECOND", 0, 0, 0, 1.056789e+10}, {"10.567890", "SECOND_MICROSECOND", 0, 0, 0, 1.056789e+10}, - {"35:10.567890", "MINUTE_SECOND", 0, 0, 0, 2.11056789e+12}, - {"11:35:10.567890", "HOUR_SECOND", 0, 0, 0, 4.171056789e+13}, + {"-10.567890", "SECOND_MICROSECOND", 0, 0, 0, -1.056789e+10}, + {"35:10567890", "MINUTE_SECOND", 0, 0, 0, 1.056999e+16}, + {"3510567890", "MINUTE_SECOND", 0, 0, 0, 3.51056789e+18}, + {"11:35:10.567890", "HOUR_MICROSECOND", 0, 0, 0, 4.171056789e+13}, + {"567890", "HOUR_MICROSECOND", 0, 0, 0, 5.6789e+08}, {"14:00", "HOUR_MINUTE", 0, 0, 0, 5.04e+13}, + {"14", "HOUR_MINUTE", 0, 0, 0, 8.4e+11}, {"12 14:00:00.345", "DAY_MICROSECOND", 0, 0, 12, 5.0400345e+13}, {"12 14:00:00", "DAY_SECOND", 0, 0, 12, 5.04e+13}, {"12 14:00", "DAY_MINUTE", 0, 0, 12, 5.04e+13}, {"12 14", "DAY_HOUR", 0, 0, 12, 5.04e+13}, + {"1:1", "DAY_HOUR", 0, 0, 1, 3.6e+12}, + {"aa1bb1", "DAY_HOUR", 0, 0, 1, 3.6e+12}, + {"-1:1", "DAY_HOUR", 0, 0, -1, -3.6e+12}, + {"-aa1bb1", "DAY_HOUR", 0, 0, -1, -3.6e+12}, {"2019-12", "YEAR_MONTH", 2019, 12, 0, 0}, + {"1 1", "YEAR_MONTH", 1, 1, 0, 0}, + {"aa1bb1", "YEAR_MONTH", 1, 1, 0, 0}, + {"-1 1", "YEAR_MONTH", -1, -1, 0, 0}, + {"-aa1bb1", "YEAR_MONTH", -1, -1, 0, 0}, + {" \t\n\r\n - aa1bb1 \t\n ", "YEAR_MONTH", -1, -1, 0, 0}, } for _, col := range tbl { res1, res2, res3, res4, err := types.ExtractTimeValue(col.unit, col.format)