diff --git a/backend/mbql/src/metabase/mbql/normalize.clj b/backend/mbql/src/metabase/mbql/normalize.clj
index 23e76a18970db..7d8fe10f28a38 100644
--- a/backend/mbql/src/metabase/mbql/normalize.clj
+++ b/backend/mbql/src/metabase/mbql/normalize.clj
@@ -250,7 +250,7 @@
"Normalize source/results metadata for a single column."
[metadata]
{:pre [(map? metadata)]}
- (-> (reduce #(m/update-existing %1 %2 keyword) metadata [:base_type :semantic_type :visibility_type :source :unit])
+ (-> (reduce #(m/update-existing %1 %2 keyword) metadata [:base_type :effective_type :semantic_type :visibility_type :source :unit])
(m/update-existing :field_ref (comp canonicalize-mbql-clauses normalize-tokens))
(m/update-existing :fingerprint walk/keywordize-keys)))
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj
index 77543152f8251..2991461f0e102 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj
@@ -166,10 +166,12 @@
[:> $date [:absolute-datetime #t "2014-01-01T00:00Z[UTC]" :default]]
[:=
$user_id
- [:value 5 {:base_type :type/Integer
- :semantic_type :type/FK
- :database_type "INTEGER"
- :name "USER_ID"}]]]
+ [:value 5 {:base_type :type/Integer
+ :effective_type :type/Integer
+ :coercion_strategy nil
+ :semantic_type :type/FK
+ :database_type "INTEGER"
+ :name "USER_ID"}]]]
::row-level-restrictions/gtap? true}
:joins [{:source-query
{:source-table $$venues
@@ -177,10 +179,12 @@
$venues.latitude $venues.longitude $venues.price]
:filter [:=
$venues.price
- [:value 1 {:base_type :type/Integer
- :semantic_type :type/Category
- :database_type "INTEGER"
- :name "PRICE"}]]
+ [:value 1 {:base_type :type/Integer
+ :effective_type :type/Integer
+ :coercion_strategy nil
+ :semantic_type :type/Category
+ :database_type "INTEGER"
+ :name "PRICE"}]]
::row-level-restrictions/gtap? true}
:alias "v"
:strategy :left-join
@@ -762,6 +766,8 @@
(is (= [:=
[:field (mt/id :products :category) {:join-alias "products"}]
[:value "Widget" {:base_type :type/Text
+ :effective_type :type/Text
+ :coercion_strategy nil
:semantic_type (db/select-one-field :semantic_type Field
:id (mt/id :products :category))
:database_type "VARCHAR"
diff --git a/frontend/src/metabase/admin/datamodel/containers/FieldApp.jsx b/frontend/src/metabase/admin/datamodel/containers/FieldApp.jsx
index dfac2b7d388ca..4d1cd69e10b85 100644
--- a/frontend/src/metabase/admin/datamodel/containers/FieldApp.jsx
+++ b/frontend/src/metabase/admin/datamodel/containers/FieldApp.jsx
@@ -25,6 +25,9 @@ import { LeftNavPane, LeftNavPaneItem } from "metabase/components/LeftNavPane";
import Section, { SectionHeader } from "../components/Section";
import SelectSeparator from "../components/SelectSeparator";
+import { is_coerceable, coercions_for_type } from "cljs/metabase.types";
+import { isFK } from "metabase/lib/types";
+
import {
FieldVisibilityPicker,
SemanticTypeAndTargetPicker,
@@ -313,6 +316,35 @@ const FieldGeneralPane = ({
/>
+ {!isFK(field.semantic_type) && is_coerceable(field.base_type) && (
+
+
+
+ )}
metadata
+ (and base_type (not (:effective_type metadata)))
+ (assoc :effective_type base_type))))
(def ^:const ga-type->base-type
"Map of Google Analytics field types to Metabase types."
diff --git a/modules/drivers/googleanalytics/test/metabase/driver/googleanalytics_test.clj b/modules/drivers/googleanalytics/test/metabase/driver/googleanalytics_test.clj
index 251a1e42f8119..b48d94405745b 100644
--- a/modules/drivers/googleanalytics/test/metabase/driver/googleanalytics_test.clj
+++ b/modules/drivers/googleanalytics/test/metabase/driver/googleanalytics_test.clj
@@ -296,21 +296,24 @@
:status :completed
:data {:rows [["Toucan Sighting" 1000]]
:native_form expected-ga-query
- :cols [{:description "This is ga:eventLabel"
- :semantic_type nil
- :name "ga:eventLabel"
- :settings nil
- :source :breakout
- :parent_id nil
- :visibility_type :normal
- :display_name "ga:eventLabel"
- :fingerprint nil
- :base_type :type/Text}
+ :cols [{:description "This is ga:eventLabel"
+ :semantic_type nil
+ :name "ga:eventLabel"
+ :settings nil
+ :source :breakout
+ :parent_id nil
+ :visibility_type :normal
+ :display_name "ga:eventLabel"
+ :fingerprint nil
+ :base_type :type/Text
+ :effective_type :type/Text
+ :coercion_strategy nil}
{:name "metric"
:display_name "ga:totalEvents"
:source :aggregation
:description "This is ga:totalEvents"
- :base_type :type/Text}]
+ :base_type :type/Text
+ :effective_type :type/Text}]
:results_timezone system-timezone-id}}
(-> (tu/doall-recursive (qp query))
(update-in [:data :cols] #(for [col %]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
index 91338c1c8549f..59db96499a33d 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
@@ -24,7 +24,7 @@
t)))
(defn- param-value->str
- [{semantic-type :semantic_type, :as field} x]
+ [{coercion :coercion_strategy, :as field} x]
(cond
;; sequences get converted to `$in`
(sequential? x)
@@ -36,11 +36,11 @@
(param-value->str field (u.date/parse (:s x)))
(and (instance? Temporal x)
- (isa? semantic-type :type/UNIXTimestampSeconds))
+ (isa? coercion :Coercion/UNIXSeconds->DateTime))
(long (/ (t/to-millis-from-epoch (->utc-instant x)) 1000))
(and (instance? Temporal x)
- (isa? semantic-type :type/UNIXTimestampMilliseconds))
+ (isa? coercion :Coercion/UNIXMilliSeconds->DateTime))
(t/to-millis-from-epoch (->utc-instant x))
;; convert temporal types to ISODate("2019-12-09T...") (etc.)
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index 5a1b17de0f040..fb786534f9302 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -110,16 +110,16 @@
x)
(defmethod ->rvalue (class Field)
- [{semantic-type :semantic_type, :as field}]
+ [{coercion :coercion_strategy, :as field}]
(let [field-name (str \$ (field->name field "."))]
(cond
- (isa? semantic-type :type/UNIXTimestampMicroseconds)
+ (isa? coercion :Coercion/UNIXMicroSeconds->DateTime)
{:$dateFromParts {:millisecond {$divide [field-name 1000]}, :year 1970}}
- (isa? semantic-type :type/UNIXTimestampMilliseconds)
+ (isa? coercion :Coercion/UNIXMilliSeconds->DateTime)
{:$dateFromParts {:millisecond field-name, :year 1970}}
- (isa? semantic-type :type/UNIXTimestampSeconds)
+ (isa? coercion :Coercion/UNIXSeconds->DateTime)
{:$dateFromParts {:second field-name, :year 1970}}
:else field-name)))
diff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.clj
index fed0cefe236f7..23e8d3d358472 100644
--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj
+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj
@@ -201,11 +201,11 @@
(hx/+ (hsql/raw "timestamp '1970-01-01 00:00:00 UTC'")
(num-to-ds-interval :second field-or-value)))
-(defmethod sql.qp/cast-temporal-string [:oracle :type/ISO8601DateTimeString]
+(defmethod sql.qp/cast-temporal-string [:oracle :Coercion/ISO8601->DateTime]
[_driver _semantic_type expr]
(hsql/call :to_timestamp expr "YYYY-MM-DD HH:mi:SS"))
-(defmethod sql.qp/cast-temporal-string [:oracle :type/ISO8601DateString]
+(defmethod sql.qp/cast-temporal-string [:oracle :Coercion/ISO8601->Date]
[_driver _semantic_type expr]
(hsql/call :to_date expr "YYYY-MM-DD"))
diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
index b27911137638b..07ad191b7810e 100644
--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
@@ -123,7 +123,9 @@
:id (mt/id :extsales :buyerid)
:visibility_type :normal
:display_name "Buyer ID"
- :base_type :type/Integer}
+ :base_type :type/Integer
+ :effective_type :type/Integer
+ :coercion_strategy nil}
{:description nil
:table_id (mt/id :extsales)
:semantic_type nil
@@ -135,7 +137,9 @@
:id (mt/id :extsales :salesid)
:visibility_type :normal
:display_name "Sale Sid"
- :base_type :type/Integer}]
+ :base_type :type/Integer
+ :effective_type :type/Integer
+ :coercion_strategy nil}]
; in different Redshift instances, the fingerprint on these
; columns is different.
(map #(dissoc % :fingerprint)
diff --git a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
index 63589d78eb1f5..26a7407b4e804 100644
--- a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
+++ b/modules/drivers/sqlite/src/metabase/driver/sqlite.clj
@@ -216,15 +216,15 @@
[_ _ expr]
(->datetime expr (hx/literal "unixepoch")))
-(defmethod sql.qp/cast-temporal-string [:sqlite :type/ISO8601DateTimeString]
+(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/ISO8601->DateTime]
[_driver _semantic_type expr]
(->datetime expr))
-(defmethod sql.qp/cast-temporal-string [:sqlite :type/ISO8601DateString]
+(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/ISO8601->Date]
[_driver _semantic_type expr]
(->date expr))
-(defmethod sql.qp/cast-temporal-string [:sqlite :type/ISO8601TimeString]
+(defmethod sql.qp/cast-temporal-string [:sqlite :Coercion/ISO8601->Time]
[_driver _semantic_type expr]
(->time expr))
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index 5714f6caac580..118c140b9968b 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -204,7 +204,7 @@
;; Work around this by converting the timestamps to minutes instead before calling DATEADD().
(date-add :minute (hx// expr 60) (hx/literal "1970-01-01")))
-(defmethod sql.qp/cast-temporal-string [:sqlserver :type/ISO8601DateTimeString]
+(defmethod sql.qp/cast-temporal-string [:sqlserver :Coercion/ISO8601->DateTime]
[_driver _semantic_type expr]
(hx/->datetime expr))
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index b750180fb901b..6766b245a9f02 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -23,7 +23,7 @@
(mt/defdataset ^:private genetic-data
[["genetic-data"
- [{:field-name "gene", :base-type {:native "VARCHAR(MAX)"}}]
+ [{:field-name "gene", :base-type {:native "VARCHAR(MAX)"}, :effective-type :type/Text}]
[[(a-gene)]]]])
(deftest clobs-should-come-back-as-text-test
diff --git a/resources/migrations/000_migrations.yaml b/resources/migrations/000_migrations.yaml
index 3f28fd9260f2b..56ac81710c48a 100644
--- a/resources/migrations/000_migrations.yaml
+++ b/resources/migrations/000_migrations.yaml
@@ -7983,8 +7983,77 @@ databaseChangeLog:
sql: |
ALTER TABLE task_history
MODIFY ended_at timestamp(6) DEFAULT current_timestamp(6) NOT NULL;
-
-
+ - changeSet:
+ id: 284
+ author: dpsutton
+ comment: Added 0.39 - Semantic type system - add effective type
+ changes:
+ - addColumn:
+ tableName: metabase_field
+ columns:
+ - column:
+ name: effective_type
+ type: varchar(255)
+ remarks: 'The effective type of the field after any coercions.'
+ - changeSet:
+ id: 285
+ author: dpsutton
+ comment: Added 0.39 - Semantic type system - add coercion column
+ changes:
+ - addColumn:
+ tableName: metabase_field
+ columns:
+ - column:
+ name: coercion_strategy
+ type: varchar(255)
+ remarks: 'A strategy to coerce the base_type into the effective_type.'
+ - changeSet:
+ id: 286
+ author: dpsutton
+ comment: Added 0.39 - Semantic type system - set effective_type default
+ changes:
+ - sql:
+ sql: UPDATE metabase_field set effective_type = base_type
+ - changeSet:
+ id: 287
+ author: dpsutton
+ comment: Added 0.39 - Semantic type system - migrate ISO8601 strings
+ changes:
+ - sql:
+ sql: >-
+ UPDATE metabase_field
+ SET semantic_type = NULL, -- special type was overriden to provide coercion so no semantic type
+ effective_type = (CASE semantic_type
+ WHEN 'type/ISO8601DateTimeString' THEN 'type/DateTime'
+ WHEN 'type/ISO8601TimeString' THEN 'type/Time'
+ WHEN 'type/ISO8601DateString' THEN 'type/Date'
+ END),
+ coercion_strategy = (CASE semantic_type
+ WHEN 'type/ISO8601DateTimeString' THEN 'Coercion/ISO8601->DateTime'
+ WHEN 'type/ISO8601TimeString' THEN 'Coercion/ISO8601->Time'
+ WHEN 'type/ISO8601DateString' THEN 'Coercion/ISO8601->Date'
+ END)
+ WHERE semantic_type IN ('type/ISO8601DateTimeString',
+ 'type/ISO8601TimeString',
+ 'type/ISO8601DateString');
+ - changeSet:
+ id: 288
+ author: dpsutton
+ comment: Added 0.39 - Semantic type system - migrate unix timestamps
+ changes:
+ - sql:
+ sql: >-
+ UPDATE metabase_field
+ set semantic_type = null,
+ effective_type = 'type/DateTime',
+ coercion_strategy = (case semantic_type
+ WHEN 'type/UNIXTimestampSeconds' THEN 'Coercion/UNIXSeconds->DateTime'
+ WHEN 'type/UNIXTimestampMilliSeconds' THEN 'Coercion/UNIXMilliSeconds->DateTime'
+ WHEN 'type/UNIXTimestampMicroSeconds' THEN 'Coercion/UNIXMicroSeconds->DateTime'
+ END)
+ WHERE semantic_type IN ('type/UNIXTimestampSeconds',
+ 'type/UNIXTimestampMilliSeconds',
+ 'type/UNIXTimestampMicroSeconds')
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/resources/sample-dataset.db.mv.db b/resources/sample-dataset.db.mv.db
index 87233d5db0106..c60626f0038cc 100644
Binary files a/resources/sample-dataset.db.mv.db and b/resources/sample-dataset.db.mv.db differ
diff --git a/shared/src/metabase/types.cljc b/shared/src/metabase/types.cljc
index c6b76b40b08d4..b92f78af07225 100644
--- a/shared/src/metabase/types.cljc
+++ b/shared/src/metabase/types.cljc
@@ -209,6 +209,18 @@
(derive :type/Boolean :type/Category)
(derive :type/Enum :type/Category)
+(derive :Coercion/String->Temporal :Coercion/*)
+(derive :Coercion/ISO8601->Temporal :Coercion/String->Temporal)
+(derive :Coercion/ISO8601->DateTime :Coercion/ISO8601->Temporal)
+(derive :Coercion/ISO8601->Time :Coercion/ISO8601->Temporal)
+(derive :Coercion/ISO8601->Date :Coercion/ISO8601->Temporal)
+
+(derive :Coercion/Number->Temporal :Coercion/*)
+(derive :Coercion/UNIXTime->Temporal :Coercion/Number->Temporal)
+(derive :Coercion/UNIXSeconds->DateTime :Coercion/UNIXTime->Temporal)
+(derive :Coercion/UNIXMilliSeconds->DateTime :Coercion/UNIXTime->Temporal)
+(derive :Coercion/UNIXMicroSeconds->DateTime :Coercion/UNIXTime->Temporal)
+
;;; ---------------------------------------------------- Util Fns ----------------------------------------------------
(defn- types->parents
@@ -225,8 +237,55 @@
"True if a Metabase `Field` instance has a temporal base or semantic type, i.e. if this Field represents a value
relating to a moment in time."
{:arglists '([field])}
- [{base-type :base_type, semantic-type :semantic_type}]
- (some #(isa? % :type/Temporal) [base-type semantic-type]))
+ [{base-type :base_type, effective-type :effective_type}]
+ (some #(isa? % :type/Temporal) [base-type effective-type]))
+
+(def ^:private coercions
+ "A map from types to maps of conversions to resulting effective types:
+
+ eg:
+ {:type/Text {:Coercion/ISO8601->Date :type/Date
+ :Coercion/ISO8601->DateTime :type/DateTime
+ :Coercion/ISO8601->Time :type/Time}}"
+ ;; Decimal seems out of place but that's the type that oracle uses Number which we map to Decimal. Not sure if
+ ;; that's an intentional mapping or not. But it does mean that lots of extra columns will be offered a conversion
+ ;; (think Price being offerred to be interpreted as a date)
+ (let [numeric-types [:type/BigInteger :type/Integer :type/Decimal]]
+ (reduce #(assoc %1 %2 {:Coercion/UNIXMicroSeconds->DateTime :type/DateTime
+ :Coercion/UNIXMilliSeconds->DateTime :type/DateTime
+ :Coercion/UNIXSeconds->DateTime :type/DateTime})
+ {:type/Text {:Coercion/ISO8601->Date :type/Date
+ :Coercion/ISO8601->DateTime :type/DateTime
+ :Coercion/ISO8601->Time :type/Time}}
+ numeric-types)))
+
+(defn ^:export is_coerceable
+ "Returns a boolean of whether a field base-type has any coercion strategies available."
+ [base-type]
+ (boolean (contains? coercions (keyword base-type))))
+
+(defn ^:export effective_type_for_coercion
+ "The effective type resulting from a coercion."
+ [coercion]
+ ;;todo: unify this with the coercions map above
+ (get {:Coercion/ISO8601->Date :type/Date
+ :Coercion/ISO8601->DateTime :type/DateTime
+ :Coercion/ISO8601->Time :type/Time
+ :Coercion/UNIXMicroSeconds->DateTime :type/DateTime
+ :Coercion/UNIXMilliSeconds->DateTime :type/DateTime
+ :Coercion/UNIXSeconds->DateTime :type/DateTime}
+ (keyword coercion)))
+
+(defn ^:export coercions_for_type
+ "Coercions available for a type. In cljs will return a js array of strings like [\"Coercion/ISO8601->Time\" ...]. In
+ clojure will return a sequence of keywords."
+ [base-type]
+ (let [applicable (keys (get coercions (keyword base-type)))]
+ #?(:cljs
+ (clj->js (map (fn [kw] (str (namespace kw) "/" (name kw)))
+ applicable))
+ :clj
+ applicable)))
#?(:cljs
(defn ^:export isa
diff --git a/src/metabase/api/field.clj b/src/metabase/api/field.clj
index d5e970c9394a5..c709262a21b32 100644
--- a/src/metabase/api/field.clj
+++ b/src/metabase/api/field.clj
@@ -12,6 +12,7 @@
[metabase.models.table :refer [Table]]
[metabase.query-processor :as qp]
[metabase.related :as related]
+ [metabase.types :as types]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
[metabase.util.schema :as su]
@@ -27,6 +28,11 @@
(su/with-api-error-message (s/constrained s/Str #(isa? (keyword %) :type/*))
"value must be a valid field type."))
+(def ^:private CoercionType
+ "Schema for a valid `Coercion` type."
+ (su/with-api-error-message (s/constrained s/Str #(isa? (keyword %) :Coercion/*))
+ "value must be a valid coercion type."))
+
(def ^:private FieldVisibilityType
"Schema for a valid `Field` visibility type."
(apply s/enum (map name field/visibility-types)))
@@ -93,7 +99,7 @@
(api/defendpoint PUT "/:id"
"Update `Field` with ID."
[id :as {{:keys [caveats description display_name fk_target_field_id points_of_interest semantic_type
- visibility_type has_field_values settings]
+ coercion_strategy visibility_type has_field_values settings]
:as body} :body}]
{caveats (s/maybe su/NonBlankString)
description (s/maybe su/NonBlankString)
@@ -101,11 +107,14 @@
fk_target_field_id (s/maybe su/IntGreaterThanZero)
points_of_interest (s/maybe su/NonBlankString)
semantic_type (s/maybe FieldType)
+ coercion_strategy (s/maybe CoercionType)
visibility_type (s/maybe FieldVisibilityType)
has_field_values (s/maybe (apply s/enum (map name field/has-field-values-options)))
settings (s/maybe su/Map)}
(let [field (hydrate (api/write-check Field id) :dimensions)
new-semantic-type (keyword (get body :semantic_type (:semantic_type field)))
+ effective-type (or (types/effective_type_for_coercion coercion_strategy)
+ (:base_type field))
removed-fk? (removed-fk-semantic-type? (:semantic_type field) new-semantic-type)
fk-target-field-id (get body :fk_target_field_id (:fk_target_field_id field))]
@@ -123,8 +132,11 @@
true)
(clear-dimension-on-type-change! field (:base_type field) new-semantic-type)
(db/update! Field id
- (u/select-keys-when (assoc body :fk_target_field_id (when-not removed-fk? fk-target-field-id))
- :present #{:caveats :description :fk_target_field_id :points_of_interest :semantic_type :visibility_type
+ (u/select-keys-when (assoc body
+ :fk_target_field_id (when-not removed-fk? fk-target-field-id)
+ :effective_type effective-type
+ :coercion_strategy coercion_strategy)
+ :present #{:caveats :description :fk_target_field_id :points_of_interest :semantic_type :visibility_type :coercion_strategy :effective_type
:has_field_values}
:non-nil #{:display_name :settings})))))
;; return updated field
diff --git a/src/metabase/driver/common/parameters/values.clj b/src/metabase/driver/common/parameters/values.clj
index fbc16158b1dc1..07f1a0fcf5c75 100644
--- a/src/metabase/driver/common/parameters/values.clj
+++ b/src/metabase/driver/common/parameters/values.clj
@@ -117,7 +117,8 @@
(i/map->FieldFilter
;; TODO - shouldn't this use the QP Store?
{:field (let [field-id (field-filter->field-id field-filter)]
- (or (db/select-one [Field :name :parent_id :table_id :base_type :semantic_type] :id field-id)
+ (or (db/select-one [Field :name :parent_id :table_id :base_type :effective_type :coercion_strategy :semantic_type]
+ :id field-id)
(throw (ex-info (str (deferred-tru "Can''t find field with ID: {0}" field-id))
{:field-id field-id, :type qp.error-type/invalid-parameter}))))
:value (if-let [value-info-or-infos (or
@@ -240,13 +241,12 @@
to parse it as appropriate based on the base type and semantic type of the Field associated with it). These are
special cases for handling types that do not have an associated parameter type (such as `date` or `number`), such as
UUID fields."
- [base-type :- su/FieldType semantic-type :- (s/maybe su/FieldType) value]
+ [effective-type :- su/FieldType value]
(cond
- (isa? base-type :type/UUID)
+ (isa? effective-type :type/UUID)
(UUID/fromString value)
- (and (isa? base-type :type/Number)
- (not (isa? semantic-type :type/Temporal)))
+ (isa? effective-type :type/Number)
(value->number value)
:else
@@ -255,17 +255,17 @@
(s/defn ^:private update-filter-for-field-type :- ParsedParamValue
"Update a Field Filter with a textual, or sequence of textual, values. The base type and semantic type of the field
are used to determine what 'semantic' type interpretation is required (e.g. for UUID fields)."
- [{{base-type :base_type, semantic-type :semantic_type} :field, {value :value} :value, :as field-filter} :- FieldFilter]
+ [{{effective_type :effective_type, :as _field} :field, {value :value} :value, :as field-filter} :- FieldFilter]
(let [new-value (cond
(string? value)
- (parse-value-for-field-type base-type semantic-type value)
+ (parse-value-for-field-type effective_type value)
(and (sequential? value)
(every? string? value))
- (mapv (partial parse-value-for-field-type base-type semantic-type) value))]
+ (mapv (partial parse-value-for-field-type effective_type) value))]
(when (not= value new-value)
- (log/tracef "update filter for base-type: %s semantic-type: %s value: %s -> %s"
- (pr-str base-type) (pr-str semantic-type) (pr-str value) (pr-str new-value)))
+ (log/tracef "update filter for base-type: %s value: %s -> %s"
+ (pr-str effective_type) (pr-str value) (pr-str new-value)))
(cond-> field-filter
new-value (assoc-in [:value :value] new-value))))
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 0e93acb7a94ff..1866e7a8fd8d7 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -162,7 +162,7 @@
(defmethod sql.qp/unix-timestamp->honeysql [:mysql :seconds] [_ _ expr]
(hsql/call :from_unixtime expr))
-(defmethod sql.qp/cast-temporal-string [:mysql :type/ISO8601DateTimeString]
+(defmethod sql.qp/cast-temporal-string [:mysql :Coercion/ISO8601->DateTime]
[_driver _semantic_type expr]
(hx/->datetime expr))
diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.clj
index a1f7345794a38..5d2e4a95b2939 100644
--- a/src/metabase/driver/sql/parameters/substitution.clj
+++ b/src/metabase/driver/sql/parameters/substitution.clj
@@ -238,9 +238,7 @@
(:replacement-snippet
(honeysql->replacement-snippet-info
driver
- (let [identifier (cond->> (sql.qp/->honeysql driver (sql.qp/field->identifier driver field))
- (isa? semantic-type :type/UNIXTimestamp)
- (sql.qp/unix-timestamp->honeysql driver (sql.qp/semantic-type->unix-timestamp-unit semantic-type)))]
+ (let [identifier (sql.qp/cast-field-if-needed driver field (sql.qp/->honeysql driver (sql.qp/field->identifier driver field)))]
(if (date-params/date-type? param-type)
(sql.qp/date driver :day identifier)
identifier)))))
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index cf0fc6fcd6140..1290a8e3e66a3 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -229,25 +229,29 @@
(->honeysql driver (mbql.u/expression-with-name *query* expression-name)))
(defn semantic-type->unix-timestamp-unit
- "Translates types like `:type/UNIXTimestampSeconds` to the corresponding unit of time to use in
- `unix-timestamp->honeysql`. Throws an AssertionError if the argument does not descend from `:type/UNIXTimestamp`
+ "Translates coercion types like `:Coercion/UNIXSeconds->DateTime` to the corresponding unit of time to use in
+ `unix-timestamp->honeysql`. Throws an AssertionError if the argument does not descend from `:UNIXTime->Temporal`
and an exception if the type does not have an associated unit."
- [semantic-type]
- (assert (isa? semantic-type :type/UNIXTimestamp) "Semantic type must be a UNIXTimestamp")
- (or (get {:type/UNIXTimestampMicroseconds :microseconds
- :type/UNIXTimestampMilliseconds :milliseconds
- :type/UNIXTimestampSeconds :seconds}
- semantic-type)
- (throw (Exception. (tru "No magnitude known for {0}" semantic-type)))))
+ [coercion-type]
+ (assert (isa? coercion-type :Coercion/UNIXTime->Temporal) "Semantic type must be a UNIXTimestamp")
+ (or (get {:Coercion/UNIXMicroSeconds->DateTime :microseconds
+ :Coercion/UNIXMilliSeconds->DateTime :milliseconds
+ :Coercion/UNIXSeconds->DateTime :seconds}
+ coercion-type)
+ (throw (Exception. (tru "No magnitude known for {0}" coercion-type)))))
(defn cast-field-if-needed
"Wrap a `field-identifier` in appropriate HoneySQL expressions if it refers to a UNIX timestamp Field."
[driver field field-identifier]
- (match [(:base_type field) (:semantic_type field)]
- [(:isa? :type/Number) (:isa? :type/UNIXTimestamp)] (unix-timestamp->honeysql driver
- (semantic-type->unix-timestamp-unit (:semantic_type field))
- field-identifier)
- [:type/Text (:isa? :type/TemporalString)] (cast-temporal-string driver (:semantic_type field) field-identifier)
+ (match [(:base_type field) (:coercion_strategy field)]
+ [(:isa? :type/Number) (:isa? :Coercion/UNIXTime->Temporal)]
+ (unix-timestamp->honeysql driver
+ (semantic-type->unix-timestamp-unit (:coercion_strategy field))
+ field-identifier)
+
+ [:type/Text (:isa? :Coercion/String->Temporal) ]
+ (cast-temporal-string driver (:coercion_strategy field) field-identifier)
+
:else field-identifier))
(defmethod ->honeysql [:sql TypedHoneySQLForm]
diff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.clj
index 3931c57f01348..e7e446854ac6d 100644
--- a/src/metabase/driver/sql_jdbc.clj
+++ b/src/metabase/driver/sql_jdbc.clj
@@ -69,14 +69,14 @@
[driver database table]
(sql-jdbc.sync/describe-table-fks driver database table))
-(defmethod sql.qp/cast-temporal-string [:sql-jdbc :type/ISO8601DateTimeString]
+(defmethod sql.qp/cast-temporal-string [:sql-jdbc :Coercion/ISO8601->DateTime]
[_driver _semantic_type expr]
(hx/->timestamp expr))
-(defmethod sql.qp/cast-temporal-string [:sql-jdbc :type/ISO8601DateString]
+(defmethod sql.qp/cast-temporal-string [:sql-jdbc :Coercion/ISO8601->Date]
[_driver _semantic_type expr]
(hx/->date expr))
-(defmethod sql.qp/cast-temporal-string [:sql-jdbc :type/ISO8601TimeString]
+(defmethod sql.qp/cast-temporal-string [:sql-jdbc :Coercion/ISO8601->Time]
[_driver _semantic_type expr]
(hx/->time expr))
diff --git a/src/metabase/models/field.clj b/src/metabase/models/field.clj
index 8dca6c06a437f..b0a65d10d918d 100644
--- a/src/metabase/models/field.clj
+++ b/src/metabase/models/field.clj
@@ -58,13 +58,17 @@
(models/defmodel Field :metabase_field)
-(defn- check-valid-types [{base-type :base_type, semantic-type :semantic_type}]
+(defn- check-valid-types [{base-type :base_type, semantic-type :semantic_type,
+ coercion-strategy :coercion_strategy}]
(when base-type
(assert (isa? (keyword base-type) :type/*)
(str "Invalid base type: " base-type)))
(when semantic-type
(assert (isa? (keyword semantic-type) :type/*)
- (str "Invalid semantic type: " semantic-type))))
+ (str "Invalid semantic type: " semantic-type)))
+ (when coercion-strategy
+ (assert (isa? (keyword coercion-strategy) :Coercion/*)
+ (str "Invalid coercion strategy: " coercion-strategy))))
(defn- pre-insert [field]
(check-valid-types field)
@@ -134,6 +138,8 @@
(merge models/IModelDefaults
{:hydration-keys (constantly [:destination :field :origin :human_readable_field])
:types (constantly {:base_type :keyword
+ :effective_type :keyword
+ :coercion_strategy :keyword
:semantic_type :keyword
:visibility_type :keyword
:has_field_values :keyword
diff --git a/src/metabase/models/field_values.clj b/src/metabase/models/field_values.clj
index c00267e5a8f3c..ade095d6e828b 100644
--- a/src/metabase/models/field_values.clj
+++ b/src/metabase/models/field_values.clj
@@ -243,8 +243,8 @@
[field-ids]
(let [fields (when (seq field-ids)
(filter field-should-have-field-values?
- (db/select ['Field :name :id :base_type :semantic_type :visibility_type :table_id
- :has_field_values]
+ (db/select ['Field :name :id :base_type :effective_type :coercion_strategy
+ :semantic_type :visibility_type :table_id :has_field_values]
:id [:in field-ids])))
table-id->is-on-demand? (table-ids->table-id->is-on-demand? (map :table_id fields))]
(doseq [{table-id :table_id, :as field} fields]
diff --git a/src/metabase/query_processor/middleware/add_implicit_clauses.clj b/src/metabase/query_processor/middleware/add_implicit_clauses.clj
index 4932d18919ebf..d5ee21b942fa6 100644
--- a/src/metabase/query_processor/middleware/add_implicit_clauses.clj
+++ b/src/metabase/query_processor/middleware/add_implicit_clauses.clj
@@ -21,7 +21,7 @@
(defn- table->sorted-fields
[table-id]
- (db/select [Field :id :base_type :semantic_type]
+ (db/select [Field :id :base_type :effective_type :coercion_strategy :semantic_type]
:table_id table-id
:active true
:visibility_type [:not-in ["sensitive" "retired"]]
diff --git a/src/metabase/query_processor/middleware/add_source_metadata.clj b/src/metabase/query_processor/middleware/add_source_metadata.clj
index 5213c6ca15929..7f2278894d9d1 100644
--- a/src/metabase/query_processor/middleware/add_source_metadata.clj
+++ b/src/metabase/query_processor/middleware/add_source_metadata.clj
@@ -55,8 +55,8 @@
;; to end up adding it again when the middleware runs at the top level
:query (assoc-in source-query [:middleware :disable-remaps?] true)}))]
(for [col cols]
- (select-keys col [:name :id :table_id :display_name :base_type :semantic_type :unit :fingerprint :settings
- :source_alias :field_ref :parent_id])))
+ (select-keys col [:name :id :table_id :display_name :base_type :effective_type :coercion_strategy
+ :semantic_type :unit :fingerprint :settings :source_alias :field_ref :parent_id])))
(catch Throwable e
(log/error e (str (trs "Error determining expected columns for query")))
nil)))
diff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.clj
index 3e0a6860e2e3d..7a48be2b856a3 100644
--- a/src/metabase/query_processor/middleware/annotate.clj
+++ b/src/metabase/query_processor/middleware/annotate.clj
@@ -27,7 +27,8 @@
:display_name s/Str
;; type of the Field. For Native queries we look at the values in the first 100 rows to make an educated guess
:base_type su/FieldType
- (s/optional-key :semantic_type) (s/maybe su/FieldType)
+ ;; effective_type, coercion, etc don't go here. probably best to rename base_type to effective type in the return
+ ;; from the metadata but that's for another day
;; where this column came from in the original query.
:source (s/enum :aggregation :fields :breakout :native)
;; a field clause that can be used to refer to this Field if this query is subsequently used as a source query.
diff --git a/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj b/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj
index fe9d5d3bfa4d0..86247a8b0173e 100644
--- a/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj
+++ b/src/metabase/query_processor/middleware/auto_bucket_datetimes.clj
@@ -2,7 +2,8 @@
"Middleware for automatically bucketing unbucketed `:type/Temporal` (but not `:type/Time`) Fields with `:day`
bucketing. Applies to any unbucketed Field in a breakout, or fields in a filter clause being compared against
`yyyy-MM-dd` format datetime strings."
- (:require [medley.core :as m]
+ (:require [clojure.set :as set]
+ [medley.core :as m]
[metabase.mbql.predicates :as mbql.preds]
[metabase.mbql.schema :as mbql.s]
[metabase.mbql.util :as mbql.u]
@@ -34,9 +35,14 @@
[id-or-name {:base-type base-type}]))
;; build map of field ID ->
(when-let [field-ids (seq (filter integer? (map second unbucketed-fields)))]
- (into {} (for [{id :id, base-type :base_type, semantic-type :semantic_type} (db/select [Field :id :base_type :semantic_type]
- :id [:in (set field-ids)])]
- [id {:base-type base-type, :semantic-type semantic-type}])))))
+ (into {} (for [{id :id, :as field}
+ (db/select [Field :id :base_type :effective_type :semantic_type]
+ :id [:in (set field-ids)])]
+ [id (set/rename-keys (select-keys field
+ [:base_type :effective_type :semantic_type])
+ {:base_type :base-type
+ :effective_type :effective-type
+ :semantic_type :semantic-type})])))))
(defn- yyyy-MM-dd-date-string? [x]
(and (string? x)
@@ -69,11 +75,11 @@
(let [[_ _ opts] x]
((some-fn :temporal-unit :binning) opts)))))
-(defn- date-or-datetime-field? [{base-type :base-type, semantic-type :semantic-type}]
+(defn- date-or-datetime-field? [{base-type :base-type, effective-type :effective-type}]
(some (fn [field-type]
(some #(isa? field-type %)
[:type/Date :type/DateTime]))
- [base-type semantic-type]))
+ [base-type effective-type]))
(s/defn ^:private wrap-unbucketed-fields
"Add `:temporal-unit` to `:field`s in breakouts and filters if appropriate; look at corresponing type information in
diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.clj
index 800685835b7cb..ac9e4e8fdb395 100644
--- a/src/metabase/query_processor/middleware/results_metadata.clj
+++ b/src/metabase/query_processor/middleware/results_metadata.clj
@@ -114,7 +114,7 @@
(mapv
(fn [{final-base-type :base_type, :as final-col} {our-base-type :base_type, :as insights-col}]
(merge
- (select-keys final-col [:name :display_name :base_type :semantic_type :id :field_ref])
+ (select-keys final-col [:name :display_name :base_type :effective_type :coercion_strategy :semantic_type :id :field_ref])
insights-col
(when (= our-base-type :type/*)
{:base_type final-base-type})))
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index b13e7b5854368..8aacfe6d50aa1 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -22,7 +22,7 @@
(defmethod type-info :default [_] nil)
(defmethod type-info (class Field) [this]
- (let [field-info (select-keys this [:base_type :semantic_type :database_type :name])]
+ (let [field-info (select-keys this [:base_type :effective_type :coercion_strategy :semantic_type :database_type :name])]
(merge
field-info
;; add in a default unit for this Field so we know to wrap datetime strings in `absolute-datetime` below based on
diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.clj
index 1e3c7af659dfd..6cd9ba614a36a 100644
--- a/src/metabase/query_processor/store.clj
+++ b/src/metabase/query_processor/store.clj
@@ -86,9 +86,11 @@
query results. Try to keep this set pared down to just what's needed by the QP and frontend, since it has to be done
for every MBQL query."
[:base_type
+ :coercion_strategy
:database_type
:description
:display_name
+ :effective_type
:fingerprint
:id
:name
@@ -101,15 +103,19 @@
(def ^:private FieldInstanceWithRequiredStorekeys
(s/both
(class Field)
- {:name su/NonBlankString
- :display_name su/NonBlankString
- :description (s/maybe s/Str)
- :database_type su/NonBlankString
- :base_type su/FieldType
- :semantic_type (s/maybe su/FieldType)
- :fingerprint (s/maybe su/Map)
- :parent_id (s/maybe su/IntGreaterThanZero)
- s/Any s/Any}))
+ {:name su/NonBlankString
+ :display_name su/NonBlankString
+ :description (s/maybe s/Str)
+ :database_type su/NonBlankString
+ :base_type su/FieldType
+ ;; there's a tension as we sometimes store fields from the db, and sometimes store computed fields. ideally we
+ ;; would make everything just use base_type.
+ (s/optional-key :effective_type) (s/maybe su/FieldType)
+ (s/optional-key :coercion_strategy) (s/maybe su/CoercionStrategy)
+ :semantic_type (s/maybe su/FieldType)
+ :fingerprint (s/maybe su/Map)
+ :parent_id (s/maybe su/IntGreaterThanZero)
+ s/Any s/Any}))
;;; ------------------------------------------ Saving objects in the Store -------------------------------------------
diff --git a/src/metabase/sync/interface.clj b/src/metabase/sync/interface.clj
index aaf23fb833ae4..7484b5af18ccf 100644
--- a/src/metabase/sync/interface.clj
+++ b/src/metabase/sync/interface.clj
@@ -21,15 +21,19 @@
(def TableMetadataField
"Schema for a given Field as provided in `describe-table`."
- {:name su/NonBlankString
- :database-type (s/maybe su/NonBlankString) ; blank if the Field is all NULL & untyped, i.e. in Mongo
- :base-type su/FieldType
- :database-position su/IntGreaterThanOrEqualToZero
- (s/optional-key :semantic-type) (s/maybe su/FieldType)
- (s/optional-key :field-comment) (s/maybe su/NonBlankString)
- (s/optional-key :pk?) s/Bool
- (s/optional-key :nested-fields) #{(s/recursive #'TableMetadataField)}
- (s/optional-key :custom) {s/Any s/Any}})
+ {:name su/NonBlankString
+ :database-type (s/maybe su/NonBlankString) ; blank if the Field is all NULL & untyped, i.e. in Mongo
+ :base-type su/FieldType
+ :database-position su/IntGreaterThanOrEqualToZero
+ (s/optional-key :semantic-type) (s/maybe su/FieldType)
+ (s/optional-key :effective-type) (s/maybe su/FieldType)
+ (s/optional-key :coercion-strategy) (s/maybe su/CoercionStrategy)
+ (s/optional-key :field-comment) (s/maybe su/NonBlankString)
+ (s/optional-key :pk?) s/Bool
+ (s/optional-key :nested-fields) #{(s/recursive #'TableMetadataField)}
+ (s/optional-key :custom) {s/Any s/Any}
+ ;; for future backwards compatability, when adding things
+ s/Keyword s/Any})
(def TableMetadata
"Schema for the expected output of `describe-table`."
diff --git a/src/metabase/sync/sync_metadata/fields/fetch_metadata.clj b/src/metabase/sync/sync_metadata/fields/fetch_metadata.clj
index 8bc6aa06ac3e9..5ba2e6b527a9b 100644
--- a/src/metabase/sync/sync_metadata/fields/fetch_metadata.clj
+++ b/src/metabase/sync/sync_metadata/fields/fetch_metadata.clj
@@ -24,6 +24,8 @@
:id (:id field)
:name (:name field)
:database-type (:database_type field)
+ :effective-type (:effective_type field)
+ :coercion-strategy (:coercion_strategy field)
:base-type (:base_type field)
:semantic-type (:semantic_type field)
:pk? (isa? (:semantic_type field) :type/PK)
@@ -61,7 +63,8 @@
(s/defn ^:private table->fields :- [i/FieldInstance]
"Fetch active Fields from the Metabase application database for a given `table`."
[table :- i/TableInstance]
- (db/select [Field :name :database_type :base_type :semantic_type :parent_id :id :description :database_position]
+(db/select [Field :name :database_type :base_type :effective_type :coercion_strategy :semantic_type
+ :parent_id :id :description :database_position]
:table_id (u/the-id table)
:active true
{:order-by table/field-order-rule}))
diff --git a/src/metabase/sync/sync_metadata/fields/sync_instances.clj b/src/metabase/sync/sync_metadata/fields/sync_instances.clj
index 6dd38ddacc99c..d9ac8dd203fec 100644
--- a/src/metabase/sync/sync_metadata/fields/sync_instances.clj
+++ b/src/metabase/sync/sync_metadata/fields/sync_instances.clj
@@ -45,6 +45,8 @@
:display_name (humanization/name->human-readable-name field-name)
:database_type (or database-type "NULL") ; placeholder for Fields w/ no type info (e.g. Mongo) & all NULL
:base_type base-type
+ ;; todo test this?
+ :effective_type base-type
:semantic_type (common/semantic-type field)
:parent_id parent-id
:description field-comment
diff --git a/src/metabase/util/schema.clj b/src/metabase/util/schema.clj
index 04c3710298f85..f142c2fbe82eb 100644
--- a/src/metabase/util/schema.clj
+++ b/src/metabase/util/schema.clj
@@ -205,6 +205,11 @@
(with-api-error-message (s/pred #(isa? % :type/*) (deferred-tru "Valid field type"))
(deferred-tru "value must be a valid field type.")))
+(def CoercionStrategy
+ "Schema for a valid Field type (does it derive from `:type/*`)?"
+ (with-api-error-message (s/pred #(isa? % :Coercion/*) (deferred-tru "Valid coercion strategy"))
+ (deferred-tru "value must be a valid coercion strategy.")))
+
(def FieldTypeKeywordOrString
"Like `FieldType` (e.g. a valid derivative of `:type/*`) but allows either a keyword or a string.
This is useful especially for validating API input or objects coming out of the DB as it is unlikely
diff --git a/test/metabase/api/database_test.clj b/test/metabase/api/database_test.clj
index a9bd567dd191f..3d4ee13ee4993 100644
--- a/test/metabase/api/database_test.clj
+++ b/test/metabase/api/database_test.clj
@@ -249,6 +249,7 @@
:display_name "ID"
:database_type "BIGINT"
:base_type "type/BigInteger"
+ :effective_type "type/BigInteger"
:visibility_type "normal"
:has_field_values "none"
:database_position 0})
@@ -260,6 +261,7 @@
:display_name "Name"
:database_type "VARCHAR"
:base_type "type/Text"
+ :effective_type "type/Text"
:visibility_type "normal"
:has_field_values "list"
:database_position 1})]
diff --git a/test/metabase/api/field_test.clj b/test/metabase/api/field_test.clj
index 3a398b255f0f7..4982c8063cf5c 100644
--- a/test/metabase/api/field_test.clj
+++ b/test/metabase/api/field_test.clj
@@ -58,6 +58,7 @@
:visibility_type "normal"
:database_type "VARCHAR"
:base_type "type/Text"
+ :effective_type "type/Text"
:has_field_values "list"
:dimensions []
:name_field nil})
diff --git a/test/metabase/api/table_test.clj b/test/metabase/api/table_test.clj
index 95e2e3a67b586..6c5c4701c4a34 100644
--- a/test/metabase/api/table_test.clj
+++ b/test/metabase/api/table_test.clj
@@ -152,6 +152,7 @@
:display_name "ID"
:database_type "BIGINT"
:base_type "type/BigInteger"
+ :effective_type "type/BigInteger"
:visibility_type "normal"
:has_field_values "none")
(assoc (field-details (Field (mt/id :users :name)))
@@ -161,6 +162,7 @@
:display_name "Name"
:database_type "VARCHAR"
:base_type "type/Text"
+ :effective_type "type/Text"
:visibility_type "normal"
:dimension_options []
:default_dimension_option nil
@@ -173,6 +175,7 @@
:display_name "Last Login"
:database_type "TIMESTAMP"
:base_type "type/DateTime"
+ :effective_type "type/DateTime"
:visibility_type "normal"
:dimension_options (var-get #'table-api/datetime-dimension-indexes)
:default_dimension_option (var-get #'table-api/date-default-index)
@@ -186,6 +189,7 @@
:display_name "Password"
:database_type "VARCHAR"
:base_type "type/Text"
+ :effective_type "type/Text"
:visibility_type "sensitive"
:has_field_values "list"
:position 3
@@ -211,6 +215,7 @@
:display_name "ID"
:database_type "BIGINT"
:base_type "type/BigInteger"
+ :effective_type "type/BigInteger"
:has_field_values "none")
(assoc (field-details (Field (mt/id :users :name)))
:table_id (mt/id :users)
@@ -219,6 +224,7 @@
:display_name "Name"
:database_type "VARCHAR"
:base_type "type/Text"
+ :effective_type "type/Text"
:has_field_values "list"
:position 1
:database_position 1)
@@ -228,6 +234,7 @@
:display_name "Last Login"
:database_type "TIMESTAMP"
:base_type "type/DateTime"
+ :effective_type "type/DateTime"
:dimension_options (var-get #'table-api/datetime-dimension-indexes)
:default_dimension_option (var-get #'table-api/date-default-index)
:has_field_values "none"
@@ -323,6 +330,7 @@
:display_name "User ID"
:database_type "INTEGER"
:base_type "type/Integer"
+ :effective_type "type/Integer"
:semantic_type "type/FK"
:database_position 2
:position 2
@@ -340,6 +348,7 @@
:name "ID"
:display_name "ID"
:base_type "type/BigInteger"
+ :effective_type "type/BigInteger"
:database_type "BIGINT"
:semantic_type "type/PK"
:table (merge
@@ -372,6 +381,7 @@
:display_name "ID"
:database_type "BIGINT"
:base_type "type/BigInteger"
+ :effective_type "type/BigInteger"
:has_field_values "none"})
(merge
(field-details (Field (mt/id :categories :name)))
@@ -381,6 +391,7 @@
:display_name "Name"
:database_type "VARCHAR"
:base_type "type/Text"
+ :effective_type "type/Text"
:dimension_options []
:default_dimension_option nil
:has_field_values "list"
diff --git a/test/metabase/driver/common/parameters/values_test.clj b/test/metabase/driver/common/parameters/values_test.clj
index fd6e4cdb4df42..ffe47d4f7e65b 100644
--- a/test/metabase/driver/common/parameters/values_test.clj
+++ b/test/metabase/driver/common/parameters/values_test.clj
@@ -29,10 +29,25 @@
(#'values/value-for-tag
{:name "id", :display-name "ID", :type :text, :required true, :default "100"} nil)))))
+(defn- extra-field-info
+ "Add extra field information like coercion_strategy, semantic_type, and effective_type."
+ [{:keys [base_type] :as field}]
+ (merge {:coercion_strategy nil, :effective_type base_type, :semantic_type nil}
+ field))
+
+(defn value-for-tag
+ "Call the private function and de-recordize the field"
+ [field-info info]
+ (mt/derecordize (#'values/value-for-tag field-info info)))
+
+(defn parse-tag
+ [field-info info]
+ (mt/derecordize (#'values/parse-tag field-info info)))
+
(deftest field-filter-test
(testing "specified"
(testing "date range for a normal :type/Temporal field"
- (is (= {:field (map->FieldInstance
+ (is (= {:field (extra-field-info
{:name "DATE"
:parent_id nil
:table_id (mt/id :checkins)
@@ -40,53 +55,54 @@
:semantic_type nil})
:value {:type :date/range
:value "2015-04-01~2015-05-01"}}
- (into {} (#'values/value-for-tag
- {:name "checkin_date"
- :display-name "Checkin Date"
- :type :dimension
- :dimension [:field-id (mt/id :checkins :date)]}
- [{:type :date/range
- :target [:dimension [:template-tag "checkin_date"]]
- :value "2015-04-01~2015-05-01"}])))))
+ (value-for-tag
+ {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :dimension [:field-id (mt/id :checkins :date)]}
+ [{:type :date/range
+ :target [:dimension [:template-tag "checkin_date"]]
+ :value "2015-04-01~2015-05-01"}]))))
(testing "date range for a UNIX timestamp field should work just like a :type/Temporal field (#11934)"
(mt/dataset tupac-sightings
(mt/$ids sightings
- (is (= {:field (map->FieldInstance
- {:name "TIMESTAMP"
- :parent_id nil
- :table_id $$sightings
- :base_type :type/BigInteger
- :semantic_type :type/UNIXTimestampSeconds})
+ (is (= {:field (extra-field-info
+ {:name "TIMESTAMP"
+ :parent_id nil
+ :table_id $$sightings
+ :base_type :type/BigInteger
+ :effective_type :type/DateTime
+ :coercion_strategy :Coercion/UNIXSeconds->DateTime})
:value {:type :date/range
:value "2020-02-01~2020-02-29"}}
- (into {} (#'values/value-for-tag
- {:name "timestamp"
- :display-name "Sighting Timestamp"
- :type :dimension
- :dimension $timestamp
- :widget-type :date/range}
- [{:type :date/range
- :target [:dimension [:template-tag "timestamp"]]
- :value "2020-02-01~2020-02-29"}]))))))))
+ (value-for-tag
+ {:name "timestamp"
+ :display-name "Sighting Timestamp"
+ :type :dimension
+ :dimension $timestamp
+ :widget-type :date/range}
+ [{:type :date/range
+ :target [:dimension [:template-tag "timestamp"]]
+ :value "2020-02-01~2020-02-29"}])))))))
(testing "unspecified"
- (is (= {:field (map->FieldInstance
+ (is (= {:field (extra-field-info
{:name "DATE"
:parent_id nil
:table_id (mt/id :checkins)
:base_type :type/Date
:semantic_type nil})
:value i/no-value}
- (into {} (#'values/value-for-tag
- {:name "checkin_date"
- :display-name "Checkin Date"
- :type :dimension
- :dimension [:field-id (mt/id :checkins :date)]}
- nil)))))
+ (value-for-tag
+ {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :dimension [:field-id (mt/id :checkins :date)]}
+ nil))))
(testing "id requiring casting"
- (is (= {:field (map->FieldInstance
+ (is (= {:field (extra-field-info
{:name "ID"
:parent_id nil
:table_id (mt/id :checkins)
@@ -94,19 +110,19 @@
:semantic_type :type/PK})
:value {:type :id
:value 5}}
- (into {} (#'values/value-for-tag
- {:name "id", :display-name "ID", :type :dimension, :dimension [:field-id (mt/id :checkins :id)]}
- [{:type :id, :target [:dimension [:template-tag "id"]], :value "5"}])))))
+ (value-for-tag
+ {:name "id", :display-name "ID", :type :dimension, :dimension [:field-id (mt/id :checkins :id)]}
+ [{:type :id, :target [:dimension [:template-tag "id"]], :value "5"}]))))
(testing "required but unspecified"
(is (thrown? Exception
- (into {} (#'values/value-for-tag
- {:name "checkin_date", :display-name "Checkin Date", :type "dimension", :required true,
- :dimension [:field (mt/id :checkins :date) nil]}
- nil)))))
+ (value-for-tag
+ {:name "checkin_date", :display-name "Checkin Date", :type "dimension", :required true,
+ :dimension [:field (mt/id :checkins :date) nil]}
+ nil))))
(testing "required and default specified"
- (is (= {:field (map->FieldInstance
+ (is (= {:field (extra-field-info
{:name "DATE"
:parent_id nil
:table_id (mt/id :checkins)
@@ -114,18 +130,18 @@
:semantic_type nil})
:value {:type :dimension
:value "2015-04-01~2015-05-01"}}
- (into {} (#'values/value-for-tag
- {:name "checkin_date"
- :display-name "Checkin Date"
- :type :dimension
- :required true
- :default "2015-04-01~2015-05-01",
- :dimension [:field-id (mt/id :checkins :date)]}
- nil)))))
+ (value-for-tag
+ {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :required true
+ :default "2015-04-01~2015-05-01",
+ :dimension [:field-id (mt/id :checkins :date)]}
+ nil))))
(testing "multiple values for the same tag should return a vector with multiple params instead of a single param"
- (is (= {:field (map->FieldInstance
+ (is (= {:field (extra-field-info
{:name "DATE"
:parent_id nil
:table_id (mt/id :checkins)
@@ -135,13 +151,13 @@
:value "2015-01-01~2016-09-01"}
{:type :date/single
:value "2015-07-01"}]}
- (into {} (#'values/value-for-tag
- {:name "checkin_date", :display-name "Checkin Date", :type :dimension, :dimension [:field-id (mt/id :checkins :date)]}
- [{:type :date/range, :target [:dimension [:template-tag "checkin_date"]], :value "2015-01-01~2016-09-01"}
- {:type :date/single, :target [:dimension [:template-tag "checkin_date"]], :value "2015-07-01"}])))))
+ (value-for-tag
+ {:name "checkin_date", :display-name "Checkin Date", :type :dimension, :dimension [:field-id (mt/id :checkins :date)]}
+ [{:type :date/range, :target [:dimension [:template-tag "checkin_date"]], :value "2015-01-01~2016-09-01"}
+ {:type :date/single, :target [:dimension [:template-tag "checkin_date"]], :value "2015-07-01"}]))))
(testing "Make sure defaults values get picked up for field filter clauses"
- (is (= {:field (map->FieldInstance
+ (is (= {:field (extra-field-info
{:name "DATE"
:parent_id nil
:table_id (mt/id :checkins)
@@ -149,14 +165,14 @@
:semantic_type nil})
:value {:type :date/all-options
:value "past5days"}}
- (into {} (#'values/parse-tag
- {:name "checkin_date"
- :display-name "Checkin Date"
- :type :dimension
- :dimension [:field-id (mt/id :checkins :date)]
- :default "past5days"
- :widget-type :date/all-options}
- nil))))))
+ (parse-tag
+ {:name "checkin_date"
+ :display-name "Checkin Date"
+ :type :dimension
+ :dimension [:field-id (mt/id :checkins :date)]
+ :default "past5days"
+ :widget-type :date/all-options}
+ nil)))))
(deftest field-filter-errors-test
(testing "error conditions for field filter (:dimension) parameters"
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index 77b07029056a9..e210c33dc6832 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -57,7 +57,7 @@
(tx/defdataset ^:private tiny-int-ones
[["number-of-cans"
[{:field-name "thing", :base-type :type/Text}
- {:field-name "number-of-cans", :base-type {:native "tinyint(1)"}}]
+ {:field-name "number-of-cans", :base-type {:native "tinyint(1)"}, :effective-type :type/Integer}]
[["Six Pack" 6]
["Toucan" 2]
["Empty Vending Machine" 0]]]])
@@ -89,7 +89,7 @@
(tx/defdataset ^:private year-db
[["years"
- [{:field-name "year_column", :base-type {:native "YEAR"}}]
+ [{:field-name "year_column", :base-type {:native "YEAR"}, :effective-type :type/Date}]
[[2001] [2002] [1999]]]])
(deftest year-test
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 2dbb5e838598a..79eac48a969c1 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -244,7 +244,7 @@
(testing "Verify that we identify JSON columns and mark metadata properly during sync"
(mt/dataset (mt/dataset-definition "Postgres with a JSON Field"
["venues"
- [{:field-name "address", :base-type {:native "json"}}]
+ [{:field-name "address", :base-type {:native "json"}, :effective-type :type/Structured}]
[[(hsql/raw "to_json('{\"street\": \"431 Natoma\", \"city\": \"San Francisco\", \"state\": \"CA\", \"zip\": 94103}'::text)")]]])
(is (= :type/SerializedJSON
(db/select-one-field :semantic_type Field, :id (mt/id :venues :address))))))))
@@ -310,7 +310,7 @@
(mt/defdataset ^:private ip-addresses
[["addresses"
- [{:field-name "ip", :base-type {:native "inet"}}]
+ [{:field-name "ip", :base-type {:native "inet"}, :effective-type :type/IPAddress}]
[[(hsql/raw "'192.168.1.1'::inet")]
[(hsql/raw "'10.4.4.15'::inet")]]]])
diff --git a/test/metabase/pulse/render/body_test.clj b/test/metabase/pulse/render/body_test.clj
index a7ae62701eeec..f494bc23db949 100644
--- a/test/metabase/pulse/render/body_test.clj
+++ b/test/metabase/pulse/render/body_test.clj
@@ -231,7 +231,8 @@
(def ^:private test-columns-with-date-semantic-type
(update test-columns 2 merge {:base_type :type/Text
- :semantic_type :type/DateTime}))
+ :effective_type :type/DateTime
+ :coercion_strategy :Coercion/ISO8601->DateTime}))
(deftest cols-with-semantic-types
(is (= [{:bar-width nil, :row [(number "1") (number "34.10") "Apr 1, 2014" "Stout Burgers & Beers"]}
diff --git a/test/metabase/query_processor/async_test.clj b/test/metabase/query_processor/async_test.clj
index 8846768bb916e..b895963ae6c4e 100644
--- a/test/metabase/query_processor/async_test.clj
+++ b/test/metabase/query_processor/async_test.clj
@@ -14,6 +14,8 @@
(is (= [{:name "NAME"
:display_name "Name"
:base_type :type/Text
+ :coercion_strategy nil
+ :effective_type :type/Text
:semantic_type :type/Name
:fingerprint {:global {:distinct-count 100, :nil% 0.0},
:type #:type {:Text
diff --git a/test/metabase/query_processor/middleware/add_source_metadata_test.clj b/test/metabase/query_processor/middleware/add_source_metadata_test.clj
index 0aab37b7425f9..7131a72b18561 100644
--- a/test/metabase/query_processor/middleware/add_source_metadata_test.clj
+++ b/test/metabase/query_processor/middleware/add_source_metadata_test.clj
@@ -16,7 +16,8 @@
(for [col (-> query-results :data :cols)]
(select-keys
col
- [:id :table_id :name :display_name :base_type :semantic_type :unit :fingerprint :settings :field_ref :parent_id])))
+ [:id :table_id :name :display_name :base_type :effective_type :coercion_strategy
+ :semantic_type :unit :fingerprint :settings :field_ref :parent_id])))
(defn- venues-source-metadata
([]
diff --git a/test/metabase/query_processor/middleware/annotate_test.clj b/test/metabase/query_processor/middleware/annotate_test.clj
index a0544589ba0a3..4f04714945388 100644
--- a/test/metabase/query_processor/middleware/annotate_test.clj
+++ b/test/metabase/query_processor/middleware/annotate_test.clj
@@ -176,10 +176,15 @@
(testing "For fields with parents we should return them with a combined name including parent's name"
(tt/with-temp* [Field [parent {:name "parent", :table_id (mt/id :venues)}]
Field [child {:name "child", :table_id (mt/id :venues), :parent_id (u/the-id parent)}]]
- (mt/with-everything-store
+ (mt/with-everything-store
(is (= {:description nil
:table_id (mt/id :venues)
:semantic_type nil
+ :effective_type nil
+ ;; these two are a gross symptom. there's some tension. sometimes it makes sense to have an effective
+ ;; type: the db type is different and we have a way to convert. Othertimes, it doesn't make sense:
+ ;; when the info is inferred. the solution to this might be quite extensive renaming
+ :coercion_strategy nil
:name "parent.child"
:settings nil
:field_ref [:field (u/the-id child) nil]
@@ -199,6 +204,8 @@
(is (= {:description nil
:table_id (mt/id :venues)
:semantic_type nil
+ :effective_type nil
+ :coercion_strategy nil
:name "grandparent.parent.child"
:settings nil
:field_ref [:field (u/the-id child) nil]
diff --git a/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj b/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj
index c4fd706c7665a..3cac754b76e3f 100644
--- a/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj
+++ b/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj
@@ -21,7 +21,7 @@
(deftest auto-bucket-in-breakout-test
(testing "does a :type/DateTime Field get auto-bucketed when present in a breakout clause?"
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:effective_type :type/DateTime}]
(is (= {:source-table 1
:breakout [[:field (u/the-id field) {:temporal-unit :day}]]}
(auto-bucket-mbql
@@ -41,7 +41,7 @@
;; [:= [:field {:temporal-unit :day}] "2018-11-19"]
;;
;; if `` is a `:type/DateTime` Field
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:base_type :type/DateTime, :effective_type :type/DateTime, :semantic_type nil}]
(is (= {:source-table 1
:filter [:= [:field (u/the-id field) {:temporal-unit :day}] "2018-11-19"]}
(auto-bucket-mbql
@@ -50,8 +50,8 @@
(deftest auto-bucket-in-compound-filter-clause-test
(testing "Fields should still get auto-bucketed when present in compound filter clauses (#9127)"
- (mt/with-temp* [Field [field-1 {:base_type :type/DateTime, :semantic_type nil}]
- Field [field-2 {:base_type :type/Text, :semantic_type nil}]]
+ (mt/with-temp* [Field [field-1 {:effective_type :type/DateTime}]
+ Field [field-2 {:effective_type :type/Text}]]
(is (= {:source-table 1
:filter [:and
[:= [:field (u/the-id field-1) {:temporal-unit :day}] "2018-11-19"]
@@ -71,7 +71,7 @@
:filter [:= [:field "timestamp" {:base-type :type/DateTime}] "2018-11-19"]})))))
(deftest do-not-autobucket-when-compared-to-non-yyyy-MM-dd-strings-test
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:base_type :type/DateTime, :effective_type :type/DateTime, :semantic_type nil}]
(testing (str "On the other hand, we shouldn't auto-bucket Fields inside a filter clause if they are being compared "
"against a datetime string that includes more than just yyyy-MM-dd:")
(is (= {:source-table 1
@@ -101,7 +101,7 @@
(deftest only-auto-bucket-appropriate-instances-test
(testing "if a Field occurs more than once we should only rewrite the instances that should be rebucketed"
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:base_type :type/DateTime, :effective_type :type/DateTime, :semantic_type nil}]
;; filter doesn't get auto-bucketed here because it's being compared to something with > date resolution
(is (= {:source-table 1
:breakout [[:field (u/the-id field) {:temporal-unit :day}]]
@@ -121,7 +121,7 @@
(deftest do-not-auto-bucket-inside-time-interval-test
(testing "We should not try to bucket Fields inside a `time-interval` clause as that would be invalid"
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:effective_type :type/DateTime}]
(is (= {:source-table 1
:filter [:time-interval [:field (u/the-id field) nil] -30 :day]}
(auto-bucket-mbql
@@ -130,7 +130,7 @@
(deftest do-not-auto-bucket-inappropriate-filter-clauses-test
(testing "Don't auto-bucket fields in non-equality or non-comparison filter clauses, for example `:is-null`:"
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:effective_type :type/DateTime}]
(is (= {:source-table 1
:filter [:is-null [:field (u/the-id field) nil]]}
(auto-bucket-mbql
@@ -140,7 +140,7 @@
(deftest do-not-auto-bucket-time-fields-test
(testing (str "we also should not auto-bucket Fields that are `:type/Time`, because grouping a Time Field by day "
"makes ZERO SENSE.")
- (mt/with-temp Field [field {:base_type :type/Time, :semantic_type nil}]
+ (mt/with-temp Field [field {:effective_type :type/Time}]
(is (= {:source-table 1
:breakout [[:field (u/the-id field) nil]]}
(auto-bucket-mbql
@@ -149,7 +149,8 @@
(deftest auto-bucket-by-semantic-type-test
(testing "should be considered to be :type/DateTime based on `semantic_type` as well"
- (mt/with-temp Field [field {:base_type :type/Integer, :semantic_type :type/DateTime}]
+ (mt/with-temp Field [field {:base_type :type/Integer, :effective_type :type/DateTime
+ :coercion_strategy :Coercion/UNIXSeconds->DateTime}]
(is (= {:source-table 1
:breakout [[:field (u/the-id field) {:temporal-unit :day}]]}
(auto-bucket-mbql
@@ -170,7 +171,7 @@
(deftest ignore-non-temporal-breakouts-test
(testing "does a breakout Field that isn't temporal pass thru unchnaged?"
- (mt/with-temp Field [field {:base_type :type/Integer, :semantic_type nil}]
+ (mt/with-temp Field [field {:effective_type :type/Integer}]
(is (= {:source-table 1
:breakout [[:field (u/the-id field) nil]]}
(auto-bucket-mbql
@@ -179,7 +180,7 @@
(deftest do-not-auto-bucket-already-bucketed-test
(testing "does a :type/DateTime breakout Field that is already bucketed pass thru unchanged?"
- (mt/with-temp Field [field {:base_type :type/DateTime, :semantic_type nil}]
+ (mt/with-temp Field [field {:effective_type :type/DateTime}]
(is (= {:source-table 1
:breakout [[:field (u/the-id field) {:temporal-unit :month}]]}
(auto-bucket-mbql
diff --git a/test/metabase/query_processor/middleware/binning_test.clj b/test/metabase/query_processor/middleware/binning_test.clj
index e54cd84763750..30017a858a434 100644
--- a/test/metabase/query_processor/middleware/binning_test.clj
+++ b/test/metabase/query_processor/middleware/binning_test.clj
@@ -88,16 +88,17 @@
;; Try an end-to-end test of the middleware
(defn- test-field []
(field/map->FieldInstance
- {:database_type "DOUBLE"
- :table_id (mt/id :checkins)
+ {:database_type "DOUBLE"
+ :table_id (mt/id :checkins)
:semantic_type :type/Income
- :name "TOTAL"
- :display_name "Total"
- :fingerprint {:global {:distinct-count 10000}
- :type {:type/Number {:min 12.061602936923117
- :max 238.32732001721533
- :avg 82.96014815230829}}}
- :base_type :type/Float}))
+ :name "TOTAL"
+ :display_name "Total"
+ :fingerprint {:global {:distinct-count 10000}
+ :type {:type/Number {:min 12.061602936923117
+ :max 238.32732001721533
+ :avg 82.96014815230829}}}
+ :base_type :type/Float
+ :effective_type :type/Float}))
(deftest update-binning-strategy-test
(mt/with-temp Field [field (test-field)]
diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.clj
index b41f0106e6166..fedfbe3633d7a 100644
--- a/test/metabase/query_processor/middleware/results_metadata_test.clj
+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj
@@ -190,6 +190,8 @@
:info {:card-id (u/the-id card)
:query-hash (qputil/query-hash {})}})
(is (= [{:base_type :type/DateTime
+ :effective_type :type/Date
+ :coercion_strategy nil
:display_name "Date"
:name "DATE"
:unit :year
diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
index 83767b78e411b..12860ba09718d 100644
--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj
@@ -24,23 +24,29 @@
(is (= (mt/mbql-query venues
{:filter [:>
$id
- [:value 50 {:base_type :type/BigInteger
- :semantic_type :type/PK
- :database_type "BIGINT"
- :name "ID"}]]})
+ [:value 50 {:base_type :type/BigInteger
+ :effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type :type/PK
+ :database_type "BIGINT"
+ :name "ID"}]]})
(wrap-value-literals
(mt/mbql-query venues
{:filter [:> $id 50]}))))
(is (= (mt/mbql-query venues
{:filter [:and
- [:> $id [:value 50 {:base_type :type/BigInteger
- :semantic_type :type/PK
- :database_type "BIGINT"
- :name "ID"}]]
- [:< $price [:value 5 {:base_type :type/Integer
- :semantic_type :type/Category
- :database_type "INTEGER"
- :name "PRICE"}]]]})
+ [:> $id [:value 50 {:base_type :type/BigInteger
+ :effective_type :type/BigInteger
+ :coercion_strategy nil
+ :semantic_type :type/PK
+ :database_type "BIGINT"
+ :name "ID"}]]
+ [:< $price [:value 5 {:base_type :type/Integer
+ :effective_type :type/Integer
+ :coercion_strategy nil
+ :semantic_type :type/Category
+ :database_type "INTEGER"
+ :name "PRICE"}]]]})
(wrap-value-literals
(mt/mbql-query venues
{:filter [:and
@@ -136,11 +142,13 @@
(is (= (mt/mbql-query checkins
{:filter [:starts-with
!month.date
- [:value "2018-10-01" {:base_type :type/Date
- :semantic_type nil
- :database_type "DATE"
- :unit :month
- :name "DATE"}]]})
+ [:value "2018-10-01" {:base_type :type/Date
+ :effective_type :type/Date
+ :coercion_strategy nil
+ :semantic_type nil
+ :database_type "DATE"
+ :unit :month
+ :name "DATE"}]]})
(wrap-value-literals
(mt/mbql-query checkins
{:filter [:starts-with !month.date "2018-10-01"]}))))))
diff --git a/test/metabase/query_processor_test.clj b/test/metabase/query_processor_test.clj
index 5bf41ef3348aa..3a017de3b00ea 100644
--- a/test/metabase/query_processor_test.clj
+++ b/test/metabase/query_processor_test.clj
@@ -69,7 +69,8 @@
[table-kw field-kw]
(merge
(col-defaults)
- (db/select-one [Field :id :table_id :semantic_type :base_type :name :display_name :fingerprint]
+ (db/select-one [Field :id :table_id :semantic_type :base_type :effective_type
+ :coercion_strategy :name :display_name :fingerprint]
:id (data/id table-kw field-kw))
{:field_ref [:field (data/id table-kw field-kw) nil]}
(when (#{:last_login :date} field-kw)
diff --git a/test/metabase/query_processor_test/alternative_date_test.clj b/test/metabase/query_processor_test/alternative_date_test.clj
index 2aa26bf187141..530ff583e51af 100644
--- a/test/metabase/query_processor_test/alternative_date_test.clj
+++ b/test/metabase/query_processor_test/alternative_date_test.clj
@@ -11,18 +11,19 @@
[metabase.util :as u]))
(deftest semantic-type->unix-timestamp-unit-test
- (testing "every descendant of `:type/UNIXTimestamp` has a unit associated with it"
- (doseq [semantic-type (descendants :type/UNIXTimestamp)]
+ (testing "every descendant of `:Coercion/UNIXTime->Temporal` has a unit associated with it"
+ (doseq [semantic-type (descendants :Coercion/UNIXTime->Temporal)]
(is (sql.qp/semantic-type->unix-timestamp-unit semantic-type))))
- (testing "throws if argument is not a descendant of `:type/UNIXTimestamp`"
+ (testing "throws if argument is not a descendant of `:Coercion/UNIXTime->Temporal`"
(is (thrown? AssertionError (sql.qp/semantic-type->unix-timestamp-unit :type/Integer)))))
(mt/defdataset toucan-microsecond-incidents
[["incidents" [{:field-name "severity"
:base-type :type/Integer}
- {:field-name "timestamp"
- :base-type :type/BigInteger
- :semantic-type :type/UNIXTimestampMicroseconds}]
+ {:field-name "timestamp"
+ :base-type :type/BigInteger
+ :effective-type :type/DateTime
+ :coercion-strategy :Coercion/UNIXMicroSeconds->DateTime}]
[[4 1433587200000000]
[0 1433965860000000]]]])
@@ -134,33 +135,40 @@
;;; :type/ISO8601DateTimeString tests
(mt/defdataset just-dates
- [["just_dates" [{:field-name "name"
- :base-type :type/Text}
- {:field-name "ts"
- :base-type :type/Text
- :semantic-type :type/ISO8601DateTimeString}
- {:field-name "d"
- :base-type :type/Text
- :semantic-type :type/ISO8601DateString}]
+ [["just_dates" [{:field-name "name"
+ :base-type :type/Text
+ :effective-type :type/Text}
+ {:field-name "ts"
+ :base-type :type/Text
+ :effective-type :type/DateTime
+ :coercion-strategy :Coercion/ISO8601->DateTime}
+ {:field-name "d"
+ :base-type :type/Text
+ :effective-type :type/Date
+ :coercion-strategy :Coercion/ISO8601->Date}]
[["foo" "2004-10-19 10:23:54" "2004-10-19"]
["bar" "2008-10-19 10:23:54" "2008-10-19"]
["baz" "2012-10-19 10:23:54" "2012-10-19"]]]])
(mt/defdataset string-times
[["times" [{:field-name "name"
- :base-type :type/Text}
- {:field-name "ts"
- :base-type :type/Text
- :semantic-type :type/ISO8601DateTimeString}
- {:field-name "d"
- :base-type :type/Text
- :semantic-type :type/ISO8601DateString}
- {:field-name "t"
- :base-type :type/Text
- :semantic-type :type/ISO8601TimeString}]
- [["foo" "2004-10-19 10:23:54" "2004-10-19" "10:23:54"]
- ["bar" "2008-10-19 10:23:54" "2008-10-19" "10:23:54"]
- ["baz" "2012-10-19 10:23:54" "2012-10-19" "10:23:54"]]]])
+ :effective-type :type/Text
+ :base-type :type/Text}
+ {:field-name "ts"
+ :base-type :type/Text
+ :effective-type :type/DateTime
+ :coercion-strategy :Coercion/ISO8601->DateTime}
+ {:field-name "d"
+ :base-type :type/Text
+ :effective-type :type/Date
+ :coercion-strategy :Coercion/ISO8601->Date}
+ {:field-name "t"
+ :base-type :type/Text
+ :effective-type :type/Time
+ :coercion-strategy :Coercion/ISO8601->Time}]
+ [["foo" "2004-10-19 10:23:54" "2004-10-19" "10:23:54"]
+ ["bar" "2008-10-19 10:23:54" "2008-10-19" "10:23:54"]
+ ["baz" "2012-10-19 10:23:54" "2012-10-19" "10:23:54"]]]])
(deftest iso-8601-text-fields
(testing "text fields with semantic_type :type/ISO8601DateTimeString"
diff --git a/test/metabase/query_processor_test/nested_queries_test.clj b/test/metabase/query_processor_test/nested_queries_test.clj
index 7d95a89783fac..fc3301f8e465b 100644
--- a/test/metabase/query_processor_test/nested_queries_test.clj
+++ b/test/metabase/query_processor_test/nested_queries_test.clj
@@ -71,7 +71,8 @@
(dissoc :description :parent_id :visibility_type))
(not has-source-metadata?)
- (dissoc :id :semantic_type :settings :fingerprint :table_id))
+ (dissoc :id :semantic_type :settings :fingerprint :table_id
+ :effective_type :coercion_strategy))
(qp.test/aggregate-col :count)]})
(deftest mbql-source-query-breakout-aggregation-test
@@ -378,6 +379,7 @@
(testing "make sure a query using a source query comes back with the correct columns metadata"
(is (= (map (partial qp.test/col :venues)
[:id :name :category_id :latitude :longitude :price])
+ ;; todo: i don't know why the results don't have the information
(mt/cols
(mt/with-temp Card [card (venues-mbql-card-def)]
(qp/process-query (query-with-source-card card)))))))
@@ -399,7 +401,8 @@
:unit :day)
;; because this field literal comes from a native query that does not include `:source-metadata` it won't have
;; the usual extra keys
- (dissoc :semantic_type :table_id :id :settings :fingerprint))
+ (dissoc :semantic_type :effective_type :coercion_strategy :table_id
+ :id :settings :fingerprint))
(qp.test/aggregate-col :count)]
(mt/cols
(mt/with-temp Card [card {:dataset_query {:database (mt/id)
@@ -809,7 +812,7 @@
(get-in result [:data :results_metadata :columns])
(u/key-by :name result)
(get result "EAN")
- (select-keys result [:name :display_name :base_type :semantic_type :id :field_ref])))]
+ (select-keys result [:name :display_name :base_type :id :field_ref])))]
(testing "Make sure metadata is correct for the 'EAN' column with"
(let [base-query (mt/mbql-query orders
{:source-table $$orders
@@ -827,7 +830,6 @@
{:name "EAN"
:display_name "Products → Ean"
:base_type :type/Text
- :semantic_type nil
:id %ean
:field_ref &Products.ean})
(ean-metadata (qp/process-query query))))))))))))))
diff --git a/test/metabase/sync/sync_metadata/fields/fetch_metadata_test.clj b/test/metabase/sync/sync_metadata/fields/fetch_metadata_test.clj
index 534cfe5a91243..536630195f9fe 100644
--- a/test/metabase/sync/sync_metadata/fields/fetch_metadata_test.clj
+++ b/test/metabase/sync/sync_metadata/fields/fetch_metadata_test.clj
@@ -18,45 +18,55 @@
(is (= #{{:name "id"
:database-type "SERIAL"
:base-type :type/Integer
+ :effective-type :type/Integer
:semantic-type :type/PK
:pk? true}
{:name "buyer"
:database-type "OBJECT"
:base-type :type/Dictionary
+ :effective-type :type/Dictionary
:pk? false
:nested-fields #{{:name "name"
:database-type "VARCHAR"
:base-type :type/Text
+ :effective-type :type/Text
:pk? false}
{:name "cc"
:database-type "VARCHAR"
:base-type :type/Text
+ :effective-type :type/Text
:pk? false}}}
{:name "ts"
:database-type "BIGINT"
:base-type :type/BigInteger
+ :effective-type :type/BigInteger
:semantic-type :type/UNIXTimestampMilliseconds
:pk? false}
{:name "toucan"
:database-type "OBJECT"
:base-type :type/Dictionary
+ :effective-type :type/Dictionary
:pk? false
:nested-fields #{{:name "name"
:database-type "VARCHAR"
:base-type :type/Text
+ :effective-type :type/Text
:pk? false}
{:name "details"
:database-type "OBJECT"
:base-type :type/Dictionary
+ :effective-type :type/Dictionary
:pk? false
:nested-fields #{{:name "weight"
:database-type "DECIMAL"
:base-type :type/Decimal
+ :effective-type :type/Decimal
:semantic-type :type/Category
:pk? false}
{:name "age"
:database-type "INT"
:base-type :type/Integer
+ :effective-type :type/Integer
:pk? false}}}}}}
(let [transactions-table-id (u/get-id (db/select-one-id Table :db_id (u/get-id db), :name "transactions"))
diff --git a/test/metabase/sync_test.clj b/test/metabase/sync_test.clj
index 3e53e2f58704c..8fb2b078b6d36 100644
--- a/test/metabase/sync_test.clj
+++ b/test/metabase/sync_test.clj
@@ -127,6 +127,7 @@
:display_name "ID"
:database_type "SERIAL"
:base_type :type/Integer
+ :effective_type :type/Integer
:semantic_type :type/PK
:database_position 0
:position 0}))
@@ -138,6 +139,7 @@
:display_name "Studio"
:database_type "VARCHAR"
:base_type :type/Text
+ :effective_type :type/Text
:fk_target_field_id true
:semantic_type :type/FK
:database_position 2
@@ -150,6 +152,7 @@
:display_name "Title"
:database_type "VARCHAR"
:base_type :type/Text
+ :effective_type :type/Text
:semantic_type :type/Title
:database_position 1
:position 1}))
@@ -161,6 +164,7 @@
:display_name "Name"
:database_type "VARCHAR"
:base_type :type/Text
+ :effective_type :type/Text
:semantic_type :type/Name
:database_position 1
:position 1}))
@@ -173,6 +177,7 @@
:display_name "Studio"
:database_type "VARCHAR"
:base_type :type/Text
+ :effective_type :type/Text
:semantic_type :type/PK
:database_position 0
:position 0}))
diff --git a/test/metabase/test/data/dataset_definitions/sad-toucan-incidents.edn b/test/metabase/test/data/dataset_definitions/sad-toucan-incidents.edn
index 3067ba8d90795..465c9ea31ad9b 100644
--- a/test/metabase/test/data/dataset_definitions/sad-toucan-incidents.edn
+++ b/test/metabase/test/data/dataset_definitions/sad-toucan-incidents.edn
@@ -2,7 +2,8 @@
:base-type :type/Integer}
{:field-name "timestamp"
:base-type :type/BigInteger
- :semantic-type :type/UNIXTimestampMilliseconds}]
+ :effective-type :type/DateTime
+ :coercion-strategy :Coercion/UNIXMilliSeconds->DateTime}]
[[4 1433587200000]
[0 1433965860000]
[5 1433864520000]
diff --git a/test/metabase/test/data/dataset_definitions/tupac-sightings.edn b/test/metabase/test/data/dataset_definitions/tupac-sightings.edn
index e2cbbcf5621c5..fd68481d20f81 100644
--- a/test/metabase/test/data/dataset_definitions/tupac-sightings.edn
+++ b/test/metabase/test/data/dataset_definitions/tupac-sightings.edn
@@ -183,9 +183,10 @@
{:field-name "category_id"
:base-type :type/Integer
:fk :categories}
- {:field-name "timestamp"
- :base-type :type/BigInteger
- :semantic-type :type/UNIXTimestampSeconds}]
+ {:field-name "timestamp"
+ :base-type :type/BigInteger
+ :effective-type :type/DateTime
+ :coercion-strategy :Coercion/UNIXSeconds->DateTime}]
[[23 14 927183600]
[65 13 978854400]
[60 10 886752000]
diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.clj
index 0564f46e8879f..01d716982eaa9 100644
--- a/test/metabase/test/data/impl.clj
+++ b/test/metabase/test/data/impl.clj
@@ -1,6 +1,7 @@
(ns metabase.test.data.impl
"Internal implementation of various helper functions in `metabase.test.data`."
- (:require [clojure.tools.logging :as log]
+ (:require [clojure.string :as str]
+ [clojure.tools.logging :as log]
[clojure.tools.reader.edn :as edn]
[metabase.config :as config]
[metabase.driver :as driver]
@@ -62,17 +63,15 @@
table-name
(u/pprint-to-str (dissoc table-definition :rows))
(u/pprint-to-str (db/select [Table :schema :name], :db_id (:id db))))))))]
- (doseq [{:keys [field-name visibility-type semantic-type], :as field-definition} (:field-definitions table-definition)]
+ (doseq [{:keys [field-name], :as field-definition} (:field-definitions table-definition)]
(let [field (delay (or (tx/metabase-instance field-definition @table)
- (throw (Exception. (format "Field '%s' not loaded from definition:\n"
+ (throw (Exception. (format "Field '%s' not loaded from definition:\n%s"
field-name
(u/pprint-to-str field-definition))))))]
- (when visibility-type
- (log/debug (format "SET VISIBILITY TYPE %s.%s -> %s" table-name field-name visibility-type))
- (db/update! Field (:id @field) :visibility_type (name visibility-type)))
- (when semantic-type
- (log/debug (format "SET SEMANTIC TYPE %s.%s -> %s" table-name field-name semantic-type))
- (db/update! Field (:id @field) :semantic_type (u/qualified-name semantic-type))))))))
+ (doseq [property [:visibility-type :semantic-type :effective-type :coercion-strategy]]
+ (when-let [v (get field-definition property)]
+ (log/debug (format "SET %s %s.%s -> %s" property table-name field-name v))
+ (db/update! Field (:id @field) (keyword (str/replace (name property) #"-" "_")) (u/qualified-name v)))))))))
(def ^:private create-database-timeout-ms
"Max amount of time to wait for driver text extensions to create a DB and load test data."
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index 2136f01bc8312..096286c85ca24 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -29,19 +29,21 @@
;;; | Dataset Definition Record Types & Protocol |
;;; +----------------------------------------------------------------------------------------------------------------+
-(p.types/defrecord+ FieldDefinition [field-name base-type semantic-type visibility-type fk field-comment])
+(p.types/defrecord+ FieldDefinition [field-name base-type effective-type coercion-strategy semantic-type visibility-type fk field-comment])
(p.types/defrecord+ TableDefinition [table-name field-definitions rows table-comment])
(p.types/defrecord+ DatabaseDefinition [database-name table-definitions])
(def ^:private FieldDefinitionSchema
- {:field-name su/NonBlankString
- :base-type (s/cond-pre {:native su/NonBlankString} su/FieldType)
- (s/optional-key :semantic-type) (s/maybe su/FieldType)
- (s/optional-key :visibility-type) (s/maybe (apply s/enum field/visibility-types))
- (s/optional-key :fk) (s/maybe su/KeywordOrString)
- (s/optional-key :field-comment) (s/maybe su/NonBlankString)})
+ {:field-name su/NonBlankString
+ :base-type (s/cond-pre {:native su/NonBlankString} su/FieldType)
+ (s/optional-key :semantic-type) (s/maybe su/FieldType)
+ (s/optional-key :effective-type) (s/maybe su/FieldType)
+ (s/optional-key :coercion-strategy) (s/maybe su/CoercionStrategy)
+ (s/optional-key :visibility-type) (s/maybe (apply s/enum field/visibility-types))
+ (s/optional-key :fk) (s/maybe su/KeywordOrString)
+ (s/optional-key :field-comment) (s/maybe su/NonBlankString)})
(def ^:private ValidFieldDefinition
(s/constrained FieldDefinitionSchema (partial instance? FieldDefinition)))
@@ -381,9 +383,10 @@
;; TODO - not sure everything below belongs in this namespace
(s/defn ^:private dataset-field-definition :- ValidFieldDefinition
- [field-definition-map :- DatasetFieldDefinition]
+ [{:keys [coercion-strategy base-type] :as field-definition-map} :- DatasetFieldDefinition]
"Parse a Field definition (from a `defdatset` form or EDN file) and return a FieldDefinition instance for
comsumption by various test-data-loading methods."
+ ;; if definition uses a coercion strategy they need to provide the effective-type
(map->FieldDefinition field-definition-map))
(s/defn ^:private dataset-table-definition :- ValidTableDefinition