adds table transforms

src/DataAugmentation.jl

@@ -28,6 +28,7 @@ include("./sequence.jl")
 include("./items/arrayitem.jl")
 include("./projective/base.jl")
 include("./items/image.jl")
+include("./items/table.jl")
 include("./items/keypoints.jl")
 include("./items/mask.jl")
 include("./projective/compose.jl")
@@ -36,6 +37,7 @@ include("./projective/affine.jl")
 include("./projective/warp.jl")
 include("./oneof.jl")
 include("./preprocessing.jl")
+include("./rowtransforms.jl")
 include("./colortransforms.jl")
 include("testing.jl")
 include("./visualization.jl")
@@ -49,6 +51,7 @@ export Item,
     Sequence,
     Project,
     Image,
+    TabularItem,
     Keypoints,
     Polygon,
     ToEltype,
@@ -88,7 +91,8 @@ export Item,
     onehot,
     showitems,
     showgrid,
-    Bounds
+    Bounds,
+    getcategorypools
 
 
 end # module

src/items/table.jl

@@ -0,0 +1,4 @@
+struct TabularItem{T} <: Item
+    data::T
+    columns
+end

src/rowtransforms.jl

@@ -0,0 +1,56 @@
+struct NormalizeRow{T, S} <: Transform
+    dict::T
+    cols::S
+end
+
+struct FillMissing{T, S} <: Transform
+    dict::T
+    cols::S
+end
+
+struct Categorify{T, S} <: Transform
+    dict::T
+    cols::S
+    function Categorify{T, S}(dict::T, cols::S) where {T, S}
+        for (col, vals) in dict
+            if any(ismissing, vals)
+                dict[col] = filter(!ismissing, vals)
+                @warn "There is a missing value present for category '$col' which will be removed from Categorify dict"
+            end
+        end
+        new{T, S}(dict, cols)
+    end
+end
+
+Categorify(dict::T, cols::S) where {T, S} = Categorify{T, S}(dict, cols)
+
+function apply(tfm::NormalizeRow, item::TabularItem; randstate=nothing)
+    x = NamedTuple(Iterators.map(item.columns, item.data) do col, val
+        if col in tfm.cols
+            colmean, colstd = tfm.dict[col]
+            val = (val - colmean)/colstd
+        end
+        (col, val)
+    end)
+    TabularItem(x, item.columns)
+end
+
+function apply(tfm::FillMissing, item::TabularItem; randstate=nothing)
+    x = NamedTuple(Iterators.map(item.columns, item.data) do col, val
+        if col in tfm.cols && ismissing(val)
+            val = tfm.dict[col]
+        end
+        (col, val)
+    end)
+    TabularItem(x, item.columns)
+end
+
+function apply(tfm::Categorify, item::TabularItem; randstate=nothing)
+    x = NamedTuple(Iterators.map(item.columns, item.data) do col, val
+        if col in tfm.cols
+            val = ismissing(val) ? 1 : findfirst(val .== tfm.dict[col]) + 1
+        end
+        (col, val)
+    end)
+    TabularItem(x, item.columns)
+end