adds table transforms

Project.toml

@@ -4,6 +4,7 @@ authors = ["lorenzoh <[email protected]>"]
 version = "0.2.3"
 
 [deps]
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 ColorBlendModes = "60508b50-96e1-4007-9d6c-f475c410f16b"
 CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"

src/DataAugmentation.jl

@@ -2,6 +2,7 @@ module DataAugmentation
 
 using ColorBlendModes
 using CoordinateTransformations
+using CategoricalArrays
 using Distributions: Sampleable, Uniform, Categorical
 using ImageDraw
 using Images
@@ -28,6 +29,7 @@ include("./sequence.jl")
 include("./items/arrayitem.jl")
 include("./projective/base.jl")
 include("./items/image.jl")
+include("./items/table.jl")
 include("./items/keypoints.jl")
 include("./items/mask.jl")
 include("./projective/compose.jl")
@@ -36,6 +38,7 @@ include("./projective/affine.jl")
 include("./projective/warp.jl")
 include("./oneof.jl")
 include("./preprocessing.jl")
+include("./rowtransforms.jl")
 include("./colortransforms.jl")
 include("testing.jl")
 include("./visualization.jl")
@@ -49,6 +52,7 @@ export Item,
     Sequence,
     Project,
     Image,
+    TabularItem,
     Keypoints,
     Polygon,
     ToEltype,
@@ -88,7 +92,8 @@ export Item,
     onehot,
     showitems,
     showgrid,
-    Bounds
+    Bounds,
+    getcategorypools
 
 
 end # module

src/items/table.jl

@@ -0,0 +1,4 @@
+struct TabularItem{T} <: Item
+	data::T
+	columns
+end

src/rowtransforms.jl

@@ -0,0 +1,50 @@
+struct NormalizeRow{T, S} <: DataAugmentation.Transform
+        normstats::T
+        normcols::S
+end
+NormalizeRow(normstats::T, normcols::S) where {T, S} = NormalizeRow{T, S}(normstats, normcols)
+
+struct Categorify{T, S} <: DataAugmentation.Transform
+        catdict::T
+        categorycols::S
+end
+Categorify(catdict::T, categorycols::S) where {T, S} = Categorify{T, S}(catdict, categorycols)
+
+struct FillMissing{T, S} <: DataAugmentation.Transform
+        fmvals::T
+        fmcols::S
+end
+FillMissing(fmvals::T, fmcols::S) where {T, S} = FillMissing{T, S}(fmvals, fmcols)
+
+function DataAugmentation.apply(tfm::FillMissing, item::TabularItem; randstate=nothing)
+    x = [val for val in item.data]
+    for col in tfm.fmcols
+            idx = findfirst(col .== item.columns)
+            if ismissing(x[idx])
+                x[idx] = tfm.fmvals[col]
+            end
+    end
+    x = (; zip(item.columns, x)...)
+    TabularItem(x, item.columns)
+end
+
+function DataAugmentation.apply(tfm::NormalizeRow, item::TabularItem; randstate=nothing)
+    x = [val for val in item.data]
+    for col in tfm.normcols
+            idx = findfirst(col .== item.columns)
+            colmean, colstd = tfm.normstats[col]
+            x[idx] = (x[idx] - colmean)/colstd
+    end
+    x = (; zip(item.columns, x)...)
+    TabularItem(x, item.columns)
+end
+
+function DataAugmentation.apply(tfm::Categorify, item::TabularItem; randstate=nothing)
+    x = [val for val in item.data]
+    for col in tfm.categorycols
+        idx = findfirst(col .== item.columns)
+        x[idx] = ismissing(x[idx]) ? 1 : findfirst(skipmissing(x[idx] .== tfm.catdict[col])) + 1
+    end
+    x = (; zip(item.columns, x)...)
+    TabularItem(x, item.columns)
+end