From 2c83640f2c84cd8fbdfd78eda03912cd47493f96 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Wed, 28 Feb 2024 22:27:30 +0900 Subject: [PATCH 01/11] adjust docstrings --- src/colortransforms.jl | 2 -- src/projective/affine.jl | 28 ++++++++++++++++++++++------ src/projective/crop.jl | 9 +++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/colortransforms.jl b/src/colortransforms.jl index 52972261..7ecc6337 100644 --- a/src/colortransforms.jl +++ b/src/colortransforms.jl @@ -13,7 +13,6 @@ Pixels are clamped to [0,1] unless `clamp=false` is passed. ## Example -{cell=AdjustBrightness} ```julia using DataAugmentation, TestImages @@ -77,7 +76,6 @@ Pixels are clamped to [0,1] unless `clamp=false` is passed. ## Example -{cell=AdjustBrightness} ```julia using DataAugmentation, TestImages diff --git a/src/projective/affine.jl b/src/projective/affine.jl index 6d92e687..99a17e3a 100644 --- a/src/projective/affine.jl +++ b/src/projective/affine.jl @@ -9,6 +9,11 @@ function scaleprojection(ratios::NTuple{N}, T = Float32) where N end +""" + ScaleRatio(minlengths) <: ProjectiveTransform + +Scales the aspect ratio +""" struct ScaleRatio{N} <: ProjectiveTransform ratios::NTuple{N} end @@ -26,12 +31,11 @@ original aspect ratio. ## Examples -{cell=ScaleKeepAspect} -```julia +```@example using DataAugmentation, TestImages image = testimage("lighthouse") tfm = ScaleKeepAspect((200, 200)) -apply(tfm, Image(image)) |> showitems +apply(tfm, Image(image)) ``` """ struct ScaleKeepAspect{N} <: ProjectiveTransform @@ -186,8 +190,13 @@ function centered(P, bounds::Bounds{2}) return recenter(P, midpoint) end - +""" +Reflect(180) +""" FlipX() = Reflect(180) +""" +Reflect(90) +""" FlipY() = Reflect(90) function reflectionmatrix(r) @@ -239,8 +248,15 @@ compose(cropped::ComposedProjectiveTransform, pin::PinOrigin) = Sequence(cropped compose(cropped::ProjectiveTransform, pin::PinOrigin) = Sequence(cropped, pin) # ## Resize crops - +""" +ScaleKeepAspect(sz) |> RandomCrop(sz) |> PinOrigin() +""" RandomResizeCrop(sz) = ScaleKeepAspect(sz) |> RandomCrop(sz) |> PinOrigin() +""" +ScaleKeepAspect(sz) |> CenterCrop(sz) |> PinOrigin() +""" CenterResizeCrop(sz) = ScaleKeepAspect(sz) |> CenterCrop(sz) |> PinOrigin() - +""" +ScaleKeepAspect(sz) |> PadDivisible(by) |> PinOrigin() +""" ResizePadDivisible(sz, by) = ScaleKeepAspect(sz) |> PadDivisible(by) |> PinOrigin() diff --git a/src/projective/crop.jl b/src/projective/crop.jl index 3ebb23b2..bcba2ecf 100644 --- a/src/projective/crop.jl +++ b/src/projective/crop.jl @@ -13,6 +13,9 @@ struct Crop{N, F<:CropFrom} <: AbstractCrop from::F end +""" +Crop(sz, FromOrigin()) +""" Crop(sz) = Crop(sz, FromOrigin()) @@ -25,7 +28,13 @@ function apply(crop::Crop, item::Item; randstate = getrandstate(crop)) end +""" +Crop(sz, FromCenter()) +""" CenterCrop(sz) = Crop(sz, FromCenter()) +""" +Crop(sz, FromRandom()) +""" RandomCrop(sz) = Crop(sz, FromRandom()) # The random state of a [`Crop`](#) consists of offsets from the origin. From 627f396d40fb2519275839ba093d5e8a1cbd19b9 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Wed, 28 Feb 2024 22:27:46 +0900 Subject: [PATCH 02/11] add documenter gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2cadb826..b01f11fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /docs/build/ +/docs-documenter/build/ Manifest.toml theme gh-pages From 0a4df135529591cfad90b37539e12f4bd85e6a85 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Thu, 29 Feb 2024 00:47:29 +0900 Subject: [PATCH 03/11] move to documenter --- src/items/image.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/items/image.jl b/src/items/image.jl index 7358c0ae..219b27bb 100644 --- a/src/items/image.jl +++ b/src/items/image.jl @@ -10,7 +10,6 @@ Item representing an N-dimensional image with element type T. ## Examples -{cell=image} ```julia using DataAugmentation, Images @@ -21,7 +20,6 @@ showitems(item) If `T` is not a color, the image will be interpreted as grayscale: -{cell=image} ```julia imagedata = rand(Float32, 100, 100) item = Image(imagedata) From 3b66c8c4ec8ede91f89a436abd7a57c368419f18 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Thu, 29 Feb 2024 00:47:44 +0900 Subject: [PATCH 04/11] add badges --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8eacba29..e56106de 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # DataAugmentation.jl +![CI](https://github.com/FluxML/DataAugmentation.jl/actions/workflows/CI/badge.svg) +![compat](https://github.com/FluxML/DataAugmentation.jl/actions/workflows/CompatHelper/badge.svg) + + [Documentation | Latest](https://fluxml.github.io/DataAugmentation.jl/dev/documents/README.md) Efficient, composable data augmentation for machine and deep learning with support for n-dimensional images, keypoints and categorical masks. From a648b97f5238d4a097fe82e01d670e580f47e53c Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Thu, 29 Feb 2024 00:47:54 +0900 Subject: [PATCH 05/11] add Documenter.jl docs --- docs-documenter/Project.toml | 10 ++ docs-documenter/make.jl | 24 +++ docs-documenter/src/buffering.md | 23 +++ docs-documenter/src/index.md | 36 +++++ docs-documenter/src/iteminterface.md | 27 ++++ docs-documenter/src/preprocessing.md | 9 ++ docs-documenter/src/projective/data.md | 9 ++ docs-documenter/src/projective/gallery.md | 171 ++++++++++++++++++++++ docs-documenter/src/projective/intro.md | 41 ++++++ docs-documenter/src/quickstart.md | 31 ++++ docs-documenter/src/ref.md | 15 ++ docs-documenter/src/tfminterface.md | 73 +++++++++ docs-documenter/src/transformations.md | 73 +++++++++ 13 files changed, 542 insertions(+) create mode 100644 docs-documenter/Project.toml create mode 100644 docs-documenter/make.jl create mode 100644 docs-documenter/src/buffering.md create mode 100644 docs-documenter/src/index.md create mode 100644 docs-documenter/src/iteminterface.md create mode 100644 docs-documenter/src/preprocessing.md create mode 100644 docs-documenter/src/projective/data.md create mode 100644 docs-documenter/src/projective/gallery.md create mode 100644 docs-documenter/src/projective/intro.md create mode 100644 docs-documenter/src/quickstart.md create mode 100644 docs-documenter/src/ref.md create mode 100644 docs-documenter/src/tfminterface.md create mode 100644 docs-documenter/src/transformations.md diff --git a/docs-documenter/Project.toml b/docs-documenter/Project.toml new file mode 100644 index 00000000..e27f93b5 --- /dev/null +++ b/docs-documenter/Project.toml @@ -0,0 +1,10 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31" +Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" +MosaicViews = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +TestImages = "5e47fb64-e119-507b-a336-dd2b206d9990" + +[compat] +Documenter = "1.2" diff --git a/docs-documenter/make.jl b/docs-documenter/make.jl new file mode 100644 index 00000000..b16ce667 --- /dev/null +++ b/docs-documenter/make.jl @@ -0,0 +1,24 @@ +using Documenter, DataAugmentation + +makedocs(sitename="DataAugmentation.jl", + pages = [ + "index.md", + "Quickstart" => "quickstart.md", + "Transformations" => "transformations.md", + "Build your transformations" =>[ + "Item Interface" => "iteminterface.md", + "Transform Interface" => "tfminterface.md", + "Projective Interface" => [ + "Intro" => "projective/intro.md", + "Data" => "projective/data.md", + "Gallery" => "projective/gallery.md", + ], + ], + "Misc" =>[ + "Buffering" => "buffering.md", + "Preprocessing" => "preprocessing.md", + "References" => "ref.md" + ], + ], + format = Documenter.HTML(prettyurls = false) +) diff --git a/docs-documenter/src/buffering.md b/docs-documenter/src/buffering.md new file mode 100644 index 00000000..3ab85a99 --- /dev/null +++ b/docs-documenter/src/buffering.md @@ -0,0 +1,23 @@ +# Buffering + +As mentioned in the section on [transformations](./tfminterface.md), you can implement [`apply!`](#) as an inplace version of [`apply`](#) to support buffered transformations. Usually, the result of a regular `apply` can be used as a buffer. You may write + +```julia +buffer = apply(tfm, item) +apply!(buffer, tfm, item) +``` + +However, for some transformations, a different buffer is needed. [`Sequence`](#), for example, needs to reuse all intermediate results. That is why the buffer creation can be customized: + +- [`makebuffer`](#)`(tfm, item)` creates a buffer `buf` that can be used in an `apply!` call: `apply!(buf, tfm, item)`. + +--- + +Managing the buffers manually quickly becomes tedious. For convenience, this library implements [`Buffered`](#), a transformation wrapper that will use a buffer internally. `btfm = Buffered(tfm)` will create a buffer the first time it is `apply`ed and then use it by internally calling `apply!`. + +```julia +buffered = Buffered(tfm) +buffer = apply(tfm, item) # uses apply! internally +``` + +Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`BufferedThreadsafe`](#), a version of `Buffered` that keeps a separate buffer for every thread. \ No newline at end of file diff --git a/docs-documenter/src/index.md b/docs-documenter/src/index.md new file mode 100644 index 00000000..d72da115 --- /dev/null +++ b/docs-documenter/src/index.md @@ -0,0 +1,36 @@ +# DataAugmentation.jl + +This library provides data transformations for machine and deep learning. At the moment, it focuses on spatial data (think images, keypoint data and masks), but that is owed only to my current work. The extensible abstractions should fit other domains as well. + +For the most part, the transformations themselves are not very complex. The challenge this library tackles is to reconcile an easy-to-use, composable interface with performant execution. + +The key abstractions are `Transform`s, the transformation to apply, and `Item`s which contain the data to be transformed. [`apply`](#)`(tfm, item)`, as the name gives away, applies a transformation to an item. For example, given an [`Image`](#) item, we can resize it with the [`CenterResizeCrop`](#) transformation. + +```julia +item = Image(image) +tfm = CenterResizeCrop((128, 128)) +apply(tfm, item) +``` + +## Requirements + +The above example is simple, but there are more requirements of data augmentation pipelines that this library adresses. They serve as a motivation to the interface I've arrived at for defining transformations. + +### Stochasticity + +A transformation is stochastic (as opposed to deterministic) if it produces different outputs based on some random state. +This randomness can become a problem when applying an transformation to an aligned pair of input and target. If we have an image and a corresponding segmentation mask, using different scaling factors results in misalignment of the two; the segmentation no longer matches up with the image pixels. + +To handle this, the random state is explicitly passed to the transformations, rendering them deterministic. A generator for the random state can be defined with [`getrandstate`](#)`(tfm)` and passed to `apply` with the `randstate` keyword argument. + +### Composition + +Most data augmentation pipelines are made up of multiple steps: augmenting an image can mean resizing, randomly rotating, cropping and then normalizing the values. So applying transformations one after another – sequencing – is one way to compose transformations. But some operations, like affine transformations, can also be *fused*, resulting in a single transformation that is more performant and produces more accurate results. + +### Buffering + +Since data augmentation pipelines often run on large amounts of data, performance can often be improved by using prealloacted output buffers for the transformations. This results in fewer memory allocations and less garbage collection which both take time. + +--- + +Let's next see how these requirements are reflected in the [item](./iteminterface.md) and [transformation](./tfminterface.md) interfaces. diff --git a/docs-documenter/src/iteminterface.md b/docs-documenter/src/iteminterface.md new file mode 100644 index 00000000..c8fc7248 --- /dev/null +++ b/docs-documenter/src/iteminterface.md @@ -0,0 +1,27 @@ +# Item interface + +As described previously, items are simply containers for data: an [`Image`](#) represents an image, and [`Keypoints`](#) some keypoints. + +### Why do I need to wrap my data in an item? + +For one, the item `struct`s may contain metadata that is useful for some transformations. More importantly, though, by wrapping data in an item type, the *meaning* of the data is separated from the *representation*, that is, the concrete type. + +An `Array{Integer, 2}` could represent an image, but also a multi-class segmentation mask. Is `Array{Float32, 3}` a 3-dimensional image or a 2-dimensional image with the color channels expanded? + +Separating the representation from the data's meaning resolves those ambiguities. + +## Creating items + +To create a new item, you can simply subtype [`Item`](#): + +```julia +struct MyItem <: Item + data +end +``` + +The only function that is expected to be implemented is [`itemdata`](#), which simply returns the wrapped data. If, as above, you simply call the field holding the data `data`, you do not need to implement it. The same goes for the [`setdata`](#) helper. + +For some items, it also makes sense to implement the following: + +- [`showitem!`](#)`(img, item::I)` creates a visual representation of an item on top of `img`. \ No newline at end of file diff --git a/docs-documenter/src/preprocessing.md b/docs-documenter/src/preprocessing.md new file mode 100644 index 00000000..9ae45a2e --- /dev/null +++ b/docs-documenter/src/preprocessing.md @@ -0,0 +1,9 @@ +# Preprocessing + +This library also implements some general transformations useful for getting data ready to be put into a model. + +- [`ToEltype`](#)`(T)` converts the element type of any [`AbstractArrayItem`](#) to `T`. +- [`ImageToTensor`](#) converts an image to an `ArrayItem` with another dimension for the color channels +- [`Normalize`](#) normalizes image tensors +- [`OneHot`](#) to one-hot encode multi-class masks ([`MaskMulti`](#)s) + diff --git a/docs-documenter/src/projective/data.md b/docs-documenter/src/projective/data.md new file mode 100644 index 00000000..86cce81c --- /dev/null +++ b/docs-documenter/src/projective/data.md @@ -0,0 +1,9 @@ +# Spatial data + +Before introducing various projective transformations, we have a look at the data that can be projected. There are three kinds of data currently supported: images, keypoints and segmentation masks. Both 2D and 3D data is supported (and technically, higher dimensions, but I've yet to find a dataset with 4 spatial dimensions). + +- [`Image`](@ref)`{N, T}` represents an `N`-dimensional image. `T` refers to the element type of the array that `Image` wraps, usually a color. When projecting images, proper interpolation methods are used to reduce artifacts like aliasing. See `src/items/image.jl` + +- [`MaskBinary`](@ref)`{N}` and [`MaskMulti`](@ref)`{N, T}` likewise represents `N`-dimensional segmentation masks. Unlike images, nearest-neighbor interpolation is used for projecting masks. See `src/items/mask.jl` + +- Lastly, [`Keypoints`](@ref)`{N}` represent keypoint data. The data should be an array of `SVector{N}`. Since there are many interpretations of keypoint data, there are also wrapper items for convenience: [`BoundingBox`](@ref) and [`Polygon`](@ref).See `src/items/keypoints.jl` diff --git a/docs-documenter/src/projective/gallery.md b/docs-documenter/src/projective/gallery.md new file mode 100644 index 00000000..c4139e6e --- /dev/null +++ b/docs-documenter/src/projective/gallery.md @@ -0,0 +1,171 @@ +# Gallery + +Let's visualize what these projective transformations look like. + +You can apply them to [`Image`](@ref)s and +the keypoint-based items [`Keypoints`](@ref), [`Polygon`](@ref), and [`BoundingBox`](@ref). + +Let's take this picture of a light house: + +```@setup deps +using DataAugmentation +using MosaicViews +using Images +using TestImages +using StaticArrays +``` + +```@example +using DataAugmentation +using MosaicViews +using Images +using TestImages +using StaticArrays + +imagedata = testimage("lighthouse") +imagedata = imresize(imagedata, ratio = 196 / size(imagedata, 1)) +``` + +```@example deps +imagedata = testimage("lighthouse") +imagedata +``` + +To apply a transformation `tfm` to it, wrap it in +`Image`, apply the transformation and unwrap it using [`itemdata`](@ref): + +```@example deps +tfm = CenterCrop((196, 196)) +image = Image(imagedata) +apply(tfm, image) |> itemdata +``` + +Now let's say we want to train a light house detector and have a bounding box for the light house. We can use the [`BoundingBox`](@ref) item to represent it. It takes the two corners of the bounding rectangle as the first argument. As the second argument we have to pass the size of the corresponding image. + +```@example deps +points = SVector{2, Float32}[SVector(23., 120.), SVector(120., 150.)] +bbox = BoundingBox(points, size(imagedata)) +``` + +[`showitems`](@ref) visualizes the two items: +```@example deps +showitems((image, bbox)) +``` +If we apply transformations like translation and cropping to the image, then the same transformations have to be applied to the bounding box. Otherwise, the bounding box will no longer match up with the light house. + +Another problem can occur with stochastic transformations like [`RandomResizeCrop`](@ref) If we apply it separately to the image and the bounding box, they will be cropped from slightly different locations: + +```@example deps +tfm = RandomResizeCrop((128, 128)) +showitems(( + apply(tfm, image), + apply(tfm, bbox) +)) +``` +Instead, pass a tuple of the items to a single `apply` call so the same random state will be used for both image and bounding box: + +```@example deps +apply(tfm, (image, bbox)) |> showitems +``` + +!!! info "3D Projective dimensions" + + We'll use a 2-dimensional [`Image`](@ref) and [`BoundingBox`](@ref) here, but you can apply most projective transformations to any spatial item (including [`Keypoints`](@ref), [`MaskBinary`](@ref) and [`MaskMulti`](@ref)) in 3 dimensions. + + Of course, you have to create a 3-dimensional transformation, i.e. `CenterCrop((128, 128, 128))` instead of `CenterCrop((128, 128))`. + +## Gallery +```julia +function showtransform(tfm, item, n = 8; ncol = 4) + return mosaicview( + [showitems(apply(tfm, item)) for _ in 1:n], + fillvalue = RGBA(1, 1, 1, 0), + npad = 8, + rowmajor = true, + ncol = ncol) +end + + +function showtransforms(tfms, item; ncol = length(tfms)) + return mosaicview( + [parent(showitems(apply(tfm, item))) for tfm in tfms], + fillvalue = RGBA(1, 1, 1, 0), + npad = 8, + rowmajor = true, + ncol = ncol) +end + +nothing # hide +``` + +### [`RandomResizeCrop`](@ref)`(sz)` + +Resizes the sides so that one of them is no longer than `sz` and crops a region of size `sz` *from a random location*. + +```julia +tfm = RandomResizeCrop((128, 128)) +``` + +```julia +o = showtransform(tfm, (image, bbox), 6, ncol=6) +``` + +### [`CenterResizeCrop`](@ref) + +Resizes the sides so that one of them is no longer than `sz` and crops a region of size `sz` *from the center*. + +```julia +tfm = CenterResizeCrop((128, 128)) +``` + +```julia +o = showtransform(tfm, (image, bbox), 1) +``` + +### [`Crop`](@ref)`(sz[, from])` + +Crops a region of size `sz` from the image, *without resizing* the image first. + +```julia +using DataAugmentation: FromOrigin, FromCenter, FromRandom +tfms = [ + Crop((128, 128), FromOrigin()), + Crop((128, 128), FromCenter()), + Crop((128, 128), FromRandom()), + Crop((128, 128), FromRandom()), + Crop((128, 128), FromRandom()), + Crop((128, 128), FromRandom()), +] +``` + +```julia +o = showtransforms(tfms, (image, bbox)) +``` + +### [`FlipX`](@ref), [`FlipY`](@ref), [`Reflect`](@ref) + +Flip the data on the horizontally and vertically, respectively. More generally, reflect around an angle from the x-axis. + +```julia +tfms = [ + FlipX(), + FlipY(), + Reflect(30), +] +``` + +```julia +o = showtransforms(tfms, (image, bbox)) +``` + +### [`Rotate`](@ref) + +Rotate counter-clockwise by an angle. + +```julia +tfm = Rotate(20) |> CenterCrop((256, 256)) +``` + +```julia +o = showtransform(tfm, (image, bbox), 1) +``` diff --git a/docs-documenter/src/projective/intro.md b/docs-documenter/src/projective/intro.md new file mode 100644 index 00000000..01d62563 --- /dev/null +++ b/docs-documenter/src/projective/intro.md @@ -0,0 +1,41 @@ +# Projective transformations +## An example pipeline + + +We can break down most augmentation used in practive into a single (possibly stochastic) projection and a crop. + +As an example, consider an image augmentation pipeline: A random horizontal flip, followed by a random resized crop. The latter resizes and crops (irregularly sized) images to a common size without distorting the aspect ratio. + +```julia +Maybe(FlipX()) |> RandomResizeCrop((h, w)) +``` + +Let's pull apart the steps involved. + +1. Half of the time, flip the image horizontally. + +2. Scale the image down without distortion so that the shorter side length is 128. With an input of size `(512, 256)` this result in scaling both dimensions by `1/2`, resulting in an image with side lengths `(256, 128)`. + +3. Crop a random `(128, 128)` portion from that image. There is only "wiggle room" on the y-axis (which, by convention, is the first). + +All of these steps can be efficiently computed in one step with two tricks: + +- Some projections like reflection, translation, scaling and rotation can be composed into a single projection matrix. This means in the above example we only need to apply one projection which represents both the flipping (a reflection) and the scaling. Especially in pipelines with many augmentation steps this avoids a lot of unnecessary computation. + +- In cases, where the result of the projection is cropped, we can save additional computing by only evaluating the parts that we want to keep. + +## Cropping + +By default, the bounds of a projected item will be chosen so they still encase all the data. So after applying a `Scale((2, 2))` to an `Image`, its bounds will also be scaled by 2. Sometimes, however, we want to crop a part of the projected output, for example so a number of images can later be batched into a single array. While the crop usually has a fixed size, the region to crop still needs to be chosen. For validation data (which should be transformed deterministically), a center crop is usually used. For training data, on the other hand, a random region is selected to add additional augmentation. + +## Projective transformations interface + +The abstract type [`ProjectiveTransform`](#) represents a projective transformation. +A `ProjectiveTransform` needs to implement [`getprojection`](#)`(tfm, bounds; randstate)` that should return a `Transformation` from [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl). + +To add support for projective transformations to an item `I`, you need to implement + +- `getbounds(item::I)` returns the spatial bounds of the item; and +- `project(P, item::I, indices)` applies the projective transformation `P` and crops to `indices` + +To support `apply!`-ing projective transformations, `project!(bufitem, P, item)` can also be implemented. diff --git a/docs-documenter/src/quickstart.md b/docs-documenter/src/quickstart.md new file mode 100644 index 00000000..05061eb6 --- /dev/null +++ b/docs-documenter/src/quickstart.md @@ -0,0 +1,31 @@ +# Quickstart + +```@setup deps +using TestImages +using ImageShow +using Images +using DataAugmentation +``` + +Import the library: +``` +using DataAugmentation # Image, CenterResizeCrop, apply, showitem +using TestImages: testimage +``` +Load your data: +```@example deps +image = testimage("lighthouse") +``` +Create an item that contains the data you want to augment: +```@example deps +item = Image(image) +``` +Create a transform: +```@example deps +tfm = CenterResizeCrop((128, 128)) +``` +Apply the transformation and unwrap the data: +```@example deps +titem = apply(tfm, item) +timage = itemdata(titem) +``` diff --git a/docs-documenter/src/ref.md b/docs-documenter/src/ref.md new file mode 100644 index 00000000..f9c4a217 --- /dev/null +++ b/docs-documenter/src/ref.md @@ -0,0 +1,15 @@ +# Reference +```@docs +BoundingBox +Image +Keypoints +MaskBinary +MaskMulti +Polygon +RandomResizeCrop +CenterResizeCrop +Crop +Rotate +itemdata +showitems +``` diff --git a/docs-documenter/src/tfminterface.md b/docs-documenter/src/tfminterface.md new file mode 100644 index 00000000..239c6bc1 --- /dev/null +++ b/docs-documenter/src/tfminterface.md @@ -0,0 +1,73 @@ + +# Transformation interface + +{style="opacity:60%;"} +*`src/base.jl`* + +The transformation interface is the centrepiece of this library. Beside straightforward transform application it also enables stochasticity, composition and buffering. + +A transformation is a type that subtypes [`Transform`](#). The only *required* function to implement for your transformation type `T` is + + +- [`apply`](#)`(tfm::T, item::I; randstate)` + + Applies the transformation `tfm` to item `item`. Implemented methods can of course dispatch on the type of `item`. `randstate` encapsulates the random state needed for stochastic transformations. The `apply` method implementation itself should be deterministic. + + You may dispatch on a specific item type `I` or use the abstract `Item` if one implementation works for all item types. + +You may additionally also implement: + +- [`getrandstate`](#)`(tfm)` for *stochastic* transformations + + Generates random state to be used inside `apply`. Calling `apply(tfm, item)` is equivalent to + `apply(tfm, item; randstate = getrandstate(tfm))`. It defaults to `nothing`, so we need not implement it for deterministic transformations. + +- [`apply!`](#)`(bufitem, tfm::T, item; randstate)` to support *buffering* + + Buffered version of `apply` that mutates `bufitem`. If not implemented, + falls back to regular `apply`. + +- [`compose`](#)`(tfm1, tfm2)` for custom *composition* with other transformations + + Composes transformations. By default, returns a [`Sequence`](#) transformation that applies the transformations one after the other. + + + +### Example + +The implementation of the [`MapElem`](#) transformation illustrates this interface well. It transforms any item with array data by mapping a function over the array's elements, just like `Base.map`. + +```julia +struct MapElem <: Transform + f +end +``` + +The `apply` implementation dispatches on [`AbstractArrayItem`](#), an abstract item type for items that wrap arrays. Note that the `randstate` keyword argument needs to be given even for implementations of deterministic transformations. We also make use of the [`setdata`](#) helper to update the item data. + +```julia +function apply(tfm::MapElem, item::AbstractArrayItem; randstate = nothing) + a = itemdata(item) + a_ = map(tfm.f, a) + return setdata(item, a_) +end +``` + +The buffered version applies the function inplace using `Base.map!`: + +```julia +function apply!( + bufitem::I, + tfm::MapElem, + item::I; + randstate = nothing) where I <: AbstractArrayItem + map!(tfm.f, itemdata(bufitem), itemdata(item)) + return bufitem +end +``` + +Finally, a `MapElem` can also be composed nicely with other `MapElem`s. Instead of applying them sequentially, the functions are fused and applied once. + +```julia +compose(tfm1::MapElem, tfm2::MapElem2) = MapElem(tfm2.f ∘ tfm1.f) +``` diff --git a/docs-documenter/src/transformations.md b/docs-documenter/src/transformations.md new file mode 100644 index 00000000..be60330d --- /dev/null +++ b/docs-documenter/src/transformations.md @@ -0,0 +1,73 @@ +```@setup tsm +using DataAugmentation +``` +# Usage + +Using transformations is easy. Simply `compose` them: + +```@example tsm +tfm = Rotate(10) |> ScaleRatio((0.7,0.1,1.2)) |> FlipX() |> Crop((128, 128)) +``` + +# Projective transformations +DataAugmentation.jl has great support for transforming spatial data like images and keypoints. Most of these transformations are projective transformations. For our purposes, a projection means a mapping between two coordinate spaces. In computer vision, these are frequently used for preprocessing and augmenting image data: images are randomly scaled, maybe flipped horizontally and finally cropped to the same size. + +This library generalizes projective transformations for different kinds of image and keypoint data in an N-dimensional Euclidean space. It also uses composition for performance improvements like fusing affine transformations. + +Unlike mathematical objects, the spatial data we want to transform has spatial bounds. For an image, these bounds are akin to the array size. But keypoint data aligned with an image has the same bounds even if they are not explicitly encoded in the representation of the data. These spatial bounds can be used to dynamically create useful transformations. For example, a rotation around the center or a horizontal flip of keypoint annotations can be calculated from the bounds. + +Often, we also want to crop an area from the projected results. By evaluating only the parts of a projection that fall inside the cropped area, a lot of unnecessary computation can be avoided. + +Projective transformations include: +1. [Affine transformations](@ref) +2. [Crops](@ref) +## Affine transformations + +Affine transformations are a subgroup of projective transformations that can be composed very efficiently: composing two affine transformations results in another affine transformation. Affine transformations can represent translation, scaling, reflection and rotation. Available `Transform`s are: + +```@docs +ScaleRatio +ScaleKeepAspect +FlipX +FlipY +Reflect +WarpAffine +``` + +## Crops + +To get a cropped result, simply `compose` any `ProjectiveTransform` with + +```@docs +CenterCrop +RandomCrop +``` + +# Color transformations + +DataAugmentation.jl currently supports the following color transformations for augmentation: + +```@docs +AdjustContrast +AdjustBrightness +``` + +# Stochastic transformations +When augmenting data, it is often useful to apply a transformation only with some probability or choose from a set of transformations. Unlike in other data augmentation libraries like *albumentations*, in DataAugmentation.jl you can use wrapper transformations for this functionality. + +- [`Maybe`](@ref)`(tfm, p = 0.5)` applies a transformation with probability `p`; and +- [`OneOf`](@ref)`([tfm1, tfm2])` randomly selects a transformation to apply. +```@docs +Maybe +OneOf +``` + +Let's say we have an image classification dataset. For most datasets, horizontally flipping the image does not change the label: a flipped image of a cat still shows a cat. So let's flip every image horizontally half of the time to improve the generalization of the model we might be training. + +```@example +using DataAugmentation, TestImages +item = Image(testimage("lighthouse")) +tfm = Maybe(FlipX()) +titems = [apply(tfm, item) for _ in 1:8] +showgrid(titems; ncol = 4, npad = 16) +``` From 418ba3f14f30d4ee83e2031d77099d2eed20bbb9 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Thu, 29 Feb 2024 00:48:24 +0900 Subject: [PATCH 06/11] remove local build --- docs-documenter/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-documenter/make.jl b/docs-documenter/make.jl index b16ce667..0efbe787 100644 --- a/docs-documenter/make.jl +++ b/docs-documenter/make.jl @@ -20,5 +20,5 @@ makedocs(sitename="DataAugmentation.jl", "References" => "ref.md" ], ], - format = Documenter.HTML(prettyurls = false) + # format = Documenter.HTML(prettyurls = false) ) From d6270702dbfdd300f747b3c7e2bcba620ecd22e9 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Thu, 7 Mar 2024 12:59:51 +0900 Subject: [PATCH 07/11] change docs-documnter docs --- docs-documenter/Project.toml | 10 - docs-documenter/make.jl | 24 --- docs-documenter/src/buffering.md | 23 --- docs-documenter/src/iteminterface.md | 27 --- docs-documenter/src/preprocessing.md | 9 - docs/Artifacts.toml | 6 - docs/Project.toml | 12 +- docs/literate/buffering.md | 23 --- docs/literate/colortransforms.md | 8 - docs/literate/intro.md | 36 ---- docs/literate/preprocessing.md | 9 - docs/literate/projective/data.md | 9 - docs/literate/projective/gallery.md | 180 ------------------ docs/literate/projective/interface.md | 13 -- docs/literate/projective/intro.md | 45 ----- docs/literate/projective/usage.md | 25 --- docs/literate/quickstart.md | 47 ----- docs/literate/stochastic.md | 19 -- docs/literate/tfminterface.md | 73 ------- docs/make.jl | 48 +++-- docs/project.jl | 27 --- docs/serve.jl | 26 --- docs/src/buffering.md | 23 +++ {docs-documenter => docs}/src/index.md | 4 +- docs/{literate => src}/iteminterface.md | 8 +- docs/src/preprocessing.md | 9 + .../src/projective/data.md | 0 .../src/projective/gallery.md | 0 .../src/projective/intro.md | 4 +- {docs-documenter => docs}/src/quickstart.md | 0 {docs-documenter => docs}/src/ref.md | 0 {docs-documenter => docs}/src/tfminterface.md | 16 +- .../src/transformations.md | 0 docs/toc.json | 26 --- 34 files changed, 74 insertions(+), 715 deletions(-) delete mode 100644 docs-documenter/Project.toml delete mode 100644 docs-documenter/make.jl delete mode 100644 docs-documenter/src/buffering.md delete mode 100644 docs-documenter/src/iteminterface.md delete mode 100644 docs-documenter/src/preprocessing.md delete mode 100644 docs/Artifacts.toml delete mode 100644 docs/literate/buffering.md delete mode 100644 docs/literate/colortransforms.md delete mode 100644 docs/literate/intro.md delete mode 100644 docs/literate/preprocessing.md delete mode 100644 docs/literate/projective/data.md delete mode 100644 docs/literate/projective/gallery.md delete mode 100644 docs/literate/projective/interface.md delete mode 100644 docs/literate/projective/intro.md delete mode 100644 docs/literate/projective/usage.md delete mode 100644 docs/literate/quickstart.md delete mode 100644 docs/literate/stochastic.md delete mode 100644 docs/literate/tfminterface.md delete mode 100644 docs/project.jl delete mode 100644 docs/serve.jl create mode 100644 docs/src/buffering.md rename {docs-documenter => docs}/src/index.md (87%) rename docs/{literate => src}/iteminterface.md (66%) create mode 100644 docs/src/preprocessing.md rename {docs-documenter => docs}/src/projective/data.md (100%) rename {docs-documenter => docs}/src/projective/gallery.md (100%) rename {docs-documenter => docs}/src/projective/intro.md (88%) rename {docs-documenter => docs}/src/quickstart.md (100%) rename {docs-documenter => docs}/src/ref.md (100%) rename {docs-documenter => docs}/src/tfminterface.md (64%) rename {docs-documenter => docs}/src/transformations.md (100%) delete mode 100644 docs/toc.json diff --git a/docs-documenter/Project.toml b/docs-documenter/Project.toml deleted file mode 100644 index e27f93b5..00000000 --- a/docs-documenter/Project.toml +++ /dev/null @@ -1,10 +0,0 @@ -[deps] -Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31" -Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" -MosaicViews = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" -StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" -TestImages = "5e47fb64-e119-507b-a336-dd2b206d9990" - -[compat] -Documenter = "1.2" diff --git a/docs-documenter/make.jl b/docs-documenter/make.jl deleted file mode 100644 index 0efbe787..00000000 --- a/docs-documenter/make.jl +++ /dev/null @@ -1,24 +0,0 @@ -using Documenter, DataAugmentation - -makedocs(sitename="DataAugmentation.jl", - pages = [ - "index.md", - "Quickstart" => "quickstart.md", - "Transformations" => "transformations.md", - "Build your transformations" =>[ - "Item Interface" => "iteminterface.md", - "Transform Interface" => "tfminterface.md", - "Projective Interface" => [ - "Intro" => "projective/intro.md", - "Data" => "projective/data.md", - "Gallery" => "projective/gallery.md", - ], - ], - "Misc" =>[ - "Buffering" => "buffering.md", - "Preprocessing" => "preprocessing.md", - "References" => "ref.md" - ], - ], - # format = Documenter.HTML(prettyurls = false) -) diff --git a/docs-documenter/src/buffering.md b/docs-documenter/src/buffering.md deleted file mode 100644 index 3ab85a99..00000000 --- a/docs-documenter/src/buffering.md +++ /dev/null @@ -1,23 +0,0 @@ -# Buffering - -As mentioned in the section on [transformations](./tfminterface.md), you can implement [`apply!`](#) as an inplace version of [`apply`](#) to support buffered transformations. Usually, the result of a regular `apply` can be used as a buffer. You may write - -```julia -buffer = apply(tfm, item) -apply!(buffer, tfm, item) -``` - -However, for some transformations, a different buffer is needed. [`Sequence`](#), for example, needs to reuse all intermediate results. That is why the buffer creation can be customized: - -- [`makebuffer`](#)`(tfm, item)` creates a buffer `buf` that can be used in an `apply!` call: `apply!(buf, tfm, item)`. - ---- - -Managing the buffers manually quickly becomes tedious. For convenience, this library implements [`Buffered`](#), a transformation wrapper that will use a buffer internally. `btfm = Buffered(tfm)` will create a buffer the first time it is `apply`ed and then use it by internally calling `apply!`. - -```julia -buffered = Buffered(tfm) -buffer = apply(tfm, item) # uses apply! internally -``` - -Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`BufferedThreadsafe`](#), a version of `Buffered` that keeps a separate buffer for every thread. \ No newline at end of file diff --git a/docs-documenter/src/iteminterface.md b/docs-documenter/src/iteminterface.md deleted file mode 100644 index c8fc7248..00000000 --- a/docs-documenter/src/iteminterface.md +++ /dev/null @@ -1,27 +0,0 @@ -# Item interface - -As described previously, items are simply containers for data: an [`Image`](#) represents an image, and [`Keypoints`](#) some keypoints. - -### Why do I need to wrap my data in an item? - -For one, the item `struct`s may contain metadata that is useful for some transformations. More importantly, though, by wrapping data in an item type, the *meaning* of the data is separated from the *representation*, that is, the concrete type. - -An `Array{Integer, 2}` could represent an image, but also a multi-class segmentation mask. Is `Array{Float32, 3}` a 3-dimensional image or a 2-dimensional image with the color channels expanded? - -Separating the representation from the data's meaning resolves those ambiguities. - -## Creating items - -To create a new item, you can simply subtype [`Item`](#): - -```julia -struct MyItem <: Item - data -end -``` - -The only function that is expected to be implemented is [`itemdata`](#), which simply returns the wrapped data. If, as above, you simply call the field holding the data `data`, you do not need to implement it. The same goes for the [`setdata`](#) helper. - -For some items, it also makes sense to implement the following: - -- [`showitem!`](#)`(img, item::I)` creates a visual representation of an item on top of `img`. \ No newline at end of file diff --git a/docs-documenter/src/preprocessing.md b/docs-documenter/src/preprocessing.md deleted file mode 100644 index 9ae45a2e..00000000 --- a/docs-documenter/src/preprocessing.md +++ /dev/null @@ -1,9 +0,0 @@ -# Preprocessing - -This library also implements some general transformations useful for getting data ready to be put into a model. - -- [`ToEltype`](#)`(T)` converts the element type of any [`AbstractArrayItem`](#) to `T`. -- [`ImageToTensor`](#) converts an image to an `ArrayItem` with another dimension for the color channels -- [`Normalize`](#) normalizes image tensors -- [`OneHot`](#) to one-hot encode multi-class masks ([`MaskMulti`](#)s) - diff --git a/docs/Artifacts.toml b/docs/Artifacts.toml deleted file mode 100644 index 5b10ec8b..00000000 --- a/docs/Artifacts.toml +++ /dev/null @@ -1,6 +0,0 @@ -[flux-theme] -git-tree-sha1 = "6e4be8bec8da9323c18d777d7855ef79dddcf524" - - [[flux-theme.download]] - sha256 = "b00941248ecdb643a72960bbfda19a3128591eca3d7cbcb3bfb80a6ab1c7f99e" - url = "https://github.com/darsnack/flux-theme/releases/download/v0.2.1/flux-theme-0.2.1.tar.gz" diff --git a/docs/Project.toml b/docs/Project.toml index 794d222a..e27f93b5 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,14 +1,10 @@ [deps] -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e" -FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" -ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" -ImageMagick = "6218d12a-5da1-5696-b52f-db25d2ecc6d1" +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" -JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" -ModuleInfo = "3c3ff5e7-c68c-4a09-80d1-9526a1e9878a" MosaicViews = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" -Pollen = "c88717ad-5130-4874-a664-5a9aba5ec443" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" TestImages = "5e47fb64-e119-507b-a336-dd2b206d9990" + +[compat] +Documenter = "1.2" diff --git a/docs/literate/buffering.md b/docs/literate/buffering.md deleted file mode 100644 index 3ab85a99..00000000 --- a/docs/literate/buffering.md +++ /dev/null @@ -1,23 +0,0 @@ -# Buffering - -As mentioned in the section on [transformations](./tfminterface.md), you can implement [`apply!`](#) as an inplace version of [`apply`](#) to support buffered transformations. Usually, the result of a regular `apply` can be used as a buffer. You may write - -```julia -buffer = apply(tfm, item) -apply!(buffer, tfm, item) -``` - -However, for some transformations, a different buffer is needed. [`Sequence`](#), for example, needs to reuse all intermediate results. That is why the buffer creation can be customized: - -- [`makebuffer`](#)`(tfm, item)` creates a buffer `buf` that can be used in an `apply!` call: `apply!(buf, tfm, item)`. - ---- - -Managing the buffers manually quickly becomes tedious. For convenience, this library implements [`Buffered`](#), a transformation wrapper that will use a buffer internally. `btfm = Buffered(tfm)` will create a buffer the first time it is `apply`ed and then use it by internally calling `apply!`. - -```julia -buffered = Buffered(tfm) -buffer = apply(tfm, item) # uses apply! internally -``` - -Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`BufferedThreadsafe`](#), a version of `Buffered` that keeps a separate buffer for every thread. \ No newline at end of file diff --git a/docs/literate/colortransforms.md b/docs/literate/colortransforms.md deleted file mode 100644 index 280c914c..00000000 --- a/docs/literate/colortransforms.md +++ /dev/null @@ -1,8 +0,0 @@ -# Color transformations - -DataAugmentation.jl currently supports the following color transformations for augmentation: - -- [`AdjustContrast`](#) randomly adjusts the contrast; and -- [`AdjustBrightness`](#) randomly adjusts the brightness. - -See the docstrings for examples. \ No newline at end of file diff --git a/docs/literate/intro.md b/docs/literate/intro.md deleted file mode 100644 index ce085c8a..00000000 --- a/docs/literate/intro.md +++ /dev/null @@ -1,36 +0,0 @@ -# DataAugmentation.jl - -This library provides data transformations for machine and deep learning. At the moment, it focuses on spatial data (think images, keypoint data and masks), but that is owed only to my current work. The extensible abstractions should fit other domains as well. - -For the most part, the transformations themselves are not very complex. The challenge this library tackles is to reconcile an easy-to-use, composable interface with performant execution. - -The key abstractions are `Transform`s, the transformation to apply, and `Item`s which contain the data to be transformed. [`apply`](#)`(tfm, item)`, as the name gives away, applies a transformation to an item. For example, given an [`Image`](#) item, we can resize it with the [`CenterResizeCrop`](#) transformation. - -```julia -item = Image(image) -tfm = CenterResizeCrop((128, 128)) -apply(tfm, item) -``` - -## Requirements - -The above example is simple, but there are more requirements of data augmentation pipelines that this library adresses. They serve as a motivation to the interface I've arrived at for defining transformations. - -### Stochasticity - -A transformation is stochastic (as opposed to deterministic) if it produces different outputs based on some random state. -This randomness can become a problem when applying an transformation to an aligned pair of input and target. If we have an image and a corresponding segmentation mask, using different scaling factors results in misalignment of the two; the segmentation no longer matches up with the image pixels. - -To handle this, the random state is explicitly passed to the transformations, rendering them deterministic. A generator for the random state can be defined with [`getrandstate`](#)`(tfm)` and passed to `apply` with the `randstate` keyword argument. - -### Composition - -Most data augmentation pipelines are made up of multiple steps: augmenting an image can mean resizing, randomly rotating, cropping and then normalizing the values. So applying transformations one after another – sequencing – is one way to compose transformations. But some operations, like affine transformations, can also be *fused*, resulting in a single transformation that is more performant and produces more accurate results. - -### Buffering - -Since data augmentation pipelines often run on large amounts of data, performance can often be improved by using prealloacted output buffers for the transformations. This results in fewer memory allocations and less garbage collection which both take time. - ---- - -Let's next see how these requirements are reflected in the [item](./iteminterface.md) and [transformation](./tfminterface.md) interfaces. \ No newline at end of file diff --git a/docs/literate/preprocessing.md b/docs/literate/preprocessing.md deleted file mode 100644 index 9ae45a2e..00000000 --- a/docs/literate/preprocessing.md +++ /dev/null @@ -1,9 +0,0 @@ -# Preprocessing - -This library also implements some general transformations useful for getting data ready to be put into a model. - -- [`ToEltype`](#)`(T)` converts the element type of any [`AbstractArrayItem`](#) to `T`. -- [`ImageToTensor`](#) converts an image to an `ArrayItem` with another dimension for the color channels -- [`Normalize`](#) normalizes image tensors -- [`OneHot`](#) to one-hot encode multi-class masks ([`MaskMulti`](#)s) - diff --git a/docs/literate/projective/data.md b/docs/literate/projective/data.md deleted file mode 100644 index 10850b59..00000000 --- a/docs/literate/projective/data.md +++ /dev/null @@ -1,9 +0,0 @@ -# Spatial data - -Before introducing various projective transformations, we have a look at the data that can be projected. There are three kinds of data currently supported: images, keypoints and segmentation masks. Both 2D and 3D data is supported (and technically, higher dimensions, but I've yet to find a dataset with 4 spatial dimensions). - -- [`Image`](#)`{N, T}` represents an `N`-dimensional image. `T` refers to the element type of the array that `Image` wraps, usually a color. When projecting images, proper interpolation methods are used to reduce artifacts like aliasing. See [`image.jl`](../../../src/items/image.jl) - -- [`MaskBinary`](#)`{N}` and [`MaskMulti`](#)`{N, T}` likewise represents `N`-dimensional segmentation masks. Unlike images, nearest-neighbor interpolation is used for projecting masks. See [`mask.jl`](../../src/items/mask.jl) - -- Lastly, [`Keypoints`](#)`{N}` represent keypoint data. The data should be an array of `SVector{N}`. Since there are many interpretations of keypoint data, there are also wrapper items for convenience: [`BoundingBox`](#) and [`Polygon`](#).See [`keypoints.jl`](../../src/items/keypoints.jl) diff --git a/docs/literate/projective/gallery.md b/docs/literate/projective/gallery.md deleted file mode 100644 index 47fb3736..00000000 --- a/docs/literate/projective/gallery.md +++ /dev/null @@ -1,180 +0,0 @@ -# Gallery - -Let's visualize what these projective transformations look like. - -You can apply them to [`Image`](#)s and -the keypoint-based items [`Keypoints`](#), [`Polygon`](#), and [`BoundingBox`](#). - -Let's take this picture of a light house: - -{cell=main style="display:none;" result=false output=false} -```julia -using DataAugmentation -using MosaicViews -using Images -using TestImages -using StaticArrays - -imagedata = testimage("lighthouse") -imagedata = imresize(imagedata, ratio = 196 / size(imagedata, 1)) -``` - -```julia -imagedata = testimage("lighthouse") -``` -{cell=main style="display:none;"} -```julia -imagedata -``` -To apply a transformation `tfm` to it, wrap it in -`Image`, apply the transformation and unwrap it using [`itemdata`](#): - -{cell=main } -```julia -tfm = CenterCrop((196, 196)) -image = Image(imagedata) -apply(tfm, image) |> itemdata -``` - -Now let's say we want to train a light house detector and have a bounding box for the light house. We can use the [`BoundingBox`](#) item to represent it. It takes the two corners of the bounding rectangle as the first argument. As the second argument we have to pass the size of the corresponding image. - -{cell=main} -```julia -points = SVector{2, Float32}[SVector(23., 120.), SVector(120., 150.)] -bbox = BoundingBox(points, size(imagedata)) -``` - -[`showitems`](#) visualizes the two items: -{cell=main} -```julia -showitems((image, bbox)) -``` -If we apply transformations like translation and cropping to the image, then the same transformations have to be applied to the bounding box. Otherwise, the bounding box will no longer match up with the light house. - -Another problem can occur with stochastic transformations like [`RandomResizeCrop`](#). If we apply it separately to the image and the bounding box, they will be cropped from slightly different locations: - -{cell=main} -```julia -tfm = RandomResizeCrop((128, 128)) -showitems(( - apply(tfm, image), - apply(tfm, bbox) -)) -``` -Instead, pass a tuple of the items to a single `apply` call so the same random state will be used for both image and bounding box: - -{cell=main} -```julia -apply(tfm, (image, bbox)) |> showitems -``` - -!!! info "3D Projective dimensions" - - We'll use a 2-dimensional [`Image`](#) and [`BoundingBox`](#) here, but you can apply most projective transformations to any spatial item (including [`Keypoints`](#), [`MaskBinary`](#) and [`MaskMulti`](#)) in 3 dimensions. - - Of course, you have to create a 3-dimensional transformation, i.e. `CenterCrop((128, 128, 128))` instead of `CenterCrop((128, 128))`. - -## Gallery -{cell=main style="display:none;" result=false} -```julia -function showtransform(tfm, item, n = 8; ncol = 4) - return mosaicview( - [showitems(apply(tfm, item)) for _ in 1:n], - fillvalue = RGBA(1, 1, 1, 0), - npad = 8, - rowmajor = true, - ncol = ncol) -end - - -function showtransforms(tfms, item; ncol = length(tfms)) - return mosaicview( - [parent(showitems(apply(tfm, item))) for tfm in tfms], - fillvalue = RGBA(1, 1, 1, 0), - npad = 8, - rowmajor = true, - ncol = ncol) -end -``` - -### [`RandomResizeCrop`](#)`(sz)` - -Resizes the sides so that one of them is no longer than `sz` and crops a region of size `sz` *from a random location*. - -{cell=main result=false} -```julia -tfm = RandomResizeCrop((128, 128)) -``` - -{cell=main style="display:none;"} -```julia -o = showtransform(tfm, (image, bbox), 6, ncol=6) -``` - -### [`CenterResizeCrop`](#) - -Resizes the sides so that one of them is no longer than `sz` and crops a region of size `sz` *from the center*. - -{cell=main result=false} -```julia -tfm = CenterResizeCrop((128, 128)) -``` - -{cell=main style="display:none;"} -```julia -o = showtransform(tfm, (image, bbox), 1) -``` - -### [`Crop`](#)`(sz[, from])` - -Crops a region of size `sz` from the image, *without resizing* the image first. - -{cell=main result=false} -```julia -using DataAugmentation: FromOrigin, FromCenter, FromRandom -tfms = [ - Crop((128, 128), FromOrigin()), - Crop((128, 128), FromCenter()), - Crop((128, 128), FromRandom()), - Crop((128, 128), FromRandom()), - Crop((128, 128), FromRandom()), - Crop((128, 128), FromRandom()), -] -``` - -{cell=main style="display:none;"} -```julia -o = showtransforms(tfms, (image, bbox)) -``` - -### [`FlipX`](#), [`FlipY`](#), [`Reflect`](#) - -Flip the data on the horizontally and vertically, respectively. More generally, reflect around an angle from the x-axis. - -{cell=main result=false} -```julia -tfms = [ - FlipX(), - FlipY(), - Reflect(30), -] -``` - -{cell=main style="display:none;"} -```julia -o = showtransforms(tfms, (image, bbox)) -``` - -### [`Rotate`](#) - -Rotate counter-clockwise by an angle. - -{cell=main result=false} -```julia -tfm = Rotate(20) |> CenterCrop((256, 256)) -``` - -{cell=main style="display:none;"} -```julia -o = showtransform(tfm, (image, bbox), 1) -``` diff --git a/docs/literate/projective/interface.md b/docs/literate/projective/interface.md deleted file mode 100644 index a0ec05cf..00000000 --- a/docs/literate/projective/interface.md +++ /dev/null @@ -1,13 +0,0 @@ - -## Projective transformations interface - - -The abstract type [`ProjectiveTransform`](#) represents a projective transformation. -A `ProjectiveTransform` needs to implement [`getprojection`](#)`(tfm, bounds; randstate)` that should return a `Transformation` from [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl). - -To add support for projective transformations to an item `I`, you need to implement - -- `getbounds(item::I)` returns the spatial bounds of the item; and -- `project(P, item::I, indices)` applies the projective transformation `P` and crops to `indices` - -To support `apply!`-ing projective transformations, `project!(bufitem, P, item)` can also be implemented. diff --git a/docs/literate/projective/intro.md b/docs/literate/projective/intro.md deleted file mode 100644 index 7a2a8c61..00000000 --- a/docs/literate/projective/intro.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# Projective transformations - -DataAugmentation.jl has great support for transforming spatial data like images and keypoints. Most of these transformations are projective transformations. For our purposes, a projection means a mapping between two coordinate spaces. In computer vision, these are frequently used for preprocessing and augmenting image data: images are randomly scaled, maybe flipped horizontally and finally cropped to the same size. - -This library generalizes projective transformations for different kinds of image and keypoint data in an N-dimensional Euclidean space. It also uses composition for performance improvements like fusing affine transformations. - -Unlike mathematical objects, the spatial data we want to transform has *spatial bounds*. For an image, these bounds are akin to the array size. But keypoint data aligned with an image has the same bounds even if they are not explicitly encoded in the representation of the data. -These spatial bounds can be used to dynamically create useful transformations. For example, a rotation around the center or a horizontal flip of keypoint annotations can be calculated from the bounds. - -Often, we also want to *crop* an area from the projected results. By evaluating only the parts of a projection that fall inside the cropped area, a lot of unnecessary computation can be avoided. - -## An example pipeline - - -We can break down most augmentation used in practive into a single (possibly stochastic) projection and a crop. - -As an example, consider an image augmentation pipeline: A random horizontal flip, followed by a random resized crop. The latter resizes and crops (irregularly sized) images to a common size without distorting the aspect ratio. - -```julia -Maybe(FlipX()) |> RandomResizeCrop((h, w)) -``` - -Let's pull apart the steps involved. - -1. Half of the time, flip the image horizontally. - -2. Scale the image down without distortion so that the shorter side length is 128. With an input of size `(512, 256)` this result in scaling both dimensions by `1/2`, resulting in an image with side lengths `(256, 128)`. - -3. Crop a random `(128, 128)` portion from that image. There is only "wiggle room" on the y-axis (which, by convention, is the first). - -All of these steps can be efficiently computed in one step with two tricks: - -- Some projections like reflection, translation, scaling and rotation can be composed into a single projection matrix. This means in the above example we only need to apply one projection which represents both the flipping (a reflection) and the scaling. Especially in pipelines with many augmentation steps this avoids a lot of unnecessary computation. - -- In cases, where the result of the projection is cropped, we can save additional computing by only evaluating the parts that we want to keep. - -## Cropping - -By default, the bounds of a projected item will be chosen so they still encase all the data. So after applying a `Scale((2, 2))` to an `Image`, its bounds will also be scaled by 2. Sometimes, however, we want to crop a part of the projected output, for example so a number of images can later be batched into a single array. While the crop usually has a fixed size, the region to crop still needs to be chosen. For validation data (which should be transformed deterministically), a center crop is usually used. For training data, on the other hand, a random region is selected to add additional augmentation. - ---- - -Read on to find out [how projective transformations are implemented](./interface.md) or jump straight to the [usage section](./usage.md). \ No newline at end of file diff --git a/docs/literate/projective/usage.md b/docs/literate/projective/usage.md deleted file mode 100644 index eb5f3699..00000000 --- a/docs/literate/projective/usage.md +++ /dev/null @@ -1,25 +0,0 @@ -# Usage - -Using projective transformations is as simple as any other transformations. Simply `compose` them: - -```julia -Rotate(-10:10) |> ScaleRatio(0.7:0.1:1.2) |> FlipX() |> Crop((128, 128)) -``` - -The composition will automatically create a single projective transformation and evaluate only the cropped area. - -## Affine transformations - -Affine transformations are a subgroup of projective transformations that can be composed very efficiently: composing two affine transformations results in another affine transformation. Affine transformations can represent translation, scaling, reflection and rotation. Available `Transform`s are: - -- [`ScaleRatio`](#), [`ScaleKeepAspect`](#) -- [`Rotate`](#) -- [`FlipX`](#), [`FlipY`](#), [`Reflect`](#) -- [`WarpAffine`](#) - -## Crops - -To get a cropped result, simply `compose` any `ProjectiveTransform` with - -- [`CenterCrop`](#) to crop a fixed-size region from the center; or -- [`RandomCrop`](#) to crop a fixed-size region from a random position diff --git a/docs/literate/quickstart.md b/docs/literate/quickstart.md deleted file mode 100644 index 9447e9e4..00000000 --- a/docs/literate/quickstart.md +++ /dev/null @@ -1,47 +0,0 @@ - -# Quickstart - -{cell=main style="display:none;"} -```julia -using TestImages -using ImageShow -using Images -using DataAugmentation -``` - -1. Import the library: - {cell=main} - ```julia - using DataAugmentation # Image, CenterResizeCrop, apply, showitem - using TestImages: testimage - ``` - -2. Load your data - - ```julia - image = testimage("lighthouse") - ``` - - {cell=main style="display:none;"} - ```julia - image = imresize(testimage("lighthouse"), ratio = 1/3) - ``` - -3. Create an item that contains the data you want to augment: - {cell=main result=false} - ```julia - item = Image(image) - ``` - -4. Create a transform: - {cell=main result=false} - ```julia - tfm = CenterResizeCrop((128, 128)) - ``` - -5. Apply the transformation and unwrap the data: - {cell=main} - ```julia - titem = apply(tfm, item) - timage = itemdata(titem) - ``` diff --git a/docs/literate/stochastic.md b/docs/literate/stochastic.md deleted file mode 100644 index b9d04dc8..00000000 --- a/docs/literate/stochastic.md +++ /dev/null @@ -1,19 +0,0 @@ -# Stochastic transformations - - -When augmenting data, it is often useful to apply a transformation only with some probability or choose from a set of transformations. Unlike in other data augmentation libraries like *albumentations*, in DataAugmentation.jl you can use wrapper transformations for this functionality. - -- [`Maybe`](#)`(tfm, p = 0.5)` applies a transformation with probability `p`; and -- [`OneOf`](#)`([tfm1, tfm2])` randomly selects a transformation to apply. - -## Example -Let's say we have an image classification dataset. For most datasets, horizontally flipping the image does not change the label: a flipped image of a cat still shows a cat. So let's flip every image horizontally half of the time to improve the generalization of the model we might be training. - -{cell=main} -```julia -using DataAugmentation, TestImages -item = Image(testimage("lighthouse")) -tfm = Maybe(FlipX()) -titems = [apply(tfm, item) for _ in 1:8] -showgrid(titems; ncol = 4, npad = 16) -``` \ No newline at end of file diff --git a/docs/literate/tfminterface.md b/docs/literate/tfminterface.md deleted file mode 100644 index a06b2bc2..00000000 --- a/docs/literate/tfminterface.md +++ /dev/null @@ -1,73 +0,0 @@ - -# Transformation interface - -{style="opacity:60%;"} -*[`base.jl`](../../src/base.jl)* - -The transformation interface is the centrepiece of this library. Beside straightforward transform application it also enables stochasticity, composition and buffering. - -A transformation is a type that subtypes [`Transform`](#). The only *required* function to implement for your transformation type `T` is - - -- [`apply`](#)`(tfm::T, item::I; randstate)` - - Applies the transformation `tfm` to item `item`. Implemented methods can of course dispatch on the type of `item`. `randstate` encapsulates the random state needed for stochastic transformations. The `apply` method implementation itself should be deterministic. - - You may dispatch on a specific item type `I` or use the abstract `Item` if one implementation works for all item types. - -You may additionally also implement: - -- [`getrandstate`](#)`(tfm)` for *stochastic* transformations - - Generates random state to be used inside `apply`. Calling `apply(tfm, item)` is equivalent to - `apply(tfm, item; randstate = getrandstate(tfm))`. It defaults to `nothing`, so we need not implement it for deterministic transformations. - -- [`apply!`](#)`(bufitem, tfm::T, item; randstate)` to support *buffering* - - Buffered version of `apply` that mutates `bufitem`. If not implemented, - falls back to regular `apply`. - -- [`compose`](#)`(tfm1, tfm2)` for custom *composition* with other transformations - - Composes transformations. By default, returns a [`Sequence`](#) transformation that applies the transformations one after the other. - - - -### Example - -The implementation of the [`MapElem`](#) transformation illustrates this interface well. It transforms any item with array data by mapping a function over the array's elements, just like `Base.map`. - -```julia -struct MapElem <: Transform - f -end -``` - -The `apply` implementation dispatches on [`AbstractArrayItem`](#), an abstract item type for items that wrap arrays. Note that the `randstate` keyword argument needs to be given even for implementations of deterministic transformations. We also make use of the [`setdata`](#) helper to update the item data. - -```julia -function apply(tfm::MapElem, item::AbstractArrayItem; randstate = nothing) - a = itemdata(item) - a_ = map(tfm.f, a) - return setdata(item, a_) -end -``` - -The buffered version applies the function inplace using `Base.map!`: - -```julia -function apply!( - bufitem::I, - tfm::MapElem, - item::I; - randstate = nothing) where I <: AbstractArrayItem - map!(tfm.f, itemdata(bufitem), itemdata(item)) - return bufitem -end -``` - -Finally, a `MapElem` can also be composed nicely with other `MapElem`s. Instead of applying them sequentially, the functions are fused and applied once. - -```julia -compose(tfm1::MapElem, tfm2::MapElem2) = MapElem(tfm2.f ∘ tfm1.f) -``` \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index 8600a40e..0efbe787 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,28 +1,24 @@ -""" -This script builds the Pollen.jl documentation so that it can be loaded -by the frontend. It accepts one argument: the path where the generated -files should be stored. +using Documenter, DataAugmentation - > julia docs/make.jl DIR - -Use `./serve.jl` for interactive development. -""" - -# Create target folder -isempty(ARGS) && error("Please pass a file path to make.jl:\n\t> julia docs/make.jl DIR ") -DIR = abspath(mkpath(ARGS[1])) - -# Create Project -project = include("project.jl") - -@info "Rewriting documents..." -Pollen.rewritesources!(project) - -@info "Writing to disk at \"$DIR\"..." -Pollen.build( - FileBuilder( - JSONFormat(), - DIR, - ), - project, +makedocs(sitename="DataAugmentation.jl", + pages = [ + "index.md", + "Quickstart" => "quickstart.md", + "Transformations" => "transformations.md", + "Build your transformations" =>[ + "Item Interface" => "iteminterface.md", + "Transform Interface" => "tfminterface.md", + "Projective Interface" => [ + "Intro" => "projective/intro.md", + "Data" => "projective/data.md", + "Gallery" => "projective/gallery.md", + ], + ], + "Misc" =>[ + "Buffering" => "buffering.md", + "Preprocessing" => "preprocessing.md", + "References" => "ref.md" + ], + ], + # format = Documenter.HTML(prettyurls = false) ) diff --git a/docs/project.jl b/docs/project.jl deleted file mode 100644 index b9fcff72..00000000 --- a/docs/project.jl +++ /dev/null @@ -1,27 +0,0 @@ -using Pollen -using Pkg - -# The main package you are documenting -using DataAugmentation -m = DataAugmentation - - -# Packages that will be indexed in the documentation. Add additional modules -# to the list. -ms = [m] - - -# Add rewriters here -project = Pollen.Project( - Pollen.Rewriter[ - DocumentFolder(Pkg.pkgdir(m), prefix = "documents"), - ParseCode(), - ExecuteCode(), - PackageDocumentation(ms), - StaticResources(), - DocumentGraph(), - SearchIndex(), - SaveAttributes((:title,)), - LoadFrontendConfig(Pkg.pkgdir(m)) - ], -) diff --git a/docs/serve.jl b/docs/serve.jl deleted file mode 100644 index c97200ca..00000000 --- a/docs/serve.jl +++ /dev/null @@ -1,26 +0,0 @@ -""" -This script serves the Pollen.jl documentation on a local file server -so that it can be loaded by the frontend in development mode. -files should be stored. - - > julia docs/serve.jl - -Use `./make.jl` to export the generated documents to disk. - -There are two modes for interactive development: Lazy and Regular. -In lazy mode, each document will be built only if it is requested in -the frontend, while for Regular mode, each document will be built -once before serving. -""" - -using Pollen - -project = include("project.jl") - - -Pollen.serve( - project; - lazy = get(ENV, "POLLEN_LAZY", "false") == "true", - port = Base.parse(Int, get(ENV, "POLLEN_PORT", "8000")), - format = JSONFormat() -) diff --git a/docs/src/buffering.md b/docs/src/buffering.md new file mode 100644 index 00000000..4b97ff33 --- /dev/null +++ b/docs/src/buffering.md @@ -0,0 +1,23 @@ +# Buffering + +As mentioned in the section on [transformations](./tfminterface.md), you can implement [`apply!`](@ref) as an inplace version of [`apply`](@ref) to support buffered transformations. Usually, the result of a regular `apply` can be used as a buffer. You may write + +```julia +buffer = apply(tfm, item) +apply!(buffer, tfm, item) +``` + +However, for some transformations, a different buffer is needed. [`Sequence`](@ref), for example, needs to reuse all intermediate results. That is why the buffer creation can be customized: + +- [`makebuffer`](@ref)`(tfm, item)` creates a buffer `buf` that can be used in an `apply!` call: `apply!(buf, tfm, item)`. + +--- + +Managing the buffers manually quickly becomes tedious. For convenience, this library implements [`Buffered`](@ref), a transformation wrapper that will use a buffer internally. `btfm = Buffered(tfm)` will create a buffer the first time it is `apply`ed and then use it by internally calling `apply!`. + +```julia +buffered = Buffered(tfm) +buffer = apply(tfm, item) # uses apply! internally +``` + +Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`BufferedThreadsafe`](@ref), a version of `Buffered` that keeps a separate buffer for every thread. \ No newline at end of file diff --git a/docs-documenter/src/index.md b/docs/src/index.md similarity index 87% rename from docs-documenter/src/index.md rename to docs/src/index.md index d72da115..4f7ee443 100644 --- a/docs-documenter/src/index.md +++ b/docs/src/index.md @@ -4,7 +4,7 @@ This library provides data transformations for machine and deep learning. At the For the most part, the transformations themselves are not very complex. The challenge this library tackles is to reconcile an easy-to-use, composable interface with performant execution. -The key abstractions are `Transform`s, the transformation to apply, and `Item`s which contain the data to be transformed. [`apply`](#)`(tfm, item)`, as the name gives away, applies a transformation to an item. For example, given an [`Image`](#) item, we can resize it with the [`CenterResizeCrop`](#) transformation. +The key abstractions are `Transform`s, the transformation to apply, and `Item`s which contain the data to be transformed. [`apply`](@ref)`(tfm, item)`, as the name gives away, applies a transformation to an item. For example, given an [`Image`](@ref) item, we can resize it with the [`CenterResizeCrop`](@ref) transformation. ```julia item = Image(image) @@ -21,7 +21,7 @@ The above example is simple, but there are more requirements of data augmentatio A transformation is stochastic (as opposed to deterministic) if it produces different outputs based on some random state. This randomness can become a problem when applying an transformation to an aligned pair of input and target. If we have an image and a corresponding segmentation mask, using different scaling factors results in misalignment of the two; the segmentation no longer matches up with the image pixels. -To handle this, the random state is explicitly passed to the transformations, rendering them deterministic. A generator for the random state can be defined with [`getrandstate`](#)`(tfm)` and passed to `apply` with the `randstate` keyword argument. +To handle this, the random state is explicitly passed to the transformations, rendering them deterministic. A generator for the random state can be defined with [`getrandstate`](@ref)`(tfm)` and passed to `apply` with the `randstate` keyword argument. ### Composition diff --git a/docs/literate/iteminterface.md b/docs/src/iteminterface.md similarity index 66% rename from docs/literate/iteminterface.md rename to docs/src/iteminterface.md index c8fc7248..cae90271 100644 --- a/docs/literate/iteminterface.md +++ b/docs/src/iteminterface.md @@ -1,6 +1,6 @@ # Item interface -As described previously, items are simply containers for data: an [`Image`](#) represents an image, and [`Keypoints`](#) some keypoints. +As described previously, items are simply containers for data: an [`Image`](@ref) represents an image, and [`Keypoints`](@ref) some keypoints. ### Why do I need to wrap my data in an item? @@ -12,7 +12,7 @@ Separating the representation from the data's meaning resolves those ambiguities ## Creating items -To create a new item, you can simply subtype [`Item`](#): +To create a new item, you can simply subtype [`Item`](@ref): ```julia struct MyItem <: Item @@ -20,8 +20,8 @@ struct MyItem <: Item end ``` -The only function that is expected to be implemented is [`itemdata`](#), which simply returns the wrapped data. If, as above, you simply call the field holding the data `data`, you do not need to implement it. The same goes for the [`setdata`](#) helper. +The only function that is expected to be implemented is [`itemdata`](@ref), which simply returns the wrapped data. If, as above, you simply call the field holding the data `data`, you do not need to implement it. The same goes for the [`setdata`](@ref) helper. For some items, it also makes sense to implement the following: -- [`showitem!`](#)`(img, item::I)` creates a visual representation of an item on top of `img`. \ No newline at end of file +- [`showitem!`](@ref)`(img, item::I)` creates a visual representation of an item on top of `img`. \ No newline at end of file diff --git a/docs/src/preprocessing.md b/docs/src/preprocessing.md new file mode 100644 index 00000000..74565847 --- /dev/null +++ b/docs/src/preprocessing.md @@ -0,0 +1,9 @@ +# Preprocessing + +This library also implements some general transformations useful for getting data ready to be put into a model. + +- [`ToEltype`](@ref)`(T)` converts the element type of any [`AbstractArrayItem`](@ref) to `T`. +- [`ImageToTensor`](@ref) converts an image to an `ArrayItem` with another dimension for the color channels +- [`Normalize`](@ref) normalizes image tensors +- [`OneHot`](@ref) to one-hot encode multi-class masks ([`MaskMulti`](@ref)s) + diff --git a/docs-documenter/src/projective/data.md b/docs/src/projective/data.md similarity index 100% rename from docs-documenter/src/projective/data.md rename to docs/src/projective/data.md diff --git a/docs-documenter/src/projective/gallery.md b/docs/src/projective/gallery.md similarity index 100% rename from docs-documenter/src/projective/gallery.md rename to docs/src/projective/gallery.md diff --git a/docs-documenter/src/projective/intro.md b/docs/src/projective/intro.md similarity index 88% rename from docs-documenter/src/projective/intro.md rename to docs/src/projective/intro.md index 01d62563..6d29a27d 100644 --- a/docs-documenter/src/projective/intro.md +++ b/docs/src/projective/intro.md @@ -30,8 +30,8 @@ By default, the bounds of a projected item will be chosen so they still encase a ## Projective transformations interface -The abstract type [`ProjectiveTransform`](#) represents a projective transformation. -A `ProjectiveTransform` needs to implement [`getprojection`](#)`(tfm, bounds; randstate)` that should return a `Transformation` from [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl). +The abstract type [`ProjectiveTransform`](@ref) represents a projective transformation. +A `ProjectiveTransform` needs to implement [`getprojection`](@ref)`(tfm, bounds; randstate)` that should return a `Transformation` from [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl). To add support for projective transformations to an item `I`, you need to implement diff --git a/docs-documenter/src/quickstart.md b/docs/src/quickstart.md similarity index 100% rename from docs-documenter/src/quickstart.md rename to docs/src/quickstart.md diff --git a/docs-documenter/src/ref.md b/docs/src/ref.md similarity index 100% rename from docs-documenter/src/ref.md rename to docs/src/ref.md diff --git a/docs-documenter/src/tfminterface.md b/docs/src/tfminterface.md similarity index 64% rename from docs-documenter/src/tfminterface.md rename to docs/src/tfminterface.md index 239c6bc1..0a8dfc83 100644 --- a/docs-documenter/src/tfminterface.md +++ b/docs/src/tfminterface.md @@ -6,10 +6,10 @@ The transformation interface is the centrepiece of this library. Beside straightforward transform application it also enables stochasticity, composition and buffering. -A transformation is a type that subtypes [`Transform`](#). The only *required* function to implement for your transformation type `T` is +A transformation is a type that subtypes [`Transform`](@ref). The only *required* function to implement for your transformation type `T` is -- [`apply`](#)`(tfm::T, item::I; randstate)` +- [`apply`](@ref)`(tfm::T, item::I; randstate)` Applies the transformation `tfm` to item `item`. Implemented methods can of course dispatch on the type of `item`. `randstate` encapsulates the random state needed for stochastic transformations. The `apply` method implementation itself should be deterministic. @@ -17,25 +17,25 @@ A transformation is a type that subtypes [`Transform`](#). The only *required* f You may additionally also implement: -- [`getrandstate`](#)`(tfm)` for *stochastic* transformations +- [`getrandstate`](@ref)`(tfm)` for *stochastic* transformations Generates random state to be used inside `apply`. Calling `apply(tfm, item)` is equivalent to `apply(tfm, item; randstate = getrandstate(tfm))`. It defaults to `nothing`, so we need not implement it for deterministic transformations. -- [`apply!`](#)`(bufitem, tfm::T, item; randstate)` to support *buffering* +- [`apply!`](@ref)`(bufitem, tfm::T, item; randstate)` to support *buffering* Buffered version of `apply` that mutates `bufitem`. If not implemented, falls back to regular `apply`. -- [`compose`](#)`(tfm1, tfm2)` for custom *composition* with other transformations +- [`compose`](@ref)`(tfm1, tfm2)` for custom *composition* with other transformations - Composes transformations. By default, returns a [`Sequence`](#) transformation that applies the transformations one after the other. + Composes transformations. By default, returns a [`Sequence`](@ref) transformation that applies the transformations one after the other. ### Example -The implementation of the [`MapElem`](#) transformation illustrates this interface well. It transforms any item with array data by mapping a function over the array's elements, just like `Base.map`. +The implementation of the [`MapElem`](@ref) transformation illustrates this interface well. It transforms any item with array data by mapping a function over the array's elements, just like `Base.map`. ```julia struct MapElem <: Transform @@ -43,7 +43,7 @@ struct MapElem <: Transform end ``` -The `apply` implementation dispatches on [`AbstractArrayItem`](#), an abstract item type for items that wrap arrays. Note that the `randstate` keyword argument needs to be given even for implementations of deterministic transformations. We also make use of the [`setdata`](#) helper to update the item data. +The `apply` implementation dispatches on [`AbstractArrayItem`](@ref), an abstract item type for items that wrap arrays. Note that the `randstate` keyword argument needs to be given even for implementations of deterministic transformations. We also make use of the [`setdata`](@ref) helper to update the item data. ```julia function apply(tfm::MapElem, item::AbstractArrayItem; randstate = nothing) diff --git a/docs-documenter/src/transformations.md b/docs/src/transformations.md similarity index 100% rename from docs-documenter/src/transformations.md rename to docs/src/transformations.md diff --git a/docs/toc.json b/docs/toc.json deleted file mode 100644 index ff094da4..00000000 --- a/docs/toc.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "README": "documents/README.md", - "Quickstart": "documents/docs/literate/quickstart.md", - "Overview": "documents/docs/literate/intro.md", - "How-to": { - "Apply transforms inplace": "documents/docs/literate/buffering.md", - "Construct projective transforms": "docs/literate/projective/usage.md" - }, - "Reference": { - "Module reference": "references/DataAugmentation", - "Transforms": { - "Projective": "documents/docs/literate/projective/gallery.md", - "Preprocessing": "documents/docs/literate/preprocessing.md", - "Color": "documents/docs/literate/colortransforms.md", - "Stochastic": "documents/docs/literate/stochastic.md" - }, - "Items": { - "Spatial data": "documents/docs/literate/projective/data.md" - }, - "Interfaces": { - "Items": "documents/docs/literate/iteminterface.md", - "Transforms": "documents/docs/literate/tfminterface.md", - "Projective transforms": "documents/docs/literate/projective/interface.md" - } - } -} From 5fe17926455c090571079636c41eddd89c753b2e Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Thu, 7 Mar 2024 13:00:11 +0900 Subject: [PATCH 08/11] change (#) to (@ref) (documenter.jl) --- src/base.jl | 14 +++++++------- src/items/image.jl | 6 +++--- src/items/keypoints.jl | 4 ++-- src/preprocessing.jl | 6 +++--- src/projective/affine.jl | 2 +- src/projective/base.jl | 2 +- src/projective/compose.jl | 2 +- src/projective/crop.jl | 2 +- src/projective/warp.jl | 2 +- src/sequence.jl | 6 +++--- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/base.jl b/src/base.jl index a6c92729..5961ee8d 100644 --- a/src/base.jl +++ b/src/base.jl @@ -6,7 +6,7 @@ abstract type AbstractItem Abstract supertype for all items. To implement items, subtype -either [`Item`](#) to create a new item or [`ItemWrapper`](#) +either [`Item`](@ref) to create a new item or [`ItemWrapper`](@ref) to wrap an existing item. """ abstract type AbstractItem end @@ -18,7 +18,7 @@ abstract type AbstractItem end Abstract supertype of concrete items. Subtype if you want to create a new item. If you want to wrap -an existing item, see [`ItemWrapper`](#). +an existing item, see [`ItemWrapper`](@ref). """ abstract type Item <: AbstractItem end @@ -38,8 +38,8 @@ Abstract supertype for all transformations. """ abstract type Transform end -# Plus the default implementations of [`itemdata`](#) and -# [`getrandstate`](#). +# Plus the default implementations of [`itemdata`](@ref) and +# [`getrandstate`](@ref). """ itemdata(item) @@ -86,7 +86,7 @@ end Compose tranformations. Use `|>` as an alias. -Defaults to creating a [`Sequence`](#) of transformations, +Defaults to creating a [`Sequence`](@ref) of transformations, but smarter behavior can be implemented. For example, `MapElem(f) |> MapElem(g) == MapElem(g ∘ f)`. """ @@ -95,7 +95,7 @@ compose(tfm::Transform) = tfm compose(tfms...) = compose(compose(tfms[1], tfms[2]), tfms[3:end]...) Base.:(|>)(tfm1::Transform, tfm2::Transform) = compose(tfm1, tfm2) -# [`Identity`](#) is the identity transformation. +# [`Identity`](@ref) is the identity transformation. """ Identity() @@ -110,7 +110,7 @@ compose(::Identity, ::Identity) = Identity() compose(tfm::Transform, ::Identity) = tfm compose(::Identity, tfm::Transform) = tfm -# Lastly, [`setdata`](#) provides a convenient way to create a copy +# Lastly, [`setdata`](@ref) provides a convenient way to create a copy # of an item, replacing only the wrapped data. This relies on the # wrapped data field being named `data`, though. diff --git a/src/items/image.jl b/src/items/image.jl index 219b27bb..7aea4dc9 100644 --- a/src/items/image.jl +++ b/src/items/image.jl @@ -1,4 +1,4 @@ -# We first define the [`Image`](#) item. Since we need to keep +# We first define the [`Image`](@ref) item. Since we need to keep # track of the spatial bounds for projective transformations # we add them as a field. By default, they will simply # correspond to the image axes. @@ -52,8 +52,8 @@ function showitem!(img, image::Image{2, <:AbstractFloat}) end -# To support projective transformations, we need to implement [`getbounds`](#) -# and [`project`](#). +# To support projective transformations, we need to implement [`getbounds`](@ref) +# and [`project`](@ref). getbounds(image::Image) = image.bounds diff --git a/src/items/keypoints.jl b/src/items/keypoints.jl index f4a34494..5d095b30 100644 --- a/src/items/keypoints.jl +++ b/src/items/keypoints.jl @@ -69,7 +69,7 @@ end Polygon(points, sz) Polygon{N, T, M}(points, bounds) -Item wrapper around [`Keypoints`](#). +Item wrapper around [`Keypoints`](@ref). ## Examples @@ -106,7 +106,7 @@ end BoundingBox(points, sz) BoundingBox{N, T, M}(points, bounds) -Item wrapper around [`Keypoints`](#). +Item wrapper around [`Keypoints`](@ref). ## Examples diff --git a/src/preprocessing.jl b/src/preprocessing.jl index 0ee1697a..98bb0c7d 100644 --- a/src/preprocessing.jl +++ b/src/preprocessing.jl @@ -1,4 +1,4 @@ -# ### [`ToEltype`](#) +# ### [`ToEltype`](@ref) """ ToEltype(T) @@ -34,7 +34,7 @@ function apply!(buf, ::ToEltype, item::AbstractArrayItem; randstate = nothing) return buf end -# ### [`Normalize`](#) +# ### [`Normalize`](@ref) """ Normalize(means, stds) @@ -225,7 +225,7 @@ end OneHot([T = Float32]) One-hot encodes a `MaskMulti` with `n` classes and size `sz` into -an array item of size `(sz..., n)` with element type `T`. Supports [`apply!`](#). +an array item of size `(sz..., n)` with element type `T`. Supports [`apply!`](@ref). ```julia item = MaskMulti(rand(1:4, 100, 100), 1:4) diff --git a/src/projective/affine.jl b/src/projective/affine.jl index 99a17e3a..861116b1 100644 --- a/src/projective/affine.jl +++ b/src/projective/affine.jl @@ -72,7 +72,7 @@ end Projective transformation that scales sides to `sizes`, disregarding aspect ratio. -See also [`ScaleKeepAspect`](#). +See also [`ScaleKeepAspect`](@ref). """ struct ScaleFixed{N} <: ProjectiveTransform sizes::NTuple{N, Int} diff --git a/src/projective/base.jl b/src/projective/base.jl index c17aa3c2..dc54fe1b 100644 --- a/src/projective/base.jl +++ b/src/projective/base.jl @@ -76,7 +76,7 @@ function project end project!(bufitem, P, item, indices) Project `item` using projection `P` and crop to `indices` if given. -Store result in `bufitem`. Inplace version of [`project`](#). +Store result in `bufitem`. Inplace version of [`project`](@ref). Default implementation falls back to `project`. """ diff --git a/src/projective/compose.jl b/src/projective/compose.jl index ba000c2f..94e01ef8 100644 --- a/src/projective/compose.jl +++ b/src/projective/compose.jl @@ -1,4 +1,4 @@ -# [`ComposedProjectiveTransform`](#) implements efficient composition +# [`ComposedProjectiveTransform`](@ref) implements efficient composition # of `ProjectiveTransform`s. """ diff --git a/src/projective/crop.jl b/src/projective/crop.jl index bcba2ecf..3b63b180 100644 --- a/src/projective/crop.jl +++ b/src/projective/crop.jl @@ -37,7 +37,7 @@ Crop(sz, FromRandom()) """ RandomCrop(sz) = Crop(sz, FromRandom()) -# The random state of a [`Crop`](#) consists of offsets from the origin. +# The random state of a [`Crop`](@ref) consists of offsets from the origin. getrandstate(crop::Crop{N, FromOrigin}) where N = Tuple(0. for _ in 1:N) getrandstate(crop::Crop{N, FromCenter}) where N = Tuple(0.5 for _ in 1:N) diff --git a/src/projective/warp.jl b/src/projective/warp.jl index ca87b5bb..01731e73 100644 --- a/src/projective/warp.jl +++ b/src/projective/warp.jl @@ -32,7 +32,7 @@ end """ threepointwarpaffine(srcps, dstps) -Calculate an affine [`CoordinateTransformations.LinearMap`](#) +Calculate an affine [`CoordinateTransformations.LinearMap`](@ref) from 3 source points to 3 destination points. Adapted from [CoordinateTransformations.jl#30](https://github.com/JuliaGeometry/CoordinateTransformations.jl/issues/30#issuecomment-610337378). diff --git a/src/sequence.jl b/src/sequence.jl index 08a890f4..9e41bf96 100644 --- a/src/sequence.jl +++ b/src/sequence.jl @@ -1,7 +1,7 @@ -# To make composition possible, we implement [`compose`](#), which -# defaults to returning a [`Sequence`](#). +# To make composition possible, we implement [`compose`](@ref), which +# defaults to returning a [`Sequence`](@ref). """ Sequence(transforms...) @@ -9,7 +9,7 @@ `Transform` that applies multiple `transformations` after each other. -You should not use this explicitly. Instead use [`compose`](#). +You should not use this explicitly. Instead use [`compose`](@ref). """ struct Sequence{T<:Tuple} <: Transform transforms::T From aa457aa44be0da39401d586548ec5a4c343acdbc Mon Sep 17 00:00:00 2001 From: darsnack Date: Sat, 9 Mar 2024 09:04:26 -0500 Subject: [PATCH 09/11] Add deployment CI --- .github/workflows/CleanPreview.yml | 27 ++++++++++++ .github/workflows/PRComment.yml | 14 +++++++ .github/workflows/ci.yml | 66 ++++++++++++++++++++++++++---- .github/workflows/pollenbuild.yml | 33 --------------- .github/workflows/pollenstatic.yml | 58 -------------------------- .gitignore | 1 - docs/Project.toml | 1 + docs/make.jl | 49 +++++++++++++--------- 8 files changed, 130 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/CleanPreview.yml create mode 100644 .github/workflows/PRComment.yml delete mode 100644 .github/workflows/pollenbuild.yml delete mode 100644 .github/workflows/pollenstatic.yml diff --git a/.github/workflows/CleanPreview.yml b/.github/workflows/CleanPreview.yml new file mode 100644 index 00000000..25946efc --- /dev/null +++ b/.github/workflows/CleanPreview.yml @@ -0,0 +1,27 @@ +# from https://github.com/CliMA/ClimaTimeSteppers.jl +name: Doc Preview Cleanup + +on: + pull_request: + types: [closed] + +jobs: + doc-preview-cleanup: + runs-on: ubuntu-latest + steps: + - name: Checkout gh-pages branch + uses: actions/checkout@v2 + with: + ref: gh-pages + - name: Delete preview and history + push changes + run: | + if [ -d "previews/PR$PRNUM" ]; then + git config user.name "Documenter.jl" + git config user.email "documenter@juliadocs.github.io" + git rm -rf "previews/PR$PRNUM" + git commit -m "delete preview" + git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree}) + git push --force origin gh-pages-new:gh-pages + fi + env: + PRNUM: ${{ github.event.number }} diff --git a/.github/workflows/PRComment.yml b/.github/workflows/PRComment.yml new file mode 100644 index 00000000..6c12a6ed --- /dev/null +++ b/.github/workflows/PRComment.yml @@ -0,0 +1,14 @@ +name: Doc Preview PR Comment +on: + pull_request: + types: [labeled] +jobs: + pr_comment: + runs-on: ubuntu-latest + steps: + - name: Create PR comment + if: github.event_name == 'pull_request' && github.repository == github.event.pull_request.head.repo.full_name && github.event.label.name == 'documentation' # if this is a pull request build AND the pull request is NOT made from a fork + uses: thollander/actions-comment-pull-request@71efef56b184328c7ef1f213577c3a90edaa4aff + with: + message: 'Once the documentation build has completed, you can preview any updated documentation at this URL: https://fluxml.ai/DataAugmentation.jl/previews/PR${{ github.event.number }}/' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce973ff..2880f4de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,28 +1,80 @@ name: CI + on: - - push - - pull_request + push: + branches: ['main'] + pull_request: + branches: ['main'] + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: version: - - '1.6' - '1' + - '1.6' # this is the oldest compat + - 'nightly' os: - ubuntu-latest - macOS-latest - windows-latest arch: - x64 + - x86 + exclude: + - os: macOS-latest + arch: x86 steps: - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-runtest@latest - + - uses: actions/cache@v3 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + docs: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: '1.6' + - run: | + julia --project=docs -e ' + using Pkg + Pkg.develop(PackageSpec(path=pwd())) + Pkg.instantiate()' + - run: | + julia --color=yes --project=docs/ -e ' + using DataAugmentation + using Documenter + using Documenter: doctest + DocMeta.setdocmeta!(DataAugmentation, + :DocTestSetup, + :(using DataAugmentation); + recursive=true) + doctest(DataAugmentation)' + - run: julia --project=docs docs/make.jl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/pollenbuild.yml b/.github/workflows/pollenbuild.yml deleted file mode 100644 index 07f7284a..00000000 --- a/.github/workflows/pollenbuild.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: pollenbuild - -on: - push: - branches: ['master', 'main'] - -jobs: - pollen: - name: "Pollen - build documentation data" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/checkout@v3 - with: - ref: pollen - path: pollen - - uses: julia-actions/setup-julia@latest - with: - version: '1.7' - - name: "Install package and docs dependencies" - run: | - julia --color=yes --project=./docs -e 'using Pkg; Pkg.add([Pkg.PackageSpec(path="."), Pkg.PackageSpec(url="https://github.com/c42f/JuliaSyntax.jl"), Pkg.PackageSpec(url="https://github.com/lorenzoh/ModuleInfo.jl"), Pkg.PackageSpec(url="https://github.com/lorenzoh/Pollen.jl", rev="main")]); Pkg.instantiate();' - - name: Build - run: | - julia --color=yes --project=./docs ./docs/make.jl ./pollen/dev/ - - name: Deploy - run: | - cd pollen - git config user.name github-actions - git config user.email github-actions@github.com - git add . - git commit -m "Build documentation data (Pollen.jl)" - git push diff --git a/.github/workflows/pollenstatic.yml b/.github/workflows/pollenstatic.yml deleted file mode 100644 index 9d7bad84..00000000 --- a/.github/workflows/pollenstatic.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: pollenstatic - -on: - push: - branches: ['pollen'] - workflow_run: - workflows: ['pollenbuild'] - types: - - completed - -jobs: - pollen: - name: "Pollen - Prerender static frontend and deploy" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: pollen - path: data - - uses: actions/checkout@v3 - with: - ref: "gh-pages" - path: pages - - uses: actions/checkout@v3 - with: - repository: lorenzoh/pollenjl-frontend - path: frontend - ref: main - - name: Move Pollen data to static folder - run: | - ls - rm -rf data/.git - cp -r data/ frontend/static/ - - uses: actions/setup-node@v2 - - name: Install dependencies - run: | - cd frontend - npm install - - name: Build - run: | - cd frontend - npm run build - - name: Build search index - run: | - cd frontend - cat static/data/dev/documents.json | node buildindex.cjs > ../data/dev/searchindex.json - - name: Deploy changes - run: | - ls -lt - rm -r frontend/build/data - cp -r frontend/build/** pages - cp -r data pages/ - cd pages - git config user.name github-actions - git config user.email github-actions@github.com - git add -f . - git commit -m "Deploy documentation (Pollen.jl)" - git push diff --git a/.gitignore b/.gitignore index b01f11fa..2cadb826 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /docs/build/ -/docs-documenter/build/ Manifest.toml theme gh-pages diff --git a/docs/Project.toml b/docs/Project.toml index e27f93b5..69d1e968 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,4 +1,5 @@ [deps] +DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" diff --git a/docs/make.jl b/docs/make.jl index 0efbe787..cc89505a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,24 +1,33 @@ using Documenter, DataAugmentation -makedocs(sitename="DataAugmentation.jl", - pages = [ - "index.md", - "Quickstart" => "quickstart.md", - "Transformations" => "transformations.md", - "Build your transformations" =>[ - "Item Interface" => "iteminterface.md", - "Transform Interface" => "tfminterface.md", - "Projective Interface" => [ - "Intro" => "projective/intro.md", - "Data" => "projective/data.md", - "Gallery" => "projective/gallery.md", - ], +makedocs(; + modules = [DataAugmentation], + sitename="DataAugmentation.jl", + pages = [ + "index.md", + "Quickstart" => "quickstart.md", + "Transformations" => "transformations.md", + "Build your transformations" =>[ + "Item Interface" => "iteminterface.md", + "Transform Interface" => "tfminterface.md", + "Projective Interface" => [ + "Intro" => "projective/intro.md", + "Data" => "projective/data.md", + "Gallery" => "projective/gallery.md", + ], + ], + "Misc" =>[ + "Buffering" => "buffering.md", + "Preprocessing" => "preprocessing.md", + "References" => "ref.md" + ], ], - "Misc" =>[ - "Buffering" => "buffering.md", - "Preprocessing" => "preprocessing.md", - "References" => "ref.md" - ], - ], - # format = Documenter.HTML(prettyurls = false) + warnonly = [:example_block, :missing_docs, :cross_references], + format = Documenter.HTML(canonical = "https://fluxml.ai/DataAugmentation.jl/stable/", + assets = ["assets/flux.css"], + prettyurls = get(ENV, "CI", nothing) == "true") ) + +deploydocs(repo = "github.com/FluxML/DataAugmentation.jl.git", + target = "build", + push_preview = true) From ebeb620c8dd01d403f80242407517c3f3d9a7a2d Mon Sep 17 00:00:00 2001 From: darsnack Date: Sat, 9 Mar 2024 09:12:42 -0500 Subject: [PATCH 10/11] Switch to master from main --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2880f4de..6702157d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: ['main'] + branches: ['master'] pull_request: - branches: ['main'] + branches: ['master'] concurrency: # Skip intermediate builds: always. From fb8aab2892e47bbaed9757441f34ada4766996f1 Mon Sep 17 00:00:00 2001 From: Zhanibek Date: Mon, 11 Mar 2024 18:28:31 +0900 Subject: [PATCH 11/11] fix links add docs --- docs/src/buffering.md | 11 ++++++--- docs/src/index.md | 2 +- docs/src/iteminterface.md | 4 ++-- docs/src/preprocessing.md | 2 +- docs/src/projective/intro.md | 4 ++-- docs/src/ref.md | 43 ++++++++++++++++++++++++++++++++++++ docs/src/tfminterface.md | 6 ++--- docs/src/transformations.md | 4 ++++ src/base.jl | 9 ++++---- src/buffered.jl | 6 +++++ src/projective/affine.jl | 3 +++ src/projective/base.jl | 1 - src/projective/warp.jl | 2 +- 13 files changed, 79 insertions(+), 18 deletions(-) diff --git a/docs/src/buffering.md b/docs/src/buffering.md index 4b97ff33..eb3a8fcd 100644 --- a/docs/src/buffering.md +++ b/docs/src/buffering.md @@ -9,15 +9,20 @@ apply!(buffer, tfm, item) However, for some transformations, a different buffer is needed. [`Sequence`](@ref), for example, needs to reuse all intermediate results. That is why the buffer creation can be customized: -- [`makebuffer`](@ref)`(tfm, item)` creates a buffer `buf` that can be used in an `apply!` call: `apply!(buf, tfm, item)`. +- [`DataAugmentation.makebuffer`](@ref)`(tfm, item)` creates a buffer `buf` that can be used in an `apply!` call: `apply!(buf, tfm, item)`. --- -Managing the buffers manually quickly becomes tedious. For convenience, this library implements [`Buffered`](@ref), a transformation wrapper that will use a buffer internally. `btfm = Buffered(tfm)` will create a buffer the first time it is `apply`ed and then use it by internally calling `apply!`. +Managing the buffers manually quickly becomes tedious. For convenience, this library implements [`DataAugmentation.Buffered`](@ref), a transformation wrapper that will use a buffer internally. `btfm = Buffered(tfm)` will create a buffer the first time it is `apply`ed and then use it by internally calling `apply!`. ```julia buffered = Buffered(tfm) buffer = apply(tfm, item) # uses apply! internally ``` -Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`BufferedThreadsafe`](@ref), a version of `Buffered` that keeps a separate buffer for every thread. \ No newline at end of file +Since `Buffered` only stores one buffer, you may run into problems when using it in a multi-threading context where different threads invalidate the buffer before it can be used. In that case, you can use [`DataAugmentation.BufferedThreadsafe`](@ref), a version of `Buffered` that keeps a separate buffer for every thread. + +```@docs +DataAugmentation.Buffered +DataAugmentation.BufferedThreadsafe +``` diff --git a/docs/src/index.md b/docs/src/index.md index 4f7ee443..3d855f9d 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -21,7 +21,7 @@ The above example is simple, but there are more requirements of data augmentatio A transformation is stochastic (as opposed to deterministic) if it produces different outputs based on some random state. This randomness can become a problem when applying an transformation to an aligned pair of input and target. If we have an image and a corresponding segmentation mask, using different scaling factors results in misalignment of the two; the segmentation no longer matches up with the image pixels. -To handle this, the random state is explicitly passed to the transformations, rendering them deterministic. A generator for the random state can be defined with [`getrandstate`](@ref)`(tfm)` and passed to `apply` with the `randstate` keyword argument. +To handle this, the random state is explicitly passed to the transformations, rendering them deterministic. A generator for the random state can be defined with [`DataAugmentation.getrandstate`](@ref)`(tfm)` and passed to `apply` with the `randstate` keyword argument. ### Composition diff --git a/docs/src/iteminterface.md b/docs/src/iteminterface.md index cae90271..da809b7b 100644 --- a/docs/src/iteminterface.md +++ b/docs/src/iteminterface.md @@ -20,8 +20,8 @@ struct MyItem <: Item end ``` -The only function that is expected to be implemented is [`itemdata`](@ref), which simply returns the wrapped data. If, as above, you simply call the field holding the data `data`, you do not need to implement it. The same goes for the [`setdata`](@ref) helper. +The only function that is expected to be implemented is [`itemdata`](@ref), which simply returns the wrapped data. If, as above, you simply call the field holding the data `data`, you do not need to implement it. The same goes for the [`DataAugmentation.setdata`](@ref) helper. For some items, it also makes sense to implement the following: -- [`showitem!`](@ref)`(img, item::I)` creates a visual representation of an item on top of `img`. \ No newline at end of file +- [`DataAugmentation.showitem!`](@ref)`(img, item::I)` creates a visual representation of an item on top of `img`. diff --git a/docs/src/preprocessing.md b/docs/src/preprocessing.md index 74565847..406f6bf7 100644 --- a/docs/src/preprocessing.md +++ b/docs/src/preprocessing.md @@ -2,7 +2,7 @@ This library also implements some general transformations useful for getting data ready to be put into a model. -- [`ToEltype`](@ref)`(T)` converts the element type of any [`AbstractArrayItem`](@ref) to `T`. +- [`ToEltype`](@ref)`(T)` converts the element type of any [`DataAugmentation.AbstractArrayItem`](@ref) to `T`. - [`ImageToTensor`](@ref) converts an image to an `ArrayItem` with another dimension for the color channels - [`Normalize`](@ref) normalizes image tensors - [`OneHot`](@ref) to one-hot encode multi-class masks ([`MaskMulti`](@ref)s) diff --git a/docs/src/projective/intro.md b/docs/src/projective/intro.md index 6d29a27d..f96e4e08 100644 --- a/docs/src/projective/intro.md +++ b/docs/src/projective/intro.md @@ -30,8 +30,8 @@ By default, the bounds of a projected item will be chosen so they still encase a ## Projective transformations interface -The abstract type [`ProjectiveTransform`](@ref) represents a projective transformation. -A `ProjectiveTransform` needs to implement [`getprojection`](@ref)`(tfm, bounds; randstate)` that should return a `Transformation` from [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl). +The abstract type [`DataAugmentation.ProjectiveTransform`](@ref) represents a projective transformation. +A `ProjectiveTransform` needs to implement [`DataAugmentation.getprojection`](@ref)`(tfm, bounds; randstate)` that should return a `Transformation` from [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl). To add support for projective transformations to an item `I`, you need to implement diff --git a/docs/src/ref.md b/docs/src/ref.md index f9c4a217..d2911a73 100644 --- a/docs/src/ref.md +++ b/docs/src/ref.md @@ -12,4 +12,47 @@ Crop Rotate itemdata showitems + +DataAugmentation.AbstractArrayItem +DataAugmentation.AbstractItem +DataAugmentation.ArrayItem +DataAugmentation.Categorify +DataAugmentation.ComposedProjectiveTransform +DataAugmentation.FillMissing +DataAugmentation.Identity +DataAugmentation.Item +DataAugmentation.ItemWrapper +DataAugmentation.MapElem +DataAugmentation.Normalize +DataAugmentation.NormalizeRow +DataAugmentation.OneHot +DataAugmentation.PinOrigin +DataAugmentation.ProjectiveTransform +DataAugmentation.ResizePadDivisible +DataAugmentation.ScaleFixed +DataAugmentation.Sequence +DataAugmentation.ToEltype +DataAugmentation.Transform +DataAugmentation.Zoom +DataAugmentation.apply +DataAugmentation.apply! +DataAugmentation.boundsof +DataAugmentation.centered +DataAugmentation.compose +DataAugmentation.getbounds +DataAugmentation.getprojection +DataAugmentation.getrandstate +DataAugmentation.makebuffer +DataAugmentation.offsetcropbounds +DataAugmentation.project +DataAugmentation.project! +DataAugmentation.projectionbounds +DataAugmentation.setdata +DataAugmentation.showitem! +DataAugmentation.testapply +DataAugmentation.testapply! +DataAugmentation.testitem +DataAugmentation.testprojective +DataAugmentation.threepointwarpaffine +DataAugmentation.transformbounds ``` diff --git a/docs/src/tfminterface.md b/docs/src/tfminterface.md index 0a8dfc83..c787be47 100644 --- a/docs/src/tfminterface.md +++ b/docs/src/tfminterface.md @@ -17,7 +17,7 @@ A transformation is a type that subtypes [`Transform`](@ref). The only *required You may additionally also implement: -- [`getrandstate`](@ref)`(tfm)` for *stochastic* transformations +- [`DataAugmentation.getrandstate`](@ref)`(tfm)` for *stochastic* transformations Generates random state to be used inside `apply`. Calling `apply(tfm, item)` is equivalent to `apply(tfm, item; randstate = getrandstate(tfm))`. It defaults to `nothing`, so we need not implement it for deterministic transformations. @@ -27,7 +27,7 @@ You may additionally also implement: Buffered version of `apply` that mutates `bufitem`. If not implemented, falls back to regular `apply`. -- [`compose`](@ref)`(tfm1, tfm2)` for custom *composition* with other transformations +- [`DataAugmentation.compose`](@ref)`(tfm1, tfm2)` for custom *composition* with other transformations Composes transformations. By default, returns a [`Sequence`](@ref) transformation that applies the transformations one after the other. @@ -43,7 +43,7 @@ struct MapElem <: Transform end ``` -The `apply` implementation dispatches on [`AbstractArrayItem`](@ref), an abstract item type for items that wrap arrays. Note that the `randstate` keyword argument needs to be given even for implementations of deterministic transformations. We also make use of the [`setdata`](@ref) helper to update the item data. +The `apply` implementation dispatches on [`DataAugmentation.AbstractArrayItem`](@ref), an abstract item type for items that wrap arrays. Note that the `randstate` keyword argument needs to be given even for implementations of deterministic transformations. We also make use of the [`DataAugmentation.setdata`](@ref) helper to update the item data. ```julia function apply(tfm::MapElem, item::AbstractArrayItem; randstate = nothing) diff --git a/docs/src/transformations.md b/docs/src/transformations.md index be60330d..449776df 100644 --- a/docs/src/transformations.md +++ b/docs/src/transformations.md @@ -71,3 +71,7 @@ tfm = Maybe(FlipX()) titems = [apply(tfm, item) for _ in 1:8] showgrid(titems; ncol = 4, npad = 16) ``` + +```@docs +DataAugmentation.ImageToTensor +``` diff --git a/src/base.jl b/src/base.jl index 5961ee8d..77622401 100644 --- a/src/base.jl +++ b/src/base.jl @@ -110,10 +110,11 @@ compose(::Identity, ::Identity) = Identity() compose(tfm::Transform, ::Identity) = tfm compose(::Identity, tfm::Transform) = tfm -# Lastly, [`setdata`](@ref) provides a convenient way to create a copy -# of an item, replacing only the wrapped data. This relies on the -# wrapped data field being named `data`, though. - +""" +Provides a convenient way to create a copy +of an item, replacing only the wrapped data. This relies on the +wrapped data field being named `data`, though. +""" function setdata(item::Item, data) item = Setfield.@set item.data = data return item diff --git a/src/buffered.jl b/src/buffered.jl index a9b1fddd..3f2f023f 100644 --- a/src/buffered.jl +++ b/src/buffered.jl @@ -41,6 +41,9 @@ end # ## Buffered transforms +""" +Buffer to store transform results +""" mutable struct Buffered{T<:Transform} <: Transform tfm::T buffers::Dict @@ -73,6 +76,9 @@ function apply!(buf, buffered::Buffered, items::T; randstate = getrandstate(buff end +""" +Buffer to store transform results (threadsafe) +""" struct BufferedThreadsafe <: Transform buffereds::Vector{Buffered} function BufferedThreadsafe(tfm; n = Threads.nthreads()) diff --git a/src/projective/affine.jl b/src/projective/affine.jl index 861116b1..3205d414 100644 --- a/src/projective/affine.jl +++ b/src/projective/affine.jl @@ -108,6 +108,9 @@ end Zoom(scales::NTuple{2, T} = (1., 1.2)) where T = Zoom(Uniform(scales[1], scales[2])) +""" +Return random state of the transform +""" getrandstate(tfm::Zoom) = rand(tfm.dist) function getprojection(tfm::Zoom, bounds::Bounds{N}; randstate = getrandstate(tfm)) where N diff --git a/src/projective/base.jl b/src/projective/base.jl index dc54fe1b..43415a7c 100644 --- a/src/projective/base.jl +++ b/src/projective/base.jl @@ -2,7 +2,6 @@ abstract type ProjectiveTransform <: Transform Abstract supertype for projective transformations. See -[Projective transformations](../docs/projective/interface.md). """ abstract type ProjectiveTransform <: Transform end diff --git a/src/projective/warp.jl b/src/projective/warp.jl index 01731e73..a954f765 100644 --- a/src/projective/warp.jl +++ b/src/projective/warp.jl @@ -32,7 +32,7 @@ end """ threepointwarpaffine(srcps, dstps) -Calculate an affine [`CoordinateTransformations.LinearMap`](@ref) +Calculate an affine `CoordinateTransformations.LinearMap` from 3 source points to 3 destination points. Adapted from [CoordinateTransformations.jl#30](https://github.com/JuliaGeometry/CoordinateTransformations.jl/issues/30#issuecomment-610337378).