Breaking: Materialize DimArray or DimStack From a Table

docs/src/tables.md

@@ -2,12 +2,22 @@
 
 [Tables.jl](https://github.com/JuliaData/Tables.jl) provides an ecosystem-wide interface to tabular data in Julia, ensuring interoperability with [DataFrames.jl](https://dataframes.juliadata.org/stable/), [CSV.jl](https://csv.juliadata.org/stable/), and hundreds of other packages that implement the standard.
 
+## Dimensional data are tables
 DimensionalData.jl implements the Tables.jl interface for `AbstractDimArray` and `AbstractDimStack`. `DimStack` layers are unrolled so they are all the same size, and dimensions loop to match the length of the largest layer.
 
 Columns are given the [`name`](@ref) of the array or stack layer, and the result of `DD.name(dimension)` for `Dimension` columns.
 
-Looping of dimensions and stack layers is done _lazily_,
-and does not allocate unless collected.
+Looping of dimensions and stack layers is done _lazily_, and does not allocate unless collected.
+
+## Materializing tables to DimArray or DimStack
+`DimArray` and `DimStack` have fallback methods to materialize any `Tables.jl`-compatible table.
+
+By default, it will treat columns such as X, Y, Z, and Band as dimensions, and other columns as data.
+Pass a `name` keyword argument to determine which column(s) are used.
+
+You have full control over which columns are dimensions - and what those dimensions look like exactly. If you pass a `Tuple` of `Symbol` or dimension types (e.g. `X`) as the second argument, those columns are treated as dimensions. Passing a `Tuple` of dimensions preserves these dimensions - with values matched to the corresponding columns.
+
+Materializing tables will worked even if the table is not ordered, and can handle missing values.
 
 ## Example
 
@@ -89,3 +99,61 @@ using CSV
 CSV.write("dimstack.csv", st)
 readlines("dimstack.csv")
 ````
+
+## Converting a DataFrame to a DimArray or DimStack
+
+The Dataframe we use will have 5 columns: X, Y, category, data1, and data2
+
+````@ansi dataframe
+df = DataFrame(st)
+````
+
+::: tabs
+
+== Create a `DimArray`
+
+Converting this DataFrame to a DimArray without other arguments will read the `category` columns as data and ignore data1 and data2:
+
+````@ansi dataframe
+DimArray(df)
+````
+
+Specify dimenion names to ensure these get treated as dimensions. Now data1 is read in instead.
+````@ansi dataframe
+DimArray(df, (X,Y,:category))
+````
+
+You can also pass in the actual dimensions.
+````@ansi dataframe
+DimArray(df, dims(st))
+````
+
+Pass in a name argument to read in data2 instead.
+````@ansi dataframe
+DimArray(df, dims(st); name = :data2)
+````
+
+== Create a `DimStack`
+
+Converting the DataFrame to a `DimStack` will by default read category, data1, and data2 as layers
+````@ansi dataframe
+DimStack(df)
+````
+
+
+Specify dimenion names to ensure these get treated as dimensions. Now data1 and data2 are layers.
+````@ansi dataframe
+DimStack(df, (X,Y,:category))
+````
+
+You can also pass in the actual dimensions. 
+````@ansi dataframe
+DimStack(df, dims(st))
+````
+
+Pass in a tuple of column names to control which columns are read.
+````@ansi dataframe
+DimStack(df, dims(st); name = (:data2,))
+````
+
+:::

src/DimensionalData.jl

@@ -92,6 +92,7 @@ const DD = DimensionalData
 # Common
 include("interface.jl")
 include("name.jl")
+include("table_ops.jl")
 
 # Arrays
 include("array/array.jl")

src/Dimensions/dimension.jl

@@ -178,6 +178,7 @@ lookup(dim::Union{DimType,Val{<:Dimension}}) = NoLookup()
 name(dim::Dimension) = name(typeof(dim))
 name(dim::Val{D}) where D = name(D)
 name(dim::Type{D}) where D<:Dimension = nameof(D)
+name(s::Symbol) = s
 
 label(x) = string(name(x))
 

src/array/array.jl

@@ -144,7 +144,8 @@ function Base.NamedTuple(A1::AbstractDimArray, As::AbstractDimArray...)
 end
 
 # undef constructor for all AbstractDimArray 
-(::Type{A})(x::UndefInitializer, dims::Dimension...; kw...) where {A<:AbstractDimArray{<:Any}} = A(x, dims; kw...)
+(::Type{A})(x::UndefInitializer, dims::Dimension...; kw...) where {A<:AbstractDimArray{T}} where T = 
+    A(x, dims; kw...)
 function (::Type{A})(x::UndefInitializer, dims::DimTuple; kw...) where {A<:AbstractDimArray{T}} where T
     basetypeof(A)(Array{T}(undef, size(dims)), dims; kw...)
 end
@@ -410,13 +411,14 @@ moves dimensions to reference dimension `refdims` after reducing operations
 
 ## Arguments
 
-- `data`: An `AbstractArray`.
+- `data`: An `AbstractArray` or a table with coordinate columns corresponding to `dims`.
 - `gen`: A generator expression. Where source iterators are `Dimension`s the dim args or kw is not needed.
 - `dims`: A `Tuple` of `Dimension`
 - `name`: A string name for the array. Shows in plots and tables.
 - `refdims`: refence dimensions. Usually set programmatically to track past
     slices and reductions of dimension for labelling and reconstruction.
 - `metadata`: `Dict` or `Metadata` object, or `NoMetadata()`
+- `selector`: The coordinate selector type to use when materializing from a table.
 
 Indexing can be done with all regular indices, or with [`Dimension`](@ref)s
 and/or [`Selector`](@ref)s. 
@@ -512,6 +514,57 @@ function DimArray(A::AbstractBasicDimArray;
     newdata = collect(data)
     DimArray(newdata, format(dims, newdata); refdims, name, metadata)
 end
+# Tables
+# Write a single column from a table with one or more coordinate columns to a DimArray
+function DimArray(table, dims; kw...)
+    # Confirm that the Tables interface is implemented
+    Tables.istable(table) || throw(ArgumentError("`obj` must be an `AbstractArray` or satisfy the `Tables.jl` interface."))
+    table = Tables.columnaccess(table) ? table : Tables.columns(table)
+    dimarray_from_table(DimArray, table, guess_dims(table, dims); kw...)
+end
+# Same as above, but guess dimension names from scratch
+function DimArray(table; kw...)
+    # Confirm that the Tables interface is implemented
+    Tables.istable(table) || throw(ArgumentError("`table` must satisfy the `Tables.jl` interface."))
+    table = Tables.columnaccess(table) ? table : Tables.columns(table)
+    # Use default dimension 
+    return dimarray_from_table(DimArray, table, guess_dims(table; kw...); kw...)
+end
+# Special-case for AbstractVectors - these might be tables
+function DimArray(data::AbstractVector, dims::Tuple; 
+    refdims=(), name=NoName(), metadata=NoMetadata(), kw...
+)
+    if !(data isa AbstractBasicDimArray) && Tables.istable(data) && 
+        all(map(d -> Dimensions.name(d) in Tables.schema(data).names, dims))
+        table = Tables.columns(data)
+        dims = guess_dims(table, dims; kw...)
+        return dimarray_from_table(DimArray, table, dims; refdims, name, metadata, kw...)
+    else
+        return DimArray(data, format(dims, data), refdims, name, metadata)
+    end
+end
+
+function dimarray_from_table(::Type{T}, table, dims; 
+    name=NoName(), 
+    selector=nothing, 
+    precision=6, 
+    missingval=missing, 
+    kw...
+) where T <: AbstractDimArray
+    # Determine row indices based on coordinate values
+    indices = coords_to_indices(table, dims; selector, atol=10.0^-precision)
+
+    # Extract the data column correspondong to `name`
+    col = name == NoName() ? data_col_names(table, dims) |> first : Symbol(name)
+    data = Tables.getcolumn(table, col)
+
+    # Restore array data
+    array = restore_array(data, indices, dims, missingval)
+
+    # Return DimArray
+    return T(array, dims, name=col; kw...)
+end
+
 """
     DimArray(f::Function, dim::Dimension; [name])
 
@@ -520,7 +573,7 @@ Apply function `f` across the values of the dimension `dim`
 the given dimension. Optionally provide a name for the result.
 """
 function DimArray(f::Function, dim::Dimension; name=Symbol(nameof(f), "(", name(dim), ")"))
-     DimArray(f.(val(dim)), (dim,); name)
+    DimArray(f.(val(dim)), (dim,); name)
 end
 
 DimArray(itr::Base.Generator; kwargs...) = rebuild(collect(itr); kwargs...)

src/stack/stack.jl

@@ -30,6 +30,11 @@ const AbstractVectorDimStack = AbstractDimStack{K,T,1} where {K,T}
 const AbstractMatrixDimStack = AbstractDimStack{K,T,2} where {K,T}
 
 (::Type{T})(st::AbstractDimStack; kw...) where T<:AbstractDimArray =
+    dimarray_from_dimstack(T, st; kw...) 
+# For ambiguity
+DimArray(st::AbstractDimStack; kw...) = dimarray_from_dimstack(DimArray, st; kw...) 
+
+dimarray_from_dimstack(T, st; kw...) =
     T([st[D] for D in DimIndices(st)]; dims=dims(st), metadata=metadata(st), kw...)
 
 data(s::AbstractDimStack) = getfield(s, :data)
@@ -101,14 +106,16 @@ and an existing stack.
 
 # Keywords
 
-Keywords are simply the fields of the stack object:
+Keywords are simply the common fields of an `AbstractDimStack` object:
 
 - `data`
 - `dims`
 - `refdims`
 - `metadata`
 - `layerdims`
 - `layermetadata`
+
+There is no promise that these keywords will be used in all cases.
 """
 function rebuild_from_arrays(
     s::AbstractDimStack{Keys}, das::Tuple{Vararg{AbstractBasicDimArray}}; kw...
@@ -340,6 +347,7 @@ end
 """
     DimStack <: AbstractDimStack
 
+    DimStack(table, [dims]; kw...)
     DimStack(data::AbstractDimArray...; kw...)
     DimStack(data::Union{AbstractArray,Tuple,NamedTuple}, [dims::DimTuple]; kw...)
     DimStack(data::AbstractDimArray; layersfrom, kw...)
@@ -512,14 +520,17 @@ function DimStack(das::NamedTuple{<:Any,<:Tuple{Vararg{AbstractDimArray}}};
 end
 DimStack(data::Union{Tuple,AbstractArray,NamedTuple}, dim::Dimension; name=uniquekeys(data), kw...) = 
     DimStack(NamedTuple{Tuple(name)}(data), (dim,); kw...)
-DimStack(data::Union{Tuple,AbstractArray}, dims::Tuple; name=uniquekeys(data), kw...) = 
+DimStack(data::Union{Tuple,AbstractArray{<:AbstractArray}}, dims::Tuple; name=uniquekeys(data), kw...) = 
     DimStack(NamedTuple{Tuple(name)}(data), dims; kw...)
 function DimStack(data::NamedTuple{K}, dims::Tuple;
     refdims=(), 
     metadata=NoMetadata(),
     layermetadata=nothing,
     layerdims=nothing
 ) where K
+    if length(data) > 0 && Tables.istable(data) && all(d -> name(d) in keys(data), dims)
+        return dimstack_from_table(DimStack, data, dims; refdims, metadata)
+    end
     layerdims = if isnothing(layerdims) 
         all(map(d -> axes(d) == axes(first(data)), data)) || _stack_size_mismatch()
         map(_ -> basedims(dims), data)
@@ -546,6 +557,53 @@ function DimStack(st::AbstractDimStack;
     DimStack(data, dims, refdims, layerdims, metadata, layermetadata)
 end
 
+# Write each column from a table with one or more coordinate columns to a layer in a DimStack
+function DimStack(data, dims::Tuple; kw...
+)
+    if Tables.istable(data)
+        table = Tables.columns(data)
+        all(map(d -> Dimensions.name(d) in Tables.columnnames(table), dims)) || throw(ArgumentError(
+            "All dimensions in dims must be in the table columns."
+        ))
+        dims = guess_dims(table, dims; kw...)
+        return dimstack_from_table(DimStack, table, dims; kw...)
+    else
+        throw(ArgumentError(
+            """data must be a table with coordinate columns, an AbstractArray, 
+            or a Tuple or NamedTuple of AbstractArrays"""
+        ))
+
+    end
+end
+function DimStack(table; kw...)
+    if Tables.istable(table)
+        table = Tables.columns(table)
+        dimstack_from_table(DimStack, table, guess_dims(table; kw...); kw...)
+    else
+        throw(ArgumentError(
+            """data must be a table with coordinate columns, an AbstractArray, 
+            or a Tuple or NamedTuple of AbstractArrays"""
+        ))    end
+end
+
+function dimstack_from_table(::Type{T}, table, dims; 
+    name=nothing, 
+    selector=nothing, 
+    precision=6, 
+    missingval=missing, 
+    kw...
+) where T<:AbstractDimStack
+    table = Tables.columnaccess(table) ? table : Tables.columns(table)
+    data_cols = isnothing(name) ? data_col_names(table, dims) : name
+    dims = guess_dims(table, dims; precision)
+    indices = coords_to_indices(table, dims; selector)
+    layers = map(data_cols) do col
+        d = Tables.getcolumn(table, col)
+        restore_array(d, indices, dims, missingval)
+    end
+    return T(layers, dims; name = data_cols, kw...)
+end
+
 layerdims(s::DimStack{<:Any,<:Any,<:Any,<:Any,<:Any,<:Any,Nothing}, name::Symbol) = dims(s)
 
 ### Skipmissing on DimStacks
@@ -573,4 +631,4 @@ Base.eltype(::Type{Base.SkipMissing{T}}) where {T<:AbstractDimStack{<:Any, NT}}
     _nonmissing_nt(NT)
 
 @generated _nonmissing_nt(NT::Type{<:NamedTuple{K,V}}) where {K,V} =
-    NamedTuple{K, Tuple{map(Base.nonmissingtype, V.parameters)...}}
+    NamedTuple{K, Tuple{map(Base.nonmissingtype, V.parameters)...}}