Ordered structs (JuliaIO#224)

* OrderedDict support for MatlabClassObject

* MatlabStructArray integer indexing

Author: matthijscox

Project.toml

@@ -1,22 +1,22 @@
 name = "MAT"
 uuid = "23992714-dd62-5051-b70f-ba57cb901cac"
-version = "0.11.2"
+version = "0.11.3"
 
 [deps]
-BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
-BufferedStreams = "0.4.1, 1"
 CodecZlib = "0.5, 0.6, 0.7"
 Dates = "1"
 HDF5 = "0.16, 0.17"
+OrderedCollections = "1"
 PooledArrays = "1.4.3"
 StringEncodings = "0.3.7"
 Tables = "1.12.1"

docs/src/object_arrays.md

@@ -84,34 +84,35 @@ Note that before v0.11 MAT.jl will read struct arrays as a Dict with concatenate
 
 You can write an old class object with the `MatlabClassObject` and arrays of objects with `MatlabStructArray` by providing the class name. These are also the types you obtain when you read files.
 
+> Please note that the order of the fields is important for MatlabClassObjects to be read properly in MATLAB. You may get `Warning: Fields of object 'tc' do not match the current constructor definition for class 'TestClass'. The object has been converted to a structure.`. Consider using `OrderedDict` if you have multiple fields.
+
 Write a single class object:
 ```julia
-d = Dict("foo" => 5.0)
-obj = MatlabClassObject(d, "TestClassOld")
-matwrite("matfile.mat", Dict("tc_old" => obj))
+d = Dict{String,Any}("foo" => 5.0)
+obj = MatlabClassObject(d, "TestClass")
+matwrite("matfile.mat", Dict("tc" => obj))
 ```
 
 A class object array
 ```julia
-class_array = MatlabStructArray(["foo"], [[5.0, "bar"]], "TestClassOld")
+class_array = MatlabStructArray(["foo"], [[5.0, "bar"]], "TestClass")
 matwrite("matfile.mat", Dict("class_array" => class_array))
 ```
 
 Also a class object array, but will be converted to `MatlabStructArray` internally:
 ```julia
 class_array = MatlabClassObject[
-    MatlabClassObject(Dict("foo" => 5.0), "TestClassOld"),
-    MatlabClassObject(Dict("foo" => "bar"), "TestClassOld")
+    MatlabClassObject(Dict{String,Any}("foo" => 5.0), "TestClass"),
+    MatlabClassObject(Dict{String,Any}("foo" => "bar"), "TestClass")
 ]
 matwrite("matfile.mat", Dict("class_array" => class_array))
 ```
 
 A cell array:
 ```julia
 cell_array = Any[
-    MatlabClassObject(Dict("foo" => 5.0), "TestClassOld"),
-    MatlabClassObject(Dict("a" => "bar"), "AnotherClass")
+    MatlabClassObject(Dict{String,Any}("foo" => 5.0), "TestClass"),
+    MatlabClassObject(Dict{String,Any}("a" => "bar"), "AnotherClass")
 ]
 matwrite("matfile.mat", Dict("cell_array" => cell_array))
 ```
-

src/MAT_HDF5.jl

@@ -660,6 +660,7 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::M
             else
                 write_attribute(g, name_type_attr_matlab, arr.class)
                 write_attribute(g, object_decode_attr_matlab, UInt32(2))
+                write_attribute(g, "MATLAB_fields", HDF5.VLen(arr.names))
             end
             for (fieldname, field_values) in arr
                 refs = _write_references!(mfile, parent, field_values)
@@ -699,7 +700,9 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M
     try
         write_attribute(g, name_type_attr_matlab, obj.class)
         write_attribute(g, object_decode_attr_matlab, UInt32(2))
-        for (ki, vi) in zip(keys(obj), values(obj))
+        all_keys = collect(keys(obj))
+        write_attribute(g, "MATLAB_fields", HDF5.VLen(all_keys))
+        for (ki, vi) in zip(all_keys, values(obj))
             m_write(mfile, g, ki, vi)
         end
     finally

src/MAT_subsys.jl

@@ -754,7 +754,7 @@ function set_fwrap_data!(subsys::Subsystem)
     push!(fwrap_data, reshape(subsys.mcos_class_alias_metadata, :, 1))
     push!(fwrap_data, reshape(subsys.prop_vals_defaults, :, 1))
 
-    fw_obj = MatlabOpaque(Dict("__filewrapper__" => reshape(fwrap_data, :, 1)), "FileWrapper__")
+    fw_obj = MatlabOpaque(Dict{String,Any}("__filewrapper__" => reshape(fwrap_data, :, 1)), "FileWrapper__")
     return fw_obj
 end
 

src/MAT_types.jl

@@ -34,6 +34,7 @@ import Dates
 import Dates: DateTime, Second, Millisecond
 import PooledArrays: PooledArray, RefArray
 using Tables: Tables
+import OrderedCollections: OrderedDict
 
 export MatlabStructArray, StructArrayField, convert_struct_array
 export MatlabClassObject
@@ -68,7 +69,7 @@ using MAT
 s_arr = MatlabStructArray(["a", "b"], [[1, 2],["foo", 5]])
 
 # write-read
-matwrite("matfile.mat", Dict("struct_array" => s_arr))
+matwrite("matfile.mat", Dict{String,Any}("struct_array" => s_arr))
 read_s_arr = matread("matfile.mat")["struct_array"]
 
 # convert to Dict Array
@@ -144,6 +145,8 @@ Base.haskey(arr::MatlabStructArray, k::AbstractString) = k in keys(arr)
 function Base.copy(arr::MatlabStructArray{N}) where {N}
     return MatlabStructArray{N}(copy(arr.names), copy(arr.values))
 end
+class(arr::MatlabStructArray) = arr.class
+class(d::AbstractDict) = ""
 
 function Base.iterate(arr::T, i=next_state(arr)) where {T<:MatlabStructArray}
     if i == 0
@@ -255,6 +258,16 @@ function Base.Array{D,N}(arr::MatlabStructArray{N}) where {D<:AbstractDict,N}
     return result
 end
 
+function Base.getindex(arr::MatlabStructArray, s::Integer)
+    row_values = [getindex(v, s) for v in arr.values]
+    if isempty(arr.class)
+        D = Dict{String,Any}
+    else
+        D = OrderedDict{String,Any}
+    end
+    return create_struct(D, arr.names, row_values, arr.class)
+end
+
 function create_struct(::Type{D}, keys, values, class::String) where {T,D<:AbstractDict{T}}
     return D(T.(keys) .=> values)
 end
@@ -284,7 +297,7 @@ Internal Marker for Empty Structs with dimensions like 1x0 or 0x0
 struct EmptyStruct
    dims::Vector{UInt64}
 end
-
+class(m::EmptyStruct) = ""
 
 """
     MatlabClassObject(
@@ -296,11 +309,13 @@ Type to store old class objects. Inside MATLAB a class named \"TestClassOld\" wo
 
 If you want to write these objects you have to make sure the keys in the Dict match the class defined properties/fields.
 """
-struct MatlabClassObject <: AbstractDict{String,Any}
-    d::Dict{String,Any}
+struct MatlabClassObject{D<:AbstractDict{String,Any}} <: AbstractDict{String,Any}
+    d::D
     class::String
 end
 
+class(m::MatlabClassObject) = m.class
+
 Base.eltype(::Type{MatlabClassObject}) = Pair{String,Any}
 Base.length(m::MatlabClassObject) = length(m.d)
 Base.keys(m::MatlabClassObject) = keys(m.d)
@@ -320,7 +335,7 @@ function Base.isapprox(m1::MatlabClassObject, m2::MatlabClassObject; kwargs...)
     return m1.class == m2.class && dict_isapprox(m1.d, m2.d; kwargs...)
 end
 
-function MatlabStructArray(arr::AbstractArray{MatlabClassObject})
+function MatlabStructArray(arr::AbstractArray{<:MatlabClassObject})
     first_obj, remaining_obj = Iterators.peel(arr)
     class = first_obj.class
     if !all(x -> isequal(class, x.class), remaining_obj)
@@ -331,7 +346,7 @@ function MatlabStructArray(arr::AbstractArray{MatlabClassObject})
     return MatlabStructArray(arr, class)
 end
 
-function convert_struct_array(d::Dict{String,Any}, class::String="")
+function convert_struct_array(d::AbstractDict{String,Any}, class::String="")
     # there is no possibility of having cell arrays mixed with struct arrays (afaik)
     field_values = first(values(d))
     if field_values isa StructArrayField
@@ -351,13 +366,15 @@ function Base.Array(arr::MatlabStructArray{N}) where {N}
     if isempty(arr.class)
         return Array{Dict{String,Any},N}(arr)
     else
-        return Array{MatlabClassObject,N}(arr)
+        # ordered dict by default, to preserve field order
+        D = OrderedDict{String,Any}
+        return Array{MatlabClassObject{D},N}(arr)
     end
 end
 
-function create_struct(::Type{D}, keys, values, class::String) where {D<:MatlabClassObject}
-    d = Dict{String,Any}(string.(keys) .=> values)
-    return MatlabClassObject(d, class)
+function create_struct(::Type{MatlabClassObject{D}}, keys, values, class::String) where {D<:AbstractDict}
+    d = D(string.(keys) .=> values)
+    return MatlabClassObject{D}(d, class)
 end
 
 """
@@ -370,10 +387,11 @@ Type to store opaque class objects.
 These are the 'modern' Matlab classes, different from the old `MatlabClassObject` types.
 
 """
-struct MatlabOpaque <: AbstractDict{String,Any}
-    d::Dict{String,Any}
+struct MatlabOpaque{D<:AbstractDict{String,Any}} <: AbstractDict{String,Any}
+    d::D
     class::String
 end
+class(m::MatlabOpaque) = m.class
 
 Base.eltype(::Type{MatlabOpaque}) = Pair{String,Any}
 Base.length(m::MatlabOpaque) = length(m.d)
@@ -492,7 +510,7 @@ end
 
 function MatlabOpaque(d::ScalarOrArray{DateTime})
     return MatlabOpaque(
-        Dict(
+        Dict{String,Any}(
             "tz"   => "",
             "data" => map_or_not(to_matlab_data, d),
             "fmt"  => "",
@@ -514,7 +532,7 @@ to_matlab_millis(d::Millisecond) = Float64(Dates.value(d))
 
 function MatlabOpaque(d::ScalarOrArray{Millisecond})
     return MatlabOpaque(
-        Dict(
+        Dict{String,Any}(
             "millis" => map_or_not(to_matlab_millis, d),
         ),
         "duration"

src/MAT_v4.jl

@@ -27,7 +27,7 @@
 # http://www.mathworks.com/help/pdf_doc/matlab/matfile_format.pdf
 
 module MAT_v4
-using BufferedStreams, HDF5, SparseArrays
+using HDF5, SparseArrays
 import Base: read, write, close
 
 round_uint8(data) = round.(UInt8, data)

src/MAT_v5.jl

@@ -26,7 +26,7 @@
 # http://www.mathworks.com/help/pdf_doc/matlab/matfile_format.pdf
 
 module MAT_v5
-using CodecZlib, BufferedStreams, HDF5, SparseArrays
+using CodecZlib, HDF5, SparseArrays
 import Base: read, write, close
 import ..MAT_types: MatlabStructArray, MatlabClassObject, MatlabTable
 

test/types.jl

@@ -64,7 +64,7 @@ using Dates
     # class object array conversion
     s_arr_class = MatlabStructArray(d_arr, "TestClass")
     c_arr = Array(s_arr_class)
-    @test c_arr isa Array{MatlabClassObject}
+    @test c_arr isa Array{<:MatlabClassObject}
     @test all(c->c.class=="TestClass", c_arr)
     @test MatlabStructArray(c_arr) == s_arr_class
     @test s_arr_class != s_arr
@@ -78,6 +78,28 @@ using Dates
     @test_throws ErrorException(msg) MatlabStructArray(wrong_sarr)
 end
 
+@testset "MatlabStructArray integer indexing" begin
+    d_arr = Dict{String, Any}[
+        Dict("x"=>[1.0,2.0], SubString("y")=>3.0),
+        Dict("x"=>[5.0,6.0], "y"=>[])
+    ]
+    s_arr = MatlabStructArray(d_arr)
+    @test s_arr[1] == Array(s_arr)[1]
+    @test s_arr[2] == Array(s_arr)[2]
+
+    s_arr = MatlabStructArray(reshape(d_arr, 1, 2))
+    @test s_arr[1] == Array(s_arr)[1]
+    @test s_arr[2] == Array(s_arr)[2]
+
+    s_arr = MatlabStructArray(reshape(d_arr, 2, 1))
+    @test s_arr[1] == Array(s_arr)[1]
+    @test s_arr[2] == Array(s_arr)[2]
+
+    s_arr = MatlabStructArray(reshape(d_arr, 2, 1), "TestClass")
+    @test s_arr[1] == Array(s_arr)[1]
+    @test s_arr[2] == Array(s_arr)[2]
+end
+
 @testset "MatlabClassObject" begin
     d = Dict{String,Any}("a" => 5)
     obj = MatlabClassObject(d, "TestClassOld")
@@ -189,7 +211,7 @@ end
 end
 
 @testset "MatlabOpaque duration" begin
-    d = Dict(
+    d = Dict{String,Any}(
         "millis" => [3.6e6 7.2e6],
         # "fmt"    => 'h' # optional format
     )
@@ -198,7 +220,7 @@ end
     @test ms == map(Millisecond, d["millis"])
     @test MatlabOpaque(ms) == obj
 
-    d = Dict(
+    d = Dict{String,Any}(
         "millis" => 12000.0,
         # "fmt"    => 'h',
     )
@@ -209,7 +231,7 @@ end
 end
 
 @testset "MatlabOpaque categorical" begin
-    d = Dict(
+    d = Dict{String,Any}(
         "isProtected"   => false,
         "codes"         => reshape(UInt8[0x02, 0x03, 0x01, 0x01, 0x01, 0x02], 3, 2),
         "categoryNames" => Any["Fair"; "Good"; "Poor";;],