Skip to content

Commit

Permalink
Tables.jl 1.0 update
Browse files Browse the repository at this point in the history
  • Loading branch information
@tk3369
tk3369 committed Feb 15, 2020
1 parent 33068e9 commit e584a1c
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 49 deletions.
12 changes: 6 additions & 6 deletions Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
name = "SASLib"
uuid = "df8f2f22-cfef-5733-af3f-96770d497d85"
authors = ["Tom Kwong <[email protected]>"]
version = "1.2.0"
version = "1.2.1"

[deps]
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d"
StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68"
TableTraits = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
TabularDisplay = "3eeacb1d-13c2-54cc-9b18-30c86af3cadb"
TableTraits = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d"

[compat]
IteratorInterfaceExtensions = "0.1.1, 1"
StringEncodings = "0.3"
TableTraits = "0.4.1, 1"
Tables = "1.0"
TabularDisplay = "1"
Tables = "0.2.3"
julia = "1"
IteratorInterfaceExtensions = "0.1.1, 1"
TableTraits = "0.4.1, 1"

[extras]
InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Expand Down
17 changes: 15 additions & 2 deletions src/tables.jl
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,25 @@
# This file implements Tables interface and provide compatibility
# to the Queryverse ecosystem.

# -----------------------------------------------------------------------------
# Tables.jl implementation

Tables.istable(::Type{<:ResultSet}) = true

Tables.rowaccess(::Type{<:ResultSet}) = true
# AbstractColumns interface
Tables.columnaccess(::Type{<:ResultSet}) = true
Tables.columns(rs::ResultSet) = rs
Tables.getcolumn(rs::ResultSet, i::Int) = rs[names(rs)[i]]

# AbstractRow interface
Tables.rowaccess(::Type{<:ResultSet}) = true
Tables.rows(rs::ResultSet) = rs
Tables.columns(rs::ResultSet) = rs

# Schema definition
Tables.schema(rs::ResultSet) = Tables.Schema(names(rs), eltype.(columns(rs)))

# -----------------------------------------------------------------------------
# Queryverse compatibility

IteratorInterfaceExtensions.getiterator(rs::ResultSet) = Tables.datavaluerows(rs)
TableTraits.isiterabletable(x::ResultSet) = true
103 changes: 62 additions & 41 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ using SharedArrays: SharedArray
using Tables
import IteratorInterfaceExtensions, TableTraits

function getpath(dir, file)
function getpath(dir, file)
path = joinpath(dir, file)
#println("================ $path ================")
path
Expand Down Expand Up @@ -59,7 +59,7 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))
end

@testset "case insensitive dict" begin
function testdict(lowercase_key, mixedcase_key, second_lowercase_key)
function testdict(lowercase_key, mixedcase_key, second_lowercase_key)

T = typeof(lowercase_key)
d = SASLib.CIDict{T,Int}()
Expand Down Expand Up @@ -108,7 +108,7 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))

@testset "read basic test files (test*.sas7bdat)" begin
dir = "data_pandas"
files = filter(x -> endswith(x, "sas7bdat") && startswith(x, "test"),
files = filter(x -> endswith(x, "sas7bdat") && startswith(x, "test"),
Base.Filesystem.readdir("$dir"))
for f in files
result = readfile(dir, f)
Expand Down Expand Up @@ -150,7 +150,7 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))
rs = readsas(fname, include_columns=[:MONTH, :YEAR])
@test size(rs, 2) == 2
@test sort(names(rs)) == sort([:MONTH, :YEAR])

rs = readsas(fname, include_columns=[1, 2, 7])
@test size(rs, 2) == 3
@test sort(names(rs)) == sort([:ACTUAL, :PREDICT, :PRODUCT])
Expand All @@ -171,12 +171,12 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))

# test bad include/exclude param
# see https://discourse.julialang.org/t/test-warn-doesnt-work-with-warn-in-0-7/9001
@test_logs (:warn, "Unknown include column blah") (:warn,
@test_logs (:warn, "Unknown include column blah") (:warn,
"Unknown include column Year") readsas(fname, include_columns=[:blah, :Year])
@test_logs (:warn, "Unknown exclude column blah") (:warn,
@test_logs (:warn, "Unknown exclude column blah") (:warn,
"Unknown exclude column Year") readsas(fname, exclude_columns=[:blah, :Year])
# error handling
@test_throws SASLib.ConfigError readsas(fname,
@test_throws SASLib.ConfigError readsas(fname,
include_columns=[1], exclude_columns=[1])
end

Expand All @@ -187,8 +187,8 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))
@test size(rs) == (1440, 10)
@test size(rs,1) == 1440
@test size(rs,2) == 10
@test length(columns(rs)) == 10
@test length(names(rs)) == 10
@test length(columns(rs)) == 10
@test length(names(rs)) == 10

# cell indexing
@test rs[1][1] ≈ 925.0
Expand All @@ -204,40 +204,61 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))

# iteration
@test sum(r[1] for r in rs) ≈ 730337.0

# portion of result set
@test typeof(rs[1:2]) == SASLib.ResultSet
@test typeof(rs[:ACTUAL, :PREDICT]) == SASLib.ResultSet
@test rs[1:2][1][1] ≈ 925.0
@test rs[:ACTUAL, :PREDICT][1][1] ≈ 925.0

# setindex!
rs[1,1] = 100.0
@test rs[1,1] ≈ 100.0
rs[1,:ACTUAL] = 200.0
@test rs[1,:ACTUAL] ≈ 200.0
let rscopy = deepcopy(rs)
rs[1,1] = 100.0
@test rs[1,1] ≈ 100.0
rs[1,:ACTUAL] = 200.0
@test rs[1,:ACTUAL] ≈ 200.0
end

# display related
@test show(rs) == nothing
@test SASLib.sizestr(rs) == "1440 rows x 10 columns"

# Tables.jl interface - getproperty test
@test rs.ACTUAL == rs[:ACTUAL]
@test names(rs) == propertynames(rs)
# Tables.jl
let twocols = rs[:ACTUAL, :PREDICT]

# General
@test Tables.istable(typeof(rs))
@test Tables.rowaccess(typeof(rs))
@test Tables.columnaccess(typeof(rs))

# AbstractRow interface
let row = twocols[3] # (ACTUAL = 608.0, PREDICT = 846.0)
@test Tables.getcolumn(row, 1) ≈ 608.0
@test Tables.getcolumn(row, :ACTUAL) ≈ 608.0
@test Tables.columnnames(row) === (:ACTUAL, :PREDICT)
end

# Tables.jl coverage - indirect tests / usage
@test Tables.schema(rs).names == Tuple(names(rs))
@test Tables.schema(rs).types == Tuple(eltype.([rs[s] for s in names(rs)]))
@test length(Tables.rowtable(rs)) == 1440
@test length(Tables.columntable(rs)) == 10
@test size(Tables.matrix(rs[:ACTUAL, :PREDICT])) == (1440,2)
# AbstractColumns interface
let tab = Tables.columns(twocols)
@test Tables.getcolumn(tab, 1) isa Array{Float64,1}
@test Tables.getcolumn(tab, 1) |> size === (1440,)
@test Tables.getcolumn(tab, :ACTUAL) isa Array{Float64,1}
@test Tables.getcolumn(tab, :ACTUAL) |> size === (1440,)
@test Tables.columnnames(tab) === propertynames(twocols)
end

# Tables.jl coverage - direct tests
@test Tables.istable(typeof(rs)) === true
@test Tables.rowaccess(typeof(rs)) === true
@test Tables.columnaccess(typeof(rs)) === true
@test Tables.rows(rs) |> first |> propertynames |> Tuple == Tuple(names(rs))
@test Tables.columns(rs) |> propertynames |> Tuple == Tuple(names(rs))
# Usages
@test size(Tables.matrix(twocols)) == (1440, 2)
end

# old tests
#@test Tables.schema(rs).names == Tuple(names(rs))
#@test Tables.schema(rs).types == Tuple(eltype.([rs[s] for s in names(rs)]))
#@test length(Tables.rowtable(rs)) == 1440
#@test length(Tables.columntable(rs)) == 10
#@test size(Tables.matrix(rs[:ACTUAL, :PREDICT])) == (1440,2)
#@test Tables.rows(rs) |> first |> propertynames |> Tuple == Tuple(names(rs))
#@test Tables.columns(rs) |> propertynames |> Tuple == Tuple(names(rs))

# Queryverse integration
@test TableTraits.isiterabletable(rs) == true
Expand Down Expand Up @@ -333,18 +354,18 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))
end

@testset "array constructors" begin

rs = readfile("data_AHS2013", "homimp.sas7bdat")
@test typeof(rs[:RAS]) == SASLib.ObjectPool{String,UInt16}

# string_array_fn test for specific string columns
rs = readfile("data_AHS2013", "homimp.sas7bdat",
rs = readfile("data_AHS2013", "homimp.sas7bdat",
string_array_fn = Dict(:RAS => REGULAR_STR_ARRAY))
@test typeof(rs[:RAS]) == Array{String,1}
@test typeof(rs[:RAH]) != Array{String,1}

# string_array_fn test for all string columns
rs = readfile("data_AHS2013", "homimp.sas7bdat",
rs = readfile("data_AHS2013", "homimp.sas7bdat",
string_array_fn = Dict(:_all_ => REGULAR_STR_ARRAY))
@test typeof(rs[:RAS]) == Array{String,1}
@test typeof(rs[:RAH]) == Array{String,1}
Expand All @@ -354,13 +375,13 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))

# number_array_fn test by column name
makesharedarray(n) = SharedArray{Float64}(n)
rs = readfile("data_misc", "numeric_1000000_2.sas7bdat",
rs = readfile("data_misc", "numeric_1000000_2.sas7bdat",
number_array_fn = Dict(:f => makesharedarray))
@test typeof(rs[:f]) == SharedArray{Float64,1}
@test typeof(rs[:x]) == Array{Float64,1}

# number_array_fn test for all numeric columns
rs = readfile("data_misc", "numeric_1000000_2.sas7bdat",
rs = readfile("data_misc", "numeric_1000000_2.sas7bdat",
number_array_fn = Dict(:_all_ => makesharedarray))
@test typeof(rs[:f]) == SharedArray{Float64,1}
@test typeof(rs[:x]) == SharedArray{Float64,1}
Expand All @@ -371,31 +392,31 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))
@testset "user specified column types" begin

# normal use case
rs = readfile("data_pandas", "productsales.sas7bdat";
rs = readfile("data_pandas", "productsales.sas7bdat";
verbose_level = 0, column_types = Dict(:YEAR => Int16, :QUARTER => Int8))
@test eltype(rs[:YEAR]) == Int16
@test eltype(rs[:QUARTER]) == Int8

# error handling - warn() when a column cannot be converted
rs = readfile("data_pandas", "productsales.sas7bdat";
rs = readfile("data_pandas", "productsales.sas7bdat";
verbose_level = 0, column_types = Dict(:YEAR => Int8, :QUARTER => Int8))
@test eltype(rs[:YEAR]) == Float64
@test eltype(rs[:QUARTER]) == Int8
#TODO expect warning for :YEAR conversion

# case insensitive column symbol
rs = readfile("data_pandas", "productsales.sas7bdat";
rs = readfile("data_pandas", "productsales.sas7bdat";
verbose_level = 0, column_types = Dict(:Quarter => Int8))
@test eltype(rs[:QUARTER]) == Int8

# conversion to custom types
rs = readfile("data_pandas", "productsales.sas7bdat";
rs = readfile("data_pandas", "productsales.sas7bdat";
verbose_level = 0, column_types = Dict(:Year => YearStr))
@test eltype(rs[:YEAR]) == YearStr

# test Union type
let T = Union{Int,Missing}
rs = readfile("data_pandas", "productsales.sas7bdat";
let T = Union{Int,Missing}
rs = readfile("data_pandas", "productsales.sas7bdat";
verbose_level = 0, column_types = Dict(:Year => T))
@test eltype(rs[:YEAR]) == T
end
Expand All @@ -410,7 +431,7 @@ Base.convert(::Type{YearStr}, v::Float64) = YearStr(string(round(Int, v)))
@testset "just reads" begin
for dir in ["data_pandas", "data_reikoch", "data_AHS2013", "data_misc"]
for f in readdir(dir)
if endswith(f, ".sas7bdat") &&
if endswith(f, ".sas7bdat") &&
!(f in ["zero_variables.sas7bdat"])
rs = readfile(dir, f)
@test size(rs, 1) > 0
Expand Down

0 comments on commit e584a1c

Please sign in to comment.