Skip to content

Commit

Permalink
Merge pull request #22 from davidanthoff/new-fileio
Browse files Browse the repository at this point in the history
New FileIO
  • Loading branch information
davidanthoff authored May 23, 2018
2 parents 4350a1a + 64896e9 commit f13713d
Show file tree
Hide file tree
Showing 5 changed files with 25 additions and 27 deletions.
12 changes: 6 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ Use ``Pkg.add("CSVFiles")`` in Julia to install CSVFiles and its dependencies.
To read a CSV file into a ``DataFrame``, use the following julia code:

````julia
using FileIO, CSVFiles, DataFrames
using CSVFiles, DataFrames

df = DataFrame(load("data.csv"))
````

The call to ``load`` returns a ``struct`` that is an [IterableTable.jl](https://github.com/davidanthoff/IterableTables.jl), so it can be passed to any function that can handle iterable tables, i.e. all the sinks in [IterableTable.jl](https://github.com/davidanthoff/IterableTables.jl). Here are some examples of materializing a CSV file into data structures that are not a ``DataFrame``:

````julia
using FileIO, CSVFiles, DataTables, IndexedTables, TimeSeries, Temporal, Gadfly
using CSVFiles, DataTables, IndexedTables, TimeSeries, Temporal, Gadfly

# Load into a DataTable
dt = DataTable(load("data.csv"))
Expand Down Expand Up @@ -81,15 +81,15 @@ These are simply the arguments from [TextParse.jl](https://github.com/JuliaCompu

The following code saves any iterable table as a CSV file:
````julia
using FileIO, CSVFiles
using CSVFiles

save("output.csv", it)
````
This will work as long as ``it`` is any of the types supported as sources in [IterableTables.jl](https://github.com/davidanthoff/IterableTables.jl).

One can also save into an arbitrary stream:
````julia
using FileIO, CSVFiles
using CSVFiles

save(Stream(format"CSV", io), it)
````
Expand All @@ -111,15 +111,15 @@ save(f::FileIO.File{FileIO.format"CSV"}, data; delim=',', quotechar='"', escapec
Both ``load`` and ``save`` also support the pipe syntax. For example, to load a CSV file into a ``DataFrame``, one can use the following code:

````julia
using FileIO, CSVFiles, DataFrame
using CSVFiles, DataFrame

df = load("data.csv") |> DataFrame
````

To save an iterable table, one can use the following form:

````julia
using FileIO, CSVFiles, DataFrame
using CSVFiles, DataFrame

df = # Aquire a DataFrame somehow

Expand Down
2 changes: 1 addition & 1 deletion REQUIRE
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ IteratorInterfaceExtensions 0.0.2
TableTraits 0.0.3
TableTraitsUtils 0.1.3
DataValues 0.1.0
FileIO 0.4.0
FileIO 0.9.0
HTTP 0.6.0
IterableTables 0.6.1
14 changes: 7 additions & 7 deletions src/CSVFiles.jl
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
module CSVFiles

using TextParse, IteratorInterfaceExtensions, TableTraits, TableTraitsUtils,
DataValues
import FileIO
using HTTP
DataValues, FileIO, HTTP
import IterableTables

export load, save

struct CSVFile
filename::String
delim
Expand All @@ -18,19 +18,19 @@ struct CSVStream
keywords
end

function load(f::FileIO.File{FileIO.format"CSV"}, delim=','; args...)
function fileio_load(f::FileIO.File{FileIO.format"CSV"}, delim=','; args...)
return CSVFile(f.filename, delim, args)
end

function load(f::FileIO.File{FileIO.format"TSV"}, delim='\t'; args...)
function fileio_load(f::FileIO.File{FileIO.format"TSV"}, delim='\t'; args...)
return CSVFile(f.filename, delim, args)
end

function load(s::FileIO.Stream{FileIO.format"CSV"}, delim=','; args...)
function fileio_load(s::FileIO.Stream{FileIO.format"CSV"}, delim=','; args...)
return CSVStream(s.io, delim, args)
end

function load(s::FileIO.Stream{FileIO.format"TSV"}, delim='\t'; args...)
function fileio_load(s::FileIO.Stream{FileIO.format"TSV"}, delim='\t'; args...)
return CSVStream(s.io, delim, args)
end

Expand Down
8 changes: 4 additions & 4 deletions src/csv_writer.jl
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,18 @@ function _save(filename::AbstractString, data; delim=',', quotechar='"', escapec
end
end

function save(f::FileIO.File{FileIO.format"CSV"}, data; delim=',', quotechar='"', escapechar='\\', header=true)
function fileio_save(f::FileIO.File{FileIO.format"CSV"}, data; delim=',', quotechar='"', escapechar='\\', header=true)
return _save(f.filename, data, delim=delim, quotechar=quotechar, escapechar=escapechar, header=header)
end

function save(f::FileIO.File{FileIO.format"TSV"}, data; delim='\t', quotechar='"', escapechar='\\', header=true)
function fileio_save(f::FileIO.File{FileIO.format"TSV"}, data; delim='\t', quotechar='"', escapechar='\\', header=true)
return _save(f.filename, data, delim=delim, quotechar=quotechar, escapechar=escapechar, header=header)
end

function save(s::FileIO.Stream{FileIO.format"CSV"}, data; delim=',', quotechar='"', escapechar='\\', header=true)
function fileio_save(s::FileIO.Stream{FileIO.format"CSV"}, data; delim=',', quotechar='"', escapechar='\\', header=true)
return _save(s.io, data, delim=delim, quotechar=quotechar, escapechar=escapechar, header=header)
end

function save(s::FileIO.Stream{FileIO.format"TSV"}, data; delim='\t', quotechar='"', escapechar='\\', header=true)
function fileio_save(s::FileIO.Stream{FileIO.format"TSV"}, data; delim='\t', quotechar='"', escapechar='\\', header=true)
return _save(s.io, data, delim=delim, quotechar=quotechar, escapechar=escapechar, header=header)
end
16 changes: 7 additions & 9 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
using FileIO
using CSVFiles
using TableTraits
using NamedTuples
using DataValues
using Base.Test
Expand All @@ -26,7 +24,7 @@ end

csvf = load(joinpath(@__DIR__, "data.csv"))

@test isiterable(csvf) == true
@test IteratorInterfaceExtensions.isiterable(csvf) == true
@test TableTraits.isiterabletable(csvf) == true

array3 = [@NT(a=DataValue(3),b="df\"e"),@NT(a=DataValue{Int}(),b="something")]
Expand Down Expand Up @@ -70,24 +68,24 @@ data = [@NT(Name="John",Age=34.,Children=2),@NT(Name="Sally",Age=54.,Children=1)

stream = IOBuffer()
mark(stream)
fileiostream = FileIO.Stream(format"CSV", stream)
fileiostream = FileIO.Stream(FileIO.format"CSV", stream)
save(fileiostream, data)
reset(stream)
csvstream = load(fileiostream)
reloaded_data = collect(csvstream)
@test isiterable(csvstream)
@test isiterabletable(csvstream)
@test IteratorInterfaceExtensions.isiterable(csvstream)
@test TableTraits.isiterabletable(csvstream)
@test reloaded_data == data

stream = IOBuffer()
mark(stream)
fileiostream = FileIO.Stream(format"TSV", stream)
fileiostream = FileIO.Stream(FileIO.format"TSV", stream)
save(fileiostream, data)
reset(stream)
csvstream = load(fileiostream)
reloaded_data = collect(csvstream)
@test isiterable(csvstream)
@test isiterabletable(csvstream)
@test IteratorInterfaceExtensions.isiterable(csvstream)
@test TableTraits.isiterabletable(csvstream)
@test reloaded_data == data

end
Expand Down

0 comments on commit f13713d

Please sign in to comment.