Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move load json & csv to use v2 protocol #38

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Move load json & csv to use v2 protocol #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
647205b
move load json & csv to use v2 protocol
NRHelmi Oct 13, 2022
5dd81cc
fix
NRHelmi Oct 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions RelationalAI.Test/DatabaseTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,7 @@ public async Task DatabaseCloneTest()

// load some data and model
var loadRsp = await client.LoadJsonAsync(Dbname, EngineName, "test_data", TestJson);
Assert.False(loadRsp.Aborted);
Assert.Empty(loadRsp.Output);
Assert.Equal(TransactionAsyncState.Completed, loadRsp.Transaction.State);
Assert.Empty(loadRsp.Problems);

var resp = await client.LoadModelsAsync(Dbname, EngineName, TestModel);
Expand Down
255 changes: 41 additions & 214 deletions RelationalAI.Test/LoadCsvTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,67 +19,26 @@ public class LoadCsvTests : UnitTest
"\"bellini\",3,12.25,\"2020-04-04\"\n";

[Fact]
public async Task LoadCsvtTest()
public async Task LoadCsvTest()
{
var client = CreateClient();

await client.CreateEngineWaitAsync(EngineName);
await client.CreateDatabaseAsync(Dbname, EngineName);

var loadRsp = await client.LoadCsvAsync(Dbname, EngineName, "sample", Sample);
Assert.False(loadRsp.Aborted);
Assert.Empty(loadRsp.Output);
Assert.Equal(TransactionAsyncState.Completed, loadRsp.Transaction.State);
Assert.Empty(loadRsp.Problems);

var rsp = await client.ExecuteV1Async(Dbname, EngineName, "def output = sample");

var rel = FindRelation(rsp.Output, ":date");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":price");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"12.50", "14.25", "11.00", "12.25"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":quantity");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"2", "4", "4", "3"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":cocktail");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"martini", "sazerac", "cosmopolitan", "bellini"}
},
rel.Columns
);
var rsp = await client.ExecuteWaitAsync(Dbname, EngineName, "def output = sample");
Assert.Equal(rsp.Results[0].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[1].Table, new List<object> { "martini", "sazerac", "cosmopolitan", "bellini" });
Assert.Equal(rsp.Results[2].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[3].Table, new List<object> { "2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04" });
Assert.Equal(rsp.Results[4].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[5].Table, new List<object> { "12.50", "14.25", "11.00", "12.25" });
Assert.Equal(rsp.Results[6].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[7].Table, new List<object> { "2", "4", "4", "3" });
}

private const string SampleNoHeader = "" +
Expand All @@ -98,60 +57,18 @@ public async Task LoadCsvNoHeaderTest()

var opts = new CsvOptions().WithHeaderRow(0);
var loadRsp = await client.LoadCsvAsync(Dbname, EngineName, "sample_no_header", SampleNoHeader, opts);
Assert.False(loadRsp.Aborted);
Assert.Empty(loadRsp.Output);
Assert.Equal(TransactionAsyncState.Completed, loadRsp.Transaction.State);
Assert.Empty(loadRsp.Problems);

var rsp = await client.ExecuteV1Async(Dbname, EngineName, "def output = sample_no_header");

var rel = FindRelation(rsp.Output, ":COL1");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {1L, 2L, 3L, 4L},
new object[] {"martini", "sazerac", "cosmopolitan", "bellini"}
},
rel.Columns
);


rel = FindRelation(rsp.Output, ":COL2");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {1L, 2L, 3L, 4L},
new object[] {"2", "4", "4", "3"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":COL3");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {1L, 2L, 3L, 4L},
new object[] {"12.50", "14.25", "11.00", "12.25"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":COL4");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {1L, 2L, 3L, 4L},
new object[] {"2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04"}
},
rel.Columns
);
var rsp = await client.ExecuteWaitAsync(Dbname, EngineName, "def output = sample_no_header");
Assert.Equal(rsp.Results[0].Table, new List<object> { 1L, 2L, 3L, 4L });
Assert.Equal(rsp.Results[1].Table, new List<object> { "martini", "sazerac", "cosmopolitan", "bellini" });
Assert.Equal(rsp.Results[2].Table, new List<object> { 1L, 2L, 3L, 4L });
Assert.Equal(rsp.Results[3].Table, new List<object> { "2", "4", "4", "3" });
Assert.Equal(rsp.Results[4].Table, new List<object> { 1L, 2L, 3L, 4L });
Assert.Equal(rsp.Results[5].Table, new List<object> { "12.50", "14.25", "11.00", "12.25" });
Assert.Equal(rsp.Results[6].Table, new List<object> { 1L, 2L, 3L, 4L });
Assert.Equal(rsp.Results[7].Table, new List<object> { "2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04" });
}

private const string SampleAltSyntax = "" +
Expand All @@ -171,59 +88,18 @@ public async Task LoadCsvAltSyntaxTest()

var opts = new CsvOptions().WithDelim('|').WithQuoteChar('\'');
var loadRsp = await client.LoadCsvAsync(Dbname, EngineName, "sample_alt_syntax", SampleAltSyntax, opts);
Assert.False(loadRsp.Aborted);
Assert.Empty(loadRsp.Output);
Assert.Equal(TransactionAsyncState.Completed, loadRsp.Transaction.State);
Assert.Empty(loadRsp.Problems);

var rsp = await client.ExecuteV1Async(Dbname, EngineName, "def output = sample_alt_syntax");

var rel = FindRelation(rsp.Output, ":date");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":price");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"12.50", "14.25", "11.00", "12.25"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":quantity");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"2", "4", "4", "3"}
},
rel.Columns
);

rel = FindRelation(rsp.Output, ":cocktail");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"martini", "sazerac", "cosmopolitan", "bellini"}
},
rel.Columns
);
var rsp = await client.ExecuteWaitAsync(Dbname, EngineName, "def output = sample_alt_syntax");
Assert.Equal(rsp.Results[0].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[1].Table, new List<object> { "martini", "sazerac", "cosmopolitan", "bellini" });
Assert.Equal(rsp.Results[2].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[3].Table, new List<object> { "2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04" });
Assert.Equal(rsp.Results[4].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[5].Table, new List<object> { "12.50", "14.25", "11.00", "12.25" });
Assert.Equal(rsp.Results[6].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[7].Table, new List<object> { "2", "4", "4", "3" });
}

[Fact]
Expand All @@ -244,67 +120,18 @@ public async Task LoadCsvWithSchemaTest()

var opts = new CsvOptions().WithSchema(schema);
var loadRsp = await client.LoadCsvAsync(Dbname, EngineName, "sample", Sample, opts);
Assert.False(loadRsp.Aborted);
Assert.Empty(loadRsp.Output);
Assert.Equal(TransactionAsyncState.Completed, loadRsp.Transaction.State);
Assert.Empty(loadRsp.Problems);

var rsp = await client.ExecuteV1Async(Dbname, EngineName, "def output = sample");

var rel = FindRelation(rsp.Output, ":date");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"2020-01-01", "2020-02-02", "2020-03-03", "2020-04-04"}
},
rel.Columns
);
Assert.Single(rel.RelKey.Values);
Assert.Equal("Dates.Date", rel.RelKey.Values[0]);

rel = FindRelation(rsp.Output, ":price");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {12.5, 14.25, 11.00, 12.25}
},
rel.Columns
);
Assert.Single(rel.RelKey.Values);
Assert.Equal("FixedPointDecimals.FixedDecimal{Int64, 2}", rel.RelKey.Values[0]);

rel = FindRelation(rsp.Output, ":quantity");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {2L, 4L, 4L, 3L}
},
rel.Columns
);
Assert.Single(rel.RelKey.Values);
Assert.Equal("Int64", rel.RelKey.Values[0]);

rel = FindRelation(rsp.Output, ":cocktail");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(
new[]
{
new object[] {2L, 3L, 4L, 5L},
new object[] {"martini", "sazerac", "cosmopolitan", "bellini"}
},
rel.Columns
);
Assert.Single(rel.RelKey.Values);
Assert.Equal("String", rel.RelKey.Values[0]);
var rsp = await client.ExecuteWaitAsync(Dbname, EngineName, "def output = sample");
Assert.Equal(rsp.Results[0].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[1].Table, new List<object> { "martini", "sazerac", "cosmopolitan", "bellini" });
Assert.Equal(rsp.Results[2].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[3].Table, new List<object> { 737425L, 737457L, 737487L, 737519L });
Assert.Equal(rsp.Results[4].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[5].Table, new List<object> { 1250L, 1425L, 1100L, 1225L });
Assert.Equal(rsp.Results[6].Table, new List<object> { 2L, 3L, 4L, 5L });
Assert.Equal(rsp.Results[7].Table, new List<object> { 2L, 4L, 4L, 3L });
}

public override async Task DisposeAsync()
Expand Down
30 changes: 6 additions & 24 deletions RelationalAI.Test/LoadJsonTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,39 +17,21 @@ public class LoadJsonTests : UnitTest
"\"pets\":[\"dog\",\"rabbit\"]}";

[Fact]
public async Task LoadJsontTest()
public async Task LoadJsonTest()
{
var client = CreateClient();

await client.CreateEngineWaitAsync(EngineName);
await client.CreateDatabaseAsync(Dbname, EngineName);

var loadRsp = await client.LoadJsonAsync(Dbname, EngineName, "sample", Sample);
Assert.False(loadRsp.Aborted);
Assert.Empty(loadRsp.Output);
Assert.Equal(TransactionAsyncState.Completed, loadRsp.Transaction.State);
Assert.Empty(loadRsp.Problems);

var rsp = await client.ExecuteV1Async(Dbname, EngineName, "def output = sample");

var rel = FindRelation(rsp.Output, ":name");
Assert.NotNull(rel);
Assert.Single(rel.Columns);
Assert.Equal(new[] { new object[] { "Amira" } }, rel.Columns);

rel = FindRelation(rsp.Output, ":age");
Assert.NotNull(rel);
Assert.Single(rel.Columns);
Assert.Equal(new[] { new object[] { 32L } }, rel.Columns);

rel = FindRelation(rsp.Output, ":height");
Assert.NotNull(rel);
Assert.Single(rel.Columns);
Assert.Equal(new[] { new object[] { null } }, rel.Columns);

rel = FindRelation(rsp.Output, ":pets");
Assert.NotNull(rel);
Assert.Equal(2, rel.Columns.Length);
Assert.Equal(new[] { new object[] { 1L, 2L }, new object[] { "dog", "rabbit" } }, rel.Columns);
// FIXME: complete the test implementation
// when this issue https://github.com/RelationalAI/rai-sdk-csharp/issues/25
// is fixed
//var rsp = await client.ExecuteAsync(Dbname, EngineName, "def output = sample");
}

public override async Task DisposeAsync()
Expand Down
8 changes: 4 additions & 4 deletions RelationalAI/Client.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public async Task<TransactionAsyncResult> ExecuteAsync(
return ReadTransactionAsyncResults(rsp as List<TransactionAsyncFile>);
}

public Task<TransactionResult> LoadJsonAsync(
public Task<TransactionAsyncResult> LoadJsonAsync(
string database,
string engine,
string relation,
Expand All @@ -509,10 +509,10 @@ public Task<TransactionResult> LoadJsonAsync(
{ "data", data }
};
var source = GenLoadJson(relation);
return ExecuteV1Async(database, engine, source, false, inputs);
return ExecuteWaitAsync(database, engine, source, false, inputs);
}

public Task<TransactionResult> LoadCsvAsync(
public Task<TransactionAsyncResult> LoadCsvAsync(
string database,
string engine,
string relation,
Expand All @@ -524,7 +524,7 @@ public Task<TransactionResult> LoadCsvAsync(
{
{ "data", data }
};
return ExecuteV1Async(database, engine, source, false, inputs);
return ExecuteWaitAsync(database, engine, source, false, inputs);
}

private static TransactionMode CreateMode(string source, bool overwrite)
Expand Down