Skip to content

Commit

Permalink
additional fixes and more testing for saving floats over doubles in t…
Browse files Browse the repository at this point in the history 
…ime-series (#256)

* fixes for mixing floats and doubles between time-series writes

includes regular and irregular time-series

* refactor test, prep for other intervals

* work in progress

irregular intervals working
1Hour, 1Minute, and 1Day working

* 1Year and 1Month working

* reduce number of data points

lowering number of values to
NUM_TS_VALUES 1221  so Weekly tests will pass

new issue #255 discovered to be handled later.

* fix typo

* update version date
  • Loading branch information
@ktarbet
ktarbet authored Oct 18, 2024
1 parent 6009ee6 commit a0173f3
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 38 deletions.
7 changes: 6 additions & 1 deletion heclib/heclib_c/src/Internal/ztsIrregStoreBlock.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,12 @@ int ztsIrregStoreBlock(long long *ifltab, zStructTimeSeries *tss, const char *pa
if (tsClone->times) {
for (size_t i = 0; i < numberToStore; i++)
{
tsClone->times[i] = (julianStartBlockDate * (86400 / tsClone->timeGranularitySeconds)) + (timesToStore[i] / tsClone->timeGranularitySeconds);
if (blockSize == 5) {
tsClone->times[i] = timesToStore[i] + julianStartBlockDate * 1440;
}
else {
tsClone->times[i] = (julianStartBlockDate * (86400 / tsClone->timeGranularitySeconds)) + (timesToStore[i] / tsClone->timeGranularitySeconds);
}
}
status = ztsStore(ifltab, tsClone, storageFlag);
}
Expand Down
19 changes: 16 additions & 3 deletions heclib/heclib_c/src/Internal/ztsRegStoreBlock.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -841,16 +841,29 @@ int ztsRegStoreBlock(long long *ifltab, zStructTimeSeries *tss, const char *path
int recordType = rb->recordType;
zstructFree(rb);

if (status == STATUS_RECORD_FOUND && recordType == DATA_TYPE_RTD
if ( status == STATUS_RECORD_FOUND && recordType == DATA_TYPE_RTD
&& dataType == DATA_TYPE_RTS
&& tss->floatValues
&& tss->doubleValues == NULL) {
// Support writing floats into a double record (calling ztsStore recursively)

zStructTimeSeries* tsClone = zstructTsClone(tss, pathname);

tsClone->startJulianDate = julianBlockDate + (blockStartPosition / (86400 / tsClone->timeIntervalSeconds));
tsClone->startTimeSeconds = (blockStartPosition + 1) * tsClone->timeIntervalSeconds % 86400;
long divisor = (SECS_IN_1_DAY / tsClone->timeIntervalSeconds);
if (tsClone->timeIntervalSeconds > SECS_IN_1_DAY) {
divisor = 1;
}

tsClone->startJulianDate = julianBlockDate + (blockStartPosition / divisor);


tsClone->startTimeSeconds = (blockStartPosition + 1) * tsClone->timeIntervalSeconds % SECS_IN_1_DAY;
if (tss->timeIntervalSeconds >= SECS_IN_1_DAY) {

incrementTime(intervalSeconds, 0, julianFirstValue,
secondsFirstValue - intervalSeconds + tss->timeOffsetSeconds,
&tsClone->startJulianDate, &tsClone->startTimeSeconds);
}

free(tsClone->floatValues);
tsClone->floatValues = NULL;
Expand Down
2 changes: 1 addition & 1 deletion heclib/heclib_c/src/headers/hecdssInternal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@

#define DSS_VERSION "7-IU"

#define DSS_VERSION_DATE "10 October 2024"
#define DSS_VERSION_DATE "18 October 2024"


const char *ztypeName(int recordType, int boolAbbreviation);
Expand Down
159 changes: 126 additions & 33 deletions test/Dss-C/source/mixed_record_types.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ int writeFloatIrregularTimeSeries(long long* ifltab, const char* path, const cha
int writeSingleTimeSeries(long long* ifltab, const char* path, const char* date, const char* time, int some_missing_data);



#define NUM_TS_VALUES 20
#define NUM_TS_VALUES 1221

/// prints out all paths, returns non-zero if any records are floats (we are expecting doubles)
int check_catalog_for_doubles(long long* ifltab) {
Expand Down Expand Up @@ -82,24 +81,19 @@ int check_values_compare(long long ifltab[], const char* path, int irregular) {
/// Writes floats to a time-series filling in the missing data.
/// </summary>
/// <returns></returns>
int write_irregular_ts_mixed(long long* ifltab, int writeDoublesFirst) {
int write_irregular_ts_mixed(long long* ifltab, char* path, int writeDoublesFirst) {
int status = 1;
const char* date = "30Jun2026";
const char* time = "2300";
const char* date = "15Jun2026";
const char* time = "1200";

char* path_float = "/ResSim//Flow//~1Hour/TSS-Floats/";
char* path_double = "/ResSim//Flow//~1Hour/TSS-Doubles/";
char* path;
if (writeDoublesFirst) {
path = path_double;
status = writeDoubleIrregularTimeSeries(ifltab, path, date, time, 1); // write doubles with gaps
if (status != STATUS_OKAY) {
return status;
}
status = writeFloatIrregularTimeSeries(ifltab, path, date, time, 0); // write floats without gaps
}
else { // write floats first
path = path_float;
status = writeFloatIrregularTimeSeries(ifltab, path, date, time, 1); // write floats with gaps
if (status != STATUS_OKAY) {
return status;
Expand All @@ -120,24 +114,19 @@ int write_irregular_ts_mixed(long long* ifltab, int writeDoublesFirst) {
/// Writes floats to a time-series filling in the missing data.
/// </summary>
/// <returns></returns>
int write_ts_mixed(long long* ifltab, int writeDoublesFirst) {
int write_ts_mixed(long long* ifltab, char* path, int writeDoublesFirst) {
int status = 1;
const char* date = "30Jun2024";
const char* time = "2300";
const char* date = "15Jun2024";
const char* time = "1200";

char* path_float = "//GAPT_DAM/FLOW-LOCAL//1Hour/GAPT_HMS_FORECAST_floats_first/";
char* path_double = "//GAPT_DAM/FLOW-LOCAL//1Hour/GAPT_HMS_FORECAST_doubles_first/";
char* path;
if (writeDoublesFirst) {
path = path_double;
status = writeDoublesTimeSeries(ifltab, path, date, time, 1); // write doubles with gaps
if (status != STATUS_OKAY) {
return status;
}
status = writeSingleTimeSeries(ifltab, path, date, time, 0); // write floats without gaps
}
else { // write floats first
path = path_float;
status = writeSingleTimeSeries(ifltab, path, date, time, 1); // write floats with gaps
if (status != STATUS_OKAY) {
return status;
Expand All @@ -164,35 +153,119 @@ int test_mixed_record_types() {
}
// -- Irregular Interval --

status = write_irregular_ts_mixed(ifltab, 0);
if (1) {
status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Hour/TSS-Floats/", 0);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Hour/TSS-Doubles/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Day/TSS-Floats/", 0);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Day/TSS-Doubles/", 1);
if (status != STATUS_OKAY) {
return status;
}
status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Week/TSS-Floats/", 0);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Week/TSS-Doubles/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Month/TSS-Floats/", 0);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Month/TSS-Doubles/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Year/TSS-Floats/", 0);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, "/ResSim//Flow//~1Year/TSS-Doubles/", 1);
if (status != STATUS_OKAY) {
return status;
}
}

// -- Regular Interval --

status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Hour/GAPT_HMS_FORECAST_floats_first/", 0);
if (status != STATUS_OKAY) {
return status;
}
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Hour/GAPT_HMS_FORECAST_doubles_first/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Minute/GAPT_HMS_FORECAST_floats_first/", 0);
if (status != STATUS_OKAY) {
return status;
}
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Minute/GAPT_HMS_FORECAST_doubles_first/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_irregular_ts_mixed(ifltab, 1);
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Day/GAPT_HMS_FORECAST_floats_first/", 0);
if (status != STATUS_OKAY) {
return status;
}
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Day/GAPT_HMS_FORECAST_doubles_first/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Week/GAPT_HMS_FORECAST_float_first/", 0);
if (status != STATUS_OKAY) {
return status;
}
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Week/GAPT_HMS_FORECAST_doubles_first/", 1);
if (status != STATUS_OKAY) {
return status;
}

// -- Regular Interval --
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Month/GAPT_HMS_FORECAST_floats_first/", 0);
if (status != STATUS_OKAY) {
return status;
}
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Month/GAPT_HMS_FORECAST_doubles_first/", 1);
if (status != STATUS_OKAY) {
return status;
}

status = write_ts_mixed(ifltab, 0);
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Year/GAPT_HMS_FORECAST_float_first/", 0);
if (status != STATUS_OKAY) {
return status;
}
status = write_ts_mixed(ifltab, 1);
status = write_ts_mixed(ifltab, "//GAPT_DAM/FLOW-LOCAL//1Year/GAPT_HMS_FORECAST_doubles_first/", 1);
if (status != STATUS_OKAY) {
return status;
}

// check that all records are doubles.

status = check_catalog_for_doubles(ifltab);


zclose(ifltab);

return status;

}
Expand Down Expand Up @@ -230,46 +303,66 @@ int writeSingleTimeSeries(long long* ifltab, const char* path, const char* date,
return status;
}


void createTimesArray(const char* path, const char* date, const char* time, int* itimes) {
int increment = 1;
// get epart determine increment
if (zfindString(path, strnlen_hec(path, MAX_PATHNAME_LENGTH), "~1Hour", 6) != -1) {
increment = SECS_IN_1_HOUR;
}
else if (zfindString(path, strnlen_hec(path, MAX_PATHNAME_LENGTH), "~1Day", 5) != -1) {
increment = SECS_IN_1_DAY;
}
else if (zfindString(path, strnlen_hec(path, MAX_PATHNAME_LENGTH), "~1Month", 7) != -1) {
increment = SECS_IN_1_MONTH;
}
else if (zfindString(path, strnlen_hec(path, MAX_PATHNAME_LENGTH), "~1Year", 6) != -1) {
increment = SECS_IN_1_YEAR;
}
int julian = dateToJulian(date);
int seconds = timeStringToSeconds(time);

for (int i = 0; i < NUM_TS_VALUES; i++) {
itimes[i] = julian * 1440 + seconds / 60;
incrementTime(increment, 1, julian, seconds, &julian, &seconds);
}
}

int writeDoubleIrregularTimeSeries(long long* ifltab, const char* path, const char* date, const char* time, int some_missing_data) {
double dvalues[NUM_TS_VALUES];
int itimes[NUM_TS_VALUES];
char* cnull = 0;

int julian = dateToJulian(date);
int seconds = timeStringToSeconds(time);
int mins = julian * 1440 + seconds / 60;
for (int i = 0; i < NUM_TS_VALUES; i++) {
if (i % 6 == 0 && some_missing_data) {
dvalues[i] = UNDEFINED_DOUBLE;
}
else {
dvalues[i] = (double)i;
}
itimes[i] = mins + (i * 60);
}
createTimesArray(path, date, time, itimes);

zStructTimeSeries* tss = zstructTsNewIrregDoubles(path, dvalues, NUM_TS_VALUES, itimes, MINUTE_GRANULARITY, cnull, "cfs", "Inst-Val");
int status = ztsStore(ifltab, tss, 0);
zstructFree(tss);
return status;
}

int writeFloatIrregularTimeSeries(long long* ifltab, const char* path, const char* date, const char* time, int some_missing_data) {
float fvalues[NUM_TS_VALUES];
int itimes[NUM_TS_VALUES];
float fvalues[NUM_TS_VALUES];
char* cnull = 0;

int julian = dateToJulian(date);
int seconds = timeStringToSeconds(time);
int mins = julian * 1440 + seconds / 60;
for (int i = 0; i < NUM_TS_VALUES; i++) {
if (i % 6 == 0 && some_missing_data) {
fvalues[i] = UNDEFINED_FLOAT;
}
else {
fvalues[i] = (float)i;
}
itimes[i] = mins + (i * 60);
}
createTimesArray(path, date, time, itimes);

zStructTimeSeries* tss = zstructTsNewIrregFloats(path, fvalues, NUM_TS_VALUES, itimes, MINUTE_GRANULARITY, cnull, "cfs", "Inst-Val");
int status = ztsStore(ifltab, tss, 0);
Expand Down

0 comments on commit a0173f3

Please sign in to comment.