From 621330d86bd624160ea9527eb53df06c1698d4a0 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 8 Nov 2024 18:53:05 +0100 Subject: [PATCH 01/36] Add KDVH migration package --- db/flags.sql | 12 + integration_tests/src/main.rs | 1 + migrations/.gitignore | 7 + migrations/README.md | 27 + migrations/go.mod | 27 + migrations/go.sum | 62 + migrations/kdvh/cache.go | 243 ++ migrations/kdvh/dump.go | 244 ++ migrations/kdvh/dump_functions.go | 276 ++ migrations/kdvh/import.go | 325 ++ migrations/kdvh/import_functions.go | 465 +++ migrations/kdvh/import_test.go | 31 + migrations/kdvh/list_tables.go | 24 + migrations/kdvh/main.go | 53 + migrations/kdvh/product_offsets.csv | 161 + migrations/kdvh/table.go | 125 + migrations/kdvh_test.go | 71 + migrations/lard/import.go | 92 + migrations/lard/main.go | 80 + migrations/lard/timeseries.go | 62 + migrations/main.go | 43 + .../tests/T_MDATA_combined/12345/TA.csv | 2645 +++++++++++++++++ migrations/utils/email.go | 60 + migrations/utils/utils.go | 62 + 24 files changed, 5198 insertions(+) create mode 100644 migrations/.gitignore create mode 100644 migrations/README.md create mode 100644 migrations/go.mod create mode 100644 migrations/go.sum create mode 100644 migrations/kdvh/cache.go create mode 100644 migrations/kdvh/dump.go create mode 100644 migrations/kdvh/dump_functions.go create mode 100644 migrations/kdvh/import.go create mode 100644 migrations/kdvh/import_functions.go create mode 100644 migrations/kdvh/import_test.go create mode 100644 migrations/kdvh/list_tables.go create mode 100644 migrations/kdvh/main.go create mode 100644 migrations/kdvh/product_offsets.csv create mode 100644 migrations/kdvh/table.go create mode 100644 migrations/kdvh_test.go create mode 100644 migrations/lard/import.go create mode 100644 migrations/lard/main.go create mode 100644 migrations/lard/timeseries.go create mode 100644 migrations/main.go create mode 100644 migrations/tests/T_MDATA_combined/12345/TA.csv create mode 100644 migrations/utils/email.go create mode 100644 migrations/utils/utils.go diff --git a/db/flags.sql b/db/flags.sql index bdb8f50f..785cf603 100644 --- a/db/flags.sql +++ b/db/flags.sql @@ -7,9 +7,21 @@ CREATE TABLE IF NOT EXISTS flags.kvdata ( corrected REAL NULL, controlinfo TEXT NULL, useinfo TEXT NULL, + -- TODO: check that this type is correct, it's stored as a string in Kvalobs? cfailed INT4 NULL, CONSTRAINT unique_kvdata_timeseries_obstime UNIQUE (timeseries, obstime) ); CREATE INDEX IF NOT EXISTS kvdata_obtime_index ON flags.kvdata (obstime); CREATE INDEX IF NOT EXISTS kvdata_timeseries_index ON flags.kvdata USING HASH (timeseries); + +CREATE TABLE IF NOT EXISTS flags.kdvh ( + timeseries INT4 REFERENCES public.timeseries, + obstime TIMESTAMPTZ NOT NULL, + controlinfo TEXT NULL, + useinfo TEXT NULL, + CONSTRAINT unique_kdvh_timeseries_obstime UNIQUE (timeseries, obstime) +); + +CREATE INDEX IF NOT EXISTS kdvh_obtime_index ON flags.kdvh (obstime); +CREATE INDEX IF NOT EXISTS kdvh_timeseries_index ON flags.kdvh USING HASH (timeseries); diff --git a/integration_tests/src/main.rs b/integration_tests/src/main.rs index 3f4cad89..a73a1241 100644 --- a/integration_tests/src/main.rs +++ b/integration_tests/src/main.rs @@ -37,6 +37,7 @@ async fn main() { } }); + // NOTE: order matters let schemas = ["db/public.sql", "db/labels.sql", "db/flags.sql"]; for schema in schemas { insert_schema(&client, schema).await.unwrap(); diff --git a/migrations/.gitignore b/migrations/.gitignore new file mode 100644 index 00000000..6305aabb --- /dev/null +++ b/migrations/.gitignore @@ -0,0 +1,7 @@ +*.txt +*.sh +migrate +tables*/ +test_*/ +.env +dumps/ diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 00000000..b32cfa74 --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,27 @@ +# Migrations + +Go package used to dump tables from old databases (KDVH, Kvalobs) and import them into LARD. + +## Usage + +1. Compile it with + + ```terminal + go build + ``` + +1. Dump tables from KDVH + + ```terminal + ./migrate kdvh dump --help + ``` + +1. Import dumps into LARD + + ```terminal + ./migrate kdvh import --help + ``` + +## Other notes + +Insightful talk on migrations: [here](https://www.youtube.com/watch?v=wqXqJfQMrqI&t=280s) diff --git a/migrations/go.mod b/migrations/go.mod new file mode 100644 index 00000000..d7233f7b --- /dev/null +++ b/migrations/go.mod @@ -0,0 +1,27 @@ +module migrate + +go 1.22.3 + +require ( + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/jackc/pgx/v5 v5.6.0 + github.com/jessevdk/go-flags v1.6.1 + github.com/joho/godotenv v1.5.1 + github.com/rickb777/period v1.0.5 + github.com/schollz/progressbar/v3 v3.16.1 +) + +require ( + github.com/govalues/decimal v0.1.29 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rickb777/plural v1.4.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.16.0 // indirect +) diff --git a/migrations/go.sum b/migrations/go.sum new file mode 100644 index 00000000..22ed9b76 --- /dev/null +++ b/migrations/go.sum @@ -0,0 +1,62 @@ +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/govalues/decimal v0.1.29 h1:GKC5g9y9oWxKIy51czdHTShOABwHm/shVuOVPwG415M= +github.com/govalues/decimal v0.1.29/go.mod h1:LUlHHucpCmA4rJfNrDvMgrWibDpYnDNWqJuNU1/gxW8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rickb777/period v1.0.5 h1:jAzlI2knYam5VMy0X8eYgqJBl0ew57N+J1djJSBOulM= +github.com/rickb777/period v1.0.5/go.mod h1:AmEwpgIShi3EEw34qbafoPJxVeRbv9VVtjLyOeRwK6c= +github.com/rickb777/plural v1.4.2 h1:Kl/syFGLFZ5EbuV8c9SVud8s5HI2HpCCtOMw2U1kS+A= +github.com/rickb777/plural v1.4.2/go.mod h1:kdmXUpmKBJTS0FtG/TFumd//VBWsNTD7zOw7x4umxNw= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= +github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ= +github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/migrations/kdvh/cache.go b/migrations/kdvh/cache.go new file mode 100644 index 00000000..2c8d718f --- /dev/null +++ b/migrations/kdvh/cache.go @@ -0,0 +1,243 @@ +package kdvh + +import ( + "context" + "fmt" + "log/slog" + "os" + "slices" + "time" + + "github.com/gocarina/gocsv" + "github.com/jackc/pgx/v5" + "github.com/rickb777/period" +) + +// Caches all the metadata needed for import. +// If any error occurs inside here the program will exit. +func (config *ImportConfig) CacheMetadata() { + config.cacheStinfo() + config.cacheKDVH() + config.cacheParamOffsets() +} + +// StinfoKey is used for lookup of parameter offsets and metadata from Stinfosys +type StinfoKey struct { + ElemCode string + TableName string +} + +// Subset of StinfoQuery with only param info +type StinfoParam struct { + TypeID int32 + ParamID int32 + Hlevel *int32 + Sensor int32 + Fromtime time.Time + IsScalar bool +} + +// Struct holding query from Stinfosys elem_map_cfnames_param +type StinfoQuery struct { + ElemCode string `db:"elem_code"` + TableName string `db:"table_name"` + TypeID int32 `db:"typeid"` + ParamID int32 `db:"paramid"` + Hlevel *int32 `db:"hlevel"` + Sensor int32 `db:"sensor"` + Fromtime time.Time `db:"fromtime"` + IsScalar bool `db:"scalar"` +} + +func (q *StinfoQuery) toParam() StinfoParam { + return StinfoParam{ + TypeID: q.TypeID, + ParamID: q.ParamID, + Hlevel: q.Hlevel, + Sensor: q.Sensor, + Fromtime: q.Fromtime, + IsScalar: q.IsScalar, + } +} +func (q *StinfoQuery) toKey() StinfoKey { + return StinfoKey{q.ElemCode, q.TableName} +} + +// Save metadata for later use by quering Stinfosys +func (config *ImportConfig) cacheStinfo() { + cache := make(map[StinfoKey]StinfoParam) + + fmt.Println("Connecting to Stinfosys to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("STINFO_STRING")) + if err != nil { + slog.Error("Could not connect to Stinfosys. Make sure to be connected to the VPN. " + err.Error()) + os.Exit(1) + } + defer conn.Close(context.TODO()) + + for _, table := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + // select paramid, elem_code, scalar from elem_map_cfnames_param join param using(paramid) where scalar = false + query := `SELECT elem_code, table_name, typeid, paramid, hlevel, sensor, fromtime, scalar + FROM elem_map_cfnames_param + JOIN param USING(paramid) + WHERE table_name = $1 + AND ($2::text[] IS NULL OR elem_code = ANY($2))` + + rows, err := conn.Query(context.TODO(), query, table.TableName, config.Elements) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[StinfoQuery]) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, meta := range metas { + cache[meta.toKey()] = meta.toParam() + } + } + + config.StinfoMap = cache +} + +// Used for lookup of fromtime and totime from KDVH +type KDVHKey struct { + Inner StinfoKey + Station int32 +} + +func newKDVHKey(elem, table string, stnr int32) KDVHKey { + return KDVHKey{StinfoKey{ElemCode: elem, TableName: table}, stnr} +} + +// Timespan stored in KDVH for a given (table, station, element) triplet +type Timespan struct { + FromTime *time.Time `db:"fdato"` + ToTime *time.Time `db:"tdato"` +} + +// Struct used to deserialize KDVH query in cacheKDVH +type MetaKDVH struct { + ElemCode string `db:"elem_code"` + TableName string `db:"table_name"` + Station int32 `db:"stnr"` + FromTime *time.Time `db:"fdato"` + ToTime *time.Time `db:"tdato"` +} + +func (m *MetaKDVH) toTimespan() Timespan { + return Timespan{m.FromTime, m.ToTime} +} + +func (m *MetaKDVH) toKey() KDVHKey { + return KDVHKey{StinfoKey{ElemCode: m.ElemCode, TableName: m.TableName}, m.Station} +} + +func (config *ImportConfig) cacheKDVH() { + cache := make(map[KDVHKey]Timespan) + + fmt.Println("Connecting to KDVH proxy to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("KDVH_PROXY_CONN")) + if err != nil { + slog.Error("Could not connect to KDVH proxy. Make sure to be connected to the VPN: " + err.Error()) + os.Exit(1) + } + defer conn.Close(context.TODO()) + + for _, t := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, t.TableName) { + continue + } + + // TODO: probably need to sanitize these inputs + query := fmt.Sprintf( + `SELECT table_name, stnr, elem_code, fdato, tdato FROM %s + WHERE ($1::bigint[] IS NULL OR stnr = ANY($1)) + AND ($2::text[] IS NULL OR elem_code = ANY($2))`, + t.ElemTableName, + ) + + rows, err := conn.Query(context.TODO(), query, config.Stations, config.Elements) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[MetaKDVH]) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, meta := range metas { + cache[meta.toKey()] = meta.toTimespan() + } + } + + config.KDVHMap = cache +} + +// Caches how to modify the obstime (in KDVH) for certain paramids +func (config *ImportConfig) cacheParamOffsets() { + cache := make(map[StinfoKey]period.Period) + + type CSVRow struct { + TableName string `csv:"table_name"` + ElemCode string `csv:"elem_code"` + ParamID int32 `csv:"paramid"` + FromtimeOffset string `csv:"fromtime_offset"` + Timespan string `csv:"timespan"` + } + + csvfile, err := os.Open("kdvh/product_offsets.csv") + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + defer csvfile.Close() + + var csvrows []CSVRow + if err := gocsv.UnmarshalFile(csvfile, &csvrows); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, row := range csvrows { + var fromtimeOffset, timespan period.Period + if row.FromtimeOffset != "" { + fromtimeOffset, err = period.Parse(row.FromtimeOffset) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + } + if row.Timespan != "" { + timespan, err = period.Parse(row.Timespan) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + } + migrationOffset, err := fromtimeOffset.Add(timespan) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[StinfoKey{ElemCode: row.ElemCode, TableName: row.TableName}] = migrationOffset + } + + config.OffsetMap = cache +} diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go new file mode 100644 index 00000000..7affa258 --- /dev/null +++ b/migrations/kdvh/dump.go @@ -0,0 +1,244 @@ +package kdvh + +import ( + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + + _ "github.com/jackc/pgx/v5/stdlib" + + "migrate/utils" +) + +type DumpConfig struct { + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + TablesCmd string `short:"t" long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + StationsCmd string `short:"s" long:"stnr" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + ElementsCmd string `short:"e" long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` + Email []string `long:"email" description:"Optional email address used to notify if the program crashed"` + + Tables []string + Stations []string + Elements []string +} + +func (config *DumpConfig) setup() { + if config.TablesCmd != "" { + config.Tables = strings.Split(config.TablesCmd, ",") + } + if config.StationsCmd != "" { + config.Stations = strings.Split(config.StationsCmd, ",") + } + if config.ElementsCmd != "" { + config.Elements = strings.Split(config.ElementsCmd, ",") + } +} + +func (config *DumpConfig) Execute([]string) error { + config.setup() + + conn, err := sql.Open("pgx", os.Getenv("KDVH_PROXY_CONN")) + if err != nil { + slog.Error(err.Error()) + return nil + } + + for _, table := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + table.Dump(conn, config) + } + + return nil +} + +func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { + defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) + + table.Path = filepath.Join(config.BaseDir, table.Path) + if _, err := os.ReadDir(table.Path); err == nil && !config.Overwrite { + slog.Info(fmt.Sprint("Skipping data dump of ", table.TableName, " because dumped folder already exists")) + return + } + + utils.SetLogFile(table.TableName, "dump") + elements, err := table.getElements(conn, config) + if err != nil { + return + } + + bar := utils.NewBar(len(elements), table.TableName) + + // TODO: should be safe to spawn goroutines/waitgroup here with connection pool? + bar.RenderBlank() + for _, element := range elements { + table.dumpElement(element, conn, config) + bar.Add(1) + } +} + +// TODO: maybe we don't do this? Or can we use pgdump/copy? +// The problem is that there are no indices on the tables, that's why the queries are super slow +// Dumping the whole table might be a lot faster (for T_MDATA it's ~10 times faster!), +// but it might be more difficult to recover if something goes wrong? +// => +// copyQuery := fmt.SPrintf("\\copy (select * from t_mdata) TO '%s/%s.csv' WITH CSV HEADER", config.BaseDir, table.TableName) +// cmd := exec.Command("psql", CONN_STRING, "-c", copyQuery) +// cmd.Stderr = &bytes.Buffer{} +// err = cmd.Run() +func (table *Table) dumpElement(element string, conn *sql.DB, config *DumpConfig) { + stations, err := table.getStationsWithElement(element, conn, config) + if err != nil { + slog.Error(fmt.Sprintf("Could not fetch stations for table %s: %v", table.TableName, err)) + return + } + + for _, station := range stations { + path := filepath.Join(table.Path, string(station)) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + slog.Error(err.Error()) + return + } + + err := table.dumpFunc( + path, + DumpMeta{ + element: element, + station: station, + dataTable: table.TableName, + flagTable: table.FlagTableName, + }, + conn, + ) + + // NOTE: Non-nil errors are logged inside each DumpFunc + if err == nil { + slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) + } + } +} + +// Fetches elements and filters them based on user input +func (table *Table) getElements(conn *sql.DB, config *DumpConfig) ([]string, error) { + elements, err := table.fetchElements(conn) + if err != nil { + return nil, err + } + + elements = utils.FilterSlice(config.Elements, elements, "") + return elements, nil +} + +// List of columns that we do not need to select when extracting the element codes from a KDVH table +var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} + +// Fetch column names for a given table +// We skip the columns defined in INVALID_COLUMNS and all columns that contain the 'kopi' string +// TODO: should we dump these invalid/kopi elements even if we are not importing them? +func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { + slog.Info(fmt.Sprintf("Fetching elements for %s...", table.TableName)) + + // TODO: not sure why we only dump these two for this table + if table.TableName == "T_HOMOGEN_MONTH" { + return []string{"rr", "tam"}, nil + } + + rows, err := conn.Query( + `SELECT column_name FROM information_schema.columns + WHERE table_name = $1 + AND NOT column_name = ANY($2::text[]) + AND column_name NOT LIKE '%kopi%'`, + // NOTE: needs to be lowercase with PG + strings.ToLower(table.TableName), + INVALID_COLUMNS, + ) + if err != nil { + slog.Error(fmt.Sprintf("Could not fetch elements for table %s: %v", table.TableName, err)) + return nil, err + } + defer rows.Close() + + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + slog.Error(fmt.Sprintf("Could not fetch elements for table %s: %v", table.TableName, err)) + return nil, err + } + elements = append(elements, name) + } + return elements, rows.Err() +} + +// Fetches station numbers and filters them based on user input +func (table *Table) getStationsWithElement(element string, conn *sql.DB, config *DumpConfig) ([]string, error) { + stations, err := table.fetchStationsWithElement(element, conn) + if err != nil { + return nil, err + } + + msg := fmt.Sprintf("Element '%s'", element) + "not available for station '%s'" + stations = utils.FilterSlice(config.Stations, stations, msg) + return stations, nil +} + +// Fetches the unique station numbers in the table for a given element (and when that element is not null) +// NOTE: splitting by element does make it a bit better, because we avoid quering for stations that have no data or flag for that element? +func (table *Table) fetchStationsWithElement(element string, conn *sql.DB) (stations []string, err error) { + slog.Info(fmt.Sprintf("Fetching station numbers for %s (this can take a while)...", element)) + + query := fmt.Sprintf( + `SELECT DISTINCT stnr FROM %s WHERE %s IS NOT NULL`, + table.TableName, + element, + ) + + rows, err := conn.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var stnr string + if err := rows.Scan(&stnr); err != nil { + return nil, err + } + stations = append(stations, stnr) + } + + return stations, rows.Err() +} + +// Fetches all unique station numbers in the table +// FIXME: the DISTINCT query can be extremely slow +// NOTE: decided to use fetchStationsWithElement instead +func (table *Table) fetchStationNumbers(conn *sql.DB) (stations []string, err error) { + slog.Info(fmt.Sprint("Fetching station numbers (this can take a while)...")) + + query := fmt.Sprintf( + `SELECT DISTINCT stnr FROM %s`, + table.TableName, + ) + + rows, err := conn.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var stnr string + if err := rows.Scan(&stnr); err != nil { + return nil, err + } + stations = append(stations, stnr) + } + + return stations, rows.Err() +} diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go new file mode 100644 index 00000000..8afe0946 --- /dev/null +++ b/migrations/kdvh/dump_functions.go @@ -0,0 +1,276 @@ +package kdvh + +import ( + "database/sql" + "encoding/csv" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "time" +) + +// Fetch min and max year from table, needed for tables that are dumped by year +func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, error) { + var beginStr, endStr string + query := fmt.Sprintf("SELECT min(to_char(dato, 'yyyy')), max(to_char(dato, 'yyyy')) FROM %s WHERE stnr = $1", tableName) + + if err := conn.QueryRow(query, station).Scan(&beginStr, &endStr); err != nil { + slog.Error(fmt.Sprint("Could not query row: ", err)) + return 0, 0, err + } + + begin, err := strconv.ParseInt(beginStr, 10, 64) + if err != nil { + slog.Error(fmt.Sprintf("Could not parse year '%s': %s", beginStr, err)) + return 0, 0, err + } + + end, err := strconv.ParseInt(endStr, 10, 64) + if err != nil { + slog.Error(fmt.Sprintf("Could not parse year '%s': %s", endStr, err)) + return 0, 0, err + } + + return begin, end, nil +} + +func dumpByYearDataOnly(path string, meta DumpMeta, conn *sql.DB) error { + begin, end, err := fetchYearRange(meta.dataTable, meta.station, conn) + if err != nil { + return err + } + + query := fmt.Sprintf( + `SELECT dato AS time, %[1]s AS data, '' AS flag FROM %[2]s + WHERE %[1]s IS NOT NULL + AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2`, + meta.element, + meta.dataTable, + ) + + for year := begin; year < end; year++ { + rows, err := conn.Query(query, meta.station, year) + if err != nil { + slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + return err + } + + path := filepath.Join(path, fmt.Sprint(year)) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + slog.Error(err.Error()) + continue + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + } + + return nil +} + +func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { + dataBegin, dataEnd, err := fetchYearRange(meta.dataTable, meta.station, conn) + if err != nil { + return err + } + + flagBegin, flagEnd, err := fetchYearRange(meta.flagTable, meta.station, conn) + if err != nil { + return err + } + + begin := min(dataBegin, flagBegin) + end := max(dataEnd, flagEnd) + + query := fmt.Sprintf( + `SELECT + dato AS time, + d.%[1]s AS data, + f.%[1]s AS flag + FROM + (SELECT dato, stnr, %[1]s FROM %[2]s + WHERE %[1]s IS NOT NULL AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2) d + FULL OUTER JOIN + (SELECT dato, stnr, %[1]s FROM %[3]s + WHERE %[1]s IS NOT NULL AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2) f + USING (dato)`, + meta.element, + meta.dataTable, + meta.flagTable, + ) + + for year := begin; year < end; year++ { + rows, err := conn.Query(query, meta.station, year) + if err != nil { + slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + return err + } + + yearPath := filepath.Join(path, fmt.Sprint(year)) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + slog.Error(err.Error()) + continue + } + + if err := dumpToFile(yearPath, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + } + + return nil +} + +func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { + query := fmt.Sprintf( + `SELECT dato AS time, %s[1]s AS data, '' AS flag FROM T_HOMOGEN_MONTH + WHERE %s[1]s IS NOT NULL AND stnr = $1 AND season BETWEEN 1 AND 12`, + // NOTE: adding a dummy argument is the only way to suppress this stupid warning + meta.element, "", + ) + + rows, err := conn.Query(query, meta.station) + if err != nil { + slog.Error(err.Error()) + return err + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + + return nil +} + +func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { + query := fmt.Sprintf( + `SELECT dato AS time, %[1]s AS data, '' AS flag FROM %[2]s + WHERE %[1]s IS NOT NULL AND stnr = $1`, + meta.element, + meta.dataTable, + ) + + rows, err := conn.Query(query, meta.station) + if err != nil { + slog.Error(err.Error()) + return err + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + + return nil +} + +func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { + query := fmt.Sprintf( + `SELECT + dato AS time, + d.%[1]s AS data, + f.%[1]s AS flag + FROM + (SELECT dato, %[1]s FROM %[2]s WHERE %[1]s IS NOT NULL AND stnr = $1) d + FULL OUTER JOIN + (SELECT dato, %[1]s FROM %[3]s WHERE %[1]s IS NOT NULL AND stnr = $1) f + USING (dato)`, + meta.element, + meta.dataTable, + meta.flagTable, + ) + + rows, err := conn.Query(query, meta.station) + if err != nil { + slog.Error(err.Error()) + return err + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + + return nil +} + +func dumpToFile(path, element string, rows *sql.Rows) error { + filename := filepath.Join(path, element+".csv") + file, err := os.Create(filename) + if err != nil { + return err + } + + lines, err := sortRows(rows) + if err != nil { + return err + } + + err = writeElementFile(lines, file) + if closeErr := file.Close(); closeErr != nil { + return errors.Join(err, closeErr) + } + return err +} + +// Struct representing a single record in the output CSV file +type Record struct { + time time.Time + data sql.NullString + flag sql.NullString +} + +// Scans the rows and collects them in a slice of chronologically sorted lines +func sortRows(rows *sql.Rows) ([]Record, error) { + defer rows.Close() + + // TODO: if we use pgx we might be able to preallocate the right size + var records []Record + var record Record + + for rows.Next() { + if err := rows.Scan(&record.time, &record.data, &record.flag); err != nil { + return nil, errors.New("Could not scan rows: " + err.Error()) + } + records = append(records, record) + } + + slices.SortFunc(records, func(a, b Record) int { + return a.time.Compare(b.time) + }) + + return records, rows.Err() +} + +// Format string for date field in CSV files +const TIMEFORMAT string = "2006-01-02_15:04:05" + +// Writes queried (time | data | flag) columns to CSV +func writeElementFile(lines []Record, file io.Writer) error { + // Write number of lines as header + file.Write([]byte(fmt.Sprintf("%v\n", len(lines)))) + + writer := csv.NewWriter(file) + + record := make([]string, 3) + for _, l := range lines { + record[0] = l.time.Format(TIMEFORMAT) + record[1] = l.data.String + record[2] = l.flag.String + + if err := writer.Write(record); err != nil { + return errors.New("Could not write to file: " + err.Error()) + } + } + + writer.Flush() + return writer.Error() +} diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go new file mode 100644 index 00000000..444ee9d9 --- /dev/null +++ b/migrations/kdvh/import.go @@ -0,0 +1,325 @@ +package kdvh + +import ( + "bufio" + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/rickb777/period" + + "migrate/lard" + "migrate/utils" +) + +type ImportConfig struct { + Verbose bool `short:"v" description:"Increase verbosity level"` + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + TablesCmd string `short:"t" long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + StationsCmd string `short:"s" long:"station" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + ElementsCmd string `short:"e" long:"elemcode" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Sep string `long:"sep" default:"," description:"Separator character in the dumped files. Needs to be quoted"` + HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` + Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` + Email []string `long:"email" description:"Optional email address used to notify if the program crashed"` + + Tables []string + Stations []string + Elements []string + + OffsetMap map[StinfoKey]period.Period // Map of offsets used to correct (?) KDVH times for specific parameters + StinfoMap map[StinfoKey]StinfoParam // Map of metadata used to query timeseries ID in LARD + KDVHMap map[KDVHKey]Timespan // Map of `from_time` and `to_time` for each (table, station, element) triplet. Not present for all parameters +} + +func (config *ImportConfig) setup() { + if len(config.Sep) > 1 { + slog.Warn("'--sep' only accepts single-byte characters. Defaulting to ','") + config.Sep = "," + } + if config.TablesCmd != "" { + config.Tables = strings.Split(config.TablesCmd, ",") + } + if config.StationsCmd != "" { + config.Stations = strings.Split(config.StationsCmd, ",") + } + if config.ElementsCmd != "" { + config.Elements = strings.Split(config.ElementsCmd, ",") + } + config.CacheMetadata() +} + +func (config *ImportConfig) Execute([]string) error { + config.setup() + + // Create connection pool for LARD + pool, err := pgxpool.New(context.TODO(), os.Getenv("LARD_STRING")) + if err != nil { + slog.Error(fmt.Sprint("Could not connect to Lard:", err)) + return err + } + defer pool.Close() + + for _, table := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + table.Import(pool, config) + } + + return nil +} + +func (table *Table) Import(pool *pgxpool.Pool, config *ImportConfig) (rowsInserted int64) { + defer utils.SendEmailOnPanic("importTable", config.Email) + + if table.importUntil == 0 { + if config.Verbose { + slog.Info("Skipping import of" + table.TableName + " because this table is not set for import") + } + return 0 + } + + utils.SetLogFile(table.TableName, "import") + + table.Path = filepath.Join(config.BaseDir, table.Path) + stations, err := os.ReadDir(table.Path) + if err != nil { + slog.Warn(fmt.Sprintf("Could not read directory %s: %s", table.Path, err)) + return 0 + } + + bar := utils.NewBar(len(stations), table.TableName) + bar.RenderBlank() + for _, station := range stations { + count, err := table.importStation(station, pool, config) + if err == nil { + rowsInserted += count + } + bar.Add(1) + } + + outputStr := fmt.Sprintf("%v: %v total rows inserted", table.TableName, rowsInserted) + slog.Info(outputStr) + fmt.Println(outputStr) + return rowsInserted +} + +// Loops over the element files present in the station directory and processes them concurrently +func (table *Table) importStation(station os.DirEntry, pool *pgxpool.Pool, config *ImportConfig) (totRows int64, err error) { + stnr, err := getStationNumber(station, config.Stations) + if err != nil { + if config.Verbose { + slog.Info(err.Error()) + } + return 0, err + } + + dir := filepath.Join(table.Path, station.Name()) + elements, err := os.ReadDir(dir) + if err != nil { + slog.Warn(fmt.Sprintf("Could not read directory %s: %s", dir, err)) + return 0, err + } + + var wg sync.WaitGroup + for _, element := range elements { + elemCode, err := getElementCode(element, config.Elements) + if err != nil { + if config.Verbose { + slog.Info(err.Error()) + } + continue + } + + wg.Add(1) + go func() { + defer wg.Done() + + tsInfo, err := config.NewTimeseriesInfo(table.TableName, elemCode, stnr) + if err != nil { + return + } + + tsid, err := getTimeseriesID(tsInfo, pool) + if err != nil { + slog.Error(tsInfo.logstr + "could not obtain timeseries - " + err.Error()) + return + } + + filename := filepath.Join(dir, element.Name()) + data, err := table.parseElementFile(filename, tsInfo, config) + if err != nil { + return + } + + ts := lard.NewTimeseries(tsid, data) + count, err := importData(ts, tsInfo, pool, config) + if err != nil { + return + } + totRows += count + }() + } + wg.Wait() + + return totRows, nil +} + +func (table *Table) parseElementFile(filename string, tsInfo *TimeseriesInfo, config *ImportConfig) ([]lard.Obs, error) { + file, err := os.Open(filename) + if err != nil { + slog.Warn(fmt.Sprintf("Could not open file '%s': %s", filename, err)) + return nil, err + } + defer file.Close() + + data, err := table.parseData(file, tsInfo, config) + if err != nil { + slog.Error(fmt.Sprintf("Could not parse data from '%s': %s", filename, err)) + return nil, err + } + + if len(data) == 0 { + slog.Info(tsInfo.logstr + "no rows to insert (all obstimes > max import time)") + return nil, err + } + + return data, nil +} + +func importData(ts *lard.Timeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, config *ImportConfig) (count int64, err error) { + if !(config.Skip == "data") { + if tsInfo.param.IsScalar { + count, err = lard.InsertData(ts, pool, tsInfo.logstr) + if err != nil { + slog.Error(tsInfo.logstr + "failed data bulk insertion - " + err.Error()) + return 0, err + } + } else { + count, err = lard.InsertNonscalarData(ts, pool, tsInfo.logstr) + if err != nil { + slog.Error(tsInfo.logstr + "failed non-scalar data bulk insertion - " + err.Error()) + return 0, err + } + // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data + // return count, nil + } + } + + if !(config.Skip == "flags") { + if err := lard.InsertFlags(ts, pool, tsInfo.logstr); err != nil { + slog.Error(tsInfo.logstr + "failed flag bulk insertion - " + err.Error()) + } + } + + return count, nil + +} + +func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { + if !station.IsDir() { + return 0, errors.New(fmt.Sprintf("%s is not a directory, skipping", station.Name())) + } + + if stationList != nil && !slices.Contains(stationList, station.Name()) { + return 0, errors.New(fmt.Sprintf("Station %v not in the list, skipping", station.Name())) + } + + stnr, err := strconv.ParseInt(station.Name(), 10, 32) + if err != nil { + return 0, errors.New("Error parsing station number:" + err.Error()) + } + + return int32(stnr), nil +} + +func getElementCode(element os.DirEntry, elementList []string) (string, error) { + elemCode := strings.ToUpper(strings.TrimSuffix(element.Name(), ".csv")) + + if elementList != nil && !slices.Contains(elementList, elemCode) { + return "", errors.New(fmt.Sprintf("Element '%s' not in the list, skipping", elemCode)) + } + + if elemcodeIsInvalid(elemCode) { + return "", errors.New(fmt.Sprintf("Element '%s' not set for import, skipping", elemCode)) + } + return elemCode, nil +} + +func getTimeseriesID(tsInfo *TimeseriesInfo, pool *pgxpool.Pool) (int32, error) { + label := lard.Label{ + StationID: tsInfo.station, + TypeID: tsInfo.param.TypeID, + ParamID: tsInfo.param.ParamID, + Sensor: &tsInfo.param.Sensor, + Level: tsInfo.param.Hlevel, + } + tsid, err := lard.GetTimeseriesID(label, tsInfo.param.Fromtime, pool) + if err != nil { + slog.Error(tsInfo.logstr + "could not obtain timeseries - " + err.Error()) + return 0, err + + } + return tsid, nil +} + +func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *ImportConfig) ([]lard.Obs, error) { + scanner := bufio.NewScanner(handle) + + var rowCount int + // Try to infer row count from header + if config.HasHeader { + scanner.Scan() + // rowCount, _ = strconv.Atoi(scanner.Text()) + if temp, err := strconv.Atoi(scanner.Text()); err == nil { + rowCount = temp + } + } + + data := make([]lard.Obs, 0, rowCount) + for scanner.Scan() { + cols := strings.Split(scanner.Text(), config.Sep) + + obsTime, err := time.Parse("2006-01-02_15:04:05", cols[0]) + if err != nil { + return nil, err + } + + // Only import data between KDVH's defined fromtime and totime + if meta.span.FromTime != nil && obsTime.Sub(*meta.span.FromTime) < 0 { + continue + } else if meta.span.ToTime != nil && obsTime.Sub(*meta.span.ToTime) > 0 { + break + } + + if obsTime.Year() >= table.importUntil { + break + } + + temp, err := table.convFunc(Obs{meta, obsTime, cols[1], cols[2]}) + if err != nil { + return nil, err + } + + data = append(data, temp) + } + + return data, nil +} + +// TODO: add CALL_SIGN? It's not in stinfosys? +var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} + +func elemcodeIsInvalid(element string) bool { + return strings.Contains(element, "KOPI") || slices.Contains(INVALID_ELEMENTS, element) +} diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go new file mode 100644 index 00000000..00060670 --- /dev/null +++ b/migrations/kdvh/import_functions.go @@ -0,0 +1,465 @@ +package kdvh + +import ( + "errors" + "migrate/lard" + "strconv" + + "github.com/rickb777/period" +) + +// In kvalobs a flag is a 16 char string containg QC information about the observation: +// Note: Missing numbers in the following lists are marked as reserved (not in use I guess?) +// +// CONTROLINFO FLAG: +// +// 0 - CONTROL LEVEL (not used) +// 1 - RANGE CHECK +// 0. Not checked +// 1. Check passed +// 2. Higher than HIGH +// 3. Lower than LOW +// 4. Higher than HIGHER +// 5. Lower that LOWER +// 6. Check failed, above HIGHEST or below LOWEST +// +// 2 - FORMAL CONSISTENCY CHECK +// 0. Not checked +// 1. Check passe +// 2. Inconsistency found, but not an error with the relevant parameter, no correction +// 3. Inconsistency found at the observation time, but not possible to determine which parameter, no correction +// 4. Inconsistency found at earliar/later observation times, but not possible to determine which parameter, no correction +// 6. Inconsistency found at the observation time, probably error with the relevant parameter, no correction +// 7. Inconsistency found at earliar/later observation times, probably error with relevant parameter, no correction +// 8. Inconsistency found, a parameter is missing, no correction +// A. Inconsistency found at the observation time, corrected automatically +// B. Inconsistency found at earliar/later observation times, corrected automatically +// D. Check failed +// +// 3 - JUMP CHECK (STEP, DIP, FREEZE, DRIFT) +// 0. Not checked +// 1. Check passed +// 2. Change higher than test value, no correction +// 3. No change in measured value (freeze check did not pass?), no correction +// 4. Suspected error in freeze check, no error in dip check (??), no correction +// 5. Suspected error in dip check, no error in freeze check (??), no correction +// 7. Observed drift, no correction +// 9. Change higher than test value, corrected automatically +// A. Freeze check did not pass, corrected automatically +// +// 4 - PROGNOSTIC CHECK +// 0. Not checked +// 1. Check passed +// 2. Deviation from model higher than HIGH +// 3. Deviation from model lower than LOW +// 4. Deviation from model higher than HIGHER +// 5. Deviation from model lower that LOWER +// 6. Check failed, deviation from model above HIGHEST or below LOWEST +// +// 5 - VALUE CHECK (FOR MOVING STATIONS) +// 0. Not checked +// 1. Check passed +// 3. Suspicious value, no correction +// 4. Suspicious value, corrected automatically +// 6. Check failed +// +// 6 - MISSING OBSERVATIONS +// 0. Original and corrected values exist +// 1. Original value missing, but corrected value exists +// 2. Corrected value missing, orginal value discarded +// 3. Original and corrected values missing +// +// 7 - TIMESERIES FITTING +// 0. Not checked +// 1. Interpolated with good fitness +// 2. Interpolated with unsure fitness +// 3. Intepolation not suitable +// +// 8 - WEATHER ANALYSIS +// 0. Not checked +// 1. Check passed +// 2. Suspicious value, not corrected +// 3. Suspicious value, corrected automatically +// +// 9 - STATISTICAL CHECK +// 0. Not checked +// 1. Check passed +// 2. Suspicious value, not corrected +// +// 10 - CLIMATOLOGICAL CONSISTENCY CHECK +// 0. Not checked +// 1. Check passed +// 2. Climatologically questionable, but not an error with the relevant parameter, no correction +// 3. Climatologically questionable at the observation time, but not possible to determine which parameter, no correction +// 4. Climatologically questionable at earliar/later observation times, but not possible to determine which parameter, no correction +// 6. Climatologically questionable at the observation time, probably error with the relevant parameter, no correction +// 7. Climatologically questionable at earliar/later observation times, probably error with relevant parameter, no correction +// A. Inconsistency found at the observation time, corrected automatically +// B. Inconsistency found at earliar/later observation times, corrected automatically +// D. Check failed +// +// 11 - CLIMATOLOGICAL CHECK +// 0. Not checked +// 1. Check passed +// 2. Suspicious value, not corrected +// 3. Suspicious value, corrected automatically +// +// 12 - DISTRIBUTION CHECK OF ACCUMULATED PARAMETERS (ESPECIALLY FOR PRECIPITATION) +// 0. Not checked +// 1. Not an accumulated value +// 2. Observation outside accumulated parameter range +// 3. Abnormal observation (??) +// 6. Accumulation calculated from numerical model +// 7. Accumulation calculated from weather analysis +// A. Accumulation calculated with 'steady rainfall' method +// B. Accumulation calculated with 'uneven rainfall' method +// +// 13 - PREQUALIFICATION (CERTAIN PAIRS OF 'STATIONID' AND 'PARAMID' CAN BE DISCARDED) +// 0. Not checked +// 5. Value is missing +// 6. Check failed, invalid original value +// 7. Check failed, original value is noisy +// +// 14 - COMBINATION CHECK +// 0. Not checked +// 1. Check passed +// 2. Outside test limit value, but no jumps detected and inside numerical model tolerance +// 9. Check failed. Outside test limit value, no jumps detected but outside numerical model tolerance +// A. Check failed. Outside test limit value, jumps detected but inside numerical model tolerance +// B. Check failed. Outside test limit value, jumps detected and outside numerical model tolerance +// +// 15 - MANUAL QUALITY CONTROL +// 0. Not checked +// 1. Check passed +// 2. Probably OK +// 5. Value manually interpolated +// 6. Value manually assigned +// 7. Value manually corrected +// A. Manually rejected + +const ( + VALUE_PASSED_QC = "00000" + "00000000000" + + // Corrected value is present, and original was remove by QC + VALUE_CORRECTED_AUTOMATICALLY = "00000" + "01000000000" + VALUE_MANUALLY_INTERPOLATED = "00000" + "01000000005" + VALUE_MANUALLY_ASSIGNED = "00000" + "01000000006" + + VALUE_REMOVED_BY_QC = "00000" + "02000000000" // Corrected value is missing, and original was remove by QC + VALUE_MISSING = "00000" + "03000000000" // Both original and corrected are missing + VALUE_PASSED_HQC = "00000" + "00000000001" // Value was sent to HQC for inspection, but it was OK + + // original value still exists, not exactly sure what the difference with VALUE_MANUALLY_INTERPOLATED is + INTERPOLATION_ADDED_MANUALLY = "00000" + "00000000005" +) + +// USEINFO FLAG: +// +// 0 - CONTROL LEVELS PASSED +// 1. Completed QC1, QC2 and HQC +// 2. Completed QC2 and HQC +// 3. Completed QC1 and HQC +// 4. Completed HQC +// 5. Completed QC1 and QC2 +// 6. Completed QC2 +// 7. Completed QC1 +// 9. Missing information +// +// 1 - DEVIATION FROM NORM (MEAN?) +// 0. Observation time and period are okay +// 1. Observation time deviates from norm +// 2. Observation period is shorter than norm +// 3. Observation perios is longer than norm +// 4. Observation time deviates from norm, and period is shorter than norm +// 5. Observation time deviates from norm, and period is longer than norm +// 8. Missing value +// 9. Missing status information +// +// 2 - QUALITY LEVEL OF ORIGNAL VALUE +// 0. Value is okay +// 1. Value is suspicious (probably correct) +// 2. Value is suspicious (probably wrong) +// 3. Value is wrong +// 9. Missing quality information +// +// 3 - TREATMENT OF ORIGINAL VALUE +// 0. Unchanged +// 1. Manually corrected +// 2. Manually interpolated +// 3. Automatically corrected +// 4. Automatically interpolated +// 5. Manually derived from accumulated value +// 6. Automatically derived from accumulated value +// 8. Rejected +// 9. Missing information +// +// 4 - MOST IMPORT CHECK RESULT (?) +// 0. Original value is okay +// 1. Range check +// 2. Consistency check +// 3. Jump check +// 4. Consistency check in relation with earlier/later observations +// 5. Prognostic check based on observation data +// 6. Prognostic check based on Timeseries +// 7. Prognostic check based on model data +// 8. Prognostic check based on statistics +// 9. Missing information +// +// 7 - DELAY INFORMATION +// 0. Observation carried out and reported at the right time +// 1. Observation carried out early and reported at the right time +// 2. Observation carried out late and reported at the right time +// 3. Observation reported early +// 4. Observation reported late +// 5. Observation carried out early and reported late +// 6. Observation carried out late and reported late +// 9. Missing information +// +// 8 - FIRST DIGIT OF HEXADECIMAL VALUE OF THE OBSERVATION CONFIDENCE LEVEL +// 9 - SECOND DIGIT OF HEXADECIMAL VALUE OF THE OBSERVATION CONFIDENCE LEVEL +// 13 - FIRST HQC OPERATOR DIGIT +// 14 - SECOND HQC OPERATOR DIGIT +// 15 - HEXADECIMAL DIGIT WITH NUMBER OF TESTS THAT DID NOT PASS (RETURNED A RESULT?) + +const ( + // Remaing 11 digits of `useinfo` that follow the 5 digits contained in `obs.Flags`. + // TODO: From the docs it looks like the '9' should be changed by kvalobs when + // the observation is inserted into the database but that's not the case? + DELAY_DEFAULT = "00900000000" + + INVALID_FLAGS = "99999" + DELAY_DEFAULT // Only returned when the flags are invalid + COMPLETED_HQC = "40000" + DELAY_DEFAULT // Specific to T_VDATA + DIURNAL_INTERPOLATED_USEINFO = "48925" + DELAY_DEFAULT // Specific to T_DIURNAL_INTERPOLATED +) + +func (obs *Obs) flagsAreValid() bool { + if len(obs.Flags) != 5 { + return false + } + _, err := strconv.ParseInt(obs.Flags, 10, 32) + return err == nil +} + +func (obs *Obs) Useinfo() string { + if !obs.flagsAreValid() { + return INVALID_FLAGS + } + return obs.Flags + DELAY_DEFAULT +} + +// The following functions try to recover the original pair of `controlinfo` +// and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` +// Different KDVH tables need different ways to perform this conversion. + +func makeDataPage(obs Obs) (lard.Obs, error) { + var valPtr *float32 + + controlinfo := VALUE_PASSED_QC + if obs.Data == "" { + controlinfo = VALUE_MISSING + } + + // NOTE: this is the only function that can return `lard.Obs` + // with non-null text data + if !obs.param.IsScalar { + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Text: &obs.Data, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil + } + + val, err := strconv.ParseFloat(obs.Data, 32) + if err == nil { + f32 := float32(val) + valPtr = &f32 + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +// modify obstimes to always use totime +func makeDataPageProduct(obs Obs) (lard.Obs, error) { + obsLard, err := makeDataPage(obs) + if !obs.offset.IsZero() { + if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { + obsLard.Obstime = temp + } + } + return obsLard, err +} + +func makeDataPageEdata(obs Obs) (lard.Obs, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + switch obs.Flags { + case "70381", "70389", "90989": + controlinfo = VALUE_REMOVED_BY_QC + default: + // Includes "70000", "70101", "99999" + controlinfo = VALUE_MISSING + } + } else { + controlinfo = VALUE_PASSED_QC + f32 := float32(val) + valPtr = &f32 + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +func makeDataPagePdata(obs Obs) (lard.Obs, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + switch obs.Flags { + case "20389", "30389", "40389", "50383", "70381", "71381": + controlinfo = VALUE_REMOVED_BY_QC + default: + // "00000", "10000", "10319", "30000", "30319", + // "40000", "40929", "48929", "48999", "50000", + // "50205", "60000", "70000", "70103", "70203", + // "71000", "71203", "90909", "99999" + controlinfo = VALUE_MISSING + } + } else { + f32 := float32(val) + valPtr = &f32 + + switch obs.Flags { + case "10319", "10329", "30319", "40319", "48929", "48999": + controlinfo = VALUE_MANUALLY_INTERPOLATED + case "20389", "30389", "40389", "50383", "70381", "71381", "99319": + controlinfo = VALUE_CORRECTED_AUTOMATICALLY + case "40929": + controlinfo = INTERPOLATION_ADDED_MANUALLY + default: + // "71000", "71203", "90909", "99999" + controlinfo = VALUE_PASSED_QC + } + + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +func makeDataPageNdata(obs Obs) (lard.Obs, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + switch obs.Flags { + case "70389": + controlinfo = VALUE_REMOVED_BY_QC + default: + // "30319", "38929", "40000", "40100", "40315" + // "40319", "43325", "48325", "49225", "49915" + // "70000", "70204", "71000", "73309", "78937" + // "90909", "93399", "98999", "99999" + controlinfo = VALUE_MISSING + } + } else { + switch obs.Flags { + case "43325", "48325": + controlinfo = VALUE_MANUALLY_ASSIGNED + case "30319", "38929", "40315", "40319": + controlinfo = VALUE_MANUALLY_INTERPOLATED + case "49225", "49915": + controlinfo = INTERPOLATION_ADDED_MANUALLY + case "70389", "73309", "78937", "93399", "98999": + controlinfo = VALUE_CORRECTED_AUTOMATICALLY + default: + // "40000", "40100", "70000", "70204", "71000", "90909", "99999" + controlinfo = VALUE_PASSED_QC + } + f32 := float32(val) + valPtr = &f32 + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +func makeDataPageVdata(obs Obs) (lard.Obs, error) { + var useinfo, controlinfo string + var valPtr *float32 + + // set useinfo based on time + if h := obs.Obstime.Hour(); h == 0 || h == 6 || h == 12 || h == 18 { + useinfo = COMPLETED_HQC + } else { + useinfo = INVALID_FLAGS + } + + // set data and controlinfo + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + controlinfo = VALUE_MISSING + } else { + // super special treatment clause of T_VDATA.OT_24, so it will be the same as in kvalobs + f32 := float32(val) + + if obs.element == "OT_24" { + // add custom offset, because OT_24 in KDVH has been treated differently than OT_24 in kvalobs + offset, err := period.Parse("PT18H") // fromtime_offset -PT6H, timespan P1D + if err != nil { + return lard.Obs{}, errors.New("could not parse period") + } + temp, ok := offset.AddTo(obs.Obstime) + if !ok { + return lard.Obs{}, errors.New("could not add period") + } + + obs.Obstime = temp + // convert from hours to minutes + f32 *= 60.0 + } + valPtr = &f32 + controlinfo = VALUE_PASSED_QC + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: useinfo, + Controlinfo: controlinfo, + }, nil +} + +func makeDataPageDiurnalInterpolated(obs Obs) (lard.Obs, error) { + val, err := strconv.ParseFloat(obs.Data, 32) + if err != nil { + return lard.Obs{}, err + } + f32 := float32(val) + + return lard.Obs{ + Obstime: obs.Obstime, + Data: &f32, + Useinfo: DIURNAL_INTERPOLATED_USEINFO, + Controlinfo: VALUE_MANUALLY_INTERPOLATED, + }, nil +} diff --git a/migrations/kdvh/import_test.go b/migrations/kdvh/import_test.go new file mode 100644 index 00000000..ba6e574b --- /dev/null +++ b/migrations/kdvh/import_test.go @@ -0,0 +1,31 @@ +package kdvh + +import "testing" + +func TestFlagsAreValid(t *testing.T) { + type testCase struct { + input Obs + expected bool + } + + cases := []testCase{ + {Obs{Flags: "12309"}, true}, + {Obs{Flags: "984.3"}, false}, + {Obs{Flags: ".1111"}, false}, + {Obs{Flags: "1234."}, false}, + {Obs{Flags: "12.2.4"}, false}, + {Obs{Flags: "12.343"}, false}, + {Obs{Flags: ""}, false}, + {Obs{Flags: "asdas"}, false}, + {Obs{Flags: "12a3a"}, false}, + {Obs{Flags: "1sdfl"}, false}, + } + + for _, c := range cases { + t.Log("Testing flag:", c.input.Flags) + + if result := c.input.flagsAreValid(); result != c.expected { + t.Errorf("Got %v, wanted %v", result, c.expected) + } + } +} diff --git a/migrations/kdvh/list_tables.go b/migrations/kdvh/list_tables.go new file mode 100644 index 00000000..c030a6c7 --- /dev/null +++ b/migrations/kdvh/list_tables.go @@ -0,0 +1,24 @@ +package kdvh + +import ( + "fmt" + "slices" +) + +type ListConfig struct{} + +func (config *ListConfig) Execute(_ []string) error { + fmt.Println("Available tables in KDVH:") + + var tables []string + for table := range KDVH { + tables = append(tables, table) + } + + slices.Sort(tables) + for _, table := range tables { + fmt.Println(" -", table) + } + + return nil +} diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go new file mode 100644 index 00000000..af104227 --- /dev/null +++ b/migrations/kdvh/main.go @@ -0,0 +1,53 @@ +package kdvh + +// Command line arguments for KDVH migrations +type Cmd struct { + Dump DumpConfig `command:"dump" description:"Dump tables from KDVH to CSV"` + Import ImportConfig `command:"import" description:"Import CSV file dumped from KDVH"` + List ListConfig `command:"list" description:"List available KDVH tables"` +} + +// The KDVH database simply contains a map of "table name" to `Table` +var KDVH map[string]*Table = map[string]*Table{ + // Section 1: tables that need to be migrated entirely + // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(makeDataPageEdata).SetImport(3000), + // NOTE(1): there is a T_METARFLAG, but it's empty + // NOTE(2): already dumped, but with wrong format? + "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), // already dumped + + // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 + "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImport(2006), + "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPagePdata).SetImport(2006), // already dumped + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageNdata).SetImport(2006), // already dumped + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageVdata).SetImport(2006), // already dumped + "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + + // Section 3: tables that should only be dumped + "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), + "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), + + // TODO: T_AVINOR, T_PROJDATA have a bunch of parameters that are not in Stinfosys? + // But it shouldn't be a problem if the goal is to only dump them? + "T_AVINOR": NewTable("T_AVINOR", "T_AVINOR_FLAG", "T_ELEM_OBS"), + // TODO: T_PROJFLAG is not in the proxy! And T_PROJDATA is not readable from the proxy + // "T_PROJDATA": newTable("T_PROJDATA", "T_PROJFLAG", "T_ELEM_PROJ"), + "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), // already dumped + "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), // already dumped + "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), // already dumped + "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), // already dumped + "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), // already dumped + + // Section 4: other special cases + // TODO: do we need to import these? + "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(makeDataPageProduct).SetImport(1957), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(makeDataPageProduct), + "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(makeDataPageProduct), + "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(makeDataPageProduct), + + // TODO: these two are the only tables seemingly missing from the KDVH proxy + // {TableName: "T_DIURNAL_INTERPOLATED", DataFunction: makeDataPageDiurnalInterpolated, ImportUntil: 3000}, + // {TableName: "T_MONTH_INTERPOLATED", DataFunction: makeDataPageDiurnalInterpolated, ImportUntil: 3000}, +} diff --git a/migrations/kdvh/product_offsets.csv b/migrations/kdvh/product_offsets.csv new file mode 100644 index 00000000..7ee4fdb1 --- /dev/null +++ b/migrations/kdvh/product_offsets.csv @@ -0,0 +1,161 @@ +table_name,elem_code,paramid,fromtime_offset,timespan +T_DIURNAL,EE,129,PT6H, +T_DIURNAL,EM,7,PT6H, +T_DIURNAL,FF2M,3044,-PT1H,P1D +T_DIURNAL,FF2N,3046,-PT1H,P1D +T_DIURNAL,FF2X,3049,-PT1H,P1D +T_DIURNAL,FFM,3050,-PT1H,P1D +T_DIURNAL,FFN,3052,-PT1H,P1D +T_DIURNAL,FFX,3054,-PT1H,P1D +T_DIURNAL,FGM,3056,-PT6H,P1D +T_DIURNAL,FGN,3058,-PT6H,P1D +T_DIURNAL,FGX,3060,-PT6H,P1D +T_DIURNAL,FLRR,,-PT18H,P1D +T_DIURNAL,FXM,3063,-PT6H,P1D +T_DIURNAL,FXN,3065,-PT6H,P1D +T_DIURNAL,FXX,3067,-PT6H,P1D +T_DIURNAL,GD17,3073,-PT1H,P1D +T_DIURNAL,HWAM,3147,-PT1H,P1D +T_DIURNAL,HWAN,3151,-PT1H,P1D +T_DIURNAL,HWAX,3149,-PT1H,P1D +T_DIURNAL,MR,3197,-PT1H,P1D +T_DIURNAL,NN04,3075,-PT1H,P1D +T_DIURNAL,NN09,3077,-PT1H,P1D +T_DIURNAL,NN20,3079,-PT1H,P1D +T_DIURNAL,NNM,3081,-PT1H,P1D +T_DIURNAL,NNN,3086,-PT1H,P1D +T_DIURNAL,NNX,3088,-PT1H,P1D +T_DIURNAL,OT,122,-P1D,P1D +T_DIURNAL,POM,3032,-PT1H,P1D +T_DIURNAL,PON,3035,-PT1H,P1D +T_DIURNAL,POX,3037,-PT1H,P1D +T_DIURNAL,PRM,3093,-PT1H,P1D +T_DIURNAL,PRN,3095,-PT1H,P1D +T_DIURNAL,PRX,3097,-PT1H,P1D +T_DIURNAL,PWAM,3141,-PT1H,P1D +T_DIURNAL,PWAN,3145,-PT1H,P1D +T_DIURNAL,PWAX,3143,-PT1H,P1D +T_DIURNAL,RR,110,-PT18H,P1D +T_DIURNAL,RR_720,3243,-P29DT18H,P30D +T_DIURNAL,RRID,117,PT6H, +T_DIURNAL,RRTA,3241,-PT6H,P1D +T_DIURNAL,SA,112,PT6H, +T_DIURNAL,SAE,3109,-PT18H,P1D +T_DIURNAL,SD,18,PT6H, +T_DIURNAL,SGN,3119,-PT1H,P1D +T_DIURNAL,SGX,3121,-PT1H,P1D +T_DIURNAL,SH,3123,-PT1H,P1D +T_DIURNAL,SLAG,10051,-PT18H,P1D +T_DIURNAL,SLAGV,10052,-PT18H,P1D +T_DIURNAL,SLAGW,10053,-PT18H,P1D +T_DIURNAL,SLAGWA,10054,-PT18H,P1D +T_DIURNAL,SS_24,114,-PT18H,P1D +T_DIURNAL,TAM,3016,-PT1H,P1D +T_DIURNAL,X1TAM,3016,-PT1H,P1D +T_DIURNAL,TAM_K,3125,-PT1H,P1D +T_DIURNAL,TAM10,3016,-PT1H,P1D +T_DIURNAL,TAMRR,3300,-PT18H,P1D +T_DIURNAL,TAN,3304,-PT6H,P1D +T_DIURNAL,X1TAN,3304,-PT6H,P1D +T_DIURNAL,TAND,3302,-PT1H,P1D +T_DIURNAL,TAX,3305,-PT6H,P1D +T_DIURNAL,X1TAX,3305,-PT6H,P1D +T_DIURNAL,TAXD,3303,-PT1H,P1D +T_DIURNAL,TD,3026,-PT1H,P1D +T_DIURNAL,TGN,3028,-PT1H,P1D +T_DIURNAL,TW,3306,-PT1H,P1D +T_DIURNAL,UM,266,-PT1H,P1D +T_DIURNAL,UM10,266,-PT1H,P1D +T_DIURNAL,UN,3006,-PT1H,P1D +T_DIURNAL,UX,3004,-PT1H,P1D +T_DIURNAL,VEKST,3134,-PT1H,P1D +T_DIURNAL,VP,3136,-PT1H,P1D +T_DIURNAL,VSUM,3138,-PT1H,P1D +T_DIURNAL,VVN,3002,-PT1H,P1D +T_DIURNAL,VVX,3000,-PT1H,P1D +T_MONTH,DRR_GE1,3194,-PT1H,P1M +T_MONTH,FF2M,3045,-PT1H,P1M +T_MONTH,FF2N,3047,-PT1H,P1M +T_MONTH,FF2X,3048,-PT1H,P1M +T_MONTH,FFM,3051,-PT1H,P1M +T_MONTH,FFN,3053,-PT1H,P1M +T_MONTH,FFX,3055,-PT1H,P1M +T_MONTH,FGM,3057,-PT6H,P1M +T_MONTH,FGN,3059,-PT6H,P1M +T_MONTH,FGX,3061,-PT6H,P1M +T_MONTH,FXM,3062,-PT6H,P1M +T_MONTH,FXN,3064,-PT6H,P1M +T_MONTH,FXX,3066,-PT6H,P1M +T_MONTH,FXXDT,3068,-PT1H,P1M +T_MONTH,GD17,3069,-PT1H,P1M +T_MONTH,GD17_I,3074,-PT1H,P1M +T_MONTH,HWAM,3148,-PT1H,P1M +T_MONTH,HWAN,3152,-PT1H,P1M +T_MONTH,HWAX,3150,-PT1H,P1M +T_MONTH,MRM,3193,-PT1H,P1M +T_MONTH,NN04,3076,-PT1H,P1M +T_MONTH,NN09,3078,-PT1H,P1M +T_MONTH,NN20,3080,-PT1H,P1M +T_MONTH,NNM,3082,-PT1H,P1M +T_MONTH,NNN,3087,-PT1H,P1M +T_MONTH,NNX,3089,-PT1H,P1M +T_MONTH,OT,3090,-PT1H,P1M +T_MONTH,OTN,3091,-PT1H,P1M +T_MONTH,OTX,3092,-PT1H,P1M +T_MONTH,POM,3033,-PT1H,P1M +T_MONTH,PON,3036,-PT1H,P1M +T_MONTH,POX,3038,-PT1H,P1M +T_MONTH,PRM,3094,-PT1H,P1M +T_MONTH,PRN,3096,-PT1H,P1M +T_MONTH,PRX,3098,-PT1H,P1M +T_MONTH,PWAM,3142,-PT1H,P1M +T_MONTH,PWAN,3146,-PT1H,P1M +T_MONTH,PWAX,3144,-PT1H,P1M +T_MONTH,RR,3102,-PT18H,P1M +T_MONTH,RR_24X,3196,-PT18H,P1M +T_MONTH,RR_24XDT,3195,-PT18H,P1M +T_MONTH,RRA,3175,-PT18H,P1M +T_MONTH,RRA_6190,3175,-PT18H,P1M +T_MONTH,RRA_9120,3163,-PT18H,P1M +T_MONTH,RRID,117,PT6H, +T_MONTH,RRTA,3240,-PT6H,P1M +T_MONTH,SAM,3110,-PT18H,P1M +T_MONTH,SAN,3112,-PT18H,P1M +T_MONTH,SAX,3114,-PT18H,P1M +T_MONTH,SDM,3116,-PT18H,P1M +T_MONTH,SDN,3117,-PT18H,P1M +T_MONTH,SDX,3118,-PT18H,P1M +T_MONTH,SGN,3120,-PT1H,P1M +T_MONTH,SGX,3122,-PT1H,P1M +T_MONTH,SHM,3124,-PT1H,P1M +T_MONTH,TAM,3015,-PT1H,P1M +T_MONTH,TAM_K,3126,-PT1H,P1M +T_MONTH,TAMA,3170,-PT1H,P1M +T_MONTH,TAMA_6190,3170,-PT1H,P1M +T_MONTH,TAMA_9120,3159,-PT1H,P1M +T_MONTH,TAMRR,3301,-PT18H,P1M +T_MONTH,TAN,3018,-PT6H,P1M +T_MONTH,TANDT,3127,-PT6H,P1M +T_MONTH,TANM,3128,-PT6H,P1M +T_MONTH,TAX,3022,-PT6H,P1M +T_MONTH,TAXDT,3129,-PT6H,P1M +T_MONTH,TAXM,3130,-PT6H,P1M +T_MONTH,TD,3027,-PT1H,P1M +T_MONTH,TGNM,3131,-PT1H,P1M +T_MONTH,TGNN,3132,-PT1H,P1M +T_MONTH,TGNX,3133,-PT1H,P1M +T_MONTH,TWM,3029,-PT1H,P1M +T_MONTH,TWN,3030,-PT1H,P1M +T_MONTH,TWX,3031,-PT1H,P1M +T_MONTH,UM,3008,-PT1H,P1M +T_MONTH,UN,3007,-PT1H,P1M +T_MONTH,UX,3005,-PT1H,P1M +T_MONTH,VEKST,3135,-PT1H,P1M +T_MONTH,VP,3137,-PT1H,P1M +T_MONTH,VSUM,3139,-PT1H,P1M +T_MONTH,VVN,3003,-PT1H,P1M +T_MONTH,VVX,3001,-PT1H,P1M +T_HOMOGEN_DIURNAL,TAM,3009,-PT1H,P1D +T_HOMOGEN_DIURNAL,RR,3247,-PT18H,P1D +T_HOMOGEN_MONTH,TAM,3010,-PT1H,P1M +T_HOMOGEN_MONTH,RR,3099,-PT18H,P1M \ No newline at end of file diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go new file mode 100644 index 00000000..6a155740 --- /dev/null +++ b/migrations/kdvh/table.go @@ -0,0 +1,125 @@ +package kdvh + +import ( + "database/sql" + "errors" + "fmt" + "log/slog" + "migrate/lard" + "time" + + "github.com/rickb777/period" +) + +// In KDVH for each table name we usually have three separate tables: +// 1. A DATA table containing observation values; +// 2. A FLAG table containing quality control (QC) flags; +// 3. A ELEM table containing metadata about the validity of the timeseries. +// +// DATA and FLAG tables have the same schema: +// | dato | stnr | ... | +// where 'dato' is the timestamp of the observation, 'stnr' is the station +// where the observation was measured, and '...' is a varying number of columns +// each with different observations, where the column name is the 'elem_code' +// (e.g. for air temperature, 'ta'). +// +// TODO: are the timestamps UTC? Otherwise we probably need to convert them during import +// +// The ELEM tables have the following schema: +// | stnr | elem_code | fdato | tdato | table_name | flag_table_name | audit_dato + +// Table contains metadata on how to treat different tables in KDVH +type Table struct { + TableName string // Name of the DATA table + FlagTableName string // Name of the FLAG table + ElemTableName string // Name of the ELEM table + Path string // Directory name of where the dumped table is stored + dumpFunc DumpFunction // Function used to dump the KDVH table (found in `dump_functions.go`) + convFunc ConvertFunction // Function that converts KDVH obs to LARD obs (found in `import_functions.go`) + importUntil int // Import data only until the year specified by this field +} + +type DumpFunction func(path string, meta DumpMeta, conn *sql.DB) error +type DumpMeta struct { + element string + station string + dataTable string + flagTable string +} + +type ConvertFunction func(Obs) (lard.Obs, error) +type Obs struct { + *TimeseriesInfo + Obstime time.Time + Data string + Flags string +} + +// Convenience struct that holds information for a specific timeseries +type TimeseriesInfo struct { + station int32 + element string + offset period.Period + param StinfoParam + span Timespan + logstr string +} + +func (config *ImportConfig) NewTimeseriesInfo(table, element string, station int32) (*TimeseriesInfo, error) { + logstr := fmt.Sprintf("%v - %v - %v: ", table, station, element) + key := newKDVHKey(element, table, station) + + meta, ok := config.StinfoMap[key.Inner] + if !ok { + // TODO: should it fail here? How do we deal with data without metadata? + slog.Error(logstr + "Missing metadata in Stinfosys") + return nil, errors.New("") + } + + // No need to check for `!ok`, will default to 0 offset + offset := config.OffsetMap[key.Inner] + + // No need to check for `!ok`, timespan will be ignored if not in the map + span := config.KDVHMap[key] + + return &TimeseriesInfo{ + station: station, + element: element, + offset: offset, + param: meta, + span: span, + logstr: logstr, + }, nil +} + +// Creates default Table +func NewTable(data, flag, elem string) *Table { + return &Table{ + TableName: data, + FlagTableName: flag, + ElemTableName: elem, + Path: data + "_combined", // NOTE: '_combined' kept for backward compatibility with original scripts + dumpFunc: dumpDataAndFlags, + convFunc: makeDataPage, + } +} + +// Sets the `ImportUntil` field if the year is greater than 0 +func (t *Table) SetImport(year int) *Table { + if year > 0 { + t.importUntil = year + } + return t +} + +// Sets the function used to dump the Table +func (t *Table) SetDumpFunc(fn DumpFunction) *Table { + t.dumpFunc = fn + return t +} + +// Sets the function used to convert observations from the table to LARD observations +func (t *Table) SetConvFunc(fn ConvertFunction) *Table { + t.convFunc = fn + return t +} diff --git a/migrations/kdvh_test.go b/migrations/kdvh_test.go new file mode 100644 index 00000000..3f564e8c --- /dev/null +++ b/migrations/kdvh_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/joho/godotenv" + // "github.com/rickb777/period" + + "migrate/kdvh" +) + +const LARD_STRING string = "host=localhost user=postgres dbname=postgres password=postgres" + +func mockConfig(t *ImportTest) *kdvh.ImportConfig { + config := kdvh.ImportConfig{ + Tables: []string{t.table}, + Stations: []string{fmt.Sprint(t.station)}, + Elements: []string{t.elem}, + BaseDir: "./tests", + HasHeader: true, + Sep: ";", + } + + config.CacheMetadata() + return &config + +} + +type ImportTest struct { + table string + station int32 + elem string + expectedRows int64 +} + +func TestImportKDVH(t *testing.T) { + err := godotenv.Load() + if err != nil { + fmt.Println(err) + return + } + + // TODO: could also define a smaller version just for tests + db := kdvh.KDVH + + pool, err := pgxpool.New(context.TODO(), LARD_STRING) + if err != nil { + t.Log("Could not connect to Lard:", err) + } + defer pool.Close() + + testCases := []ImportTest{ + {table: "T_MDATA", station: 12345, elem: "TA", expectedRows: 2644}, + } + + for _, c := range testCases { + config := mockConfig(&c) + table, ok := db[c.table] + if !ok { + t.Fatal("Table does not exist in database") + } + + insertedRows := table.Import(pool, config) + if insertedRows != c.expectedRows { + t.Fail() + } + } +} diff --git a/migrations/lard/import.go b/migrations/lard/import.go new file mode 100644 index 00000000..408cc641 --- /dev/null +++ b/migrations/lard/import.go @@ -0,0 +1,92 @@ +package lard + +import ( + "context" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func InsertData(data DataInserter, pool *pgxpool.Pool, logStr string) (int64, error) { + size := data.Len() + count, err := pool.CopyFrom( + context.TODO(), + pgx.Identifier{"public", "data"}, + []string{"timeseries", "obstime", "obsvalue"}, + pgx.CopyFromSlice(size, func(i int) ([]any, error) { + return []any{ + data.ID(), + data.Obstime(i), + data.Data(i), + }, nil + }), + ) + if err != nil { + return count, err + } + + logStr += fmt.Sprintf("%v/%v data rows inserted", count, size) + if int(count) != size { + slog.Warn(logStr) + } else { + slog.Info(logStr) + } + return count, nil +} + +func InsertNonscalarData(data TextInserter, pool *pgxpool.Pool, logStr string) (int64, error) { + size := data.Len() + count, err := pool.CopyFrom( + context.TODO(), + pgx.Identifier{"public", "nonscalar_data"}, + []string{"timeseries", "obstime", "obsvalue"}, + pgx.CopyFromSlice(size, func(i int) ([]any, error) { + return []any{ + data.ID(), + data.Obstime(i), + data.Text(i), + }, nil + }), + ) + if err != nil { + return count, err + } + + logStr += fmt.Sprintf("%v/%v non-scalar data rows inserted", count, size) + if int(count) != size { + slog.Warn(logStr) + } else { + slog.Info(logStr) + } + return count, nil +} + +func InsertFlags(data FlagInserter, pool *pgxpool.Pool, logStr string) error { + size := data.Len() + count, err := pool.CopyFrom( + context.TODO(), + pgx.Identifier{"flags", "kdvh"}, + []string{"timeseries", "obstime", "controlinfo", "useinfo"}, + pgx.CopyFromSlice(size, func(i int) ([]any, error) { + return []any{ + data.ID(), + data.Obstime(i), + data.Controlinfo(i), + data.Useinfo(i), + }, nil + }), + ) + if err != nil { + return err + } + + logStr += fmt.Sprintf("%v/%v flag rows inserted", count, size) + if int(count) != size { + slog.Warn(logStr) + } else { + slog.Info(logStr) + } + return nil +} diff --git a/migrations/lard/main.go b/migrations/lard/main.go new file mode 100644 index 00000000..d245f5de --- /dev/null +++ b/migrations/lard/main.go @@ -0,0 +1,80 @@ +package lard + +import "time" + +// Timeseries in LARD have and ID and associated observations +type Timeseries struct { + id int32 + data []Obs +} + +func NewTimeseries(id int32, data []Obs) *Timeseries { + return &Timeseries{id, data} +} + +func (ts *Timeseries) Len() int { + return len(ts.data) +} + +func (ts *Timeseries) ID() int32 { + return ts.id +} + +func (ts *Timeseries) Obstime(i int) time.Time { + return ts.data[i].Obstime +} + +func (ts *Timeseries) Text(i int) string { + return *ts.data[i].Text +} + +func (ts *Timeseries) Data(i int) float32 { + return *ts.data[i].Data +} + +func (ts *Timeseries) Controlinfo(i int) string { + return ts.data[i].Controlinfo +} + +func (ts *Timeseries) Useinfo(i int) string { + return ts.data[i].Useinfo +} + +// Struct containg all the fields we want to save in LARD +type Obs struct { + // Time of observation + Obstime time.Time + // Observation data formatted as a single precision floating point number + Data *float32 + // Observation data that cannot be represented as a float, therefore stored as a string + Text *string + // Flag encoding quality control status + Controlinfo string + // Flag encoding quality control status + Useinfo string +} + +// TODO: I'm not sure I like the interface solution +type DataInserter interface { + Obstime(i int) time.Time + Data(i int) float32 + ID() int32 + Len() int +} + +type TextInserter interface { + Obstime(i int) time.Time + Text(i int) string + ID() int32 + Len() int +} + +// TODO: This maybe needs different implementation for each system +// i.e. insert to different tables and different columns +type FlagInserter interface { + ID() int32 + Obstime(i int) time.Time + Controlinfo(i int) string + Useinfo(i int) string + Len() int +} diff --git a/migrations/lard/timeseries.go b/migrations/lard/timeseries.go new file mode 100644 index 00000000..5629b3c4 --- /dev/null +++ b/migrations/lard/timeseries.go @@ -0,0 +1,62 @@ +package lard + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// Struct that mimics `labels.met` table structure +type Label struct { + StationID int32 + TypeID int32 + ParamID int32 + Sensor *int32 + Level *int32 +} + +func GetTimeseriesID(label Label, fromtime time.Time, pool *pgxpool.Pool) (tsid int32, err error) { + // Query LARD labels table + err = pool.QueryRow( + context.TODO(), + `SELECT timeseries FROM labels.met + WHERE station_id = $1 + AND param_id = $2 + AND type_id = $3 + AND (($4::int IS NULL AND lvl IS NULL) OR (lvl = $4)) + AND (($5::int IS NULL AND sensor IS NULL) OR (sensor = $5))`, + label.StationID, label.ParamID, label.TypeID, label.Level, label.Sensor).Scan(&tsid) + + // If timeseries exists, return its ID + if err == nil { + return tsid, nil + } + + // Otherwise insert new timeseries + transaction, err := pool.Begin(context.TODO()) + if err != nil { + return tsid, err + } + + err = transaction.QueryRow( + context.TODO(), + `INSERT INTO public.timeseries (fromtime) VALUES ($1) RETURNING id`, + fromtime, + ).Scan(&tsid) + if err != nil { + return tsid, err + } + + _, err = transaction.Exec( + context.TODO(), + `INSERT INTO labels.met (timeseries, station_id, param_id, type_id, lvl, sensor) + VALUES ($1, $2, $3, $4, $5, $6)`, + tsid, label.StationID, label.ParamID, label.TypeID, label.Level, label.Sensor) + if err != nil { + return tsid, err + } + + err = transaction.Commit(context.TODO()) + return tsid, err +} diff --git a/migrations/main.go b/migrations/main.go new file mode 100644 index 00000000..78ae62c8 --- /dev/null +++ b/migrations/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "log" + + "github.com/jessevdk/go-flags" + "github.com/joho/godotenv" + + "migrate/kdvh" +) + +type CmdArgs struct { + KDVH kdvh.Cmd `command:"kdvh" description:"Perform KDVH migrations"` +} + +func main() { + log.SetFlags(log.LstdFlags | log.Lshortfile) + + // The following env variables are needed: + // 1. Dump + // - kdvh: "KDVH_PROXY_CONN" + // + // 2. Import + // - kdvh: "LARD_STRING", "STINFO_STRING", "KDVH_PROXY_CONN" + err := godotenv.Load() + if err != nil { + fmt.Println(err) + return + } + + // NOTE: go-flags calls the Execute method on the parsed subcommand + _, err = flags.Parse(&CmdArgs{}) + if err != nil { + if flagsErr, ok := err.(*flags.Error); ok { + if flagsErr.Type == flags.ErrHelp { + return + } + } + fmt.Println("Type './migrate -h' for help") + return + } +} diff --git a/migrations/tests/T_MDATA_combined/12345/TA.csv b/migrations/tests/T_MDATA_combined/12345/TA.csv new file mode 100644 index 00000000..dd6cb263 --- /dev/null +++ b/migrations/tests/T_MDATA_combined/12345/TA.csv @@ -0,0 +1,2645 @@ +2644 +2001-07-01_09:00:00;12.9;70000 +2001-07-01_10:00:00;13;70000 +2001-07-01_11:00:00;13;70000 +2001-07-01_12:00:00;13.1;70000 +2001-07-01_13:00:00;13.1;70000 +2001-07-01_14:00:00;13;70000 +2001-07-01_15:00:00;12.9;70000 +2001-07-01_16:00:00;12.8;70000 +2001-07-01_17:00:00;12.8;70000 +2001-07-01_18:00:00;12.7;70000 +2001-07-01_19:00:00;12.8;70000 +2001-07-01_20:00:00;12.6;70000 +2001-07-01_21:00:00;12.6;70000 +2001-07-01_22:00:00;12.6;70000 +2001-07-01_23:00:00;12.6;70000 +2001-07-02_00:00:00;12.5;70000 +2001-07-02_01:00:00;12.4;70000 +2001-07-02_02:00:00;12.4;70000 +2001-07-02_03:00:00;12.3;70000 +2001-07-02_04:00:00;12.3;70000 +2001-07-02_05:00:00;12.3;70000 +2001-07-02_06:00:00;12.4;70000 +2001-07-02_07:00:00;12.5;70000 +2001-07-02_08:00:00;12.6;70000 +2001-07-02_09:00:00;12.7;70000 +2001-07-02_10:00:00;12.9;70000 +2001-07-02_11:00:00;13;70000 +2001-07-02_12:00:00;13.2;70000 +2001-07-02_13:00:00;13.3;70000 +2001-07-02_14:00:00;13.3;70000 +2001-07-02_15:00:00;13.4;70000 +2001-07-02_16:00:00;13.3;70000 +2001-07-02_17:00:00;13.3;70000 +2001-07-02_18:00:00;13.2;70000 +2001-07-02_19:00:00;13.2;70000 +2001-07-02_20:00:00;13.1;70000 +2001-07-02_21:00:00;12.9;70000 +2001-07-02_22:00:00;12.9;70000 +2001-07-02_23:00:00;12.9;70000 +2001-07-03_00:00:00;12.8;58927 +2001-07-03_01:00:00;12.7;70000 +2001-07-03_02:00:00;12.6;70000 +2001-07-03_03:00:00;12.7;70000 +2001-07-03_04:00:00;12.5;70000 +2001-07-03_05:00:00;12.2;70000 +2001-07-03_06:00:00;12.3;70000 +2001-07-03_07:00:00;12.4;70000 +2001-07-03_08:00:00;12.5;70000 +2001-07-03_09:00:00;12.6;70000 +2001-07-03_10:00:00;12.6;70000 +2001-07-03_11:00:00;12.8;70000 +2001-07-03_12:00:00;12.8;70000 +2001-07-03_13:00:00;13;70000 +2001-07-03_14:00:00;13.1;70000 +2001-07-03_15:00:00;13.2;70000 +2001-07-03_16:00:00;13.2;70000 +2001-07-03_17:00:00;13.1;70000 +2001-07-03_18:00:00;13.1;70000 +2001-07-03_19:00:00;13.1;70000 +2001-07-03_20:00:00;12.8;70000 +2001-07-03_21:00:00;12.8;70000 +2001-07-03_22:00:00;12.8;70000 +2001-07-03_23:00:00;12.8;70000 +2001-07-04_00:00:00;12.6;70000 +2001-07-04_01:00:00;12.8;70000 +2001-07-04_02:00:00;12.3;70000 +2001-07-04_03:00:00;12.6;70000 +2001-07-04_04:00:00;12.5;70000 +2001-07-04_05:00:00;12.5;70000 +2001-07-04_06:00:00;12.5;70000 +2001-07-04_07:00:00;12.5;70000 +2001-07-04_08:00:00;12.4;70000 +2001-07-04_09:00:00;12.5;70000 +2001-07-04_10:00:00;12.6;70000 +2001-07-04_11:00:00;12.6;70000 +2001-07-04_12:00:00;12.6;58927 +2001-07-04_13:00:00;12.5;70000 +2001-07-04_14:00:00;12.6;70000 +2001-07-04_15:00:00;12.5;70000 +2001-07-04_16:00:00;12.6;70000 +2001-07-04_17:00:00;12.6;70000 +2001-07-04_18:00:00;12.6;70000 +2001-07-04_19:00:00;12.5;70000 +2001-07-04_20:00:00;12.5;70000 +2001-07-04_21:00:00;12.5;70000 +2001-07-04_22:00:00;12.4;70000 +2001-07-04_23:00:00;12.4;70000 +2001-07-05_00:00:00;12.5;70000 +2001-07-05_01:00:00;12.4;70000 +2001-07-05_02:00:00;12.1;70000 +2001-07-05_03:00:00;11.9;70000 +2001-07-05_04:00:00;12;70000 +2001-07-05_05:00:00;12;70000 +2001-07-05_06:00:00;12.1;70000 +2001-07-05_07:00:00;12.3;70000 +2001-07-05_08:00:00;12.6;70000 +2001-07-05_09:00:00;12.9;70000 +2001-07-05_10:00:00;13;70000 +2001-07-05_11:00:00;13.2;70000 +2001-07-05_12:00:00;13.5;70000 +2001-07-05_13:00:00;13.8;70000 +2001-07-05_14:00:00;13.9;70000 +2001-07-05_15:00:00;13.4;70000 +2001-07-05_16:00:00;13.9;70000 +2001-07-05_17:00:00;13.8;70000 +2001-07-05_18:00:00;13.7;70000 +2001-07-05_19:00:00;13.6;70000 +2001-07-05_20:00:00;13.5;70000 +2001-07-05_21:00:00;13.3;70000 +2001-07-05_22:00:00;13.2;70000 +2001-07-05_23:00:00;13.1;70000 +2001-07-06_00:00:00;13.1;70000 +2001-07-06_01:00:00;13;70000 +2001-07-06_02:00:00;12.9;70000 +2001-07-06_03:00:00;12.8;70000 +2001-07-06_04:00:00;12.9;58927 +2001-07-06_05:00:00;12.9;70000 +2001-07-06_06:00:00;13.2;70000 +2001-07-06_07:00:00;13.2;70000 +2001-07-06_08:00:00;13.3;70000 +2001-07-06_09:00:00;13.8;70000 +2001-07-06_10:00:00;14.3;70000 +2001-07-06_11:00:00;14.7;70000 +2001-07-06_12:00:00;15.8;70000 +2001-07-06_13:00:00;14.9;70000 +2001-07-06_14:00:00;14.6;70000 +2001-07-06_15:00:00;14.7;70000 +2001-07-06_16:00:00;14.6;70000 +2001-07-06_17:00:00;15.5;70000 +2001-07-06_18:00:00;16.6;70000 +2001-07-06_19:00:00;15.5;70000 +2001-07-06_20:00:00;14.8;70000 +2001-07-06_21:00:00;14.7;70000 +2001-07-06_22:00:00;16.2;70000 +2001-07-06_23:00:00;15.6;70000 +2001-07-07_00:00:00;15.1;70000 +2001-07-07_01:00:00;14.4;70000 +2001-07-07_02:00:00;13.8;70000 +2001-07-07_03:00:00;13.2;70000 +2001-07-07_04:00:00;13.3;70000 +2001-07-07_05:00:00;13.6;70000 +2001-07-07_06:00:00;14;70000 +2001-07-07_07:00:00;14.1;70000 +2001-07-07_08:00:00;14.1;70000 +2001-07-07_09:00:00;14.3;70000 +2001-07-07_10:00:00;14.4;70000 +2001-07-07_11:00:00;14.5;70000 +2001-07-07_12:00:00;14.6;70000 +2001-07-07_13:00:00;14.9;70000 +2001-07-07_14:00:00;15;70000 +2001-07-07_15:00:00;14.9;70000 +2001-07-07_16:00:00;15;70000 +2001-07-07_17:00:00;14.9;70000 +2001-07-07_18:00:00;14.9;70000 +2001-07-07_19:00:00;14.8;70000 +2001-07-07_20:00:00;14.8;70000 +2001-07-07_21:00:00;15;70000 +2001-07-07_22:00:00;15;70000 +2001-07-07_23:00:00;15.3;70000 +2001-07-08_00:00:00;14.9;70000 +2001-07-08_01:00:00;14.6;70000 +2001-07-08_02:00:00;14.5;70000 +2001-07-08_03:00:00;14.4;70000 +2001-07-08_04:00:00;14.4;70000 +2001-07-08_05:00:00;14.7;70000 +2001-07-08_06:00:00;14.6;70000 +2001-07-08_07:00:00;14.3;70000 +2001-07-08_08:00:00;14.5;70000 +2001-07-08_09:00:00;14.5;70000 +2001-07-08_10:00:00;14.5;70000 +2001-07-08_11:00:00;15.1;70000 +2001-07-08_12:00:00;15.2;70000 +2001-07-08_13:00:00;15.5;70000 +2001-07-08_14:00:00;14.6;70000 +2001-07-08_15:00:00;16.9;78947 +2001-07-08_16:00:00;17.1;78947 +2001-07-08_17:00:00;16.9;78947 +2001-07-08_18:00:00;16;78947 +2001-07-08_19:00:00;15.5;78947 +2001-07-08_20:00:00;15.1;78947 +2001-07-08_21:00:00;14.9;78947 +2001-07-08_22:00:00;14.6;78947 +2001-07-08_23:00:00;14.3;78947 +2001-07-09_00:00:00;14.1;78947 +2001-07-09_01:00:00;14.2;78947 +2001-07-09_02:00:00;14.3;78947 +2001-07-09_03:00:00;14;78947 +2001-07-09_04:00:00;14;78947 +2001-07-09_05:00:00;14.2;78947 +2001-07-09_06:00:00;14;78947 +2001-07-09_07:00:00;14.6;78947 +2001-07-09_08:00:00;14.5;78947 +2001-07-09_09:00:00;15.3;78947 +2001-07-09_10:00:00;16.3;78947 +2001-07-09_11:00:00;15.1;78947 +2001-07-09_12:00:00;16.2;78947 +2001-07-09_13:00:00;15.2;78947 +2001-07-09_14:00:00;15.6;78947 +2001-07-09_15:00:00;15.4;78947 +2001-07-09_16:00:00;15.6;78947 +2001-07-09_17:00:00;15;78947 +2001-07-09_18:00:00;14.2;78947 +2001-07-09_19:00:00;13.7;78947 +2001-07-09_20:00:00;13.5;78947 +2001-07-09_21:00:00;13.2;78947 +2001-07-09_22:00:00;13.4;78947 +2001-07-09_23:00:00;13.5;78947 +2001-07-10_00:00:00;12.8;78947 +2001-07-10_01:00:00;12.9;78947 +2001-07-10_02:00:00;12.9;78947 +2001-07-10_03:00:00;13.2;78947 +2001-07-10_04:00:00;13.1;78947 +2001-07-10_05:00:00;13.3;78947 +2001-07-10_06:00:00;13.8;78947 +2001-07-10_07:00:00;13.9;78947 +2001-07-10_08:00:00;14.3;78947 +2001-07-10_09:00:00;14.7;78947 +2001-07-10_10:00:00;15.1;78947 +2001-07-10_11:00:00;15.3;78947 +2001-07-10_12:00:00;15.3;78947 +2001-07-10_13:00:00;16;78947 +2001-07-10_14:00:00;16.1;78947 +2001-07-10_15:00:00;15.6;78947 +2001-07-10_16:00:00;15;78947 +2001-07-10_17:00:00;14.5;78947 +2001-07-10_18:00:00;14.3;78947 +2001-07-10_19:00:00;13.5;78947 +2001-07-10_20:00:00;13.3;78947 +2001-07-10_21:00:00;12.9;78947 +2001-07-10_22:00:00;12.2;78947 +2001-07-10_23:00:00;11.9;78947 +2001-07-11_00:00:00;13;78947 +2001-07-11_01:00:00;12.7;78947 +2001-07-11_02:00:00;12.7;78947 +2001-07-11_03:00:00;12.6;78947 +2001-07-11_04:00:00;12.7;78947 +2001-07-11_05:00:00;12.8;78947 +2001-07-11_06:00:00;13.7;78947 +2001-07-11_07:00:00;13.7;78947 +2001-07-11_08:00:00;13.7;78947 +2001-07-11_09:00:00;14.4;78947 +2001-07-11_10:00:00;14.7;78947 +2001-07-11_11:00:00;15.2;78947 +2001-07-11_12:00:00;15.3;78947 +2001-07-11_13:00:00;13.7;78947 +2001-07-11_14:00:00;14.5;78947 +2001-07-11_15:00:00;15;78947 +2001-07-11_16:00:00;13.2;78947 +2001-07-11_17:00:00;12.9;78947 +2001-07-11_18:00:00;12.5;78947 +2001-07-11_19:00:00;12.3;78947 +2001-07-11_20:00:00;12.4;78947 +2001-07-11_21:00:00;12.4;78947 +2001-07-11_22:00:00;12.4;78947 +2001-07-11_23:00:00;12.5;78947 +2001-07-12_00:00:00;12;78947 +2001-07-12_01:00:00;12.1;78947 +2001-07-12_02:00:00;12.2;78947 +2001-07-12_03:00:00;12.2;78947 +2001-07-12_04:00:00;12.3;78947 +2001-07-12_05:00:00;12.3;78947 +2001-07-12_06:00:00;12.1;78947 +2001-07-12_07:00:00;12.4;78947 +2001-07-12_08:00:00;13.5;78947 +2001-07-12_09:00:00;13.1;78947 +2001-07-12_10:00:00;14;78947 +2001-07-12_11:00:00;15.2;78947 +2001-07-12_12:00:00;14.3;78947 +2001-07-12_13:00:00;13.9;78947 +2001-07-12_14:00:00;14.3;78947 +2001-07-12_15:00:00;14;78947 +2001-07-12_16:00:00;13.9;78947 +2001-07-12_17:00:00;13.7;78947 +2001-07-12_18:00:00;13.5;78947 +2001-07-12_19:00:00;13.1;78947 +2001-07-12_20:00:00;12.6;78947 +2001-07-12_21:00:00;12.2;78947 +2001-07-12_22:00:00;11.9;78947 +2001-07-12_23:00:00;11.9;78947 +2001-07-13_00:00:00;11.7;78947 +2001-07-13_01:00:00;11.5;78947 +2001-07-13_02:00:00;11.3;78947 +2001-07-13_03:00:00;11.1;78947 +2001-07-13_04:00:00;11.3;78947 +2001-07-13_05:00:00;12;78947 +2001-07-13_06:00:00;13.4;78947 +2001-07-13_08:00:00;15.5;78947 +2001-07-13_09:00:00;16.5;78947 +2001-07-13_10:00:00;17.4;78947 +2001-07-13_11:00:00;17.7;78947 +2001-07-13_12:00:00;17.3;78947 +2001-07-13_13:00:00;17.3;78947 +2001-07-13_14:00:00;17.3;78947 +2001-07-13_15:00:00;17;78947 +2001-07-13_16:00:00;16.4;78947 +2001-07-13_17:00:00;15.5;78947 +2001-07-13_18:00:00;14.9;78947 +2001-07-13_19:00:00;14.1;78947 +2001-07-13_20:00:00;13.2;78947 +2001-07-13_21:00:00;12.3;78947 +2001-07-13_23:00:00;11.5;78947 +2001-07-14_00:00:00;11.2;78947 +2001-07-14_01:00:00;10.9;78947 +2001-07-14_02:00:00;10.7;78947 +2001-07-14_03:00:00;10.6;78947 +2001-07-14_04:00:00;10.7;78947 +2001-07-14_05:00:00;11.6;78947 +2001-07-14_06:00:00;13.1;78947 +2001-07-14_07:00:00;14.5;78947 +2001-07-14_08:00:00;15.9;78947 +2001-07-14_09:00:00;17.2;78947 +2001-07-14_10:00:00;18.3;78947 +2001-07-14_11:00:00;18.8;78947 +2001-07-14_12:00:00;18.5;78947 +2001-07-14_13:00:00;17.9;78947 +2001-07-14_14:00:00;17.4;78947 +2001-07-14_15:00:00;17.1;78947 +2001-07-14_16:00:00;17;78947 +2001-07-14_17:00:00;16.6;78947 +2001-07-14_18:00:00;16.4;78947 +2001-07-14_19:00:00;15.4;78947 +2001-07-14_20:00:00;14.6;78947 +2001-07-14_21:00:00;13.7;78947 +2001-07-14_23:00:00;12.9;78947 +2001-07-15_00:00:00;12.7;78947 +2001-07-15_01:00:00;12.5;78947 +2001-07-15_02:00:00;12.5;78947 +2001-07-15_03:00:00;12.4;78947 +2001-07-15_05:00:00;12.8;78947 +2001-07-15_06:00:00;13.6;78947 +2001-07-15_10:00:00;16.7;78947 +2001-07-15_11:00:00;16.6;78947 +2001-07-15_13:00:00;16.1;78947 +2001-07-15_15:00:00;15.9;78947 +2001-07-15_16:00:00;15.5;78947 +2001-07-15_17:00:00;15.1;78947 +2001-07-15_18:00:00;14.7;78947 +2001-07-15_19:00:00;14.2;78947 +2001-07-15_20:00:00;13.5;78947 +2001-07-15_21:00:00;12.4;78947 +2001-07-15_22:00:00;11.5;78947 +2001-07-15_23:00:00;10.9;78947 +2001-07-16_00:00:00;10.1;78947 +2001-07-16_01:00:00;10.6;78947 +2001-07-16_02:00:00;11.8;78947 +2001-07-16_03:00:00;12.7;78947 +2001-07-16_04:00:00;13.2;78947 +2001-07-16_05:00:00;13.6;78947 +2001-07-16_06:00:00;14;78947 +2001-07-16_07:00:00;15.2;78947 +2001-07-16_08:00:00;16.6;78947 +2001-07-16_09:00:00;17.7;78947 +2001-07-16_10:00:00;18.8;78947 +2001-07-16_11:00:00;19.5;78947 +2001-07-16_12:00:00;20.5;78947 +2001-07-16_13:00:00;20.6;78947 +2001-07-16_14:00:00;20.7;78947 +2001-07-16_15:00:00;19.3;78947 +2001-07-16_16:00:00;19.4;78947 +2001-07-16_17:00:00;18.5;78947 +2001-07-16_18:00:00;17.1;78947 +2001-07-16_19:00:00;15.8;78947 +2001-07-16_20:00:00;15.1;78947 +2001-07-16_21:00:00;15.1;78947 +2001-07-16_22:00:00;15.5;78947 +2001-07-16_23:00:00;15.1;78947 +2001-07-17_00:00:00;15.3;78947 +2001-07-17_01:00:00;15.1;78947 +2001-07-17_02:00:00;15.3;78947 +2001-07-17_03:00:00;15.3;78947 +2001-07-17_04:00:00;15.2;78947 +2001-07-17_05:00:00;15.1;78947 +2001-07-17_06:00:00;14.8;78947 +2001-07-17_07:00:00;14.8;78947 +2001-07-17_08:00:00;14.8;78947 +2001-07-17_09:00:00;15;78947 +2001-07-17_10:00:00;15.2;78947 +2001-07-17_11:00:00;15.2;78947 +2001-07-17_12:00:00;15.3;78947 +2001-07-17_13:00:00;15.4;78947 +2001-07-17_14:00:00;15.5;78947 +2001-07-17_15:00:00;15.5;78947 +2001-07-17_16:00:00;16.4;78947 +2001-07-17_17:00:00;16.2;78947 +2001-07-17_18:00:00;14.6;78947 +2001-07-17_19:00:00;14.7;78947 +2001-07-17_20:00:00;14.8;78947 +2001-07-17_21:00:00;14.1;78947 +2001-07-17_22:00:00;13.7;78947 +2001-07-17_23:00:00;13.7;78947 +2001-07-18_00:00:00;13.4;78947 +2001-07-18_01:00:00;14;78947 +2001-07-18_02:00:00;14.3;78947 +2001-07-18_03:00:00;13.9;78947 +2001-07-18_04:00:00;13.4;78947 +2001-07-18_05:00:00;13.7;78947 +2001-07-18_06:00:00;16.2;78947 +2001-07-18_07:00:00;17.4;78947 +2001-07-18_08:00:00;18.3;78947 +2001-07-18_11:00:00;20.1;78947 +2001-07-18_12:00:00;19.7;78947 +2001-07-18_13:00:00;18.6;78947 +2001-07-18_14:00:00;19.3;78947 +2001-07-18_15:00:00;18.3;78947 +2001-07-18_16:00:00;16.6;78947 +2001-07-18_17:00:00;17;78947 +2001-07-18_18:00:00;16.9;78947 +2001-07-18_19:00:00;16.5;78947 +2001-07-18_20:00:00;15.1;78947 +2001-07-18_21:00:00;14.7;78947 +2001-07-18_22:00:00;14.3;78947 +2001-07-18_23:00:00;14;78947 +2001-07-19_00:00:00;14.1;78947 +2001-07-19_01:00:00;14;78947 +2001-07-19_02:00:00;14.1;78947 +2001-07-19_03:00:00;14.2;78947 +2001-07-19_04:00:00;13.9;78947 +2001-07-19_05:00:00;13.8;78947 +2001-07-19_06:00:00;14.7;78947 +2001-07-19_07:00:00;15.7;78947 +2001-07-19_08:00:00;15.7;78947 +2001-07-19_09:00:00;17.2;78947 +2001-07-19_10:00:00;18.4;78947 +2001-07-19_11:00:00;18.3;78947 +2001-07-19_12:00:00;16.1;78947 +2001-07-19_13:00:00;15.5;78947 +2001-07-19_14:00:00;16;78947 +2001-07-19_15:00:00;16.5;78947 +2001-07-19_16:00:00;15.3;78947 +2001-07-19_17:00:00;15.4;78947 +2001-07-19_18:00:00;15;78947 +2001-07-19_19:00:00;14.4;78947 +2001-07-19_20:00:00;14.2;78947 +2001-07-19_21:00:00;14.1;78947 +2001-07-19_22:00:00;14;78947 +2001-07-19_23:00:00;13.6;78947 +2001-07-20_00:00:00;13.8;78947 +2001-07-20_01:00:00;13.8;78947 +2001-07-20_02:00:00;13.6;78947 +2001-07-20_03:00:00;13.7;78947 +2001-07-20_04:00:00;13.6;78947 +2001-07-20_05:00:00;14;78947 +2001-07-20_06:00:00;15.1;78947 +2001-07-20_07:00:00;15.6;78947 +2001-07-20_08:00:00;15.4;78947 +2001-07-20_09:00:00;16;78947 +2001-07-20_10:00:00;16.6;78947 +2001-07-20_11:00:00;17.1;78947 +2001-07-20_12:00:00;17.3;78947 +2001-07-20_13:00:00;17;78947 +2001-07-20_14:00:00;16.5;78947 +2001-07-20_15:00:00;16.4;78947 +2001-07-20_16:00:00;15.7;78947 +2001-07-20_17:00:00;14.9;78947 +2001-07-20_18:00:00;14.4;78947 +2001-07-20_19:00:00;14.1;78947 +2001-07-20_20:00:00;13.8;78947 +2001-07-20_21:00:00;13.7;78947 +2001-07-20_22:00:00;13.5;78947 +2001-07-20_23:00:00;13.4;78947 +2001-07-21_00:00:00;13.4;78947 +2001-07-21_01:00:00;13.4;78947 +2001-07-21_02:00:00;13.4;78947 +2001-07-21_03:00:00;13.3;78947 +2001-07-21_04:00:00;13.2;78947 +2001-07-21_05:00:00;13.2;78947 +2001-07-21_06:00:00;13.2;78947 +2001-07-21_07:00:00;13.4;78947 +2001-07-21_08:00:00;14;78947 +2001-07-21_09:00:00;14.6;78947 +2001-07-21_10:00:00;15.2;78947 +2001-07-21_11:00:00;15.4;78947 +2001-07-21_12:00:00;16.5;78947 +2001-07-21_13:00:00;16.2;78947 +2001-07-21_14:00:00;15.8;78947 +2001-07-21_15:00:00;15.4;78947 +2001-07-21_16:00:00;15.1;78947 +2001-07-21_17:00:00;14.7;78947 +2001-07-21_18:00:00;13.9;78947 +2001-07-21_19:00:00;13.4;78947 +2001-07-21_20:00:00;13;78947 +2001-07-21_21:00:00;12.8;78947 +2001-07-21_22:00:00;12.8;78947 +2001-07-21_23:00:00;12.9;78947 +2001-07-22_00:00:00;13;78947 +2001-07-22_01:00:00;13.1;78947 +2001-07-22_02:00:00;13.2;78947 +2001-07-22_03:00:00;13.2;78947 +2001-07-22_04:00:00;13.3;78947 +2001-07-22_05:00:00;13.5;78947 +2001-07-22_06:00:00;14;78947 +2001-07-22_07:00:00;14.7;78947 +2001-07-22_08:00:00;15.5;78947 +2001-07-22_09:00:00;15.8;78947 +2001-07-22_10:00:00;16.7;78947 +2001-07-22_11:00:00;17;78947 +2001-07-22_12:00:00;16.5;78947 +2001-07-22_13:00:00;17.4;78947 +2001-07-22_14:00:00;17.3;78947 +2001-07-22_15:00:00;17.5;78947 +2001-07-22_16:00:00;17;78947 +2001-07-22_17:00:00;16.7;78947 +2001-07-22_18:00:00;15.8;78947 +2001-07-22_19:00:00;15.4;78947 +2001-07-22_20:00:00;15.5;78947 +2001-07-22_21:00:00;15.2;78947 +2001-07-22_22:00:00;15.2;78947 +2001-07-22_23:00:00;15.1;78947 +2001-07-23_00:00:00;14.9;78947 +2001-07-23_01:00:00;14.8;78947 +2001-07-23_02:00:00;14.8;78947 +2001-07-23_03:00:00;14.6;78947 +2001-07-23_04:00:00;14.5;78947 +2001-07-23_05:00:00;14.7;78947 +2001-07-23_06:00:00;15.1;78947 +2001-07-23_07:00:00;15.5;78947 +2001-07-23_08:00:00;15.7;78947 +2001-07-23_09:00:00;16.6;78947 +2001-07-23_10:00:00;18.7;78947 +2001-07-23_11:00:00;19.6;78947 +2001-07-23_12:00:00;16.8;78947 +2001-07-23_13:00:00;17.3;78947 +2001-07-23_14:00:00;17.6;78947 +2001-07-23_15:00:00;19.6;78947 +2001-07-23_16:00:00;17.4;78947 +2001-07-23_17:00:00;17.5;78947 +2001-07-23_18:00:00;16.7;78947 +2001-07-23_19:00:00;16.1;78947 +2001-07-23_20:00:00;15.4;78947 +2001-07-23_21:00:00;15.2;78947 +2001-07-23_22:00:00;14.9;78947 +2001-07-23_23:00:00;15.2;78947 +2001-07-24_00:00:00;15.4;78947 +2001-07-24_01:00:00;15;78947 +2001-07-24_02:00:00;14.1;78947 +2001-07-24_03:00:00;14.1;78947 +2001-07-24_04:00:00;14.5;78947 +2001-07-24_05:00:00;15;78947 +2001-07-24_06:00:00;15.6;78947 +2001-07-24_07:00:00;16.1;78947 +2001-07-24_08:00:00;17.4;78947 +2001-07-24_09:00:00;18.6;78947 +2001-07-24_10:00:00;19.9;78947 +2001-07-24_11:00:00;19.9;78947 +2001-07-24_12:00:00;18.2;78947 +2001-07-24_13:00:00;17.6;78947 +2001-07-24_14:00:00;17.9;78947 +2001-07-24_15:00:00;18.4;78947 +2001-07-24_16:00:00;17.9;78947 +2001-07-24_17:00:00;17.6;78947 +2001-07-24_18:00:00;17.3;78947 +2001-07-24_19:00:00;16.4;78947 +2001-07-24_20:00:00;15.4;78947 +2001-07-24_21:00:00;15.1;78947 +2001-07-24_22:00:00;15.2;78947 +2001-07-24_23:00:00;15.1;78947 +2001-07-25_00:00:00;15.1;78947 +2001-07-25_01:00:00;15;78947 +2001-07-25_02:00:00;14.9;78947 +2001-07-25_03:00:00;15;78947 +2001-07-25_04:00:00;14.9;78947 +2001-07-25_05:00:00;15.2;78947 +2001-07-25_06:00:00;15.7;78947 +2001-07-25_07:00:00;16.3;78947 +2001-07-25_09:00:00;17.7;78947 +2001-07-25_10:00:00;18.6;78947 +2001-07-25_11:00:00;19.1;78947 +2001-07-25_12:00:00;18.1;78947 +2001-07-25_13:00:00;18.7;78947 +2001-07-25_14:00:00;18.8;78947 +2001-07-25_15:00:00;18.9;78947 +2001-07-25_16:00:00;18.7;78947 +2001-07-25_17:00:00;17.8;78947 +2001-07-25_18:00:00;16.9;78947 +2001-07-25_19:00:00;16.4;78947 +2001-07-25_20:00:00;16;78947 +2001-07-25_21:00:00;15.7;78947 +2001-07-25_22:00:00;15.4;78947 +2001-07-25_23:00:00;15.1;78947 +2001-07-26_00:00:00;14.7;78947 +2001-07-26_01:00:00;14.7;78947 +2001-07-26_02:00:00;14.6;78947 +2001-07-26_03:00:00;14.6;78947 +2001-07-26_04:00:00;14.7;78947 +2001-07-26_05:00:00;14.7;78947 +2001-07-26_06:00:00;14.6;78947 +2001-07-26_07:00:00;14.7;78947 +2001-07-26_09:00:00;15.2;78947 +2001-07-26_10:00:00;15.7;78947 +2001-07-26_11:00:00;15.8;78947 +2001-07-26_12:00:00;14.8;78947 +2001-07-26_13:00:00;14.9;78947 +2001-07-26_14:00:00;15.4;78947 +2001-07-26_15:00:00;15.7;78947 +2001-07-26_16:00:00;15.5;78947 +2001-07-26_17:00:00;15.3;78947 +2001-07-26_18:00:00;15.2;78947 +2001-07-26_19:00:00;14.6;78947 +2001-07-26_20:00:00;13.9;78947 +2001-07-26_21:00:00;13.3;78947 +2001-07-26_22:00:00;13.2;78947 +2001-07-26_23:00:00;13.2;78947 +2001-07-27_01:00:00;13.4;78947 +2001-07-27_02:00:00;13.4;78947 +2001-07-27_03:00:00;13.2;78947 +2001-07-27_04:00:00;13;78947 +2001-07-27_05:00:00;13.1;78947 +2001-07-27_06:00:00;13.8;78947 +2001-07-27_07:00:00;14.4;78947 +2001-07-27_08:00:00;15.1;78947 +2001-07-27_09:00:00;16;78947 +2001-07-27_10:00:00;16.7;78947 +2001-07-27_11:00:00;16.7;78947 +2001-07-27_12:00:00;16.8;78947 +2001-07-27_13:00:00;16.5;78947 +2001-07-27_14:00:00;16.1;78947 +2001-07-27_15:00:00;15.6;78947 +2001-07-27_16:00:00;15;78947 +2001-07-27_17:00:00;14.5;78947 +2001-07-27_18:00:00;14.1;78947 +2001-07-27_19:00:00;13.5;78947 +2001-07-27_20:00:00;12.9;78947 +2001-07-27_21:00:00;12.6;78947 +2001-07-27_22:00:00;12.5;78947 +2001-07-27_23:00:00;12.4;78947 +2001-07-28_00:00:00;12.6;78947 +2001-07-28_01:00:00;12.8;78947 +2001-07-28_02:00:00;12.9;78947 +2001-07-28_03:00:00;12.9;78947 +2001-07-28_04:00:00;13;78947 +2001-07-28_05:00:00;13.2;78947 +2001-07-28_06:00:00;13.4;78947 +2001-07-28_07:00:00;13.7;78947 +2001-07-28_09:00:00;14.1;78947 +2001-07-28_10:00:00;14.4;78947 +2001-07-28_11:00:00;14.7;78947 +2001-07-28_12:00:00;15.4;78947 +2001-07-28_13:00:00;15.3;78947 +2001-07-28_14:00:00;14.8;78947 +2001-07-28_15:00:00;14.5;78947 +2001-07-28_16:00:00;14.3;78947 +2001-07-28_17:00:00;14.1;78947 +2001-07-28_18:00:00;13.4;78947 +2001-07-28_19:00:00;12.8;78947 +2001-07-28_20:00:00;12.4;78947 +2001-07-28_21:00:00;12.3;78947 +2001-07-28_22:00:00;12.5;78947 +2001-07-28_23:00:00;12.7;78947 +2001-07-29_00:00:00;12.4;78947 +2001-07-29_01:00:00;12.3;78947 +2001-07-29_02:00:00;12.1;78947 +2001-07-29_03:00:00;12;78947 +2001-07-29_04:00:00;12.1;78947 +2001-07-29_05:00:00;12.3;78947 +2001-07-29_06:00:00;12.9;78947 +2001-07-29_07:00:00;13.5;78947 +2001-07-29_09:00:00;14.6;78947 +2001-07-29_10:00:00;15;78947 +2001-07-29_11:00:00;15.2;78947 +2001-07-29_12:00:00;15.5;78947 +2001-07-29_13:00:00;15.4;78947 +2001-07-29_14:00:00;15.2;78947 +2001-07-29_15:00:00;14.8;78947 +2001-07-29_16:00:00;14.4;78947 +2001-07-29_17:00:00;14;78947 +2001-07-29_18:00:00;13.6;78947 +2001-07-29_19:00:00;13.1;78947 +2001-07-29_20:00:00;12.7;78947 +2001-07-29_21:00:00;12.5;78947 +2001-07-29_22:00:00;12.4;78947 +2001-07-29_23:00:00;12.3;78947 +2001-07-30_00:00:00;12.3;78947 +2001-07-30_01:00:00;12.2;78947 +2001-07-30_02:00:00;12.3;78947 +2001-07-30_03:00:00;12.3;78947 +2001-07-30_05:00:00;12.6;78947 +2001-07-30_06:00:00;13.4;78947 +2001-07-30_07:00:00;14;78947 +2001-07-30_08:00:00;14.8;78947 +2001-07-30_09:00:00;15.4;78947 +2001-07-30_10:00:00;15.8;78947 +2001-07-30_11:00:00;16.1;78947 +2001-07-30_12:00:00;16.6;78947 +2001-07-30_13:00:00;16.5;78947 +2001-07-30_14:00:00;16.3;78947 +2001-07-30_15:00:00;16;78947 +2001-07-30_16:00:00;15.6;78947 +2001-07-30_17:00:00;15;78947 +2001-07-30_18:00:00;14.4;78947 +2001-07-30_19:00:00;13.9;78947 +2001-07-30_20:00:00;13.3;78947 +2001-07-30_21:00:00;12.8;78947 +2001-07-30_22:00:00;12.5;78947 +2001-07-30_23:00:00;12.6;78947 +2001-07-31_00:00:00;13;78947 +2001-07-31_01:00:00;13.1;78947 +2001-07-31_02:00:00;13.3;78947 +2001-07-31_03:00:00;13.3;78947 +2001-07-31_04:00:00;13.2;78947 +2001-07-31_05:00:00;13.5;78947 +2001-07-31_06:00:00;14.2;78947 +2001-07-31_07:00:00;14.6;78947 +2001-07-31_08:00:00;15;78947 +2001-07-31_09:00:00;15.8;78947 +2001-07-31_10:00:00;16.4;78947 +2001-07-31_11:00:00;16.8;78947 +2001-07-31_12:00:00;17.1;78947 +2001-07-31_13:00:00;17.1;78947 +2001-07-31_14:00:00;16.6;78947 +2001-07-31_15:00:00;16.2;78947 +2001-07-31_16:00:00;15.8;78947 +2001-07-31_17:00:00;15.4;78947 +2001-07-31_18:00:00;14.8;78947 +2001-07-31_19:00:00;14;78947 +2001-07-31_20:00:00;13.2;78947 +2001-07-31_21:00:00;12.6;78947 +2001-07-31_22:00:00;12;78947 +2001-07-31_23:00:00;12.3;78947 +2001-08-01_00:00:00;13.1;78947 +2001-08-01_01:00:00;13.2;78947 +2001-08-01_02:00:00;13.3;78947 +2001-08-01_03:00:00;13.3;78947 +2001-08-01_04:00:00;13.3;78947 +2001-08-01_05:00:00;13.6;78947 +2001-08-01_06:00:00;14.4;78947 +2001-08-01_07:00:00;14.9;78947 +2001-08-01_09:00:00;16.2;78947 +2001-08-01_10:00:00;16.6;78947 +2001-08-01_11:00:00;17;78947 +2001-08-01_12:00:00;17.2;78947 +2001-08-01_13:00:00;17.4;78947 +2001-08-01_14:00:00;17.4;78947 +2001-08-01_15:00:00;17.1;78947 +2001-08-01_16:00:00;16.7;78947 +2001-08-01_17:00:00;16.2;78947 +2001-08-01_18:00:00;15.4;78947 +2001-08-01_19:00:00;14.6;78947 +2001-08-01_20:00:00;13.7;78947 +2001-08-01_21:00:00;13;78947 +2001-08-01_22:00:00;12.5;78947 +2001-08-01_23:00:00;12.3;78947 +2001-08-02_00:00:00;12.6;78947 +2001-08-02_01:00:00;12.6;78947 +2001-08-02_02:00:00;12.6;78947 +2001-08-02_03:00:00;12.8;78947 +2001-08-02_04:00:00;13.2;78947 +2001-08-02_05:00:00;14;78947 +2001-08-02_06:00:00;15.6;78947 +2001-08-02_07:00:00;16.1;78947 +2001-08-02_09:00:00;17;78947 +2001-08-02_10:00:00;17;78947 +2001-08-02_11:00:00;17.1;78947 +2001-08-02_12:00:00;17.2;78947 +2001-08-02_13:00:00;18.2;78947 +2001-08-02_14:00:00;19.2;78947 +2001-08-02_15:00:00;19.4;78947 +2001-08-02_16:00:00;19.8;78947 +2001-08-02_17:00:00;20;78947 +2001-08-02_18:00:00;18.4;78947 +2001-08-02_19:00:00;17.8;78947 +2001-08-02_20:00:00;16.6;78947 +2001-08-02_21:00:00;15.6;78947 +2001-08-02_22:00:00;15.6;78947 +2001-08-02_23:00:00;15.7;78947 +2001-08-03_00:00:00;15.6;78947 +2001-08-03_01:00:00;16;78947 +2001-08-03_02:00:00;16.5;78947 +2001-08-03_03:00:00;16.7;78947 +2001-08-03_04:00:00;16.3;78947 +2001-08-03_05:00:00;16.5;78947 +2001-08-03_06:00:00;17.5;78947 +2001-08-03_07:00:00;17;78947 +2001-08-03_09:00:00;17.4;78947 +2001-08-03_10:00:00;18.7;78947 +2001-08-03_11:00:00;20.4;78947 +2001-08-03_12:00:00;21.9;78947 +2001-08-03_13:00:00;21.3;78947 +2001-08-03_14:00:00;21.9;78947 +2001-08-03_15:00:00;21.7;78947 +2001-08-03_16:00:00;19.8;78947 +2001-08-03_17:00:00;17.8;78947 +2001-08-03_18:00:00;19.1;78947 +2001-08-03_19:00:00;18.6;78947 +2001-08-03_20:00:00;18;78947 +2001-08-03_21:00:00;17.7;78947 +2001-08-03_22:00:00;17.7;78947 +2001-08-03_23:00:00;17.8;78947 +2001-08-04_00:00:00;17;78947 +2001-08-04_01:00:00;16.8;78947 +2001-08-04_02:00:00;16.9;78947 +2001-08-04_03:00:00;16.9;78947 +2001-08-04_04:00:00;16.9;78947 +2001-08-04_05:00:00;16.6;78947 +2001-08-04_06:00:00;17.5;78947 +2001-08-04_07:00:00;19;78947 +2001-08-04_09:00:00;20.2;78947 +2001-08-04_10:00:00;19.4;78947 +2001-08-04_11:00:00;20;78947 +2001-08-04_12:00:00;20.3;78947 +2001-08-04_13:00:00;21.3;78947 +2001-08-04_14:00:00;22.2;78947 +2001-08-04_15:00:00;22.1;78947 +2001-08-04_16:00:00;22.5;78947 +2001-08-04_17:00:00;22;78947 +2001-08-04_18:00:00;18.6;78947 +2001-08-04_19:00:00;17.1;78947 +2001-08-04_20:00:00;16.5;78947 +2001-08-04_21:00:00;16.2;78947 +2001-08-04_22:00:00;16.2;78947 +2001-08-04_23:00:00;16.7;78947 +2001-08-05_00:00:00;16.4;78947 +2001-08-05_01:00:00;16.7;78947 +2001-08-05_02:00:00;16.5;78947 +2001-08-05_03:00:00;16.4;78947 +2001-08-05_04:00:00;16.6;78947 +2001-08-05_05:00:00;16.5;78947 +2001-08-05_06:00:00;16.1;78947 +2001-08-05_07:00:00;16.3;78947 +2001-08-05_09:00:00;16.5;78947 +2001-08-05_10:00:00;16.7;78947 +2001-08-05_11:00:00;16.8;78947 +2001-08-05_12:00:00;17;78947 +2001-08-05_13:00:00;16.9;78947 +2001-08-05_14:00:00;16.6;78947 +2001-08-05_15:00:00;16.3;78947 +2001-08-05_16:00:00;16.1;78947 +2001-08-05_17:00:00;15.9;78947 +2001-08-05_18:00:00;16;78947 +2001-08-05_19:00:00;15.8;78947 +2001-08-05_20:00:00;15.6;78947 +2001-08-05_21:00:00;15.6;78947 +2001-08-05_22:00:00;15.5;78947 +2001-08-05_23:00:00;15.4;78947 +2001-08-06_00:00:00;15;78947 +2001-08-06_01:00:00;14.8;78947 +2001-08-06_02:00:00;14.6;78947 +2001-08-06_03:00:00;14.5;78947 +2001-08-06_04:00:00;14.5;78947 +2001-08-06_05:00:00;14.5;78947 +2001-08-06_06:00:00;14.7;78947 +2001-08-06_08:00:00;14.8;78947 +2001-08-06_09:00:00;15.7;78947 +2001-08-06_10:00:00;16.1;78947 +2001-08-06_11:00:00;15.8;78947 +2001-08-06_12:00:00;16.5;78947 +2001-08-06_13:00:00;15.8;78947 +2001-08-06_14:00:00;16.5;78947 +2001-08-06_15:00:00;16.2;78947 +2001-08-06_16:00:00;16.1;78947 +2001-08-06_17:00:00;15.7;78947 +2001-08-06_18:00:00;14.8;78947 +2001-08-06_19:00:00;14.2;78947 +2001-08-06_20:00:00;13.1;78947 +2001-08-06_21:00:00;12.3;78947 +2001-08-06_22:00:00;11.7;78947 +2001-08-06_23:00:00;11.3;78947 +2001-08-07_00:00:00;11.2;78947 +2001-08-07_01:00:00;11.5;78947 +2001-08-07_02:00:00;11.6;78947 +2001-08-07_03:00:00;11.8;78947 +2001-08-07_04:00:00;11.7;78947 +2001-08-07_05:00:00;12.3;78947 +2001-08-07_06:00:00;13.1;78947 +2001-08-07_07:00:00;15;78947 +2001-08-07_08:00:00;14.6;78947 +2001-08-07_09:00:00;13.8;78947 +2001-08-07_10:00:00;14.3;78947 +2001-08-07_11:00:00;15.5;78947 +2001-08-07_12:00:00;15.6;78947 +2001-08-07_13:00:00;15.5;78947 +2001-08-07_14:00:00;16.2;78947 +2001-08-07_15:00:00;16.5;78947 +2001-08-07_16:00:00;16;78947 +2001-08-07_17:00:00;15.9;78947 +2001-08-07_18:00:00;14.8;78947 +2001-08-07_19:00:00;14.5;78947 +2001-08-07_20:00:00;14.2;78947 +2001-08-07_21:00:00;14.5;78947 +2001-08-07_22:00:00;14.6;78947 +2001-08-07_23:00:00;13.5;78947 +2001-08-08_00:00:00;13.2;78947 +2001-08-08_01:00:00;13.1;78947 +2001-08-08_02:00:00;13.4;78947 +2001-08-08_03:00:00;13.2;78947 +2001-08-08_04:00:00;12.9;78947 +2001-08-08_05:00:00;12.7;78947 +2001-08-08_06:00:00;14.1;78947 +2001-08-08_07:00:00;14.9;78947 +2001-08-08_08:00:00;14.7;78947 +2001-08-08_09:00:00;15.5;78947 +2001-08-08_10:00:00;15;78947 +2001-08-08_11:00:00;16.7;78947 +2001-08-08_12:00:00;14.6;70000 +2001-08-08_13:00:00;14;70000 +2001-08-08_14:00:00;13.8;58927 +2001-08-08_15:00:00;13.6;70000 +2001-08-08_16:00:00;14.1;58927 +2001-08-08_17:00:00;14.5;70000 +2001-08-08_18:00:00;14.7;58927 +2001-08-08_19:00:00;14.9;70000 +2001-08-08_20:00:00;13.7;78947 +2001-08-08_21:00:00;13.8;78947 +2001-08-08_22:00:00;15.8;70000 +2001-08-08_23:00:00;15.9;70000 +2001-08-09_00:00:00;12.5;78947 +2001-08-09_01:00:00;13.1;78947 +2001-08-09_02:00:00;13.1;78947 +2001-08-09_03:00:00;13.2;78947 +2001-08-09_04:00:00;13.5;78947 +2001-08-09_05:00:00;13.7;78947 +2001-08-09_06:00:00;13.8;78947 +2001-08-09_07:00:00;14.7;70000 +2001-08-09_08:00:00;14.7;58927 +2001-08-09_09:00:00;14.7;70000 +2001-08-09_10:00:00;14.6;70000 +2001-08-09_11:00:00;14.6;70000 +2001-08-09_12:00:00;14.4;70000 +2001-08-09_13:00:00;14.2;70000 +2001-08-09_14:00:00;14.4;70000 +2001-08-09_15:00:00;14.3;70000 +2001-08-09_16:00:00;14.2;70000 +2001-08-09_17:00:00;14;70000 +2001-08-09_18:00:00;13.9;70000 +2001-08-09_19:00:00;12.9;70000 +2001-08-09_20:00:00;13.2;70000 +2001-08-09_21:00:00;13.1;70000 +2001-08-09_22:00:00;13.1;70000 +2001-08-09_23:00:00;13.1;70000 +2001-08-10_00:00:00;12.8;70000 +2001-08-10_01:00:00;11.6;70000 +2001-08-10_02:00:00;12.2;70000 +2001-08-10_03:00:00;12.7;70000 +2001-08-10_04:00:00;12.5;70000 +2001-08-10_05:00:00;12.4;70000 +2001-08-10_06:00:00;12.5;70000 +2001-08-10_07:00:00;12.6;70000 +2001-08-10_08:00:00;12.6;70000 +2001-08-10_09:00:00;11.8;70000 +2001-08-10_10:00:00;12.1;70000 +2001-08-10_11:00:00;12.1;70000 +2001-08-10_12:00:00;12.7;70000 +2001-08-10_13:00:00;13;70000 +2001-08-10_14:00:00;13.1;70000 +2001-08-10_15:00:00;13.3;70000 +2001-08-10_16:00:00;13.4;70000 +2001-08-10_17:00:00;13.5;70000 +2001-08-10_18:00:00;13.5;70000 +2001-08-10_19:00:00;13.6;70000 +2001-08-10_20:00:00;13.8;70000 +2001-08-10_21:00:00;13.9;70000 +2001-08-10_22:00:00;13.9;70000 +2001-08-10_23:00:00;13.7;58927 +2001-08-11_00:00:00;13.5;70000 +2001-08-11_01:00:00;13.4;70000 +2001-08-11_02:00:00;13.1;70000 +2001-08-11_03:00:00;12.5;70000 +2001-08-11_04:00:00;12.7;70000 +2001-08-11_05:00:00;13.2;70000 +2001-08-11_06:00:00;12.9;70000 +2001-08-11_07:00:00;14.3;70000 +2001-08-11_08:00:00;15;70000 +2001-08-11_09:00:00;15.8;70000 +2001-08-11_10:00:00;15.8;58927 +2001-08-11_11:00:00;15.7;70000 +2001-08-11_12:00:00;15.9;70000 +2001-08-11_13:00:00;16;70000 +2001-08-11_14:00:00;16.1;70000 +2001-08-11_15:00:00;16.2;70000 +2001-08-11_16:00:00;16;70000 +2001-08-11_17:00:00;16.1;70000 +2001-08-11_18:00:00;16.1;70000 +2001-08-11_19:00:00;16;70000 +2001-08-11_20:00:00;15.9;70000 +2001-08-11_21:00:00;15.7;70000 +2001-08-11_22:00:00;15.6;70000 +2001-08-11_23:00:00;15.6;70000 +2001-08-12_00:00:00;15.6;70000 +2001-08-12_01:00:00;15.3;70000 +2001-08-12_02:00:00;15.2;70000 +2001-08-12_03:00:00;15.1;70000 +2001-08-12_04:00:00;15;70000 +2001-08-12_05:00:00;14.7;70000 +2001-08-12_06:00:00;14.7;70000 +2001-08-12_07:00:00;14.9;70000 +2001-08-12_08:00:00;14.9;70000 +2001-08-12_09:00:00;14.9;70000 +2001-08-12_10:00:00;15.3;70000 +2001-08-12_11:00:00;15.6;70000 +2001-08-12_12:00:00;15.8;70000 +2001-08-12_13:00:00;15.6;70000 +2001-08-12_14:00:00;15.9;70000 +2001-08-12_15:00:00;16.2;70000 +2001-08-12_16:00:00;16.2;70000 +2001-08-12_17:00:00;16.3;70000 +2001-08-12_18:00:00;16.9;70000 +2001-08-12_19:00:00;16.6;70000 +2001-08-12_20:00:00;15.7;70000 +2001-08-12_21:00:00;16.2;70000 +2001-08-12_22:00:00;16.5;70000 +2001-08-12_23:00:00;16.3;70000 +2001-08-13_00:00:00;16.1;70000 +2001-08-13_01:00:00;16;70000 +2001-08-13_02:00:00;15.9;70000 +2001-08-13_03:00:00;15.7;70000 +2001-08-13_04:00:00;15.8;70000 +2001-08-13_05:00:00;15.5;70000 +2001-08-13_06:00:00;15.3;70000 +2001-08-13_07:00:00;15.6;70000 +2001-08-13_08:00:00;16.2;70000 +2001-08-13_09:00:00;16.6;70000 +2001-08-13_10:00:00;16.9;70000 +2001-08-13_11:00:00;17.2;70000 +2001-08-13_12:00:00;17.5;70000 +2001-08-13_13:00:00;17.4;70000 +2001-08-13_14:00:00;17.6;70000 +2001-08-13_15:00:00;17.2;70000 +2001-08-13_16:00:00;17.1;70000 +2001-08-13_17:00:00;17.2;70000 +2001-08-13_18:00:00;17.2;70000 +2001-08-13_19:00:00;17.9;70000 +2001-08-13_20:00:00;17.5;70000 +2001-08-13_21:00:00;17.3;70000 +2001-08-13_22:00:00;17.2;70000 +2001-08-13_23:00:00;17.1;70000 +2001-08-14_00:00:00;17.1;70000 +2001-08-14_01:00:00;17.2;70000 +2001-08-14_02:00:00;16.9;70000 +2001-08-14_03:00:00;17.2;70000 +2001-08-14_04:00:00;17;70000 +2001-08-14_05:00:00;16.9;70000 +2001-08-14_06:00:00;16.8;70000 +2001-08-14_07:00:00;17;70000 +2001-08-14_08:00:00;17.3;70000 +2001-08-14_09:00:00;17;70000 +2001-08-14_10:00:00;17.1;70000 +2001-08-14_11:00:00;17.3;70000 +2001-08-14_12:00:00;17.1;70000 +2001-08-14_13:00:00;17;70000 +2001-08-14_14:00:00;17.1;70000 +2001-08-14_15:00:00;16.9;70000 +2001-08-14_16:00:00;16.7;70000 +2001-08-14_17:00:00;16.5;70000 +2001-08-14_18:00:00;16.3;70000 +2001-08-14_19:00:00;16.2;70000 +2001-08-14_20:00:00;16.2;70000 +2001-08-14_21:00:00;16.2;70000 +2001-08-14_22:00:00;15.9;70000 +2001-08-14_23:00:00;16;70000 +2001-08-15_00:00:00;15.4;70000 +2001-08-15_01:00:00;15.3;70000 +2001-08-15_02:00:00;15.3;70000 +2001-08-15_03:00:00;15.3;70000 +2001-08-15_04:00:00;14.9;70000 +2001-08-15_05:00:00;15;70000 +2001-08-15_06:00:00;15;70000 +2001-08-15_07:00:00;15.2;70000 +2001-08-15_08:00:00;15.1;70000 +2001-08-15_09:00:00;15.4;70000 +2001-08-15_10:00:00;15.3;70000 +2001-08-15_11:00:00;15.3;70000 +2001-08-15_12:00:00;15.6;70000 +2001-08-15_13:00:00;15.6;70000 +2001-08-15_14:00:00;15.5;70000 +2001-08-15_15:00:00;15.5;70000 +2001-08-15_16:00:00;15.5;70000 +2001-08-15_17:00:00;15.5;70000 +2001-08-15_18:00:00;15.3;70000 +2001-08-15_19:00:00;15.3;70000 +2001-08-15_20:00:00;15.2;70000 +2001-08-15_21:00:00;15.2;70000 +2001-08-15_22:00:00;15.1;70000 +2001-08-15_23:00:00;15;70000 +2001-08-16_00:00:00;15;70000 +2001-08-16_01:00:00;14.8;70000 +2001-08-16_02:00:00;14.8;70000 +2001-08-16_03:00:00;14.8;70000 +2001-08-16_04:00:00;14.6;70000 +2001-08-16_05:00:00;14.6;70000 +2001-08-16_06:00:00;14.5;70000 +2001-08-16_07:00:00;14.6;70000 +2001-08-16_08:00:00;14.5;70000 +2001-08-16_09:00:00;14.5;70000 +2001-08-16_10:00:00;14.8;70000 +2001-08-16_11:00:00;14.9;70000 +2001-08-16_12:00:00;15;70000 +2001-08-16_13:00:00;15.1;70000 +2001-08-16_14:00:00;15.2;70000 +2001-08-16_15:00:00;15.3;70000 +2001-08-16_16:00:00;15.4;70000 +2001-08-16_17:00:00;15.3;70000 +2001-08-16_18:00:00;15.4;70000 +2001-08-16_19:00:00;15.4;70000 +2001-08-16_20:00:00;15.5;70000 +2001-08-16_21:00:00;15.6;70000 +2001-08-16_22:00:00;15.7;70000 +2001-08-16_23:00:00;15.3;70000 +2001-08-17_00:00:00;15.2;70000 +2001-08-17_01:00:00;14.9;70000 +2001-08-17_02:00:00;14.6;70000 +2001-08-17_03:00:00;14.6;70000 +2001-08-17_04:00:00;14.3;70000 +2001-08-17_05:00:00;14.1;70000 +2001-08-17_06:00:00;14.3;70000 +2001-08-17_07:00:00;14.5;70000 +2001-08-17_08:00:00;14.8;70000 +2001-08-17_09:00:00;15.1;70000 +2001-08-17_10:00:00;15.4;70000 +2001-08-17_11:00:00;15.8;70000 +2001-08-17_12:00:00;16;70000 +2001-08-17_13:00:00;16.1;70000 +2001-08-17_14:00:00;16.3;70000 +2001-08-17_15:00:00;16.5;70000 +2001-08-17_16:00:00;16.7;70000 +2001-08-17_17:00:00;16.5;70000 +2001-08-17_18:00:00;16.5;70000 +2001-08-17_19:00:00;16.3;70000 +2001-08-17_20:00:00;15.9;70000 +2001-08-17_21:00:00;15.9;70000 +2001-08-17_22:00:00;15.8;70000 +2001-08-17_23:00:00;15.6;70000 +2001-08-18_00:00:00;15.5;70000 +2001-08-18_01:00:00;15.4;70000 +2001-08-18_02:00:00;15.3;70000 +2001-08-18_03:00:00;15.2;70000 +2001-08-18_04:00:00;15.2;70000 +2001-08-18_05:00:00;15;70000 +2001-08-18_06:00:00;15;70000 +2001-08-18_07:00:00;15;70000 +2001-08-18_08:00:00;15.2;70000 +2001-08-18_09:00:00;15.2;70000 +2001-08-18_10:00:00;15.2;70000 +2001-08-18_11:00:00;15.2;70000 +2001-08-18_12:00:00;15.2;70000 +2001-08-18_13:00:00;15.2;70203 +2001-08-18_14:00:00;15.3;70000 +2001-08-18_15:00:00;15.3;70000 +2001-08-18_16:00:00;15.3;70000 +2001-08-18_17:00:00;15.3;70000 +2001-08-18_18:00:00;15.2;70000 +2001-08-18_19:00:00;15.2;70000 +2001-08-18_20:00:00;15.2;70000 +2001-08-18_21:00:00;15.2;70000 +2001-08-18_22:00:00;15.2;70000 +2001-08-18_23:00:00;15.2;70203 +2001-08-19_00:00:00;15.2;70203 +2001-08-19_01:00:00;15.2;70203 +2001-08-19_02:00:00;15.2;70203 +2001-08-19_03:00:00;15;70000 +2001-08-19_04:00:00;15;70000 +2001-08-19_05:00:00;15;70000 +2001-08-19_06:00:00;14.9;70000 +2001-08-19_07:00:00;15.1;70000 +2001-08-19_08:00:00;15;70000 +2001-08-19_09:00:00;15;70000 +2001-08-19_10:00:00;15.1;70000 +2001-08-19_11:00:00;15;70000 +2001-08-19_12:00:00;15.1;70000 +2001-08-19_13:00:00;15.1;70000 +2001-08-19_14:00:00;15.1;70000 +2001-08-19_15:00:00;15;70000 +2001-08-19_16:00:00;15;70000 +2001-08-19_17:00:00;15;70000 +2001-08-19_18:00:00;14.9;70000 +2001-08-19_19:00:00;14.8;70000 +2001-08-19_20:00:00;14.8;70000 +2001-08-19_21:00:00;14.6;70000 +2001-08-19_22:00:00;14.6;70000 +2001-08-19_23:00:00;14.5;70000 +2001-08-20_00:00:00;14.8;70000 +2001-08-20_01:00:00;14.6;70000 +2001-08-20_02:00:00;14.8;70000 +2001-08-20_03:00:00;14.1;70000 +2001-08-20_04:00:00;14.9;70000 +2001-08-20_05:00:00;15.2;70000 +2001-08-20_06:00:00;15.2;70000 +2001-08-20_07:00:00;13.9;70000 +2001-08-20_08:00:00;13.5;70000 +2001-08-20_09:00:00;14.6;70000 +2001-08-20_10:00:00;15.8;70000 +2001-08-20_11:00:00;15.8;70000 +2001-08-20_12:00:00;15.9;70000 +2001-08-20_13:00:00;15.8;70000 +2001-08-20_14:00:00;15.8;70000 +2001-08-20_15:00:00;15.9;70000 +2001-08-20_16:00:00;16;70000 +2001-08-20_17:00:00;16;70000 +2001-08-20_18:00:00;15.8;70000 +2001-08-20_19:00:00;15.6;70000 +2001-08-20_20:00:00;15.5;70000 +2001-08-20_21:00:00;15.5;70000 +2001-08-20_22:00:00;15.4;70000 +2001-08-20_23:00:00;15.3;70000 +2001-08-21_00:00:00;15.4;70000 +2001-08-21_01:00:00;15.4;70000 +2001-08-21_02:00:00;15.5;70000 +2001-08-21_03:00:00;15.6;70000 +2001-08-21_04:00:00;15.8;70000 +2001-08-21_05:00:00;15.4;70000 +2001-08-21_06:00:00;14.7;70000 +2001-08-21_07:00:00;14.9;70000 +2001-08-21_08:00:00;14.9;70000 +2001-08-21_09:00:00;14.4;70000 +2001-08-21_10:00:00;14.9;70000 +2001-08-21_11:00:00;15.9;70000 +2001-08-21_12:00:00;15.8;70000 +2001-08-21_13:00:00;16.1;70000 +2001-08-21_14:00:00;16.1;70000 +2001-08-21_15:00:00;15.9;70000 +2001-08-21_16:00:00;15.8;70000 +2001-08-21_17:00:00;15.8;70000 +2001-08-21_18:00:00;15.7;70000 +2001-08-21_19:00:00;15.7;70000 +2001-08-21_20:00:00;15.6;70000 +2001-08-21_21:00:00;15.5;70000 +2001-08-21_22:00:00;15.3;70000 +2001-08-21_23:00:00;15.2;70000 +2001-08-22_00:00:00;15.2;70000 +2001-08-22_01:00:00;15.3;70000 +2001-08-22_02:00:00;15.4;70000 +2001-08-22_03:00:00;15.4;70000 +2001-08-22_04:00:00;15.2;70000 +2001-08-22_05:00:00;15.2;70000 +2001-08-22_06:00:00;15.5;70000 +2001-08-22_07:00:00;15.5;70000 +2001-08-22_08:00:00;15.5;70000 +2001-08-22_09:00:00;15.5;70000 +2001-08-22_10:00:00;15.4;70000 +2001-08-22_11:00:00;15.3;70000 +2001-08-22_12:00:00;15.2;70000 +2001-08-22_13:00:00;15.2;70000 +2001-08-22_14:00:00;15.2;70000 +2001-08-22_15:00:00;15.2;70000 +2001-08-22_16:00:00;15;70000 +2001-08-22_17:00:00;15.3;70000 +2001-08-22_18:00:00;15.2;70000 +2001-08-22_19:00:00;15.2;70000 +2001-08-22_20:00:00;15.3;70000 +2001-08-22_21:00:00;15.3;70000 +2001-08-22_22:00:00;15;70000 +2001-08-22_23:00:00;15.2;70000 +2001-08-23_00:00:00;15.2;70000 +2001-08-23_01:00:00;15;70000 +2001-08-23_02:00:00;14.9;70000 +2001-08-23_03:00:00;14.9;70000 +2001-08-23_04:00:00;14.8;70000 +2001-08-23_05:00:00;14.9;70000 +2001-08-23_06:00:00;15;70000 +2001-08-23_07:00:00;15.1;70000 +2001-08-23_08:00:00;14.8;70000 +2001-08-23_09:00:00;15.1;70000 +2001-08-23_10:00:00;15.2;70000 +2001-08-23_11:00:00;15.2;70000 +2001-08-23_12:00:00;15.5;70000 +2001-08-23_13:00:00;15.6;70000 +2001-08-23_14:00:00;15.7;70000 +2001-08-23_15:00:00;15.9;70000 +2001-08-23_16:00:00;16;70000 +2001-08-23_17:00:00;16.2;70000 +2001-08-23_18:00:00;16.3;70000 +2001-08-23_19:00:00;16.3;70000 +2001-08-23_20:00:00;16.5;70000 +2001-08-23_21:00:00;16.5;70000 +2001-08-23_22:00:00;16.9;70000 +2001-08-23_23:00:00;16.6;70000 +2001-08-24_00:00:00;16.5;70000 +2001-08-24_01:00:00;16.2;70000 +2001-08-24_02:00:00;16.6;70000 +2001-08-24_03:00:00;16;70000 +2001-08-24_04:00:00;16.6;70000 +2001-08-24_05:00:00;16.7;70000 +2001-08-24_06:00:00;15.8;70000 +2001-08-24_07:00:00;14.9;70000 +2001-08-24_08:00:00;14.6;70000 +2001-08-24_09:00:00;15.2;70000 +2001-08-24_10:00:00;15.7;70000 +2001-08-24_11:00:00;15.7;70000 +2001-08-24_12:00:00;15.8;70000 +2001-08-24_13:00:00;16.3;70000 +2001-08-24_14:00:00;17.2;70000 +2001-08-24_15:00:00;17.3;70000 +2001-08-24_16:00:00;17.4;70000 +2001-08-24_17:00:00;17.4;70000 +2001-08-24_18:00:00;15.8;70000 +2001-08-24_19:00:00;15.4;70000 +2001-08-24_20:00:00;15.3;70000 +2001-08-24_21:00:00;15;70000 +2001-08-24_22:00:00;15.5;70000 +2001-08-24_23:00:00;15.5;70000 +2001-08-25_00:00:00;15.5;70000 +2001-08-25_01:00:00;15.3;70000 +2001-08-25_02:00:00;15.3;70000 +2001-08-25_03:00:00;15.3;70000 +2001-08-25_04:00:00;15.3;70000 +2001-08-25_05:00:00;15.3;70000 +2001-08-25_06:00:00;15.3;70203 +2001-08-25_07:00:00;15.3;70203 +2001-08-25_08:00:00;15.6;70000 +2001-08-25_09:00:00;15.7;70000 +2001-08-25_10:00:00;15.8;70000 +2001-08-25_11:00:00;15.8;70000 +2001-08-25_12:00:00;15.8;70000 +2001-08-25_13:00:00;15.9;70000 +2001-08-25_14:00:00;16.1;70000 +2001-08-25_15:00:00;16.2;70000 +2001-08-25_16:00:00;16.3;70000 +2001-08-25_17:00:00;16.2;70000 +2001-08-25_18:00:00;16.2;70000 +2001-08-25_19:00:00;16.3;70000 +2001-08-25_20:00:00;16.3;70000 +2001-08-25_21:00:00;16.3;70000 +2001-08-25_22:00:00;16.3;70000 +2001-08-25_23:00:00;16.1;70000 +2001-08-26_00:00:00;15.8;70000 +2001-08-26_01:00:00;16.4;70000 +2001-08-26_02:00:00;16.4;70000 +2001-08-26_03:00:00;16.5;70000 +2001-08-26_04:00:00;16.5;70000 +2001-08-26_05:00:00;16.2;70000 +2001-08-26_06:00:00;16.3;70000 +2001-08-26_07:00:00;16.5;70000 +2001-08-26_08:00:00;16.4;70000 +2001-08-26_09:00:00;16.6;70000 +2001-08-26_10:00:00;16.9;70000 +2001-08-26_11:00:00;17.1;70000 +2001-08-26_12:00:00;16.6;70000 +2001-08-26_13:00:00;16.2;70000 +2001-08-26_14:00:00;16.7;70000 +2001-08-26_15:00:00;17.3;70000 +2001-08-26_16:00:00;17.7;70000 +2001-08-26_17:00:00;16.4;70000 +2001-08-26_18:00:00;17.9;70000 +2001-08-26_19:00:00;18.1;70000 +2001-08-26_20:00:00;18.5;70000 +2001-08-26_21:00:00;19;70000 +2001-08-26_22:00:00;18.4;70000 +2001-08-26_23:00:00;18.4;70000 +2001-08-27_00:00:00;18.1;70000 +2001-08-27_01:00:00;17.8;70000 +2001-08-27_02:00:00;17.7;70000 +2001-08-27_03:00:00;17.6;70000 +2001-08-27_04:00:00;17.5;70000 +2001-08-27_05:00:00;17.5;70000 +2001-08-27_06:00:00;17.6;70000 +2001-08-27_07:00:00;17.1;70000 +2001-08-27_08:00:00;16.9;70000 +2001-08-27_09:00:00;16.7;70000 +2001-08-27_10:00:00;15;78947 +2001-08-27_11:00:00;15;78947 +2001-08-27_12:00:00;18.1;78947 +2001-08-27_13:00:00;18.1;78947 +2001-08-27_14:00:00;16.5;78947 +2001-08-27_15:00:00;16.9;78947 +2001-08-27_16:00:00;15.1;78947 +2001-08-27_17:00:00;15.2;78947 +2001-08-27_18:00:00;15.4;78947 +2001-08-27_19:00:00;14.9;78947 +2001-08-27_20:00:00;14.7;78947 +2001-08-27_21:00:00;14.3;78947 +2001-08-27_22:00:00;14.4;78947 +2001-08-27_23:00:00;14.1;78947 +2001-08-28_00:00:00;12.8;78947 +2001-08-28_01:00:00;12.7;78947 +2001-08-28_02:00:00;12.6;78947 +2001-08-28_03:00:00;12.4;78947 +2001-08-28_04:00:00;12.6;78947 +2001-08-28_05:00:00;11.9;78947 +2001-08-28_06:00:00;12.6;78947 +2001-08-28_07:00:00;12.7;78947 +2001-08-28_08:00:00;13.5;78947 +2001-08-28_09:00:00;12.8;78947 +2001-08-28_10:00:00;12.9;78947 +2001-08-28_11:00:00;13.2;78947 +2001-08-28_12:00:00;14.4;78947 +2001-08-28_13:00:00;13;78947 +2001-08-28_14:00:00;14.1;78947 +2001-08-28_15:00:00;13.2;78947 +2001-08-28_16:00:00;13;78947 +2001-08-28_17:00:00;12.7;78947 +2001-08-28_18:00:00;12.8;78947 +2001-08-28_19:00:00;13;78947 +2001-08-28_20:00:00;13.1;78947 +2001-08-28_21:00:00;12.9;78947 +2001-08-28_22:00:00;13;78947 +2001-08-28_23:00:00;13.4;78947 +2001-08-29_00:00:00;14.1;78947 +2001-08-29_01:00:00;13.3;78947 +2001-08-29_02:00:00;13.6;78947 +2001-08-29_03:00:00;13.8;78947 +2001-08-29_04:00:00;13.8;78947 +2001-08-29_05:00:00;13.9;78947 +2001-08-29_06:00:00;15.7;70000 +2001-08-29_07:00:00;15.9;70000 +2001-08-29_08:00:00;16.1;70000 +2001-08-29_09:00:00;16.1;70000 +2001-08-29_10:00:00;15.8;70000 +2001-08-29_11:00:00;15.4;70000 +2001-08-29_12:00:00;15.3;70000 +2001-08-29_13:00:00;15.7;70000 +2001-08-29_14:00:00;15.8;70000 +2001-08-29_15:00:00;15;70000 +2001-08-29_16:00:00;14.8;70000 +2001-08-29_17:00:00;14.5;70000 +2001-08-29_18:00:00;13.2;70000 +2001-08-29_19:00:00;13.6;70000 +2001-08-29_20:00:00;13.4;70000 +2001-08-29_21:00:00;13.3;70000 +2001-08-29_22:00:00;13.5;70000 +2001-08-29_23:00:00;13.2;70000 +2001-08-30_00:00:00;13.5;70000 +2001-08-30_01:00:00;13.3;70000 +2001-08-30_02:00:00;13.2;70000 +2001-08-30_03:00:00;13.6;70000 +2001-08-30_04:00:00;13.5;70000 +2001-08-30_05:00:00;13.5;70000 +2001-08-30_06:00:00;13.5;70000 +2001-08-30_07:00:00;13.6;70000 +2001-08-30_08:00:00;13.5;70000 +2001-08-30_09:00:00;13.5;70000 +2001-08-30_10:00:00;13.1;70000 +2001-08-30_11:00:00;13.2;70000 +2001-08-30_12:00:00;13.2;70000 +2001-08-30_13:00:00;13.4;70000 +2001-08-30_14:00:00;13.2;70000 +2001-08-30_15:00:00;13;70000 +2001-08-30_16:00:00;12.6;70000 +2001-08-30_17:00:00;12.8;70000 +2001-08-30_18:00:00;13.3;70000 +2001-08-30_19:00:00;14;70000 +2001-08-30_20:00:00;14.7;70000 +2001-08-30_21:00:00;14.8;70000 +2001-08-30_22:00:00;14.7;70000 +2001-08-30_23:00:00;14.6;70000 +2001-08-31_00:00:00;14.7;70000 +2001-08-31_01:00:00;14.6;70000 +2001-08-31_02:00:00;14.3;70000 +2001-08-31_03:00:00;14.4;70000 +2001-08-31_04:00:00;14;70000 +2001-08-31_05:00:00;14.2;70000 +2001-08-31_06:00:00;13.6;70000 +2001-08-31_07:00:00;14.5;70000 +2001-08-31_08:00:00;14.6;70000 +2001-08-31_09:00:00;14.3;70000 +2001-08-31_10:00:00;14.2;70000 +2001-08-31_11:00:00;14.5;70000 +2001-08-31_12:00:00;14.3;70000 +2001-08-31_13:00:00;14.2;70000 +2001-08-31_14:00:00;14.3;70000 +2001-08-31_15:00:00;14.3;70000 +2001-08-31_16:00:00;14.2;70000 +2001-08-31_17:00:00;14.3;70000 +2001-08-31_18:00:00;14.2;70000 +2001-08-31_19:00:00;14.1;70000 +2001-08-31_20:00:00;14.2;70000 +2001-08-31_21:00:00;14.3;70000 +2001-08-31_22:00:00;14.2;70000 +2001-08-31_23:00:00;14.3;70000 +2001-09-01_00:00:00;14.6;70000 +2001-09-01_01:00:00;14.8;70000 +2001-09-01_02:00:00;10.5;78947 +2001-09-01_03:00:00;9.7;78947 +2001-09-01_04:00:00;9.2;78947 +2001-09-01_05:00:00;8.9;78947 +2001-09-01_06:00:00;14.9;70000 +2001-09-01_07:00:00;14.8;70000 +2001-09-01_08:00:00;14.8;70000 +2001-09-01_09:00:00;14.6;70000 +2001-09-01_10:00:00;14.7;70000 +2001-09-01_11:00:00;14.9;70000 +2001-09-01_12:00:00;14.8;70000 +2001-09-01_13:00:00;14.4;70000 +2001-09-01_14:00:00;14.1;70000 +2001-09-01_15:00:00;14.1;70000 +2001-09-01_16:00:00;14.3;70000 +2001-09-01_17:00:00;14.3;70000 +2001-09-01_18:00:00;14.2;70000 +2001-09-01_19:00:00;14.1;70000 +2001-09-01_20:00:00;13.9;70000 +2001-09-01_21:00:00;13.9;70000 +2001-09-01_22:00:00;13.8;70000 +2001-09-01_23:00:00;13.6;70000 +2001-09-02_00:00:00;13.6;70000 +2001-09-02_01:00:00;13.3;70000 +2001-09-02_02:00:00;13.2;70000 +2001-09-02_03:00:00;13.2;70000 +2001-09-02_04:00:00;13.1;70000 +2001-09-02_05:00:00;13.1;70000 +2001-09-02_06:00:00;13.2;70000 +2001-09-02_07:00:00;13;70000 +2001-09-02_08:00:00;13.1;70000 +2001-09-02_09:00:00;13.4;70000 +2001-09-02_10:00:00;13.6;70000 +2001-09-02_11:00:00;13.7;70000 +2001-09-02_12:00:00;13.8;70000 +2001-09-02_13:00:00;13.9;70000 +2001-09-02_14:00:00;13.5;70000 +2001-09-02_15:00:00;13.6;70000 +2001-09-02_16:00:00;14;70000 +2001-09-02_17:00:00;13.6;70000 +2001-09-02_18:00:00;13.6;70000 +2001-09-02_19:00:00;13.9;70000 +2001-09-02_20:00:00;14.3;70000 +2001-09-02_21:00:00;14.6;70000 +2001-09-02_22:00:00;14.6;70000 +2001-09-02_23:00:00;14.9;70000 +2001-09-03_00:00:00;14.5;70000 +2001-09-03_01:00:00;14.6;70000 +2001-09-03_02:00:00;14.8;70000 +2001-09-03_03:00:00;14.7;70000 +2001-09-03_04:00:00;14.8;70000 +2001-09-03_05:00:00;14.9;70000 +2001-09-03_06:00:00;15.1;70000 +2001-09-03_07:00:00;15.1;70000 +2001-09-03_08:00:00;15.2;70000 +2001-09-03_09:00:00;15.5;70000 +2001-09-03_10:00:00;15.8;70000 +2001-09-03_11:00:00;15.8;70000 +2001-09-03_12:00:00;15.9;70000 +2001-09-03_13:00:00;16.2;70000 +2001-09-03_14:00:00;16.2;70000 +2001-09-03_15:00:00;16.2;70000 +2001-09-03_16:00:00;16.1;70000 +2001-09-03_17:00:00;16.1;70000 +2001-09-03_18:00:00;16.2;70000 +2001-09-03_19:00:00;16.3;70000 +2001-09-03_20:00:00;16.4;70000 +2001-09-03_21:00:00;16.4;70000 +2001-09-03_22:00:00;16.1;70000 +2001-09-03_23:00:00;16.2;70000 +2001-09-04_00:00:00;15.9;70000 +2001-09-04_01:00:00;15.6;70000 +2001-09-04_02:00:00;15.3;70000 +2001-09-04_03:00:00;15.3;70000 +2001-09-04_04:00:00;15.6;70000 +2001-09-04_05:00:00;15.6;70000 +2001-09-04_06:00:00;15.4;70000 +2001-09-04_07:00:00;15.4;70000 +2001-09-04_08:00:00;15.3;70000 +2001-09-04_09:00:00;15.4;70000 +2001-09-04_10:00:00;15.5;70000 +2001-09-04_11:00:00;15.8;70000 +2001-09-04_12:00:00;16;70000 +2001-09-04_13:00:00;15.3;70000 +2001-09-04_14:00:00;15.3;70000 +2001-09-04_15:00:00;15.3;70000 +2001-09-04_16:00:00;15.3;70000 +2001-09-04_17:00:00;15.3;70000 +2001-09-04_18:00:00;15.3;70203 +2001-09-04_19:00:00;15.3;70203 +2001-09-04_20:00:00;15;70000 +2001-09-04_21:00:00;15.4;70000 +2001-09-04_22:00:00;15.4;70000 +2001-09-04_23:00:00;15.6;70000 +2001-09-05_00:00:00;15.6;70000 +2001-09-05_01:00:00;15.4;70000 +2001-09-05_02:00:00;15.7;70000 +2001-09-05_03:00:00;15.8;70000 +2001-09-05_04:00:00;15.9;70000 +2001-09-05_05:00:00;15.9;70000 +2001-09-05_06:00:00;16;70000 +2001-09-05_07:00:00;16;70000 +2001-09-05_08:00:00;16.1;70000 +2001-09-05_09:00:00;16.2;70000 +2001-09-05_10:00:00;15.9;70000 +2001-09-05_11:00:00;15.9;70000 +2001-09-05_12:00:00;15.7;70000 +2001-09-05_13:00:00;15.5;70000 +2001-09-05_14:00:00;15;70000 +2001-09-05_15:00:00;14.7;70000 +2001-09-05_16:00:00;14.9;70000 +2001-09-05_17:00:00;14.8;70000 +2001-09-05_18:00:00;14.6;70000 +2001-09-05_19:00:00;14.5;70000 +2001-09-05_20:00:00;14.6;70000 +2001-09-05_21:00:00;14.8;70000 +2001-09-05_22:00:00;14.3;70000 +2001-09-05_23:00:00;14.2;70000 +2001-09-06_00:00:00;14.2;70000 +2001-09-06_01:00:00;14.2;70000 +2001-09-06_02:00:00;14.1;70000 +2001-09-06_03:00:00;14.1;70000 +2001-09-06_04:00:00;14.1;70000 +2001-09-06_05:00:00;14;70000 +2001-09-06_06:00:00;14.1;70000 +2001-09-06_07:00:00;14.3;70000 +2001-09-06_08:00:00;14.6;70000 +2001-09-06_09:00:00;14.9;70000 +2001-09-06_10:00:00;13.8;70000 +2001-09-06_11:00:00;14.3;70000 +2001-09-06_12:00:00;14.8;70000 +2001-09-06_13:00:00;14.9;70000 +2001-09-06_14:00:00;14.9;70000 +2001-09-06_15:00:00;14.9;70000 +2001-09-06_16:00:00;14.9;70000 +2001-09-06_17:00:00;14.8;70000 +2001-09-06_18:00:00;14.7;70000 +2001-09-06_19:00:00;14.6;70000 +2001-09-06_20:00:00;14.6;70000 +2001-09-06_21:00:00;14.5;70000 +2001-09-06_22:00:00;14.7;70000 +2001-09-06_23:00:00;14.7;70000 +2001-09-07_00:00:00;14.6;70000 +2001-09-07_01:00:00;14.6;70000 +2001-09-07_02:00:00;14.5;70000 +2001-09-07_03:00:00;14.3;70000 +2001-09-07_04:00:00;14.6;70000 +2001-09-07_05:00:00;14;70000 +2001-09-07_06:00:00;13.6;70000 +2001-09-07_07:00:00;13.7;70000 +2001-09-07_08:00:00;12.9;70000 +2001-09-07_09:00:00;13.6;70000 +2001-09-07_10:00:00;13.4;70000 +2001-09-07_11:00:00;13.2;70000 +2001-09-07_12:00:00;12.3;70000 +2001-09-07_13:00:00;13.6;70000 +2001-09-07_14:00:00;13.6;70000 +2001-09-07_15:00:00;13.3;70000 +2001-09-07_16:00:00;13.5;70000 +2001-09-07_17:00:00;13.6;70000 +2001-09-07_18:00:00;13.5;70000 +2001-09-07_19:00:00;13.4;70000 +2001-09-07_20:00:00;13.5;70000 +2001-09-07_21:00:00;13.1;70000 +2001-09-07_22:00:00;12.5;70000 +2001-09-07_23:00:00;12.5;70000 +2001-09-08_00:00:00;12.5;70000 +2001-09-08_01:00:00;12.8;70000 +2001-09-08_02:00:00;11.8;70000 +2001-09-08_03:00:00;12.5;70000 +2001-09-08_04:00:00;13.1;70000 +2001-09-08_05:00:00;13.2;70000 +2001-09-08_06:00:00;12.6;70000 +2001-09-08_07:00:00;12.9;70000 +2001-09-08_08:00:00;13.5;70000 +2001-09-08_09:00:00;13.5;70000 +2001-09-08_10:00:00;13.4;70000 +2001-09-08_11:00:00;13.6;70000 +2001-09-08_12:00:00;12.5;70000 +2001-09-08_13:00:00;13.2;70000 +2001-09-08_14:00:00;13.5;70000 +2001-09-08_15:00:00;13.6;70000 +2001-09-08_16:00:00;13.6;70000 +2001-09-08_17:00:00;13.6;70000 +2001-09-08_18:00:00;13.6;70000 +2001-09-08_19:00:00;13.6;70000 +2001-09-08_20:00:00;13.6;70203 +2001-09-08_21:00:00;13.7;70000 +2001-09-08_22:00:00;13.6;70000 +2001-09-08_23:00:00;13.6;70000 +2001-09-09_00:00:00;13.8;70000 +2001-09-09_01:00:00;13.6;70000 +2001-09-09_02:00:00;13.5;70000 +2001-09-09_03:00:00;13.6;70000 +2001-09-09_04:00:00;13.5;70000 +2001-09-09_05:00:00;12.3;70000 +2001-09-09_06:00:00;13.3;70000 +2001-09-09_07:00:00;13.2;70000 +2001-09-09_08:00:00;13.2;70000 +2001-09-09_09:00:00;13.3;70000 +2001-09-09_10:00:00;13.4;58927 +2001-09-09_11:00:00;13.5;70000 +2001-09-09_12:00:00;13.6;70000 +2001-09-09_13:00:00;13.6;70000 +2001-09-09_14:00:00;13.6;70000 +2001-09-09_15:00:00;13.6;70000 +2001-09-09_16:00:00;13.7;70000 +2001-09-09_17:00:00;13.8;70000 +2001-09-09_18:00:00;12.1;70000 +2001-09-09_19:00:00;13.3;70000 +2001-09-09_20:00:00;13.1;70000 +2001-09-09_21:00:00;12.9;70000 +2001-09-09_22:00:00;11.8;70000 +2001-09-09_23:00:00;11.6;70000 +2001-09-10_00:00:00;11.8;70000 +2001-09-10_01:00:00;11.5;70000 +2001-09-10_02:00:00;11.6;70000 +2001-09-10_03:00:00;11.9;70000 +2001-09-10_04:00:00;12.4;70000 +2001-09-10_05:00:00;12.5;70000 +2001-09-10_06:00:00;12.9;70000 +2001-09-10_07:00:00;12.8;70000 +2001-09-10_08:00:00;13;70000 +2001-09-10_09:00:00;14.6;70000 +2001-09-10_10:00:00;14.8;70000 +2001-09-10_11:00:00;15;70000 +2001-09-10_12:00:00;15.1;70000 +2001-09-10_13:00:00;15.3;70000 +2001-09-10_14:00:00;15.3;70000 +2001-09-10_15:00:00;15.2;70000 +2001-09-10_16:00:00;15.6;70000 +2001-09-10_17:00:00;15.4;70000 +2001-09-10_18:00:00;15.4;70000 +2001-09-10_19:00:00;15;70000 +2001-09-10_20:00:00;15.6;70000 +2001-09-10_21:00:00;16.4;70000 +2001-09-10_22:00:00;16.6;70000 +2001-09-10_23:00:00;16.4;70000 +2001-09-11_00:00:00;16.3;70000 +2001-09-11_01:00:00;16.1;70000 +2001-09-11_02:00:00;15.9;70000 +2001-09-11_03:00:00;15.9;70000 +2001-09-11_04:00:00;15.9;70000 +2001-09-11_05:00:00;15.6;70000 +2001-09-11_06:00:00;15.6;70000 +2001-09-11_07:00:00;15.4;70000 +2001-09-11_08:00:00;15.7;70000 +2001-09-11_09:00:00;15.7;70000 +2001-09-11_10:00:00;15.6;70000 +2001-09-11_11:00:00;15.3;70000 +2001-09-11_12:00:00;15.2;70000 +2001-09-11_13:00:00;15.2;70000 +2001-09-11_14:00:00;15.2;70000 +2001-09-11_15:00:00;15.3;70000 +2001-09-11_16:00:00;15.4;70000 +2001-09-11_17:00:00;15.5;70000 +2001-09-11_18:00:00;15.4;70000 +2001-09-11_19:00:00;15.1;70000 +2001-09-11_20:00:00;15;70000 +2001-09-11_21:00:00;14.9;70000 +2001-09-11_22:00:00;14.9;70000 +2001-09-11_23:00:00;15;70000 +2001-09-12_00:00:00;15.1;70000 +2001-09-12_01:00:00;14.9;70000 +2001-09-12_02:00:00;14.3;70000 +2001-09-12_03:00:00;13.6;70000 +2001-09-12_04:00:00;14.2;70000 +2001-09-12_05:00:00;14.3;70000 +2001-09-12_06:00:00;15;70000 +2001-09-12_07:00:00;14.9;70000 +2001-09-12_08:00:00;15.1;70000 +2001-09-12_09:00:00;14.9;70000 +2001-09-12_10:00:00;14.8;70000 +2001-09-12_11:00:00;14.8;70000 +2001-09-12_12:00:00;13.6;70000 +2001-09-12_13:00:00;13.3;70000 +2001-09-12_14:00:00;13.6;70000 +2001-09-12_15:00:00;14.1;70000 +2001-09-12_16:00:00;15;70000 +2001-09-12_17:00:00;15.2;70000 +2001-09-12_18:00:00;14.2;70000 +2001-09-12_19:00:00;14.2;70000 +2001-09-12_20:00:00;14.1;70000 +2001-09-12_21:00:00;14.2;70000 +2001-09-12_22:00:00;14.2;70000 +2001-09-12_23:00:00;14.2;70000 +2001-09-13_00:00:00;14.3;70000 +2001-09-13_01:00:00;14.4;70000 +2001-09-13_02:00:00;14.2;70000 +2001-09-13_03:00:00;14.3;70000 +2001-09-13_04:00:00;14.2;70000 +2001-09-13_05:00:00;13.9;70000 +2001-09-13_06:00:00;13.9;70000 +2001-09-13_07:00:00;13.9;70000 +2001-09-13_08:00:00;14.2;70000 +2001-09-13_09:00:00;14.2;70000 +2001-09-13_10:00:00;14.1;70000 +2001-09-13_11:00:00;13.8;70000 +2001-09-13_12:00:00;13.5;70000 +2001-09-13_13:00:00;14.2;70000 +2001-09-13_14:00:00;14;70000 +2001-09-13_15:00:00;14.1;70000 +2001-09-13_16:00:00;13.6;70000 +2001-09-13_17:00:00;13.9;70000 +2001-09-13_18:00:00;13.5;70000 +2001-09-13_19:00:00;13.9;70000 +2001-09-13_20:00:00;14.1;70000 +2001-09-13_21:00:00;14.1;70000 +2001-09-13_22:00:00;13.9;70000 +2001-09-13_23:00:00;13.9;70000 +2001-09-14_00:00:00;13.9;70000 +2001-09-14_01:00:00;14;70000 +2001-09-14_02:00:00;14;70000 +2001-09-14_03:00:00;13.9;70000 +2001-09-14_04:00:00;14.1;70000 +2001-09-14_05:00:00;14.1;70000 +2001-09-14_06:00:00;14.1;70000 +2001-09-14_07:00:00;13.9;70000 +2001-09-14_08:00:00;13.8;70000 +2001-09-14_09:00:00;13.3;70000 +2001-09-14_10:00:00;13.2;70000 +2001-09-14_11:00:00;13.2;70000 +2001-09-14_12:00:00;12.9;70000 +2001-09-14_13:00:00;12.9;70000 +2001-09-14_14:00:00;12.9;70000 +2001-09-14_15:00:00;12.9;70000 +2001-09-14_16:00:00;12.7;70000 +2001-09-14_17:00:00;12.8;70000 +2001-09-14_18:00:00;12.5;70000 +2001-09-14_19:00:00;12.6;70000 +2001-09-14_20:00:00;12.6;70000 +2001-09-14_21:00:00;12;70000 +2001-09-14_22:00:00;11.8;70000 +2001-09-14_23:00:00;12.9;70000 +2001-09-15_00:00:00;13.3;70000 +2001-09-15_01:00:00;12.8;70000 +2001-09-15_02:00:00;11.9;70000 +2001-09-15_03:00:00;12.6;70000 +2001-09-15_04:00:00;13.2;70000 +2001-09-15_05:00:00;13.2;70000 +2001-09-15_06:00:00;12.9;70000 +2001-09-15_07:00:00;12.8;70000 +2001-09-15_08:00:00;12.5;70000 +2001-09-15_09:00:00;11.4;70000 +2001-09-15_10:00:00;12.7;70000 +2001-09-15_11:00:00;11.1;70000 +2001-09-15_12:00:00;12.2;70000 +2001-09-15_13:00:00;12.1;70000 +2001-09-15_14:00:00;12.1;70000 +2001-09-15_15:00:00;12.2;70000 +2001-09-15_16:00:00;12.2;70000 +2001-09-15_17:00:00;12.2;70000 +2001-09-15_18:00:00;12.2;70000 +2001-09-15_19:00:00;12.2;70000 +2001-09-15_20:00:00;12.1;70000 +2001-09-15_21:00:00;12.1;70000 +2001-09-15_22:00:00;12.1;70000 +2001-09-15_23:00:00;12.2;70000 +2001-09-16_00:00:00;11.2;70000 +2001-09-16_01:00:00;11.6;70000 +2001-09-16_02:00:00;11.1;70000 +2001-09-16_03:00:00;11.5;70000 +2001-09-16_04:00:00;11.3;70000 +2001-09-16_05:00:00;11.5;70000 +2001-09-16_06:00:00;11.2;70000 +2001-09-16_07:00:00;11.5;70000 +2001-09-16_08:00:00;11.2;70000 +2001-09-16_09:00:00;11.4;70000 +2001-09-16_10:00:00;11.3;70000 +2001-09-16_11:00:00;11.5;70000 +2001-09-16_12:00:00;11.7;70000 +2001-09-16_13:00:00;11.8;70000 +2001-09-16_14:00:00;11.8;70000 +2001-09-16_15:00:00;11.9;70000 +2001-09-16_16:00:00;12.1;70000 +2001-09-16_17:00:00;12.1;70000 +2001-09-16_18:00:00;11.5;70000 +2001-09-16_19:00:00;11.5;70000 +2001-09-16_20:00:00;11.7;70000 +2001-09-16_21:00:00;11.8;70000 +2001-09-16_22:00:00;11.7;70000 +2001-09-16_23:00:00;11.9;70000 +2001-09-17_00:00:00;12;70000 +2001-09-17_01:00:00;11.6;70000 +2001-09-17_02:00:00;11.8;70000 +2001-09-17_03:00:00;11.9;70000 +2001-09-17_04:00:00;12;70000 +2001-09-17_05:00:00;12;70000 +2001-09-17_06:00:00;12.3;70000 +2001-09-17_07:00:00;12.5;70000 +2001-09-17_08:00:00;12.4;70000 +2001-09-17_09:00:00;12.6;70000 +2001-09-17_10:00:00;13.2;70000 +2001-09-17_11:00:00;13.1;70000 +2001-09-17_12:00:00;13.3;70000 +2001-09-17_13:00:00;12.8;70000 +2001-09-17_14:00:00;12.6;70000 +2001-09-17_15:00:00;12.9;70000 +2001-09-17_16:00:00;12.9;70000 +2001-09-17_17:00:00;14;70000 +2001-09-17_18:00:00;14.1;70000 +2001-09-17_19:00:00;14.1;70000 +2001-09-17_20:00:00;14.2;70000 +2001-09-17_21:00:00;14.2;70000 +2001-09-17_22:00:00;14.5;70000 +2001-09-17_23:00:00;14.2;70000 +2001-09-18_00:00:00;14.3;70000 +2001-09-18_01:00:00;14.2;70000 +2001-09-18_02:00:00;14.5;70000 +2001-09-18_03:00:00;14.3;70000 +2001-09-18_04:00:00;12.8;70000 +2001-09-18_05:00:00;12.2;70000 +2001-09-18_06:00:00;12.7;70000 +2001-09-18_07:00:00;12.8;70000 +2001-09-18_08:00:00;12.8;70000 +2001-09-18_09:00:00;13.2;70000 +2001-09-18_10:00:00;13.7;70000 +2001-09-18_11:00:00;14.3;70000 +2001-09-18_12:00:00;14.3;70000 +2001-09-18_13:00:00;14.5;70000 +2001-09-18_14:00:00;14.4;70000 +2001-09-18_15:00:00;14.5;70000 +2001-09-18_16:00:00;14.5;70000 +2001-09-18_17:00:00;14.6;70000 +2001-09-18_18:00:00;14.5;70000 +2001-09-18_19:00:00;14.2;70000 +2001-09-18_20:00:00;13.3;70000 +2001-09-18_21:00:00;13.1;70000 +2001-09-18_22:00:00;12.8;70000 +2001-09-18_23:00:00;12.7;70000 +2001-09-19_00:00:00;12.6;70000 +2001-09-19_01:00:00;12.4;70000 +2001-09-19_02:00:00;12.1;70000 +2001-09-19_03:00:00;11.9;70000 +2001-09-19_04:00:00;12.1;70000 +2001-09-19_05:00:00;11;78947 +2001-09-19_06:00:00;11.6;78947 +2001-09-19_07:00:00;11.7;78947 +2001-09-19_09:00:00;13.1;78947 +2001-09-19_10:00:00;13.5;78947 +2001-09-19_11:00:00;13.7;78947 +2001-09-19_12:00:00;12.2;78947 +2001-09-19_13:00:00;12.5;78947 +2001-09-19_14:00:00;12.2;78947 +2001-09-19_15:00:00;11.9;78947 +2001-09-19_16:00:00;11.8;78947 +2001-09-19_17:00:00;11.8;78947 +2001-09-19_18:00:00;11.6;78947 +2001-09-19_19:00:00;11.7;78947 +2001-09-19_20:00:00;11.8;78947 +2001-09-19_21:00:00;11.7;78947 +2001-09-19_22:00:00;11.7;78947 +2001-09-19_23:00:00;11.5;78947 +2001-09-20_00:00:00;11.4;78947 +2001-09-20_01:00:00;11.7;78947 +2001-09-20_02:00:00;12.2;78947 +2001-09-20_03:00:00;12.7;78947 +2001-09-20_04:00:00;12.7;78947 +2001-09-20_06:00:00;12.4;78947 +2001-09-20_07:00:00;12.5;78947 +2001-09-20_08:00:00;12.9;78947 +2001-09-20_09:00:00;13.2;78947 +2001-09-20_10:00:00;13.2;78947 +2001-09-20_11:00:00;13.4;78947 +2001-09-20_12:00:00;13.9;78947 +2001-09-20_13:00:00;14.4;78947 +2001-09-20_14:00:00;14.4;78947 +2001-09-20_15:00:00;14;78947 +2001-09-20_16:00:00;13.3;78947 +2001-09-20_17:00:00;12.7;78947 +2001-09-20_18:00:00;11.7;78947 +2001-09-20_19:00:00;11.6;78947 +2001-09-20_20:00:00;11.3;78947 +2001-09-20_21:00:00;11.3;78947 +2001-09-20_22:00:00;11.2;78947 +2001-09-20_23:00:00;11.3;78947 +2001-09-21_00:00:00;10.7;78947 +2001-09-21_01:00:00;10.8;78947 +2001-09-21_02:00:00;10.8;78947 +2001-09-21_03:00:00;11.3;78947 +2001-09-21_04:00:00;11.4;78947 +2001-09-21_05:00:00;11.3;78947 +2001-09-21_06:00:00;11.5;78947 +2001-09-21_07:00:00;11.9;78947 +2001-09-21_09:00:00;13.2;78947 +2001-09-21_10:00:00;13.3;78947 +2001-09-21_11:00:00;12.9;78947 +2001-09-21_12:00:00;13.3;78947 +2001-09-21_13:00:00;13.6;78947 +2001-09-21_14:00:00;12.7;78947 +2001-09-21_15:00:00;11.7;78947 +2001-09-21_16:00:00;12;78947 +2001-09-21_17:00:00;11.7;78947 +2001-09-21_18:00:00;11.1;78947 +2001-09-21_19:00:00;11.3;78947 +2001-09-21_20:00:00;11.4;78947 +2001-09-21_21:00:00;11.4;78947 +2001-09-21_22:00:00;10.8;78947 +2001-09-21_23:00:00;10.9;78947 +2001-09-22_00:00:00;10.9;78947 +2001-09-22_01:00:00;11.2;78947 +2001-09-22_02:00:00;11.2;78947 +2001-09-22_03:00:00;11.1;78947 +2001-09-22_04:00:00;11.2;78947 +2001-09-22_05:00:00;11.1;78947 +2001-09-22_06:00:00;10.6;78947 +2001-09-22_07:00:00;10.7;78947 +2001-09-22_09:00:00;11.3;78947 +2001-09-22_10:00:00;12.3;78947 +2001-09-22_11:00:00;12.9;78947 +2001-09-22_12:00:00;11.7;78947 +2001-09-22_13:00:00;11.8;78947 +2001-09-22_14:00:00;11.6;78947 +2001-09-22_15:00:00;11.7;78947 +2001-09-22_16:00:00;11.6;78947 +2001-09-22_17:00:00;11.6;78947 +2001-09-22_18:00:00;11.2;78947 +2001-09-22_19:00:00;11.1;78947 +2001-09-22_20:00:00;11;78947 +2001-09-22_21:00:00;10.8;78947 +2001-09-22_22:00:00;10.7;78947 +2001-09-22_23:00:00;10.8;78947 +2001-09-23_00:00:00;10.7;78947 +2001-09-23_01:00:00;10.8;78947 +2001-09-23_02:00:00;10.9;78947 +2001-09-23_03:00:00;10.8;78947 +2001-09-23_04:00:00;10.7;78947 +2001-09-23_05:00:00;10.9;78947 +2001-09-23_06:00:00;10.8;78947 +2001-09-23_07:00:00;11;78947 +2001-09-23_08:00:00;11.1;78947 +2001-09-23_09:00:00;11.7;78947 +2001-09-23_10:00:00;12.2;78947 +2001-09-23_11:00:00;12.6;78947 +2001-09-23_12:00:00;11.6;78947 +2001-09-23_13:00:00;12.6;78947 +2001-09-23_14:00:00;12.8;78947 +2001-09-23_15:00:00;12.7;78947 +2001-09-23_16:00:00;12.1;78947 +2001-09-23_17:00:00;11.6;78947 +2001-09-23_18:00:00;10.5;78947 +2001-09-23_19:00:00;10.6;78947 +2001-09-23_20:00:00;11.2;78947 +2001-09-23_21:00:00;11.6;78947 +2001-09-23_22:00:00;11.8;78947 +2001-09-23_23:00:00;12;78947 +2001-09-24_00:00:00;11.8;78947 +2001-09-24_01:00:00;12.1;78947 +2001-09-24_02:00:00;12.2;78947 +2001-09-24_03:00:00;12.4;78947 +2001-09-24_04:00:00;12.5;78947 +2001-09-24_05:00:00;12.7;78947 +2001-09-24_06:00:00;12.5;78947 +2001-09-24_07:00:00;12.8;78947 +2001-09-24_08:00:00;13.4;78947 +2001-09-24_09:00:00;13.6;78947 +2001-09-24_10:00:00;13.9;78947 +2001-09-24_11:00:00;15;78947 +2001-09-24_12:00:00;14.7;78947 +2001-09-24_13:00:00;14.6;78947 +2001-09-24_14:00:00;14.2;78947 +2001-09-24_15:00:00;13.8;78947 +2001-09-24_16:00:00;13.7;78947 +2001-09-24_17:00:00;13.5;78947 +2001-09-24_18:00:00;13.3;78947 +2001-09-24_19:00:00;13.1;78947 +2001-09-24_20:00:00;13;78947 +2001-09-24_21:00:00;12.9;78947 +2001-09-24_22:00:00;12.9;78947 +2001-09-24_23:00:00;12.8;78947 +2001-09-25_00:00:00;12.4;78947 +2001-09-25_01:00:00;12.4;78947 +2001-09-25_02:00:00;12.4;78947 +2001-09-25_03:00:00;12.4;78947 +2001-09-25_04:00:00;12.4;78947 +2001-09-25_05:00:00;12.5;78947 +2001-09-25_06:00:00;13.1;78947 +2001-09-25_07:00:00;13.3;78947 +2001-09-25_08:00:00;13.7;78947 +2001-09-25_09:00:00;13.8;78947 +2001-09-25_10:00:00;13.9;78947 +2001-09-25_12:00:00;15.2;78947 +2001-09-25_13:00:00;15.5;78947 +2001-09-25_14:00:00;15.8;78947 +2001-09-25_15:00:00;15.2;78947 +2001-09-25_16:00:00;14.5;78947 +2001-09-25_17:00:00;14;78947 +2001-09-25_18:00:00;14.9;78947 +2001-09-25_19:00:00;14.7;78947 +2001-09-25_20:00:00;14.3;78947 +2001-09-25_21:00:00;13.3;78947 +2001-09-25_22:00:00;13.3;78947 +2001-09-25_23:00:00;13.5;78947 +2001-09-26_00:00:00;13.1;78947 +2001-09-26_01:00:00;13.2;78947 +2001-09-26_02:00:00;13.3;78947 +2001-09-26_03:00:00;13.3;78947 +2001-09-26_04:00:00;13.4;78947 +2001-09-26_05:00:00;13.5;78947 +2001-09-26_06:00:00;13.7;78947 +2001-09-26_07:00:00;13.4;78947 +2001-09-26_08:00:00;13.2;78947 +2001-09-26_09:00:00;12.9;78947 +2001-09-26_10:00:00;12.5;78947 +2001-09-26_11:00:00;13.4;78947 +2001-09-26_12:00:00;13.3;78947 +2001-09-26_13:00:00;13.7;78947 +2001-09-26_14:00:00;13.5;78947 +2001-09-26_15:00:00;13;78947 +2001-09-26_16:00:00;12.5;78947 +2001-09-26_17:00:00;12.4;78947 +2001-09-26_18:00:00;12;78947 +2001-09-26_19:00:00;11.7;78947 +2001-09-26_20:00:00;11.5;78947 +2001-09-26_21:00:00;11.4;78947 +2001-09-26_22:00:00;11.3;78947 +2001-09-26_23:00:00;11.1;78947 +2001-09-27_00:00:00;10.8;78947 +2001-09-27_01:00:00;10.5;78947 +2001-09-27_02:00:00;10.1;78947 +2001-09-27_03:00:00;9.8;78947 +2001-09-27_04:00:00;9.7;78947 +2001-09-27_05:00:00;10.1;78947 +2001-09-27_06:00:00;9.5;78947 +2001-09-27_07:00:00;10.9;78947 +2001-09-27_09:00:00;12.5;78947 +2001-09-27_10:00:00;13.2;78947 +2001-09-27_11:00:00;13.9;78947 +2001-09-27_12:00:00;13.8;78947 +2001-09-27_13:00:00;13.7;78947 +2001-09-27_14:00:00;13.5;78947 +2001-09-27_15:00:00;12.9;78947 +2001-09-27_16:00:00;12.6;78947 +2001-09-27_17:00:00;12.4;78947 +2001-09-27_18:00:00;12.2;78947 +2001-09-27_19:00:00;12.3;78947 +2001-09-27_20:00:00;12.5;78947 +2001-09-27_21:00:00;12.7;78947 +2001-09-27_22:00:00;12.8;78947 +2001-09-27_23:00:00;12.9;78947 +2001-09-28_00:00:00;13;78947 +2001-09-28_01:00:00;13.1;78947 +2001-09-28_02:00:00;13.2;78947 +2001-09-28_03:00:00;13.3;78947 +2001-09-28_04:00:00;13.4;78947 +2001-09-28_05:00:00;13.4;78947 +2001-09-28_06:00:00;13.3;78947 +2001-09-28_07:00:00;13.3;78947 +2001-09-28_08:00:00;13.5;78947 +2001-09-28_09:00:00;14.1;78947 +2001-09-28_10:00:00;14.5;78947 +2001-09-28_11:00:00;14.2;78947 +2001-09-28_12:00:00;15.2;78947 +2001-09-28_13:00:00;14.6;78947 +2001-09-28_14:00:00;14.2;78947 +2001-09-28_15:00:00;14;78947 +2001-09-28_16:00:00;14.1;78947 +2001-09-28_17:00:00;14.1;78947 +2001-09-28_18:00:00;14.1;78947 +2001-09-28_19:00:00;14.1;78947 +2001-09-28_20:00:00;14;78947 +2001-09-28_21:00:00;13.9;78947 +2001-09-28_22:00:00;13.9;78947 +2001-09-28_23:00:00;13.8;78947 +2001-09-29_00:00:00;14.1;78947 +2001-09-29_01:00:00;14.1;78947 +2001-09-29_02:00:00;14;78947 +2001-09-29_03:00:00;13.7;78947 +2001-09-29_04:00:00;13.3;78947 +2001-09-29_05:00:00;13;78947 +2001-09-29_06:00:00;12.9;78947 +2001-09-29_07:00:00;13.1;78947 +2001-09-29_09:00:00;14.1;78947 +2001-09-29_10:00:00;14.7;78947 +2001-09-29_11:00:00;15.3;78947 +2001-09-29_12:00:00;14.1;78947 +2001-09-29_13:00:00;14.2;78947 +2001-09-29_14:00:00;14.6;78947 +2001-09-29_15:00:00;14.7;78947 +2001-09-29_16:00:00;14.5;78947 +2001-09-29_17:00:00;14.3;78947 +2001-09-29_18:00:00;14.1;78947 +2001-09-29_19:00:00;14.1;78947 +2001-09-29_20:00:00;14.1;78947 +2001-09-29_21:00:00;14.1;78947 +2001-09-29_22:00:00;14;78947 +2001-09-29_23:00:00;13.5;78947 +2001-09-30_00:00:00;13.2;78947 +2001-09-30_01:00:00;13;78947 +2001-09-30_02:00:00;12.9;78947 +2001-09-30_03:00:00;12.8;78947 +2001-09-30_04:00:00;12.7;78947 +2001-09-30_05:00:00;12.7;78947 +2001-09-30_06:00:00;12.9;78947 +2001-09-30_07:00:00;13;78947 +2001-09-30_09:00:00;13.9;78947 +2001-09-30_10:00:00;14.7;78947 +2001-09-30_11:00:00;15.4;78947 +2001-09-30_12:00:00;16.7;78947 +2001-09-30_13:00:00;16.7;78947 +2001-09-30_14:00:00;16.6;78947 +2001-09-30_15:00:00;16.4;78947 +2001-09-30_16:00:00;16.1;78947 +2001-09-30_17:00:00;15.3;78947 +2001-09-30_18:00:00;14.9;78947 +2001-09-30_19:00:00;14.8;78947 +2001-09-30_20:00:00;14.9;78947 +2001-09-30_21:00:00;14.8;78947 +2001-09-30_22:00:00;14.7;78947 +2001-09-30_23:00:00;14.7;78947 +2001-10-01_00:00:00;14.1;78947 +2001-10-01_01:00:00;14.2;78947 +2001-10-01_02:00:00;14.2;78947 +2001-10-01_03:00:00;14.1;78947 +2001-10-01_04:00:00;13.8;78947 +2001-10-01_05:00:00;13.6;78947 +2001-10-01_06:00:00;13.4;78947 +2001-10-01_07:00:00;13.8;78947 +2001-10-01_08:00:00;14.6;78947 +2001-10-01_09:00:00;15.5;78947 +2001-10-01_10:00:00;16.2;78947 +2001-10-01_11:00:00;16.7;78947 +2001-10-01_12:00:00;17.7;78947 +2001-10-01_13:00:00;18;78947 +2001-10-01_14:00:00;17.8;78947 +2001-10-01_15:00:00;17.2;78947 +2001-10-01_16:00:00;16.2;78947 +2001-10-01_17:00:00;15.3;78947 +2001-10-01_18:00:00;14.9;78947 +2001-10-01_19:00:00;14.8;78947 +2001-10-01_20:00:00;14.5;78947 +2001-10-01_21:00:00;14.2;78947 +2001-10-01_22:00:00;14.1;78947 +2001-10-01_23:00:00;14;78947 +2001-10-02_00:00:00;14.1;78947 +2001-10-02_01:00:00;13.9;78947 +2001-10-02_02:00:00;13.6;78947 +2001-10-02_03:00:00;13.6;78947 +2001-10-02_04:00:00;13.5;78947 +2001-10-02_05:00:00;13.5;78947 +2001-10-02_06:00:00;13.8;78947 +2001-10-02_07:00:00;13.9;78947 +2001-10-02_08:00:00;14;78947 +2001-10-02_09:00:00;14.4;78947 +2001-10-02_10:00:00;14.4;78947 +2001-10-02_11:00:00;14.1;78947 +2001-10-02_12:00:00;15;78947 +2001-10-02_13:00:00;14.7;78947 +2001-10-02_14:00:00;13.9;78947 +2001-10-02_15:00:00;13.4;78947 +2001-10-02_16:00:00;13;78947 +2001-10-02_17:00:00;12.7;78947 +2001-10-02_18:00:00;12.3;78947 +2001-10-02_19:00:00;12;78947 +2001-10-02_20:00:00;11.3;78947 +2001-10-02_21:00:00;10.6;78947 +2001-10-02_22:00:00;11.4;78947 +2001-10-02_23:00:00;11.1;78947 +2001-10-03_00:00:00;10.9;78947 +2001-10-03_01:00:00;11.3;78947 +2001-10-03_02:00:00;11.6;78947 +2001-10-03_03:00:00;11.8;78947 +2001-10-03_04:00:00;11.3;78947 +2001-10-03_05:00:00;11.7;78947 +2001-10-03_06:00:00;12.2;78947 +2001-10-03_07:00:00;12.8;78947 +2001-10-03_08:00:00;13.3;78947 +2001-10-03_09:00:00;13.1;78947 +2001-10-03_10:00:00;13.4;78947 +2001-10-03_11:00:00;13.7;78947 +2001-10-03_12:00:00;13.8;78947 +2001-10-03_13:00:00;13.9;78947 +2001-10-03_14:00:00;13.6;78947 +2001-10-03_15:00:00;13.9;78947 +2001-10-03_16:00:00;14.2;78947 +2001-10-03_17:00:00;14.3;78947 +2001-10-03_18:00:00;13.9;78947 +2001-10-03_19:00:00;14.1;78947 +2001-10-03_20:00:00;14.4;78947 +2001-10-03_21:00:00;12.9;78947 +2001-10-03_22:00:00;12.8;78947 +2001-10-03_23:00:00;13.1;78947 +2001-10-04_00:00:00;12.1;78947 +2001-10-04_01:00:00;12.3;78947 +2001-10-04_02:00:00;11.9;78947 +2001-10-04_03:00:00;11.5;78947 +2001-10-04_04:00:00;11.7;78947 +2001-10-04_05:00:00;12.3;78947 +2001-10-04_06:00:00;11;78947 +2001-10-04_07:00:00;11.1;78947 +2001-10-04_09:00:00;11.5;78947 +2001-10-04_10:00:00;11.6;78947 +2001-10-04_11:00:00;12;78947 +2001-10-04_12:00:00;11.6;78947 +2001-10-04_13:00:00;12.2;78947 +2001-10-04_14:00:00;13;78947 +2001-10-04_15:00:00;12;78947 +2001-10-04_16:00:00;12.4;78947 +2001-10-04_17:00:00;11.5;78947 +2001-10-04_18:00:00;11.1;78947 +2001-10-04_19:00:00;11.2;78947 +2001-10-04_20:00:00;11.8;78947 +2001-10-04_21:00:00;11.5;78947 +2001-10-04_22:00:00;11.4;78947 +2001-10-04_23:00:00;11.2;78947 +2001-10-05_00:00:00;10.5;78947 +2001-10-05_01:00:00;10.8;78947 +2001-10-05_02:00:00;10.7;78947 +2001-10-05_03:00:00;10.9;78947 +2001-10-05_04:00:00;11.1;78947 +2001-10-05_05:00:00;10.9;78947 +2001-10-05_06:00:00;10.8;78947 +2001-10-05_07:00:00;10.9;78947 +2001-10-05_09:00:00;12.2;78947 +2001-10-05_10:00:00;12.8;78947 +2001-10-05_11:00:00;12.6;78947 +2001-10-05_12:00:00;11.9;78947 +2001-10-05_13:00:00;12.3;78947 +2001-10-05_14:00:00;13;78947 +2001-10-05_15:00:00;13.4;78947 +2001-10-05_16:00:00;13.5;78947 +2001-10-05_17:00:00;13.6;78947 +2001-10-05_18:00:00;13.6;78947 +2001-10-05_19:00:00;13.7;78947 +2001-10-05_20:00:00;13.7;78947 +2001-10-05_21:00:00;13.6;78947 +2001-10-05_22:00:00;13.4;78947 +2001-10-05_23:00:00;13.3;78947 +2001-10-06_00:00:00;12.4;78947 +2001-10-06_01:00:00;12;78947 +2001-10-06_02:00:00;11.9;78947 +2001-10-06_03:00:00;11.9;78947 +2001-10-06_04:00:00;11.2;78947 +2001-10-06_05:00:00;11;78947 +2001-10-06_06:00:00;9.5;78947 +2001-10-06_07:00:00;9.7;78947 +2001-10-06_08:00:00;10.2;78947 +2001-10-06_09:00:00;10.9;78947 +2001-10-06_10:00:00;11.1;78947 +2001-10-06_11:00:00;11.3;78947 +2001-10-06_12:00:00;10.5;78947 +2001-10-06_13:00:00;9.8;78947 +2001-10-06_14:00:00;10;78947 +2001-10-06_15:00:00;10.1;78947 +2001-10-06_16:00:00;10.1;78947 +2001-10-06_17:00:00;9.9;78947 +2001-10-06_18:00:00;9.1;78947 +2001-10-06_19:00:00;8.8;78947 +2001-10-06_20:00:00;8.5;78947 +2001-10-06_21:00:00;9;78947 +2001-10-06_22:00:00;9;78947 +2001-10-06_23:00:00;9.1;78947 +2001-10-07_00:00:00;9.2;78947 +2001-10-07_01:00:00;9.6;78947 +2001-10-07_02:00:00;9.5;78947 +2001-10-07_03:00:00;9.9;78947 +2001-10-07_04:00:00;10.6;78947 +2001-10-07_05:00:00;10.1;78947 +2001-10-07_06:00:00;9.5;78947 +2001-10-07_07:00:00;9.4;78947 +2001-10-07_09:00:00;10.7;78947 +2001-10-07_10:00:00;10.7;78947 +2001-10-07_11:00:00;9.5;78947 +2001-10-07_12:00:00;9.9;78947 +2001-10-07_13:00:00;9.8;78947 +2001-10-07_14:00:00;10.5;78947 +2001-10-07_15:00:00;9.1;78947 +2001-10-07_16:00:00;9.5;78947 +2001-10-07_17:00:00;9.5;78947 +2001-10-07_18:00:00;8.1;78947 +2001-10-07_19:00:00;9.1;78947 +2001-10-07_20:00:00;9.3;78947 +2001-10-07_21:00:00;9.3;78947 +2001-10-07_22:00:00;9.2;78947 +2001-10-07_23:00:00;9.2;78947 +2001-10-08_00:00:00;7.8;78947 +2001-10-08_01:00:00;7.7;78947 +2001-10-08_02:00:00;7.9;78947 +2001-10-08_03:00:00;7.6;78947 +2001-10-08_04:00:00;7.8;78947 +2001-10-08_05:00:00;7.9;78947 +2001-10-08_06:00:00;7.1;78947 +2001-10-08_07:00:00;6.8;78947 +2001-10-08_08:00:00;7.1;78947 +2001-10-08_09:00:00;8.2;78947 +2001-10-08_10:00:00;9.3;78947 +2001-10-08_11:00:00;10;78947 +2001-10-08_12:00:00;9.8;78947 +2001-10-08_13:00:00;10.2;78947 +2001-10-08_14:00:00;10.2;78947 +2001-10-08_15:00:00;10;78947 +2001-10-08_16:00:00;9.6;78947 +2001-10-08_17:00:00;8.5;78947 +2001-10-08_18:00:00;5.9;78947 +2001-10-08_19:00:00;4.4;78947 +2001-10-08_20:00:00;3.5;78947 +2001-10-08_21:00:00;3.6;78947 +2001-10-08_22:00:00;4.6;78947 +2001-10-08_23:00:00;6.2;78947 +2001-10-09_00:00:00;8;78947 +2001-10-09_01:00:00;8.8;78947 +2001-10-09_02:00:00;8.9;78947 +2001-10-09_03:00:00;8.8;78947 +2001-10-09_04:00:00;8.9;78947 +2001-10-09_05:00:00;8.9;78947 +2001-10-09_06:00:00;9.6;78947 +2001-10-09_07:00:00;9.5;78947 +2001-10-09_08:00:00;9.1;78947 +2001-10-09_09:00:00;9.2;78947 +2001-10-09_10:00:00;9.4;78947 +2001-10-09_11:00:00;9.7;78947 +2001-10-09_12:00:00;9.7;78947 +2001-10-09_13:00:00;10.5;78947 +2001-10-09_14:00:00;11.2;78947 +2001-10-09_15:00:00;10.2;78947 +2001-10-09_16:00:00;9.9;78947 +2001-10-09_17:00:00;10.2;78947 +2001-10-09_18:00:00;10.8;78947 +2001-10-09_19:00:00;11.1;78947 +2001-10-09_20:00:00;10.9;78947 +2001-10-09_21:00:00;10.6;78947 +2001-10-09_22:00:00;10.4;78947 +2001-10-09_23:00:00;10.2;78947 +2001-10-10_00:00:00;9.8;78947 +2001-10-10_01:00:00;9.9;78947 +2001-10-10_02:00:00;9.9;78947 +2001-10-10_03:00:00;9.9;78947 +2001-10-10_04:00:00;10;78947 +2001-10-10_05:00:00;10.2;78947 +2001-10-10_06:00:00;10.2;78947 +2001-10-10_07:00:00;10.3;78947 +2001-10-10_08:00:00;10;78947 +2001-10-10_09:00:00;9.7;78947 +2001-10-10_10:00:00;10.7;78947 +2001-10-10_11:00:00;10.2;78947 +2001-10-10_12:00:00;10.5;78947 +2001-10-10_13:00:00;10.4;78947 +2001-10-10_14:00:00;10.9;78947 +2001-10-10_15:00:00;11;78947 +2001-10-10_16:00:00;10.4;78947 +2001-10-10_17:00:00;10.1;78947 +2001-10-10_18:00:00;9.5;78947 +2001-10-10_19:00:00;9.5;78947 +2001-10-10_20:00:00;9.7;78947 +2001-10-10_21:00:00;9.7;78947 +2001-10-10_22:00:00;9.8;78947 +2001-10-10_23:00:00;9.8;78947 +2001-10-11_00:00:00;9.6;78947 +2001-10-11_01:00:00;9.3;78947 +2001-10-11_02:00:00;9.7;78947 +2001-10-11_03:00:00;10;78947 +2001-10-11_04:00:00;9.8;78947 +2001-10-11_05:00:00;9.6;78947 +2001-10-11_06:00:00;9.2;78947 +2001-10-11_07:00:00;9.2;78947 +2001-10-11_08:00:00;9.4;78947 +2001-10-11_09:00:00;10.3;78947 +2001-10-11_10:00:00;9.6;78947 +2001-10-11_11:00:00;10.3;78947 +2001-10-11_12:00:00;10.9;78947 +2001-10-11_13:00:00;11;78947 +2001-10-11_14:00:00;10.7;78947 +2001-10-11_15:00:00;8.7;78947 +2001-10-11_16:00:00;8.7;78947 +2001-10-11_17:00:00;8.8;78947 +2001-10-11_18:00:00;8.2;78947 +2001-10-11_19:00:00;8.6;78947 +2001-10-11_20:00:00;8.7;78947 +2001-10-11_21:00:00;9.2;78947 +2001-10-11_22:00:00;9.3;78947 +2001-10-11_23:00:00;9.1;78947 +2001-10-12_00:00:00;9.3;78947 +2001-10-12_01:00:00;9.4;78947 +2001-10-12_02:00:00;9.6;78947 +2001-10-12_03:00:00;9.7;78947 +2001-10-12_04:00:00;9.6;78947 +2001-10-12_05:00:00;9.3;78947 +2001-10-12_06:00:00;9.1;78947 +2001-10-12_07:00:00;9.2;78947 +2001-10-12_08:00:00;9.3;78947 +2001-10-12_11:00:00;9.5;78947 +2001-10-12_12:00:00;10.6;78947 +2001-10-12_13:00:00;10.5;78947 +2001-10-12_14:00:00;11;78947 +2001-10-12_15:00:00;10.6;78947 +2001-10-12_16:00:00;10.1;78947 +2001-10-12_17:00:00;9.8;78947 +2001-10-12_18:00:00;8.7;78947 +2001-10-12_19:00:00;8.4;78947 +2001-10-12_20:00:00;8;78947 +2001-10-12_21:00:00;7.8;78947 +2001-10-12_22:00:00;7.3;78947 +2001-10-12_23:00:00;6.8;78947 +2001-10-13_00:00:00;6.4;78947 +2001-10-13_01:00:00;6.1;78947 +2001-10-13_02:00:00;5.5;78947 +2001-10-13_03:00:00;4.6;78947 +2001-10-13_04:00:00;3.5;78947 +2001-10-13_05:00:00;2.7;78947 +2001-10-13_06:00:00;2.2;78947 +2001-10-13_07:00:00;2;78947 +2001-10-13_09:00:00;6.4;78947 +2001-10-13_10:00:00;9.4;78947 +2001-10-13_11:00:00;10.5;78947 +2001-10-13_12:00:00;10.3;78947 +2001-10-13_13:00:00;10.4;78947 +2001-10-13_14:00:00;10.3;78947 +2001-10-13_15:00:00;9.9;78947 +2001-10-13_16:00:00;8.7;78947 +2001-10-13_17:00:00;6.7;78947 +2001-10-13_18:00:00;5.2;78947 +2001-10-13_19:00:00;4.6;78947 +2001-10-13_20:00:00;4.7;78947 +2001-10-13_21:00:00;4.7;78947 +2001-10-13_22:00:00;4.7;78947 +2001-10-13_23:00:00;5.4;78947 +2001-10-14_00:00:00;6.1;78947 +2001-10-14_01:00:00;6.7;78947 +2001-10-14_02:00:00;7.6;78947 +2001-10-14_03:00:00;8.1;78947 +2001-10-14_04:00:00;8.3;78947 +2001-10-14_06:00:00;9.5;78947 +2001-10-14_07:00:00;9.6;78947 +2001-10-14_09:00:00;9.9;78947 +2001-10-14_10:00:00;9.9;78947 +2001-10-14_11:00:00;10.3;78947 +2001-10-14_12:00:00;11.2;78947 +2001-10-14_13:00:00;11.1;78947 +2001-10-14_14:00:00;11.2;78947 +2001-10-14_15:00:00;11.3;78947 +2001-10-14_16:00:00;11.1;78947 +2001-10-14_17:00:00;10.4;78947 +2001-10-14_18:00:00;10.2;78947 +2001-10-14_19:00:00;10.3;78947 +2001-10-14_20:00:00;9.8;78947 +2001-10-14_21:00:00;9.6;78947 +2001-10-14_22:00:00;9.6;78947 +2001-10-14_23:00:00;9.8;78947 +2001-10-15_00:00:00;9.9;78947 +2001-10-15_01:00:00;10.3;78947 +2001-10-15_02:00:00;10.4;78947 +2001-10-15_03:00:00;10.5;78947 +2001-10-15_04:00:00;10.6;78947 +2001-10-15_05:00:00;10.8;78947 +2001-10-15_06:00:00;9.7;78947 +2001-10-15_07:00:00;9.9;78947 +2001-10-15_08:00:00;10.4;78947 +2001-10-15_09:00:00;10.6;78947 +2001-10-15_10:00:00;11;78947 +2001-10-15_11:00:00;11.1;78947 +2001-10-15_12:00:00;12.1;78947 +2001-10-15_13:00:00;11.1;78947 +2001-10-15_14:00:00;11.3;78947 +2001-10-15_15:00:00;11;78947 +2001-10-15_16:00:00;10.5;78947 +2001-10-15_17:00:00;10.3;78947 +2001-10-15_18:00:00;10.4;78947 +2001-10-15_19:00:00;10.1;78947 +2001-10-15_20:00:00;10.2;78947 +2001-10-15_21:00:00;10.1;78947 +2001-10-15_22:00:00;9.9;78947 +2001-10-15_23:00:00;10.1;78947 +2001-10-16_00:00:00;9.8;78947 +2001-10-16_01:00:00;9.8;78947 +2001-10-16_02:00:00;9.9;78947 +2001-10-16_03:00:00;9.9;78947 +2001-10-16_04:00:00;10.1;78947 +2001-10-16_05:00:00;10.3;78947 +2001-10-16_06:00:00;10.4;78947 +2001-10-16_07:00:00;10.4;78947 +2001-10-16_08:00:00;10.6;78947 +2001-10-16_09:00:00;11;78947 +2001-10-16_10:00:00;11.5;78947 +2001-10-16_11:00:00;11.7;78947 +2001-10-16_12:00:00;11.6;78947 +2001-10-16_13:00:00;11.5;78947 +2001-10-16_14:00:00;11.3;78947 +2001-10-16_15:00:00;11.1;78947 +2001-10-16_16:00:00;11.1;78947 +2001-10-16_17:00:00;11.1;78947 +2001-10-16_18:00:00;10.8;78947 +2001-10-16_19:00:00;11;78947 +2001-10-16_20:00:00;11.2;78947 +2001-10-16_21:00:00;11.3;78947 +2001-10-16_22:00:00;11.4;78947 +2001-10-16_23:00:00;11.4;78947 +2001-10-17_00:00:00;10.8;78947 +2001-10-17_01:00:00;10.5;78947 +2001-10-17_02:00:00;10.9;78947 +2001-10-17_03:00:00;11.1;78947 +2001-10-17_04:00:00;11.1;78947 +2001-10-17_05:00:00;10.7;78947 +2001-10-17_06:00:00;10.7;78947 +2001-10-17_07:00:00;10.5;78947 +2001-10-17_09:00:00;11.8;78947 +2001-10-17_10:00:00;11.5;78947 +2001-10-17_11:00:00;11.5;78947 +2001-10-17_12:00:00;11.4;78947 +2001-10-17_13:00:00;11.3;78947 +2001-10-17_14:00:00;11.1;78947 +2001-10-17_15:00:00;11;78947 +2001-10-17_16:00:00;10.8;78947 +2001-10-17_17:00:00;10.8;78947 +2001-10-17_18:00:00;10.5;78947 +2001-10-17_19:00:00;10.6;78947 +2001-10-17_20:00:00;10.7;78947 +2001-10-17_21:00:00;10.5;78947 +2001-10-17_22:00:00;10.6;78947 +2001-10-17_23:00:00;11.1;78947 +2001-10-18_00:00:00;8.5;78947 +2001-10-18_01:00:00;6.8;78947 +2001-10-18_02:00:00;7.5;78947 +2001-10-18_03:00:00;8.7;78947 +2001-10-18_04:00:00;8.6;78947 +2001-10-18_05:00:00;8.7;78947 +2001-10-18_06:00:00;8.5;78947 +2001-10-18_07:00:00;8.5;78947 +2001-10-18_09:00:00;9.5;78947 +2001-10-18_10:00:00;9.5;78947 +2001-10-18_11:00:00;9.9;78947 +2001-10-18_12:00:00;9.7;78947 +2001-10-18_13:00:00;9.1;78947 +2001-10-18_14:00:00;9.3;78947 +2001-10-18_15:00:00;9;78947 +2001-10-18_16:00:00;8.5;78947 +2001-10-18_17:00:00;8.3;78947 +2001-10-18_18:00:00;8.2;78947 +2001-10-18_19:00:00;8.3;78947 +2001-10-18_20:00:00;8.1;78947 +2001-10-18_21:00:00;7.6;78947 +2001-10-18_22:00:00;6.2;78947 +2001-10-18_23:00:00;6.7;78947 +2001-10-19_00:00:00;7.4;78947 +2001-10-19_01:00:00;7.1;78947 +2001-10-19_02:00:00;7.8;78947 +2001-10-19_03:00:00;8;78947 +2001-10-19_04:00:00;7.6;78947 +2001-10-19_05:00:00;6.3;78947 +2001-10-19_06:00:00;7.2;78947 +2001-10-19_07:00:00;8;78947 +2001-10-19_09:00:00;7.6;78947 +2001-10-19_10:00:00;7.6;78947 +2001-10-19_11:00:00;7.6;78947 +2001-10-19_12:00:00;6.9;78947 +2001-10-19_13:00:00;6.8;78947 +2001-10-19_14:00:00;6.6;78947 +2001-10-19_15:00:00;6.4;78947 +2001-10-19_16:00:00;6.3;78947 +2001-10-19_17:00:00;6.2;78947 +2001-10-19_18:00:00;6.9;78947 +2001-10-19_19:00:00;6.6;78947 +2001-10-19_20:00:00;6.4;78947 +2001-10-19_21:00:00;6;78947 +2001-10-19_22:00:00;5.9;78947 +2001-10-19_23:00:00;5.7;78947 +2001-10-20_00:00:00;6.1;78947 +2001-10-20_01:00:00;6.2;78947 +2001-10-20_02:00:00;7.1;78947 +2001-10-20_03:00:00;6.4;78947 +2001-10-20_04:00:00;6.3;78947 +2001-10-20_05:00:00;6.6;78947 +2001-10-20_06:00:00;7;78947 +2001-10-20_07:00:00;6.1;78947 +2001-10-20_09:00:00;5.6;78947 +2001-10-20_10:00:00;6.1;78947 +2001-10-20_11:00:00;6.6;78947 +2001-10-20_12:00:00;6.8;78947 +2001-10-20_13:00:00;6.4;78947 +2001-10-20_14:00:00;6.7;78947 +2001-10-20_15:00:00;6.2;78947 +2001-10-20_16:00:00;5.7;78947 +2001-10-20_17:00:00;5.2;78947 +2001-10-20_18:00:00;4;78947 +2001-10-20_19:00:00;3.1;78947 +2001-10-20_20:00:00;3.1;78947 +2001-10-20_21:00:00;3.1;78947 +2001-10-20_22:00:00;2.1;78947 +2001-10-20_23:00:00;2;78947 +2001-10-21_00:00:00;2.2;78947 +2001-10-21_01:00:00;3.7;78947 +2001-10-21_02:00:00;5.1;78947 +2001-10-21_03:00:00;6;78947 +2001-10-21_04:00:00;6.4;78947 +2001-10-21_05:00:00;6.7;78947 +2001-10-21_06:00:00;6.9;78947 +2001-10-21_07:00:00;7.3;78947 diff --git a/migrations/utils/email.go b/migrations/utils/email.go new file mode 100644 index 00000000..17ba4885 --- /dev/null +++ b/migrations/utils/email.go @@ -0,0 +1,60 @@ +package utils + +import ( + "encoding/base64" + "fmt" + "log/slog" + "net/smtp" + "os" + "runtime/debug" + "strings" +) + +func sendEmail(subject, body string, to []string) { + // server and from/to + host := "aspmx.l.google.com" + port := "25" + from := "oda-noreply@met.no" + + // add stuff to headers and make the message body + header := make(map[string]string) + header["From"] = from + header["To"] = strings.Join(to, ",") + header["Subject"] = subject + header["MIME-Version"] = "1.0" + header["Content-Type"] = "text/plain; charset=\"utf-8\"" + header["Content-Transfer-Encoding"] = "base64" + message := "" + for k, v := range header { + message += fmt.Sprintf("%s: %s\r\n", k, v) + } + + body = body + "\n\n" + fmt.Sprintf("Ran with the following command:\n%s", strings.Join(os.Args, " ")) + message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) + + // send the email + err := smtp.SendMail(host+":"+port, nil, from, to, []byte(message)) + if err != nil { + slog.Error(err.Error()) + return + } + slog.Info("Email sent successfully!") +} + +// TODO: modify this to be more flexible +// send an email and resume the panic +func SendEmailOnPanic(function string, recipients []string) { + if r := recover(); r != nil { + if recipients != nil { + body := "KDVH importer was unable to finish successfully, and the error was not handled." + + " This email is sent from a recover function triggered in " + + function + + ".\n\nError message:" + + fmt.Sprint(r) + + "\n\nStack trace:\n\n" + + string(debug.Stack()) + sendEmail("LARD – KDVH importer panicked", body, recipients) + } + panic(r) + } +} diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go new file mode 100644 index 00000000..d8c70859 --- /dev/null +++ b/migrations/utils/utils.go @@ -0,0 +1,62 @@ +package utils + +import ( + "fmt" + "log" + "log/slog" + "os" + "slices" + + "github.com/schollz/progressbar/v3" +) + +func NewBar(size int, description string) *progressbar.ProgressBar { + return progressbar.NewOptions(size, + progressbar.OptionOnCompletion(func() { fmt.Println() }), + progressbar.OptionSetDescription(description), + progressbar.OptionShowCount(), + progressbar.OptionSetPredictTime(false), + progressbar.OptionShowElapsedTimeOnFinish(), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "=", + SaucerHead: ">", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + ) +} + +// Filters elements of a slice by comparing them to the elements of a reference slice. +// formatMsg is an optional format string with a single format argument that can be used +// to add context on why the element may be missing from the reference slice +func FilterSlice[T comparable](slice, reference []T, formatMsg string) []T { + if slice == nil { + return reference + } + + if formatMsg == "" { + formatMsg = "User input '%s' not present in reference, skipping" + } + + // I hate this so much + out := slice[:0] + for _, s := range slice { + if !slices.Contains(reference, s) { + slog.Warn(fmt.Sprintf(formatMsg, s)) + continue + } + out = append(out, s) + } + return out +} + +func SetLogFile(tableName, procedure string) { + filename := fmt.Sprintf("%s_%s_log.txt", tableName, procedure) + fh, err := os.Create(filename) + if err != nil { + slog.Error(fmt.Sprintf("Could not create log '%s': %s", filename, err)) + return + } + log.SetOutput(fh) +} From 3344e4af291dbbcdbf26e3f4fba01bc55cd15a76 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 8 Nov 2024 18:53:16 +0100 Subject: [PATCH 02/36] Switch to justfile --- integration_tests/Makefile | 38 ------------------------------------- integration_tests/README.md | 15 +++++++-------- justfile | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 46 deletions(-) delete mode 100644 integration_tests/Makefile create mode 100644 justfile diff --git a/integration_tests/Makefile b/integration_tests/Makefile deleted file mode 100644 index 363d8a4a..00000000 --- a/integration_tests/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -# Basically the same as defining all targets .PHONY -MAKEFLAGS += --always-make - -unit_tests: - cargo build --workspace --tests - cargo test --no-fail-fast --workspace --exclude lard_tests -- --nocapture - -test_all: --test_all clean ---test_all: setup - cargo test --workspace --no-fail-fast -- --nocapture --test-threads=1 - -end_to_end: --end_to_end clean ---end_to_end: setup - cargo test --test end_to_end --no-fail-fast -- --nocapture --test-threads=1 - -debug_kafka: --kafka clean ---kafka: setup - cargo test --test end_to_end test_kafka --features debug --no-fail-fast -- --nocapture --test-threads=1 - -# With the `debug` feature, the database is not cleaned up after running the test, -# so it can be inspected with psql. Run with: -# TEST= make debug_test -debug_test: setup - cargo test "$(TEST)" --features debug --no-fail-fast -- --nocapture --test-threads=1 - -setup: - @echo "Starting Postgres docker container..." - docker run --name lard_tests -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres - @echo; sleep 5 - cargo build --workspace --tests - @echo; echo "Loading DB schema..."; echo - @cd ..; target/debug/prepare_postgres - -clean: - @echo "Stopping Postgres container..." - docker stop lard_tests - @echo "Removing Postgres container..." - docker rm lard_tests diff --git a/integration_tests/README.md b/integration_tests/README.md index fa585df7..21e482be 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -20,22 +20,21 @@ End-to-end tests are implemented inside `integration_tests\tests\end_to_end.rs`. > defined in the `mock_permit_tables` function, otherwise the ingestor will not be able to > insert the data into the database. -If you have Docker installed, you can run the tests locally using the provided -`Makefile`: +If you have Docker installed, you can run the tests locally using the provided `justfile`: ```terminal # Run all tests -make test_all +just test_all # Run unit tests only -make unit_tests +just test_unit # Run integration tests only -make end_to_end +just test_end_to_end -# Debug a specific test (does not clean up the DB if `my_test_name` is an integration test) -TEST=my_test_name make debug_test +# Debug a specific test (does not clean up the DB if `test_name` is an integration test) +just debug_test test_name # If any error occurs while running integration tests, you might need to reset the DB container manually -make clean +just clean ``` diff --git a/justfile b/justfile new file mode 100644 index 00000000..9e111a00 --- /dev/null +++ b/justfile @@ -0,0 +1,36 @@ +test_unit: + cargo build --workspace --tests + cargo test --no-fail-fast --workspace --exclude lard_tests -- --nocapture + +test_all: setup && clean + cargo test --workspace --no-fail-fast -- --nocapture --test-threads=1 + +test_end_to_end: setup && clean + cargo test --test end_to_end --no-fail-fast -- --nocapture --test-threads=1 + +test_migrations: debug_migrations && clean + +# Debug commands don't perfom the clean up action after running. +# This allows to manually check the state of the database. +debug_kafka: setup + cargo test --test end_to_end test_kafka --features debug --no-fail-fast -- --nocapture --test-threads=1 + +debug_test TEST: setup + cargo test {{TEST}} --features debug --no-fail-fast -- --nocapture --test-threads=1 + +debug_migrations: setup + @ cd migrations && go test -v ./... + +setup: + @ echo "Starting Postgres docker container..." + docker run --name lard_tests -e POSTGRES_PASSWORD=postgres -p 5432:5432 -d postgres + @ echo; sleep 5 + cargo build --workspace --tests + @ echo; echo "Loading DB schema..."; echo + @target/debug/prepare_postgres + +clean: + @ echo "Stopping Postgres container..." + docker stop lard_tests + @ echo "Removing Postgres container..." + docker rm lard_tests From 574cc0fda674153123195dd33d2aa5757227eb4b Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 11 Nov 2024 09:29:19 +0100 Subject: [PATCH 03/36] Rework interfaces and move lard.Timeseries to kdvh.LardTimeseries --- migrations/kdvh/import.go | 15 +++--- migrations/kdvh/import_functions.go | 37 +++++++------ migrations/kdvh/lard.go | 63 +++++++++++++++++++++++ migrations/kdvh/table.go | 7 ++- migrations/lard/import.go | 57 ++++++++++---------- migrations/lard/main.go | 80 ----------------------------- 6 files changed, 118 insertions(+), 141 deletions(-) create mode 100644 migrations/kdvh/lard.go delete mode 100644 migrations/lard/main.go diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go index 444ee9d9..d8761bdd 100644 --- a/migrations/kdvh/import.go +++ b/migrations/kdvh/import.go @@ -162,7 +162,7 @@ func (table *Table) importStation(station os.DirEntry, pool *pgxpool.Pool, confi return } - ts := lard.NewTimeseries(tsid, data) + ts := NewTimeseries(tsid, data) count, err := importData(ts, tsInfo, pool, config) if err != nil { return @@ -175,7 +175,7 @@ func (table *Table) importStation(station os.DirEntry, pool *pgxpool.Pool, confi return totRows, nil } -func (table *Table) parseElementFile(filename string, tsInfo *TimeseriesInfo, config *ImportConfig) ([]lard.Obs, error) { +func (table *Table) parseElementFile(filename string, tsInfo *TimeseriesInfo, config *ImportConfig) ([]LardObs, error) { file, err := os.Open(filename) if err != nil { slog.Warn(fmt.Sprintf("Could not open file '%s': %s", filename, err)) @@ -197,7 +197,7 @@ func (table *Table) parseElementFile(filename string, tsInfo *TimeseriesInfo, co return data, nil } -func importData(ts *lard.Timeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, config *ImportConfig) (count int64, err error) { +func importData(ts *LardTimeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, config *ImportConfig) (count int64, err error) { if !(config.Skip == "data") { if tsInfo.param.IsScalar { count, err = lard.InsertData(ts, pool, tsInfo.logstr) @@ -206,7 +206,7 @@ func importData(ts *lard.Timeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, return 0, err } } else { - count, err = lard.InsertNonscalarData(ts, pool, tsInfo.logstr) + count, err = lard.InsertTextData(ts, pool, tsInfo.logstr) if err != nil { slog.Error(tsInfo.logstr + "failed non-scalar data bulk insertion - " + err.Error()) return 0, err @@ -217,13 +217,12 @@ func importData(ts *lard.Timeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, } if !(config.Skip == "flags") { - if err := lard.InsertFlags(ts, pool, tsInfo.logstr); err != nil { + if err := lard.InsertFlags(ts, FLAGS_TABLE, FLAGS_COLS, pool, tsInfo.logstr); err != nil { slog.Error(tsInfo.logstr + "failed flag bulk insertion - " + err.Error()) } } return count, nil - } func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { @@ -273,7 +272,7 @@ func getTimeseriesID(tsInfo *TimeseriesInfo, pool *pgxpool.Pool) (int32, error) return tsid, nil } -func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *ImportConfig) ([]lard.Obs, error) { +func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *ImportConfig) ([]LardObs, error) { scanner := bufio.NewScanner(handle) var rowCount int @@ -286,7 +285,7 @@ func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *Imp } } - data := make([]lard.Obs, 0, rowCount) + data := make([]LardObs, 0, rowCount) for scanner.Scan() { cols := strings.Split(scanner.Text(), config.Sep) diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go index 00060670..8c783ccf 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import_functions.go @@ -2,7 +2,6 @@ package kdvh import ( "errors" - "migrate/lard" "strconv" "github.com/rickb777/period" @@ -251,7 +250,7 @@ func (obs *Obs) Useinfo() string { // and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` // Different KDVH tables need different ways to perform this conversion. -func makeDataPage(obs Obs) (lard.Obs, error) { +func makeDataPage(obs Obs) (LardObs, error) { var valPtr *float32 controlinfo := VALUE_PASSED_QC @@ -259,10 +258,10 @@ func makeDataPage(obs Obs) (lard.Obs, error) { controlinfo = VALUE_MISSING } - // NOTE: this is the only function that can return `lard.Obs` + // NOTE: this is the only function that can return `LardObs` // with non-null text data if !obs.param.IsScalar { - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: valPtr, Text: &obs.Data, @@ -277,7 +276,7 @@ func makeDataPage(obs Obs) (lard.Obs, error) { valPtr = &f32 } - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: valPtr, Useinfo: obs.Useinfo(), @@ -286,7 +285,7 @@ func makeDataPage(obs Obs) (lard.Obs, error) { } // modify obstimes to always use totime -func makeDataPageProduct(obs Obs) (lard.Obs, error) { +func makeDataPageProduct(obs Obs) (LardObs, error) { obsLard, err := makeDataPage(obs) if !obs.offset.IsZero() { if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { @@ -296,7 +295,7 @@ func makeDataPageProduct(obs Obs) (lard.Obs, error) { return obsLard, err } -func makeDataPageEdata(obs Obs) (lard.Obs, error) { +func makeDataPageEdata(obs Obs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -314,7 +313,7 @@ func makeDataPageEdata(obs Obs) (lard.Obs, error) { valPtr = &f32 } - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: valPtr, Useinfo: obs.Useinfo(), @@ -322,7 +321,7 @@ func makeDataPageEdata(obs Obs) (lard.Obs, error) { }, nil } -func makeDataPagePdata(obs Obs) (lard.Obs, error) { +func makeDataPagePdata(obs Obs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -355,7 +354,7 @@ func makeDataPagePdata(obs Obs) (lard.Obs, error) { } - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: valPtr, Useinfo: obs.Useinfo(), @@ -363,7 +362,7 @@ func makeDataPagePdata(obs Obs) (lard.Obs, error) { }, nil } -func makeDataPageNdata(obs Obs) (lard.Obs, error) { +func makeDataPageNdata(obs Obs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -396,7 +395,7 @@ func makeDataPageNdata(obs Obs) (lard.Obs, error) { valPtr = &f32 } - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: valPtr, Useinfo: obs.Useinfo(), @@ -404,7 +403,7 @@ func makeDataPageNdata(obs Obs) (lard.Obs, error) { }, nil } -func makeDataPageVdata(obs Obs) (lard.Obs, error) { +func makeDataPageVdata(obs Obs) (LardObs, error) { var useinfo, controlinfo string var valPtr *float32 @@ -426,11 +425,11 @@ func makeDataPageVdata(obs Obs) (lard.Obs, error) { // add custom offset, because OT_24 in KDVH has been treated differently than OT_24 in kvalobs offset, err := period.Parse("PT18H") // fromtime_offset -PT6H, timespan P1D if err != nil { - return lard.Obs{}, errors.New("could not parse period") + return LardObs{}, errors.New("could not parse period") } temp, ok := offset.AddTo(obs.Obstime) if !ok { - return lard.Obs{}, errors.New("could not add period") + return LardObs{}, errors.New("could not add period") } obs.Obstime = temp @@ -441,7 +440,7 @@ func makeDataPageVdata(obs Obs) (lard.Obs, error) { controlinfo = VALUE_PASSED_QC } - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: valPtr, Useinfo: useinfo, @@ -449,14 +448,14 @@ func makeDataPageVdata(obs Obs) (lard.Obs, error) { }, nil } -func makeDataPageDiurnalInterpolated(obs Obs) (lard.Obs, error) { +func makeDataPageDiurnalInterpolated(obs Obs) (LardObs, error) { val, err := strconv.ParseFloat(obs.Data, 32) if err != nil { - return lard.Obs{}, err + return LardObs{}, err } f32 := float32(val) - return lard.Obs{ + return LardObs{ Obstime: obs.Obstime, Data: &f32, Useinfo: DIURNAL_INTERPOLATED_USEINFO, diff --git a/migrations/kdvh/lard.go b/migrations/kdvh/lard.go new file mode 100644 index 00000000..0a4f0be1 --- /dev/null +++ b/migrations/kdvh/lard.go @@ -0,0 +1,63 @@ +package kdvh + +import ( + "time" + + "github.com/jackc/pgx/v5" +) + +// LardTimeseries in LARD have and ID and associated observations +type LardTimeseries struct { + id int32 + data []LardObs +} + +func NewTimeseries(id int32, data []LardObs) *LardTimeseries { + return &LardTimeseries{id, data} +} + +func (ts *LardTimeseries) Len() int { + return len(ts.data) +} + +func (ts *LardTimeseries) InsertData(i int) ([]any, error) { + return []any{ + ts.id, + ts.data[i].Obstime, + ts.data[i].Data, + }, nil +} + +func (ts *LardTimeseries) InsertText(i int) ([]any, error) { + return []any{ + ts.id, + ts.data[i].Obstime, + ts.data[i].Text, + }, nil +} + +var FLAGS_TABLE pgx.Identifier = pgx.Identifier{"flags", "kdvh"} +var FLAGS_COLS []string = []string{"timeseries", "obstime", "controlinfo", "useinfo"} + +func (ts *LardTimeseries) InsertFlags(i int) ([]any, error) { + return []any{ + ts.id, + ts.data[i].Obstime, + ts.data[i].Controlinfo, + ts.data[i].Useinfo, + }, nil +} + +// Struct containg all the fields we want to save in LARD from KDVH +type LardObs struct { + // Time of observation + Obstime time.Time + // Observation data formatted as a single precision floating point number + Data *float32 + // Observation data that cannot be represented as a float, therefore stored as a string + Text *string + // Flag encoding quality control status + Controlinfo string + // Flag encoding quality control status + Useinfo string +} diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 6a155740..a2d8b0c7 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "log/slog" - "migrate/lard" "time" "github.com/rickb777/period" @@ -35,7 +34,7 @@ type Table struct { ElemTableName string // Name of the ELEM table Path string // Directory name of where the dumped table is stored dumpFunc DumpFunction // Function used to dump the KDVH table (found in `dump_functions.go`) - convFunc ConvertFunction // Function that converts KDVH obs to LARD obs (found in `import_functions.go`) + convFunc ConvertFunction // Function that converts KDVH obs to Lardobs (found in `import_functions.go`) importUntil int // Import data only until the year specified by this field } @@ -47,7 +46,7 @@ type DumpMeta struct { flagTable string } -type ConvertFunction func(Obs) (lard.Obs, error) +type ConvertFunction func(Obs) (LardObs, error) type Obs struct { *TimeseriesInfo Obstime time.Time @@ -118,7 +117,7 @@ func (t *Table) SetDumpFunc(fn DumpFunction) *Table { return t } -// Sets the function used to convert observations from the table to LARD observations +// Sets the function used to convert observations from the table to Lardobservations func (t *Table) SetConvFunc(fn ConvertFunction) *Table { t.convFunc = fn return t diff --git a/migrations/lard/import.go b/migrations/lard/import.go index 408cc641..696107ee 100644 --- a/migrations/lard/import.go +++ b/migrations/lard/import.go @@ -9,19 +9,29 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -func InsertData(data DataInserter, pool *pgxpool.Pool, logStr string) (int64, error) { - size := data.Len() +// TODO: I'm not sure I like the interface solution +type DataInserter interface { + InsertData(i int) ([]any, error) + Len() int +} + +type TextInserter interface { + InsertText(i int) ([]any, error) + Len() int +} + +type FlagInserter interface { + InsertFlags(i int) ([]any, error) + Len() int +} + +func InsertData(ts DataInserter, pool *pgxpool.Pool, logStr string) (int64, error) { + size := ts.Len() count, err := pool.CopyFrom( context.TODO(), pgx.Identifier{"public", "data"}, []string{"timeseries", "obstime", "obsvalue"}, - pgx.CopyFromSlice(size, func(i int) ([]any, error) { - return []any{ - data.ID(), - data.Obstime(i), - data.Data(i), - }, nil - }), + pgx.CopyFromSlice(size, ts.InsertData), ) if err != nil { return count, err @@ -36,19 +46,13 @@ func InsertData(data DataInserter, pool *pgxpool.Pool, logStr string) (int64, er return count, nil } -func InsertNonscalarData(data TextInserter, pool *pgxpool.Pool, logStr string) (int64, error) { - size := data.Len() +func InsertTextData(ts TextInserter, pool *pgxpool.Pool, logStr string) (int64, error) { + size := ts.Len() count, err := pool.CopyFrom( context.TODO(), pgx.Identifier{"public", "nonscalar_data"}, []string{"timeseries", "obstime", "obsvalue"}, - pgx.CopyFromSlice(size, func(i int) ([]any, error) { - return []any{ - data.ID(), - data.Obstime(i), - data.Text(i), - }, nil - }), + pgx.CopyFromSlice(size, ts.InsertText), ) if err != nil { return count, err @@ -63,20 +67,13 @@ func InsertNonscalarData(data TextInserter, pool *pgxpool.Pool, logStr string) ( return count, nil } -func InsertFlags(data FlagInserter, pool *pgxpool.Pool, logStr string) error { - size := data.Len() +func InsertFlags(ts FlagInserter, table pgx.Identifier, columns []string, pool *pgxpool.Pool, logStr string) error { + size := ts.Len() count, err := pool.CopyFrom( context.TODO(), - pgx.Identifier{"flags", "kdvh"}, - []string{"timeseries", "obstime", "controlinfo", "useinfo"}, - pgx.CopyFromSlice(size, func(i int) ([]any, error) { - return []any{ - data.ID(), - data.Obstime(i), - data.Controlinfo(i), - data.Useinfo(i), - }, nil - }), + table, + columns, + pgx.CopyFromSlice(size, ts.InsertFlags), ) if err != nil { return err diff --git a/migrations/lard/main.go b/migrations/lard/main.go deleted file mode 100644 index d245f5de..00000000 --- a/migrations/lard/main.go +++ /dev/null @@ -1,80 +0,0 @@ -package lard - -import "time" - -// Timeseries in LARD have and ID and associated observations -type Timeseries struct { - id int32 - data []Obs -} - -func NewTimeseries(id int32, data []Obs) *Timeseries { - return &Timeseries{id, data} -} - -func (ts *Timeseries) Len() int { - return len(ts.data) -} - -func (ts *Timeseries) ID() int32 { - return ts.id -} - -func (ts *Timeseries) Obstime(i int) time.Time { - return ts.data[i].Obstime -} - -func (ts *Timeseries) Text(i int) string { - return *ts.data[i].Text -} - -func (ts *Timeseries) Data(i int) float32 { - return *ts.data[i].Data -} - -func (ts *Timeseries) Controlinfo(i int) string { - return ts.data[i].Controlinfo -} - -func (ts *Timeseries) Useinfo(i int) string { - return ts.data[i].Useinfo -} - -// Struct containg all the fields we want to save in LARD -type Obs struct { - // Time of observation - Obstime time.Time - // Observation data formatted as a single precision floating point number - Data *float32 - // Observation data that cannot be represented as a float, therefore stored as a string - Text *string - // Flag encoding quality control status - Controlinfo string - // Flag encoding quality control status - Useinfo string -} - -// TODO: I'm not sure I like the interface solution -type DataInserter interface { - Obstime(i int) time.Time - Data(i int) float32 - ID() int32 - Len() int -} - -type TextInserter interface { - Obstime(i int) time.Time - Text(i int) string - ID() int32 - Len() int -} - -// TODO: This maybe needs different implementation for each system -// i.e. insert to different tables and different columns -type FlagInserter interface { - ID() int32 - Obstime(i int) time.Time - Controlinfo(i int) string - Useinfo(i int) string - Len() int -} From 00f649d935ae8e3e0353905f9cc68e46a8eaafd4 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 11 Nov 2024 09:42:49 +0100 Subject: [PATCH 04/36] Remove useless field initialization --- migrations/kdvh/import_functions.go | 1 - 1 file changed, 1 deletion(-) diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go index 8c783ccf..f475aa07 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import_functions.go @@ -263,7 +263,6 @@ func makeDataPage(obs Obs) (LardObs, error) { if !obs.param.IsScalar { return LardObs{ Obstime: obs.Obstime, - Data: valPtr, Text: &obs.Data, Useinfo: obs.Useinfo(), Controlinfo: controlinfo, From bf647fe532a8b5845bb17aef1cca4e41a6798ce7 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 11 Nov 2024 12:59:36 +0100 Subject: [PATCH 05/36] Keep elapsed time for progressbar --- migrations/utils/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go index d8c70859..ce0191e4 100644 --- a/migrations/utils/utils.go +++ b/migrations/utils/utils.go @@ -16,6 +16,7 @@ func NewBar(size int, description string) *progressbar.ProgressBar { progressbar.OptionSetDescription(description), progressbar.OptionShowCount(), progressbar.OptionSetPredictTime(false), + progressbar.OptionSetElapsedTime(true), progressbar.OptionShowElapsedTimeOnFinish(), progressbar.OptionSetTheme(progressbar.Theme{ Saucer: "=", From 5ccf60d6b9216f4fc9f8a69e4bdd82abe44dc54c Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 11 Nov 2024 15:36:31 +0100 Subject: [PATCH 06/36] Check for overwrite at the element file level --- migrations/kdvh/dump.go | 6 +-- migrations/kdvh/dump_functions.go | 82 +++++++++++++++++++++++-------- migrations/kdvh/table.go | 1 + 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index 7affa258..41dc7af9 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -62,12 +62,9 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) table.Path = filepath.Join(config.BaseDir, table.Path) - if _, err := os.ReadDir(table.Path); err == nil && !config.Overwrite { - slog.Info(fmt.Sprint("Skipping data dump of ", table.TableName, " because dumped folder already exists")) - return - } utils.SetLogFile(table.TableName, "dump") + elements, err := table.getElements(conn, config) if err != nil { return @@ -113,6 +110,7 @@ func (table *Table) dumpElement(element string, conn *sql.DB, config *DumpConfig station: station, dataTable: table.TableName, flagTable: table.FlagTableName, + overwrite: config.Overwrite, }, conn, ) diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go index 8afe0946..b7db673b 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump_functions.go @@ -14,6 +14,17 @@ import ( "time" ) +func fileExists(filename string, overwrite bool) error { + if _, err := os.Stat(filename); err == nil && !overwrite { + return errors.New( + fmt.Sprintf( + "Skipping dump of '%s' because dumped file already exists and the --overwrite flag was not provided", + filename, + )) + } + return nil +} + // Fetch min and max year from table, needed for tables that are dumped by year func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, error) { var beginStr, endStr string @@ -54,21 +65,27 @@ func dumpByYearDataOnly(path string, meta DumpMeta, conn *sql.DB) error { ) for year := begin; year < end; year++ { + yearPath := filepath.Join(path, fmt.Sprint(year)) + if err := os.MkdirAll(yearPath, os.ModePerm); err != nil { + slog.Error(err.Error()) + continue + } + + filename := filepath.Join(yearPath, meta.element+".csv") + if err := fileExists(filename, meta.overwrite); err != nil { + slog.Warn(err.Error()) + continue + } + rows, err := conn.Query(query, meta.station, year) if err != nil { slog.Error(fmt.Sprint("Could not query KDVH: ", err)) - return err - } - - path := filepath.Join(path, fmt.Sprint(year)) - if err := os.MkdirAll(path, os.ModePerm); err != nil { - slog.Error(err.Error()) continue } - if err := dumpToFile(path, meta.element, rows); err != nil { + if err := dumpToFile(filename, rows); err != nil { slog.Error(err.Error()) - return err + continue } } @@ -107,21 +124,27 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { ) for year := begin; year < end; year++ { - rows, err := conn.Query(query, meta.station, year) - if err != nil { - slog.Error(fmt.Sprint("Could not query KDVH: ", err)) - return err - } - yearPath := filepath.Join(path, fmt.Sprint(year)) if err := os.MkdirAll(path, os.ModePerm); err != nil { slog.Error(err.Error()) continue } - if err := dumpToFile(yearPath, meta.element, rows); err != nil { + filename := filepath.Join(yearPath, meta.element+".csv") + if err := fileExists(filename, meta.overwrite); err != nil { + slog.Warn(err.Error()) + continue + } + + rows, err := conn.Query(query, meta.station, year) + if err != nil { + slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + continue + } + + if err := dumpToFile(filename, rows); err != nil { slog.Error(err.Error()) - return err + continue } } @@ -129,6 +152,12 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { } func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { + filename := filepath.Join(path, meta.element+".csv") + if err := fileExists(filename, meta.overwrite); err != nil { + slog.Warn(err.Error()) + return err + } + query := fmt.Sprintf( `SELECT dato AS time, %s[1]s AS data, '' AS flag FROM T_HOMOGEN_MONTH WHERE %s[1]s IS NOT NULL AND stnr = $1 AND season BETWEEN 1 AND 12`, @@ -142,7 +171,7 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { return err } - if err := dumpToFile(path, meta.element, rows); err != nil { + if err := dumpToFile(filename, rows); err != nil { slog.Error(err.Error()) return err } @@ -151,6 +180,12 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { } func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { + filename := filepath.Join(path, meta.element+".csv") + if err := fileExists(filename, meta.overwrite); err != nil { + slog.Warn(err.Error()) + return err + } + query := fmt.Sprintf( `SELECT dato AS time, %[1]s AS data, '' AS flag FROM %[2]s WHERE %[1]s IS NOT NULL AND stnr = $1`, @@ -164,7 +199,7 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { return err } - if err := dumpToFile(path, meta.element, rows); err != nil { + if err := dumpToFile(filename, rows); err != nil { slog.Error(err.Error()) return err } @@ -173,6 +208,12 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { } func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { + filename := filepath.Join(path, meta.element+".csv") + if err := fileExists(filename, meta.overwrite); err != nil { + slog.Warn(err.Error()) + return err + } + query := fmt.Sprintf( `SELECT dato AS time, @@ -194,7 +235,7 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { return err } - if err := dumpToFile(path, meta.element, rows); err != nil { + if err := dumpToFile(path, rows); err != nil { slog.Error(err.Error()) return err } @@ -202,8 +243,7 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { return nil } -func dumpToFile(path, element string, rows *sql.Rows) error { - filename := filepath.Join(path, element+".csv") +func dumpToFile(filename string, rows *sql.Rows) error { file, err := os.Create(filename) if err != nil { return err diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index a2d8b0c7..5c688077 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -44,6 +44,7 @@ type DumpMeta struct { station string dataTable string flagTable string + overwrite bool } type ConvertFunction func(Obs) (LardObs, error) From 928c0bae9e46fd6837a42da55589e1f395e0697f Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 12 Nov 2024 11:47:45 +0100 Subject: [PATCH 07/36] Save available elements and stations in separate files --- migrations/kdvh/dump.go | 9 +++++++++ migrations/utils/utils.go | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index 41dc7af9..a70e5f49 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -62,6 +62,10 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) table.Path = filepath.Join(config.BaseDir, table.Path) + if err := os.MkdirAll(table.Path, os.ModePerm); err != nil { + slog.Error(err.Error()) + return + } utils.SetLogFile(table.TableName, "dump") @@ -129,6 +133,11 @@ func (table *Table) getElements(conn *sql.DB, config *DumpConfig) ([]string, err return nil, err } + filename := filepath.Join(table.Path, "elements.txt") + if err := utils.SaveToFile(elements, filename); err != nil { + slog.Warn("Could not save element list to " + filename) + } + elements = utils.FilterSlice(config.Elements, elements, "") return elements, nil } diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go index ce0191e4..59fe3b98 100644 --- a/migrations/utils/utils.go +++ b/migrations/utils/utils.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "slices" + "strings" "github.com/schollz/progressbar/v3" ) @@ -52,6 +53,16 @@ func FilterSlice[T comparable](slice, reference []T, formatMsg string) []T { return out } +// Saves a slice to a file +func SaveToFile(values []string, filename string) error { + file, err := os.Create(filename) + if err != nil { + return err + } + file.WriteString(strings.Join(values, "\n")) + return file.Close() +} + func SetLogFile(tableName, procedure string) { filename := fmt.Sprintf("%s_%s_log.txt", tableName, procedure) fh, err := os.Create(filename) From a40d91c5ddfd593da41da2b78a64a1c31257f72f Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 12 Nov 2024 13:36:29 +0100 Subject: [PATCH 08/36] Add comment about T_HOMOGEN_MONTH --- migrations/kdvh/dump.go | 3 ++- migrations/kdvh/dump_functions.go | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index a70e5f49..973a5c3a 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -151,7 +151,8 @@ var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { slog.Info(fmt.Sprintf("Fetching elements for %s...", table.TableName)) - // TODO: not sure why we only dump these two for this table + // NOTE: T_HOMOGEN_MONTH is a special case, refer to `dumpHomogenMonth` in + // `dump_functions.go` for more information if table.TableName == "T_HOMOGEN_MONTH" { return []string{"rr", "tam"}, nil } diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go index b7db673b..0210f8dd 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump_functions.go @@ -151,6 +151,12 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { return nil } +// T_HOMOGEN_MONTH contains seasonal and annual data, plus other derivative +// data combining both of these. We decided to dump only the monthly data (season BETWEEN 1 AND 12) for +// - TAM (mean hourly temperature), and +// - RR (hourly precipitations, note that in Stinfosys this parameter is 'RR_1') +// +// We calculate the other data on the fly (outside this program) if needed. func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { From 603ee092468ac11bb1b9c6044c7e61037604ec1a Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 12 Nov 2024 14:28:31 +0100 Subject: [PATCH 09/36] Upgrade logging inside dump functions --- migrations/kdvh/dump.go | 27 ++++++++-------- migrations/kdvh/dump_functions.go | 51 ++++++++++++++++--------------- migrations/kdvh/table.go | 1 + 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index 973a5c3a..e36391e0 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -107,21 +107,18 @@ func (table *Table) dumpElement(element string, conn *sql.DB, config *DumpConfig return } - err := table.dumpFunc( - path, - DumpMeta{ - element: element, - station: station, - dataTable: table.TableName, - flagTable: table.FlagTableName, - overwrite: config.Overwrite, - }, - conn, - ) - - // NOTE: Non-nil errors are logged inside each DumpFunc - if err == nil { - slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) + meta := DumpMeta{ + element: element, + station: station, + dataTable: table.TableName, + flagTable: table.FlagTableName, + overwrite: config.Overwrite, + logStr: fmt.Sprintf("%s - %s - %s: ", table.TableName, station, element), + } + + if err := table.dumpFunc(path, meta, conn); err == nil { + // NOTE: Non-nil errors are logged inside each DumpFunc + slog.Info(meta.logStr + "dumped successfully") } } } diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go index 0210f8dd..1f00d933 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump_functions.go @@ -14,6 +14,9 @@ import ( "time" ) +// Error returned if no observations are found for a (station, element) pair +var EMPTY_QUERY_ERR error = errors.New("The query did not return any rows") + func fileExists(filename string, overwrite bool) error { if _, err := os.Stat(filename); err == nil && !overwrite { return errors.New( @@ -31,20 +34,17 @@ func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, erro query := fmt.Sprintf("SELECT min(to_char(dato, 'yyyy')), max(to_char(dato, 'yyyy')) FROM %s WHERE stnr = $1", tableName) if err := conn.QueryRow(query, station).Scan(&beginStr, &endStr); err != nil { - slog.Error(fmt.Sprint("Could not query row: ", err)) - return 0, 0, err + return 0, 0, fmt.Errorf("Could not query row: %v", err) } begin, err := strconv.ParseInt(beginStr, 10, 64) if err != nil { - slog.Error(fmt.Sprintf("Could not parse year '%s': %s", beginStr, err)) - return 0, 0, err + return 0, 0, fmt.Errorf("Could not parse year '%s': %s", beginStr, err) } end, err := strconv.ParseInt(endStr, 10, 64) if err != nil { - slog.Error(fmt.Sprintf("Could not parse year '%s': %s", endStr, err)) - return 0, 0, err + return 0, 0, fmt.Errorf("Could not parse year '%s': %s", endStr, err) } return begin, end, nil @@ -53,6 +53,7 @@ func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, erro func dumpByYearDataOnly(path string, meta DumpMeta, conn *sql.DB) error { begin, end, err := fetchYearRange(meta.dataTable, meta.station, conn) if err != nil { + slog.Error(meta.logStr + err.Error()) return err } @@ -67,24 +68,24 @@ func dumpByYearDataOnly(path string, meta DumpMeta, conn *sql.DB) error { for year := begin; year < end; year++ { yearPath := filepath.Join(path, fmt.Sprint(year)) if err := os.MkdirAll(yearPath, os.ModePerm); err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) continue } filename := filepath.Join(yearPath, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { - slog.Warn(err.Error()) + slog.Warn(meta.logStr + err.Error()) continue } rows, err := conn.Query(query, meta.station, year) if err != nil { - slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + slog.Error(meta.logStr + fmt.Sprint("Could not query KDVH: ", err)) continue } if err := dumpToFile(filename, rows); err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) continue } } @@ -126,24 +127,24 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { for year := begin; year < end; year++ { yearPath := filepath.Join(path, fmt.Sprint(year)) if err := os.MkdirAll(path, os.ModePerm); err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) continue } filename := filepath.Join(yearPath, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { - slog.Warn(err.Error()) + slog.Warn(meta.logStr + err.Error()) continue } rows, err := conn.Query(query, meta.station, year) if err != nil { - slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + slog.Error(meta.logStr + fmt.Sprint("Could not query KDVH: ", err)) continue } if err := dumpToFile(filename, rows); err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) continue } } @@ -160,7 +161,7 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { - slog.Warn(err.Error()) + slog.Warn(meta.logStr + err.Error()) return err } @@ -173,12 +174,12 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { rows, err := conn.Query(query, meta.station) if err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) return err } if err := dumpToFile(filename, rows); err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) return err } @@ -188,7 +189,7 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { - slog.Warn(err.Error()) + slog.Warn(meta.logStr + err.Error()) return err } @@ -201,12 +202,12 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { rows, err := conn.Query(query, meta.station) if err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) return err } if err := dumpToFile(filename, rows); err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) return err } @@ -216,7 +217,7 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { - slog.Warn(err.Error()) + slog.Warn(meta.logStr + err.Error()) return err } @@ -237,12 +238,14 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { rows, err := conn.Query(query, meta.station) if err != nil { - slog.Error(err.Error()) + slog.Error(meta.logStr + err.Error()) return err } - if err := dumpToFile(path, rows); err != nil { - slog.Error(err.Error()) + if err := dumpToFile(filename, rows); err != nil { + if !errors.Is(err, EMPTY_QUERY_ERR) { + slog.Error(meta.logStr + err.Error()) + } return err } diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 5c688077..86b6b397 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -45,6 +45,7 @@ type DumpMeta struct { dataTable string flagTable string overwrite bool + logStr string } type ConvertFunction func(Obs) (LardObs, error) From 81fe231fecfa02f7597347a06cc2f37fc7f453d8 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Tue, 12 Nov 2024 16:04:04 +0100 Subject: [PATCH 10/36] Rework some comments based on Ketil's feedback --- migrations/kdvh/main.go | 49 ++++++++++++++++++---------------------- migrations/kdvh/table.go | 2 -- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go index af104227..ae47a599 100644 --- a/migrations/kdvh/main.go +++ b/migrations/kdvh/main.go @@ -11,43 +11,38 @@ type Cmd struct { var KDVH map[string]*Table = map[string]*Table{ // Section 1: tables that need to be migrated entirely // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? - "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(makeDataPageEdata).SetImport(3000), - // NOTE(1): there is a T_METARFLAG, but it's empty - // NOTE(2): already dumped, but with wrong format? - "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), // already dumped + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(makeDataPageEdata).SetImport(3000), + "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImport(2006), - "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), // already dumped - "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), // already dumped - "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPagePdata).SetImport(2006), // already dumped - "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageNdata).SetImport(2006), // already dumped - "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageVdata).SetImport(2006), // already dumped - "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), + "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPagePdata).SetImport(2006), + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageNdata).SetImport(2006), + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageVdata).SetImport(2006), + "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), // Section 3: tables that should only be dumped "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), + "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), + "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), + "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), + "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), + "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), - // TODO: T_AVINOR, T_PROJDATA have a bunch of parameters that are not in Stinfosys? - // But it shouldn't be a problem if the goal is to only dump them? - "T_AVINOR": NewTable("T_AVINOR", "T_AVINOR_FLAG", "T_ELEM_OBS"), - // TODO: T_PROJFLAG is not in the proxy! And T_PROJDATA is not readable from the proxy - // "T_PROJDATA": newTable("T_PROJDATA", "T_PROJFLAG", "T_ELEM_PROJ"), - "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), // already dumped - "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), // already dumped - "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), // already dumped - "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), // already dumped - "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), // already dumped - - // Section 4: other special cases - // TODO: do we need to import these? + // Section 4: special cases, namely digitized historical data "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(makeDataPageProduct).SetImport(1957), - "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(makeDataPageProduct), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(makeDataPageProduct).SetImport(2006), "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(makeDataPageProduct), "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(makeDataPageProduct), - // TODO: these two are the only tables seemingly missing from the KDVH proxy - // {TableName: "T_DIURNAL_INTERPOLATED", DataFunction: makeDataPageDiurnalInterpolated, ImportUntil: 3000}, - // {TableName: "T_MONTH_INTERPOLATED", DataFunction: makeDataPageDiurnalInterpolated, ImportUntil: 3000}, + // Tables missing in the KDVH proxy: + // 1. these exist in a separate database + // - "T_AVINOR" + // - "T_PROJDATA" + // 2. these are not in active use and don't need to be imported in LARD + // - "T_DIURNAL_INTERPOLATED" + // - "T_MONTH_INTERPOLATED" } diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 86b6b397..097ae763 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -22,8 +22,6 @@ import ( // each with different observations, where the column name is the 'elem_code' // (e.g. for air temperature, 'ta'). // -// TODO: are the timestamps UTC? Otherwise we probably need to convert them during import -// // The ELEM tables have the following schema: // | stnr | elem_code | fdato | tdato | table_name | flag_table_name | audit_dato From f4bf5677a31c3bc9fb29239474e428392265541b Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 13 Nov 2024 12:06:45 +0100 Subject: [PATCH 11/36] Update README.md --- migrations/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/migrations/README.md b/migrations/README.md index b32cfa74..192738fb 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -13,15 +13,17 @@ Go package used to dump tables from old databases (KDVH, Kvalobs) and import the 1. Dump tables from KDVH ```terminal - ./migrate kdvh dump --help + ./migrate kdvh dump ``` -1. Import dumps into LARD +1. Import dumps into LARD (you can use the `--help` flag to see all available options) ```terminal - ./migrate kdvh import --help + ./migrate kdvh import ``` +For each command, you can use the `--help` flag to see all available options. + ## Other notes Insightful talk on migrations: [here](https://www.youtube.com/watch?v=wqXqJfQMrqI&t=280s) From c0a6cd13d53b0ee4394d4e967dff4798a37b7155 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 13 Nov 2024 12:09:44 +0100 Subject: [PATCH 12/36] Remove redundant comment --- migrations/kdvh/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go index ae47a599..2cb4b33a 100644 --- a/migrations/kdvh/main.go +++ b/migrations/kdvh/main.go @@ -7,7 +7,6 @@ type Cmd struct { List ListConfig `command:"list" description:"List available KDVH tables"` } -// The KDVH database simply contains a map of "table name" to `Table` var KDVH map[string]*Table = map[string]*Table{ // Section 1: tables that need to be migrated entirely // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? From 65ea8b60bebb883375b723186d12c085e4e0ab5c Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 13 Nov 2024 14:59:15 +0100 Subject: [PATCH 13/36] Exit if separator is not valid --- migrations/kdvh/import.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go index d8761bdd..0153ce41 100644 --- a/migrations/kdvh/import.go +++ b/migrations/kdvh/import.go @@ -43,8 +43,8 @@ type ImportConfig struct { func (config *ImportConfig) setup() { if len(config.Sep) > 1 { - slog.Warn("'--sep' only accepts single-byte characters. Defaulting to ','") - config.Sep = "," + fmt.Printf("Error: '--sep' only accepts single-byte characters. Got %s", config.Sep) + os.Exit(1) } if config.TablesCmd != "" { config.Tables = strings.Split(config.TablesCmd, ",") From 255e2f8a8a3e8dc4187cefe4debab32b77679e38 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 13 Nov 2024 15:06:52 +0100 Subject: [PATCH 14/36] Move Obs to KdvhObs --- migrations/kdvh/import.go | 2 +- migrations/kdvh/import_functions.go | 18 +++++++++--------- migrations/kdvh/import_test.go | 22 +++++++++++----------- migrations/kdvh/table.go | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go index 0153ce41..b593fac6 100644 --- a/migrations/kdvh/import.go +++ b/migrations/kdvh/import.go @@ -305,7 +305,7 @@ func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *Imp break } - temp, err := table.convFunc(Obs{meta, obsTime, cols[1], cols[2]}) + temp, err := table.convFunc(KdvhObs{meta, obsTime, cols[1], cols[2]}) if err != nil { return nil, err } diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go index f475aa07..0ab06a70 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import_functions.go @@ -231,7 +231,7 @@ const ( DIURNAL_INTERPOLATED_USEINFO = "48925" + DELAY_DEFAULT // Specific to T_DIURNAL_INTERPOLATED ) -func (obs *Obs) flagsAreValid() bool { +func (obs *KdvhObs) flagsAreValid() bool { if len(obs.Flags) != 5 { return false } @@ -239,7 +239,7 @@ func (obs *Obs) flagsAreValid() bool { return err == nil } -func (obs *Obs) Useinfo() string { +func (obs *KdvhObs) Useinfo() string { if !obs.flagsAreValid() { return INVALID_FLAGS } @@ -250,7 +250,7 @@ func (obs *Obs) Useinfo() string { // and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` // Different KDVH tables need different ways to perform this conversion. -func makeDataPage(obs Obs) (LardObs, error) { +func makeDataPage(obs KdvhObs) (LardObs, error) { var valPtr *float32 controlinfo := VALUE_PASSED_QC @@ -284,7 +284,7 @@ func makeDataPage(obs Obs) (LardObs, error) { } // modify obstimes to always use totime -func makeDataPageProduct(obs Obs) (LardObs, error) { +func makeDataPageProduct(obs KdvhObs) (LardObs, error) { obsLard, err := makeDataPage(obs) if !obs.offset.IsZero() { if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { @@ -294,7 +294,7 @@ func makeDataPageProduct(obs Obs) (LardObs, error) { return obsLard, err } -func makeDataPageEdata(obs Obs) (LardObs, error) { +func makeDataPageEdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -320,7 +320,7 @@ func makeDataPageEdata(obs Obs) (LardObs, error) { }, nil } -func makeDataPagePdata(obs Obs) (LardObs, error) { +func makeDataPagePdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -361,7 +361,7 @@ func makeDataPagePdata(obs Obs) (LardObs, error) { }, nil } -func makeDataPageNdata(obs Obs) (LardObs, error) { +func makeDataPageNdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -402,7 +402,7 @@ func makeDataPageNdata(obs Obs) (LardObs, error) { }, nil } -func makeDataPageVdata(obs Obs) (LardObs, error) { +func makeDataPageVdata(obs KdvhObs) (LardObs, error) { var useinfo, controlinfo string var valPtr *float32 @@ -447,7 +447,7 @@ func makeDataPageVdata(obs Obs) (LardObs, error) { }, nil } -func makeDataPageDiurnalInterpolated(obs Obs) (LardObs, error) { +func makeDataPageDiurnalInterpolated(obs KdvhObs) (LardObs, error) { val, err := strconv.ParseFloat(obs.Data, 32) if err != nil { return LardObs{}, err diff --git a/migrations/kdvh/import_test.go b/migrations/kdvh/import_test.go index ba6e574b..1ffdf836 100644 --- a/migrations/kdvh/import_test.go +++ b/migrations/kdvh/import_test.go @@ -4,21 +4,21 @@ import "testing" func TestFlagsAreValid(t *testing.T) { type testCase struct { - input Obs + input KdvhObs expected bool } cases := []testCase{ - {Obs{Flags: "12309"}, true}, - {Obs{Flags: "984.3"}, false}, - {Obs{Flags: ".1111"}, false}, - {Obs{Flags: "1234."}, false}, - {Obs{Flags: "12.2.4"}, false}, - {Obs{Flags: "12.343"}, false}, - {Obs{Flags: ""}, false}, - {Obs{Flags: "asdas"}, false}, - {Obs{Flags: "12a3a"}, false}, - {Obs{Flags: "1sdfl"}, false}, + {KdvhObs{Flags: "12309"}, true}, + {KdvhObs{Flags: "984.3"}, false}, + {KdvhObs{Flags: ".1111"}, false}, + {KdvhObs{Flags: "1234."}, false}, + {KdvhObs{Flags: "12.2.4"}, false}, + {KdvhObs{Flags: "12.343"}, false}, + {KdvhObs{Flags: ""}, false}, + {KdvhObs{Flags: "asdas"}, false}, + {KdvhObs{Flags: "12a3a"}, false}, + {KdvhObs{Flags: "1sdfl"}, false}, } for _, c := range cases { diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 097ae763..9f6b8881 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -46,8 +46,8 @@ type DumpMeta struct { logStr string } -type ConvertFunction func(Obs) (LardObs, error) -type Obs struct { +type ConvertFunction func(KdvhObs) (LardObs, error) +type KdvhObs struct { *TimeseriesInfo Obstime time.Time Data string From c98411f4aab3a6ab9d661323f88df754b2754237 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 13 Nov 2024 15:27:02 +0100 Subject: [PATCH 15/36] Update importUntil doc string --- migrations/kdvh/table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 9f6b8881..2eee55a5 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -33,7 +33,7 @@ type Table struct { Path string // Directory name of where the dumped table is stored dumpFunc DumpFunction // Function used to dump the KDVH table (found in `dump_functions.go`) convFunc ConvertFunction // Function that converts KDVH obs to Lardobs (found in `import_functions.go`) - importUntil int // Import data only until the year specified by this field + importUntil int // Import data only until the year specified by this field. If this field is not explicitly set, table import is skipped. } type DumpFunction func(path string, meta DumpMeta, conn *sql.DB) error From 2b5ebf9f1c34acbf9377a78b497f28a22c63f54e Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 13 Nov 2024 18:24:01 +0100 Subject: [PATCH 16/36] Use fork that is able to correctly parse comma separated strings into slices --- migrations/go.mod | 2 ++ migrations/go.sum | 4 ++-- migrations/kdvh/dump.go | 30 ++++++------------------------ migrations/kdvh/import.go | 31 +++++++++---------------------- 4 files changed, 19 insertions(+), 48 deletions(-) diff --git a/migrations/go.mod b/migrations/go.mod index d7233f7b..5deeb931 100644 --- a/migrations/go.mod +++ b/migrations/go.mod @@ -25,3 +25,5 @@ require ( golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.16.0 // indirect ) + +replace github.com/jessevdk/go-flags => github.com/Lun4m/go-flags v0.0.0-20241113125827-68757125e949 diff --git a/migrations/go.sum b/migrations/go.sum index 22ed9b76..2d0a2d0b 100644 --- a/migrations/go.sum +++ b/migrations/go.sum @@ -1,3 +1,5 @@ +github.com/Lun4m/go-flags v0.0.0-20241113125827-68757125e949 h1:7xyEGIr1X5alOjBjlNTDF+aRBcRIo60YX5sdlziLE5w= +github.com/Lun4m/go-flags v0.0.0-20241113125827-68757125e949/go.mod h1:42/L0FDbP0qe91I+81tBqjU3uoz1tn1GDMZAhcCE2PE= github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -35,8 +37,6 @@ github.com/rickb777/plural v1.4.2 h1:Kl/syFGLFZ5EbuV8c9SVud8s5HI2HpCCtOMw2U1kS+A github.com/rickb777/plural v1.4.2/go.mod h1:kdmXUpmKBJTS0FtG/TFumd//VBWsNTD7zOw7x4umxNw= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= -github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ= github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index e36391e0..db069c71 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -15,33 +15,15 @@ import ( ) type DumpConfig struct { - BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` - TablesCmd string `short:"t" long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` - StationsCmd string `short:"s" long:"stnr" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` - ElementsCmd string `short:"e" long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` - Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` - Email []string `long:"email" description:"Optional email address used to notify if the program crashed"` - - Tables []string - Stations []string - Elements []string -} - -func (config *DumpConfig) setup() { - if config.TablesCmd != "" { - config.Tables = strings.Split(config.TablesCmd, ",") - } - if config.StationsCmd != "" { - config.Stations = strings.Split(config.StationsCmd, ",") - } - if config.ElementsCmd != "" { - config.Elements = strings.Split(config.ElementsCmd, ",") - } + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + Tables []string `short:"t" delimiter:"," long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + Stations []string `short:"s" delimiter:"," long:"stnr" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + Elements []string `short:"e" delimiter:"," long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` + Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` } func (config *DumpConfig) Execute([]string) error { - config.setup() - conn, err := sql.Open("pgx", os.Getenv("KDVH_PROXY_CONN")) if err != nil { slog.Error(err.Error()) diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go index b593fac6..b323d651 100644 --- a/migrations/kdvh/import.go +++ b/migrations/kdvh/import.go @@ -22,19 +22,15 @@ import ( ) type ImportConfig struct { - Verbose bool `short:"v" description:"Increase verbosity level"` - BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` - TablesCmd string `short:"t" long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` - StationsCmd string `short:"s" long:"station" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` - ElementsCmd string `short:"e" long:"elemcode" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` - Sep string `long:"sep" default:"," description:"Separator character in the dumped files. Needs to be quoted"` - HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` - Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` - Email []string `long:"email" description:"Optional email address used to notify if the program crashed"` - - Tables []string - Stations []string - Elements []string + Verbose bool `short:"v" description:"Increase verbosity level"` + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + Tables []string `short:"t" long:"table" delimiter:"," default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + Stations []string `short:"s" long:"station" delimiter:"," default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + Elements []string `short:"e" long:"elemcode" delimiter:"," default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Sep string `long:"sep" default:"," description:"Separator character in the dumped files. Needs to be quoted"` + HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` + Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` + Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` OffsetMap map[StinfoKey]period.Period // Map of offsets used to correct (?) KDVH times for specific parameters StinfoMap map[StinfoKey]StinfoParam // Map of metadata used to query timeseries ID in LARD @@ -46,15 +42,6 @@ func (config *ImportConfig) setup() { fmt.Printf("Error: '--sep' only accepts single-byte characters. Got %s", config.Sep) os.Exit(1) } - if config.TablesCmd != "" { - config.Tables = strings.Split(config.TablesCmd, ",") - } - if config.StationsCmd != "" { - config.Stations = strings.Split(config.StationsCmd, ",") - } - if config.ElementsCmd != "" { - config.Elements = strings.Split(config.ElementsCmd, ",") - } config.CacheMetadata() } From 954774b261ac061867713c34f858af29a4d6094d Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 09:54:25 +0100 Subject: [PATCH 17/36] Use ELEM table to query unique stations in DATA table --- migrations/kdvh/dump.go | 113 ++++++++++-------------------- migrations/kdvh/dump_functions.go | 31 ++++---- migrations/utils/utils.go | 2 +- 3 files changed, 56 insertions(+), 90 deletions(-) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index db069c71..ee65c83e 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -14,6 +14,9 @@ import ( "migrate/utils" ) +// List of columns that we do not need to select when extracting the element codes from a KDVH table +var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} + type DumpConfig struct { BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` Tables []string `short:"t" delimiter:"," long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` @@ -56,32 +59,14 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { return } - bar := utils.NewBar(len(elements), table.TableName) - - // TODO: should be safe to spawn goroutines/waitgroup here with connection pool? - bar.RenderBlank() - for _, element := range elements { - table.dumpElement(element, conn, config) - bar.Add(1) - } -} - -// TODO: maybe we don't do this? Or can we use pgdump/copy? -// The problem is that there are no indices on the tables, that's why the queries are super slow -// Dumping the whole table might be a lot faster (for T_MDATA it's ~10 times faster!), -// but it might be more difficult to recover if something goes wrong? -// => -// copyQuery := fmt.SPrintf("\\copy (select * from t_mdata) TO '%s/%s.csv' WITH CSV HEADER", config.BaseDir, table.TableName) -// cmd := exec.Command("psql", CONN_STRING, "-c", copyQuery) -// cmd.Stderr = &bytes.Buffer{} -// err = cmd.Run() -func (table *Table) dumpElement(element string, conn *sql.DB, config *DumpConfig) { - stations, err := table.getStationsWithElement(element, conn, config) + stations, err := table.getStations(conn, config) if err != nil { - slog.Error(fmt.Sprintf("Could not fetch stations for table %s: %v", table.TableName, err)) return } + bar := utils.NewBar(len(stations), table.TableName) + + bar.RenderBlank() for _, station := range stations { path := filepath.Join(table.Path, string(station)) if err := os.MkdirAll(path, os.ModePerm); err != nil { @@ -89,19 +74,26 @@ func (table *Table) dumpElement(element string, conn *sql.DB, config *DumpConfig return } - meta := DumpMeta{ - element: element, - station: station, - dataTable: table.TableName, - flagTable: table.FlagTableName, - overwrite: config.Overwrite, - logStr: fmt.Sprintf("%s - %s - %s: ", table.TableName, station, element), - } + for _, element := range elements { + err := table.dumpFunc( + path, + DumpMeta{ + element: element, + station: station, + dataTable: table.TableName, + flagTable: table.FlagTableName, + overwrite: config.Overwrite, + }, + conn, + ) - if err := table.dumpFunc(path, meta, conn); err == nil { // NOTE: Non-nil errors are logged inside each DumpFunc - slog.Info(meta.logStr + "dumped successfully") + if err == nil { + slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) + } } + + bar.Add(1) } } @@ -121,9 +113,6 @@ func (table *Table) getElements(conn *sql.DB, config *DumpConfig) ([]string, err return elements, nil } -// List of columns that we do not need to select when extracting the element codes from a KDVH table -var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} - // Fetch column names for a given table // We skip the columns defined in INVALID_COLUMNS and all columns that contain the 'kopi' string // TODO: should we dump these invalid/kopi elements even if we are not importing them? @@ -163,57 +152,29 @@ func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { } // Fetches station numbers and filters them based on user input -func (table *Table) getStationsWithElement(element string, conn *sql.DB, config *DumpConfig) ([]string, error) { - stations, err := table.fetchStationsWithElement(element, conn) +func (table *Table) getStations(conn *sql.DB, config *DumpConfig) ([]string, error) { + stations, err := table.fetchStnrFromElemTable(conn) if err != nil { return nil, err } - msg := fmt.Sprintf("Element '%s'", element) + "not available for station '%s'" - stations = utils.FilterSlice(config.Stations, stations, msg) + stations = utils.FilterSlice(config.Stations, stations, "") return stations, nil } -// Fetches the unique station numbers in the table for a given element (and when that element is not null) -// NOTE: splitting by element does make it a bit better, because we avoid quering for stations that have no data or flag for that element? -func (table *Table) fetchStationsWithElement(element string, conn *sql.DB) (stations []string, err error) { - slog.Info(fmt.Sprintf("Fetching station numbers for %s (this can take a while)...", element)) - - query := fmt.Sprintf( - `SELECT DISTINCT stnr FROM %s WHERE %s IS NOT NULL`, - table.TableName, - element, - ) +// This function uses the ELEM table to fetch the station numbers +func (table *Table) fetchStnrFromElemTable(conn *sql.DB) (stations []string, err error) { + slog.Info(fmt.Sprint("Fetching station numbers...")) - rows, err := conn.Query(query) - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var stnr string - if err := rows.Scan(&stnr); err != nil { - return nil, err - } - stations = append(stations, stnr) + var rows *sql.Rows + if table.ElemTableName == "T_ELEM_OBS" { + query := `SELECT DISTINCT stnr FROM t_elem_obs WHERE table_name = $1` + rows, err = conn.Query(query, table.TableName) + } else { + query := fmt.Sprintf("SELECT DISTINCT stnr FROM %s", strings.ToLower(table.ElemTableName)) + rows, err = conn.Query(query) } - return stations, rows.Err() -} - -// Fetches all unique station numbers in the table -// FIXME: the DISTINCT query can be extremely slow -// NOTE: decided to use fetchStationsWithElement instead -func (table *Table) fetchStationNumbers(conn *sql.DB) (stations []string, err error) { - slog.Info(fmt.Sprint("Fetching station numbers (this can take a while)...")) - - query := fmt.Sprintf( - `SELECT DISTINCT stnr FROM %s`, - table.TableName, - ) - - rows, err := conn.Query(query) if err != nil { return nil, err } diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go index 1f00d933..56f8d524 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump_functions.go @@ -14,9 +14,19 @@ import ( "time" ) +// Format string for date field in CSV files +const TIMEFORMAT string = "2006-01-02_15:04:05" + // Error returned if no observations are found for a (station, element) pair var EMPTY_QUERY_ERR error = errors.New("The query did not return any rows") +// Struct representing a single record in the output CSV file +type Record struct { + time time.Time + data sql.NullString + flag sql.NullString +} + func fileExists(filename string, overwrite bool) error { if _, err := os.Stat(filename); err == nil && !overwrite { return errors.New( @@ -28,7 +38,7 @@ func fileExists(filename string, overwrite bool) error { return nil } -// Fetch min and max year from table, needed for tables that are dumped by year +// Helper function for dumpByYear functinos Fetch min and max year from table, needed for tables that are dumped by year func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, error) { var beginStr, endStr string query := fmt.Sprintf("SELECT min(to_char(dato, 'yyyy')), max(to_char(dato, 'yyyy')) FROM %s WHERE stnr = $1", tableName) @@ -253,12 +263,17 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { } func dumpToFile(filename string, rows *sql.Rows) error { - file, err := os.Create(filename) + lines, err := sortRows(rows) if err != nil { return err } - lines, err := sortRows(rows) + // Return if query was empty + if len(lines) == 0 { + return EMPTY_QUERY_ERR + } + + file, err := os.Create(filename) if err != nil { return err } @@ -270,13 +285,6 @@ func dumpToFile(filename string, rows *sql.Rows) error { return err } -// Struct representing a single record in the output CSV file -type Record struct { - time time.Time - data sql.NullString - flag sql.NullString -} - // Scans the rows and collects them in a slice of chronologically sorted lines func sortRows(rows *sql.Rows) ([]Record, error) { defer rows.Close() @@ -299,9 +307,6 @@ func sortRows(rows *sql.Rows) ([]Record, error) { return records, rows.Err() } -// Format string for date field in CSV files -const TIMEFORMAT string = "2006-01-02_15:04:05" - // Writes queried (time | data | flag) columns to CSV func writeElementFile(lines []Record, file io.Writer) error { // Write number of lines as header diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go index 59fe3b98..ea333364 100644 --- a/migrations/utils/utils.go +++ b/migrations/utils/utils.go @@ -38,7 +38,7 @@ func FilterSlice[T comparable](slice, reference []T, formatMsg string) []T { } if formatMsg == "" { - formatMsg = "User input '%s' not present in reference, skipping" + formatMsg = "Value '%s' not present in reference slice, skipping" } // I hate this so much From c231e9056c9d9ac48bf6e5b1aeffacef6a189b02 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 11:05:29 +0100 Subject: [PATCH 18/36] Add go ci --- .github/workflows/ci.yml | 18 ++++++++++++++++++ migrations/kdvh_test.go | 30 +++++++++++++----------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75dbfe4d..ba95b16a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,3 +64,21 @@ jobs: - name: Run unit and integration tests run: cargo test --no-fail-fast -- --nocapture --test-threads=1 + + - name: Setup Go 1.22.x + uses: actions/setup-go@v5 + with: + go-version: '1.22.x' + cache-dependency-path: migrations/go.sum + + - name: Install Go dependencies + working-directory: migrations + run: go get + + - name: Build migration package + working-directory: migrations + run: go build -v ./... + + - name: Run Go tests + working-directory: migrations + run: go test -v ./... diff --git a/migrations/kdvh_test.go b/migrations/kdvh_test.go index 3f564e8c..b7fbc2d0 100644 --- a/migrations/kdvh_test.go +++ b/migrations/kdvh_test.go @@ -4,10 +4,9 @@ import ( "context" "fmt" "testing" + "time" "github.com/jackc/pgx/v5/pgxpool" - "github.com/joho/godotenv" - // "github.com/rickb777/period" "migrate/kdvh" ) @@ -15,18 +14,24 @@ import ( const LARD_STRING string = "host=localhost user=postgres dbname=postgres password=postgres" func mockConfig(t *ImportTest) *kdvh.ImportConfig { - config := kdvh.ImportConfig{ + return &kdvh.ImportConfig{ Tables: []string{t.table}, Stations: []string{fmt.Sprint(t.station)}, Elements: []string{t.elem}, BaseDir: "./tests", HasHeader: true, Sep: ";", + StinfoMap: map[kdvh.StinfoKey]kdvh.StinfoParam{ + {ElemCode: t.elem, TableName: t.table}: { + TypeID: 501, + ParamID: 212, + Hlevel: nil, + Sensor: 0, + Fromtime: time.Date(2001, 7, 1, 9, 0, 0, 0, time.UTC), + IsScalar: true, + }, + }, } - - config.CacheMetadata() - return &config - } type ImportTest struct { @@ -37,15 +42,6 @@ type ImportTest struct { } func TestImportKDVH(t *testing.T) { - err := godotenv.Load() - if err != nil { - fmt.Println(err) - return - } - - // TODO: could also define a smaller version just for tests - db := kdvh.KDVH - pool, err := pgxpool.New(context.TODO(), LARD_STRING) if err != nil { t.Log("Could not connect to Lard:", err) @@ -58,7 +54,7 @@ func TestImportKDVH(t *testing.T) { for _, c := range testCases { config := mockConfig(&c) - table, ok := db[c.table] + table, ok := kdvh.KDVH[c.table] if !ok { t.Fatal("Table does not exist in database") } From f30c704794da2a3e9842e5ecfc539b4dd641695d Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 13:31:13 +0100 Subject: [PATCH 19/36] Add some docs about implemented `DumpFunction`s --- migrations/kdvh/dump_functions.go | 51 +++++-------------------------- migrations/kdvh/table.go | 1 + 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go index 56f8d524..adc1930f 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump_functions.go @@ -60,49 +60,8 @@ func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, erro return begin, end, nil } -func dumpByYearDataOnly(path string, meta DumpMeta, conn *sql.DB) error { - begin, end, err := fetchYearRange(meta.dataTable, meta.station, conn) - if err != nil { - slog.Error(meta.logStr + err.Error()) - return err - } - - query := fmt.Sprintf( - `SELECT dato AS time, %[1]s AS data, '' AS flag FROM %[2]s - WHERE %[1]s IS NOT NULL - AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2`, - meta.element, - meta.dataTable, - ) - - for year := begin; year < end; year++ { - yearPath := filepath.Join(path, fmt.Sprint(year)) - if err := os.MkdirAll(yearPath, os.ModePerm); err != nil { - slog.Error(meta.logStr + err.Error()) - continue - } - - filename := filepath.Join(yearPath, meta.element+".csv") - if err := fileExists(filename, meta.overwrite); err != nil { - slog.Warn(meta.logStr + err.Error()) - continue - } - - rows, err := conn.Query(query, meta.station, year) - if err != nil { - slog.Error(meta.logStr + fmt.Sprint("Could not query KDVH: ", err)) - continue - } - - if err := dumpToFile(filename, rows); err != nil { - slog.Error(meta.logStr + err.Error()) - continue - } - } - - return nil -} - +// This function is used when the table contains large amount of data +// (T_SECOND, T_MINUTE, T_10MINUTE) func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { dataBegin, dataEnd, err := fetchYearRange(meta.dataTable, meta.station, conn) if err != nil { @@ -196,6 +155,8 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { return nil } +// This function is used to dump tables that don't have a FLAG table, +// (T_METARDATA, T_HOMOGEN_DIURNAL) func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { @@ -224,6 +185,9 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { return nil } +// This is the default dump function. +// It selects both data and flag tables for a specific (station, element) pair, +// and then performs a full outer join on the two subqueries func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { @@ -262,6 +226,7 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { return nil } +// Dumps queried rows to file func dumpToFile(filename string, rows *sql.Rows) error { lines, err := sortRows(rows) if err != nil { diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 2eee55a5..bb2ffc89 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -36,6 +36,7 @@ type Table struct { importUntil int // Import data only until the year specified by this field. If this field is not explicitly set, table import is skipped. } +// Implementation of these functions can be found in `dump_functions.go` type DumpFunction func(path string, meta DumpMeta, conn *sql.DB) error type DumpMeta struct { element string From 681b9d13a24f7127366b7d5e1d5c5d550b0339de Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 13:44:23 +0100 Subject: [PATCH 20/36] Change name of ConvertFunction and add some docs --- migrations/kdvh/import_functions.go | 24 ++++++++++++++---------- migrations/kdvh/main.go | 16 ++++++++-------- migrations/kdvh/table.go | 3 ++- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go index 0ab06a70..cafebe79 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import_functions.go @@ -250,7 +250,8 @@ func (obs *KdvhObs) Useinfo() string { // and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` // Different KDVH tables need different ways to perform this conversion. -func makeDataPage(obs KdvhObs) (LardObs, error) { +// Default ConvertFunction +func Convert(obs KdvhObs) (LardObs, error) { var valPtr *float32 controlinfo := VALUE_PASSED_QC @@ -283,9 +284,10 @@ func makeDataPage(obs KdvhObs) (LardObs, error) { }, nil } -// modify obstimes to always use totime -func makeDataPageProduct(obs KdvhObs) (LardObs, error) { - obsLard, err := makeDataPage(obs) +// This function modifies obstimes to always use totime +// This is needed because KDVH used incorrect and incosistent timestamps +func ConvertProduct(obs KdvhObs) (LardObs, error) { + obsLard, err := Convert(obs) if !obs.offset.IsZero() { if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { obsLard.Obstime = temp @@ -294,7 +296,7 @@ func makeDataPageProduct(obs KdvhObs) (LardObs, error) { return obsLard, err } -func makeDataPageEdata(obs KdvhObs) (LardObs, error) { +func ConvertEdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -320,7 +322,7 @@ func makeDataPageEdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPagePdata(obs KdvhObs) (LardObs, error) { +func ConvertPdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -361,7 +363,7 @@ func makeDataPagePdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPageNdata(obs KdvhObs) (LardObs, error) { +func ConvertNdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -402,7 +404,7 @@ func makeDataPageNdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPageVdata(obs KdvhObs) (LardObs, error) { +func ConvertVdata(obs KdvhObs) (LardObs, error) { var useinfo, controlinfo string var valPtr *float32 @@ -447,13 +449,15 @@ func makeDataPageVdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPageDiurnalInterpolated(obs KdvhObs) (LardObs, error) { +// Specific conversionfunction for diurnal interpolated, +// with hardcoded useinfo and controlinfo +func ConvertDiurnalInterpolated(obs KdvhObs) (LardObs, error) { val, err := strconv.ParseFloat(obs.Data, 32) if err != nil { return LardObs{}, err } - f32 := float32(val) + f32 := float32(val) return LardObs{ Obstime: obs.Obstime, Data: &f32, diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go index 2cb4b33a..ed41472a 100644 --- a/migrations/kdvh/main.go +++ b/migrations/kdvh/main.go @@ -10,16 +10,16 @@ type Cmd struct { var KDVH map[string]*Table = map[string]*Table{ // Section 1: tables that need to be migrated entirely // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? - "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(makeDataPageEdata).SetImport(3000), + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(ConvertEdata).SetImport(3000), "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImport(2006), "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), - "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPagePdata).SetImport(2006), - "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageNdata).SetImport(2006), - "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageVdata).SetImport(2006), + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(ConvertPdata).SetImport(2006), + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(ConvertNdata).SetImport(2006), + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(ConvertVdata).SetImport(2006), "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), // Section 3: tables that should only be dumped @@ -32,10 +32,10 @@ var KDVH map[string]*Table = map[string]*Table{ "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), // Section 4: special cases, namely digitized historical data - "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(makeDataPageProduct).SetImport(1957), - "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(makeDataPageProduct).SetImport(2006), - "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(makeDataPageProduct), - "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(makeDataPageProduct), + "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(ConvertProduct).SetImport(1957), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(ConvertProduct).SetImport(2006), + "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(ConvertProduct), + "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(ConvertProduct), // Tables missing in the KDVH proxy: // 1. these exist in a separate database diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index bb2ffc89..1d0147c9 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -47,6 +47,7 @@ type DumpMeta struct { logStr string } +// Implementation of these functions can be found in `import_functions.go` type ConvertFunction func(KdvhObs) (LardObs, error) type KdvhObs struct { *TimeseriesInfo @@ -100,7 +101,7 @@ func NewTable(data, flag, elem string) *Table { ElemTableName: elem, Path: data + "_combined", // NOTE: '_combined' kept for backward compatibility with original scripts dumpFunc: dumpDataAndFlags, - convFunc: makeDataPage, + convFunc: Convert, } } From 85a65faada95f2d38526f5043265e9050c0d5784 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 13:49:56 +0100 Subject: [PATCH 21/36] Concurrent dump with semaphore for limiting total number of connections --- migrations/kdvh/dump.go | 52 +++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index ee65c83e..13d77b8e 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "strings" + "sync" _ "github.com/jackc/pgx/v5/stdlib" @@ -24,6 +25,7 @@ type DumpConfig struct { Elements []string `short:"e" delimiter:"," long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` + MaxConn int `long:"conns" default:"10" description:"Max number of concurrent connections allowed"` } func (config *DumpConfig) Execute([]string) error { @@ -64,8 +66,10 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { return } - bar := utils.NewBar(len(stations), table.TableName) + // Used to limit connections to the database + semaphore := make(chan struct{}, config.MaxConn) + bar := utils.NewBar(len(stations), table.TableName) bar.RenderBlank() for _, station := range stations { path := filepath.Join(table.Path, string(station)) @@ -74,25 +78,37 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { return } + var wg sync.WaitGroup for _, element := range elements { - err := table.dumpFunc( - path, - DumpMeta{ - element: element, - station: station, - dataTable: table.TableName, - flagTable: table.FlagTableName, - overwrite: config.Overwrite, - }, - conn, - ) - - // NOTE: Non-nil errors are logged inside each DumpFunc - if err == nil { - slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) - } + // This blocks if the channel is full + semaphore <- struct{}{} + + wg.Add(1) + go func() { + defer wg.Done() + + err := table.dumpFunc( + path, + DumpMeta{ + element: element, + station: station, + dataTable: table.TableName, + flagTable: table.FlagTableName, + overwrite: config.Overwrite, + }, + conn, + ) + + // NOTE: Non-nil errors are logged inside each DumpFunc + if err == nil { + slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) + } + + // Release semaphore + <-semaphore + }() } - + wg.Wait() bar.Add(1) } } From a4150c07134086eb9c01599227e6290f2afdc605 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 16:51:14 +0100 Subject: [PATCH 22/36] Create three separate slices inside parseData that can be fed directly to pgx.CopyFromRows --- db/flags.sql | 8 +- migrations/kdvh/import.go | 128 +++++++--------- migrations/kdvh/import_functions.go | 221 ++++++++++++++++++---------- migrations/kdvh/import_test.go | 22 +-- migrations/kdvh/lard.go | 63 -------- migrations/kdvh/table.go | 11 +- migrations/lard/import.go | 38 ++--- migrations/lard/main.go | 52 +++++++ 8 files changed, 282 insertions(+), 261 deletions(-) delete mode 100644 migrations/kdvh/lard.go create mode 100644 migrations/lard/main.go diff --git a/db/flags.sql b/db/flags.sql index 785cf603..0d61629b 100644 --- a/db/flags.sql +++ b/db/flags.sql @@ -15,13 +15,15 @@ CREATE TABLE IF NOT EXISTS flags.kvdata ( CREATE INDEX IF NOT EXISTS kvdata_obtime_index ON flags.kvdata (obstime); CREATE INDEX IF NOT EXISTS kvdata_timeseries_index ON flags.kvdata USING HASH (timeseries); -CREATE TABLE IF NOT EXISTS flags.kdvh ( +CREATE TABLE IF NOT EXISTS flags.old_databases ( timeseries INT4 REFERENCES public.timeseries, obstime TIMESTAMPTZ NOT NULL, + corrected REAL NULL, controlinfo TEXT NULL, useinfo TEXT NULL, + cfailed TEXT NULL , CONSTRAINT unique_kdvh_timeseries_obstime UNIQUE (timeseries, obstime) ); -CREATE INDEX IF NOT EXISTS kdvh_obtime_index ON flags.kdvh (obstime); -CREATE INDEX IF NOT EXISTS kdvh_timeseries_index ON flags.kdvh USING HASH (timeseries); +CREATE INDEX IF NOT EXISTS kdvh_obtime_index ON flags.old_databases (obstime); +CREATE INDEX IF NOT EXISTS kdvh_timeseries_index ON flags.old_databases USING HASH (timeseries); diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go index b323d651..6833cbe9 100644 --- a/migrations/kdvh/import.go +++ b/migrations/kdvh/import.go @@ -21,6 +21,9 @@ import ( "migrate/utils" ) +// TODO: add CALL_SIGN? It's not in stinfosys? +var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} + type ImportConfig struct { Verbose bool `short:"v" description:"Increase verbosity level"` BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` @@ -67,7 +70,7 @@ func (config *ImportConfig) Execute([]string) error { } func (table *Table) Import(pool *pgxpool.Pool, config *ImportConfig) (rowsInserted int64) { - defer utils.SendEmailOnPanic("importTable", config.Email) + defer utils.SendEmailOnPanic("table.Import", config.Email) if table.importUntil == 0 { if config.Verbose { @@ -144,72 +147,54 @@ func (table *Table) importStation(station os.DirEntry, pool *pgxpool.Pool, confi } filename := filepath.Join(dir, element.Name()) - data, err := table.parseElementFile(filename, tsInfo, config) + file, err := os.Open(filename) if err != nil { + slog.Warn(fmt.Sprintf("Could not open file '%s': %s", filename, err)) return } + defer file.Close() - ts := NewTimeseries(tsid, data) - count, err := importData(ts, tsInfo, pool, config) + data, text, flag, err := table.parseData(file, tsid, tsInfo, config) if err != nil { return } - totRows += count - }() - } - wg.Wait() - - return totRows, nil -} -func (table *Table) parseElementFile(filename string, tsInfo *TimeseriesInfo, config *ImportConfig) ([]LardObs, error) { - file, err := os.Open(filename) - if err != nil { - slog.Warn(fmt.Sprintf("Could not open file '%s': %s", filename, err)) - return nil, err - } - defer file.Close() - - data, err := table.parseData(file, tsInfo, config) - if err != nil { - slog.Error(fmt.Sprintf("Could not parse data from '%s': %s", filename, err)) - return nil, err - } - - if len(data) == 0 { - slog.Info(tsInfo.logstr + "no rows to insert (all obstimes > max import time)") - return nil, err - } - - return data, nil -} + if len(data) == 0 { + slog.Info(tsInfo.logstr + "no rows to insert (all obstimes > max import time)") + return + } -func importData(ts *LardTimeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, config *ImportConfig) (count int64, err error) { - if !(config.Skip == "data") { - if tsInfo.param.IsScalar { - count, err = lard.InsertData(ts, pool, tsInfo.logstr) - if err != nil { - slog.Error(tsInfo.logstr + "failed data bulk insertion - " + err.Error()) - return 0, err + var count int64 + if !(config.Skip == "data") { + if tsInfo.param.IsScalar { + count, err = lard.InsertData(data, pool, tsInfo.logstr) + if err != nil { + slog.Error(tsInfo.logstr + "failed data bulk insertion - " + err.Error()) + return + } + } else { + count, err = lard.InsertTextData(text, pool, tsInfo.logstr) + if err != nil { + slog.Error(tsInfo.logstr + "failed non-scalar data bulk insertion - " + err.Error()) + return + } + // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data + // return count, nil + } } - } else { - count, err = lard.InsertTextData(ts, pool, tsInfo.logstr) - if err != nil { - slog.Error(tsInfo.logstr + "failed non-scalar data bulk insertion - " + err.Error()) - return 0, err + + if !(config.Skip == "flags") { + if err := lard.InsertFlags(flag, pool, tsInfo.logstr); err != nil { + slog.Error(tsInfo.logstr + "failed flag bulk insertion - " + err.Error()) + } } - // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data - // return count, nil - } - } - if !(config.Skip == "flags") { - if err := lard.InsertFlags(ts, FLAGS_TABLE, FLAGS_COLS, pool, tsInfo.logstr); err != nil { - slog.Error(tsInfo.logstr + "failed flag bulk insertion - " + err.Error()) - } + totRows += count + }() } + wg.Wait() - return count, nil + return totRows, nil } func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { @@ -229,6 +214,10 @@ func getStationNumber(station os.DirEntry, stationList []string) (int32, error) return int32(stnr), nil } +func elemcodeIsInvalid(element string) bool { + return strings.Contains(element, "KOPI") || slices.Contains(INVALID_ELEMENTS, element) +} + func getElementCode(element os.DirEntry, elementList []string) (string, error) { elemCode := strings.ToUpper(strings.TrimSuffix(element.Name(), ".csv")) @@ -259,26 +248,27 @@ func getTimeseriesID(tsInfo *TimeseriesInfo, pool *pgxpool.Pool) (int32, error) return tsid, nil } -func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *ImportConfig) ([]LardObs, error) { +func (table *Table) parseData(handle *os.File, id int32, meta *TimeseriesInfo, config *ImportConfig) ([][]any, [][]any, [][]any, error) { scanner := bufio.NewScanner(handle) var rowCount int // Try to infer row count from header if config.HasHeader { scanner.Scan() - // rowCount, _ = strconv.Atoi(scanner.Text()) - if temp, err := strconv.Atoi(scanner.Text()); err == nil { - rowCount = temp - } + rowCount, _ = strconv.Atoi(scanner.Text()) } - data := make([]LardObs, 0, rowCount) + // Prepare slices for pgx.CopyFromRows + data := make([][]any, 0, rowCount) + text := make([][]any, 0, rowCount) + flag := make([][]any, 0, rowCount) + for scanner.Scan() { cols := strings.Split(scanner.Text(), config.Sep) obsTime, err := time.Parse("2006-01-02_15:04:05", cols[0]) if err != nil { - return nil, err + return nil, nil, nil, err } // Only import data between KDVH's defined fromtime and totime @@ -292,20 +282,14 @@ func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *Imp break } - temp, err := table.convFunc(KdvhObs{meta, obsTime, cols[1], cols[2]}) + dataRow, textRow, flagRow, err := table.convFunc(KdvhObs{meta, id, obsTime, cols[1], cols[2]}) if err != nil { - return nil, err + return nil, nil, nil, err } - - data = append(data, temp) + data = append(data, dataRow.ToRow()) + text = append(text, textRow.ToRow()) + flag = append(flag, flagRow.ToRow()) } - return data, nil -} - -// TODO: add CALL_SIGN? It's not in stinfosys? -var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} - -func elemcodeIsInvalid(element string) bool { - return strings.Contains(element, "KOPI") || slices.Contains(INVALID_ELEMENTS, element) + return data, text, flag, nil } diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go index cafebe79..27962366 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import_functions.go @@ -5,6 +5,8 @@ import ( "strconv" "github.com/rickb777/period" + + "migrate/lard" ) // In kvalobs a flag is a 16 char string containg QC information about the observation: @@ -231,19 +233,24 @@ const ( DIURNAL_INTERPOLATED_USEINFO = "48925" + DELAY_DEFAULT // Specific to T_DIURNAL_INTERPOLATED ) +// Work around to return reference to consts +func addr[T any](t T) *T { + return &t +} + func (obs *KdvhObs) flagsAreValid() bool { - if len(obs.Flags) != 5 { + if len(obs.flags) != 5 { return false } - _, err := strconv.ParseInt(obs.Flags, 10, 32) + _, err := strconv.ParseInt(obs.flags, 10, 32) return err == nil } -func (obs *KdvhObs) Useinfo() string { +func (obs *KdvhObs) Useinfo() *string { if !obs.flagsAreValid() { - return INVALID_FLAGS + return addr(INVALID_FLAGS) } - return obs.Flags + DELAY_DEFAULT + return addr(obs.flags + DELAY_DEFAULT) } // The following functions try to recover the original pair of `controlinfo` @@ -251,57 +258,59 @@ func (obs *KdvhObs) Useinfo() string { // Different KDVH tables need different ways to perform this conversion. // Default ConvertFunction -func Convert(obs KdvhObs) (LardObs, error) { +// NOTE: this should be the only function that can return `lard.TextObs` with non-null text data. +func Convert(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { var valPtr *float32 controlinfo := VALUE_PASSED_QC - if obs.Data == "" { + if obs.data == "" { controlinfo = VALUE_MISSING } - // NOTE: this is the only function that can return `LardObs` - // with non-null text data - if !obs.param.IsScalar { - return LardObs{ - Obstime: obs.Obstime, - Text: &obs.Data, - Useinfo: obs.Useinfo(), - Controlinfo: controlinfo, - }, nil - } - - val, err := strconv.ParseFloat(obs.Data, 32) + val, err := strconv.ParseFloat(obs.data, 32) if err == nil { f32 := float32(val) valPtr = &f32 } - return LardObs{ - Obstime: obs.Obstime, - Data: valPtr, - Useinfo: obs.Useinfo(), - Controlinfo: controlinfo, - }, nil + return lard.DataObs{ + Id: obs.id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil } // This function modifies obstimes to always use totime // This is needed because KDVH used incorrect and incosistent timestamps -func ConvertProduct(obs KdvhObs) (LardObs, error) { - obsLard, err := Convert(obs) +func ConvertProduct(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + data, text, flag, err := Convert(obs) if !obs.offset.IsZero() { - if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { - obsLard.Obstime = temp + if temp, ok := obs.offset.AddTo(data.Obstime); ok { + data.Obstime = temp + text.Obstime = temp + flag.Obstime = temp } } - return obsLard, err + return data, text, flag, err } -func ConvertEdata(obs KdvhObs) (LardObs, error) { +func ConvertEdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { var controlinfo string var valPtr *float32 - if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { - switch obs.Flags { + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + switch obs.flags { case "70381", "70389", "90989": controlinfo = VALUE_REMOVED_BY_QC default: @@ -314,20 +323,30 @@ func ConvertEdata(obs KdvhObs) (LardObs, error) { valPtr = &f32 } - return LardObs{ - Obstime: obs.Obstime, - Data: valPtr, - Useinfo: obs.Useinfo(), - Controlinfo: controlinfo, - }, nil + return lard.DataObs{ + Id: obs.id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil } -func ConvertPdata(obs KdvhObs) (LardObs, error) { +func ConvertPdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { var controlinfo string var valPtr *float32 - if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { - switch obs.Flags { + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + switch obs.flags { case "20389", "30389", "40389", "50383", "70381", "71381": controlinfo = VALUE_REMOVED_BY_QC default: @@ -341,7 +360,7 @@ func ConvertPdata(obs KdvhObs) (LardObs, error) { f32 := float32(val) valPtr = &f32 - switch obs.Flags { + switch obs.flags { case "10319", "10329", "30319", "40319", "48929", "48999": controlinfo = VALUE_MANUALLY_INTERPOLATED case "20389", "30389", "40389", "50383", "70381", "71381", "99319": @@ -355,20 +374,30 @@ func ConvertPdata(obs KdvhObs) (LardObs, error) { } - return LardObs{ - Obstime: obs.Obstime, - Data: valPtr, - Useinfo: obs.Useinfo(), - Controlinfo: controlinfo, - }, nil + return lard.DataObs{ + Id: obs.id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil } -func ConvertNdata(obs KdvhObs) (LardObs, error) { +func ConvertNdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { var controlinfo string var valPtr *float32 - if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { - switch obs.Flags { + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + switch obs.flags { case "70389": controlinfo = VALUE_REMOVED_BY_QC default: @@ -379,7 +408,7 @@ func ConvertNdata(obs KdvhObs) (LardObs, error) { controlinfo = VALUE_MISSING } } else { - switch obs.Flags { + switch obs.flags { case "43325", "48325": controlinfo = VALUE_MANUALLY_ASSIGNED case "30319", "38929", "40315", "40319": @@ -396,27 +425,37 @@ func ConvertNdata(obs KdvhObs) (LardObs, error) { valPtr = &f32 } - return LardObs{ - Obstime: obs.Obstime, - Data: valPtr, - Useinfo: obs.Useinfo(), - Controlinfo: controlinfo, - }, nil + return lard.DataObs{ + Id: obs.id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil } -func ConvertVdata(obs KdvhObs) (LardObs, error) { +func ConvertVdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { var useinfo, controlinfo string var valPtr *float32 // set useinfo based on time - if h := obs.Obstime.Hour(); h == 0 || h == 6 || h == 12 || h == 18 { + if h := obs.obstime.Hour(); h == 0 || h == 6 || h == 12 || h == 18 { useinfo = COMPLETED_HQC } else { useinfo = INVALID_FLAGS } // set data and controlinfo - if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { controlinfo = VALUE_MISSING } else { // super special treatment clause of T_VDATA.OT_24, so it will be the same as in kvalobs @@ -426,14 +465,14 @@ func ConvertVdata(obs KdvhObs) (LardObs, error) { // add custom offset, because OT_24 in KDVH has been treated differently than OT_24 in kvalobs offset, err := period.Parse("PT18H") // fromtime_offset -PT6H, timespan P1D if err != nil { - return LardObs{}, errors.New("could not parse period") + return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, errors.New("could not parse period") } - temp, ok := offset.AddTo(obs.Obstime) + temp, ok := offset.AddTo(obs.obstime) if !ok { - return LardObs{}, errors.New("could not add period") + return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, errors.New("could not add period") } - obs.Obstime = temp + obs.obstime = temp // convert from hours to minutes f32 *= 60.0 } @@ -441,27 +480,47 @@ func ConvertVdata(obs KdvhObs) (LardObs, error) { controlinfo = VALUE_PASSED_QC } - return LardObs{ - Obstime: obs.Obstime, - Data: valPtr, - Useinfo: useinfo, - Controlinfo: controlinfo, - }, nil + return lard.DataObs{ + Id: obs.id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.id, + Obstime: obs.obstime, + Useinfo: &useinfo, + Controlinfo: &controlinfo, + }, nil } // Specific conversionfunction for diurnal interpolated, // with hardcoded useinfo and controlinfo -func ConvertDiurnalInterpolated(obs KdvhObs) (LardObs, error) { - val, err := strconv.ParseFloat(obs.Data, 32) +func ConvertDiurnalInterpolated(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + val, err := strconv.ParseFloat(obs.data, 32) if err != nil { - return LardObs{}, err + return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, err } f32 := float32(val) - return LardObs{ - Obstime: obs.Obstime, - Data: &f32, - Useinfo: DIURNAL_INTERPOLATED_USEINFO, - Controlinfo: VALUE_MANUALLY_INTERPOLATED, - }, nil + return lard.DataObs{ + Id: obs.id, + Obstime: obs.obstime, + Data: &f32, + }, + lard.TextObs{ + Id: obs.id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.id, + Obstime: obs.obstime, + Useinfo: addr(DIURNAL_INTERPOLATED_USEINFO), + Controlinfo: addr(VALUE_MANUALLY_INTERPOLATED), + }, nil } diff --git a/migrations/kdvh/import_test.go b/migrations/kdvh/import_test.go index 1ffdf836..6b7e5a28 100644 --- a/migrations/kdvh/import_test.go +++ b/migrations/kdvh/import_test.go @@ -9,20 +9,20 @@ func TestFlagsAreValid(t *testing.T) { } cases := []testCase{ - {KdvhObs{Flags: "12309"}, true}, - {KdvhObs{Flags: "984.3"}, false}, - {KdvhObs{Flags: ".1111"}, false}, - {KdvhObs{Flags: "1234."}, false}, - {KdvhObs{Flags: "12.2.4"}, false}, - {KdvhObs{Flags: "12.343"}, false}, - {KdvhObs{Flags: ""}, false}, - {KdvhObs{Flags: "asdas"}, false}, - {KdvhObs{Flags: "12a3a"}, false}, - {KdvhObs{Flags: "1sdfl"}, false}, + {KdvhObs{flags: "12309"}, true}, + {KdvhObs{flags: "984.3"}, false}, + {KdvhObs{flags: ".1111"}, false}, + {KdvhObs{flags: "1234."}, false}, + {KdvhObs{flags: "12.2.4"}, false}, + {KdvhObs{flags: "12.343"}, false}, + {KdvhObs{flags: ""}, false}, + {KdvhObs{flags: "asdas"}, false}, + {KdvhObs{flags: "12a3a"}, false}, + {KdvhObs{flags: "1sdfl"}, false}, } for _, c := range cases { - t.Log("Testing flag:", c.input.Flags) + t.Log("Testing flag:", c.input.flags) if result := c.input.flagsAreValid(); result != c.expected { t.Errorf("Got %v, wanted %v", result, c.expected) diff --git a/migrations/kdvh/lard.go b/migrations/kdvh/lard.go deleted file mode 100644 index 0a4f0be1..00000000 --- a/migrations/kdvh/lard.go +++ /dev/null @@ -1,63 +0,0 @@ -package kdvh - -import ( - "time" - - "github.com/jackc/pgx/v5" -) - -// LardTimeseries in LARD have and ID and associated observations -type LardTimeseries struct { - id int32 - data []LardObs -} - -func NewTimeseries(id int32, data []LardObs) *LardTimeseries { - return &LardTimeseries{id, data} -} - -func (ts *LardTimeseries) Len() int { - return len(ts.data) -} - -func (ts *LardTimeseries) InsertData(i int) ([]any, error) { - return []any{ - ts.id, - ts.data[i].Obstime, - ts.data[i].Data, - }, nil -} - -func (ts *LardTimeseries) InsertText(i int) ([]any, error) { - return []any{ - ts.id, - ts.data[i].Obstime, - ts.data[i].Text, - }, nil -} - -var FLAGS_TABLE pgx.Identifier = pgx.Identifier{"flags", "kdvh"} -var FLAGS_COLS []string = []string{"timeseries", "obstime", "controlinfo", "useinfo"} - -func (ts *LardTimeseries) InsertFlags(i int) ([]any, error) { - return []any{ - ts.id, - ts.data[i].Obstime, - ts.data[i].Controlinfo, - ts.data[i].Useinfo, - }, nil -} - -// Struct containg all the fields we want to save in LARD from KDVH -type LardObs struct { - // Time of observation - Obstime time.Time - // Observation data formatted as a single precision floating point number - Data *float32 - // Observation data that cannot be represented as a float, therefore stored as a string - Text *string - // Flag encoding quality control status - Controlinfo string - // Flag encoding quality control status - Useinfo string -} diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 1d0147c9..90b11bb7 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "migrate/lard" "time" "github.com/rickb777/period" @@ -48,12 +49,14 @@ type DumpMeta struct { } // Implementation of these functions can be found in `import_functions.go` -type ConvertFunction func(KdvhObs) (LardObs, error) +// It returns three structs for each of the lard tables we are inserting into +type ConvertFunction func(KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) type KdvhObs struct { *TimeseriesInfo - Obstime time.Time - Data string - Flags string + id int32 + obstime time.Time + data string + flags string } // Convenience struct that holds information for a specific timeseries diff --git a/migrations/lard/import.go b/migrations/lard/import.go index 696107ee..9729f95e 100644 --- a/migrations/lard/import.go +++ b/migrations/lard/import.go @@ -9,29 +9,13 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -// TODO: I'm not sure I like the interface solution -type DataInserter interface { - InsertData(i int) ([]any, error) - Len() int -} - -type TextInserter interface { - InsertText(i int) ([]any, error) - Len() int -} - -type FlagInserter interface { - InsertFlags(i int) ([]any, error) - Len() int -} - -func InsertData(ts DataInserter, pool *pgxpool.Pool, logStr string) (int64, error) { - size := ts.Len() +func InsertData(ts [][]any, pool *pgxpool.Pool, logStr string) (int64, error) { + size := len(ts) count, err := pool.CopyFrom( context.TODO(), pgx.Identifier{"public", "data"}, []string{"timeseries", "obstime", "obsvalue"}, - pgx.CopyFromSlice(size, ts.InsertData), + pgx.CopyFromRows(ts), ) if err != nil { return count, err @@ -46,13 +30,13 @@ func InsertData(ts DataInserter, pool *pgxpool.Pool, logStr string) (int64, erro return count, nil } -func InsertTextData(ts TextInserter, pool *pgxpool.Pool, logStr string) (int64, error) { - size := ts.Len() +func InsertTextData(ts [][]any, pool *pgxpool.Pool, logStr string) (int64, error) { + size := len(ts) count, err := pool.CopyFrom( context.TODO(), pgx.Identifier{"public", "nonscalar_data"}, []string{"timeseries", "obstime", "obsvalue"}, - pgx.CopyFromSlice(size, ts.InsertText), + pgx.CopyFromRows(ts), ) if err != nil { return count, err @@ -67,13 +51,13 @@ func InsertTextData(ts TextInserter, pool *pgxpool.Pool, logStr string) (int64, return count, nil } -func InsertFlags(ts FlagInserter, table pgx.Identifier, columns []string, pool *pgxpool.Pool, logStr string) error { - size := ts.Len() +func InsertFlags(ts [][]any, pool *pgxpool.Pool, logStr string) error { + size := len(ts) count, err := pool.CopyFrom( context.TODO(), - table, - columns, - pgx.CopyFromSlice(size, ts.InsertFlags), + pgx.Identifier{"flags", "old_databases"}, + []string{"timeseries", "obstime", "corrected", "controlinfo", "useinfo", "cfailed"}, + pgx.CopyFromRows(ts), ) if err != nil { return err diff --git a/migrations/lard/main.go b/migrations/lard/main.go new file mode 100644 index 00000000..e0787999 --- /dev/null +++ b/migrations/lard/main.go @@ -0,0 +1,52 @@ +package lard + +import "time" + +// Struct mimicking the `public.data` table +type DataObs struct { + // Timeseries ID + Id int32 + // Time of observation + Obstime time.Time + // Observation data formatted as a single precision floating point number + Data *float32 +} + +func (o *DataObs) ToRow() []any { + return []any{o.Id, o.Obstime, o.Data} +} + +// Struct mimicking the `public.nonscalar_data` table +type TextObs struct { + // Timeseries ID + Id int32 + // Time of observation + Obstime time.Time + // Observation data that cannot be represented as a float, therefore stored as a string + Text *string +} + +func (o *TextObs) ToRow() []any { + return []any{o.Id, o.Obstime, o.Text} +} + +// Struct mimicking the `flags.old_databases` table +type Flag struct { + // Timeseries ID + Id int32 + // Time of observation + Obstime time.Time + // Corrected value after QC tests + Corrected *float32 + // Flag encoding quality control status + Controlinfo *string + // Flag encoding quality control status + Useinfo *string + // Number of tests that failed? + Cfailed *string +} + +func (o *Flag) ToRow() []any { + // "timeseries", "obstime", "corrected","controlinfo", "useinfo", "cfailed" + return []any{o.Id, o.Obstime, o.Corrected, o.Controlinfo, o.Useinfo, o.Cfailed} +} From 9b87e5e50d5566e0a4c2da53573ee08594055cc6 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 16:53:47 +0100 Subject: [PATCH 23/36] Fix README.md --- migrations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/README.md b/migrations/README.md index 192738fb..bfa03aa5 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -16,7 +16,7 @@ Go package used to dump tables from old databases (KDVH, Kvalobs) and import the ./migrate kdvh dump ``` -1. Import dumps into LARD (you can use the `--help` flag to see all available options) +1. Import dumps into LARD ```terminal ./migrate kdvh import From 8b7be5f0bf63f3e2e49592f08a2a46e5fe93f6b4 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 17:07:07 +0100 Subject: [PATCH 24/36] Use pgx in dump code --- migrations/kdvh/dump.go | 38 ++++++++++---------- migrations/kdvh/dump_functions.go | 58 +++++++++++++++---------------- migrations/kdvh/table.go | 4 +-- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go index 13d77b8e..930a6e07 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump.go @@ -1,7 +1,7 @@ package kdvh import ( - "database/sql" + "context" "fmt" "log/slog" "os" @@ -10,7 +10,8 @@ import ( "strings" "sync" - _ "github.com/jackc/pgx/v5/stdlib" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" "migrate/utils" ) @@ -29,7 +30,7 @@ type DumpConfig struct { } func (config *DumpConfig) Execute([]string) error { - conn, err := sql.Open("pgx", os.Getenv("KDVH_PROXY_CONN")) + pool, err := pgxpool.New(context.Background(), os.Getenv("KDVH_PROXY_CONN")) if err != nil { slog.Error(err.Error()) return nil @@ -39,13 +40,13 @@ func (config *DumpConfig) Execute([]string) error { if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { continue } - table.Dump(conn, config) + table.Dump(pool, config) } return nil } -func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { +func (table *Table) Dump(pool *pgxpool.Pool, config *DumpConfig) { defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) table.Path = filepath.Join(config.BaseDir, table.Path) @@ -56,12 +57,12 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { utils.SetLogFile(table.TableName, "dump") - elements, err := table.getElements(conn, config) + elements, err := table.getElements(pool, config) if err != nil { return } - stations, err := table.getStations(conn, config) + stations, err := table.getStations(pool, config) if err != nil { return } @@ -96,7 +97,7 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { flagTable: table.FlagTableName, overwrite: config.Overwrite, }, - conn, + pool, ) // NOTE: Non-nil errors are logged inside each DumpFunc @@ -114,8 +115,8 @@ func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { } // Fetches elements and filters them based on user input -func (table *Table) getElements(conn *sql.DB, config *DumpConfig) ([]string, error) { - elements, err := table.fetchElements(conn) +func (table *Table) getElements(pool *pgxpool.Pool, config *DumpConfig) ([]string, error) { + elements, err := table.fetchElements(pool) if err != nil { return nil, err } @@ -132,7 +133,7 @@ func (table *Table) getElements(conn *sql.DB, config *DumpConfig) ([]string, err // Fetch column names for a given table // We skip the columns defined in INVALID_COLUMNS and all columns that contain the 'kopi' string // TODO: should we dump these invalid/kopi elements even if we are not importing them? -func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { +func (table *Table) fetchElements(pool *pgxpool.Pool) (elements []string, err error) { slog.Info(fmt.Sprintf("Fetching elements for %s...", table.TableName)) // NOTE: T_HOMOGEN_MONTH is a special case, refer to `dumpHomogenMonth` in @@ -141,7 +142,8 @@ func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { return []string{"rr", "tam"}, nil } - rows, err := conn.Query( + rows, err := pool.Query( + context.TODO(), `SELECT column_name FROM information_schema.columns WHERE table_name = $1 AND NOT column_name = ANY($2::text[]) @@ -168,8 +170,8 @@ func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { } // Fetches station numbers and filters them based on user input -func (table *Table) getStations(conn *sql.DB, config *DumpConfig) ([]string, error) { - stations, err := table.fetchStnrFromElemTable(conn) +func (table *Table) getStations(pool *pgxpool.Pool, config *DumpConfig) ([]string, error) { + stations, err := table.fetchStnrFromElemTable(pool) if err != nil { return nil, err } @@ -179,16 +181,16 @@ func (table *Table) getStations(conn *sql.DB, config *DumpConfig) ([]string, err } // This function uses the ELEM table to fetch the station numbers -func (table *Table) fetchStnrFromElemTable(conn *sql.DB) (stations []string, err error) { +func (table *Table) fetchStnrFromElemTable(pool *pgxpool.Pool) (stations []string, err error) { slog.Info(fmt.Sprint("Fetching station numbers...")) - var rows *sql.Rows + var rows pgx.Rows if table.ElemTableName == "T_ELEM_OBS" { query := `SELECT DISTINCT stnr FROM t_elem_obs WHERE table_name = $1` - rows, err = conn.Query(query, table.TableName) + rows, err = pool.Query(context.TODO(), query, table.TableName) } else { query := fmt.Sprintf("SELECT DISTINCT stnr FROM %s", strings.ToLower(table.ElemTableName)) - rows, err = conn.Query(query) + rows, err = pool.Query(context.TODO(), query) } if err != nil { diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go index adc1930f..e712e949 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump_functions.go @@ -1,6 +1,7 @@ package kdvh import ( + "context" "database/sql" "encoding/csv" "errors" @@ -12,6 +13,9 @@ import ( "slices" "strconv" "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" ) // Format string for date field in CSV files @@ -22,9 +26,9 @@ var EMPTY_QUERY_ERR error = errors.New("The query did not return any rows") // Struct representing a single record in the output CSV file type Record struct { - time time.Time - data sql.NullString - flag sql.NullString + Time time.Time `db:"time"` + Data sql.NullString `db:"data"` + Flag sql.NullString `db:"flag"` } func fileExists(filename string, overwrite bool) error { @@ -39,11 +43,11 @@ func fileExists(filename string, overwrite bool) error { } // Helper function for dumpByYear functinos Fetch min and max year from table, needed for tables that are dumped by year -func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, error) { +func fetchYearRange(tableName, station string, pool *pgxpool.Pool) (int64, int64, error) { var beginStr, endStr string query := fmt.Sprintf("SELECT min(to_char(dato, 'yyyy')), max(to_char(dato, 'yyyy')) FROM %s WHERE stnr = $1", tableName) - if err := conn.QueryRow(query, station).Scan(&beginStr, &endStr); err != nil { + if err := pool.QueryRow(context.TODO(), query, station).Scan(&beginStr, &endStr); err != nil { return 0, 0, fmt.Errorf("Could not query row: %v", err) } @@ -62,13 +66,13 @@ func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, erro // This function is used when the table contains large amount of data // (T_SECOND, T_MINUTE, T_10MINUTE) -func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { - dataBegin, dataEnd, err := fetchYearRange(meta.dataTable, meta.station, conn) +func dumpByYear(path string, meta DumpMeta, pool *pgxpool.Pool) error { + dataBegin, dataEnd, err := fetchYearRange(meta.dataTable, meta.station, pool) if err != nil { return err } - flagBegin, flagEnd, err := fetchYearRange(meta.flagTable, meta.station, conn) + flagBegin, flagEnd, err := fetchYearRange(meta.flagTable, meta.station, pool) if err != nil { return err } @@ -106,7 +110,7 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { continue } - rows, err := conn.Query(query, meta.station, year) + rows, err := pool.Query(context.TODO(), query, meta.station, year) if err != nil { slog.Error(meta.logStr + fmt.Sprint("Could not query KDVH: ", err)) continue @@ -127,7 +131,7 @@ func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { // - RR (hourly precipitations, note that in Stinfosys this parameter is 'RR_1') // // We calculate the other data on the fly (outside this program) if needed. -func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { +func dumpHomogenMonth(path string, meta DumpMeta, pool *pgxpool.Pool) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { slog.Warn(meta.logStr + err.Error()) @@ -141,7 +145,7 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { meta.element, "", ) - rows, err := conn.Query(query, meta.station) + rows, err := pool.Query(context.TODO(), query, meta.station) if err != nil { slog.Error(meta.logStr + err.Error()) return err @@ -157,7 +161,7 @@ func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { // This function is used to dump tables that don't have a FLAG table, // (T_METARDATA, T_HOMOGEN_DIURNAL) -func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { +func dumpDataOnly(path string, meta DumpMeta, pool *pgxpool.Pool) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { slog.Warn(meta.logStr + err.Error()) @@ -171,7 +175,7 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { meta.dataTable, ) - rows, err := conn.Query(query, meta.station) + rows, err := pool.Query(context.TODO(), query, meta.station) if err != nil { slog.Error(meta.logStr + err.Error()) return err @@ -188,7 +192,7 @@ func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { // This is the default dump function. // It selects both data and flag tables for a specific (station, element) pair, // and then performs a full outer join on the two subqueries -func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { +func dumpDataAndFlags(path string, meta DumpMeta, pool *pgxpool.Pool) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { slog.Warn(meta.logStr + err.Error()) @@ -210,7 +214,7 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { meta.flagTable, ) - rows, err := conn.Query(query, meta.station) + rows, err := pool.Query(context.TODO(), query, meta.station) if err != nil { slog.Error(meta.logStr + err.Error()) return err @@ -227,7 +231,7 @@ func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { } // Dumps queried rows to file -func dumpToFile(filename string, rows *sql.Rows) error { +func dumpToFile(filename string, rows pgx.Rows) error { lines, err := sortRows(rows) if err != nil { return err @@ -251,22 +255,16 @@ func dumpToFile(filename string, rows *sql.Rows) error { } // Scans the rows and collects them in a slice of chronologically sorted lines -func sortRows(rows *sql.Rows) ([]Record, error) { +func sortRows(rows pgx.Rows) ([]Record, error) { defer rows.Close() - // TODO: if we use pgx we might be able to preallocate the right size - var records []Record - var record Record - - for rows.Next() { - if err := rows.Scan(&record.time, &record.data, &record.flag); err != nil { - return nil, errors.New("Could not scan rows: " + err.Error()) - } - records = append(records, record) + records, err := pgx.CollectRows(rows, pgx.RowToStructByName[Record]) + if err != nil { + return nil, errors.New("Could not collect rows: " + err.Error()) } slices.SortFunc(records, func(a, b Record) int { - return a.time.Compare(b.time) + return a.Time.Compare(b.Time) }) return records, rows.Err() @@ -281,9 +279,9 @@ func writeElementFile(lines []Record, file io.Writer) error { record := make([]string, 3) for _, l := range lines { - record[0] = l.time.Format(TIMEFORMAT) - record[1] = l.data.String - record[2] = l.flag.String + record[0] = l.Time.Format(TIMEFORMAT) + record[1] = l.Data.String + record[2] = l.Flag.String if err := writer.Write(record); err != nil { return errors.New("Could not write to file: " + err.Error()) diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index 90b11bb7..cc628de1 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -1,13 +1,13 @@ package kdvh import ( - "database/sql" "errors" "fmt" "log/slog" "migrate/lard" "time" + "github.com/jackc/pgx/v5/pgxpool" "github.com/rickb777/period" ) @@ -38,7 +38,7 @@ type Table struct { } // Implementation of these functions can be found in `dump_functions.go` -type DumpFunction func(path string, meta DumpMeta, conn *sql.DB) error +type DumpFunction func(path string, meta DumpMeta, pool *pgxpool.Pool) error type DumpMeta struct { element string station string From 8212649a0720d11e65768533790afc919afc6022 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 15 Nov 2024 16:25:20 +0100 Subject: [PATCH 25/36] Separate dump and import in different packages for better module navigation --- migrations/kdvh/cache.go | 243 ------------ migrations/kdvh/db/main.go | 41 +++ migrations/kdvh/db/table.go | 54 +++ migrations/kdvh/{ => dump}/dump.go | 74 ++-- migrations/kdvh/{ => dump}/dump_functions.go | 135 +++---- migrations/kdvh/dump/main.go | 42 +++ migrations/kdvh/dump/write.go | 89 +++++ migrations/kdvh/import.go | 295 --------------- migrations/kdvh/import/cache/kdvh.go | 97 +++++ migrations/kdvh/import/cache/main.go | 82 +++++ migrations/kdvh/import/cache/offsets.go | 65 ++++ migrations/kdvh/import/cache/stinfosys.go | 105 ++++++ migrations/kdvh/import/convert_functions.go | 345 ++++++++++++++++++ .../{import_functions.go => import/flags.go} | 303 +-------------- migrations/kdvh/import/import.go | 220 +++++++++++ migrations/kdvh/{ => import}/import_test.go | 2 +- migrations/kdvh/import/main.go | 63 ++++ migrations/kdvh/kdvh_test.go | 73 ++++ .../kdvh/{list_tables.go => list/main.go} | 10 +- migrations/kdvh/main.go | 51 +-- migrations/kdvh/table.go | 129 ------- .../tests/T_MDATA_combined/12345/TA.csv | 0 migrations/kdvh_test.go | 67 ---- migrations/utils/utils.go | 4 +- 24 files changed, 1365 insertions(+), 1224 deletions(-) delete mode 100644 migrations/kdvh/cache.go create mode 100644 migrations/kdvh/db/main.go create mode 100644 migrations/kdvh/db/table.go rename migrations/kdvh/{ => dump}/dump.go (61%) rename migrations/kdvh/{ => dump}/dump_functions.go (69%) create mode 100644 migrations/kdvh/dump/main.go create mode 100644 migrations/kdvh/dump/write.go delete mode 100644 migrations/kdvh/import.go create mode 100644 migrations/kdvh/import/cache/kdvh.go create mode 100644 migrations/kdvh/import/cache/main.go create mode 100644 migrations/kdvh/import/cache/offsets.go create mode 100644 migrations/kdvh/import/cache/stinfosys.go create mode 100644 migrations/kdvh/import/convert_functions.go rename migrations/kdvh/{import_functions.go => import/flags.go} (55%) create mode 100644 migrations/kdvh/import/import.go rename migrations/kdvh/{ => import}/import_test.go (98%) create mode 100644 migrations/kdvh/import/main.go create mode 100644 migrations/kdvh/kdvh_test.go rename migrations/kdvh/{list_tables.go => list/main.go} (63%) delete mode 100644 migrations/kdvh/table.go rename migrations/{ => kdvh}/tests/T_MDATA_combined/12345/TA.csv (100%) delete mode 100644 migrations/kdvh_test.go diff --git a/migrations/kdvh/cache.go b/migrations/kdvh/cache.go deleted file mode 100644 index 2c8d718f..00000000 --- a/migrations/kdvh/cache.go +++ /dev/null @@ -1,243 +0,0 @@ -package kdvh - -import ( - "context" - "fmt" - "log/slog" - "os" - "slices" - "time" - - "github.com/gocarina/gocsv" - "github.com/jackc/pgx/v5" - "github.com/rickb777/period" -) - -// Caches all the metadata needed for import. -// If any error occurs inside here the program will exit. -func (config *ImportConfig) CacheMetadata() { - config.cacheStinfo() - config.cacheKDVH() - config.cacheParamOffsets() -} - -// StinfoKey is used for lookup of parameter offsets and metadata from Stinfosys -type StinfoKey struct { - ElemCode string - TableName string -} - -// Subset of StinfoQuery with only param info -type StinfoParam struct { - TypeID int32 - ParamID int32 - Hlevel *int32 - Sensor int32 - Fromtime time.Time - IsScalar bool -} - -// Struct holding query from Stinfosys elem_map_cfnames_param -type StinfoQuery struct { - ElemCode string `db:"elem_code"` - TableName string `db:"table_name"` - TypeID int32 `db:"typeid"` - ParamID int32 `db:"paramid"` - Hlevel *int32 `db:"hlevel"` - Sensor int32 `db:"sensor"` - Fromtime time.Time `db:"fromtime"` - IsScalar bool `db:"scalar"` -} - -func (q *StinfoQuery) toParam() StinfoParam { - return StinfoParam{ - TypeID: q.TypeID, - ParamID: q.ParamID, - Hlevel: q.Hlevel, - Sensor: q.Sensor, - Fromtime: q.Fromtime, - IsScalar: q.IsScalar, - } -} -func (q *StinfoQuery) toKey() StinfoKey { - return StinfoKey{q.ElemCode, q.TableName} -} - -// Save metadata for later use by quering Stinfosys -func (config *ImportConfig) cacheStinfo() { - cache := make(map[StinfoKey]StinfoParam) - - fmt.Println("Connecting to Stinfosys to cache metadata") - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - conn, err := pgx.Connect(ctx, os.Getenv("STINFO_STRING")) - if err != nil { - slog.Error("Could not connect to Stinfosys. Make sure to be connected to the VPN. " + err.Error()) - os.Exit(1) - } - defer conn.Close(context.TODO()) - - for _, table := range KDVH { - if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { - continue - } - // select paramid, elem_code, scalar from elem_map_cfnames_param join param using(paramid) where scalar = false - query := `SELECT elem_code, table_name, typeid, paramid, hlevel, sensor, fromtime, scalar - FROM elem_map_cfnames_param - JOIN param USING(paramid) - WHERE table_name = $1 - AND ($2::text[] IS NULL OR elem_code = ANY($2))` - - rows, err := conn.Query(context.TODO(), query, table.TableName, config.Elements) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - - metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[StinfoQuery]) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - - for _, meta := range metas { - cache[meta.toKey()] = meta.toParam() - } - } - - config.StinfoMap = cache -} - -// Used for lookup of fromtime and totime from KDVH -type KDVHKey struct { - Inner StinfoKey - Station int32 -} - -func newKDVHKey(elem, table string, stnr int32) KDVHKey { - return KDVHKey{StinfoKey{ElemCode: elem, TableName: table}, stnr} -} - -// Timespan stored in KDVH for a given (table, station, element) triplet -type Timespan struct { - FromTime *time.Time `db:"fdato"` - ToTime *time.Time `db:"tdato"` -} - -// Struct used to deserialize KDVH query in cacheKDVH -type MetaKDVH struct { - ElemCode string `db:"elem_code"` - TableName string `db:"table_name"` - Station int32 `db:"stnr"` - FromTime *time.Time `db:"fdato"` - ToTime *time.Time `db:"tdato"` -} - -func (m *MetaKDVH) toTimespan() Timespan { - return Timespan{m.FromTime, m.ToTime} -} - -func (m *MetaKDVH) toKey() KDVHKey { - return KDVHKey{StinfoKey{ElemCode: m.ElemCode, TableName: m.TableName}, m.Station} -} - -func (config *ImportConfig) cacheKDVH() { - cache := make(map[KDVHKey]Timespan) - - fmt.Println("Connecting to KDVH proxy to cache metadata") - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - conn, err := pgx.Connect(ctx, os.Getenv("KDVH_PROXY_CONN")) - if err != nil { - slog.Error("Could not connect to KDVH proxy. Make sure to be connected to the VPN: " + err.Error()) - os.Exit(1) - } - defer conn.Close(context.TODO()) - - for _, t := range KDVH { - if config.Tables != nil && !slices.Contains(config.Tables, t.TableName) { - continue - } - - // TODO: probably need to sanitize these inputs - query := fmt.Sprintf( - `SELECT table_name, stnr, elem_code, fdato, tdato FROM %s - WHERE ($1::bigint[] IS NULL OR stnr = ANY($1)) - AND ($2::text[] IS NULL OR elem_code = ANY($2))`, - t.ElemTableName, - ) - - rows, err := conn.Query(context.TODO(), query, config.Stations, config.Elements) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - - metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[MetaKDVH]) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - - for _, meta := range metas { - cache[meta.toKey()] = meta.toTimespan() - } - } - - config.KDVHMap = cache -} - -// Caches how to modify the obstime (in KDVH) for certain paramids -func (config *ImportConfig) cacheParamOffsets() { - cache := make(map[StinfoKey]period.Period) - - type CSVRow struct { - TableName string `csv:"table_name"` - ElemCode string `csv:"elem_code"` - ParamID int32 `csv:"paramid"` - FromtimeOffset string `csv:"fromtime_offset"` - Timespan string `csv:"timespan"` - } - - csvfile, err := os.Open("kdvh/product_offsets.csv") - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - defer csvfile.Close() - - var csvrows []CSVRow - if err := gocsv.UnmarshalFile(csvfile, &csvrows); err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - - for _, row := range csvrows { - var fromtimeOffset, timespan period.Period - if row.FromtimeOffset != "" { - fromtimeOffset, err = period.Parse(row.FromtimeOffset) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - } - if row.Timespan != "" { - timespan, err = period.Parse(row.Timespan) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - } - migrationOffset, err := fromtimeOffset.Add(timespan) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } - - cache[StinfoKey{ElemCode: row.ElemCode, TableName: row.TableName}] = migrationOffset - } - - config.OffsetMap = cache -} diff --git a/migrations/kdvh/db/main.go b/migrations/kdvh/db/main.go new file mode 100644 index 00000000..fe14d2bc --- /dev/null +++ b/migrations/kdvh/db/main.go @@ -0,0 +1,41 @@ +package db + +// Map of all tables found in KDVH, with set max import year +var KDVH map[string]*Table = map[string]*Table{ + // Section 1: tables that need to be migrated entirely + // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetImportYear(3000), + "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetImportYear(3000), + + // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 + "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImportYear(2006), + + // Section 3: tables that should only be dumped + "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS"), + "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), + "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS"), + "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS"), + "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), + "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), + "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), + + // Section 4: special cases, namely digitized historical data + "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetImportYear(1957), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetImportYear(2006), + "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH"), + "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", ""), + + // Section 5: tables missing in the KDVH proxy: + // 1. these exist in a separate database + "T_AVINOR": NewTable("T_AVINOR", "T_AVINOR_FLAG", "T_ELEM_OBS"), + "T_PROJDATA": NewTable("T_PROJDATA", "T_PROJFLAG", "T_ELEM_PROJ"), + // 2. these are not in active use and don't need to be imported in LARD + "T_DIURNAL_INTERPOLATED": NewTable("T_DIURNAL_INTERPOLATED", "", ""), + "T_MONTH_INTERPOLATED": NewTable("T_MONTH_INTERPOLATED", "", ""), +} diff --git a/migrations/kdvh/db/table.go b/migrations/kdvh/db/table.go new file mode 100644 index 00000000..a1b9b787 --- /dev/null +++ b/migrations/kdvh/db/table.go @@ -0,0 +1,54 @@ +package db + +// In KDVH for each table name we usually have three separate tables: +// 1. A DATA table containing observation values; +// 2. A FLAG table containing quality control (QC) flags; +// 3. A ELEM table containing metadata about the validity of the timeseries. +// +// DATA and FLAG tables have the same schema: +// | dato | stnr | ... | +// where 'dato' is the timestamp of the observation, 'stnr' is the station +// where the observation was measured, and '...' is a varying number of columns +// each with different observations, where the column name is the 'elem_code' +// (e.g. for air temperature, 'ta'). +// +// The ELEM tables have the following schema: +// | stnr | elem_code | fdato | tdato | table_name | flag_table_name | audit_dato + +// This struct contains basic metadata for a KDVH table +type Table struct { + TableName string // Name of the DATA table + FlagTableName string // Name of the FLAG table + ElemTableName string // Name of the ELEM table + Path string // Directory name of where the dumped table is stored + importUntil int // Import data only until the year specified by this field. Table import will be skipped, if `SetImportYear` is not called. +} + +// Creates default Table +func NewTable(data, flag, elem string) *Table { + return &Table{ + TableName: data, + FlagTableName: flag, + ElemTableName: elem, + // NOTE: '_combined' kept for backward compatibility with original scripts + Path: data + "_combined", + } +} + +// Specify the year until data should be imported +func (t *Table) SetImportYear(year int) *Table { + if year > 0 { + t.importUntil = year + } + return t +} + +// Checks if the table is set for import +func (t *Table) ShouldImport() bool { + return t.importUntil > 0 +} + +// Checks if the table max import year was reached +func (t *Table) MaxImportYearReached(year int) bool { + return t.importUntil < 0 || year >= t.importUntil +} diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump/dump.go similarity index 61% rename from migrations/kdvh/dump.go rename to migrations/kdvh/dump/dump.go index 930a6e07..2c91f642 100644 --- a/migrations/kdvh/dump.go +++ b/migrations/kdvh/dump/dump.go @@ -1,4 +1,4 @@ -package kdvh +package dump import ( "context" @@ -6,74 +6,41 @@ import ( "log/slog" "os" "path/filepath" - "slices" "strings" "sync" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "migrate/kdvh/db" "migrate/utils" ) // List of columns that we do not need to select when extracting the element codes from a KDVH table var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} -type DumpConfig struct { - BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` - Tables []string `short:"t" delimiter:"," long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` - Stations []string `short:"s" delimiter:"," long:"stnr" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` - Elements []string `short:"e" delimiter:"," long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` - Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` - Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` - MaxConn int `long:"conns" default:"10" description:"Max number of concurrent connections allowed"` -} - -func (config *DumpConfig) Execute([]string) error { - pool, err := pgxpool.New(context.Background(), os.Getenv("KDVH_PROXY_CONN")) - if err != nil { - slog.Error(err.Error()) - return nil - } - - for _, table := range KDVH { - if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { - continue - } - table.Dump(pool, config) - } - - return nil -} - -func (table *Table) Dump(pool *pgxpool.Pool, config *DumpConfig) { +func DumpTable(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) { defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) - table.Path = filepath.Join(config.BaseDir, table.Path) - if err := os.MkdirAll(table.Path, os.ModePerm); err != nil { - slog.Error(err.Error()) - return - } - - utils.SetLogFile(table.TableName, "dump") - - elements, err := table.getElements(pool, config) + elements, err := getElements(table, pool, config) if err != nil { return } - stations, err := table.getStations(pool, config) + stations, err := getStations(table, pool, config) if err != nil { return } + dumpFunc := DumpFunc(table) + // Used to limit connections to the database semaphore := make(chan struct{}, config.MaxConn) bar := utils.NewBar(len(stations), table.TableName) bar.RenderBlank() for _, station := range stations { - path := filepath.Join(table.Path, string(station)) + path := filepath.Join(config.BaseDir, table.Path, string(station)) if err := os.MkdirAll(path, os.ModePerm); err != nil { slog.Error(err.Error()) return @@ -88,9 +55,9 @@ func (table *Table) Dump(pool *pgxpool.Pool, config *DumpConfig) { go func() { defer wg.Done() - err := table.dumpFunc( + err := dumpFunc( path, - DumpMeta{ + DumpArgs{ element: element, station: station, dataTable: table.TableName, @@ -99,8 +66,6 @@ func (table *Table) Dump(pool *pgxpool.Pool, config *DumpConfig) { }, pool, ) - - // NOTE: Non-nil errors are logged inside each DumpFunc if err == nil { slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) } @@ -115,13 +80,13 @@ func (table *Table) Dump(pool *pgxpool.Pool, config *DumpConfig) { } // Fetches elements and filters them based on user input -func (table *Table) getElements(pool *pgxpool.Pool, config *DumpConfig) ([]string, error) { - elements, err := table.fetchElements(pool) +func getElements(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) ([]string, error) { + elements, err := fetchElements(table, pool) if err != nil { return nil, err } - filename := filepath.Join(table.Path, "elements.txt") + filename := filepath.Join(config.BaseDir, table.Path, "elements.txt") if err := utils.SaveToFile(elements, filename); err != nil { slog.Warn("Could not save element list to " + filename) } @@ -133,7 +98,7 @@ func (table *Table) getElements(pool *pgxpool.Pool, config *DumpConfig) ([]strin // Fetch column names for a given table // We skip the columns defined in INVALID_COLUMNS and all columns that contain the 'kopi' string // TODO: should we dump these invalid/kopi elements even if we are not importing them? -func (table *Table) fetchElements(pool *pgxpool.Pool) (elements []string, err error) { +func fetchElements(table *db.Table, pool *pgxpool.Pool) (elements []string, err error) { slog.Info(fmt.Sprintf("Fetching elements for %s...", table.TableName)) // NOTE: T_HOMOGEN_MONTH is a special case, refer to `dumpHomogenMonth` in @@ -170,18 +135,23 @@ func (table *Table) fetchElements(pool *pgxpool.Pool) (elements []string, err er } // Fetches station numbers and filters them based on user input -func (table *Table) getStations(pool *pgxpool.Pool, config *DumpConfig) ([]string, error) { - stations, err := table.fetchStnrFromElemTable(pool) +func getStations(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) ([]string, error) { + stations, err := fetchStnrFromElemTable(table, pool) if err != nil { return nil, err } + filename := filepath.Join(config.BaseDir, table.Path, "stations.txt") + if err := utils.SaveToFile(stations, filename); err != nil { + slog.Warn("Could not save element list to " + filename) + } + stations = utils.FilterSlice(config.Stations, stations, "") return stations, nil } // This function uses the ELEM table to fetch the station numbers -func (table *Table) fetchStnrFromElemTable(pool *pgxpool.Pool) (stations []string, err error) { +func fetchStnrFromElemTable(table *db.Table, pool *pgxpool.Pool) (stations []string, err error) { slog.Info(fmt.Sprint("Fetching station numbers...")) var rows pgx.Rows diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump/dump_functions.go similarity index 69% rename from migrations/kdvh/dump_functions.go rename to migrations/kdvh/dump/dump_functions.go index e712e949..6fbb24bd 100644 --- a/migrations/kdvh/dump_functions.go +++ b/migrations/kdvh/dump/dump_functions.go @@ -1,34 +1,57 @@ -package kdvh +package dump import ( "context" - "database/sql" - "encoding/csv" "errors" "fmt" - "io" "log/slog" "os" "path/filepath" - "slices" "strconv" - "time" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" -) -// Format string for date field in CSV files -const TIMEFORMAT string = "2006-01-02_15:04:05" + "migrate/kdvh/db" +) -// Error returned if no observations are found for a (station, element) pair -var EMPTY_QUERY_ERR error = errors.New("The query did not return any rows") +// Function used to dump the KDVH table, see below +type DumpFunction func(path string, meta DumpArgs, pool *pgxpool.Pool) error +type DumpArgs struct { + element string + station string + dataTable string + flagTable string + overwrite bool + logStr string +} -// Struct representing a single record in the output CSV file -type Record struct { - Time time.Time `db:"time"` - Data sql.NullString `db:"data"` - Flag sql.NullString `db:"flag"` +// var DUMP_MAP map[string]DumpFunction = map[string]DumpFunction{ +// "T_METARDATA": dumpDataOnly, +// "T_HOMOGEN_DIURNAL": dumpDataOnly, +// "T_SECOND_DATA": dumpByYear, +// "T_MINUTE_DATA": dumpByYear, +// "T_10MINUTE_DATA": dumpByYear, +// "T_HOMOGEN_MONTH": dumpHomogenMonth, +// } + +// func DumpFunc(table *db.Table) DumpFunction { +// fn, ok := DUMP_MAP[table.TableName] +// if !ok { +// return dumpDataAndFlags +// } +// return fn +// } + +func DumpFunc(table *db.Table) DumpFunction { + switch table.TableName { + case "T_METARDATA", "T_HOMOGEN_DIURNAL": + return dumpDataOnly + case "T_SECOND_DATA", "T_MINUTE_DATA", "T_10MINUTE_DATA": + return dumpByYear + case "T_HOMOGEN_MONTH": + return dumpHomogenMonth + } + return dumpDataAndFlags } func fileExists(filename string, overwrite bool) error { @@ -66,7 +89,7 @@ func fetchYearRange(tableName, station string, pool *pgxpool.Pool) (int64, int64 // This function is used when the table contains large amount of data // (T_SECOND, T_MINUTE, T_10MINUTE) -func dumpByYear(path string, meta DumpMeta, pool *pgxpool.Pool) error { +func dumpByYear(path string, meta DumpArgs, pool *pgxpool.Pool) error { dataBegin, dataEnd, err := fetchYearRange(meta.dataTable, meta.station, pool) if err != nil { return err @@ -116,7 +139,7 @@ func dumpByYear(path string, meta DumpMeta, pool *pgxpool.Pool) error { continue } - if err := dumpToFile(filename, rows); err != nil { + if err := writeToCsv(filename, rows); err != nil { slog.Error(meta.logStr + err.Error()) continue } @@ -131,7 +154,7 @@ func dumpByYear(path string, meta DumpMeta, pool *pgxpool.Pool) error { // - RR (hourly precipitations, note that in Stinfosys this parameter is 'RR_1') // // We calculate the other data on the fly (outside this program) if needed. -func dumpHomogenMonth(path string, meta DumpMeta, pool *pgxpool.Pool) error { +func dumpHomogenMonth(path string, meta DumpArgs, pool *pgxpool.Pool) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { slog.Warn(meta.logStr + err.Error()) @@ -151,7 +174,7 @@ func dumpHomogenMonth(path string, meta DumpMeta, pool *pgxpool.Pool) error { return err } - if err := dumpToFile(filename, rows); err != nil { + if err := writeToCsv(filename, rows); err != nil { slog.Error(meta.logStr + err.Error()) return err } @@ -161,7 +184,7 @@ func dumpHomogenMonth(path string, meta DumpMeta, pool *pgxpool.Pool) error { // This function is used to dump tables that don't have a FLAG table, // (T_METARDATA, T_HOMOGEN_DIURNAL) -func dumpDataOnly(path string, meta DumpMeta, pool *pgxpool.Pool) error { +func dumpDataOnly(path string, meta DumpArgs, pool *pgxpool.Pool) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { slog.Warn(meta.logStr + err.Error()) @@ -181,7 +204,7 @@ func dumpDataOnly(path string, meta DumpMeta, pool *pgxpool.Pool) error { return err } - if err := dumpToFile(filename, rows); err != nil { + if err := writeToCsv(filename, rows); err != nil { slog.Error(meta.logStr + err.Error()) return err } @@ -192,7 +215,7 @@ func dumpDataOnly(path string, meta DumpMeta, pool *pgxpool.Pool) error { // This is the default dump function. // It selects both data and flag tables for a specific (station, element) pair, // and then performs a full outer join on the two subqueries -func dumpDataAndFlags(path string, meta DumpMeta, pool *pgxpool.Pool) error { +func dumpDataAndFlags(path string, meta DumpArgs, pool *pgxpool.Pool) error { filename := filepath.Join(path, meta.element+".csv") if err := fileExists(filename, meta.overwrite); err != nil { slog.Warn(meta.logStr + err.Error()) @@ -220,7 +243,7 @@ func dumpDataAndFlags(path string, meta DumpMeta, pool *pgxpool.Pool) error { return err } - if err := dumpToFile(filename, rows); err != nil { + if err := writeToCsv(filename, rows); err != nil { if !errors.Is(err, EMPTY_QUERY_ERR) { slog.Error(meta.logStr + err.Error()) } @@ -229,65 +252,3 @@ func dumpDataAndFlags(path string, meta DumpMeta, pool *pgxpool.Pool) error { return nil } - -// Dumps queried rows to file -func dumpToFile(filename string, rows pgx.Rows) error { - lines, err := sortRows(rows) - if err != nil { - return err - } - - // Return if query was empty - if len(lines) == 0 { - return EMPTY_QUERY_ERR - } - - file, err := os.Create(filename) - if err != nil { - return err - } - - err = writeElementFile(lines, file) - if closeErr := file.Close(); closeErr != nil { - return errors.Join(err, closeErr) - } - return err -} - -// Scans the rows and collects them in a slice of chronologically sorted lines -func sortRows(rows pgx.Rows) ([]Record, error) { - defer rows.Close() - - records, err := pgx.CollectRows(rows, pgx.RowToStructByName[Record]) - if err != nil { - return nil, errors.New("Could not collect rows: " + err.Error()) - } - - slices.SortFunc(records, func(a, b Record) int { - return a.Time.Compare(b.Time) - }) - - return records, rows.Err() -} - -// Writes queried (time | data | flag) columns to CSV -func writeElementFile(lines []Record, file io.Writer) error { - // Write number of lines as header - file.Write([]byte(fmt.Sprintf("%v\n", len(lines)))) - - writer := csv.NewWriter(file) - - record := make([]string, 3) - for _, l := range lines { - record[0] = l.Time.Format(TIMEFORMAT) - record[1] = l.Data.String - record[2] = l.Flag.String - - if err := writer.Write(record); err != nil { - return errors.New("Could not write to file: " + err.Error()) - } - } - - writer.Flush() - return writer.Error() -} diff --git a/migrations/kdvh/dump/main.go b/migrations/kdvh/dump/main.go new file mode 100644 index 00000000..3b94975d --- /dev/null +++ b/migrations/kdvh/dump/main.go @@ -0,0 +1,42 @@ +package dump + +import ( + "context" + "log/slog" + "os" + "slices" + + "github.com/jackc/pgx/v5/pgxpool" + + "migrate/kdvh/db" + "migrate/utils" +) + +type DumpConfig struct { + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + Tables []string `short:"t" delimiter:"," long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + Stations []string `short:"s" delimiter:"," long:"stnr" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + Elements []string `short:"e" delimiter:"," long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` + Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` + MaxConn int `long:"conns" default:"10" description:"Max number of concurrent connections allowed to KDVH"` +} + +func (config *DumpConfig) Execute([]string) error { + pool, err := pgxpool.New(context.Background(), os.Getenv("KDVH_PROXY_CONN")) + if err != nil { + slog.Error(err.Error()) + return nil + } + + for _, table := range db.KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + + utils.SetLogFile(table.TableName, "dump") + DumpTable(table, pool, config) + } + + return nil +} diff --git a/migrations/kdvh/dump/write.go b/migrations/kdvh/dump/write.go new file mode 100644 index 00000000..5e4aec9d --- /dev/null +++ b/migrations/kdvh/dump/write.go @@ -0,0 +1,89 @@ +package dump + +import ( + "database/sql" + "encoding/csv" + "errors" + "fmt" + "io" + "os" + "slices" + "time" + + "github.com/jackc/pgx/v5" +) + +// Format string for date field in CSV files +const TIMEFORMAT string = "2006-01-02_15:04:05" + +// Error returned if no observations are found for a (station, element) pair +var EMPTY_QUERY_ERR error = errors.New("The query did not return any rows") + +// Struct representing a single record in the output CSV file +type Record struct { + Time time.Time `db:"time"` + Data sql.NullString `db:"data"` + Flag sql.NullString `db:"flag"` +} + +// Dumps queried rows to file +func writeToCsv(filename string, rows pgx.Rows) error { + lines, err := sortRows(rows) + if err != nil { + return err + } + + // Return if query was empty + if len(lines) == 0 { + return EMPTY_QUERY_ERR + } + + file, err := os.Create(filename) + if err != nil { + return err + } + + err = writeElementFile(lines, file) + if closeErr := file.Close(); closeErr != nil { + return errors.Join(err, closeErr) + } + return err +} + +// Scans the rows and collects them in a slice of chronologically sorted lines +func sortRows(rows pgx.Rows) ([]Record, error) { + defer rows.Close() + + records, err := pgx.CollectRows(rows, pgx.RowToStructByName[Record]) + if err != nil { + return nil, errors.New("Could not collect rows: " + err.Error()) + } + + slices.SortFunc(records, func(a, b Record) int { + return a.Time.Compare(b.Time) + }) + + return records, rows.Err() +} + +// Writes queried (time | data | flag) columns to CSV +func writeElementFile(lines []Record, file io.Writer) error { + // Write number of lines as header + file.Write([]byte(fmt.Sprintf("%v\n", len(lines)))) + + writer := csv.NewWriter(file) + + record := make([]string, 3) + for _, l := range lines { + record[0] = l.Time.Format(TIMEFORMAT) + record[1] = l.Data.String + record[2] = l.Flag.String + + if err := writer.Write(record); err != nil { + return errors.New("Could not write to file: " + err.Error()) + } + } + + writer.Flush() + return writer.Error() +} diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go deleted file mode 100644 index 6833cbe9..00000000 --- a/migrations/kdvh/import.go +++ /dev/null @@ -1,295 +0,0 @@ -package kdvh - -import ( - "bufio" - "context" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "sync" - "time" - - "github.com/jackc/pgx/v5/pgxpool" - "github.com/rickb777/period" - - "migrate/lard" - "migrate/utils" -) - -// TODO: add CALL_SIGN? It's not in stinfosys? -var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} - -type ImportConfig struct { - Verbose bool `short:"v" description:"Increase verbosity level"` - BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` - Tables []string `short:"t" long:"table" delimiter:"," default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` - Stations []string `short:"s" long:"station" delimiter:"," default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` - Elements []string `short:"e" long:"elemcode" delimiter:"," default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` - Sep string `long:"sep" default:"," description:"Separator character in the dumped files. Needs to be quoted"` - HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` - Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` - Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` - - OffsetMap map[StinfoKey]period.Period // Map of offsets used to correct (?) KDVH times for specific parameters - StinfoMap map[StinfoKey]StinfoParam // Map of metadata used to query timeseries ID in LARD - KDVHMap map[KDVHKey]Timespan // Map of `from_time` and `to_time` for each (table, station, element) triplet. Not present for all parameters -} - -func (config *ImportConfig) setup() { - if len(config.Sep) > 1 { - fmt.Printf("Error: '--sep' only accepts single-byte characters. Got %s", config.Sep) - os.Exit(1) - } - config.CacheMetadata() -} - -func (config *ImportConfig) Execute([]string) error { - config.setup() - - // Create connection pool for LARD - pool, err := pgxpool.New(context.TODO(), os.Getenv("LARD_STRING")) - if err != nil { - slog.Error(fmt.Sprint("Could not connect to Lard:", err)) - return err - } - defer pool.Close() - - for _, table := range KDVH { - if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { - continue - } - table.Import(pool, config) - } - - return nil -} - -func (table *Table) Import(pool *pgxpool.Pool, config *ImportConfig) (rowsInserted int64) { - defer utils.SendEmailOnPanic("table.Import", config.Email) - - if table.importUntil == 0 { - if config.Verbose { - slog.Info("Skipping import of" + table.TableName + " because this table is not set for import") - } - return 0 - } - - utils.SetLogFile(table.TableName, "import") - - table.Path = filepath.Join(config.BaseDir, table.Path) - stations, err := os.ReadDir(table.Path) - if err != nil { - slog.Warn(fmt.Sprintf("Could not read directory %s: %s", table.Path, err)) - return 0 - } - - bar := utils.NewBar(len(stations), table.TableName) - bar.RenderBlank() - for _, station := range stations { - count, err := table.importStation(station, pool, config) - if err == nil { - rowsInserted += count - } - bar.Add(1) - } - - outputStr := fmt.Sprintf("%v: %v total rows inserted", table.TableName, rowsInserted) - slog.Info(outputStr) - fmt.Println(outputStr) - return rowsInserted -} - -// Loops over the element files present in the station directory and processes them concurrently -func (table *Table) importStation(station os.DirEntry, pool *pgxpool.Pool, config *ImportConfig) (totRows int64, err error) { - stnr, err := getStationNumber(station, config.Stations) - if err != nil { - if config.Verbose { - slog.Info(err.Error()) - } - return 0, err - } - - dir := filepath.Join(table.Path, station.Name()) - elements, err := os.ReadDir(dir) - if err != nil { - slog.Warn(fmt.Sprintf("Could not read directory %s: %s", dir, err)) - return 0, err - } - - var wg sync.WaitGroup - for _, element := range elements { - elemCode, err := getElementCode(element, config.Elements) - if err != nil { - if config.Verbose { - slog.Info(err.Error()) - } - continue - } - - wg.Add(1) - go func() { - defer wg.Done() - - tsInfo, err := config.NewTimeseriesInfo(table.TableName, elemCode, stnr) - if err != nil { - return - } - - tsid, err := getTimeseriesID(tsInfo, pool) - if err != nil { - slog.Error(tsInfo.logstr + "could not obtain timeseries - " + err.Error()) - return - } - - filename := filepath.Join(dir, element.Name()) - file, err := os.Open(filename) - if err != nil { - slog.Warn(fmt.Sprintf("Could not open file '%s': %s", filename, err)) - return - } - defer file.Close() - - data, text, flag, err := table.parseData(file, tsid, tsInfo, config) - if err != nil { - return - } - - if len(data) == 0 { - slog.Info(tsInfo.logstr + "no rows to insert (all obstimes > max import time)") - return - } - - var count int64 - if !(config.Skip == "data") { - if tsInfo.param.IsScalar { - count, err = lard.InsertData(data, pool, tsInfo.logstr) - if err != nil { - slog.Error(tsInfo.logstr + "failed data bulk insertion - " + err.Error()) - return - } - } else { - count, err = lard.InsertTextData(text, pool, tsInfo.logstr) - if err != nil { - slog.Error(tsInfo.logstr + "failed non-scalar data bulk insertion - " + err.Error()) - return - } - // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data - // return count, nil - } - } - - if !(config.Skip == "flags") { - if err := lard.InsertFlags(flag, pool, tsInfo.logstr); err != nil { - slog.Error(tsInfo.logstr + "failed flag bulk insertion - " + err.Error()) - } - } - - totRows += count - }() - } - wg.Wait() - - return totRows, nil -} - -func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { - if !station.IsDir() { - return 0, errors.New(fmt.Sprintf("%s is not a directory, skipping", station.Name())) - } - - if stationList != nil && !slices.Contains(stationList, station.Name()) { - return 0, errors.New(fmt.Sprintf("Station %v not in the list, skipping", station.Name())) - } - - stnr, err := strconv.ParseInt(station.Name(), 10, 32) - if err != nil { - return 0, errors.New("Error parsing station number:" + err.Error()) - } - - return int32(stnr), nil -} - -func elemcodeIsInvalid(element string) bool { - return strings.Contains(element, "KOPI") || slices.Contains(INVALID_ELEMENTS, element) -} - -func getElementCode(element os.DirEntry, elementList []string) (string, error) { - elemCode := strings.ToUpper(strings.TrimSuffix(element.Name(), ".csv")) - - if elementList != nil && !slices.Contains(elementList, elemCode) { - return "", errors.New(fmt.Sprintf("Element '%s' not in the list, skipping", elemCode)) - } - - if elemcodeIsInvalid(elemCode) { - return "", errors.New(fmt.Sprintf("Element '%s' not set for import, skipping", elemCode)) - } - return elemCode, nil -} - -func getTimeseriesID(tsInfo *TimeseriesInfo, pool *pgxpool.Pool) (int32, error) { - label := lard.Label{ - StationID: tsInfo.station, - TypeID: tsInfo.param.TypeID, - ParamID: tsInfo.param.ParamID, - Sensor: &tsInfo.param.Sensor, - Level: tsInfo.param.Hlevel, - } - tsid, err := lard.GetTimeseriesID(label, tsInfo.param.Fromtime, pool) - if err != nil { - slog.Error(tsInfo.logstr + "could not obtain timeseries - " + err.Error()) - return 0, err - - } - return tsid, nil -} - -func (table *Table) parseData(handle *os.File, id int32, meta *TimeseriesInfo, config *ImportConfig) ([][]any, [][]any, [][]any, error) { - scanner := bufio.NewScanner(handle) - - var rowCount int - // Try to infer row count from header - if config.HasHeader { - scanner.Scan() - rowCount, _ = strconv.Atoi(scanner.Text()) - } - - // Prepare slices for pgx.CopyFromRows - data := make([][]any, 0, rowCount) - text := make([][]any, 0, rowCount) - flag := make([][]any, 0, rowCount) - - for scanner.Scan() { - cols := strings.Split(scanner.Text(), config.Sep) - - obsTime, err := time.Parse("2006-01-02_15:04:05", cols[0]) - if err != nil { - return nil, nil, nil, err - } - - // Only import data between KDVH's defined fromtime and totime - if meta.span.FromTime != nil && obsTime.Sub(*meta.span.FromTime) < 0 { - continue - } else if meta.span.ToTime != nil && obsTime.Sub(*meta.span.ToTime) > 0 { - break - } - - if obsTime.Year() >= table.importUntil { - break - } - - dataRow, textRow, flagRow, err := table.convFunc(KdvhObs{meta, id, obsTime, cols[1], cols[2]}) - if err != nil { - return nil, nil, nil, err - } - data = append(data, dataRow.ToRow()) - text = append(text, textRow.ToRow()) - flag = append(flag, flagRow.ToRow()) - } - - return data, text, flag, nil -} diff --git a/migrations/kdvh/import/cache/kdvh.go b/migrations/kdvh/import/cache/kdvh.go new file mode 100644 index 00000000..36a012c1 --- /dev/null +++ b/migrations/kdvh/import/cache/kdvh.go @@ -0,0 +1,97 @@ +package cache + +import ( + "context" + "fmt" + "log/slog" + "os" + "slices" + "time" + + "github.com/jackc/pgx/v5" + + "migrate/kdvh/db" +) + +// Map of `from_time` and `to_time` for each (table, station, element) triplet. Not present for all parameters +type KDVHMap = map[KDVHKey]Timespan + +// Used for lookup of fromtime and totime from KDVH +type KDVHKey struct { + Inner StinfoKey + Station int32 +} + +func newKDVHKey(elem, table string, stnr int32) KDVHKey { + return KDVHKey{StinfoKey{ElemCode: elem, TableName: table}, stnr} +} + +// Timespan stored in KDVH for a given (table, station, element) triplet +type Timespan struct { + FromTime *time.Time `db:"fdato"` + ToTime *time.Time `db:"tdato"` +} + +// Struct used to deserialize KDVH query in cacheKDVH +type MetaKDVH struct { + ElemCode string `db:"elem_code"` + TableName string `db:"table_name"` + Station int32 `db:"stnr"` + FromTime *time.Time `db:"fdato"` + ToTime *time.Time `db:"tdato"` +} + +func (m *MetaKDVH) toTimespan() Timespan { + return Timespan{m.FromTime, m.ToTime} +} + +func (m *MetaKDVH) toKey() KDVHKey { + return KDVHKey{StinfoKey{ElemCode: m.ElemCode, TableName: m.TableName}, m.Station} +} + +func cacheKDVH(tables, stations, elements []string) KDVHMap { + cache := make(KDVHMap) + + fmt.Println("Connecting to KDVH proxy to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("KDVH_PROXY_CONN")) + if err != nil { + slog.Error("Could not connect to KDVH proxy. Make sure to be connected to the VPN: " + err.Error()) + os.Exit(1) + } + defer conn.Close(context.TODO()) + + for _, t := range db.KDVH { + if tables != nil && !slices.Contains(tables, t.TableName) { + continue + } + + // TODO: probably need to sanitize these inputs + query := fmt.Sprintf( + `SELECT table_name, stnr, elem_code, fdato, tdato FROM %s + WHERE ($1::bigint[] IS NULL OR stnr = ANY($1)) + AND ($2::text[] IS NULL OR elem_code = ANY($2))`, + t.ElemTableName, + ) + + rows, err := conn.Query(context.TODO(), query, stations, elements) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[MetaKDVH]) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, meta := range metas { + cache[meta.toKey()] = meta.toTimespan() + } + } + + return cache +} diff --git a/migrations/kdvh/import/cache/main.go b/migrations/kdvh/import/cache/main.go new file mode 100644 index 00000000..3f06004c --- /dev/null +++ b/migrations/kdvh/import/cache/main.go @@ -0,0 +1,82 @@ +package cache + +import ( + "errors" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/rickb777/period" + + "migrate/lard" +) + +type Cache struct { + OffsetMap OffsetMap + StinfoMap StinfoMap + KDVHMap KDVHMap +} + +// TODO: cache permissions + +// Caches all the metadata needed for import. +// If any error occurs inside here the program will exit. +func CacheMetadata(tables, stations, elements []string) *Cache { + return &Cache{ + OffsetMap: cacheParamOffsets(), + StinfoMap: cacheStinfo(tables, elements), + KDVHMap: cacheKDVH(tables, stations, elements), + } +} + +// Convenience struct that holds information for a specific timeseries +type TsInfo struct { + Id int32 + Station int32 + Element string + Offset period.Period + Param StinfoParam + Span Timespan + Logstr string +} + +func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpool.Pool) (*TsInfo, error) { + logstr := fmt.Sprintf("%v - %v - %v: ", table, station, element) + key := newKDVHKey(element, table, station) + + param, ok := cache.StinfoMap[key.Inner] + if !ok { + // TODO: should it fail here? How do we deal with data without metadata? + slog.Error(logstr + "Missing metadata in Stinfosys") + return nil, errors.New("") + } + + // No need to check for `!ok`, will default to 0 offset + offset := cache.OffsetMap[key.Inner] + + // No need to check for `!ok`, timespan will be ignored if not in the map + span := cache.KDVHMap[key] + + label := lard.Label{ + StationID: station, + TypeID: param.TypeID, + ParamID: param.ParamID, + Sensor: ¶m.Sensor, + Level: param.Hlevel, + } + tsid, err := lard.GetTimeseriesID(label, param.Fromtime, pool) + if err != nil { + slog.Error(logstr + "could not obtain timeseries - " + err.Error()) + return nil, err + } + + return &TsInfo{ + Id: tsid, + Station: station, + Element: element, + Offset: offset, + Param: param, + Span: span, + Logstr: logstr, + }, nil +} diff --git a/migrations/kdvh/import/cache/offsets.go b/migrations/kdvh/import/cache/offsets.go new file mode 100644 index 00000000..e39a934b --- /dev/null +++ b/migrations/kdvh/import/cache/offsets.go @@ -0,0 +1,65 @@ +package cache + +import ( + "log/slog" + "os" + + "github.com/gocarina/gocsv" + "github.com/rickb777/period" +) + +// Map of offsets used to correct KDVH times for specific parameters +type OffsetMap = map[StinfoKey]period.Period + +// Caches how to modify the obstime (in KDVH) for certain paramids +func cacheParamOffsets() OffsetMap { + cache := make(OffsetMap) + + type CSVRow struct { + TableName string `csv:"table_name"` + ElemCode string `csv:"elem_code"` + ParamID int32 `csv:"paramid"` + FromtimeOffset string `csv:"fromtime_offset"` + Timespan string `csv:"timespan"` + } + + csvfile, err := os.Open("kdvh/product_offsets.csv") + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + defer csvfile.Close() + + var csvrows []CSVRow + if err := gocsv.UnmarshalFile(csvfile, &csvrows); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, row := range csvrows { + var fromtimeOffset, timespan period.Period + if row.FromtimeOffset != "" { + fromtimeOffset, err = period.Parse(row.FromtimeOffset) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + } + if row.Timespan != "" { + timespan, err = period.Parse(row.Timespan) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + } + migrationOffset, err := fromtimeOffset.Add(timespan) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[StinfoKey{ElemCode: row.ElemCode, TableName: row.TableName}] = migrationOffset + } + + return cache +} diff --git a/migrations/kdvh/import/cache/stinfosys.go b/migrations/kdvh/import/cache/stinfosys.go new file mode 100644 index 00000000..7316ff66 --- /dev/null +++ b/migrations/kdvh/import/cache/stinfosys.go @@ -0,0 +1,105 @@ +package cache + +import ( + "context" + "fmt" + "log/slog" + "os" + "slices" + "time" + + "github.com/jackc/pgx/v5" + + "migrate/kdvh/db" +) + +// Map of metadata used to query timeseries ID in LARD +type StinfoMap = map[StinfoKey]StinfoParam + +// StinfoKey is used for lookup of parameter offsets and metadata from Stinfosys +type StinfoKey struct { + ElemCode string + TableName string +} + +// Subset of StinfoQuery with only param info +type StinfoParam struct { + TypeID int32 + ParamID int32 + Hlevel *int32 + Sensor int32 + Fromtime time.Time + IsScalar bool +} + +// Struct holding query from Stinfosys elem_map_cfnames_param +type StinfoQuery struct { + ElemCode string `db:"elem_code"` + TableName string `db:"table_name"` + TypeID int32 `db:"typeid"` + ParamID int32 `db:"paramid"` + Hlevel *int32 `db:"hlevel"` + Sensor int32 `db:"sensor"` + Fromtime time.Time `db:"fromtime"` + IsScalar bool `db:"scalar"` +} + +func (q *StinfoQuery) toParam() StinfoParam { + return StinfoParam{ + TypeID: q.TypeID, + ParamID: q.ParamID, + Hlevel: q.Hlevel, + Sensor: q.Sensor, + Fromtime: q.Fromtime, + IsScalar: q.IsScalar, + } +} +func (q *StinfoQuery) toKey() StinfoKey { + return StinfoKey{q.ElemCode, q.TableName} +} + +// Save metadata for later use by quering Stinfosys +func cacheStinfo(tables, elements []string) StinfoMap { + cache := make(StinfoMap) + + fmt.Println("Connecting to Stinfosys to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("STINFO_STRING")) + if err != nil { + slog.Error("Could not connect to Stinfosys. Make sure to be connected to the VPN. " + err.Error()) + os.Exit(1) + } + defer conn.Close(context.TODO()) + + for _, table := range db.KDVH { + if tables != nil && !slices.Contains(tables, table.TableName) { + continue + } + // select paramid, elem_code, scalar from elem_map_cfnames_param join param using(paramid) where scalar = false + query := `SELECT elem_code, table_name, typeid, paramid, hlevel, sensor, fromtime, scalar + FROM elem_map_cfnames_param + JOIN param USING(paramid) + WHERE table_name = $1 + AND ($2::text[] IS NULL OR elem_code = ANY($2))` + + rows, err := conn.Query(context.TODO(), query, table.TableName, elements) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[StinfoQuery]) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, meta := range metas { + cache[meta.toKey()] = meta.toParam() + } + } + + return cache +} diff --git a/migrations/kdvh/import/convert_functions.go b/migrations/kdvh/import/convert_functions.go new file mode 100644 index 00000000..e9c2ac0a --- /dev/null +++ b/migrations/kdvh/import/convert_functions.go @@ -0,0 +1,345 @@ +package port + +import ( + "errors" + "strconv" + "time" + + "github.com/rickb777/period" + + "migrate/kdvh/db" + "migrate/kdvh/import/cache" + "migrate/lard" +) + +// The following ConvertFunctions try to recover the original pair of `controlinfo` +// and `useinfo` generated by Kvalobs for an observation, based on `Obs.Flags` and `Obs.Data` +// Different KDVH tables need different ways to perform this conversion (defined in CONV_MAP). +// +// It returns three structs for each of the lard tables we are inserting into +type ConvertFunction func(KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) + +// var CONV_MAP map[string]ConvertFunction = map[string]ConvertFunction{ +// "T_EDATA": ConvertEdata, +// "T_PDATA": ConvertPdata, +// "T_NDATA": ConvertNdata, +// "T_VDATA": ConvertVdata, +// "T_MONTH": ConvertProduct, +// "T_DIURNAL": ConvertProduct, +// "T_HOMOGEN_DIURNAL": ConvertProduct, +// "T_HOMOGEN_MONTH": ConvertProduct, +// "T_DIURNAL_INTERPOLATED": ConvertDiurnalInterpolated, +// } + +// func ConvertFunc(table *db.Table) ConvertFunction { +// fn, ok := CONV_MAP[table.TableName] +// if !ok { +// return Convert +// } +// return fn +// } + +func ConvertFunc(table *db.Table) ConvertFunction { + switch table.TableName { + case "T_EDATA": + return ConvertEdata + case "T_PDATA": + return ConvertPdata + case "T_NDATA": + return ConvertNdata + case "T_VDATA": + return ConvertVdata + case "T_MONTH", "T_DIURNAL", "T_HOMOGEN_DIURNAL", "T_HOMOGEN_MONTH": + return ConvertProduct + case "T_DIURNAL_INTERPOLATED": + return ConvertDiurnalInterpolated + } + return Convert +} + +type KdvhObs struct { + *cache.TsInfo + obstime time.Time + data string + flags string +} + +// Work around to return reference to consts +func addr[T any](t T) *T { + return &t +} + +func (obs *KdvhObs) flagsAreValid() bool { + if len(obs.flags) != 5 { + return false + } + _, err := strconv.ParseInt(obs.flags, 10, 32) + return err == nil +} + +func (obs *KdvhObs) Useinfo() *string { + if !obs.flagsAreValid() { + return addr(INVALID_FLAGS) + } + return addr(obs.flags + DELAY_DEFAULT) +} + +// Default ConvertFunction +// NOTE: this should be the only function that can return `lard.TextObs` with non-null text data. +func Convert(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + var valPtr *float32 + + controlinfo := VALUE_PASSED_QC + if obs.data == "" { + controlinfo = VALUE_MISSING + } + + val, err := strconv.ParseFloat(obs.data, 32) + if err == nil { + valPtr = addr(float32(val)) + } + + return lard.DataObs{ + Id: obs.Id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.Id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.Id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil +} + +// This function modifies obstimes to always use totime +// This is needed because KDVH used incorrect and incosistent timestamps +func ConvertProduct(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + data, text, flag, err := Convert(obs) + if !obs.Offset.IsZero() { + if temp, ok := obs.Offset.AddTo(data.Obstime); ok { + data.Obstime = temp + text.Obstime = temp + flag.Obstime = temp + } + } + return data, text, flag, err +} + +func ConvertEdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + switch obs.flags { + case "70381", "70389", "90989": + controlinfo = VALUE_REMOVED_BY_QC + default: + // Includes "70000", "70101", "99999" + controlinfo = VALUE_MISSING + } + } else { + controlinfo = VALUE_PASSED_QC + valPtr = addr(float32(val)) + } + + return lard.DataObs{ + Id: obs.Id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.Id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.Id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil +} + +func ConvertPdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + switch obs.flags { + case "20389", "30389", "40389", "50383", "70381", "71381": + controlinfo = VALUE_REMOVED_BY_QC + default: + // "00000", "10000", "10319", "30000", "30319", + // "40000", "40929", "48929", "48999", "50000", + // "50205", "60000", "70000", "70103", "70203", + // "71000", "71203", "90909", "99999" + controlinfo = VALUE_MISSING + } + } else { + valPtr = addr(float32(val)) + + switch obs.flags { + case "10319", "10329", "30319", "40319", "48929", "48999": + controlinfo = VALUE_MANUALLY_INTERPOLATED + case "20389", "30389", "40389", "50383", "70381", "71381", "99319": + controlinfo = VALUE_CORRECTED_AUTOMATICALLY + case "40929": + controlinfo = INTERPOLATION_ADDED_MANUALLY + default: + // "71000", "71203", "90909", "99999" + controlinfo = VALUE_PASSED_QC + } + } + + return lard.DataObs{ + Id: obs.Id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.Id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.Id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil +} + +func ConvertNdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + switch obs.flags { + case "70389": + controlinfo = VALUE_REMOVED_BY_QC + default: + // "30319", "38929", "40000", "40100", "40315" + // "40319", "43325", "48325", "49225", "49915" + // "70000", "70204", "71000", "73309", "78937" + // "90909", "93399", "98999", "99999" + controlinfo = VALUE_MISSING + } + } else { + valPtr = addr(float32(val)) + + switch obs.flags { + case "43325", "48325": + controlinfo = VALUE_MANUALLY_ASSIGNED + case "30319", "38929", "40315", "40319": + controlinfo = VALUE_MANUALLY_INTERPOLATED + case "49225", "49915": + controlinfo = INTERPOLATION_ADDED_MANUALLY + case "70389", "73309", "78937", "93399", "98999": + controlinfo = VALUE_CORRECTED_AUTOMATICALLY + default: + // "40000", "40100", "70000", "70204", "71000", "90909", "99999" + controlinfo = VALUE_PASSED_QC + } + } + + return lard.DataObs{ + Id: obs.Id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.Id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.Id, + Obstime: obs.obstime, + Useinfo: obs.Useinfo(), + Controlinfo: &controlinfo, + }, nil +} + +func ConvertVdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + var useinfo, controlinfo string + var valPtr *float32 + + // set useinfo based on time + if h := obs.obstime.Hour(); h == 0 || h == 6 || h == 12 || h == 18 { + useinfo = COMPLETED_HQC + } else { + useinfo = INVALID_FLAGS + } + + // set data and controlinfo + if val, err := strconv.ParseFloat(obs.data, 32); err != nil { + controlinfo = VALUE_MISSING + } else { + // super special treatment clause of T_VDATA.OT_24, so it will be the same as in kvalobs + // add custom offset, because OT_24 in KDVH has been treated differently than OT_24 in kvalobs + if obs.Element == "OT_24" { + offset, err := period.Parse("PT18H") // fromtime_offset -PT6H, timespan P1D + if err != nil { + return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, errors.New("could not parse period") + } + temp, ok := offset.AddTo(obs.obstime) + if !ok { + return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, errors.New("could not add period") + } + + obs.obstime = temp + // convert from hours to minutes + val *= 60.0 + } + + valPtr = addr(float32(val)) + controlinfo = VALUE_PASSED_QC + } + + return lard.DataObs{ + Id: obs.Id, + Obstime: obs.obstime, + Data: valPtr, + }, + lard.TextObs{ + Id: obs.Id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.Id, + Obstime: obs.obstime, + Useinfo: &useinfo, + Controlinfo: &controlinfo, + }, nil +} + +func ConvertDiurnalInterpolated(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { + val, err := strconv.ParseFloat(obs.data, 32) + if err != nil { + return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, err + } + + return lard.DataObs{ + Id: obs.Id, + Obstime: obs.obstime, + Data: addr(float32(val)), + }, + lard.TextObs{ + Id: obs.Id, + Obstime: obs.obstime, + Text: &obs.data, + }, + lard.Flag{ + Id: obs.Id, + Obstime: obs.obstime, + Useinfo: addr(DIURNAL_INTERPOLATED_USEINFO), + Controlinfo: addr(VALUE_MANUALLY_INTERPOLATED), + }, nil +} diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import/flags.go similarity index 55% rename from migrations/kdvh/import_functions.go rename to migrations/kdvh/import/flags.go index 27962366..8fdc511b 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import/flags.go @@ -1,13 +1,4 @@ -package kdvh - -import ( - "errors" - "strconv" - - "github.com/rickb777/period" - - "migrate/lard" -) +package port // In kvalobs a flag is a 16 char string containg QC information about the observation: // Note: Missing numbers in the following lists are marked as reserved (not in use I guess?) @@ -232,295 +223,3 @@ const ( COMPLETED_HQC = "40000" + DELAY_DEFAULT // Specific to T_VDATA DIURNAL_INTERPOLATED_USEINFO = "48925" + DELAY_DEFAULT // Specific to T_DIURNAL_INTERPOLATED ) - -// Work around to return reference to consts -func addr[T any](t T) *T { - return &t -} - -func (obs *KdvhObs) flagsAreValid() bool { - if len(obs.flags) != 5 { - return false - } - _, err := strconv.ParseInt(obs.flags, 10, 32) - return err == nil -} - -func (obs *KdvhObs) Useinfo() *string { - if !obs.flagsAreValid() { - return addr(INVALID_FLAGS) - } - return addr(obs.flags + DELAY_DEFAULT) -} - -// The following functions try to recover the original pair of `controlinfo` -// and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` -// Different KDVH tables need different ways to perform this conversion. - -// Default ConvertFunction -// NOTE: this should be the only function that can return `lard.TextObs` with non-null text data. -func Convert(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - var valPtr *float32 - - controlinfo := VALUE_PASSED_QC - if obs.data == "" { - controlinfo = VALUE_MISSING - } - - val, err := strconv.ParseFloat(obs.data, 32) - if err == nil { - f32 := float32(val) - valPtr = &f32 - } - - return lard.DataObs{ - Id: obs.id, - Obstime: obs.obstime, - Data: valPtr, - }, - lard.TextObs{ - Id: obs.id, - Obstime: obs.obstime, - Text: &obs.data, - }, - lard.Flag{ - Id: obs.id, - Obstime: obs.obstime, - Useinfo: obs.Useinfo(), - Controlinfo: &controlinfo, - }, nil -} - -// This function modifies obstimes to always use totime -// This is needed because KDVH used incorrect and incosistent timestamps -func ConvertProduct(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - data, text, flag, err := Convert(obs) - if !obs.offset.IsZero() { - if temp, ok := obs.offset.AddTo(data.Obstime); ok { - data.Obstime = temp - text.Obstime = temp - flag.Obstime = temp - } - } - return data, text, flag, err -} - -func ConvertEdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - var controlinfo string - var valPtr *float32 - - if val, err := strconv.ParseFloat(obs.data, 32); err != nil { - switch obs.flags { - case "70381", "70389", "90989": - controlinfo = VALUE_REMOVED_BY_QC - default: - // Includes "70000", "70101", "99999" - controlinfo = VALUE_MISSING - } - } else { - controlinfo = VALUE_PASSED_QC - f32 := float32(val) - valPtr = &f32 - } - - return lard.DataObs{ - Id: obs.id, - Obstime: obs.obstime, - Data: valPtr, - }, - lard.TextObs{ - Id: obs.id, - Obstime: obs.obstime, - Text: &obs.data, - }, - lard.Flag{ - Id: obs.id, - Obstime: obs.obstime, - Useinfo: obs.Useinfo(), - Controlinfo: &controlinfo, - }, nil -} - -func ConvertPdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - var controlinfo string - var valPtr *float32 - - if val, err := strconv.ParseFloat(obs.data, 32); err != nil { - switch obs.flags { - case "20389", "30389", "40389", "50383", "70381", "71381": - controlinfo = VALUE_REMOVED_BY_QC - default: - // "00000", "10000", "10319", "30000", "30319", - // "40000", "40929", "48929", "48999", "50000", - // "50205", "60000", "70000", "70103", "70203", - // "71000", "71203", "90909", "99999" - controlinfo = VALUE_MISSING - } - } else { - f32 := float32(val) - valPtr = &f32 - - switch obs.flags { - case "10319", "10329", "30319", "40319", "48929", "48999": - controlinfo = VALUE_MANUALLY_INTERPOLATED - case "20389", "30389", "40389", "50383", "70381", "71381", "99319": - controlinfo = VALUE_CORRECTED_AUTOMATICALLY - case "40929": - controlinfo = INTERPOLATION_ADDED_MANUALLY - default: - // "71000", "71203", "90909", "99999" - controlinfo = VALUE_PASSED_QC - } - - } - - return lard.DataObs{ - Id: obs.id, - Obstime: obs.obstime, - Data: valPtr, - }, - lard.TextObs{ - Id: obs.id, - Obstime: obs.obstime, - Text: &obs.data, - }, - lard.Flag{ - Id: obs.id, - Obstime: obs.obstime, - Useinfo: obs.Useinfo(), - Controlinfo: &controlinfo, - }, nil -} - -func ConvertNdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - var controlinfo string - var valPtr *float32 - - if val, err := strconv.ParseFloat(obs.data, 32); err != nil { - switch obs.flags { - case "70389": - controlinfo = VALUE_REMOVED_BY_QC - default: - // "30319", "38929", "40000", "40100", "40315" - // "40319", "43325", "48325", "49225", "49915" - // "70000", "70204", "71000", "73309", "78937" - // "90909", "93399", "98999", "99999" - controlinfo = VALUE_MISSING - } - } else { - switch obs.flags { - case "43325", "48325": - controlinfo = VALUE_MANUALLY_ASSIGNED - case "30319", "38929", "40315", "40319": - controlinfo = VALUE_MANUALLY_INTERPOLATED - case "49225", "49915": - controlinfo = INTERPOLATION_ADDED_MANUALLY - case "70389", "73309", "78937", "93399", "98999": - controlinfo = VALUE_CORRECTED_AUTOMATICALLY - default: - // "40000", "40100", "70000", "70204", "71000", "90909", "99999" - controlinfo = VALUE_PASSED_QC - } - f32 := float32(val) - valPtr = &f32 - } - - return lard.DataObs{ - Id: obs.id, - Obstime: obs.obstime, - Data: valPtr, - }, - lard.TextObs{ - Id: obs.id, - Obstime: obs.obstime, - Text: &obs.data, - }, - lard.Flag{ - Id: obs.id, - Obstime: obs.obstime, - Useinfo: obs.Useinfo(), - Controlinfo: &controlinfo, - }, nil -} - -func ConvertVdata(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - var useinfo, controlinfo string - var valPtr *float32 - - // set useinfo based on time - if h := obs.obstime.Hour(); h == 0 || h == 6 || h == 12 || h == 18 { - useinfo = COMPLETED_HQC - } else { - useinfo = INVALID_FLAGS - } - - // set data and controlinfo - if val, err := strconv.ParseFloat(obs.data, 32); err != nil { - controlinfo = VALUE_MISSING - } else { - // super special treatment clause of T_VDATA.OT_24, so it will be the same as in kvalobs - f32 := float32(val) - - if obs.element == "OT_24" { - // add custom offset, because OT_24 in KDVH has been treated differently than OT_24 in kvalobs - offset, err := period.Parse("PT18H") // fromtime_offset -PT6H, timespan P1D - if err != nil { - return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, errors.New("could not parse period") - } - temp, ok := offset.AddTo(obs.obstime) - if !ok { - return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, errors.New("could not add period") - } - - obs.obstime = temp - // convert from hours to minutes - f32 *= 60.0 - } - valPtr = &f32 - controlinfo = VALUE_PASSED_QC - } - - return lard.DataObs{ - Id: obs.id, - Obstime: obs.obstime, - Data: valPtr, - }, - lard.TextObs{ - Id: obs.id, - Obstime: obs.obstime, - Text: &obs.data, - }, - lard.Flag{ - Id: obs.id, - Obstime: obs.obstime, - Useinfo: &useinfo, - Controlinfo: &controlinfo, - }, nil -} - -// Specific conversionfunction for diurnal interpolated, -// with hardcoded useinfo and controlinfo -func ConvertDiurnalInterpolated(obs KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) { - val, err := strconv.ParseFloat(obs.data, 32) - if err != nil { - return lard.DataObs{}, lard.TextObs{}, lard.Flag{}, err - } - - f32 := float32(val) - return lard.DataObs{ - Id: obs.id, - Obstime: obs.obstime, - Data: &f32, - }, - lard.TextObs{ - Id: obs.id, - Obstime: obs.obstime, - Text: &obs.data, - }, - lard.Flag{ - Id: obs.id, - Obstime: obs.obstime, - Useinfo: addr(DIURNAL_INTERPOLATED_USEINFO), - Controlinfo: addr(VALUE_MANUALLY_INTERPOLATED), - }, nil -} diff --git a/migrations/kdvh/import/import.go b/migrations/kdvh/import/import.go new file mode 100644 index 00000000..9086b8f4 --- /dev/null +++ b/migrations/kdvh/import/import.go @@ -0,0 +1,220 @@ +package port + +import ( + "bufio" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "migrate/kdvh/db" + "migrate/kdvh/import/cache" + "migrate/lard" + "migrate/utils" +) + +// TODO: add CALL_SIGN? It's not in stinfosys? +var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} + +func ImportTable(table *db.Table, cache *cache.Cache, pool *pgxpool.Pool, config *Config) (rowsInserted int64) { + defer utils.SendEmailOnPanic("table.Import", config.Email) + + stations, err := os.ReadDir(filepath.Join(config.BaseDir, table.Path)) + if err != nil { + slog.Warn(err.Error()) + return 0 + } + + bar := utils.NewBar(len(stations), table.TableName) + bar.RenderBlank() + for _, station := range stations { + count, err := importStation(table, station, cache, pool, config) + if err == nil { + rowsInserted += count + } + bar.Add(1) + } + + outputStr := fmt.Sprintf("%v: %v total rows inserted", table.TableName, rowsInserted) + slog.Info(outputStr) + fmt.Println(outputStr) + + return rowsInserted +} + +// Loops over the element files present in the station directory and processes them concurrently +func importStation(table *db.Table, station os.DirEntry, cache *cache.Cache, pool *pgxpool.Pool, config *Config) (totRows int64, err error) { + stnr, err := getStationNumber(station, config.Stations) + if err != nil { + if config.Verbose { + slog.Info(err.Error()) + } + return 0, err + } + + dir := filepath.Join(config.BaseDir, table.Path, station.Name()) + elements, err := os.ReadDir(dir) + if err != nil { + slog.Warn(err.Error()) + return 0, err + } + + var wg sync.WaitGroup + for _, element := range elements { + elemCode, err := getElementCode(element, config.Elements) + if err != nil { + if config.Verbose { + slog.Info(err.Error()) + } + continue + } + + wg.Add(1) + go func() { + defer wg.Done() + + tsInfo, err := cache.NewTsInfo(table.TableName, elemCode, stnr, pool) + if err != nil { + return + } + + file, err := os.Open(filepath.Join(dir, element.Name())) + if err != nil { + slog.Warn(err.Error()) + return + } + defer file.Close() + + data, text, flag, err := parseData(file, tsInfo, table, config) + if err != nil { + return + } + + if len(data) == 0 { + slog.Info(tsInfo.Logstr + "no rows to insert (all obstimes > max import time)") + return + } + + var count int64 + if !(config.Skip == "data") { + if tsInfo.Param.IsScalar { + count, err = lard.InsertData(data, pool, tsInfo.Logstr) + if err != nil { + slog.Error(tsInfo.Logstr + "failed data bulk insertion - " + err.Error()) + return + } + } else { + count, err = lard.InsertTextData(text, pool, tsInfo.Logstr) + if err != nil { + slog.Error(tsInfo.Logstr + "failed non-scalar data bulk insertion - " + err.Error()) + return + } + // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data + // return count, nil + } + } + + if !(config.Skip == "flags") { + if err := lard.InsertFlags(flag, pool, tsInfo.Logstr); err != nil { + slog.Error(tsInfo.Logstr + "failed flag bulk insertion - " + err.Error()) + } + } + + totRows += count + }() + } + wg.Wait() + + return totRows, nil +} + +func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { + if !station.IsDir() { + return 0, errors.New(fmt.Sprintf("%s is not a directory, skipping", station.Name())) + } + + if stationList != nil && !slices.Contains(stationList, station.Name()) { + return 0, errors.New(fmt.Sprintf("Station %v not in the list, skipping", station.Name())) + } + + stnr, err := strconv.ParseInt(station.Name(), 10, 32) + if err != nil { + return 0, errors.New("Error parsing station number:" + err.Error()) + } + + return int32(stnr), nil +} + +func elemcodeIsInvalid(element string) bool { + return strings.Contains(element, "KOPI") || slices.Contains(INVALID_ELEMENTS, element) +} + +func getElementCode(element os.DirEntry, elementList []string) (string, error) { + elemCode := strings.ToUpper(strings.TrimSuffix(element.Name(), ".csv")) + + if elementList != nil && !slices.Contains(elementList, elemCode) { + return "", errors.New(fmt.Sprintf("Element '%s' not in the list, skipping", elemCode)) + } + + if elemcodeIsInvalid(elemCode) { + return "", errors.New(fmt.Sprintf("Element '%s' not set for import, skipping", elemCode)) + } + return elemCode, nil +} + +// Parses the observations in the CSV file, converts them with the table +// ConvertFunction and returns three arrays that can be passed to pgx.CopyFromRows +func parseData(handle *os.File, tsInfo *cache.TsInfo, table *db.Table, config *Config) ([][]any, [][]any, [][]any, error) { + scanner := bufio.NewScanner(handle) + + var rowCount int + // Try to infer row count from header + if config.HasHeader { + scanner.Scan() + rowCount, _ = strconv.Atoi(scanner.Text()) + } + + data := make([][]any, 0, rowCount) + text := make([][]any, 0, rowCount) + flag := make([][]any, 0, rowCount) + + convFunc := ConvertFunc(table) + + for scanner.Scan() { + cols := strings.Split(scanner.Text(), config.Sep) + + obsTime, err := time.Parse("2006-01-02_15:04:05", cols[0]) + if err != nil { + return nil, nil, nil, err + } + + // Only import data between KDVH's defined fromtime and totime + if tsInfo.Span.FromTime != nil && obsTime.Sub(*tsInfo.Span.FromTime) < 0 { + continue + } else if tsInfo.Span.ToTime != nil && obsTime.Sub(*tsInfo.Span.ToTime) > 0 { + break + } + + if table.MaxImportYearReached(obsTime.Year()) { + break + } + + dataRow, textRow, flagRow, err := convFunc(KdvhObs{tsInfo, obsTime, cols[1], cols[2]}) + if err != nil { + return nil, nil, nil, err + } + data = append(data, dataRow.ToRow()) + text = append(text, textRow.ToRow()) + flag = append(flag, flagRow.ToRow()) + } + + return data, text, flag, nil +} diff --git a/migrations/kdvh/import_test.go b/migrations/kdvh/import/import_test.go similarity index 98% rename from migrations/kdvh/import_test.go rename to migrations/kdvh/import/import_test.go index 6b7e5a28..d5f8eafb 100644 --- a/migrations/kdvh/import_test.go +++ b/migrations/kdvh/import/import_test.go @@ -1,4 +1,4 @@ -package kdvh +package port import "testing" diff --git a/migrations/kdvh/import/main.go b/migrations/kdvh/import/main.go new file mode 100644 index 00000000..b0e22927 --- /dev/null +++ b/migrations/kdvh/import/main.go @@ -0,0 +1,63 @@ +package port + +import ( + "context" + "fmt" + "log/slog" + "os" + "slices" + + "github.com/jackc/pgx/v5/pgxpool" + + "migrate/kdvh/db" + "migrate/kdvh/import/cache" + "migrate/utils" +) + +type Config struct { + Verbose bool `short:"v" description:"Increase verbosity level"` + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + Tables []string `short:"t" long:"table" delimiter:"," default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + Stations []string `short:"s" long:"station" delimiter:"," default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + Elements []string `short:"e" long:"elemcode" delimiter:"," default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Sep string `long:"sep" default:"," description:"Separator character in the dumped files. Needs to be quoted"` + HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` + Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` + Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` +} + +func (config *Config) Execute([]string) error { + if len(config.Sep) > 1 { + fmt.Printf("Error: '--sep' only accepts single-byte characters. Got %s", config.Sep) + os.Exit(1) + } + + // Cache metadata from Stinfosys, KDVH, and local `product_offsets.csv` + cache := cache.CacheMetadata(config.Tables, config.Stations, config.Elements) + + // Create connection pool for LARD + pool, err := pgxpool.New(context.TODO(), os.Getenv("LARD_STRING")) + if err != nil { + slog.Error(fmt.Sprint("Could not connect to Lard:", err)) + return err + } + defer pool.Close() + + for _, table := range db.KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + + if !table.ShouldImport() { + if config.Verbose { + slog.Info("Skipping import of " + table.TableName + " because this table is not set for import") + } + continue + } + + utils.SetLogFile(table.TableName, "import") + ImportTable(table, cache, pool, config) + } + + return nil +} diff --git a/migrations/kdvh/kdvh_test.go b/migrations/kdvh/kdvh_test.go new file mode 100644 index 00000000..19326b95 --- /dev/null +++ b/migrations/kdvh/kdvh_test.go @@ -0,0 +1,73 @@ +package kdvh + +import ( + "context" + "fmt" + "log" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "migrate/kdvh/db" + port "migrate/kdvh/import" + "migrate/kdvh/import/cache" +) + +const LARD_STRING string = "host=localhost user=postgres dbname=postgres password=postgres" + +type ImportTest struct { + table string + station int32 + elem string + expectedRows int64 +} + +func (t *ImportTest) mockConfig() (*port.Config, *cache.Cache) { + return &port.Config{ + Tables: []string{t.table}, + Stations: []string{fmt.Sprint(t.station)}, + Elements: []string{t.elem}, + BaseDir: "./tests", + HasHeader: true, + Sep: ";", + }, + &cache.Cache{ + StinfoMap: cache.StinfoMap{ + {ElemCode: t.elem, TableName: t.table}: { + Fromtime: time.Date(2001, 7, 1, 9, 0, 0, 0, time.UTC), + IsScalar: true, + }, + }, + } +} + +func TestImportKDVH(t *testing.T) { + log.SetFlags(log.LstdFlags | log.Lshortfile) + + pool, err := pgxpool.New(context.TODO(), LARD_STRING) + if err != nil { + t.Log("Could not connect to Lard:", err) + } + defer pool.Close() + + testCases := []ImportTest{ + {table: "T_MDATA", station: 12345, elem: "TA", expectedRows: 2644}, + } + + // TODO: test does not fail, if flags are not inserted + // TODO: bar does not work well with log print outs + for _, c := range testCases { + config, cache := c.mockConfig() + + table, ok := db.KDVH[c.table] + if !ok { + t.Fatal("Table does not exist in database") + } + + insertedRows := port.ImportTable(table, cache, pool, config) + if insertedRows != c.expectedRows { + t.Fail() + } + } +} diff --git a/migrations/kdvh/list_tables.go b/migrations/kdvh/list/main.go similarity index 63% rename from migrations/kdvh/list_tables.go rename to migrations/kdvh/list/main.go index c030a6c7..41a890e1 100644 --- a/migrations/kdvh/list_tables.go +++ b/migrations/kdvh/list/main.go @@ -1,17 +1,19 @@ -package kdvh +package list import ( "fmt" "slices" + + "migrate/kdvh/db" ) -type ListConfig struct{} +type Config struct{} -func (config *ListConfig) Execute(_ []string) error { +func (config *Config) Execute(_ []string) error { fmt.Println("Available tables in KDVH:") var tables []string - for table := range KDVH { + for table := range db.KDVH { tables = append(tables, table) } diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go index ed41472a..2ad6c06f 100644 --- a/migrations/kdvh/main.go +++ b/migrations/kdvh/main.go @@ -1,47 +1,14 @@ package kdvh +import ( + "migrate/kdvh/dump" + port "migrate/kdvh/import" + "migrate/kdvh/list" +) + // Command line arguments for KDVH migrations type Cmd struct { - Dump DumpConfig `command:"dump" description:"Dump tables from KDVH to CSV"` - Import ImportConfig `command:"import" description:"Import CSV file dumped from KDVH"` - List ListConfig `command:"list" description:"List available KDVH tables"` -} - -var KDVH map[string]*Table = map[string]*Table{ - // Section 1: tables that need to be migrated entirely - // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? - "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(ConvertEdata).SetImport(3000), - "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), - - // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 - "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImport(2006), - "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), - "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), - "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(ConvertPdata).SetImport(2006), - "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(ConvertNdata).SetImport(2006), - "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(ConvertVdata).SetImport(2006), - "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), - - // Section 3: tables that should only be dumped - "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), - "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), - "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), - "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), - "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), - "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), - "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), - - // Section 4: special cases, namely digitized historical data - "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(ConvertProduct).SetImport(1957), - "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(ConvertProduct).SetImport(2006), - "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(ConvertProduct), - "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(ConvertProduct), - - // Tables missing in the KDVH proxy: - // 1. these exist in a separate database - // - "T_AVINOR" - // - "T_PROJDATA" - // 2. these are not in active use and don't need to be imported in LARD - // - "T_DIURNAL_INTERPOLATED" - // - "T_MONTH_INTERPOLATED" + Dump dump.DumpConfig `command:"dump" description:"Dump tables from KDVH to CSV"` + Import port.Config `command:"import" description:"Import CSV file dumped from KDVH"` + List list.Config `command:"list" description:"List available KDVH tables"` } diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go deleted file mode 100644 index cc628de1..00000000 --- a/migrations/kdvh/table.go +++ /dev/null @@ -1,129 +0,0 @@ -package kdvh - -import ( - "errors" - "fmt" - "log/slog" - "migrate/lard" - "time" - - "github.com/jackc/pgx/v5/pgxpool" - "github.com/rickb777/period" -) - -// In KDVH for each table name we usually have three separate tables: -// 1. A DATA table containing observation values; -// 2. A FLAG table containing quality control (QC) flags; -// 3. A ELEM table containing metadata about the validity of the timeseries. -// -// DATA and FLAG tables have the same schema: -// | dato | stnr | ... | -// where 'dato' is the timestamp of the observation, 'stnr' is the station -// where the observation was measured, and '...' is a varying number of columns -// each with different observations, where the column name is the 'elem_code' -// (e.g. for air temperature, 'ta'). -// -// The ELEM tables have the following schema: -// | stnr | elem_code | fdato | tdato | table_name | flag_table_name | audit_dato - -// Table contains metadata on how to treat different tables in KDVH -type Table struct { - TableName string // Name of the DATA table - FlagTableName string // Name of the FLAG table - ElemTableName string // Name of the ELEM table - Path string // Directory name of where the dumped table is stored - dumpFunc DumpFunction // Function used to dump the KDVH table (found in `dump_functions.go`) - convFunc ConvertFunction // Function that converts KDVH obs to Lardobs (found in `import_functions.go`) - importUntil int // Import data only until the year specified by this field. If this field is not explicitly set, table import is skipped. -} - -// Implementation of these functions can be found in `dump_functions.go` -type DumpFunction func(path string, meta DumpMeta, pool *pgxpool.Pool) error -type DumpMeta struct { - element string - station string - dataTable string - flagTable string - overwrite bool - logStr string -} - -// Implementation of these functions can be found in `import_functions.go` -// It returns three structs for each of the lard tables we are inserting into -type ConvertFunction func(KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) -type KdvhObs struct { - *TimeseriesInfo - id int32 - obstime time.Time - data string - flags string -} - -// Convenience struct that holds information for a specific timeseries -type TimeseriesInfo struct { - station int32 - element string - offset period.Period - param StinfoParam - span Timespan - logstr string -} - -func (config *ImportConfig) NewTimeseriesInfo(table, element string, station int32) (*TimeseriesInfo, error) { - logstr := fmt.Sprintf("%v - %v - %v: ", table, station, element) - key := newKDVHKey(element, table, station) - - meta, ok := config.StinfoMap[key.Inner] - if !ok { - // TODO: should it fail here? How do we deal with data without metadata? - slog.Error(logstr + "Missing metadata in Stinfosys") - return nil, errors.New("") - } - - // No need to check for `!ok`, will default to 0 offset - offset := config.OffsetMap[key.Inner] - - // No need to check for `!ok`, timespan will be ignored if not in the map - span := config.KDVHMap[key] - - return &TimeseriesInfo{ - station: station, - element: element, - offset: offset, - param: meta, - span: span, - logstr: logstr, - }, nil -} - -// Creates default Table -func NewTable(data, flag, elem string) *Table { - return &Table{ - TableName: data, - FlagTableName: flag, - ElemTableName: elem, - Path: data + "_combined", // NOTE: '_combined' kept for backward compatibility with original scripts - dumpFunc: dumpDataAndFlags, - convFunc: Convert, - } -} - -// Sets the `ImportUntil` field if the year is greater than 0 -func (t *Table) SetImport(year int) *Table { - if year > 0 { - t.importUntil = year - } - return t -} - -// Sets the function used to dump the Table -func (t *Table) SetDumpFunc(fn DumpFunction) *Table { - t.dumpFunc = fn - return t -} - -// Sets the function used to convert observations from the table to Lardobservations -func (t *Table) SetConvFunc(fn ConvertFunction) *Table { - t.convFunc = fn - return t -} diff --git a/migrations/tests/T_MDATA_combined/12345/TA.csv b/migrations/kdvh/tests/T_MDATA_combined/12345/TA.csv similarity index 100% rename from migrations/tests/T_MDATA_combined/12345/TA.csv rename to migrations/kdvh/tests/T_MDATA_combined/12345/TA.csv diff --git a/migrations/kdvh_test.go b/migrations/kdvh_test.go deleted file mode 100644 index b7fbc2d0..00000000 --- a/migrations/kdvh_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/jackc/pgx/v5/pgxpool" - - "migrate/kdvh" -) - -const LARD_STRING string = "host=localhost user=postgres dbname=postgres password=postgres" - -func mockConfig(t *ImportTest) *kdvh.ImportConfig { - return &kdvh.ImportConfig{ - Tables: []string{t.table}, - Stations: []string{fmt.Sprint(t.station)}, - Elements: []string{t.elem}, - BaseDir: "./tests", - HasHeader: true, - Sep: ";", - StinfoMap: map[kdvh.StinfoKey]kdvh.StinfoParam{ - {ElemCode: t.elem, TableName: t.table}: { - TypeID: 501, - ParamID: 212, - Hlevel: nil, - Sensor: 0, - Fromtime: time.Date(2001, 7, 1, 9, 0, 0, 0, time.UTC), - IsScalar: true, - }, - }, - } -} - -type ImportTest struct { - table string - station int32 - elem string - expectedRows int64 -} - -func TestImportKDVH(t *testing.T) { - pool, err := pgxpool.New(context.TODO(), LARD_STRING) - if err != nil { - t.Log("Could not connect to Lard:", err) - } - defer pool.Close() - - testCases := []ImportTest{ - {table: "T_MDATA", station: 12345, elem: "TA", expectedRows: 2644}, - } - - for _, c := range testCases { - config := mockConfig(&c) - table, ok := kdvh.KDVH[c.table] - if !ok { - t.Fatal("Table does not exist in database") - } - - insertedRows := table.Import(pool, config) - if insertedRows != c.expectedRows { - t.Fail() - } - } -} diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go index ea333364..f76634a0 100644 --- a/migrations/utils/utils.go +++ b/migrations/utils/utils.go @@ -63,8 +63,8 @@ func SaveToFile(values []string, filename string) error { return file.Close() } -func SetLogFile(tableName, procedure string) { - filename := fmt.Sprintf("%s_%s_log.txt", tableName, procedure) +func SetLogFile(table, procedure string) { + filename := fmt.Sprintf("%s_%s_log.txt", table, procedure) fh, err := os.Create(filename) if err != nil { slog.Error(fmt.Sprintf("Could not create log '%s': %s", filename, err)) From efed05f55bf223f1b00867078a6ecff477f49657 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 15 Nov 2024 17:43:45 +0100 Subject: [PATCH 26/36] Add permission caching and rework cache package --- migrations/kdvh/import/cache/kdvh.go | 44 +++++------ migrations/kdvh/import/cache/main.go | 46 ++++++++--- migrations/kdvh/import/cache/permissions.go | 85 +++++++++++++++++++++ migrations/kdvh/import/cache/stinfosys.go | 70 ++++++----------- migrations/kdvh/kdvh_test.go | 2 +- 5 files changed, 166 insertions(+), 81 deletions(-) create mode 100644 migrations/kdvh/import/cache/permissions.go diff --git a/migrations/kdvh/import/cache/kdvh.go b/migrations/kdvh/import/cache/kdvh.go index 36a012c1..6319c630 100644 --- a/migrations/kdvh/import/cache/kdvh.go +++ b/migrations/kdvh/import/cache/kdvh.go @@ -32,23 +32,6 @@ type Timespan struct { ToTime *time.Time `db:"tdato"` } -// Struct used to deserialize KDVH query in cacheKDVH -type MetaKDVH struct { - ElemCode string `db:"elem_code"` - TableName string `db:"table_name"` - Station int32 `db:"stnr"` - FromTime *time.Time `db:"fdato"` - ToTime *time.Time `db:"tdato"` -} - -func (m *MetaKDVH) toTimespan() Timespan { - return Timespan{m.FromTime, m.ToTime} -} - -func (m *MetaKDVH) toKey() KDVHKey { - return KDVHKey{StinfoKey{ElemCode: m.ElemCode, TableName: m.TableName}, m.Station} -} - func cacheKDVH(tables, stations, elements []string) KDVHMap { cache := make(KDVHMap) @@ -82,15 +65,30 @@ func cacheKDVH(tables, stations, elements []string) KDVHMap { os.Exit(1) } - metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[MetaKDVH]) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) + for rows.Next() { + var key KDVHKey + var span Timespan + err := rows.Scan( + &key.Inner.TableName, + &key.Station, + &key.Inner.ElemCode, + &span.FromTime, + &span.ToTime, + ) + + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[key] = span } - for _, meta := range metas { - cache[meta.toKey()] = meta.toTimespan() + if rows.Err() != nil { + slog.Error(rows.Err().Error()) + os.Exit(1) } + } return cache diff --git a/migrations/kdvh/import/cache/main.go b/migrations/kdvh/import/cache/main.go index 3f06004c..81c18a14 100644 --- a/migrations/kdvh/import/cache/main.go +++ b/migrations/kdvh/import/cache/main.go @@ -1,10 +1,14 @@ package cache import ( + "context" "errors" "fmt" "log/slog" + "os" + "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/rickb777/period" @@ -12,20 +16,38 @@ import ( ) type Cache struct { - OffsetMap OffsetMap - StinfoMap StinfoMap - KDVHMap KDVHMap + Offsets OffsetMap + Stinfo StinfoMap + KDVH KDVHMap + ParamPermits ParamPermitMap + StationPermits StationPermitMap } -// TODO: cache permissions - // Caches all the metadata needed for import. // If any error occurs inside here the program will exit. func CacheMetadata(tables, stations, elements []string) *Cache { + fmt.Println("Connecting to Stinfosys to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("STINFO_STRING")) + if err != nil { + slog.Error("Could not connect to Stinfosys. Make sure to be connected to the VPN. " + err.Error()) + os.Exit(1) + } + + stinfoMeta := cacheStinfoMeta(tables, elements, conn) + stationPermits := cacheStationPermits(conn) + paramPermits := cacheParamPermits(conn) + + conn.Close(context.TODO()) + return &Cache{ - OffsetMap: cacheParamOffsets(), - StinfoMap: cacheStinfo(tables, elements), - KDVHMap: cacheKDVH(tables, stations, elements), + Stinfo: stinfoMeta, + StationPermits: stationPermits, + ParamPermits: paramPermits, + Offsets: cacheParamOffsets(), + KDVH: cacheKDVH(tables, stations, elements), } } @@ -44,7 +66,7 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo logstr := fmt.Sprintf("%v - %v - %v: ", table, station, element) key := newKDVHKey(element, table, station) - param, ok := cache.StinfoMap[key.Inner] + param, ok := cache.Stinfo[key.Inner] if !ok { // TODO: should it fail here? How do we deal with data without metadata? slog.Error(logstr + "Missing metadata in Stinfosys") @@ -52,10 +74,10 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo } // No need to check for `!ok`, will default to 0 offset - offset := cache.OffsetMap[key.Inner] + offset := cache.Offsets[key.Inner] // No need to check for `!ok`, timespan will be ignored if not in the map - span := cache.KDVHMap[key] + span := cache.KDVH[key] label := lard.Label{ StationID: station, @@ -70,6 +92,8 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo return nil, err } + // TODO: check if station is restricted + return &TsInfo{ Id: tsid, Station: station, diff --git a/migrations/kdvh/import/cache/permissions.go b/migrations/kdvh/import/cache/permissions.go new file mode 100644 index 00000000..2a53412f --- /dev/null +++ b/migrations/kdvh/import/cache/permissions.go @@ -0,0 +1,85 @@ +package cache + +import ( + "context" + "log/slog" + "os" + + "github.com/jackc/pgx/v5" +) + +type StationId = int32 +type PermitId = int32 + +type ParamPermitMap map[StationId][]ParamPermit +type StationPermitMap map[StationId]PermitId + +type ParamPermit struct { + TypeId int + ParamdId int + PermitId int +} + +func cacheParamPermits(conn *pgx.Conn) ParamPermitMap { + cache := make(ParamPermitMap) + + rows, err := conn.Query( + context.TODO(), + "SELECT stationid, message_formatid, paramid, permitid FROM v_station_param_policy", + ) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for rows.Next() { + var stnr StationId + var permit ParamPermit + + if err := rows.Scan(&stnr, &permit.TypeId, &permit.ParamdId, &permit.PermitId); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[stnr] = append(cache[stnr], permit) + } + + if rows.Err() != nil { + slog.Error(rows.Err().Error()) + os.Exit(1) + } + + return cache +} + +func cacheStationPermits(conn *pgx.Conn) StationPermitMap { + cache := make(StationPermitMap) + + rows, err := conn.Query( + context.TODO(), + "SELECT stationid, permitid FROM station_policy", + ) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for rows.Next() { + var stnr StationId + var permit PermitId + + if err := rows.Scan(&stnr, &permit); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[stnr] = permit + } + + if rows.Err() != nil { + slog.Error(rows.Err().Error()) + os.Exit(1) + } + + return cache +} diff --git a/migrations/kdvh/import/cache/stinfosys.go b/migrations/kdvh/import/cache/stinfosys.go index 7316ff66..2658ec5c 100644 --- a/migrations/kdvh/import/cache/stinfosys.go +++ b/migrations/kdvh/import/cache/stinfosys.go @@ -2,7 +2,6 @@ package cache import ( "context" - "fmt" "log/slog" "os" "slices" @@ -22,7 +21,7 @@ type StinfoKey struct { TableName string } -// Subset of StinfoQuery with only param info +// Subset of elem_map_cfnames_param query with only param info type StinfoParam struct { TypeID int32 ParamID int32 @@ -32,47 +31,10 @@ type StinfoParam struct { IsScalar bool } -// Struct holding query from Stinfosys elem_map_cfnames_param -type StinfoQuery struct { - ElemCode string `db:"elem_code"` - TableName string `db:"table_name"` - TypeID int32 `db:"typeid"` - ParamID int32 `db:"paramid"` - Hlevel *int32 `db:"hlevel"` - Sensor int32 `db:"sensor"` - Fromtime time.Time `db:"fromtime"` - IsScalar bool `db:"scalar"` -} - -func (q *StinfoQuery) toParam() StinfoParam { - return StinfoParam{ - TypeID: q.TypeID, - ParamID: q.ParamID, - Hlevel: q.Hlevel, - Sensor: q.Sensor, - Fromtime: q.Fromtime, - IsScalar: q.IsScalar, - } -} -func (q *StinfoQuery) toKey() StinfoKey { - return StinfoKey{q.ElemCode, q.TableName} -} - // Save metadata for later use by quering Stinfosys -func cacheStinfo(tables, elements []string) StinfoMap { +func cacheStinfoMeta(tables, elements []string, conn *pgx.Conn) StinfoMap { cache := make(StinfoMap) - fmt.Println("Connecting to Stinfosys to cache metadata") - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - conn, err := pgx.Connect(ctx, os.Getenv("STINFO_STRING")) - if err != nil { - slog.Error("Could not connect to Stinfosys. Make sure to be connected to the VPN. " + err.Error()) - os.Exit(1) - } - defer conn.Close(context.TODO()) - for _, table := range db.KDVH { if tables != nil && !slices.Contains(tables, table.TableName) { continue @@ -90,14 +52,30 @@ func cacheStinfo(tables, elements []string) StinfoMap { os.Exit(1) } - metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[StinfoQuery]) - if err != nil { - slog.Error(err.Error()) - os.Exit(1) + for rows.Next() { + var key StinfoKey + var param StinfoParam + err := rows.Scan( + &key.ElemCode, + &key.TableName, + ¶m.TypeID, + ¶m.ParamID, + ¶m.Hlevel, + ¶m.Sensor, + ¶m.Fromtime, + ¶m.IsScalar, + ) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[key] = param } - for _, meta := range metas { - cache[meta.toKey()] = meta.toParam() + if rows.Err() != nil { + slog.Error(rows.Err().Error()) + os.Exit(1) } } diff --git a/migrations/kdvh/kdvh_test.go b/migrations/kdvh/kdvh_test.go index 19326b95..f1e32e0c 100644 --- a/migrations/kdvh/kdvh_test.go +++ b/migrations/kdvh/kdvh_test.go @@ -33,7 +33,7 @@ func (t *ImportTest) mockConfig() (*port.Config, *cache.Cache) { Sep: ";", }, &cache.Cache{ - StinfoMap: cache.StinfoMap{ + Stinfo: cache.StinfoMap{ {ElemCode: t.elem, TableName: t.table}: { Fromtime: time.Date(2001, 7, 1, 9, 0, 0, 0, time.UTC), IsScalar: true, From 6143007e8028a7d03cc74442d31c8e9bfd8606eb Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 18 Nov 2024 09:57:55 +0100 Subject: [PATCH 27/36] Filter out restricted station/elements --- migrations/kdvh/import/cache/main.go | 6 +++++ migrations/kdvh/import/cache/permissions.go | 25 ++++++++++++++++++--- migrations/kdvh/import/import.go | 6 +++++ migrations/kdvh/kdvh_test.go | 7 +++++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/migrations/kdvh/import/cache/main.go b/migrations/kdvh/import/cache/main.go index 81c18a14..05526bf8 100644 --- a/migrations/kdvh/import/cache/main.go +++ b/migrations/kdvh/import/cache/main.go @@ -60,6 +60,7 @@ type TsInfo struct { Param StinfoParam Span Timespan Logstr string + IsOpen bool } func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpool.Pool) (*TsInfo, error) { @@ -73,6 +74,9 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo return nil, errors.New("") } + // Check if data for this station/element is restricted + isOpen := cache.timeseriesIsOpen(station, param.TypeID, param.ParamID) + // No need to check for `!ok`, will default to 0 offset offset := cache.Offsets[key.Inner] @@ -86,6 +90,7 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo Sensor: ¶m.Sensor, Level: param.Hlevel, } + tsid, err := lard.GetTimeseriesID(label, param.Fromtime, pool) if err != nil { slog.Error(logstr + "could not obtain timeseries - " + err.Error()) @@ -102,5 +107,6 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo Param: param, Span: span, Logstr: logstr, + IsOpen: isOpen, }, nil } diff --git a/migrations/kdvh/import/cache/permissions.go b/migrations/kdvh/import/cache/permissions.go index 2a53412f..a820226c 100644 --- a/migrations/kdvh/import/cache/permissions.go +++ b/migrations/kdvh/import/cache/permissions.go @@ -15,9 +15,9 @@ type ParamPermitMap map[StationId][]ParamPermit type StationPermitMap map[StationId]PermitId type ParamPermit struct { - TypeId int - ParamdId int - PermitId int + TypeId int32 + ParamdId int32 + PermitId int32 } func cacheParamPermits(conn *pgx.Conn) ParamPermitMap { @@ -83,3 +83,22 @@ func cacheStationPermits(conn *pgx.Conn) StationPermitMap { return cache } + +func (c *Cache) timeseriesIsOpen(stnr, typeid, paramid int32) bool { + // First check param permit table + if permits, ok := c.ParamPermits[stnr]; ok { + for _, permit := range permits { + if (permit.TypeId == 0 || permit.TypeId == typeid) && + (permit.ParamdId == 0 || permit.ParamdId == paramid) { + return permit.PermitId == 1 + } + } + } + + // Otherwise check station permit table + if permit, ok := c.StationPermits[stnr]; ok { + return permit == 1 + } + + return false +} diff --git a/migrations/kdvh/import/import.go b/migrations/kdvh/import/import.go index 9086b8f4..ae4a4377 100644 --- a/migrations/kdvh/import/import.go +++ b/migrations/kdvh/import/import.go @@ -86,6 +86,12 @@ func importStation(table *db.Table, station os.DirEntry, cache *cache.Cache, poo return } + // TODO: use this to choose which table to use on insert + if !tsInfo.IsOpen { + slog.Warn(tsInfo.Logstr + "Timeseries data is restricted") + return + } + file, err := os.Open(filepath.Join(dir, element.Name())) if err != nil { slog.Warn(err.Error()) diff --git a/migrations/kdvh/kdvh_test.go b/migrations/kdvh/kdvh_test.go index f1e32e0c..f33a886e 100644 --- a/migrations/kdvh/kdvh_test.go +++ b/migrations/kdvh/kdvh_test.go @@ -20,6 +20,7 @@ type ImportTest struct { table string station int32 elem string + permit int32 expectedRows int64 } @@ -39,6 +40,9 @@ func (t *ImportTest) mockConfig() (*port.Config, *cache.Cache) { IsScalar: true, }, }, + StationPermits: cache.StationPermitMap{ + t.station: t.permit, + }, } } @@ -52,7 +56,8 @@ func TestImportKDVH(t *testing.T) { defer pool.Close() testCases := []ImportTest{ - {table: "T_MDATA", station: 12345, elem: "TA", expectedRows: 2644}, + {table: "T_MDATA", station: 12345, elem: "TA", permit: 0, expectedRows: 0}, // restricted TS + {table: "T_MDATA", station: 12345, elem: "TA", permit: 1, expectedRows: 2644}, // open TS } // TODO: test does not fail, if flags are not inserted From d5d43992fa75feb4dedc325407736dd9c6657c1d Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 18 Nov 2024 10:32:19 +0100 Subject: [PATCH 28/36] Use struct instead of global map --- migrations/kdvh/db/main.go | 72 ++++++++++++----------- migrations/kdvh/dump/main.go | 3 +- migrations/kdvh/import/cache/kdvh.go | 4 +- migrations/kdvh/import/cache/main.go | 9 +-- migrations/kdvh/import/cache/stinfosys.go | 4 +- migrations/kdvh/import/main.go | 6 +- migrations/kdvh/kdvh_test.go | 4 +- migrations/kdvh/list/main.go | 4 +- 8 files changed, 60 insertions(+), 46 deletions(-) diff --git a/migrations/kdvh/db/main.go b/migrations/kdvh/db/main.go index fe14d2bc..81141221 100644 --- a/migrations/kdvh/db/main.go +++ b/migrations/kdvh/db/main.go @@ -1,41 +1,47 @@ package db // Map of all tables found in KDVH, with set max import year -var KDVH map[string]*Table = map[string]*Table{ - // Section 1: tables that need to be migrated entirely - // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? - "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetImportYear(3000), - "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetImportYear(3000), +type KDVH struct { + Tables map[string]*Table +} + +func Init() *KDVH { + return &KDVH{map[string]*Table{ + // Section 1: tables that need to be migrated entirely + // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetImportYear(3000), + "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetImportYear(3000), - // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 - "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImportYear(2006), - "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImportYear(2006), - "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImportYear(2006), - "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetImportYear(2006), - "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetImportYear(2006), - "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetImportYear(2006), - "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImportYear(2006), + // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 + "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetImportYear(2006), + "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImportYear(2006), - // Section 3: tables that should only be dumped - "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS"), - "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), - "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS"), - "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS"), - "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), - "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), - "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), + // Section 3: tables that should only be dumped + "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS"), + "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), + "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS"), + "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS"), + "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), + "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), + "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), - // Section 4: special cases, namely digitized historical data - "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetImportYear(1957), - "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetImportYear(2006), - "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH"), - "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", ""), + // Section 4: special cases, namely digitized historical data + "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetImportYear(1957), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetImportYear(2006), + "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH"), + "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", ""), - // Section 5: tables missing in the KDVH proxy: - // 1. these exist in a separate database - "T_AVINOR": NewTable("T_AVINOR", "T_AVINOR_FLAG", "T_ELEM_OBS"), - "T_PROJDATA": NewTable("T_PROJDATA", "T_PROJFLAG", "T_ELEM_PROJ"), - // 2. these are not in active use and don't need to be imported in LARD - "T_DIURNAL_INTERPOLATED": NewTable("T_DIURNAL_INTERPOLATED", "", ""), - "T_MONTH_INTERPOLATED": NewTable("T_MONTH_INTERPOLATED", "", ""), + // Section 5: tables missing in the KDVH proxy: + // 1. these exist in a separate database + "T_AVINOR": NewTable("T_AVINOR", "T_AVINOR_FLAG", "T_ELEM_OBS"), + "T_PROJDATA": NewTable("T_PROJDATA", "T_PROJFLAG", "T_ELEM_PROJ"), + // 2. these are not in active use and don't need to be imported in LARD + "T_DIURNAL_INTERPOLATED": NewTable("T_DIURNAL_INTERPOLATED", "", ""), + "T_MONTH_INTERPOLATED": NewTable("T_MONTH_INTERPOLATED", "", ""), + }} } diff --git a/migrations/kdvh/dump/main.go b/migrations/kdvh/dump/main.go index 3b94975d..4a88b75a 100644 --- a/migrations/kdvh/dump/main.go +++ b/migrations/kdvh/dump/main.go @@ -29,7 +29,8 @@ func (config *DumpConfig) Execute([]string) error { return nil } - for _, table := range db.KDVH { + kdvh := db.Init() + for _, table := range kdvh.Tables { if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { continue } diff --git a/migrations/kdvh/import/cache/kdvh.go b/migrations/kdvh/import/cache/kdvh.go index 6319c630..8165b505 100644 --- a/migrations/kdvh/import/cache/kdvh.go +++ b/migrations/kdvh/import/cache/kdvh.go @@ -32,7 +32,7 @@ type Timespan struct { ToTime *time.Time `db:"tdato"` } -func cacheKDVH(tables, stations, elements []string) KDVHMap { +func cacheKDVH(tables, stations, elements []string, kdvh *db.KDVH) KDVHMap { cache := make(KDVHMap) fmt.Println("Connecting to KDVH proxy to cache metadata") @@ -46,7 +46,7 @@ func cacheKDVH(tables, stations, elements []string) KDVHMap { } defer conn.Close(context.TODO()) - for _, t := range db.KDVH { + for _, t := range kdvh.Tables { if tables != nil && !slices.Contains(tables, t.TableName) { continue } diff --git a/migrations/kdvh/import/cache/main.go b/migrations/kdvh/import/cache/main.go index 05526bf8..13d153f0 100644 --- a/migrations/kdvh/import/cache/main.go +++ b/migrations/kdvh/import/cache/main.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/rickb777/period" + "migrate/kdvh/db" "migrate/lard" ) @@ -23,9 +24,9 @@ type Cache struct { StationPermits StationPermitMap } -// Caches all the metadata needed for import. +// Caches all the metadata needed for import of KDVH tables. // If any error occurs inside here the program will exit. -func CacheMetadata(tables, stations, elements []string) *Cache { +func CacheMetadata(tables, stations, elements []string, kdvh *db.KDVH) *Cache { fmt.Println("Connecting to Stinfosys to cache metadata") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -36,7 +37,7 @@ func CacheMetadata(tables, stations, elements []string) *Cache { os.Exit(1) } - stinfoMeta := cacheStinfoMeta(tables, elements, conn) + stinfoMeta := cacheStinfoMeta(tables, elements, kdvh, conn) stationPermits := cacheStationPermits(conn) paramPermits := cacheParamPermits(conn) @@ -47,7 +48,7 @@ func CacheMetadata(tables, stations, elements []string) *Cache { StationPermits: stationPermits, ParamPermits: paramPermits, Offsets: cacheParamOffsets(), - KDVH: cacheKDVH(tables, stations, elements), + KDVH: cacheKDVH(tables, stations, elements, kdvh), } } diff --git a/migrations/kdvh/import/cache/stinfosys.go b/migrations/kdvh/import/cache/stinfosys.go index 2658ec5c..3a0d6456 100644 --- a/migrations/kdvh/import/cache/stinfosys.go +++ b/migrations/kdvh/import/cache/stinfosys.go @@ -32,10 +32,10 @@ type StinfoParam struct { } // Save metadata for later use by quering Stinfosys -func cacheStinfoMeta(tables, elements []string, conn *pgx.Conn) StinfoMap { +func cacheStinfoMeta(tables, elements []string, kdvh *db.KDVH, conn *pgx.Conn) StinfoMap { cache := make(StinfoMap) - for _, table := range db.KDVH { + for _, table := range kdvh.Tables { if tables != nil && !slices.Contains(tables, table.TableName) { continue } diff --git a/migrations/kdvh/import/main.go b/migrations/kdvh/import/main.go index b0e22927..ec4b5043 100644 --- a/migrations/kdvh/import/main.go +++ b/migrations/kdvh/import/main.go @@ -32,8 +32,10 @@ func (config *Config) Execute([]string) error { os.Exit(1) } + kdvh := db.Init() + // Cache metadata from Stinfosys, KDVH, and local `product_offsets.csv` - cache := cache.CacheMetadata(config.Tables, config.Stations, config.Elements) + cache := cache.CacheMetadata(config.Tables, config.Stations, config.Elements, kdvh) // Create connection pool for LARD pool, err := pgxpool.New(context.TODO(), os.Getenv("LARD_STRING")) @@ -43,7 +45,7 @@ func (config *Config) Execute([]string) error { } defer pool.Close() - for _, table := range db.KDVH { + for _, table := range kdvh.Tables { if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { continue } diff --git a/migrations/kdvh/kdvh_test.go b/migrations/kdvh/kdvh_test.go index f33a886e..196f12c5 100644 --- a/migrations/kdvh/kdvh_test.go +++ b/migrations/kdvh/kdvh_test.go @@ -60,12 +60,14 @@ func TestImportKDVH(t *testing.T) { {table: "T_MDATA", station: 12345, elem: "TA", permit: 1, expectedRows: 2644}, // open TS } + kdvh := db.Init() + // TODO: test does not fail, if flags are not inserted // TODO: bar does not work well with log print outs for _, c := range testCases { config, cache := c.mockConfig() - table, ok := db.KDVH[c.table] + table, ok := kdvh.Tables[c.table] if !ok { t.Fatal("Table does not exist in database") } diff --git a/migrations/kdvh/list/main.go b/migrations/kdvh/list/main.go index 41a890e1..579d620f 100644 --- a/migrations/kdvh/list/main.go +++ b/migrations/kdvh/list/main.go @@ -12,8 +12,10 @@ type Config struct{} func (config *Config) Execute(_ []string) error { fmt.Println("Available tables in KDVH:") + kdvh := db.Init() + var tables []string - for table := range db.KDVH { + for table := range kdvh.Tables { tables = append(tables, table) } From 6a18c98dd35514b8c18b49858aa45a7e63b36974 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 18 Nov 2024 11:42:45 +0100 Subject: [PATCH 29/36] Update version of go-flags fork (fix bug if flag is not passed) --- migrations/go.mod | 2 +- migrations/go.sum | 2 ++ migrations/kdvh/dump/dump.go | 9 +++++++-- migrations/kdvh/dump/main.go | 4 ++-- migrations/kdvh/import/cache/kdvh.go | 6 +++--- migrations/kdvh/import/cache/stinfosys.go | 5 +++-- migrations/kdvh/import/import.go | 4 ++-- migrations/kdvh/import/main.go | 2 +- migrations/utils/email.go | 2 +- migrations/utils/utils.go | 2 +- 10 files changed, 23 insertions(+), 15 deletions(-) diff --git a/migrations/go.mod b/migrations/go.mod index 5deeb931..4153ee93 100644 --- a/migrations/go.mod +++ b/migrations/go.mod @@ -26,4 +26,4 @@ require ( golang.org/x/text v0.16.0 // indirect ) -replace github.com/jessevdk/go-flags => github.com/Lun4m/go-flags v0.0.0-20241113125827-68757125e949 +replace github.com/jessevdk/go-flags => github.com/Lun4m/go-flags v0.0.0-20241118100134-6375192b7985 diff --git a/migrations/go.sum b/migrations/go.sum index 2d0a2d0b..54140c04 100644 --- a/migrations/go.sum +++ b/migrations/go.sum @@ -1,5 +1,7 @@ github.com/Lun4m/go-flags v0.0.0-20241113125827-68757125e949 h1:7xyEGIr1X5alOjBjlNTDF+aRBcRIo60YX5sdlziLE5w= github.com/Lun4m/go-flags v0.0.0-20241113125827-68757125e949/go.mod h1:42/L0FDbP0qe91I+81tBqjU3uoz1tn1GDMZAhcCE2PE= +github.com/Lun4m/go-flags v0.0.0-20241118100134-6375192b7985 h1:eUA/sFZ1CtY9+9y/fPpUivYW8fJBlXqB4/8CjC+yXqk= +github.com/Lun4m/go-flags v0.0.0-20241118100134-6375192b7985/go.mod h1:42/L0FDbP0qe91I+81tBqjU3uoz1tn1GDMZAhcCE2PE= github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/migrations/kdvh/dump/dump.go b/migrations/kdvh/dump/dump.go index 2c91f642..31521374 100644 --- a/migrations/kdvh/dump/dump.go +++ b/migrations/kdvh/dump/dump.go @@ -22,6 +22,11 @@ var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} func DumpTable(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) { defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) + if err := os.MkdirAll(filepath.Join(config.BaseDir, table.Path), os.ModePerm); err != nil { + slog.Error(err.Error()) + return + } + elements, err := getElements(table, pool, config) if err != nil { return @@ -88,7 +93,7 @@ func getElements(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) ([]str filename := filepath.Join(config.BaseDir, table.Path, "elements.txt") if err := utils.SaveToFile(elements, filename); err != nil { - slog.Warn("Could not save element list to " + filename) + slog.Warn(err.Error()) } elements = utils.FilterSlice(config.Elements, elements, "") @@ -143,7 +148,7 @@ func getStations(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) ([]str filename := filepath.Join(config.BaseDir, table.Path, "stations.txt") if err := utils.SaveToFile(stations, filename); err != nil { - slog.Warn("Could not save element list to " + filename) + slog.Warn(err.Error()) } stations = utils.FilterSlice(config.Stations, stations, "") diff --git a/migrations/kdvh/dump/main.go b/migrations/kdvh/dump/main.go index 4a88b75a..6227b989 100644 --- a/migrations/kdvh/dump/main.go +++ b/migrations/kdvh/dump/main.go @@ -19,7 +19,7 @@ type DumpConfig struct { Elements []string `short:"e" delimiter:"," long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` - MaxConn int `long:"conns" default:"10" description:"Max number of concurrent connections allowed to KDVH"` + MaxConn int `short:"n" long:"conn" default:"4" description:"Max number of concurrent connections allowed to KDVH"` } func (config *DumpConfig) Execute([]string) error { @@ -31,7 +31,7 @@ func (config *DumpConfig) Execute([]string) error { kdvh := db.Init() for _, table := range kdvh.Tables { - if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + if len(config.Tables) > 0 && !slices.Contains(config.Tables, table.TableName) { continue } diff --git a/migrations/kdvh/import/cache/kdvh.go b/migrations/kdvh/import/cache/kdvh.go index 8165b505..ea2d77ab 100644 --- a/migrations/kdvh/import/cache/kdvh.go +++ b/migrations/kdvh/import/cache/kdvh.go @@ -47,15 +47,15 @@ func cacheKDVH(tables, stations, elements []string, kdvh *db.KDVH) KDVHMap { defer conn.Close(context.TODO()) for _, t := range kdvh.Tables { - if tables != nil && !slices.Contains(tables, t.TableName) { + if len(tables) > 0 && !slices.Contains(tables, t.TableName) { continue } // TODO: probably need to sanitize these inputs query := fmt.Sprintf( `SELECT table_name, stnr, elem_code, fdato, tdato FROM %s - WHERE ($1::bigint[] IS NULL OR stnr = ANY($1)) - AND ($2::text[] IS NULL OR elem_code = ANY($2))`, + WHERE ($1::bigint[] = '{}' OR stnr = ANY($1)) + AND ($2::text[] = '{}' OR elem_code = ANY($2))`, t.ElemTableName, ) diff --git a/migrations/kdvh/import/cache/stinfosys.go b/migrations/kdvh/import/cache/stinfosys.go index 3a0d6456..c6af589f 100644 --- a/migrations/kdvh/import/cache/stinfosys.go +++ b/migrations/kdvh/import/cache/stinfosys.go @@ -36,15 +36,16 @@ func cacheStinfoMeta(tables, elements []string, kdvh *db.KDVH, conn *pgx.Conn) S cache := make(StinfoMap) for _, table := range kdvh.Tables { - if tables != nil && !slices.Contains(tables, table.TableName) { + if len(tables) > 0 && !slices.Contains(tables, table.TableName) { continue } + // select paramid, elem_code, scalar from elem_map_cfnames_param join param using(paramid) where scalar = false query := `SELECT elem_code, table_name, typeid, paramid, hlevel, sensor, fromtime, scalar FROM elem_map_cfnames_param JOIN param USING(paramid) WHERE table_name = $1 - AND ($2::text[] IS NULL OR elem_code = ANY($2))` + AND ($2::text[] = '{}' OR elem_code = ANY($2))` rows, err := conn.Query(context.TODO(), query, table.TableName, elements) if err != nil { diff --git a/migrations/kdvh/import/import.go b/migrations/kdvh/import/import.go index ae4a4377..80d52607 100644 --- a/migrations/kdvh/import/import.go +++ b/migrations/kdvh/import/import.go @@ -147,7 +147,7 @@ func getStationNumber(station os.DirEntry, stationList []string) (int32, error) return 0, errors.New(fmt.Sprintf("%s is not a directory, skipping", station.Name())) } - if stationList != nil && !slices.Contains(stationList, station.Name()) { + if len(stationList) > 0 && !slices.Contains(stationList, station.Name()) { return 0, errors.New(fmt.Sprintf("Station %v not in the list, skipping", station.Name())) } @@ -166,7 +166,7 @@ func elemcodeIsInvalid(element string) bool { func getElementCode(element os.DirEntry, elementList []string) (string, error) { elemCode := strings.ToUpper(strings.TrimSuffix(element.Name(), ".csv")) - if elementList != nil && !slices.Contains(elementList, elemCode) { + if len(elementList) > 0 && !slices.Contains(elementList, elemCode) { return "", errors.New(fmt.Sprintf("Element '%s' not in the list, skipping", elemCode)) } diff --git a/migrations/kdvh/import/main.go b/migrations/kdvh/import/main.go index ec4b5043..23589b93 100644 --- a/migrations/kdvh/import/main.go +++ b/migrations/kdvh/import/main.go @@ -46,7 +46,7 @@ func (config *Config) Execute([]string) error { defer pool.Close() for _, table := range kdvh.Tables { - if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + if len(config.Tables) > 0 && !slices.Contains(config.Tables, table.TableName) { continue } diff --git a/migrations/utils/email.go b/migrations/utils/email.go index 17ba4885..9017d678 100644 --- a/migrations/utils/email.go +++ b/migrations/utils/email.go @@ -45,7 +45,7 @@ func sendEmail(subject, body string, to []string) { // send an email and resume the panic func SendEmailOnPanic(function string, recipients []string) { if r := recover(); r != nil { - if recipients != nil { + if len(recipients) > 0 { body := "KDVH importer was unable to finish successfully, and the error was not handled." + " This email is sent from a recover function triggered in " + function + diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go index f76634a0..31974362 100644 --- a/migrations/utils/utils.go +++ b/migrations/utils/utils.go @@ -33,7 +33,7 @@ func NewBar(size int, description string) *progressbar.ProgressBar { // formatMsg is an optional format string with a single format argument that can be used // to add context on why the element may be missing from the reference slice func FilterSlice[T comparable](slice, reference []T, formatMsg string) []T { - if slice == nil { + if len(slice) == 0 { return reference } From ea811bf6e327f39afe633ea4f20eaa7652ccfbfc Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 18 Nov 2024 13:08:14 +0100 Subject: [PATCH 30/36] Add possibility to remove indices before bulk insertion --- db/drop_indices.sql | 7 ++++++ db/flags.sql | 8 +++---- migrations/kdvh/import/main.go | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 db/drop_indices.sql diff --git a/db/drop_indices.sql b/db/drop_indices.sql new file mode 100644 index 00000000..1be451d6 --- /dev/null +++ b/db/drop_indices.sql @@ -0,0 +1,7 @@ +-- Remove indices before bulk insertion +DROP INDEX IF EXISTS data_timestamp_index, + data_timeseries_index, + nonscalar_data_timestamp_index, + nonscalar_data_timeseries_index, + old_flags_obtime_index, + old_flags_timeseries_index; diff --git a/db/flags.sql b/db/flags.sql index 0d61629b..b58c329a 100644 --- a/db/flags.sql +++ b/db/flags.sql @@ -11,7 +11,6 @@ CREATE TABLE IF NOT EXISTS flags.kvdata ( cfailed INT4 NULL, CONSTRAINT unique_kvdata_timeseries_obstime UNIQUE (timeseries, obstime) ); - CREATE INDEX IF NOT EXISTS kvdata_obtime_index ON flags.kvdata (obstime); CREATE INDEX IF NOT EXISTS kvdata_timeseries_index ON flags.kvdata USING HASH (timeseries); @@ -22,8 +21,7 @@ CREATE TABLE IF NOT EXISTS flags.old_databases ( controlinfo TEXT NULL, useinfo TEXT NULL, cfailed TEXT NULL , - CONSTRAINT unique_kdvh_timeseries_obstime UNIQUE (timeseries, obstime) + CONSTRAINT unique_old_flags_timeseries_obstime UNIQUE (timeseries, obstime) ); - -CREATE INDEX IF NOT EXISTS kdvh_obtime_index ON flags.old_databases (obstime); -CREATE INDEX IF NOT EXISTS kdvh_timeseries_index ON flags.old_databases USING HASH (timeseries); +CREATE INDEX IF NOT EXISTS old_flags_obtime_index ON flags.old_databases (obstime); +CREATE INDEX IF NOT EXISTS old_flags_timeseries_index ON flags.old_databases USING HASH (timeseries); diff --git a/migrations/kdvh/import/main.go b/migrations/kdvh/import/main.go index 23589b93..40b88a76 100644 --- a/migrations/kdvh/import/main.go +++ b/migrations/kdvh/import/main.go @@ -24,6 +24,7 @@ type Config struct { HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` Email []string `long:"email" delimiter:"," description:"Optional comma separated list of email addresses used to notify if the program crashed"` + Reindex bool `long:"reindex" description:"Drops PG indices before insertion. Might improve performance"` } func (config *Config) Execute([]string) error { @@ -45,6 +46,10 @@ func (config *Config) Execute([]string) error { } defer pool.Close() + if config.Reindex { + dropIndices(pool) + } + for _, table := range kdvh.Tables { if len(config.Tables) > 0 && !slices.Contains(config.Tables, table.TableName) { continue @@ -61,5 +66,40 @@ func (config *Config) Execute([]string) error { ImportTable(table, cache, pool, config) } + if config.Reindex { + createIndices(pool) + } + return nil } + +func dropIndices(pool *pgxpool.Pool) { + fmt.Println("Dropping table indices...") + + file, err := os.ReadFile("../db/drop_indices.sql") + if err != nil { + panic(err.Error()) + } + + _, err = pool.Exec(context.Background(), string(file)) + if err != nil { + panic(err.Error()) + } +} + +func createIndices(pool *pgxpool.Pool) { + fmt.Println("Recreating table indices...") + + files := []string{"../db/public.sql", "../db/flags.sql"} + for _, filename := range files { + file, err := os.ReadFile(filename) + if err != nil { + panic(err.Error()) + } + + _, err = pool.Exec(context.Background(), string(file)) + if err != nil { + panic(err.Error()) + } + } +} From e497c961715b5fd00daaaf3e397ece5e1019fd9f Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 18 Nov 2024 13:33:54 +0100 Subject: [PATCH 31/36] Add a couple more log statements --- migrations/kdvh/import/cache/kdvh.go | 2 +- migrations/kdvh/import/cache/main.go | 2 +- migrations/kdvh/import/main.go | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/migrations/kdvh/import/cache/kdvh.go b/migrations/kdvh/import/cache/kdvh.go index ea2d77ab..0ca938cd 100644 --- a/migrations/kdvh/import/cache/kdvh.go +++ b/migrations/kdvh/import/cache/kdvh.go @@ -35,7 +35,7 @@ type Timespan struct { func cacheKDVH(tables, stations, elements []string, kdvh *db.KDVH) KDVHMap { cache := make(KDVHMap) - fmt.Println("Connecting to KDVH proxy to cache metadata") + slog.Info("Connecting to KDVH proxy to cache metadata") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() diff --git a/migrations/kdvh/import/cache/main.go b/migrations/kdvh/import/cache/main.go index 13d153f0..ed877e63 100644 --- a/migrations/kdvh/import/cache/main.go +++ b/migrations/kdvh/import/cache/main.go @@ -27,7 +27,7 @@ type Cache struct { // Caches all the metadata needed for import of KDVH tables. // If any error occurs inside here the program will exit. func CacheMetadata(tables, stations, elements []string, kdvh *db.KDVH) *Cache { - fmt.Println("Connecting to Stinfosys to cache metadata") + slog.Info("Connecting to Stinfosys to cache metadata") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() diff --git a/migrations/kdvh/import/main.go b/migrations/kdvh/import/main.go index 40b88a76..b1be2206 100644 --- a/migrations/kdvh/import/main.go +++ b/migrations/kdvh/import/main.go @@ -3,6 +3,7 @@ package port import ( "context" "fmt" + "log" "log/slog" "os" "slices" @@ -33,6 +34,7 @@ func (config *Config) Execute([]string) error { os.Exit(1) } + slog.Info("Import started!") kdvh := db.Init() // Cache metadata from Stinfosys, KDVH, and local `product_offsets.csv` @@ -66,15 +68,17 @@ func (config *Config) Execute([]string) error { ImportTable(table, cache, pool, config) } + log.SetOutput(os.Stdout) if config.Reindex { createIndices(pool) } + slog.Info("Import complete!") return nil } func dropIndices(pool *pgxpool.Pool) { - fmt.Println("Dropping table indices...") + slog.Info("Dropping table indices...") file, err := os.ReadFile("../db/drop_indices.sql") if err != nil { @@ -88,7 +92,7 @@ func dropIndices(pool *pgxpool.Pool) { } func createIndices(pool *pgxpool.Pool) { - fmt.Println("Recreating table indices...") + slog.Info("Recreating table indices...") files := []string{"../db/public.sql", "../db/flags.sql"} for _, filename := range files { From 0d69485cc8304f8aac877056042bd305de113796 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Mon, 18 Nov 2024 17:24:16 +0100 Subject: [PATCH 32/36] Remove email code --- migrations/kdvh/dump/dump.go | 2 -- migrations/kdvh/import/import.go | 2 -- migrations/utils/email.go | 60 -------------------------------- 3 files changed, 64 deletions(-) delete mode 100644 migrations/utils/email.go diff --git a/migrations/kdvh/dump/dump.go b/migrations/kdvh/dump/dump.go index 31521374..b47ab3c3 100644 --- a/migrations/kdvh/dump/dump.go +++ b/migrations/kdvh/dump/dump.go @@ -20,8 +20,6 @@ import ( var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} func DumpTable(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) { - defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) - if err := os.MkdirAll(filepath.Join(config.BaseDir, table.Path), os.ModePerm); err != nil { slog.Error(err.Error()) return diff --git a/migrations/kdvh/import/import.go b/migrations/kdvh/import/import.go index 80d52607..66ffdfb1 100644 --- a/migrations/kdvh/import/import.go +++ b/migrations/kdvh/import/import.go @@ -25,8 +25,6 @@ import ( var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} func ImportTable(table *db.Table, cache *cache.Cache, pool *pgxpool.Pool, config *Config) (rowsInserted int64) { - defer utils.SendEmailOnPanic("table.Import", config.Email) - stations, err := os.ReadDir(filepath.Join(config.BaseDir, table.Path)) if err != nil { slog.Warn(err.Error()) diff --git a/migrations/utils/email.go b/migrations/utils/email.go deleted file mode 100644 index 9017d678..00000000 --- a/migrations/utils/email.go +++ /dev/null @@ -1,60 +0,0 @@ -package utils - -import ( - "encoding/base64" - "fmt" - "log/slog" - "net/smtp" - "os" - "runtime/debug" - "strings" -) - -func sendEmail(subject, body string, to []string) { - // server and from/to - host := "aspmx.l.google.com" - port := "25" - from := "oda-noreply@met.no" - - // add stuff to headers and make the message body - header := make(map[string]string) - header["From"] = from - header["To"] = strings.Join(to, ",") - header["Subject"] = subject - header["MIME-Version"] = "1.0" - header["Content-Type"] = "text/plain; charset=\"utf-8\"" - header["Content-Transfer-Encoding"] = "base64" - message := "" - for k, v := range header { - message += fmt.Sprintf("%s: %s\r\n", k, v) - } - - body = body + "\n\n" + fmt.Sprintf("Ran with the following command:\n%s", strings.Join(os.Args, " ")) - message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) - - // send the email - err := smtp.SendMail(host+":"+port, nil, from, to, []byte(message)) - if err != nil { - slog.Error(err.Error()) - return - } - slog.Info("Email sent successfully!") -} - -// TODO: modify this to be more flexible -// send an email and resume the panic -func SendEmailOnPanic(function string, recipients []string) { - if r := recover(); r != nil { - if len(recipients) > 0 { - body := "KDVH importer was unable to finish successfully, and the error was not handled." + - " This email is sent from a recover function triggered in " + - function + - ".\n\nError message:" + - fmt.Sprint(r) + - "\n\nStack trace:\n\n" + - string(debug.Stack()) - sendEmail("LARD – KDVH importer panicked", body, recipients) - } - panic(r) - } -} From b80c8113bab7d0b5cbe23b6b8c6079abcabe6b77 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 20 Nov 2024 11:14:47 +0100 Subject: [PATCH 33/36] Simplify ImportTable --- migrations/kdvh/import/cache/main.go | 8 +- migrations/kdvh/import/convert_functions.go | 22 +-- migrations/kdvh/import/import.go | 159 +++++++++----------- 3 files changed, 80 insertions(+), 109 deletions(-) diff --git a/migrations/kdvh/import/cache/main.go b/migrations/kdvh/import/cache/main.go index ed877e63..243c6f6a 100644 --- a/migrations/kdvh/import/cache/main.go +++ b/migrations/kdvh/import/cache/main.go @@ -72,12 +72,18 @@ func (cache *Cache) NewTsInfo(table, element string, station int32, pool *pgxpoo if !ok { // TODO: should it fail here? How do we deal with data without metadata? slog.Error(logstr + "Missing metadata in Stinfosys") - return nil, errors.New("") + return nil, errors.New("No metadata") } // Check if data for this station/element is restricted isOpen := cache.timeseriesIsOpen(station, param.TypeID, param.ParamID) + // TODO: eventually use this to choose which table to use on insert + if !isOpen { + slog.Warn(logstr + "Timeseries data is restricted") + return nil, errors.New("Restricted data") + } + // No need to check for `!ok`, will default to 0 offset offset := cache.Offsets[key.Inner] diff --git a/migrations/kdvh/import/convert_functions.go b/migrations/kdvh/import/convert_functions.go index e9c2ac0a..c0dcf881 100644 --- a/migrations/kdvh/import/convert_functions.go +++ b/migrations/kdvh/import/convert_functions.go @@ -19,27 +19,7 @@ import ( // It returns three structs for each of the lard tables we are inserting into type ConvertFunction func(KdvhObs) (lard.DataObs, lard.TextObs, lard.Flag, error) -// var CONV_MAP map[string]ConvertFunction = map[string]ConvertFunction{ -// "T_EDATA": ConvertEdata, -// "T_PDATA": ConvertPdata, -// "T_NDATA": ConvertNdata, -// "T_VDATA": ConvertVdata, -// "T_MONTH": ConvertProduct, -// "T_DIURNAL": ConvertProduct, -// "T_HOMOGEN_DIURNAL": ConvertProduct, -// "T_HOMOGEN_MONTH": ConvertProduct, -// "T_DIURNAL_INTERPOLATED": ConvertDiurnalInterpolated, -// } - -// func ConvertFunc(table *db.Table) ConvertFunction { -// fn, ok := CONV_MAP[table.TableName] -// if !ok { -// return Convert -// } -// return fn -// } - -func ConvertFunc(table *db.Table) ConvertFunction { +func getConvertFunc(table *db.Table) ConvertFunction { switch table.TableName { case "T_EDATA": return ConvertEdata diff --git a/migrations/kdvh/import/import.go b/migrations/kdvh/import/import.go index 66ffdfb1..dd48fbf0 100644 --- a/migrations/kdvh/import/import.go +++ b/migrations/kdvh/import/import.go @@ -31,43 +31,12 @@ func ImportTable(table *db.Table, cache *cache.Cache, pool *pgxpool.Pool, config return 0 } + convFunc := getConvertFunc(table) + bar := utils.NewBar(len(stations), table.TableName) bar.RenderBlank() for _, station := range stations { - count, err := importStation(table, station, cache, pool, config) - if err == nil { - rowsInserted += count - } - bar.Add(1) - } - - outputStr := fmt.Sprintf("%v: %v total rows inserted", table.TableName, rowsInserted) - slog.Info(outputStr) - fmt.Println(outputStr) - - return rowsInserted -} - -// Loops over the element files present in the station directory and processes them concurrently -func importStation(table *db.Table, station os.DirEntry, cache *cache.Cache, pool *pgxpool.Pool, config *Config) (totRows int64, err error) { - stnr, err := getStationNumber(station, config.Stations) - if err != nil { - if config.Verbose { - slog.Info(err.Error()) - } - return 0, err - } - - dir := filepath.Join(config.BaseDir, table.Path, station.Name()) - elements, err := os.ReadDir(dir) - if err != nil { - slog.Warn(err.Error()) - return 0, err - } - - var wg sync.WaitGroup - for _, element := range elements { - elemCode, err := getElementCode(element, config.Elements) + stnr, err := getStationNumber(station, config.Stations) if err != nil { if config.Verbose { slog.Info(err.Error()) @@ -75,69 +44,75 @@ func importStation(table *db.Table, station os.DirEntry, cache *cache.Cache, poo continue } - wg.Add(1) - go func() { - defer wg.Done() + dir := filepath.Join(config.BaseDir, table.Path, station.Name()) + elements, err := os.ReadDir(dir) + if err != nil { + slog.Warn(err.Error()) + continue + } - tsInfo, err := cache.NewTsInfo(table.TableName, elemCode, stnr, pool) + var wg sync.WaitGroup + for _, element := range elements { + elemCode, err := getElementCode(element, config.Elements) if err != nil { - return + if config.Verbose { + slog.Info(err.Error()) + } + continue } - // TODO: use this to choose which table to use on insert - if !tsInfo.IsOpen { - slog.Warn(tsInfo.Logstr + "Timeseries data is restricted") - return - } + wg.Add(1) + go func() { + defer wg.Done() - file, err := os.Open(filepath.Join(dir, element.Name())) - if err != nil { - slog.Warn(err.Error()) - return - } - defer file.Close() - - data, text, flag, err := parseData(file, tsInfo, table, config) - if err != nil { - return - } + tsInfo, err := cache.NewTsInfo(table.TableName, elemCode, stnr, pool) + if err != nil { + return + } - if len(data) == 0 { - slog.Info(tsInfo.Logstr + "no rows to insert (all obstimes > max import time)") - return - } + filename := filepath.Join(dir, element.Name()) + data, text, flag, err := parseData(filename, tsInfo, convFunc, table, config) + if err != nil { + return + } - var count int64 - if !(config.Skip == "data") { - if tsInfo.Param.IsScalar { - count, err = lard.InsertData(data, pool, tsInfo.Logstr) - if err != nil { - slog.Error(tsInfo.Logstr + "failed data bulk insertion - " + err.Error()) - return + var count int64 + if !(config.Skip == "data") { + if tsInfo.Param.IsScalar { + count, err = lard.InsertData(data, pool, tsInfo.Logstr) + if err != nil { + slog.Error(tsInfo.Logstr + "failed data bulk insertion - " + err.Error()) + return + } + } else { + count, err = lard.InsertTextData(text, pool, tsInfo.Logstr) + if err != nil { + slog.Error(tsInfo.Logstr + "failed non-scalar data bulk insertion - " + err.Error()) + return + } + // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data + // return count, nil } - } else { - count, err = lard.InsertTextData(text, pool, tsInfo.Logstr) - if err != nil { - slog.Error(tsInfo.Logstr + "failed non-scalar data bulk insertion - " + err.Error()) - return - } - // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data - // return count, nil } - } - if !(config.Skip == "flags") { - if err := lard.InsertFlags(flag, pool, tsInfo.Logstr); err != nil { - slog.Error(tsInfo.Logstr + "failed flag bulk insertion - " + err.Error()) + if !(config.Skip == "flags") { + if err := lard.InsertFlags(flag, pool, tsInfo.Logstr); err != nil { + slog.Error(tsInfo.Logstr + "failed flag bulk insertion - " + err.Error()) + } } - } - totRows += count - }() + rowsInserted += count + }() + } + wg.Wait() + bar.Add(1) } - wg.Wait() - return totRows, nil + outputStr := fmt.Sprintf("%v: %v total rows inserted", table.TableName, rowsInserted) + slog.Info(outputStr) + fmt.Println(outputStr) + + return rowsInserted } func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { @@ -176,8 +151,15 @@ func getElementCode(element os.DirEntry, elementList []string) (string, error) { // Parses the observations in the CSV file, converts them with the table // ConvertFunction and returns three arrays that can be passed to pgx.CopyFromRows -func parseData(handle *os.File, tsInfo *cache.TsInfo, table *db.Table, config *Config) ([][]any, [][]any, [][]any, error) { - scanner := bufio.NewScanner(handle) +func parseData(filename string, tsInfo *cache.TsInfo, convFunc ConvertFunction, table *db.Table, config *Config) ([][]any, [][]any, [][]any, error) { + file, err := os.Open(filename) + if err != nil { + slog.Warn(err.Error()) + return nil, nil, nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) var rowCount int // Try to infer row count from header @@ -190,8 +172,6 @@ func parseData(handle *os.File, tsInfo *cache.TsInfo, table *db.Table, config *C text := make([][]any, 0, rowCount) flag := make([][]any, 0, rowCount) - convFunc := ConvertFunc(table) - for scanner.Scan() { cols := strings.Split(scanner.Text(), config.Sep) @@ -220,5 +200,10 @@ func parseData(handle *os.File, tsInfo *cache.TsInfo, table *db.Table, config *C flag = append(flag, flagRow.ToRow()) } + if len(data) == 0 { + slog.Info(tsInfo.Logstr + "no rows to insert (all obstimes > max import time)") + return nil, nil, nil, errors.New("No rows to insert") + } + return data, text, flag, nil } From 3c7456d3cf7dbe5ae043b3b79fd0d96818480d3f Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 20 Nov 2024 11:15:08 +0100 Subject: [PATCH 34/36] Update func name to be symmetric with import --- migrations/kdvh/dump/dump.go | 2 +- migrations/kdvh/dump/dump_functions.go | 19 +------------------ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/migrations/kdvh/dump/dump.go b/migrations/kdvh/dump/dump.go index b47ab3c3..23898e61 100644 --- a/migrations/kdvh/dump/dump.go +++ b/migrations/kdvh/dump/dump.go @@ -35,7 +35,7 @@ func DumpTable(table *db.Table, pool *pgxpool.Pool, config *DumpConfig) { return } - dumpFunc := DumpFunc(table) + dumpFunc := getDumpFunc(table) // Used to limit connections to the database semaphore := make(chan struct{}, config.MaxConn) diff --git a/migrations/kdvh/dump/dump_functions.go b/migrations/kdvh/dump/dump_functions.go index 6fbb24bd..db6fb82f 100644 --- a/migrations/kdvh/dump/dump_functions.go +++ b/migrations/kdvh/dump/dump_functions.go @@ -25,24 +25,7 @@ type DumpArgs struct { logStr string } -// var DUMP_MAP map[string]DumpFunction = map[string]DumpFunction{ -// "T_METARDATA": dumpDataOnly, -// "T_HOMOGEN_DIURNAL": dumpDataOnly, -// "T_SECOND_DATA": dumpByYear, -// "T_MINUTE_DATA": dumpByYear, -// "T_10MINUTE_DATA": dumpByYear, -// "T_HOMOGEN_MONTH": dumpHomogenMonth, -// } - -// func DumpFunc(table *db.Table) DumpFunction { -// fn, ok := DUMP_MAP[table.TableName] -// if !ok { -// return dumpDataAndFlags -// } -// return fn -// } - -func DumpFunc(table *db.Table) DumpFunction { +func getDumpFunc(table *db.Table) DumpFunction { switch table.TableName { case "T_METARDATA", "T_HOMOGEN_DIURNAL": return dumpDataOnly From f3216e1de0c8e453179690d38721a0d4f2de32ff Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 20 Nov 2024 14:55:17 +0100 Subject: [PATCH 35/36] Defer createIndices --- migrations/kdvh/import/main.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/migrations/kdvh/import/main.go b/migrations/kdvh/import/main.go index b1be2206..e45f9dd9 100644 --- a/migrations/kdvh/import/main.go +++ b/migrations/kdvh/import/main.go @@ -52,6 +52,18 @@ func (config *Config) Execute([]string) error { dropIndices(pool) } + // Recreate indices even in case the main function panics + defer func() { + r := recover() + if config.Reindex { + createIndices(pool) + } + + if r != nil { + panic(r) + } + }() + for _, table := range kdvh.Tables { if len(config.Tables) > 0 && !slices.Contains(config.Tables, table.TableName) { continue @@ -69,10 +81,6 @@ func (config *Config) Execute([]string) error { } log.SetOutput(os.Stdout) - if config.Reindex { - createIndices(pool) - } - slog.Info("Import complete!") return nil } From c6365be9fd533498c7e16b4b4039979a85c13cb1 Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Wed, 20 Nov 2024 14:58:21 +0100 Subject: [PATCH 36/36] Change cfailed type --- db/flags.sql | 3 +-- migrations/lard/main.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/db/flags.sql b/db/flags.sql index b58c329a..d3d5de2d 100644 --- a/db/flags.sql +++ b/db/flags.sql @@ -7,7 +7,6 @@ CREATE TABLE IF NOT EXISTS flags.kvdata ( corrected REAL NULL, controlinfo TEXT NULL, useinfo TEXT NULL, - -- TODO: check that this type is correct, it's stored as a string in Kvalobs? cfailed INT4 NULL, CONSTRAINT unique_kvdata_timeseries_obstime UNIQUE (timeseries, obstime) ); @@ -20,7 +19,7 @@ CREATE TABLE IF NOT EXISTS flags.old_databases ( corrected REAL NULL, controlinfo TEXT NULL, useinfo TEXT NULL, - cfailed TEXT NULL , + cfailed INT4 NULL , CONSTRAINT unique_old_flags_timeseries_obstime UNIQUE (timeseries, obstime) ); CREATE INDEX IF NOT EXISTS old_flags_obtime_index ON flags.old_databases (obstime); diff --git a/migrations/lard/main.go b/migrations/lard/main.go index e0787999..be99c2da 100644 --- a/migrations/lard/main.go +++ b/migrations/lard/main.go @@ -43,7 +43,7 @@ type Flag struct { // Flag encoding quality control status Useinfo *string // Number of tests that failed? - Cfailed *string + Cfailed *int32 } func (o *Flag) ToRow() []any {