Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Devel #13

Merged
merged 38 commits into from
Jan 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
b0f19bb
Add basic GPS parsers
frafra Nov 22, 2023
3a77350
Make dbmate wait for Postgres
frafra Nov 22, 2023
0579949
Drop old references to Openrefine
frafra Nov 22, 2023
1158f7c
limit empty row
nicokant Jan 15, 2024
6ff024e
#7 add reload button, prettier error output
nicokant Jan 16, 2024
f9e1cfc
block upload loggers if spreadsheet upload fails
nicokant Jan 16, 2024
4ef926d
upload only spreadsheets fix #12
nicokant Jan 16, 2024
aefae8b
check excel columns match database columns
nicokant Jan 17, 2024
9ea1899
remove pdm-python
nicokant Jan 17, 2024
9bdb70a
limit logger extensions to upload fix #6
nicokant Jan 17, 2024
9e2a0d5
convert detail to json before rendering
nicokant Jan 17, 2024
2a1b539
version schema
nicokant Jan 17, 2024
30507a5
handle non json errors
nicokant Jan 17, 2024
5a04104
add constraints on loggers existence
nicokant Jan 17, 2024
0e5624f
rename accelerometer to other_sensor
nicokant Jan 17, 2024
66f8ea8
fix filename presence logic
nicokant Jan 17, 2024
c18c922
separate wizard for metadata and loggers
nicokant Jan 17, 2024
5da0d3f
publish uploaded data
nicokant Jan 17, 2024
977de98
upload all files, use temp location
nicokant Jan 17, 2024
221b95a
keep only filename, not absolute path
nicokant Jan 17, 2024
940bb4c
make unknown fields optional
nicokant Jan 17, 2024
067b26d
improve not implemented error report
nicokant Jan 17, 2024
2eeb923
parse and convert to parquet
nicokant Jan 19, 2024
7099409
various gps parsers support
nicokant Jan 19, 2024
2c95c17
gpx support
nicokant Jan 19, 2024
f7b38fc
ignore some folders in test
nicokant Jan 22, 2024
0caf78e
disable index col
nicokant Jan 22, 2024
715944a
add parser type and data type as field to the parquet
nicokant Jan 22, 2024
6944561
handle accelerometer loggers
nicokant Jan 22, 2024
adfec92
handle tdr
nicokant Jan 22, 2024
1540688
open with the correct encoding
nicokant Jan 22, 2024
076234a
fix command line script to write csv
nicokant Jan 22, 2024
6bfbcef
parquet without filetype specified
nicokant Jan 22, 2024
6e2c31f
configure extensions, disable conversion to parquet on upload
nicokant Jan 22, 2024
0097972
fix migrations
nicokant Jan 24, 2024
7abccb7
fix filename missing
nicokant Jan 26, 2024
0d6b819
try parsing
nicokant Jan 26, 2024
a98c4f1
add links to outputs
nicokant Jan 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions data/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*
!.gitignore
5 changes: 4 additions & 1 deletion db/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
FROM ghcr.io/amacneil/dbmate as dbmate
COPY ./ db/
COPY migrations/ db/migrations/
COPY entrypoint.sh .
ENTRYPOINT ["/entrypoint.sh"]
CMD ["dbmate", "--no-dump-schema", "up"]
7 changes: 7 additions & 0 deletions db/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/sh

set -ex

dbmate wait

exec "$@"
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
-- migrate:up

alter table logger_instrumentation add constraint logger_instrumentation_filename_key unique (filename);

-- migrate:down
13 changes: 13 additions & 0 deletions db/migrations/20240116110423_import_fields_view.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- migrate:up

create materialized view import_fields as
(SELECT column_name, is_nullable
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'import');

grant select on import_fields to web_anon;


-- migrate:down

36 changes: 36 additions & 0 deletions db/migrations/20240117072241_row_to_json_errors_detail.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
-- migrate:up

create or replace function import() returns trigger language plpgsql
as $$
begin
if new.ring_number is null then
new.ring_number = 'AUTO_' || nextval('auto_ring_id_seq')::text;
end if;
if new.gps_deployment_date is not null and new.gps_logger_id is null then
new.gps_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
if new.gls_deployment_date is not null and new.gls_logger_id is null then
new.gls_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
if new.tdr_deployment_date is not null and new.tdr_logger_id is null then
new.tdr_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
if new.accelerometer_deployment_date is not null and new.accelerometer_logger_id is null then
new.accelerometer_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
perform import_animal_and_ring(new);
perform import_colony(new);
perform import_deployment_and_chick(new);
perform import_logger_and_logger_instrumentation(new);
return null;
exception
when others then
raise exception using
errcode = sqlstate,
message = sqlerrm,
detail = row_to_json(new);
end;
$$;

-- migrate:down

53 changes: 53 additions & 0 deletions db/migrations/20240117081051_import_filename_constraint.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
-- migrate:up

create or replace function import() returns trigger language plpgsql
as $$
begin
-- perform some checks
if new.gps_deployment_date is not null and new.gps_raw_datafile_name is null then
raise exception 'gps_raw_datafile_name cannot be empty if gps_deployment_date is defined';
end if;

if new.gls_deployment_date is not null and new.gls_raw_datafile_name is null then
raise exception 'gls_raw_datafile_name cannot be empty if gls_deployment_date is defined';
end if;

if new.tdr_deployment_date is not null and new.tdr_raw_datafile_name is null then
raise exception 'tdr_raw_datafile_name cannot be empty if tdr_deployment_date is defined';
end if;

if new.accelerometer_deployment_date is not null and new.accelerometer_raw_datafile_name is null then
raise exception 'accelerometer_raw_datafile_name cannot be empty if accelerometer_deployment_date is defined';
end if;

if new.ring_number is null then
new.ring_number = 'AUTO_' || nextval('auto_ring_id_seq')::text;
end if;
if new.gps_deployment_date is not null and new.gps_logger_id is null then
new.gps_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
if new.gls_deployment_date is not null and new.gls_logger_id is null then
new.gls_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
if new.tdr_deployment_date is not null and new.tdr_logger_id is null then
new.tdr_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
if new.accelerometer_deployment_date is not null and new.accelerometer_logger_id is null then
new.accelerometer_logger_id = 'AUTO_' || nextval('auto_logger_id_seq')::text;
end if;
perform import_animal_and_ring(new);
perform import_colony(new);
perform import_deployment_and_chick(new);
perform import_logger_and_logger_instrumentation(new);
return null;
exception
when others then
raise exception using
errcode = sqlstate,
message = sqlerrm,
detail = row_to_json(new);
end;
$$;

-- migrate:down

Loading
Loading