diff --git a/target_snowflake/snowflake.py b/target_snowflake/snowflake.py index 93250e6..5b6f015 100644 --- a/target_snowflake/snowflake.py +++ b/target_snowflake/snowflake.py @@ -526,7 +526,7 @@ def transform(): row = next(rows_iter) with io.StringIO() as out: - writer = csv.DictWriter(out, csv_headers) + writer = csv.DictWriter(out, csv_headers, quoting=csv.QUOTE_NONNUMERIC) writer.writerow(row) return out.getvalue() except StopIteration: diff --git a/target_snowflake/sql.py b/target_snowflake/sql.py index 2c64b81..60621bb 100644 --- a/target_snowflake/sql.py +++ b/target_snowflake/sql.py @@ -21,11 +21,11 @@ def valid_identifier(x): IDENTIFIER_FIELD_LENGTH, len(x), x)) - - if not re.match(r'^[a-zA-Z_]\w+$', x): + + if not re.match(r'^[a-zA-Z_](\w+)?$', x): raise SQLError( - 'Identifier must only contain alphanumerics, or underscores, and start with alphas. Got `{}` for `{}`'.format( - re.findall(r'[^0-9]', '1234a567')[0], + 'Identifier must only contain alphanumerics, or underscores, and start with alphas. Found `{}` from name `{}`'.format( + re.findall(r'[^0-9a-zA-Z_]', x), x )) diff --git a/tests/fixtures.py b/tests/fixtures.py index 1bf63c5..71722dc 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -221,7 +221,6 @@ def generate_record(self): 'adoption': adoption } - class InvalidCatStream(CatStream): def generate_record(self): record = CatStream.generate_record(self) @@ -433,6 +432,29 @@ def generate_record(self): } +SINGLE_CHAR_SCHEMA = { + 'type': 'SCHEMA', + 'stream': 'root', + 'schema': { + 'additionalProperties': False, + 'properties': { + 'x': { + 'type': 'integer' + } + } + }, + 'key_properties': [] +} + +class SingleCharStream(FakeStream): + stream = 'root' + schema = SINGLE_CHAR_SCHEMA + + def generate_record(self): + return { + 'x': random.randint(-314159265359, 314159265359) + } + def clear_schema(): with connect(**TEST_DB) as conn: diff --git a/tests/test_target_snowflake.py b/tests/test_target_snowflake.py index 58fcbbf..c035e57 100644 --- a/tests/test_target_snowflake.py +++ b/tests/test_target_snowflake.py @@ -4,7 +4,7 @@ from psycopg2 import sql import pytest -from fixtures import CatStream, CONFIG, db_prep, MultiTypeStream, NestedStream, S3_CONFIG, TEST_DB +from fixtures import CatStream, CONFIG, db_prep, MultiTypeStream, NestedStream, SingleCharStream, S3_CONFIG, TEST_DB from target_postgres import singer_stream from target_postgres.target_tools import TargetError @@ -652,6 +652,40 @@ def test_loading__multi_types_columns(db_prep): assert stream_count == len([x for x in persisted_records if isinstance(x[0], float)]) +def test_loading__single_char_columns(db_prep): + stream_count = 50 + main(CONFIG, input_stream=SingleCharStream(stream_count)) + + with connect(**TEST_DB) as conn: + with conn.cursor() as cur: + assert_columns_equal(cur, + 'ROOT', + { + ('_SDC_PRIMARY_KEY', 'TEXT', 'NO'), + ('_SDC_BATCHED_AT', 'TIMESTAMP_TZ', 'YES'), + ('_SDC_RECEIVED_AT', 'TIMESTAMP_TZ', 'YES'), + ('_SDC_SEQUENCE', 'NUMBER', 'YES'), + ('_SDC_TABLE_VERSION', 'NUMBER', 'YES'), + ('_SDC_TARGET_SNOWFLAKE_CREATE_TABLE_PLACEHOLDER', 'BOOLEAN', 'YES'), + ('X', 'NUMBER', 'YES') + }) + + cur.execute(''' + SELECT {} FROM {}.{}.{} + '''.format( + sql.identifier('X'), + sql.identifier(CONFIG['snowflake_database']), + sql.identifier(CONFIG['snowflake_schema']), + sql.identifier('ROOT') + )) + persisted_records = cur.fetchall() + + ## Assert that the column is has migrated data + assert stream_count == len(persisted_records) + assert stream_count == len([x for x in persisted_records if isinstance(x[0], float)]) + + + def test_upsert(db_prep): stream = CatStream(100) main(CONFIG, input_stream=stream)