fix: Avoid write empty state

singer_sdk/streams/core.py

@@ -817,8 +817,10 @@ def _increment_stream_state(
 
     def _write_state_message(self) -> None:
         """Write out a STATE message with the latest state."""
-        if (not self._is_state_flushed) and (
-            self.tap_state != self._last_emitted_state
+        if (
+            (not self._is_state_flushed)
+            and self.tap_state
+            and (self.tap_state != self._last_emitted_state)
         ):
             self._tap.write_message(singer.StateMessage(value=self.tap_state))
             self._last_emitted_state = copy.deepcopy(self.tap_state)

singer_sdk/tap_base.py

@@ -460,7 +460,8 @@ def sync_all(self) -> None:
         """Sync all streams."""
         self._reset_state_progress_markers()
         self._set_compatible_replication_methods()
-        self.write_message(StateMessage(value=self.state))
+        if self.state:
+            self.write_message(StateMessage(value=self.state))
 
         stream: Stream
         for stream in self.streams.values():

tests/core/test_parent_child.py

@@ -104,19 +104,19 @@ def test_parent_context_fields_in_child(tap: MyTap):
     messages = _get_messages(tap)
 
     # Parent schema is emitted
-    assert messages[1]
-    assert messages[1]["type"] == SingerMessageType.SCHEMA
-    assert messages[1]["stream"] == parent_stream.name
-    assert messages[1]["schema"] == parent_stream.schema
+    assert messages[0]
+    assert messages[0]["type"] == SingerMessageType.SCHEMA
+    assert messages[0]["stream"] == parent_stream.name
+    assert messages[0]["schema"] == parent_stream.schema
 
     # Child schema is emitted
-    assert messages[2]
-    assert messages[2]["type"] == SingerMessageType.SCHEMA
-    assert messages[2]["stream"] == child_stream.name
-    assert messages[2]["schema"] == child_stream.schema
+    assert messages[1]
+    assert messages[1]["type"] == SingerMessageType.SCHEMA
+    assert messages[1]["stream"] == child_stream.name
+    assert messages[1]["schema"] == child_stream.schema
 
     # Child records are emitted
-    child_record_messages = messages[3:6]
+    child_record_messages = messages[2:5]
     assert child_record_messages
     assert all(msg["type"] == SingerMessageType.RECORD for msg in child_record_messages)
     assert all(msg["stream"] == child_stream.name for msg in child_record_messages)
@@ -156,13 +156,13 @@ def test_child_deselected_parent(tap_with_deselected_parent: MyTap):
     messages = _get_messages(tap_with_deselected_parent)
 
     # First message is a schema for the child stream, not the parent
-    assert messages[1]
-    assert messages[1]["type"] == SingerMessageType.SCHEMA
-    assert messages[1]["stream"] == child_stream.name
-    assert messages[1]["schema"] == child_stream.schema
+    assert messages[0]
+    assert messages[0]["type"] == SingerMessageType.SCHEMA
+    assert messages[0]["stream"] == child_stream.name
+    assert messages[0]["schema"] == child_stream.schema
 
     # Child records are emitted
-    child_record_messages = messages[2:5]
+    child_record_messages = messages[1:4]
     assert child_record_messages
     assert all(msg["type"] == SingerMessageType.RECORD for msg in child_record_messages)
     assert all(msg["stream"] == child_stream.name for msg in child_record_messages)
@@ -237,30 +237,30 @@ def discover_streams(self):
     messages = _get_messages(tap)
 
     # Parent schema is emitted
-    assert messages[1]
-    assert messages[1]["type"] == SingerMessageType.SCHEMA
-    assert messages[1]["stream"] == parent_stream.name
-    assert messages[1]["schema"] == parent_stream.schema
+    assert messages[0]
+    assert messages[0]["type"] == SingerMessageType.SCHEMA
+    assert messages[0]["stream"] == parent_stream.name
+    assert messages[0]["schema"] == parent_stream.schema
 
     # Child schemas are emitted
-    schema_messages = messages[2:9:3]
+    schema_messages = messages[1:8:3]
     assert schema_messages
     assert all(msg["type"] == SingerMessageType.SCHEMA for msg in schema_messages)
     assert all(msg["stream"] == child_stream.name for msg in schema_messages)
     assert all(msg["schema"] == child_stream.schema for msg in schema_messages)
 
     # Child records are emitted
-    child_record_messages = messages[3:10:3]
+    child_record_messages = messages[2:9:3]
     assert child_record_messages
     assert all(msg["type"] == SingerMessageType.RECORD for msg in child_record_messages)
     assert all(msg["stream"] == child_stream.name for msg in child_record_messages)
     assert all("pid" in msg["record"] for msg in child_record_messages)
 
     # State messages are emitted
-    state_messages = messages[4:11:3]
+    state_messages = messages[3:10:3]
     assert state_messages
     assert all(msg["type"] == SingerMessageType.STATE for msg in state_messages)
 
     # Parent record is emitted
-    assert messages[11]
-    assert messages[11]["type"] == SingerMessageType.RECORD
+    assert messages[10]
+    assert messages[10]["type"] == SingerMessageType.RECORD

tests/samples/test_tap_countries.py

@@ -140,7 +140,7 @@ def tally_messages(messages: list) -> t.Counter:
     assert counter["SCHEMA", "countries"] == 1
     assert counter["BATCH", "countries"] == 1
 
-    assert counter["STATE",] == 3
+    assert counter["STATE",] == 2
 
 
 @pytest.mark.snapshot

tests/samples/test_target_sqlite.py

@@ -121,6 +121,7 @@ def test_sync_sqlite_to_sqlite(
     - Confirm the STDOUT from the original sample DB matches with the
       STDOUT from the re-tapped target DB.
     """
+    initial_state = deepcopy(sqlite_sample_tap.state)
     orig_stdout, _, _, _ = tap_to_target_sync_test(
         sqlite_sample_tap,
         sqlite_sample_target,
@@ -129,6 +130,7 @@ def test_sync_sqlite_to_sqlite(
     tapped_config = dict(sqlite_sample_target.config)
     tapped_target = SQLiteTap(
         config=tapped_config,
+        state=initial_state,
         catalog=sqlite_sample_db_catalog.to_dict(),
     )
     new_stdout, _ = tap_sync_test(tapped_target)
@@ -138,6 +140,8 @@ def test_sync_sqlite_to_sqlite(
     new_lines = new_stdout.readlines()
     assert len(orig_lines) > 0, "Orig tap output should not be empty."
     assert len(new_lines) > 0, "(Re-)tapped target output should not be empty."
+    assert orig_lines[0] == new_lines[0]
+    assert "STATE" in new_lines[0]
     assert len(orig_lines) == len(new_lines)
 
     line_num = 0

tests/snapshots/mapped_stream/aliased_stream.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"aliased_stream","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"aliased_stream","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/aliased_stream_batch.jsonl

@@ -1,5 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"BATCH","stream":"aliased_stream","encoding":{"format":"jsonl","compression":"gzip"},"manifest":["file:///tmp/stream.json.gz"]}
-{"type":"STATE","value":{}}
 {"type":"STATE","value":{"bookmarks":{"mystream":{}}}}

tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"aliased.stream","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"aliased.stream","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"__stream_name__","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"__stream_name__","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/builtin_variable_self.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"source_table":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"aliased_mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"aliased_mystream","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"aliased_mystream","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"MYSTREAM","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"MYSTREAM","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"MYSTREAM","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/builtin_variable_underscore.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/changed_key_properties.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":["email_hash"]}
 {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/dates.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"joined_date":{"type":["string","null"],"format":"date"},"joined_timestamp":{"type":["number","null"]},"some_datetime":{"type":["string","null"],"format":"date-time"}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","joined_date":"2022-01-01","joined_timestamp":1640995200.0,"some_datetime":"2024-10-10T10:10:10+00:00"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","joined_date":"2022-01-01","joined_timestamp":1640995200.0,"some_datetime":"2024-10-10T10:10:10+00:00"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/drop_property.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/drop_property_null_string.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/fake_credit_card_number.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"cc":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"cc":"4201040137208265027"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"cc":"675987782884"},"time_extracted":"2022-01-01T00:00:00+00:00"}

tests/snapshots/mapped_stream/fake_email_seed_class.jsonl

@@ -1,4 +1,3 @@
-{"type":"STATE","value":{}}
 {"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]"},"time_extracted":"2022-01-01T00:00:00+00:00"}
 {"type":"RECORD","stream":"mystream","record":{"email":"[email protected]"},"time_extracted":"2022-01-01T00:00:00+00:00"}