Merge pull request #179 from pushkarmoi/pugupta/comments

raw kafka offset storage comments

ikv/src/index/offset_store.rs

@@ -20,6 +20,9 @@ pub struct OffsetStore {
     file: File,
 }
 
+/// NOTE - it is okay to store raw kafka offsets
+/// An offset is always valid (w.r.t being b/w low/high watermark) even
+/// with time/size based retention in play (auto expiry by kafka).
 impl OffsetStore {
     pub fn open_or_create(mount_directory: String) -> io::Result<Self> {
         let filename = format!("{}/kafka_offsets", mount_directory);

ikv/src/kafka/consumer.rs

@@ -273,6 +273,9 @@ async fn initialize_stream_consumer(
     seek_consumer(&consumer, topic, partition, rdkafka::Offset::Beginning)?;
 
     // seek - using persisted offsets
+    // NOTE - it is okay to store raw kafka offsets
+    // An offset is always valid (w.r.t being b/w low/high watermark) even
+    // with time/size based retention in play (auto expiry by kafka).
     let stored_topic_partition_list = offset_store.read_all_offsets()?;
     for entry in stored_topic_partition_list.iter() {
         if (&entry.topic == topic) && (entry.partition == partition) {