Add query/document marker tokens

giga_cherche/models/colbert.py

@@ -367,6 +367,10 @@ def __init__(
         # Pass the model to the model card data for later use in generating a model card upon saving this model
         self.model_card_data.register_model(self)
 
+        # this will add the query and document prefix to the tokenizer vocab if they are not already there and resize the embeddings accordingly
+        # self.tokenizer.add_tokens([self.query_prefix, self.document_prefix])
+        # self._first_module().auto_model.resize_token_embeddings(len(self.tokenizer))
+
         self.document_prefix_id = self.tokenizer.convert_tokens_to_ids(
             self.document_prefix
         )