Tests for fine-tuned models in encoding_for_model

openai · Jun 1, 2023 · 96b2158 · 96b2158
1 parent 07ba412
commit 96b2158
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/tests/test_simple_public.py b/tests/test_simple_public.py
@@ -32,6 +32,16 @@ def test_encoding_for_model():
     enc = tiktoken.encoding_for_model("gpt-3.5-turbo-0301")
     assert enc.name == "cl100k_base"
 
+    # fine-tuned models
+    enc = tiktoken.encoding_for_model("davinci:ft-personal:finetunedmodel-2023-05-23-20-00-00")
+    assert enc.name == "r50k_base"
+    enc = tiktoken.encoding_for_model("curie:ft-personal:finetunedmodel-2023-05-23-20-00-00")
+    assert enc.name == "r50k_base"
+    enc = tiktoken.encoding_for_model("babbage:ft-personal:finetunedmodel-2023-05-23-20-00-00")
+    assert enc.name == "r50k_base"
+    enc = tiktoken.encoding_for_model("ada:ft-personal:finetunedmodel-2023-05-23-20-00-00")
+    assert enc.name == "r50k_base"
+
 
 def test_optional_blobfile_dependency():
     prog = """