feat: Added ability to get encoder/encoding by model name.

README.md

@@ -21,8 +21,7 @@ We will be happy to accept any PR.
 using Tiktoken.Encodings;
 using Tiktoken;
 
-var encoding = new O200KBase();
-var encoder = new Encoder(encoding);
+var encoder = Encoders.ForModel("gpt-4o"); // or explicitly new Encoder(new O200KBase())
 var tokens = encoder.Encode("hello world"); // [15339, 1917]
 var text = encoder.Decode(tokens); // hello world
 var numberOfTokens = encoder.CountTokens(text); // 2

src/libs/Directory.Build.props

@@ -9,7 +9,7 @@
     </ItemGroup>
 
     <PropertyGroup Label="Nuget">
-        <Version>2.0.0</Version>
+        <Version>2.0.1</Version>
         <Description>The fastest tokenizer for GPT-3.5 and GPT-4 inspired by Tiktoken.</Description>
         <PackageTags>chatgpt;openai;tiktoken;tokens;gpt-4;gpt-3.5-turbo;cl100k_base;p50k_base</PackageTags>
         <GeneratePackageOnBuild Condition=" '$(Configuration)' == 'Release' ">true</GeneratePackageOnBuild>

src/libs/Tiktoken/EncoderHelpers.cs → src/libs/Tiktoken/Encoders.cs

@@ -2,66 +2,70 @@
 
 namespace Tiktoken;
 
-// ReSharper disable InconsistentNaming
-
-internal static class EncoderHelpers
+/// <summary>
+/// 
+/// </summary>
+public static class Encoders
 {
     /// <summary>
-    /// Returns encoding by model name.
+    /// Returns encoder by model name.
     /// </summary>
     /// <param name="modelName">gpt-3.5-turbo</param>
     /// <returns></returns>
     public static Encoder ForModel(string modelName)
     {
-        return new Encoder(GetNameByModel(modelName));
+        return new Encoder(GetEncodingByModel(modelName));
     }
 
     /// <summary>
-    /// Returns encoding by model name or null.
+    /// Returns encoder by model name or null.
     /// </summary>
     /// <param name="modelName">gpt-3.5-turbo</param>
     /// <returns></returns>
     public static Encoder? TryForModel(string modelName)
     {
-        var encoding = TryGetNameByModel(modelName);
+        var encoding = TryGetEncodingByModel(modelName);
 
         return encoding == null
             ? null
             : new Encoder(encoding);
     }
 
-    private static Dictionary<string, Encodings.Encoding> ModelToEncoding { get; } = new()
+    private static Dictionary<string, Encoding> ModelToEncoding { get; } = new()
     {
         // chat
         { "gpt-4o", new O200KBase() },
         { "gpt-4", new Cl100KBase() },
         { "gpt-3.5-turbo", new Cl100KBase() },
         { "gpt-35-turbo", new Cl100KBase() }, // Azure deployment name
+
         // embeddings
         { "text-embedding-ada-002", new Cl100KBase() },
+        { "text-embedding-3-small", new Cl100KBase() },
+        { "text-embedding-3-large", new Cl100KBase() },
     };
 
     /// <summary>
-    /// Returns encoding name by model name or null.
+    /// Returns encoding by model name or null.
     /// </summary>
     /// <param name="modelName">gpt-4 gpt-3.5-turbo ...</param>
     /// <exception cref="ArgumentException"></exception>
     /// <returns></returns>
-    public static Encoding? TryGetNameByModel(string modelName)
+    public static Encoding? TryGetEncodingByModel(string modelName)
     {
         return ModelToEncoding
             .FirstOrDefault(a => modelName.StartsWith(a.Key, StringComparison.Ordinal)).Value;
     }
 
     /// <summary>
-    /// Returns encoding name by model name or throws exception.
+    /// Returns encoding by model name or throws exception.
     /// </summary>
     /// <param name="modelName">gpt-4 gpt-3.5-turbo ...</param>
     /// <exception cref="ArgumentException"></exception>
     /// <returns></returns>
-    public static Encoding GetNameByModel(string modelName)
+    public static Encoding GetEncodingByModel(string modelName)
     {
-        return TryGetNameByModel(modelName) ??
+        return TryGetEncodingByModel(modelName) ??
                throw new ArgumentException($"Model name {modelName} is not supported.");
     }
 }