Merge pull request #29 from cnblogs/use-tokenizer-from-sdk

feat: use tokenizer from sdk

Author: ikesnowy

src/KernelMemory.DashScope/DashScopeEmbeddedResource.cs

@@ -22,18 +22,19 @@ public class DashScopeTextGenerator(
     int maxToken = 6000) : ITextGenerator
 {
     private readonly ILogger<DashScopeTextGenerator> _logger = loggerFactory?.CreateLogger<DashScopeTextGenerator>()
-                                                                ?? DefaultLogger<DashScopeTextGenerator>.Instance;
+                                                               ?? DefaultLogger<DashScopeTextGenerator>.Instance;
 
     /// <inheritdoc />
     public int CountTokens(string text)
     {
-        return tokenizer?.CountTokens(text) ?? QWenTokenizer.CountTokensStatic(text);
+        return tokenizer?.CountTokens(text) ?? QWenTokenizer.CountTokens(text);
     }
 
     /// <inheritdoc />
     public IReadOnlyList<string> GetTokens(string text)
     {
-        return tokenizer?.GetTokens(text) ?? QWenTokenizer.GetTokensStatic(text);
+        return tokenizer?.GetTokens(text)
+               ?? QWenTokenizer.Tokenizer.EncodeToTokens(text, out _).Select(x => x.Value).ToList();
     }
 
     /// <inheritdoc />
@@ -47,7 +48,9 @@ public async IAsyncEnumerable<string> GenerateTextAsync(
             TopP = options.NucleusSampling == 0 ? null : (float)options.NucleusSampling,
             Temperature = options.Temperature == 0 ? null : (float)options.Temperature,
             RepetitionPenalty =
-                options.FrequencyPenalty == 0 ? null : ((float)options.FrequencyPenalty + 1), // dashScope's default value is 1.0, kernel memory is 0.0
+                options.FrequencyPenalty == 0
+                    ? null
+                    : ((float)options.FrequencyPenalty + 1), // dashScope's default value is 1.0, kernel memory is 0.0
             MaxTokens = options.MaxTokens == 0 ? null : options.MaxTokens,
             Stop = options.StopSequences.ToArray(),
             IncrementalOutput = true,

src/KernelMemory.DashScope/DashScopeTextGenerator.cs

@@ -36,7 +36,7 @@ public static IKernelMemoryBuilder WithDashScopeDefaults(
         ITextTokenizer? textEmbeddingTokenizer = null,
         bool onlyForRetrieval = false)
     {
-        textGenerationTokenizer ??= new QWenTokenizer();
+        textGenerationTokenizer ??= new QWenTextTokenizer();
         textEmbeddingTokenizer ??= new LengthTokenizer();
 
         var config = new DashScopeConfig
@@ -97,7 +97,7 @@ public static IKernelMemoryBuilder WithDashScope(
     {
         config.EnsureValid();
         embeddingTokenizer ??= new LengthTokenizer();
-        textTokenizer ??= new QWenTokenizer();
+        textTokenizer ??= new QWenTextTokenizer();
         dashScopeClient ??= new DashScopeClient(config.ApiKey);
         builder.WithDashScopeTextGeneration(config, textTokenizer, dashScopeClient);
         builder.WithDashScopeTextEmbeddingGeneration(config, embeddingTokenizer, onlyForRetrieval, dashScopeClient);
@@ -119,7 +119,7 @@ public static IKernelMemoryBuilder WithDashScopeTextGeneration(
         IDashScopeClient? dashScopeClient = null)
     {
         config.EnsureValid();
-        tokenizer ??= new QWenTokenizer();
+        tokenizer ??= new QWenTextTokenizer();
         dashScopeClient ??= new DashScopeClient(config.ApiKey);
         builder.Services.AddDashScopeTextGeneration(config, tokenizer, dashScopeClient);
         return builder;
@@ -188,7 +188,7 @@ public static IServiceCollection AddDashScopeTextEmbeddingGeneration(
     /// </summary>
     /// <param name="services">The <see cref="IServiceCollection"/>.</param>
     /// <param name="config">Settings for DashScope.</param>
-    /// <param name="tokenizer">The tokenizer to use, defaults to <see cref="QWenTokenizer"/>.</param>
+    /// <param name="tokenizer">The tokenizer to use, defaults to <see cref="QWenTextTokenizer"/>.</param>
     /// <param name="dashScopeClient">The underlying <see cref="IDashScopeClient"/>.</param>
     /// <returns></returns>
     public static IServiceCollection AddDashScopeTextGeneration(
@@ -198,7 +198,7 @@ public static IServiceCollection AddDashScopeTextGeneration(
         IDashScopeClient? dashScopeClient = null)
     {
         config.EnsureValid();
-        tokenizer ??= new QWenTokenizer();
+        tokenizer ??= new QWenTextTokenizer();
 
         return services.AddSingleton<ITextGenerator>(
             sp => new DashScopeTextGenerator(

src/KernelMemory.DashScope/DependencyInjector.cs

@@ -18,9 +18,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.DeepDev.TokenizerLib" Version="1.3.3" />
     <PackageReference Include="Microsoft.KernelMemory.Abstractions" Version="0.95.241216.2" />
-    <PackageReference Include="Cnblogs.DashScope.Core" Version="0.5.1" />
+    <PackageReference Include="Cnblogs.DashScope.Core" Version="0.5.2" />
   </ItemGroup>
 
   <ItemGroup>
@@ -31,8 +30,4 @@
     <None Include="../../README.md" Pack="true" PackagePath="\" />
   </ItemGroup>
 
-  <ItemGroup>
-    <EmbeddedResource Include="qwen.tiktoken" />
-  </ItemGroup>
-
 </Project>

src/KernelMemory.DashScope/KernelMemory.DashScope.csproj

@@ -0,0 +1,22 @@
+using Cnblogs.DashScope.Core;
+using Microsoft.KernelMemory.AI;
+
+namespace Cnblogs.KernelMemory.AI.DashScope;
+
+/// <summary>
+/// Tokenizer using QWen
+/// </summary>
+public class QWenTextTokenizer : ITextTokenizer
+{
+    /// <inheritdoc />
+    public int CountTokens(string text)
+    {
+        return QWenTokenizer.CountTokens(text);
+    }
+
+    /// <inheritdoc />
+    public IReadOnlyList<string> GetTokens(string text)
+    {
+        return QWenTokenizer.Tokenizer.EncodeToTokens(text, out _).Select(x => x.Value).ToList();
+    }
+}