Update Microsoft.ML.Tokenizers and drop VS2019 support

Documentation/Topics/Automatic-Translations.md

@@ -31,11 +31,35 @@ The API URL for Deepl is https://api.deepl.com/v2/translate, but for the free ve
 
 #### Configuration
 - Add a new Azure OpenAI resource using the portal or CLI.
-- Model availability may vary depending on e.g. region, check the model availabilty [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-preview-model-availability)
-- Deploy the "gpt-3.5-turbo-instruct" (completion based) or "gpt-3.5-turbo"/"gpt-4-turbo" (chat based) model.
-- Copy the API key, URL to the endpoint and name of the deployment and model into the settings of the translator
-
-#### Addtional Settings
+- Model availability may vary depending on e.g. region, check the model availability [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-preview-model-availability)
+- Deploy one of the supported models (see below).
+- Copy the API key, URL to the endpoint, deployment name, and model name into the settings of the translator.
+
+**Important: Understanding Model Name vs. Model Deployment Name**
+- **Model Name**: This is the base model identifier used by OpenAI (e.g., "gpt-3.5-turbo", "gpt-4", "gpt-4o"). This is used internally for tokenization and determining which API endpoint to use.
+- **Model Deployment Name**: This is the custom name YOU chose when deploying the model in Azure (e.g., "my-gpt4-deployment"). This is what Azure uses to route your API requests.
+
+> A list of supported model names can be found [here](https://github.com/dotnet/machinelearning/blob/4c8b3579d1053257b213ca54be2681359b66cf65/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs#L1069-L1146)
+
+**Supported Models**:
+- **GPT-3.5 Models**:
+  - `gpt-3.5-turbo-instruct` (completion-based, legacy)
+  - `gpt-3.5-turbo` (chat-based, recommended for GPT-3.5)
+  - Note: You can also use `gpt-35-turbo` or `gpt-35-turbo-instruct` (Azure naming), which will be automatically mapped.
+
+- **GPT-4 Models**:
+  - `gpt-4` (chat-based)
+  - `gpt-4-turbo` (chat-based, faster and more cost-effective)
+  - `gpt-4o` (chat-based, latest multimodal model)
+  - `gpt-4o-mini` (chat-based, smaller and faster variant)
+
+**Example Configuration**:
+- If you deployed GPT-4 Turbo in Azure with deployment name "my-translation-model":
+  - **Model Name**: `gpt-4-turbo`
+  - **Model Deployment Name**: `my-translation-model`
+  - **Endpoint URL**: `https://your-resource-name.openai.azure.com`
+
+#### Additional Settings
 - You can add a custom prompt to your request to improve the translation quality or behavior, e.g. "preserve the html tags in the results"
 - You can include the comments in your resources in the prompt, to guide the model with additional hints about the context
 

src/Directory.Build.props

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
   <PropertyGroup>
-    <Version>1.105.0.0</Version>
+    <Version>1.106.0.0</Version>
     <Company>tom-englert.de</Company>
     <Product>ResXManager</Product>
     <Copyright>Copyright (c) .NET Foundation and Contributors.</Copyright>
@@ -27,7 +27,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <GlobalPackageReference Include="Microsoft.SourceLink.GitHub" Version="10.0.201" />
+    <GlobalPackageReference Include="Microsoft.SourceLink.GitHub" Version="10.0.300" />
     <GlobalPackageReference Include="Nullable.Extended.Analyzer" Version="1.15.6581" />
   </ItemGroup>
 

src/Directory.Packages.props

@@ -1,52 +1,47 @@
 <Project>
   <PropertyGroup>
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
-    <CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
+    <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="AutoProperties.Fody" Version="1.25.0" />
     <PackageVersion Include="DataGridExtensions" Version="2.9.1" />
-    <PackageVersion Include="DocumentFormat.OpenXml" Version="[2.20.0]" />
-    <PackageVersion Include="Community.VisualStudio.Toolkit" Version="[15.0.76.309]" />
+    <PackageVersion Include="DocumentFormat.OpenXml" Version="3.5.1" />
+    <PackageVersion Include="Community.VisualStudio.Toolkit" Version="16.0.76.309" IsPinned="True" Justification="Support for VS2022" />
     <PackageVersion Include="Fody" Version="6.9.3" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.12.4" />
     <PackageVersion Include="ILMerge.Fody" Version="1.24.0" />
     <PackageVersion Include="JetBrains.Annotations" Version="2025.2.4" />
     <PackageVersion Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="[0.22.0-preview.24378.1]" Justification="VS2019" />
-    <PackageVersion Include="Microsoft.VisualStudio.SDK" Version="[15.0.1]" />
-    <PackageVersion Include="Microsoft.VSSDK.BuildTools" Version="[17.6.2164]" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.CL100KBase" Version="2.0.0" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="2.0.0" />
+    <PackageVersion Include="Microsoft.VisualStudio.SDK" Version="17.14.40265" />
+    <PackageVersion Include="Microsoft.VSSDK.BuildTools" Version="18.5.40034" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
     <PackageVersion Include="NSubstitute" Version="5.3.0" />
     <PackageVersion Include="PropertyChanged.Fody" Version="4.1.0" />
     <PackageVersion Include="Resourcer.Fody" Version="1.8.1" />
-    <PackageVersion Include="System.Composition.AttributedModel" Version="[1.4.1]" />
     <PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageVersion Include="System.Data.DataSetExtensions" Version="4.5.0" />
     <PackageVersion Include="System.Net.Http" Version="4.3.4" />
-    <PackageVersion Include="System.Security.Cryptography.Pkcs" Version="[6.0.4]" />
-    <PackageVersion Include="System.ServiceModel.Primitives" Version="[4.10.2]" />
-    <PackageVersion Include="System.ServiceModel.Http" Version="[4.10.2]" />
-    <PackageVersion Include="System.Text.Json" Version="[8.0.5]" Justification="VS2019" />
+    <PackageVersion Include="System.ServiceModel.Primitives" Version="10.0.652802" />
+    <PackageVersion Include="System.Text.Json" Version="10.0.8" />
     <PackageVersion Include="Throttle.Fody" Version="1.7.0" />
-    <PackageVersion Include="TomsToolbox.Composition" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.Composition.Ninject" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.Essentials" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.ObservableCollections" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.Wpf" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.Wpf.Composition" Version="2.22.2" />
-    <PackageVersion Include="TomsToolbox.Wpf.Composition.AttributedModel" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.Wpf.Composition.Styles" Version="2.23.0" />
-    <PackageVersion Include="TomsToolbox.Wpf.Styles" Version="2.23.0" />
-    <PackageVersion Include="Verify.XunitV3" Version="31.15.0" />
-    <PackageVersion Include="VSIX-SdkProjectAdapter" Version="3.0.0" />
+    <PackageVersion Include="TomsToolbox.Composition" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.Composition.Ninject" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.Essentials" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.ObservableCollections" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.Wpf" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.Wpf.Composition.AttributedModel" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.Wpf.Composition.Styles" Version="2.24.0" />
+    <PackageVersion Include="TomsToolbox.Wpf.Styles" Version="2.24.0" />
+    <PackageVersion Include="Verify.XunitV3" Version="31.19.0" />
+    <PackageVersion Include="VSIX-SdkProjectAdapter" Version="4.0.0" />
     <PackageVersion Include="WindowsAPICodePack-Shell" Version="1.1.1" />
     <PackageVersion Include="xunit.v3" Version="3.2.2" />
   </ItemGroup>
-
   <ItemGroup>
-    <PackageMitigation Include="Newtonsoft.Json" Version="8.0.3" Justification="VS 2019 requirement" />
-    <PackageMitigation Include="MessagePack" Version="2.2.85" Justification="VS Sdk dependency" />
+    <PackageMitigation Include="Microsoft.Bcl.Memory" Version="9.0.4" Justification="Microsoft.ML.Tokenizers@2.0.0 won't load with another version" />
   </ItemGroup>
-</Project>
-
+</Project>

src/ResXManager.Infrastructure/ResXManager.Infrastructure.csproj

@@ -14,7 +14,7 @@
     <PackageReference Include="HtmlAgilityPack" PrivateAssets="all" />
     <PackageReference Include="ILMerge.Fody" PrivateAssets="all" />
     <PackageReference Include="JetBrains.Annotations" PrivateAssets="all" />
-    <PackageReference Include="Newtonsoft.Json" VersionOverride="[9.0.1]" Justification="VS2019" />
+    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="PropertyChanged.Fody" PrivateAssets="all" />
     <PackageReference Include="TomsToolbox.Composition" />
   </ItemGroup>

src/ResXManager.Model/ExcelExtensions.cs

@@ -27,7 +27,7 @@ public enum ExcelExportMode
 
     public static partial class ResourceEntityExtensions
     {
-        private static readonly string[] _singleSheetFixedColumnHeaders = { "Project", "File", "Key" };
+        private static readonly string[] _singleSheetFixedColumnHeaders = ["Project", "File", "Key"];
 
         public static void ExportExcelFile(this ResourceManager resourceManager, string filePath, IResourceScope? scope, ExcelExportMode exportMode)
         {
@@ -256,12 +256,12 @@ private static IEnumerable<Row> GetRows(this Sheet sheet, WorkbookPart workbookP
         {
             var sheetId = sheet.Id;
 
-            if ((sheetId is not { Value: {} value}) || string.IsNullOrEmpty(value))
-                return Enumerable.Empty<Row>();
+            if ((sheetId is not { Value: { } value }) || string.IsNullOrEmpty(value))
+                return [];
 
             var worksheetPart = (WorksheetPart)workbookPart.GetPartById(value);
 
-            return worksheetPart.Worksheet.ChildElements.OfType<SheetData>().FirstOrDefault()?.OfType<Row>() ?? Enumerable.Empty<Row>();
+            return worksheetPart.Worksheet?.ChildElements.OfType<SheetData>().FirstOrDefault()?.OfType<Row>() ?? [];
         }
 
         private static ResourceEntity FindResourceEntity(this IEnumerable<MultipleSheetEntity> entities, Sheet sheet)
@@ -286,7 +286,7 @@ public DataAppender(int numberOfFixedColumns)
 
             public SheetData AppendRow(SheetData sheetData, IEnumerable<string> rowData)
             {
-                var row = (sheetData.ChildElements?.OfType<Row>()?.Count() ?? 0) + 1;
+                var row = (sheetData.ChildElements.OfType<Row>()?.Count() ?? 0) + 1;
                 var column = 1;
                 return sheetData.AppendItem(rowData.Aggregate(new Row(), (seed, item) => seed.AppendItem(CreateCell(item, row, column++))));
             }
@@ -707,4 +707,4 @@ public static void AddStylesToWorkbookPart(WorkbookPart part)
             }
         }
     }
-}
+}

src/ResXManager.Model/ResXManager.Model.csproj

@@ -22,7 +22,7 @@
     <PackageReference Include="Fody" PrivateAssets="all" />
     <PackageReference Include="PropertyChanged.Fody" PrivateAssets="all" />
     <PackageReference Include="System.ComponentModel.Annotations" />
-    <PackageReference Include="System.Composition.AttributedModel" />
+    <PackageReference Include="System.Composition.AttributedModel" VersionOverride="1.4.1" IsPinned="True" Justification="Ensure compatibility with other packages" />
     <PackageReference Include="Throttle.Fody" PrivateAssets="all" />
     <PackageReference Include="TomsToolbox.ObservableCollections" />
   </ItemGroup>

src/ResXManager.Tests/WebExporterOutput/src/resources.it.json

@@ -1,8 +1,4 @@
 {
   "_comment": "Auto-generated; do not modify!",
-  "WebExportTest": {
-    "SimpleString": "",
-    "WithInterpolation_TEMPLATE": "",
-    "WithBadInterpolation_TEMPLATE": ""
-  }
+  "WebExportTest": {}
 }

src/ResXManager.Tests/WebExporterOutput/src/resources.zh-Hans.json

@@ -1,8 +1,4 @@
 {
   "_comment": "Auto-generated; do not modify!",
-  "WebExportTest": {
-    "SimpleString": "",
-    "WithInterpolation_TEMPLATE": "",
-    "WithBadInterpolation_TEMPLATE": ""
-  }
+  "WebExportTest": {}
 }

src/ResXManager.Translators/ResXManager.Translators.csproj

@@ -53,13 +53,13 @@
     <PackageReference Include="Fody" PrivateAssets="all" />
     <PackageReference Include="ILMerge.Fody" PrivateAssets="all" />
     <PackageReference Include="Microsoft.ML.Tokenizers" />
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.CL100KBase" />
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" />
     <PackageReference Include="PropertyChanged.Fody" PrivateAssets="all" />
     <PackageReference Include="System.Net.Http" />
-    <PackageReference Include="System.Security.Cryptography.Pkcs" />
+    <PackageReference Include="System.ServiceModel.Primitives" />
     <PackageReference Include="System.Text.Json" />
     <PackageReference Include="Throttle.Fody" PrivateAssets="all" />
-    <PackageReference Include="System.ServiceModel.Primitives" />
-    <PackageReference Include="System.ServiceModel.Http" />
     <PackageReference Include="TomsToolbox.Wpf" />
     <PackageReference Include="TomsToolbox.Wpf.Composition.AttributedModel" />
     <PackageReference Include="TomsToolbox.Wpf.Styles" />

src/ResXManager.VSIX.Compatibility.Shared/CustomToolRunner.cs → src/ResXManager.VSIX.Compatibility.x64/CustomToolRunner.cs

@@ -1,4 +1,4 @@
-namespace ResXManager.VSIX;
+namespace ResXManager.VSIX.Compatibility.x64;
 
 using System;
 using System.Collections.Generic;
@@ -44,4 +44,4 @@ public void Dispose()
     {
         RunCustomTool();
     }
-}
+}

src/ResXManager.VSIX.Compatibility.Shared/DteConfiguration.cs → src/ResXManager.VSIX.Compatibility.x64/DteConfiguration.cs

@@ -1,4 +1,4 @@
-namespace ResXManager.VSIX;
+namespace ResXManager.VSIX.Compatibility.x64;
 
 using System;
 using System.ComponentModel;
@@ -14,7 +14,7 @@
 
 using static Microsoft.VisualStudio.Shell.ThreadHelper;
 
-using Configuration = ResXManager.Model.Configuration;
+using Configuration = Model.Configuration;
 
 [Shared]
 [Export(typeof(IConfiguration))]
@@ -48,16 +48,16 @@ public DteConfiguration(DteSolution solution, ITracer tracer)
 
         var solutionKey = GetSolutionKey(key);
 
-        if (!TryGetValueFromSolutionGlobals<T>(solutionKey, out var value)) 
+        if (!TryGetValueFromSolutionGlobals<T>(solutionKey, out var value))
             return base.InternalGetValue(defaultValue, key);
 
         Tracer.WriteLine("Convert old solution settings to new file based settings for key {0}, value {1}", solutionKey, value);
 
         // Convert old solution settings to new ones.
         TryClearValueFromSolutionGlobals(solutionKey);
-        
+
         base.InternalSetValue(value, key, false);
-        
+
         return value;
     }
 
@@ -66,7 +66,7 @@ public DteConfiguration(DteSolution solution, ITracer tracer)
         ThrowIfNotOnUIThread();
 
         TryClearValueFromSolutionGlobals(GetSolutionKey(key));
-        
+
         base.InternalSetValue(value, key, forceGlobal);
     }
 

src/ResXManager.VSIX.Compatibility.Shared/DteExtensions.cs → src/ResXManager.VSIX.Compatibility.x64/DteExtensions.cs

@@ -1,4 +1,4 @@
-namespace ResXManager.VSIX;
+namespace ResXManager.VSIX.Compatibility.x64;
 
 using System;
 using System.Collections.Generic;
@@ -10,14 +10,13 @@
 using Community.VisualStudio.Toolkit;
 
 using Microsoft.VisualStudio;
-
 using Microsoft.VisualStudio.Shell.Interop;
 
-using static Microsoft.VisualStudio.Shell.ThreadHelper;
-
 using ResXManager.View;
 using ResXManager.VSIX.Compatibility;
 
+using static Microsoft.VisualStudio.Shell.ThreadHelper;
+
 internal static class DteExtensions
 {
     public static EnvDTE.Document? TryGetDocument(this EnvDTE.ProjectItem? projectItem)

src/ResXManager.VSIX.Compatibility.Shared/DteProjectFile.cs → src/ResXManager.VSIX.Compatibility.x64/DteProjectFile.cs

@@ -1,12 +1,10 @@
-namespace ResXManager.VSIX;
+namespace ResXManager.VSIX.Compatibility.x64;
 
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 
-using EnvDTE;
-
 using ResXManager.Model;
 using ResXManager.VSIX.Compatibility;
 using ResXManager.VSIX.Compatibility.Properties;
@@ -186,6 +184,9 @@ private void SetCodeGenerator(CodeGenerator value)
                     case CodeGenerator.Unknown:
                     case CodeGenerator.WinForms:
                         break;
+
+                    default:
+                        throw new ArgumentOutOfRangeException(nameof(value), value, null);
                 }
             }
         }
@@ -233,12 +234,12 @@ private void SetTextTemplateCodeGenerator(EnvDTE.ProjectItem projectItem)
         item.RunCustomTool();
     }
 
-    private static void ReferenceDataAnnotations(ProjectItem projectItem)
+    private static void ReferenceDataAnnotations(EnvDTE.ProjectItem projectItem)
     {
         try
         {
             ThrowIfNotOnUIThread();
-                
+
             const string dataAnnotations = "System.ComponentModel.DataAnnotations";
 
             ThrowIfNotOnUIThread();
@@ -269,4 +270,4 @@ private static void SetCustomToolCodeGenerator(EnvDTE.ProjectItem projectItem, C
 
         projectItem.SetCustomTool(value.ToString());
     }
-}
+}