fix: position encoding to ensure correct formatting for special chars

apps/engine/lib/engine/code_mod/diff.ex

@@ -101,7 +101,7 @@ defmodule Engine.CodeMod.Diff do
   end
 
   defp advance(<<c::utf8, rest::binary>>, {line, unit}, edits) do
-    increment = CodeUnit.count(:utf8, <<c::utf8>>)
+    increment = CodeUnit.count(:utf16, <<c::utf8>>)
     advance(rest, {line, unit + increment}, edits)
   end
 

apps/engine/test/engine/code_mod/diff_test.exs

@@ -180,7 +180,7 @@ defmodule Engine.CodeMod.DiffTest do
       final = ~S[{"🎸", "after"}]
 
       assert [edit] = diff(orig, final)
-      assert_normalized(edit == edit(1, 10, 1, 12, ""))
+      assert_normalized(edit == edit(1, 8, 1, 10, ""))
       assert_edited(orig, final)
     end
 
@@ -189,7 +189,7 @@ defmodule Engine.CodeMod.DiffTest do
       final = ~S[🎸🎺🎸]
 
       assert [edit] = diff(orig, final)
-      assert_normalized(edit == edit(1, 5, 1, 5, "🎺"))
+      assert_normalized(edit == edit(1, 3, 1, 3, "🎺"))
       assert_edited(orig, final)
     end
 
@@ -198,7 +198,7 @@ defmodule Engine.CodeMod.DiffTest do
       final = ~S[🎸🎸]
 
       assert [edit] = diff(orig, final)
-      assert_normalized(edit == edit(1, 5, 1, 13, ""))
+      assert_normalized(edit == edit(1, 3, 1, 7, ""))
       assert_edited(orig, final)
     end
 

apps/engine/test/engine/code_mod/format_test.exs

@@ -114,5 +114,23 @@ defmodule Engine.CodeMod.FormatTest do
 
       assert result == formatted()
     end
+
+    test "it handles special characters", %{project: project} do
+      assert {:ok, result} =
+               ~q"""
+               [
+                 {"Karolína Plíšková","Kristýna Plíšková"}
+               ]
+               """
+               |> modify(project: project)
+
+      assert result ==
+               """
+               [
+                 {"Karolína Plíšková", "Kristýna Plíšková"}
+               ]
+               """
+               |> String.trim()
+    end
   end
 end

apps/forge/lib/test/code_mod_case.ex

@@ -1,4 +1,5 @@
 defmodule Forge.Test.CodeMod.Case do
+  alias Forge.CodeUnit
   alias Forge.Document
   alias Forge.Test.CodeSigil
 
@@ -41,7 +42,8 @@ defmodule Forge.Test.CodeMod.Case do
 
   def apply_edits(original, text_edits, opts) do
     document = Document.new("file:///file.ex", original, 0)
-    {:ok, edited_document} = Document.apply_content_changes(document, 1, text_edits)
+    utf8_edits = Enum.map(text_edits, &convert_edit_utf16_to_utf8(document, &1))
+    {:ok, edited_document} = Document.apply_content_changes(document, 1, utf8_edits)
     edited_document = Document.to_string(edited_document)
 
     if Keyword.get(opts, :trim, true) do
@@ -50,4 +52,35 @@ defmodule Forge.Test.CodeMod.Case do
       edited_document
     end
   end
+
+  defp convert_edit_utf16_to_utf8(document, %Document.Edit{} = edit) do
+    case edit.range do
+      nil ->
+        edit
+
+      range ->
+        start_pos = convert_position_utf16_to_utf8(document, range.start)
+        end_pos = convert_position_utf16_to_utf8(document, range.end)
+        %{edit | range: %{range | start: start_pos, end: end_pos}}
+    end
+  end
+
+  defp convert_position_utf16_to_utf8(document, %Document.Position{} = position) do
+    case Document.fetch_text_at(document, position.line) do
+      {:ok, line_text} ->
+        case CodeUnit.utf16_offset_to_utf8_offset(line_text, position.character - 1) do
+          {:ok, utf8_position} ->
+            Document.Position.new(document, position.line, utf8_position)
+
+          {:error, :out_of_bounds} ->
+            Document.Position.new(document, position.line, byte_size(line_text) + 1)
+
+          {:error, :misaligned} ->
+            position
+        end
+
+      :error ->
+        position
+    end
+  end
 end