Merge pull request #4 from tolgee/stepangranat/fix-unicode-chars-inde…

…xing

fix: double char unicode characters breaking placeholders

src/App.tsx

@@ -5,7 +5,7 @@ import { EditorMulti } from "./editor/EditorMulti";
 
 function App() {
   const [value, setValue] = useState(
-    `I have {value, number, ::short} items in basket. This is <a href="test">italic</i>.`
+    `🐭-I have {value, number, ::short} items in basket. This is <a href="test">italic</i>.`
   );
   const [placeholders, setPlaceholders] = useState(true);
 

src/parser/lezer/tokenizer/InputStreamLike.ts

@@ -1,17 +1,17 @@
 export const InputStreamLike = (input: string) => {
-  const codePoints = [...input].map((ch) => ch.charCodeAt(0));
-
   let position = 0;
 
   const self = {
     get next() {
-      return codePoints[position] ?? -1;
+      const char = input[position];
+      return char ? input.charCodeAt(position) : -1;
     },
     get pos() {
       return position;
     },
     peek(val: number) {
-      return codePoints[position + val] ?? -1;
+      const char = input[position + val];
+      return char ? input.charCodeAt(position + val) : -1;
     },
     advance(val = 1) {
       position += val;

src/parser/placeholders/getPlaceholders.test.ts

@@ -82,4 +82,19 @@ describe("get placeholders", () => {
     expect(placeholders![3].normalizedValue).toEqual("</a>");
     expect(placeholders![4].normalizedValue).toEqual("{test}");
   });
+
+  it("correctly indexes characters when unicode chars present - tags", () => {
+    const placeholders = getPlaceholders("🐭-<link>blog</link>");
+    const first = placeholders![0];
+    expect(first.normalizedValue).toEqual("<link>");
+    expect(first.position.start).toEqual(3);
+    expect(placeholders![1].normalizedValue).toEqual("</link>");
+  });
+
+  it("correctly indexes characters when unicode chars present - placeholders", () => {
+    const placeholders = getPlaceholders("🐭-{test}-");
+    const first = placeholders![0];
+    expect(first.normalizedValue).toEqual("{test}");
+    expect(first.position.start).toEqual(3);
+  });
 });