Skip to content

Commit

Permalink
feat: support '=' in the plural parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
stepan662 committed Feb 27, 2024
1 parent 1e7a571 commit 2ef5054
Show file tree
Hide file tree
Showing 6 changed files with 58 additions and 28 deletions.
6 changes: 3 additions & 3 deletions src/parser/lezer/tolgee.grammar
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ VariantContent { contentNested? }

Offset { offset_definition }
SelectFunction { select_function }
Param { name }
Param { identifier }
VariantDescriptor { variant_descriptor }
ExpressionOpen { "{" }
ExpressionClose { "}" }
Expand Down Expand Up @@ -48,9 +48,9 @@ HtmlTagNested { HtmlTagOpenNested | HtmlTagCloseNested }
plural_placeholder { '#' }
// non-White_Space and non-PatternSyntax
// https://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
name { ![\u{0009}-\u{000d}\u{0020}\u{0085}\u{00A0}\u{1680}\u{2000}-\u{200A}\u{2028}\u{2029}\u{202F}\u{205F}\u{3000}\u{0021}-\u{0023}\u{0024}\u{0025}-\u{0027}\u{0028}\u{0029}\u{002A}\u{002B}\u{002C}\u{002D}\u{002E}-\u{002F}\u{003A}-\u{003B}\u{003C}-\u{003E}\u{003F}-\u{0040}\u{005B}\u{005C}\u{005D}\u{005E}\u{0060}\u{007B}\u{007C}\u{007D}\u{007E}\u{00A1}\u{00A2}-\u{00A5}\u{00A6}\u{00A7}\u{00A9}\u{00AB}\u{00AC}\u{00AE}\u{00B0}\u{00B1}\u{00B6}\u{00BB}\u{00BF}\u{00D7}\u{00F7}\u{2010}-\u{2015}\u{2016}-\u{2017}\u{2018}\u{2019}\u{201A}\u{201B}-\u{201C}\u{201D}\u{201E}\u{201F}\u{2020}-\u{2027}\u{2030}-\u{2038}\u{2039}\u{203A}\u{203B}-\u{203E}\u{2041}-\u{2043}\u{2044}\u{2045}\u{2046}\u{2047}-\u{2051}\u{2052}\u{2053}\u{2055}-\u{205E}\u{2190}-\u{2194}\u{2195}-\u{2199}\u{219A}-\u{219B}\u{219C}-\u{219F}\u{21A0}\u{21A1}-\u{21A2}\u{21A3}\u{21A4}-\u{21A5}\u{21A6}\u{21A7}-\u{21AD}\u{21AE}\u{21AF}-\u{21CD}\u{21CE}-\u{21CF}\u{21D0}-\u{21D1}\u{21D2}\u{21D3}\u{21D4}\u{21D5}-\u{21F3}\u{21F4}-\u{22FF}\u{2300}-\u{2307}\u{2308}\u{2309}\u{230A}\u{230B}\u{230C}-\u{231F}\u{2320}-\u{2321}\u{2322}-\u{2328}\u{2329}\u{232A}\u{232B}-\u{237B}\u{237C}\u{237D}-\u{239A}\u{239B}-\u{23B3}\u{23B4}-\u{23DB}\u{23DC}-\u{23E1}\u{23E2}-\u{2426}\u{2427}-\u{243F}\u{2440}-\u{244A}\u{244B}-\u{245F}\u{2500}-\u{25B6}\u{25B7}\u{25B8}-\u{25C0}\u{25C1}\u{25C2}-\u{25F7}\u{25F8}-\u{25FF}\u{2600}-\u{266E}\u{266F}\u{2670}-\u{2767}\u{2768}\u{2769}\u{276A}\u{276B}\u{276C}\u{276D}\u{276E}\u{276F}\u{2770}\u{2771}\u{2772}\u{2773}\u{2774}\u{2775}\u{2794}-\u{27BF}\u{27C0}-\u{27C4}\u{27C5}\u{27C6}\u{27C7}-\u{27E5}\u{27E6}\u{27E7}\u{27E8}\u{27E9}\u{27EA}\u{27EB}\u{27EC}\u{27ED}\u{27EE}\u{27EF}\u{27F0}-\u{27FF}\u{2800}-\u{28FF}\u{2900}-\u{2982}\u{2983}\u{2984}\u{2985}\u{2986}\u{2987}\u{2988}\u{2989}\u{298A}\u{298B}\u{298C}\u{298D}\u{298E}\u{298F}\u{2990}\u{2991}\u{2992}\u{2993}\u{2994}\u{2995}\u{2996}\u{2997}\u{2998}\u{2999}-\u{29D7}\u{29D8}\u{29D9}\u{29DA}\u{29DB}\u{29DC}-\u{29FB}\u{29FC}\u{29FD}\u{29FE}-\u{2AFF}\u{2B00}-\u{2B2F}\u{2B30}-\u{2B44}\u{2B45}-\u{2B46}\u{2B47}-\u{2B4C}\u{2B4D}-\u{2B73}\u{2B74}-\u{2B75}\u{2B76}-\u{2B95}\u{2B96}\u{2B97}-\u{2BFF}\u{2E00}-\u{2E01}\u{2E02}\u{2E03}\u{2E04}\u{2E05}\u{2E06}-\u{2E08}\u{2E09}\u{2E0A}\u{2E0B}\u{2E0C}\u{2E0D}\u{2E0E}-\u{2E16}\u{2E17}\u{2E18}-\u{2E19}\u{2E1A}\u{2E1B}\u{2E1C}\u{2E1D}\u{2E1E}-\u{2E1F}\u{2E20}\u{2E21}\u{2E22}\u{2E23}\u{2E24}\u{2E25}\u{2E26}\u{2E27}\u{2E28}\u{2E29}\u{2E2A}-\u{2E2E}\u{2E2F}\u{2E30}-\u{2E39}\u{2E3A}-\u{2E3B}\u{2E3C}-\u{2E3F}\u{2E40}\u{2E41}\u{2E42}\u{2E43}-\u{2E4F}\u{2E50}-\u{2E51}\u{2E52}-\u{2E54}\u{2E55}\u{2E56}\u{2E57}\u{2E58}\u{2E59}\u{2E5A}\u{2E5B}\u{2E5C}\u{2E5D}\u{2E5E}-\u{2E7F}\u{3001}-\u{3003}\u{3008}\u{3009}\u{300A}\u{300B}\u{300C}\u{300D}\u{300E}\u{300F}\u{3010}\u{3011}\u{3012}-\u{3013}\u{3014}\u{3015}\u{3016}\u{3017}\u{3018}\u{3019}\u{301A}\u{301B}\u{301C}\u{301D}\u{301E}-\u{301F}\u{3020}\u{3030}\u{FD3E}\u{FD3F}\u{FE45}-\u{FE46}]+ }
identifier { ![\u{0009}-\u{000d}\u{0020}\u{0085}\u{00A0}\u{1680}\u{2000}-\u{200A}\u{2028}\u{2029}\u{202F}\u{205F}\u{3000}\u{0021}-\u{0023}\u{0024}\u{0025}-\u{0027}\u{0028}\u{0029}\u{002A}\u{002B}\u{002C}\u{002D}\u{002E}-\u{002F}\u{003A}-\u{003B}\u{003C}-\u{003E}\u{003F}-\u{0040}\u{005B}\u{005C}\u{005D}\u{005E}\u{0060}\u{007B}\u{007C}\u{007D}\u{007E}\u{00A1}\u{00A2}-\u{00A5}\u{00A6}\u{00A7}\u{00A9}\u{00AB}\u{00AC}\u{00AE}\u{00B0}\u{00B1}\u{00B6}\u{00BB}\u{00BF}\u{00D7}\u{00F7}\u{2010}-\u{2015}\u{2016}-\u{2017}\u{2018}\u{2019}\u{201A}\u{201B}-\u{201C}\u{201D}\u{201E}\u{201F}\u{2020}-\u{2027}\u{2030}-\u{2038}\u{2039}\u{203A}\u{203B}-\u{203E}\u{2041}-\u{2043}\u{2044}\u{2045}\u{2046}\u{2047}-\u{2051}\u{2052}\u{2053}\u{2055}-\u{205E}\u{2190}-\u{2194}\u{2195}-\u{2199}\u{219A}-\u{219B}\u{219C}-\u{219F}\u{21A0}\u{21A1}-\u{21A2}\u{21A3}\u{21A4}-\u{21A5}\u{21A6}\u{21A7}-\u{21AD}\u{21AE}\u{21AF}-\u{21CD}\u{21CE}-\u{21CF}\u{21D0}-\u{21D1}\u{21D2}\u{21D3}\u{21D4}\u{21D5}-\u{21F3}\u{21F4}-\u{22FF}\u{2300}-\u{2307}\u{2308}\u{2309}\u{230A}\u{230B}\u{230C}-\u{231F}\u{2320}-\u{2321}\u{2322}-\u{2328}\u{2329}\u{232A}\u{232B}-\u{237B}\u{237C}\u{237D}-\u{239A}\u{239B}-\u{23B3}\u{23B4}-\u{23DB}\u{23DC}-\u{23E1}\u{23E2}-\u{2426}\u{2427}-\u{243F}\u{2440}-\u{244A}\u{244B}-\u{245F}\u{2500}-\u{25B6}\u{25B7}\u{25B8}-\u{25C0}\u{25C1}\u{25C2}-\u{25F7}\u{25F8}-\u{25FF}\u{2600}-\u{266E}\u{266F}\u{2670}-\u{2767}\u{2768}\u{2769}\u{276A}\u{276B}\u{276C}\u{276D}\u{276E}\u{276F}\u{2770}\u{2771}\u{2772}\u{2773}\u{2774}\u{2775}\u{2794}-\u{27BF}\u{27C0}-\u{27C4}\u{27C5}\u{27C6}\u{27C7}-\u{27E5}\u{27E6}\u{27E7}\u{27E8}\u{27E9}\u{27EA}\u{27EB}\u{27EC}\u{27ED}\u{27EE}\u{27EF}\u{27F0}-\u{27FF}\u{2800}-\u{28FF}\u{2900}-\u{2982}\u{2983}\u{2984}\u{2985}\u{2986}\u{2987}\u{2988}\u{2989}\u{298A}\u{298B}\u{298C}\u{298D}\u{298E}\u{298F}\u{2990}\u{2991}\u{2992}\u{2993}\u{2994}\u{2995}\u{2996}\u{2997}\u{2998}\u{2999}-\u{29D7}\u{29D8}\u{29D9}\u{29DA}\u{29DB}\u{29DC}-\u{29FB}\u{29FC}\u{29FD}\u{29FE}-\u{2AFF}\u{2B00}-\u{2B2F}\u{2B30}-\u{2B44}\u{2B45}-\u{2B46}\u{2B47}-\u{2B4C}\u{2B4D}-\u{2B73}\u{2B74}-\u{2B75}\u{2B76}-\u{2B95}\u{2B96}\u{2B97}-\u{2BFF}\u{2E00}-\u{2E01}\u{2E02}\u{2E03}\u{2E04}\u{2E05}\u{2E06}-\u{2E08}\u{2E09}\u{2E0A}\u{2E0B}\u{2E0C}\u{2E0D}\u{2E0E}-\u{2E16}\u{2E17}\u{2E18}-\u{2E19}\u{2E1A}\u{2E1B}\u{2E1C}\u{2E1D}\u{2E1E}-\u{2E1F}\u{2E20}\u{2E21}\u{2E22}\u{2E23}\u{2E24}\u{2E25}\u{2E26}\u{2E27}\u{2E28}\u{2E29}\u{2E2A}-\u{2E2E}\u{2E2F}\u{2E30}-\u{2E39}\u{2E3A}-\u{2E3B}\u{2E3C}-\u{2E3F}\u{2E40}\u{2E41}\u{2E42}\u{2E43}-\u{2E4F}\u{2E50}-\u{2E51}\u{2E52}-\u{2E54}\u{2E55}\u{2E56}\u{2E57}\u{2E58}\u{2E59}\u{2E5A}\u{2E5B}\u{2E5C}\u{2E5D}\u{2E5E}-\u{2E7F}\u{3001}-\u{3003}\u{3008}\u{3009}\u{300A}\u{300B}\u{300C}\u{300D}\u{300E}\u{300F}\u{3010}\u{3011}\u{3012}-\u{3013}\u{3014}\u{3015}\u{3016}\u{3017}\u{3018}\u{3019}\u{301A}\u{301B}\u{301C}\u{301D}\u{301E}-\u{301F}\u{3020}\u{3030}\u{FD3E}\u{FD3F}\u{FE45}-\u{FE46}]+ }
select_function { 'plural' | 'selectordinal' | 'select' }
variant_descriptor { name }
variant_descriptor { '='? ('-' | '+')? identifier }
space { @whitespace+ }
format_function { 'number' | 'date' | 'time' }
// non-White_Space
Expand Down
40 changes: 40 additions & 0 deletions src/parser/lezer/tolgeeParser.tags.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { parse } from "@formatjs/icu-messageformat-parser";
import { parser } from "./tolgeeParser";

function icu(text: string, ignoreTag = true) {
return parse(text, {
captureLocation: true,
ignoreTag,
});
}

function tolgee(text: string) {
return parser.configure({ strict: true, dialect: "tags" }).parse(text);
}

function getText() {
return expect.getState().currentTestName!.replace("simple formatter ", "");
}

function matchIcu(text = getText()) {
expect(() => icu(text)).not.toThrow();
expect(() => tolgee(text)).not.toThrow();
}

describe("tolgee parser with tags", () => {
test("This is < invalid > tag", () => {
matchIcu();
});

test("Test <a>valid</a> tag", () => {
matchIcu();
});

test("Test '<a>' escaped tag", () => {
matchIcu();
});

test("tag escaping causing also more stuff to be escaped", () => {
matchIcu("This '<a> {} is all escaped");
});
});
33 changes: 11 additions & 22 deletions src/parser/lezer/tolgeeParser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ function icu(text: string, ignoreTag = true) {
}

function tolgee(text: string) {
return parser.configure({ strict: true, dialect: "tags" }).parse(text);
return parser.configure({ strict: true }).parse(text);
}

function getText() {
Expand All @@ -21,14 +21,12 @@ function matchIcu(text = getText()) {
expect(() => tolgee(text)).not.toThrow();
}

function matchIcuWithTags(text = getText()) {
expect(() => icu(text, false)).not.toThrow();
expect(() => tolgee(text)).not.toThrow();
function expectToThrowWithIcu(text = getText()) {
expect(() => icu(text)).toThrow();
expect(() => tolgee(text)).toThrow();
}

function expectToThrowWithIcu() {
const text = getText();
expect(() => icu(text)).toThrow();
function expectToThrow(text = getText()) {
expect(() => tolgee(text)).toThrow();
}

Expand Down Expand Up @@ -142,22 +140,13 @@ describe("simple formatter", () => {
expectToThrowWithIcu();
});

test("This is < invalid > tag", () => {
matchIcu();
});

test("Test <a>valid</a> tag", () => {
matchIcuWithTags();
matchIcu();
});

test("Test '<a>' escaped tag", () => {
matchIcuWithTags();
matchIcu();
test("Wont take tags as escapabable", () => {
expectToThrow("{value, plural, other {'<>'}}");
});

test("tag escaping causing also more stuff to be escaped", () => {
matchIcuWithTags("This '<a> {} is all escaped");
matchIcu("This '<a> {} is all escaped");
test("Accepts ={number} format", () => {
matchIcu(
"Auto translated {test, plural, =1 {# translation} other {# translations}}"
);
});
});
Loading

0 comments on commit 2ef5054

Please sign in to comment.