feat: reading skipped in unidic lex that matches surface in katakana

rust/src/japanese.rs

@@ -3,6 +3,7 @@ pub trait JapaneseChar {
     fn is_kana(&self) -> bool;
     fn is_hiragana(&self) -> bool;
     fn is_katakana(&self) -> bool;
+    fn to_katakana(&self) -> char;
 }
 
 impl JapaneseChar for char {
@@ -17,16 +18,29 @@ impl JapaneseChar for char {
     fn is_katakana(&self) -> bool {
         matches!(*self, '\u{30a0}'..='\u{30ff}')
     }
+
+    fn to_katakana(&self) -> char {
+        if *self >= '\u{3041}' && *self <= '\u{3096}' {
+            char::from_u32(*self as u32 + 96).unwrap()
+        } else {
+            *self
+        }
+    }
 }
 
 pub trait JapaneseString {
     fn contains_only_kana(&self) -> bool;
+    fn to_katakana(&self) -> String;
 }
 
 impl JapaneseString for str {
     fn contains_only_kana(&self) -> bool {
         self.chars().all(|c| c.is_kana())
     }
+
+    fn to_katakana(&self) -> String {
+        self.chars().map(|c| c.to_katakana()).collect()
+    }
 }
 
 /// 五段

rust/src/tokenize.rs

@@ -141,7 +141,7 @@ impl TokenDetails {
             .next()
             .map(|r| {
                 if r.is_empty() {
-                    surface.into()
+                    surface.to_katakana()
                 } else {
                     r.to_string()
                 }

unidic/src/transform.rs

@@ -354,7 +354,8 @@ fn identical_base_to_empty_string(items: &mut Vec<LexItem>) {
 /// if item.surface == item.reading (most kana-only words), set reading to "" to save space
 fn identical_reading_to_empty_string(items: &mut Vec<LexItem>) {
     for item in items {
-        if item.surface == item.reading {
+        let katakana_surface: String = item.surface.chars().map(|c| c.to_katakana()).collect();
+        if katakana_surface == item.reading {
             item.reading = "".into()
         }
     }
@@ -492,6 +493,7 @@ pub trait JapaneseChar {
     fn is_kana(&self) -> bool;
     fn is_hiragana(&self) -> bool;
     fn is_katakana(&self) -> bool;
+    fn to_katakana(&self) -> char;
 }
 
 impl JapaneseChar for char {
@@ -506,4 +508,12 @@ impl JapaneseChar for char {
     fn is_katakana(&self) -> bool {
         matches!(*self, '\u{30a0}'..='\u{30ff}')
     }
+
+    fn to_katakana(&self) -> char {
+        if *self >= '\u{3041}' && *self <= '\u{3096}' {
+            char::from_u32(*self as u32 + 96).unwrap()
+        } else {
+            *self
+        }
+    }
 }