here we go!

README.md

@@ -1,6 +1,6 @@
 ---
-title: Mms Tts Demo
-emoji: 📊
+title: MMS-TTS Demo
+emoji: 🥳
 colorFrom: indigo
 colorTo: green
 sdk: gradio

app.py

@@ -0,0 +1,140 @@
+import gradio as gr
+import numpy as np
+import torch
+import os
+import re
+import tempfile
+
+from transformers import VitsModel, VitsTokenizer
+
+
+models = {
+    "English": VitsModel.from_pretrained("Matthijs/mms-tts-eng"),
+    "German": VitsModel.from_pretrained("Matthijs/mms-tts-deu"),
+    "Korean": VitsModel.from_pretrained("Matthijs/mms-tts-kor"),
+}
+
+tokenizers = {
+    "English": VitsTokenizer.from_pretrained("Matthijs/mms-tts-eng"),
+    "German": VitsTokenizer.from_pretrained("Matthijs/mms-tts-deu"),
+    "Korean": VitsTokenizer.from_pretrained("Matthijs/mms-tts-kor"),
+}
+
+
+# For certain checkpoints, the text needs to be romanized.
+# MMS-TTS uses uromanize.pl for this from https://github.com/isi-nlp/uroman
+# This needs to be installed in the folder "uroman"
+def uromanize(text, uroman_pl):
+    iso = "xxx"
+    with tempfile.NamedTemporaryFile() as tf, tempfile.NamedTemporaryFile() as tf2:
+        with open(tf.name, "w") as f:
+            f.write("\n".join([text]))
+        cmd = f"perl " + uroman_pl
+        cmd += f" -l {iso} "
+        cmd += f" < {tf.name} > {tf2.name}"
+        os.system(cmd)
+        outtexts = []
+        with open(tf2.name) as f:
+            for line in f:
+                line = re.sub(r"\s+", " ", line).strip()
+                outtexts.append(line)
+        outtext = outtexts[0]
+    return outtext
+
+
+def predict(text, language=None):
+    if len(text.strip()) == 0:
+        return (16000, np.zeros(0).astype(np.int16))
+
+    if language == "Korean":
+        uroman_pl = os.path.join("uroman", "bin", "uroman.pl")
+        text = uromanize(text, uroman_pl)
+
+    tokenizer = tokenizers[language]
+    inputs = tokenizer(text, return_tensors="pt")
+    input_ids = inputs["input_ids"]
+
+    if language != "Korean":
+        text = tokenizer.batch_decode(input_ids)[0]
+
+    model = models[language]
+    with torch.no_grad():
+        outputs = model(input_ids)
+
+    speech = outputs.audio[0]
+    speech = (speech.numpy() * 32767).astype(np.int16)
+    return (16000, speech), text
+
+
+title = "MMS-TTS speech synthesis"
+
+description = """
+Facebook's [Massively Multilingual Speech](https://arxiv.org/abs/2305.13516) project aims to provide
+speech technology across a diverse range of languages. The MMS-TTS project contains a collection of
+over 1000 text-to-speech (TTS) models.
+
+This demo shows how to use MMS-TTS using 🤗 Transformers. Since MMS-TTS is based on the VITS
+model, this code can also be used to run VITS checkpoints.
+For a full list of checkpoints, [click here](https://huggingface.co/models?filter=vits).
+
+As the model performs random sampling, the generated speech is slightly different each time.
+The voice may also vary between runs, or sometimes even in the same sentence.
+(Note that 🤗 Transformers also supports multispeaker VITS checkpoints but the MMS-TTS checkpoints
+are not conditioned on a speaker ID.)
+"""
+
+article = """
+<div style='margin:20px auto;'>
+
+<p>References: <a href="https://arxiv.org/abs/2305.13516">MMS paper</a> |
+<a href="https://ai.facebook.com/blog/multilingual-model-speech-recognition/">blog post</a> |
+<a href="https://huggingface.co/facebook/mms-tts">original weights</a> |
+<a href="https://huggingface.co/spaces/mms-meta/MMS">original MMS space</a>
+</p>
+
+<pre>
+@article{pratap2023mms,
+  title={Scaling Speech Technology to 1,000+ Languages},
+  author={Vineel Pratap and Andros Tjandra and Bowen Shi and Paden Tomasello and Arun Babu and Sayani Kundu and Ali Elkahky and Zhaoheng Ni and Apoorv Vyas and Maryam Fazel-Zarandi and Alexei Baevski and Yossi Adi and Xiaohui Zhang and Wei-Ning Hsu and Alexis Conneau and Michael Auli},
+  journal={arXiv},
+  year={2023}
+}
+</pre>
+
+</div>
+"""
+
+examples = [
+    ["It is not in the stars to hold our destiny but in ourselves.", "English"],
+    ["The octopus and Oliver went to the opera in October.", "English"],
+    ["She sells seashells by the seashore. I saw a kitten eating chicken in the kitchen.", "English"],
+    ["Brisk brave brigadiers brandished broad bright blades, blunderbusses, and bludgeons—balancing them badly.", "English"],
+    ["A synonym for cinnamon is a cinnamon synonym.", "English"],
+    ["How much wood would a woodchuck chuck if a woodchuck could chuck wood?", "English"],
+
+    ["Eins, zwei, Polizei. Drei, vier, Grenadier. Fünf, sechs, alte Keks. Sieben, acht, gute Nacht.", "German"],
+    ["Alle meine Entchen, schwimmen auf dem See. Köpfchen in das Wasser, Schwänzchen in die Höh.", "German"],
+
+    ["안녕 세상, 날씨는 아름다워", "Korean"],  # Hello world, the weather is beautiful (Google Translate)
+]
+
+gr.Interface(
+    fn=predict,
+    inputs=[
+        gr.Text(label="Input Text"),
+        gr.Radio(label="Language", choices=[
+            "English",
+            "German",
+            "Korean",
+        ],
+        value="English"),
+    ],
+    outputs=[
+        gr.Audio(label="Generated Speech", type="numpy"),
+        gr.Text(label="Processed text"),
+    ],
+    title=title,
+    description=description,
+    article=article,
+    examples=examples,
+).launch()

requirements.txt

@@ -0,0 +1,8 @@
+git+https://github.com/hollance/transformers.git@vits
+torch
+torchaudio
+soundfile
+librosa
+samplerate
+resampy
+sentencepiece

uroman/.gitignore

@@ -0,0 +1,35 @@
+!Build/
+.last_cover_stats
+/META.yml
+/META.json
+/MYMETA.*
+*.o
+*.pm.tdy
+*.bs
+
+# Devel::Cover
+cover_db/
+
+# Devel::NYTProf
+nytprof.out
+
+# Dizt::Zilla
+/.build/
+
+# Module::Build
+_build/
+Build
+Build.bat
+
+# Module::Install
+inc/
+
+# ExtUtils::MakeMaker
+/blib/
+/_eumm/
+/*.gz
+/Makefile
+/Makefile.old
+/MANIFEST.bak
+/pm_to_blib
+/*.zip

uroman/LICENSE.txt

@@ -0,0 +1,11 @@
+Copyright (C) 2015-2020 Ulf Hermjakob, USC Information Sciences Institute
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+Any publication of projects using uroman shall acknowledge its use: "This project uses the universal romanizer software 'uroman' written by Ulf Hermjakob, USC Information Sciences Institute (2015-2020)".
+Bibliography: Ulf Hermjakob, Jonathan May, and Kevin Knight. 2018. Out-of-the-box universal romanization tool uroman. In Proceedings of the 56th Annual Meeting of Association for Computational Linguistics, Demo Track.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

uroman/README.md

@@ -0,0 +1,165 @@
+# uroman
+
+*uroman* is a *universal romanizer*. It converts text in any script to the Latin alphabet.
+
+Version: 1.2.8
+Release date: April 23, 2021  
+Author: Ulf Hermjakob, USC Information Sciences Institute  
+
+
+### Usage
+```bash
+$ uroman.pl [-l <lang-code>] [--chart] [--no-cache] < STDIN
+       where the optional <lang-code> is a 3-letter languages code, e.g. ara, bel, bul, deu, ell, eng, fas,
+            grc, ell, eng, heb, kaz, kir, lav, lit, mkd, mkd2, oss, pnt, pus, rus, srp, srp2, tur, uig, ukr, yid.
+       --chart specifies chart output (in JSON format) to represent alternative romanizations.
+       --no-cache disables caching.
+```
+### Examples
+```bash
+$ bin/uroman.pl < text/zho.txt
+$ bin/uroman.pl -l tur < text/tur.txt
+$ bin/uroman.pl -l heb --chart < text/heb.txt
+$ bin/uroman.pl < test/multi-script.txt > test/multi-script.uroman.txt
+```
+
+Identifying the input as Arabic, Belarusian, Bulgarian, English, Farsi, German,
+Ancient Greek, Modern Greek, Pontic Greek, Hebrew, Kazakh, Kyrgyz, Latvian,
+Lithuanian, North Macedonian, Russian, Serbian, Turkish, Ukrainian, Uyghur or 
+Yiddish will improve romanization for those languages as some letters in those 
+languages have different sound values from other languages using the same script 
+(French, Russian, Hebrew respectively).
+No effect for other languages in this version.
+
+### Bibliography
+Ulf Hermjakob, Jonathan May, and Kevin Knight. 2018. Out-of-the-box universal romanization tool uroman. In Proceedings of the 56th Annual Meeting of Association for Computational Linguistics, Demo Track. ACL-2018 Best Demo Paper Award. [Paper in ACL Anthology](https://www.aclweb.org/anthology/P18-4003) | [Poster](https://www.isi.edu/~ulf/papers/poster-uroman-acl2018.pdf) | [BibTex](https://www.aclweb.org/anthology/P18-4003.bib)
+
+### Change History
+Changes in version 1.2.8
+ * Updated to Unicode 13.0 (2021), which supports several new scripts (10% larger UnicodeData.txt).
+ * Improved support for Georgian.
+ * Preserve various symbols (as opposed to mapping to the symbols' names).
+ * Various small improvements.
+
+Changes in version 1.2.7
+ * Improved support for Pashto.
+
+Changes in version 1.2.6
+ * Improved support for Ukrainian, Russian and Ogham (ancient Irish script).
+ * Added support for English Braille.
+ * Added alternative Romanization for North Macedonian and Serbian (mkd2/srp2)
+   reflecting a casual style that many native speakers of those languages use
+   when writing text in Latin script, e.g. non-accented single letters (e.g. "s")
+   rather than phonetically motivated combinations of letters (e.g. "sh").
+ * When a line starts with "::lcode xyz ", the new uroman version will switch to
+   that language for that line. This is used for the new reference test file.
+ * Various small improvements.
+
+Changes in version 1.2.5
+ * Improved support for Armenian and eight languages using Cyrillic scripts.
+   -- For Serbian and Macedonian, which are often written in both Cyrillic
+      and Latin scripts, uroman will map both official versions to the same
+      romanized text, e.g. both "Ниш" and "Niš" will be mapped to "Nish" (which
+      properly reflects the pronunciation of the city's name).
+      For both Serbian and Macedonian, casual writers often use a simplified
+      Latin form without diacritics, e.g. "s" to represent not only Cyrillic "с"
+      and Latin "s", but also "ш" or "š", even if this conflates "s" and "sh" and
+      other such pairs. The casual romanization can be simulated by using
+      alternative uroman language codes "srp2" and "mkd2", which romanize
+      both "Ниш" and "Niš" to "Nis" to reflect the casual Latin spelling.
+ * Various small improvements.
+
+Changes in version 1.2.4
+  * Bug-fix that generated two emtpy lines for each empty line in cache mode.
+
+Changes in version 1.2
+ * Run-time improvement based on (1) token-based caching and (2) shortcut 
+   romanization (identity) of ASCII strings for default 1-best (non-chart) 
+   output. Speed-up by a factor of 10 for Bengali and Uyghur on medium and 
+   large size texts.
+ * Incremental improvements for Farsi, Amharic, Russian, Hebrew and related
+   languages.
+ * Richer lattice structure (more alternatives) for "Romanization" of English
+   to support better matching to romanizations of other languages.
+   Changes output only when --chart option is specified. No change in output for
+   default 1-best output, which for ASCII characters is always the input string.
+
+Changes in version 1.1 (major upgrade)
+ * Offers chart output (in JSON format) to represent alternative romanizations.
+   -- Location of first character is defined to be "line: 1, start:0, end:0".
+ * Incremental improvements of Hebrew and Greek romanization; Chinese numbers.
+ * Improved web-interface at http://www.isi.edu/~ulf/uroman.html
+   -- Shows corresponding original and romanization text in red
+      when hovering over a text segment.
+   -- Shows alternative romanizations when hovering over romanized text
+      marked by dotted underline.
+   -- Added right-to-left script detection and improved display for right-to-left
+      script text (as determined line by line).
+   -- On-page support for some scripts that are often not pre-installed on users'
+      computers (Burmese, Egyptian, Klingon).
+
+Changes in version 1.0 (major upgrade)
+ * Upgraded principal internal data structure from string to lattice.
+ * Improvements mostly in vowelization of South and Southeast Asian languages.
+ * Vocalic 'r' more consistently treated as vowel (no additional vowel added).
+ * Repetition signs (Japanese/Chinese/Thai/Khmer/Lao) are mapped to superscript 2.
+ * Japanese Katakana middle dots now mapped to ASCII space.
+ * Tibetan intersyllabic mark now mapped to middle dot (U+00B7).
+ * Some corrections regarding analysis of Chinese numbers.
+ * Many more foreign diacritics and punctuation marks dropped or mapped to ASCII.
+ * Zero-width characters dropped, except line/sentence-initial byte order marks.
+ * Spaces normalized to ASCII space.
+ * Fixed bug that in some cases mapped signs (such as dagger or bullet) to their verbal descriptions.
+ * Tested against previous version of uroman with a new uroman visual diff tool.
+ * Almost an order of magnitude faster.
+
+Changes in version 0.7 (minor upgrade)
+ * Added script uroman-quick.pl for Arabic script languages, incl. Uyghur.
+   Much faster, pre-caching mapping of Arabic to Latin characters, simple greedy processing.
+   Will not convert material from non-Arabic blocks such as any (somewhat unusual) Cyrillic
+   or Chinese characters in Uyghur texts.
+
+Changes in version 0.6 (minor upgrade)
+ * Added support for two letter characters used in Uzbek:
+   (1) character "ʻ" ("modifier letter turned comma", which modifies preceding "g" and "u" letters)
+   (2) character "ʼ" ("modifier letter apostrophe", which Uzbek uses to mark a glottal stop).
+   Both are now mapped to "'" (plain ASCII apostrophe).
+ * Added support for Uyghur vowel characters such as "ې" (Arabic e) and "ۆ" (Arabic oe)
+   even when they are not preceded by "ئ" (yeh with hamza above).
+ * Added support for Arabic semicolon "؛", Arabic ligature forms for phrases such as "ﷺ"
+   ("sallallahou alayhe wasallam" = "prayer of God be upon him and his family and peace")
+ * Added robustness for Arabic letter presentation forms (initial/medial/final/isolated).
+   However, it is strongly recommended to normalize any presentation form Arabic letters
+   to their non-presentation form before calling uroman.
+ * Added force flush directive ($|=1;).
+
+Changes in version 0.5 (minor upgrade)
+ * Improvements for Uyghur (make sure to use language option: -l uig)
+
+Changes in version 0.4 (minor upgrade)
+ * Improvements for Thai (special cases for vowel/consonant reordering, e.g. for "sara o"; dropped some aspiration 'h's)
+ * Minor change for Arabic (added "alef+fathatan" = "an")
+
+New features in version 0.3
+ * Covers Mandarin (Chinese)
+ * Improved romanization for numerous languages
+ * Preserves capitalization (e.g. from Latin, Cyrillic, Greek scripts)
+ * Maps from native digits to Western numbers
+ * Faster for South Asian languages
+
+### Other features
+ * Web interface: http://www.isi.edu/~ulf/uroman.html
+ * Vowelization is provided when locally computable, e.g. for many South Asian languages and Tibetan.
+
+### Limitations
+ * The current version of uroman has a few limitations, some of which we plan to address in future versions.
+   For Japanese, *uroman* currently romanizes hiragana and katakana as expected, but kanji are interpreted as Chinese characters and romanized as such. 
+   For Egyptian hieroglyphs, only single-sound phonetic characters and numbers are currently romanized. 
+   For Linear B, only phonetic syllabic characters are romanized. 
+   For some other extinct scripts such as cuneiform, no romanization is provided.
+ * A romanizer is not a full transliterator. For example, this version of
+   uroman does not vowelize text that lacks explicit vowelization such as
+   normal text in Arabic and Hebrew (without diacritics/points).
+
+### Acknowledgments
+This research is based upon work supported in part by the Office of the Director of National Intelligence (ODNI), Intelligence Advanced Research Projects Activity (IARPA), via contract # FA8650-17-C-9116, and by research sponsored by Air Force Research Laboratory (AFRL) under agreement number FA8750-19-1-1000. The views and conclusions contained herein are those of the authors and should not be interpreted as necessarily representing the official policies, either expressed or implied, of ODNI, IARPA, Air Force Laboratory, DARPA, or the U.S. Government. The U.S. Government is authorized to reproduce and distribute reprints for governmental purposes notwithstanding any copyright annotation therein.

uroman/README.txt

@@ -0,0 +1,141 @@
+uroman version 1.2.8
+Release date: April 23, 2021
+Author: Ulf Hermjakob, USC Information Sciences Institute
+
+uroman is a universal romanizer. It converts text in any script to the Latin alphabet.
+
+Usage: uroman.pl [-l <lang-code>] [--chart] [--no-cache] < STDIN
+       where the optional <lang-code> is a 3-letter languages code, e.g. ara, bel, bul, deu, ell, eng, fas,
+            grc, ell, eng, heb, kaz, kir, lav, lit, mkd, mkd2, oss, pnt, pus, rus, srp, srp2, tur, uig, ukr, yid.
+       --chart specifies chart output (in JSON format) to represent alternative romanizations.
+       --no-cache disables caching.
+Examples: bin/uroman.pl < text/zho.txt
+          bin/uroman.pl -l tur < text/tur.txt
+	  bin/uroman.pl -l heb --chart < text/heb.txt
+	  bin/uroman.pl < test/multi-script.txt > test/multi-script.uroman.txt
+
+Identifying the input as Arabic, Belarusian, Bulgarian, English, Farsi, German, 
+Ancient Greek, Modern Greek, Pontic Greek, Hebrew, Kazakh, Kyrgyz, Latvian, 
+Lithuanian, North Macedonian, Russian, Serbian, Turkish, Ukrainian, Uyghur or Yiddish 
+will improve romanization for those languages as some letters in those languages 
+have different sound values from other languages using the same script.
+No effect for other languages in this version.
+
+Bibliography: Ulf Hermjakob, Jonathan May, and Kevin Knight. 2018. Out-of-the-box universal romanization tool uroman. In Proceedings of the 56th Annual Meeting of Association for Computational Linguistics, Demo Track. [Best Demo Paper Award]
+
+Changes in version 1.2.8
+ * Improved support for Georgian.
+ * Updated UnicodeData.txt to version 13 (2021) with several new scripts (10% larger).
+ * Preserve various symbols (as opposed to mapping to the symbols' names).
+ * Various small improvements.
+Changes in version 1.2.7
+ * Improved support for Pashto.
+Changes in version 1.2.6
+ * Improved support for Ukrainian, Russian and Ogham (ancient Irish script).
+ * Added support for English Braille.
+ * Added alternative Romanization for North Macedonian and Serbian (mkd2/srp2)
+   reflecting a casual style that many native speakers of those languages use
+   when writing text in Latin script, e.g. non-accented single letters (e.g. "s")
+   rather than phonetically motivated combinations of letters (e.g. "sh").
+ * When a line starts with "::lcode xyz ", the new uroman version will switch to
+   that language for that line. This is used for the new reference test file.
+ * Various small improvements.
+Changes in version 1.2.5
+ * Improved support for Armenian and eight languages using Cyrillic scripts.
+   -- For Serbian and Macedonian, which are often written in both Cyrillic
+      and Latin scripts, uroman will map both official versions to the same
+      romanized text, e.g. both "Ниш" and "Niš" will be mapped to "Nish" (which
+      properly reflects the pronunciation of the city's name).
+      For both Serbian and Macedonian, casual writers often use a simplified
+      Latin form without diacritics, e.g. "s" to represent not only Cyrillic "с"
+      and Latin "s", but also "ш" or "š", even if this conflates "s" and "sh" and
+      other such pairs. The casual romanization can be simulated by using
+      alternative uroman language codes "srp2" and "mkd2", which romanize
+      both "Ниш" and "Niš" to "Nis" to reflect the casual Latin spelling.
+ * Various small improvements.
+Changes in version 1.2.4
+ * Added support for Tifinagh (a script used for Berber languages).
+ * Bug-fix that generated two emtpy lines for each empty line in cache mode.
+Changes in version 1.2.3
+ * Exclude emojis, dingbats, many other pictographs from being romanized (e.g. to "face")
+Changes in version 1.2
+ * Run-time improvement based on (1) token-based caching and (2) shortcut 
+   romanization (identity) of ASCII strings for default 1-best (non-chart) 
+   output. Speed-up by a factor of 10 for Bengali and Uyghur on medium and 
+   large size texts.
+ * Incremental improvements for Farsi, Amharic, Russian, Hebrew and related
+   languages.
+ * Richer lattice structure (more alternatives) for "Romanization" of English
+   to support better matching to romanizations of other languages.
+   Changes output only when --chart option is specified. No change in output for
+   default 1-best output, which for ASCII characters is always the input string.
+Changes in version 1.1 (major upgrade)
+ * Offers chart output (in JSON format) to represent alternative romanizations.
+   -- Location of first character is defined to be "line: 1, start:0, end:0".
+ * Incremental improvements of Hebrew and Greek romanization; Chinese numbers.
+ * Improved web-interface at http://www.isi.edu/~ulf/uroman.html
+   -- Shows corresponding original and romanization text in red
+      when hovering over a text segment.
+   -- Shows alternative romanizations when hovering over romanized text
+      marked by dotted underline.
+   -- Added right-to-left script detection and improved display for right-to-left
+      script text (as determined line by line).
+   -- On-page support for some scripts that are often not pre-installed on users'
+      computers (Burmese, Egyptian, Klingon).
+Changes in version 1.0 (major upgrade)
+ * Upgraded principal internal data structure from string to lattice.
+ * Improvements mostly in vowelization of South and Southeast Asian languages.
+ * Vocalic 'r' more consistently treated as vowel (no additional vowel added).
+ * Repetition signs (Japanese/Chinese/Thai/Khmer/Lao) are mapped to superscript 2.
+ * Japanese Katakana middle dots now mapped to ASCII space.
+ * Tibetan intersyllabic mark now mapped to middle dot (U+00B7).
+ * Some corrections regarding analysis of Chinese numbers.
+ * Many more foreign diacritics and punctuation marks dropped or mapped to ASCII.
+ * Zero-width characters dropped, except line/sentence-initial byte order marks.
+ * Spaces normalized to ASCII space.
+ * Fixed bug that in some cases mapped signs (such as dagger or bullet) to their verbal descriptions.
+ * Tested against previous version of uroman with a new uroman visual diff tool.
+ * Almost an order of magnitude faster.
+Changes in version 0.7 (minor upgrade)
+ * Added script uroman-quick.pl for Arabic script languages, incl. Uyghur.
+   Much faster, pre-caching mapping of Arabic to Latin characters, simple greedy processing.
+   Will not convert material from non-Arabic blocks such as any (somewhat unusual) Cyrillic
+   or Chinese characters in Uyghur texts.
+Changes in version 0.6 (minor upgrade)
+ * Added support for two letter characters used in Uzbek:
+   (1) character "ʻ" ("modifier letter turned comma", which modifies preceding "g" and "u" letters)
+   (2) character "ʼ" ("modifier letter apostrophe", which Uzbek uses to mark a glottal stop).
+   Both are now mapped to "'" (plain ASCII apostrophe).
+ * Added support for Uyghur vowel characters such as "ې" (Arabic e) and "ۆ" (Arabic oe)
+   even when they are not preceded by "ئ" (yeh with hamza above).
+ * Added support for Arabic semicolon "؛", Arabic ligature forms for phrases such as "ﷺ"
+   ("sallallahou alayhe wasallam" = "prayer of God be upon him and his family and peace")
+ * Added robustness for Arabic letter presentation forms (initial/medial/final/isolated).
+   However, it is strongly recommended to normalize any presentation form Arabic letters
+   to their non-presentation form before calling uroman.
+ * Added force flush directive ($|=1;).
+Changes in version 0.5 (minor upgrade)
+ * Improvements for Uyghur (make sure to use language option: -l uig)
+Changes in version 0.4 (minor upgrade)
+ * Improvements for Thai (special cases for vowel/consonant reordering, e.g. for "sara o"; dropped some aspiration 'h's)
+ * Minor change for Arabic (added "alef+fathatan" = "an")
+New features in version 0.3
+ * Covers Mandarin (Chinese)
+ * Improved romanization for numerous languages
+ * Preserves capitalization (e.g. from Latin, Cyrillic, Greek scripts)
+ * Maps from native digits to Western numbers
+ * Faster for South Asian languages
+
+Other features
+ * Web interface: http://www.isi.edu/~ulf/uroman.html
+ * Vowelization is provided when locally computable, e.g. for many South Asian
+   languages and Tibetan.
+
+Limitations
+ * This version of uroman assumes all CJK ideographs to be Mandarin (Chinese).
+   This means that Japanese kanji are incorrectly romanized; however, Japanese
+   hiragana and katakana are properly romanized.
+ * A romanizer is not a full transliterator. For example, this version of
+   uroman does not vowelize text that lacks explicit vowelization such as
+   normal text in Arabic and Hebrew (without diacritics/points).
+

uroman/bin/de-accent.pl

@@ -0,0 +1,201 @@
+#!/usr/bin/perl -w
+
+sub print_version {
+   print STDERR "$0 version 1.1\n";
+   print STDERR "   Author: Ulf Hermjakob\n";
+   print STDERR "   Last changed: March 14, 2011\n";
+}
+
+sub print_usage {
+   print STDERR "$0 [options] < with_accents.txt > without_accents.txt\n";
+   print STDERR "   -h or -help\n";
+   print STDERR "   -v or -version\n";
+}
+
+sub de_accent_string {
+   local($s) = @_;
+
+   # $s =~ tr/A-Z/a-z/;
+   unless (0) {
+      # Latin-1
+      if ($s =~ /\xC3[\x80-\xBF]/) {
+         $s =~ s/(À|Á|Â|Ã|Ä|Å)/A/g;
+         $s =~ s/Æ/Ae/g;
+         $s =~ s/Ç/C/g;
+         $s =~ s/Ð/D/g;
+         $s =~ s/(È|É|Ê|Ë)/E/g;
+         $s =~ s/(Ì|Í|Î|Ï)/I/g;
+         $s =~ s/Ñ/N/g;
+         $s =~ s/(Ò|Ó|Ô|Õ|Ö|Ø)/O/g;
+         $s =~ s/(Ù|Ú|Û|Ü)/U/g;
+         $s =~ s/Þ/Th/g;
+         $s =~ s/Ý/Y/g;
+         $s =~ s/(à|á|â|ã|ä|å)/a/g;
+         $s =~ s/æ/ae/g;
+         $s =~ s/ç/c/g;
+         $s =~ s/(è|é|ê|ë)/e/g;
+         $s =~ s/(ì|í|î|ï)/i/g;
+         $s =~ s/ð/d/g;
+         $s =~ s/ñ/n/g;
+         $s =~ s/(ò|ó|ô|õ|ö)/o/g;
+         $s =~ s/ß/ss/g;
+         $s =~ s/þ/th/g;
+         $s =~ s/(ù|ú|û|ü)/u/g;
+         $s =~ s/(ý|ÿ)/y/g;
+      }
+      # Latin Extended-A
+      if ($s =~ /[\xC4-\xC5][\x80-\xBF]/) {
+         $s =~ s/(Ā|Ă|Ą)/A/g;
+         $s =~ s/(ā|ă|ą)/a/g;
+         $s =~ s/(Ć|Ĉ|Ċ|Č)/C/g;
+         $s =~ s/(ć|ĉ|ċ|č)/c/g;
+         $s =~ s/(Ď|Đ)/D/g;
+         $s =~ s/(ď|đ)/d/g;
+         $s =~ s/(Ē|Ĕ|Ė|Ę|Ě)/E/g;
+         $s =~ s/(ē|ĕ|ė|ę|ě)/e/g;
+         $s =~ s/(Ĝ|Ğ|Ġ|Ģ)/G/g;
+         $s =~ s/(ĝ|ğ|ġ|ģ)/g/g;
+         $s =~ s/(Ĥ|Ħ)/H/g;
+         $s =~ s/(ĥ|ħ)/h/g;
+         $s =~ s/(Ĩ|Ī|Ĭ|Į|İ)/I/g;
+         $s =~ s/(ĩ|ī|ĭ|į|ı)/i/g;
+         $s =~ s/IJ/Ij/g;
+         $s =~ s/ij/ij/g;
+         $s =~ s/Ĵ/J/g;
+         $s =~ s/ĵ/j/g;
+         $s =~ s/Ķ/K/g;
+         $s =~ s/(ķ|ĸ)/k/g;
+         $s =~ s/(Ĺ|Ļ|Ľ|Ŀ|Ł)/L/g;
+         $s =~ s/(ļ|ľ|ŀ|ł)/l/g;
+         $s =~ s/(Ń|Ņ|Ň|Ŋ)/N/g;
+         $s =~ s/(ń|ņ|ň|ʼn|ŋ)/n/g;
+         $s =~ s/(Ō|Ŏ|Ő)/O/g;
+         $s =~ s/(ō|ŏ|ő)/o/g;
+         $s =~ s/Œ/Oe/g;
+         $s =~ s/œ/oe/g;
+         $s =~ s/(Ŕ|Ŗ|Ř)/R/g;
+         $s =~ s/(ŕ|ŗ|ř)/r/g;
+         $s =~ s/(Ś|Ŝ|Ş|Š)/S/g;
+         $s =~ s/(ś|ŝ|ş|š|ſ)/s/g;
+         $s =~ s/(Ţ|Ť|Ŧ)/T/g;
+         $s =~ s/(ţ|ť|ŧ)/t/g;
+         $s =~ s/(Ũ|Ū|Ŭ|Ů|Ű|Ų)/U/g;
+         $s =~ s/(ũ|ū|ŭ|ů|ű|ų)/u/g;
+         $s =~ s/Ŵ/W/g;
+         $s =~ s/ŵ/w/g;
+         $s =~ s/(Ŷ|Ÿ)/Y/g;
+         $s =~ s/ŷ/y/g;
+         $s =~ s/(Ź|Ż|Ž)/Z/g;
+         $s =~ s/(ź|ż|ž)/z/g;
+      }
+      # Latin Extended Additional
+      if ($s =~ /\xE1[\xB8-\xBF][\x80-\xBF]/) {
+          $s =~ s/(ḁ|ạ|ả|ấ|ầ|ẩ|ẫ|ậ|ắ|ằ|ẳ|ẵ|ặ|ẚ)/a/g;
+          $s =~ s/(ḃ|ḅ|ḇ)/b/g;
+          $s =~ s/(ḉ)/c/g;
+          $s =~ s/(ḋ|ḍ|ḏ|ḑ|ḓ)/d/g;
+          $s =~ s/(ḕ|ḗ|ḙ|ḛ|ḝ|ẹ|ẻ|ẽ|ế|ề|ể|ễ|ệ)/e/g;
+          $s =~ s/(ḟ)/f/g;
+          $s =~ s/(ḡ)/g/g;
+          $s =~ s/(ḣ|ḥ|ḧ|ḩ|ḫ)/h/g;
+          $s =~ s/(ḭ|ḯ|ỉ|ị)/i/g;
+          $s =~ s/(ḱ|ḳ|ḵ)/k/g;
+          $s =~ s/(ḷ|ḹ|ḻ|ḽ)/l/g;
+          $s =~ s/(ḿ|ṁ|ṃ)/m/g;
+          $s =~ s/(ṅ|ṇ|ṉ|ṋ)/m/g;
+          $s =~ s/(ọ|ỏ|ố|ồ|ổ|ỗ|ộ|ớ|ờ|ở|ỡ|ợ|ṍ|ṏ|ṑ|ṓ)/o/g;
+          $s =~ s/(ṕ|ṗ)/p/g;
+          $s =~ s/(ṙ|ṛ|ṝ|ṟ)/r/g;
+          $s =~ s/(ṡ|ṣ|ṥ|ṧ|ṩ|ẛ)/s/g;
+          $s =~ s/(ṫ|ṭ|ṯ|ṱ)/t/g;
+          $s =~ s/(ṳ|ṵ|ṷ|ṹ|ṻ|ụ|ủ|ứ|ừ|ử|ữ|ự)/u/g;
+          $s =~ s/(ṽ|ṿ)/v/g;
+          $s =~ s/(ẁ|ẃ|ẅ|ẇ|ẉ|ẘ)/w/g;
+          $s =~ s/(ẋ|ẍ)/x/g;
+          $s =~ s/(ẏ|ỳ|ỵ|ỷ|ỹ|ẙ)/y/g;
+          $s =~ s/(ẑ|ẓ|ẕ)/z/g;
+          $s =~ s/(Ḁ|Ạ|Ả|Ấ|Ầ|Ẩ|Ẫ|Ậ|Ắ|Ằ|Ẳ|Ẵ|Ặ)/A/g;
+          $s =~ s/(Ḃ|Ḅ|Ḇ)/B/g;
+          $s =~ s/(Ḉ)/C/g;
+          $s =~ s/(Ḋ|Ḍ|Ḏ|Ḑ|Ḓ)/D/g;
+          $s =~ s/(Ḕ|Ḗ|Ḙ|Ḛ|Ḝ|Ẹ|Ẻ|Ẽ|Ế|Ề|Ể|Ễ|Ệ)/E/g;
+          $s =~ s/(Ḟ)/F/g;
+          $s =~ s/(Ḡ)/G/g;
+          $s =~ s/(Ḣ|Ḥ|Ḧ|Ḩ|Ḫ)/H/g;
+          $s =~ s/(Ḭ|Ḯ|Ỉ|Ị)/I/g;
+          $s =~ s/(Ḱ|Ḳ|Ḵ)/K/g;
+          $s =~ s/(Ḷ|Ḹ|Ḻ|Ḽ)/L/g;
+          $s =~ s/(Ḿ|Ṁ|Ṃ)/M/g;
+          $s =~ s/(Ṅ|Ṇ|Ṉ|Ṋ)/N/g;
+          $s =~ s/(Ṍ|Ṏ|Ṑ|Ṓ|Ọ|Ỏ|Ố|Ồ|Ổ|Ỗ|Ộ|Ớ|Ờ|Ở|Ỡ|Ợ)/O/g;
+          $s =~ s/(Ṕ|Ṗ)/P/g;
+          $s =~ s/(Ṙ|Ṛ|Ṝ|Ṟ)/R/g;
+          $s =~ s/(Ṡ|Ṣ|Ṥ|Ṧ|Ṩ)/S/g;
+          $s =~ s/(Ṫ|Ṭ|Ṯ|Ṱ)/T/g;
+          $s =~ s/(Ṳ|Ṵ|Ṷ|Ṹ|Ṻ|Ụ|Ủ|Ứ|Ừ|Ử|Ữ|Ự)/U/g;
+          $s =~ s/(Ṽ|Ṿ)/V/g;
+          $s =~ s/(Ẁ|Ẃ|Ẅ|Ẇ|Ẉ)/W/g;
+          $s =~ s/(Ẍ)/X/g;
+          $s =~ s/(Ẏ|Ỳ|Ỵ|Ỷ|Ỹ)/Y/g; 
+          $s =~ s/(Ẑ|Ẓ|Ẕ)/Z/g;
+      }
+      # Greek letters
+      if ($s =~ /\xCE[\x86-\xAB]/) {
+          $s =~ s/ά/α/g;
+          $s =~ s/έ/ε/g;
+          $s =~ s/ί/ι/g;
+          $s =~ s/ϊ/ι/g;
+          $s =~ s/ΐ/ι/g;
+          $s =~ s/ό/ο/g;
+          $s =~ s/ύ/υ/g;
+          $s =~ s/ϋ/υ/g;
+          $s =~ s/ΰ/υ/g;
+          $s =~ s/ώ/ω/g;
+          $s =~ s/Ά/Α/g;
+          $s =~ s/Έ/Ε/g;
+          $s =~ s/Ή/Η/g;
+          $s =~ s/Ί/Ι/g;
+          $s =~ s/Ϊ/Ι/g;
+          $s =~ s/Ύ/Υ/g;
+          $s =~ s/Ϋ/Υ/g;
+          $s =~ s/Ώ/Ω/g;
+      }
+      # Cyrillic letters
+      if ($s =~ /\xD0[\x80-\xAF]/) {
+          $s =~ s/Ѐ/Е/g;
+          $s =~ s/Ё/Е/g;
+          $s =~ s/Ѓ/Г/g;
+          $s =~ s/Ќ/К/g;
+          $s =~ s/Ѝ/И/g;
+          $s =~ s/Й/И/g;
+          $s =~ s/ѐ/е/g;
+          $s =~ s/ё/е/g;
+          $s =~ s/ѓ/г/g;
+          $s =~ s/ќ/к/g;
+          $s =~ s/ѝ/и/g;
+          $s =~ s/й/и/g;
+      }
+   }
+   return $s;
+}
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-*(h|help)$/i) {
+      &print_usage;
+      exit 1;
+   } elsif ($arg =~ /^-*(v|version)$/i) {
+      &print_version;
+      exit 1;
+   } else {
+      print STDERR "Ignoring unrecognized argument $arg\n";
+   }
+}
+
+$line_number = 0;
+while (<>) {
+   $line_number++;
+   print &de_accent_string($_);
+}
+exit 0;
+

uroman/bin/string-distance.pl

@@ -0,0 +1,99 @@
+#!/usr/bin/perl -w
+
+# Author: Ulf Hermjakob
+# Release date: October 13, 2019
+
+# Usage: string-distance.pl {-lc1 <language-code>} {-lc2 <language-code>} < STDIN > STDOUT
+# Example: string-distance.pl -lc1 rus -lc2 ukr < STDIN > STDOUT
+# Example: string-distance.pl < ../test/string-similarity-test-input.txt
+# Input format: two strings per line (tab-separated, in Latin script)
+#    Strings in non-Latin scripts should first be romanized. (Recommended script: uroman.pl)
+# Output format: repetition of the two input strings, plus the string distance between them (tab-separated).
+#    Additional output meta info lines at the top are marked with an initial #.
+#
+# The script uses data from a string-distance-cost-rules file that lists costs,
+# where the default cost is "1" with lower costs for differences in vowels,
+# duplicate consonants, "f" vs. "ph" etc.
+# Language cost rules can be language-specific and context-sensitive.
+
+$|=1;
+
+use FindBin;
+use Cwd "abs_path";
+use File::Basename qw(dirname);
+use File::Spec;
+
+my $bin_dir = abs_path(dirname($0));
+my $root_dir = File::Spec->catfile($bin_dir, File::Spec->updir());
+my $data_dir = File::Spec->catfile($root_dir, "data");
+my $lib_dir = File::Spec->catfile($root_dir, "lib");
+
+use lib "$FindBin::Bin/../lib";
+use List::Util qw(min max);
+use NLP::utilities;
+use NLP::stringDistance;
+$util = NLP::utilities;
+$sd = NLP::stringDistance;
+$verbose = 0;
+$separator = "\t";
+
+$cost_rule_filename = File::Spec->catfile($data_dir, "string-distance-cost-rules.txt");
+
+$lang_code1 = "eng";
+$lang_code2 = "eng";
+%ht = ();
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-+lc1$/) {
+      $lang_code_candidate = shift @ARGV;
+      $lang_code1 = $lang_code_candidate if $lang_code_candidate =~ /^[a-z]{3,3}$/;
+   } elsif ($arg =~ /^-+lc2$/) {
+      $lang_code_candidate = shift @ARGV;
+      $lang_code2 = $lang_code_candidate if $lang_code_candidate =~ /^[a-z]{3,3}$/;
+   } elsif ($arg =~ /^-+(v|verbose)$/) {
+      $verbose = shift @ARGV;
+   } else {
+      print STDERR "Ignoring unrecognized arg $arg\n";
+   }
+}
+
+$sd->load_string_distance_data($cost_rule_filename, *ht, $verbose);
+print STDERR "Loaded resources.\n" if $verbose;
+
+my $chart_id = 0;
+my $line_number = 0;
+print "# Lang-code-1: $lang_code1 Lang-code-2: $lang_code2\n";
+while (<>) {
+   $line_number++;
+   if ($verbose) {
+      if ($line_number =~ /000$/) {
+         if ($line_number =~ /0000$/) {
+	    print STDERR $line_number;
+         } else {
+	    print STDERR ".";
+         }
+      }
+   }
+   my $line = $_;
+   $line =~ s/^\xEF\xBB\xBF//;
+   next if $line =~ /^\s*(\#.*)?$/;
+   my $s1;
+   my $s2;
+   if (($s1, $s2) = ($line =~ /^("(?:\\"|[^"])*"|\S+)$separator("(?:\\"|[^"])*"|\S+)\s*$/)) {
+      $s1 = $util->dequote_string($s1);
+      $s2 = $util->dequote_string($s2);
+   } elsif ($line =~ /^\s*(#.*)$/) {
+   } else {
+      print STDERR "Could not process line $line_number: $line" if $verbose;
+      print "\n";
+      next;
+   }
+
+   $cost = $sd->quick_romanized_string_distance_by_chart($s1, $s2, *ht, "", $lang_code1, $lang_code2);
+   print "$s1\t$s2\t$cost\n";
+}
+print STDERR "\n" if $verbose;
+
+exit 0;
+

uroman/bin/uroman-quick.pl

@@ -0,0 +1,58 @@
+#!/usr/bin/perl -w
+
+# uroman  Nov. 12, 2015 - July 25, 2016
+# version v0.7
+# Author: Ulf Hermjakob
+
+# Usage: uroman-quick.pl {-l [tur|uig|ukr|yid]} < STDIN
+# currently only for Arabic script languages, incl. Uyghur
+
+$|=1;
+
+use FindBin;
+use Cwd "abs_path";
+use File::Basename qw(dirname);
+use File::Spec;
+
+my $bin_dir = abs_path(dirname($0));
+my $root_dir = File::Spec->catfile($bin_dir, File::Spec->updir());
+my $data_dir = File::Spec->catfile($root_dir, "data");
+my $lib_dir = File::Spec->catfile($root_dir, "lib");
+
+use lib "$FindBin::Bin/../lib";
+use NLP::Romanizer;
+use NLP::UTF8;
+$romanizer = NLP::Romanizer;
+%ht = ();
+$lang_code = "";
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-+(l|lc|lang-code)$/) {
+      $lang_code = lc (shift @ARGV || "")
+   } else {
+      print STDERR "Ignoring unrecognized arg $arg\n";
+   }
+}
+
+$romanization_table_arabic_block_filename = File::Spec->catfile($data_dir, "romanization-table-arabic-block.txt");
+$romanization_table_filename = File::Spec->catfile($data_dir, "romanization-table.txt");
+
+$romanizer->load_romanization_table(*ht, $romanization_table_arabic_block_filename);
+$romanizer->load_romanization_table(*ht, $romanization_table_filename);
+
+$line_number = 0;
+while (<>) {
+   $line_number++;
+   my $line = $_;
+   print $romanizer->quick_romanize($line, $lang_code, *ht) . "\n";
+   if ($line_number =~ /0000$/) {
+      print STDERR $line_number;
+   } elsif ($line_number =~ /000$/) {
+      print STDERR ".";
+   }
+}
+print STDERR "\n";
+
+exit 0;
+

uroman/bin/uroman-tsv.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Created by Thamme Gowda on June 17, 2019
+
+DIR=$(dirname "${BASH_SOURCE[0]}")  # get the directory name
+# DIR=$(realpath "${DIR}")    # resolve its full path if need be
+
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+    >&2 echo "ERROR: invalid args"
+    >&2 echo "Usage: <input.tsv> [<output.tsv>]"
+    exit 2
+fi
+
+INP=$1
+OUT=$2
+
+CMD=$DIR/uroman.pl
+
+function romanize(){
+    paste <(cut -f1 $INP) <(cut -f2 $INP | $CMD)
+}
+
+if [[ -n $OUT ]]; then
+    romanize > $OUT
+else
+    romanize
+fi
+
+

uroman/bin/uroman.pl

@@ -0,0 +1,138 @@
+#!/usr/bin/perl -w
+
+# uroman  Nov. 12, 2015 - Apr. 23, 2021
+$version = "v1.2.8";
+# Author: Ulf Hermjakob
+
+# Usage: uroman.pl {-l [ara|bel|bul|deu|ell|eng|fas|grc|heb|kaz|kir|lav|lit|mkd|mkd2|oss|pnt|rus|srp|srp2|tur|uig|ukr|yid]} {--chart|--offset-mapping} {--no-cache} {--workset} < STDIN
+# Example: cat workset.txt | uroman.pl --offset-mapping --workset
+
+$|=1;
+
+use FindBin;
+use Cwd "abs_path";
+use File::Basename qw(dirname);
+use File::Spec;
+
+my $bin_dir = abs_path(dirname($0));
+my $root_dir = File::Spec->catfile($bin_dir, File::Spec->updir());
+my $data_dir = File::Spec->catfile($root_dir, "data");
+my $lib_dir = File::Spec->catfile($root_dir, "lib");
+
+use lib "$FindBin::Bin/../lib";
+use NLP::Chinese;
+use NLP::Romanizer;
+use NLP::UTF8;
+use NLP::utilities;
+use JSON;
+$chinesePM = NLP::Chinese;
+$romanizer = NLP::Romanizer;
+$util = NLP::utilities;
+%ht = ();
+%pinyin_ht = ();
+$lang_code = "";
+$return_chart_p = 0;
+$return_offset_mappings_p = 0;
+$workset_p = 0;
+$cache_rom_tokens_p = 1;
+
+$script_data_filename = File::Spec->catfile($data_dir, "Scripts.txt");
+$unicode_data_overwrite_filename = File::Spec->catfile($data_dir, "UnicodeDataOverwrite.txt");
+$unicode_data_filename = File::Spec->catfile($data_dir, "UnicodeData.txt");
+$romanization_table_filename = File::Spec->catfile($data_dir, "romanization-table.txt");
+$chinese_tonal_pinyin_filename = File::Spec->catfile($data_dir, "Chinese_to_Pinyin.txt");
+
+while (@ARGV) {
+   $arg = shift @ARGV;
+   if ($arg =~ /^-+(l|lc|lang-code)$/) {
+      $lang_code = lc (shift @ARGV || "")
+   } elsif ($arg =~ /^-+chart$/i) {
+      $return_chart_p = 1;
+   } elsif ($arg =~ /^-+workset$/i) {
+      $workset_p = 1;
+   } elsif ($arg =~ /^-+offset[-_]*map/i) {
+      $return_offset_mappings_p = 1;
+   } elsif ($arg =~ /^-+unicode[-_]?data/i) {
+      $filename = shift @ARGV;
+      if (-r $filename) {
+         $unicode_data_filename = $filename;
+      } else {
+         print STDERR "Ignoring invalid UnicodeData filename $filename\n";
+      }
+   } elsif ($arg =~ /^-+(no-tok-cach|no-cach)/i) {
+      $cache_rom_tokens_p = 0;
+   } else {
+      print STDERR "Ignoring unrecognized arg $arg\n";
+   }
+}
+
+$romanizer->load_script_data(*ht, $script_data_filename);
+$romanizer->load_unicode_data(*ht, $unicode_data_filename);
+$romanizer->load_unicode_overwrite_romanization(*ht, $unicode_data_overwrite_filename);
+$romanizer->load_romanization_table(*ht, $romanization_table_filename);
+$chinese_to_pinyin_not_yet_loaded_p = 1;
+$current_date = $util->datetime("dateTtime");
+$lang_code_clause = ($lang_code) ? " \"lang-code\":\"$lang_code\",\n" : "";
+
+print "{\n \"romanizer\":\"uroman $version (Ulf Hermjakob, USC/ISI)\",\n \"date\":\"$current_date\",\n$lang_code_clause \"romanization\": [\n" if $return_chart_p;
+my $line_number = 0;
+my $chart_result = "";
+while (<>) {
+   $line_number++;
+   my $line = $_;
+   my $snt_id = "";
+   if ($workset_p) {
+      next if $line =~ /^#/;
+      if (($i_value, $s_value) = ($line =~ /^(\S+\.\d+)\s(.*)$/)) {
+	 $snt_id = $i_value;
+	 $line = "$s_value\n";
+      } else {
+	 next;
+      }
+   }
+   if ($chinese_to_pinyin_not_yet_loaded_p && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($line)) {
+      $chinesePM->read_chinese_tonal_pinyin_files(*pinyin_ht, $chinese_tonal_pinyin_filename);
+      $chinese_to_pinyin_not_yet_loaded_p = 0;
+   }
+   if ($return_chart_p) {
+      print $chart_result;
+      *chart_ht = $romanizer->romanize($line, $lang_code, "", *ht, *pinyin_ht, 0, "return chart", $line_number);
+      $chart_result = $romanizer->chart_to_json_romanization_elements(0, $chart_ht{N_CHARS}, *chart_ht, $line_number);
+   } elsif ($return_offset_mappings_p) {
+      ($best_romanization, $offset_mappings) = $romanizer->romanize($line, $lang_code, "", *ht, *pinyin_ht, 0, "return offset mappings", $line_number, 0);
+      print "::snt-id $snt_id\n" if $workset_p;
+      print "::orig $line";
+      print "::rom $best_romanization\n";
+      print "::align $offset_mappings\n\n";
+   } elsif ($cache_rom_tokens_p) {
+      print $romanizer->romanize_by_token_with_caching($line, $lang_code, "", *ht, *pinyin_ht, 0, "", $line_number) . "\n";
+   } else {
+      print $romanizer->romanize($line, $lang_code, "", *ht, *pinyin_ht, 0, "", $line_number) . "\n";
+   }
+}
+$chart_result =~ s/,(\s*)$/$1/;
+print $chart_result;
+print " ]\n}\n" if $return_chart_p;
+
+$dev_test_p = 0;
+if ($dev_test_p) {
+   $n_suspicious_code_points = 0;
+   $n_instances = 0;
+   foreach $char_name (sort { hex($ht{UTF_NAME_TO_UNICODE}->{$a}) <=> hex($ht{UTF_NAME_TO_UNICODE}->{$b}) }
+                            keys %{$ht{SUSPICIOUS_ROMANIZATION}}) {
+      $unicode_value = $ht{UTF_NAME_TO_UNICODE}->{$char_name};
+      $utf8_string = $ht{UTF_NAME_TO_CODE}->{$char_name};
+      foreach $romanization (sort keys %{$ht{SUSPICIOUS_ROMANIZATION}->{$char_name}}) {
+         $count = $ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization};
+	 $s = ($count == 1) ? "" : "s";
+         print STDERR "*** Suspiciously lengthy romanization:\n" unless $n_suspicious_code_points;
+         print STDERR "::s $utf8_string ::t $romanization ::comment $char_name (U+$unicode_value)\n";
+	 $n_suspicious_code_points++;
+	 $n_instances += $count;
+      }
+   }
+   print STDERR "  *** Total of $n_suspicious_code_points suspicious code points ($n_instances instance$s)\n" if $n_suspicious_code_points;
+} 
+
+exit 0;
+

uroman/data/Scripts.txt

@@ -0,0 +1,135 @@
+::script-name Aegean
+::script-name Ahom
+::script-name Anatolian Hieroglyph
+::script-name Arabic ::direction right-to-left
+::script-name Armenian
+::script-name Avestan
+::script-name Balinese
+::script-name Bamum
+::script-name Bassa Vah
+::script-name Batak
+::script-name Bengali ::abugida-default-vowel a
+::script-name Bhaiksuki
+::script-name Bopomofo ::language Chinese
+::script-name Brahmi ::abugida-default-vowel a
+::script-name Braille
+::script-name Buginese
+::script-name Buhid
+::script-name Canadian Syllabics
+::script-name Carian
+::script-name Caucasian Albanian
+::script-name Chakma
+::script-name Cham
+::script-name Cherokee
+::script-name Coptic
+::script-name Cuneiform
+::script-name Cypriot
+::script-name Cyrillic
+::script-name CJK ::alt-script-name Chinese, Kanji ::language Chinese, Japanese, Korean, Mandarin
+::script-name Deseret
+::script-name Devanagari ::abugida-default-vowel a
+::script-name Duployan
+::script-name Egyptian Hieroglyph
+::script-name Elbasan
+::script-name Ethiopic
+::script-name Georgian
+::script-name Glagolitic
+::script-name Gothic
+::script-name Grantha
+::script-name Greek
+::script-name Gujarati ::abugida-default-vowel a
+::script-name Gurmukhi ::abugida-default-vowel a
+::script-name Hangul ::language Korean
+::script-name Hanunoo
+::script-name Hatran
+::script-name Hebrew ::direction right-to-left
+::script-name Hiragana ::language Japanese
+::script-name Imperial Aramaic
+::script-name Inscriptional Pahlavi
+::script-name Inscriptional Parthian
+::script-name Javanese
+::script-name Kaithi
+::script-name Kannada ::abugida-default-vowel a
+::script-name Katakana ::language Japanese
+::script-name Kayah Li
+::script-name Kharoshthi
+::script-name Khmer ::abugida-default-vowel a, o
+::script-name Khojki
+::script-name Khudawadi
+::script-name Klingon
+::script-name Lao
+::script-name Lepcha
+::script-name Latin
+::script-name Limbu
+::script-name Linear A
+::script-name Linear B
+::script-name Lycian
+::script-name Lydian
+::script-name Mahajani
+::script-name Malayalam ::abugida-default-vowel a
+::script-name Mandaic
+::script-name Manichaean
+::script-name Marchen
+::script-name Meetei Mayek
+::script-name Meroitic Cursive
+::script-name Meroitic Hieroglyphic
+::script-name Miao
+::script-name Modi ::abugida-default-vowel a
+::script-name Mongolian
+::script-name Mro
+::script-name Multani
+::script-name Myanmar ::alt-script-name Burmese ::abugida-default-vowel a
+::script-name Nabataean
+::script-name New Tai Lue
+::script-name Newa
+::script-name Nko ::direction right-to-left
+::script-name Ogham
+::script-name Ol Chiki
+::script-name Old Hungarian
+::script-name Old Italic
+::script-name Old Permic
+::script-name Old Persian
+::script-name Old North Arabian
+::script-name Old South Arabian
+::script-name Old Turkic
+::script-name Oriya ::alt-script-name Odia ::abugida-default-vowel a
+::script-name Osage
+::script-name Osmanya
+::script-name Pahawh Hmong
+::script-name Palmyrene
+::script-name Pau Cin Hau
+::script-name Phags-pa
+::script-name Phaistos Disc
+::script-name Phoenician
+::script-name Psalter Pahlavi
+::script-name Rejang
+::script-name Runic
+::script-name Samaritan
+::script-name Saurashtra
+::script-name Sharada
+::script-name Shavian
+::script-name Siddham
+::script-name Sinhala ::abugida-default-vowel a
+::script-name Sora Sompeng
+::script-name Sundanese ::abugida-default-vowel a
+::script-name Syloti Nagri
+::script-name Syriac
+::script-name Tagalog
+::script-name Tagbanwa
+::script-name Tai Le
+::script-name Tai Tham
+::script-name Tai Viet
+::script-name Takri
+::script-name Tamil ::abugida-default-vowel a
+::script-name Tangut
+::script-name Telugu ::abugida-default-vowel a
+::script-name Thaana ::direction right-to-left
+::script-name Thai
+::script-name Tibetan ::abugida-default-vowel a
+::script-name Tifinagh
+::script-name Tirhuta
+::script-name Ugaritic
+::script-name Vai
+::script-name Vedic
+::script-name Warang Citi
+::script-name Yi

uroman/data/UnicodeDataOverwrite.txt

@@ -0,0 +1,442 @@
+## UnicodeDataOverwrite.txt
+::u 00A0 ::r " "  ::comment no-break space
+::u 01BF ::r w  ::comment ƿ Latin Character Wynn (Old English)
+::u 0294 ::r ' ::comment gottal stop
+::u 0295 ::r ' ::comment ʕ voiced pharyngeal fricative
+::u 0305 ::r ""  ::comment ̅ Combining overline
+::u 0306 ::r ""  ::comment ̆ Combining breve
+::u 0307 ::r ""  ::comment ̇ Combining dot above
+::u 030A ::r ""  ::comment ̊ Combining ring above
+::u 030C ::r ""  ::comment ̌ Combining caron
+::u 0311 ::r ""  ::comment ̑ Combining inverted breve
+::u 031D ::r ""  ::comment ̝ Combining down up below
+::u 031E ::r ""  ::comment ̞ Combining down tack below
+::u 031F ::r ""  ::comment ̟ Combining plus sign below
+::u 0323 ::r ""  ::comment ̣ Combining dot below
+::u 0325 ::r ""  ::comment ̥ Combining ring below
+::u 0329 ::r ""  ::comment ̩ Combining vertical line below
+::u 032A ::r ""  ::comment ̪ Combining bridge below
+::u 032F ::r ""  ::comment ̯ Combining inverted breve below
+::u 0342 ::r ""  ::comment ͂ Combining Greek perispomeni (circumflex accent)
+::u 0343 ::r ""  ::comment ̓ Combining Greek koronis
+::u 0361 ::r ""  ::comment Combining double inverted breve
+::u 0384 ::r ""  ::comment ΄ Greek tonos
+::u 0482 ::r 1000·  ::comment ҂ Cyrillic thousands sign
+::u 0483 ::r ""  ::comment ҃ Combining Cyrillic Titlo ::annotation titlo 
+::u 0484 ::r ""  ::comment ҄ Combining Cyrillic Palatalization ::annotation palatalization 
+::u 055B ::r ""  ::comment ՛ Armenian emphasis mark
+::u 055F ::r ""  ::comment ՟ Armenian abbreviation mark ::annotation abbreviation
+
+::u 0901 ::r +m  ::comment Devanagari sign candrabindu
+::u 0902 ::r +m  ::comment Devanagari sign anusvara
+::u 0903 ::r +h  ::comment Devanagari sign visarga
+::u 093D ::r '   ::comment Devanagari sign avagraha
+::u 0950 ::r om  ::comment ॐ Devanagari om symbol
+::u 0951 ::r ""  ::comment ॑ Devanagari stress sign "udatta"
+::u 0952 ::r ""  ::comment ॒ Devanagari stress sign "anudatta" 
+::u 0981 ::r +n  ::comment Bengali sign candrabindu ("chôndrôbindu")
+::u 0982 ::r +ng ::comment Bengali sign anusvara ("ônushar")
+::u 0983 ::r +h  ::comment Bengali sign visarga ("bishôrgô")
+::u 099A ::r ch  ::comment instead of Bengali C(A)
+::u 099B ::r chh ::comment instead of Bengali CC(A)
+::u 0A02 ::r +m  ::comment Gurmukhi sign bindi
+::u 0A70 ::r +m  ::comment Gurmukhi tippi
+# ::u 0A72 ::r "" ::comment Gurmukhi addak
+::u 0A72 ::r ""  ::comment Gurmukhi iri
+::u 0A73 ::r ""  ::comment Gurmukhi ura
+::u 0B01 ::r +m  ::comment Oriya sign candrabindu
+::u 0B03 ::r +h  ::comment Oriya sign visarga
+::u 0B5F ::r ya  ::comment ୟ Oriya letter yya
+::u 0B82 ::r +m  ::comment Tamil sign anusvara (not to be used?)
+::u 0B83 ::r +h  ::comment Tamil sign visarga ("āytam")
+::u 0B9F ::r t   ::comment instead of Tamil TT(A)
+::u 0BA3 ::r n   ::comment instead of Tamil NN(A)
+::u 0BA9 ::r n   ::comment instead of Tamil NNN(A)
+::u 0BB1 ::r r   ::comment instead of Tamil RR(A)
+::u 0BB3 ::r l   ::comment instead of Tamil LL(A)
+::u 0BB4 ::r l   ::comment instead of Tamil LLL(A)
+::u 0C03 ::r +h  ::comment ః Telugu sign visarga
+::u 0C83 ::r +h  ::comment Kannada sign visarga
+::u 0D02 ::r +m  ::comment Malayalam sign anusvara
+::u 0D03 ::r +h  ::comment Malayalam sign visarga
+::u 0D82 ::r +n  ::comment Sinhala sign anusvaraya
+::u 0DA4 ::r ny ::comment Sinhala ඤ
+::u 0DA5 ::r gn ::comment Sinhala ඥ
+::u 0DCA ::r ""  ::comment Sinhala sign al-lakuna (virama = no vowel)
+::u 0DCF ::r aa ::comment Sinhala ා
+::u 0DD0 ::r ae ::comment Sinhala ැ
+::u 0DD1 ::r ae ::comment Sinhala ෑ
+::u 0DD2 ::r i ::comment Sinhala ි
+::u 0DD3 ::r ii ::comment Sinhala ී
+::u 0DD4 ::r u ::comment Sinhala ු
+::u 0DD6 ::r uu ::comment Sinhala ූ
+::u 0DD8 ::r r ::comment Sinhala ෘ
+::u 0DD9 ::r e ::comment Sinhala ෙ
+::u 0DDA ::r ee ::comment Sinhala ේ
+::u 0DDB ::r ai ::comment Sinhala ෛ
+::u 0DDC ::r o ::comment Sinhala ො
+::u 0DDD ::r oo ::comment Sinhala ෝ
+::u 0DDE ::r au ::comment Sinhala ෞ
+::u 0DDF ::r aa ::comment Sinhala ා
+::u 0DF2 ::r rr ::comment Sinhala ෲ
+
+::u 0E02 ::r k   ::comment Thai character KHO KHAI
+::u 0E03 ::r k   ::comment Thai character KHO KHUAT
+::u 0E04 ::r k   ::comment Thai character KHO KHWAI
+::u 0E05 ::r k   ::comment Thai character KHO KHON
+::u 0E06 ::r k   ::comment Thai character KHO RAKHANG
+::u 0E10 ::r t   ::comment Thai character THO THAN
+::u 0E11 ::r t   ::comment Thai character THO NANGMONTHO
+::u 0E12 ::r t   ::comment Thai character THO PHUTHAO
+::u 0E16 ::r t   ::comment Thai character THO THUNG
+::u 0E17 ::r t   ::comment Thai character THO THAHAN
+::u 0E18 ::r t   ::comment Thai character THO THONG
+::u 0E1C ::r p   ::comment Thai character PHO PHUNG
+::u 0E1E ::r p   ::comment Thai character PHO PHAN
+::u 0E20 ::r p   ::comment Thai character PHO SAMPHAO
+::u 0E2D ::r o   ::comment Thai character O ANG
+::u 0E2F ::r ...  ::comment ฯ Thai character PAIYANNOI (ellipsis, abbreviation)
+::u 0E31 ::r a   ::comment Thai character MAI HAN-AKAT
+::u 0E3A ::r ""  ::comment Thai character PHINTHU (Pali virama)
+::u 0E40 ::r e   ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA E
+::u 0E41 ::r ae  ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA AE
+::u 0E42 ::r o   ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA O
+::u 0E43 ::r ai  ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA AI MAIMUAN
+::u 0E44 ::r ai  ::syllable-info written-pre-consonant-spoken-post-consonant ::comment Thai character SARA AI MAIMALAI
+::u 0E45 ::r ""  ::comment Thai character LAKKHANGYAO vowel lengthener
+::u 0E47 ::r o   ::comment Thai character MAITAIKHU vowel shortener
+::u 0E48 ::r ""  ::tone-mark non-standard ::comment Thai tone mark MAI EK
+::u 0E49 ::r ""  ::tone-mark standard ::comment Thai tone mark MAI THO
+::u 0E4A ::r ""  ::tone-mark high ::comment Thai tone mark MAI TRI
+::u 0E4B ::r ""  ::tone-mark rising ::comment Thai tone mark MAI CHATTAWA
+::u 0E4C ::r ""  ::comment Thai character THANTHAKHAT cancellation mark (cf. virama)
+::u 0E4D ::r +m  ::comment ํ Thai character NIKHAHIT final nasal (cf. anusvara)
+::u 0ECC ::r ""  ::comment ໌ Lao cancellation mark ::annotation cancellation
+::u 0F0B ::r ·  ::comment ་ Tibetan mark intersyllabic tsheg
+::u 0F0C ::r ""  ::comment ༌ Tibetan mark delimiter tsheg bstar
+::u 0F84 ::r ""  ::comment ྄ Tibetan halanta
+::u 1036 ::r +n  ::comment Myanmar sign anusvara ("auk myit")
+::u 1037 ::r ""  ::tone-mark creaky ::comment Myanmar sign dot below
+::u 1038 ::r ""  ::tone-mark high ::comment Myanmar sign visarga
+
+::u 16A0 ::r f ::comment ᚠ RUNIC LETTER FEHU FEOH FE F
+::u 16A1 ::r v ::comment ᚡ RUNIC LETTER V
+::u 16A2 ::r u ::comment ᚢ RUNIC LETTER URUZ UR U
+::u 16A3 ::r y ::comment ᚣ RUNIC LETTER YR
+::u 16A4 ::r y ::comment ᚤ RUNIC LETTER Y
+::u 16A5 ::r w ::comment ᚥ RUNIC LETTER W
+::u 16A6 ::r th ::comment ᚦ RUNIC LETTER THURISAZ THURS THORN
+::u 16A7 ::r th ::comment ᚧ RUNIC LETTER ETH
+::u 16A8 ::r a ::comment ᚨ RUNIC LETTER ANSUZ A
+::u 16A9 ::r o ::comment ᚩ RUNIC LETTER OS O
+::u 16AA ::r a ::comment ᚪ RUNIC LETTER AC A
+::u 16AB ::r ae ::comment ᚫ RUNIC LETTER AESC
+::u 16AC ::r o ::comment ᚬ RUNIC LETTER LONG-BRANCH-OSS O
+::u 16AD ::r o ::comment ᚭ RUNIC LETTER SHORT-TWIG-OSS O
+::u 16AE ::r o ::comment ᚮ RUNIC LETTER O
+::u 16AF ::r oe ::comment ᚯ RUNIC LETTER OE
+::u 16B0 ::r on ::comment ᚰ RUNIC LETTER ON
+::u 16B1 ::r r ::comment ᚱ RUNIC LETTER RAIDO RAD REID R
+::u 16B2 ::r k ::comment ᚲ RUNIC LETTER KAUNA
+::u 16B3 ::r c ::comment ᚳ RUNIC LETTER CEN
+::u 16B4 ::r k ::comment ᚴ RUNIC LETTER KAUN K
+::u 16B5 ::r g ::comment ᚵ RUNIC LETTER G
+::u 16B6 ::r ng ::comment ᚶ RUNIC LETTER ENG
+::u 16B7 ::r g ::comment ᚷ RUNIC LETTER GEBO GYFU G
+::u 16B8 ::r g ::comment ᚸ RUNIC LETTER GAR
+::u 16B9 ::r w ::comment ᚹ RUNIC LETTER WUNJO WYNN W
+::u 16BA ::r h ::comment ᚺ RUNIC LETTER HAGLAZ H
+::u 16BB ::r h ::comment ᚻ RUNIC LETTER HAEGL H
+::u 16BC ::r h ::comment ᚼ RUNIC LETTER LONG-BRANCH-HAGALL H
+::u 16BD ::r h ::comment ᚽ RUNIC LETTER SHORT-TWIG-HAGALL H
+::u 16BE ::r n ::comment ᚾ RUNIC LETTER NAUDIZ NYD NAUD N
+::u 16BF ::r n ::comment ᚿ RUNIC LETTER SHORT-TWIG-NAUD N
+::u 16C0 ::r n ::comment ᛀ RUNIC LETTER DOTTED-N
+::u 16C1 ::r i ::comment ᛁ RUNIC LETTER ISAZ IS ISS I
+::u 16C2 ::r e ::comment ᛂ RUNIC LETTER E
+::u 16C3 ::r j ::comment ᛃ RUNIC LETTER JERAN J
+::u 16C4 ::r j ::comment ᛄ RUNIC LETTER GER
+::u 16C5 ::r ae ::comment ᛅ RUNIC LETTER LONG-BRANCH-AR AE
+::u 16C6 ::r a ::comment ᛆ RUNIC LETTER SHORT-TWIG-AR A
+::u 16C7 ::r i ::comment ᛇ RUNIC LETTER IWAZ EOH
+::u 16C8 ::r p ::comment ᛈ RUNIC LETTER PERTHO PEORTH P
+::u 16C9 ::r z ::comment ᛉ RUNIC LETTER ALGIZ EOLHX
+::u 16CA ::r s ::comment ᛊ RUNIC LETTER SOWILO S
+::u 16CB ::r s ::comment ᛋ RUNIC LETTER SIGEL LONG-BRANCH-SOL S
+::u 16CC ::r s ::comment ᛌ RUNIC LETTER SHORT-TWIG-SOL S
+::u 16CD ::r c ::comment ᛍ RUNIC LETTER C
+::u 16CE ::r z ::comment ᛎ RUNIC LETTER Z
+::u 16CF ::r t ::comment ᛏ RUNIC LETTER TIWAZ TIR TYR T
+::u 16D0 ::r t ::comment ᛐ RUNIC LETTER SHORT-TWIG-TYR T
+::u 16D1 ::r d ::comment ᛑ RUNIC LETTER D
+::u 16D2 ::r b ::comment ᛒ RUNIC LETTER BERKANAN BEORC BJARKAN B
+::u 16D3 ::r b ::comment ᛓ RUNIC LETTER SHORT-TWIG-BJARKAN B
+::u 16D4 ::r p ::comment ᛔ RUNIC LETTER DOTTED-P
+::u 16D5 ::r p ::comment ᛕ RUNIC LETTER OPEN-P
+::u 16D6 ::r e ::comment ᛖ RUNIC LETTER EHWAZ EH E
+::u 16D7 ::r m ::comment ᛗ RUNIC LETTER MANNAZ MAN M
+::u 16D8 ::r m ::comment ᛘ RUNIC LETTER LONG-BRANCH-MADR M
+::u 16D9 ::r m ::comment ᛙ RUNIC LETTER SHORT-TWIG-MADR M
+::u 16DA ::r l ::comment ᛚ RUNIC LETTER LAUKAZ LAGU LOGR L
+::u 16DB ::r l ::comment ᛛ RUNIC LETTER DOTTED-L
+::u 16DC ::r ng ::comment ᛜ RUNIC LETTER INGWAZ
+::u 16DD ::r ng ::comment ᛝ RUNIC LETTER ING
+::u 16DE ::r d ::comment ᛞ RUNIC LETTER DAGAZ DAEG D
+::u 16DF ::r o ::comment ᛟ RUNIC LETTER OTHALAN ETHEL O
+::u 16E0 ::r ea ::comment ᛠ RUNIC LETTER EAR
+::u 16E1 ::r io ::comment ᛡ RUNIC LETTER IOR
+::u 16E2 ::r q ::comment ᛢ RUNIC LETTER CWEORTH
+::u 16E3 ::r k ::comment ᛣ RUNIC LETTER CALC
+::u 16E4 ::r k ::comment ᛤ RUNIC LETTER CEALC
+::u 16E5 ::r st ::comment ᛥ RUNIC LETTER STAN
+::u 16E6 ::r r ::comment ᛦ RUNIC LETTER LONG-BRANCH-YR
+::u 16E7 ::r r ::comment ᛧ RUNIC LETTER SHORT-TWIG-YR
+::u 16E8 ::r r ::comment ᛨ RUNIC LETTER ICELANDIC-YR
+::u 16E9 ::r q ::comment ᛩ RUNIC LETTER Q
+::u 16EA ::r x ::comment ᛪ RUNIC LETTER X
+
+::u 17B9 ::r oe ::comment Khmer vowel sign y (short)
+::u 17BA ::r oe ::comment Khmer vowel sign yy (long)
+::u 17C6 ::r +m  ::comment Khmer sign nikahit (cf. anusvara)
+::u 17C7 ::r +h  ::comment Khmer sign reahmuk (cf. visarga)
+::u 17C8 ::r '   ::comment Khmer sign yuukaleapintu (short vowel and glottal stop)
+::u 17C9 ::r ""  ::comment Khmer sign muusikatoan: changes the second register to the first
+::u 17CA ::r ""  ::comment Khmer sign triisap: changes the first register to the second
+::u 17CB ::r ""  ::comment Khmer sign bantoc (vowel shortener)
+::u 17D2 ::r ""  ::comment Khmer sign coeng (foot/subscript, cf. virama = no vowel)
+::u 17D5 ::r .  ::comment Khmer sign bariyoosan; period ending entire text or chapter
+
+::u 180E ::r '  ::comment ᠎ Mongolian vowel separator
+
+::u 1B80 ::r +ng  ::comment ᮀ Sundanese sign panyecek
+::u 1B81 ::r +r  ::comment ᮁ Sundanese sign panglayar
+::u 1B82 ::r +h  ::comment ᮂ Sundanese sign pangwisad
+::u 1BA1 ::r ya  ::comment ᮡ Sundanese consonant sign pamingkal
+::u 1BA2 ::r ra  ::comment ᮢ Sundanese consonant sign panyakr
+::u 1BA3 ::r la  ::comment ᮣ Sundanese consonant sign panyiku
+::u 1BA4 ::r i  ::comment ᮤ Sundanese consonant sign panghulu
+::u 1BA5 ::r u  ::comment ᮥ Sundanese consonant sign panyuku
+::u 1BA6 ::r e  ::comment ᮦ Sundanese vowel sign panaelaeng
+::u 1BA7 ::r o  ::comment ᮧ Sundanese vowel sign panolong
+::u 1BA8 ::r e  ::comment ᮨ Sundanese vowel sign pamepet
+::u 1BA9 ::r eu  ::comment ᮩ Sundanese vowel sign paneuleung
+::u 1BAA ::r ""  ::comment ᮪ Sundanese sign pamaaeh or patén (no vowel/virama)
+
+::u 1FBD ::r ""  ::comment ᾽ Greek koronis 
+::u 1FFE ::r ""  ::comment Greek dasia (rough breathing)
+
+::u 2002 ::r " "  ::comment en space
+::u 2003 ::r " "  ::comment em space
+::u 2004 ::r " "  ::comment three-per-em space
+::u 2005 ::r " "  ::comment four-per-em space
+::u 2006 ::r " "  ::comment six-per-em space
+::u 2007 ::r " "  ::comment figure space
+::u 2008 ::r " "  ::comment punctuation space
+::u 2009 ::r " "  ::comment thin space
+::u 200A ::r " "  ::comment hair space
+::u 202F ::r " "  ::comment narrow no-break space
+
+::u 2D30 ::r a ::comment TIFINAGH LETTER YA ⴰ
+::u 2D31 ::r b ::comment TIFINAGH LETTER YAB ⴱ
+::u 2D32 ::r bh ::comment TIFINAGH LETTER YABH ⴲ
+::u 2D33 ::r g ::comment TIFINAGH LETTER YAG ⴳ
+::u 2D34 ::r ghh ::comment TIFINAGH LETTER YAGHH ⴴ
+::u 2D35 ::r j ::comment TIFINAGH LETTER BERBER ACADEMY YAJ ⴵ
+::u 2D36 ::r j ::comment TIFINAGH LETTER YAJ ⴶ
+::u 2D37 ::r d ::comment TIFINAGH LETTER YAD ⴷ
+::u 2D38 ::r dh ::comment TIFINAGH LETTER YADH ⴸ
+::u 2D39 ::r dd ::comment TIFINAGH LETTER YADD ⴹ
+::u 2D3A ::r ddh ::comment TIFINAGH LETTER YADDH ⴺ
+::u 2D3B ::r e ::comment TIFINAGH LETTER YEY ⴻ
+::u 2D3C ::r f ::comment TIFINAGH LETTER YAF ⴼ
+::u 2D3D ::r k ::comment TIFINAGH LETTER YAK ⴽ
+::u 2D3E ::r k ::comment TIFINAGH LETTER TUAREG YAK ⴾ
+::u 2D3F ::r khh ::comment TIFINAGH LETTER YAKHH ⴿ
+::u 2D40 ::r h ::comment TIFINAGH LETTER YAH ⵀ
+::u 2D41 ::r h ::comment TIFINAGH LETTER BERBER ACADEMY YAH ⵁ
+::u 2D42 ::r h ::comment TIFINAGH LETTER TUAREG YAH ⵂ
+::u 2D43 ::r hh ::comment TIFINAGH LETTER YAHH ⵃ
+::u 2D44 ::r ' ::comment TIFINAGH LETTER YAA ⵄ
+::u 2D45 ::r kh ::comment TIFINAGH LETTER YAKH ⵅ
+::u 2D46 ::r kh ::comment TIFINAGH LETTER TUAREG YAKH ⵆ
+::u 2D47 ::r q ::comment TIFINAGH LETTER YAQ ⵇ
+::u 2D48 ::r q ::comment TIFINAGH LETTER TUAREG YAQ ⵈ
+::u 2D49 ::r i ::comment TIFINAGH LETTER YI ⵉ
+::u 2D4A ::r zh ::comment TIFINAGH LETTER YAZH ⵊ
+::u 2D4B ::r zh ::comment TIFINAGH LETTER AHAGGAR YAZH ⵋ
+::u 2D4C ::r zh ::comment TIFINAGH LETTER TUAREG YAZH ⵌ
+::u 2D4D ::r l ::comment TIFINAGH LETTER YAL ⵍ
+::u 2D4E ::r m ::comment TIFINAGH LETTER YAM ⵎ
+::u 2D4F ::r n ::comment TIFINAGH LETTER YAN ⵏ
+::u 2D50 ::r gn ::comment TIFINAGH LETTER TUAREG YAGN ⵐ
+::u 2D51 ::r ng ::comment TIFINAGH LETTER TUAREG YANG ⵑ
+::u 2D52 ::r p ::comment TIFINAGH LETTER YAP ⵒ
+::u 2D53 ::r u ::comment TIFINAGH LETTER YU ⵓ
+::u 2D54 ::r r ::comment TIFINAGH LETTER YAR ⵔ
+::u 2D55 ::r rr ::comment TIFINAGH LETTER YARR ⵕ
+::u 2D56 ::r gh ::comment TIFINAGH LETTER YAGH ⵖ
+::u 2D57 ::r gh ::comment TIFINAGH LETTER TUAREG YAGH ⵗ
+::u 2D58 ::r gh ::comment TIFINAGH LETTER AYER YAGH ⵘ
+::u 2D59 ::r s ::comment TIFINAGH LETTER YAS ⵙ
+::u 2D5A ::r ss ::comment TIFINAGH LETTER YASS ⵚ
+::u 2D5B ::r sh ::comment TIFINAGH LETTER YASH ⵛ
+::u 2D5C ::r t ::comment TIFINAGH LETTER YAT ⵜ
+::u 2D5D ::r th ::comment TIFINAGH LETTER YATH ⵝ
+::u 2D5E ::r ch ::comment TIFINAGH LETTER YACH ⵞ
+::u 2D5F ::r tt ::comment TIFINAGH LETTER YATT ⵟ
+::u 2D60 ::r v ::comment TIFINAGH LETTER YAV ⵠ
+::u 2D61 ::r w ::comment TIFINAGH LETTER YAW ⵡ
+::u 2D62 ::r y ::comment TIFINAGH LETTER YAY ⵢ
+::u 2D63 ::r z ::comment TIFINAGH LETTER YAZ ⵣ
+::u 2D64 ::r z ::comment TIFINAGH LETTER TAWELLEMET YAZ ⵤ
+::u 2D65 ::r zz ::comment TIFINAGH LETTER YAZZ ⵥ
+::u 2D66 ::r ye ::comment TIFINAGH LETTER YE ⵦ
+::u 2D67 ::r yo ::comment TIFINAGH LETTER YO ⵧ
+::u 2D6F ::r "" ::comment TIFINAGH MODIFIER LETTER LABIALIZATION MARK ⵯ
+::u 2D70 ::r "" ::comment TIFINAGH SEPARATOR MARK ⵰
+::u 2D7F ::r "" ::comment TIFINAGH CONSONANT JOINER ⵿
+
+::u 3063 ::r tsu ::comment Hiragana letter small tsu
+::u 30C3 ::r tsu ::comment Katakana letter small tsu
+
+::u ABE3 ::r o  ::comment ꯣ Meetei Mayek vowel sign onap
+::u ABE7 ::r ou  ::comment ꯧ Meetei Mayek vowel sign sounap
+
+::u F008 ::r ""  ::comment Yoruba diacritic in private use area
+::u F00F ::r ""  ::comment Yoruba diacritic in private use area
+::u F023 ::r ""  ::comment Yoruba diacritic in private use area
+::u F025 ::r ""  ::comment Yoruba diacritic in private use area
+
+::u F8D0 ::r a   ::name KLINGON LETTER A
+::u F8D1 ::r b   ::name KLINGON LETTER B
+::u F8D2 ::r ch  ::name KLINGON LETTER CH
+::u F8D3 ::r D   ::name KLINGON LETTER D
+::u F8D4 ::r e   ::name KLINGON LETTER E
+::u F8D5 ::r gh  ::name KLINGON LETTER GH
+::u F8D6 ::r H   ::name KLINGON LETTER H
+::u F8D7 ::r I   ::name KLINGON LETTER I
+::u F8D8 ::r j   ::name KLINGON LETTER J
+::u F8D9 ::r l   ::name KLINGON LETTER L
+::u F8DA ::r m   ::name KLINGON LETTER M
+::u F8DB ::r n   ::name KLINGON LETTER N
+::u F8DC ::r ng  ::name KLINGON LETTER NG
+::u F8DD ::r o   ::name KLINGON LETTER O
+::u F8DE ::r p   ::name KLINGON LETTER P
+::u F8DF ::r q   ::name KLINGON LETTER Q
+::u F8E0 ::r Q   ::name KLINGON LETTER Q
+::u F8E1 ::r r   ::name KLINGON LETTER R
+::u F8E2 ::r S   ::name KLINGON LETTER S
+::u F8E3 ::r t   ::name KLINGON LETTER T
+::u F8E4 ::r tlh ::name KLINGON LETTER TLH
+::u F8E5 ::r u   ::name KLINGON LETTER U
+::u F8E6 ::r v   ::name KLINGON LETTER V
+::u F8E7 ::r w   ::name KLINGON LETTER W
+::u F8E8 ::r y   ::name KLINGON LETTER Y
+::u F8E9 ::r '   ::name KLINGON LETTER GLOTTAL STOP
+::u F8F0 ::num 0 ::name KLINGON DIGIT ZERO
+::u F8F1 ::num 1 ::name KLINGON DIGIT ONE
+::u F8F2 ::num 2 ::name KLINGON DIGIT TWO
+::u F8F3 ::num 3 ::name KLINGON DIGIT THREE
+::u F8F4 ::num 4 ::name KLINGON DIGIT FOUR
+::u F8F5 ::num 5 ::name KLINGON DIGIT FIVE
+::u F8F6 ::num 6 ::name KLINGON DIGIT SIX
+::u F8F7 ::num 7 ::name KLINGON DIGIT SEVEN
+::u F8F8 ::num 8 ::name KLINGON DIGIT EIGHT
+::u F8F9 ::num 9 ::name KLINGON DIGIT NINE
+::u F8FD ::r ,   ::name KLINGON COMMA
+::u F8FE ::r .   ::name KLINGON FULL STOP
+::u F8FF ::name KLINGON MUMMIFICATION GLYPH
+
+::u 1163D ::r +m ::comment Modi sign anusvara
+::u 1163E ::r +h ::comment Modi sign visarga
+
+::u 13068 ::num 1000000 ::comment Egyptian Hieroglyph
+::u 1308B ::r r ::comment Egyptian Hieroglyph ::pic mouth
+::u 1309D ::r ' ::comment Egyptian Hieroglyph (ayn) ::pic forearm
+::u 130A7 ::r d ::comment Egyptian Hieroglyph ::pic hand
+::u 130AD ::num 10000 ::comment Egyptian Hieroglyph
+::u 130AE ::num 20000 ::comment Egyptian Hieroglyph
+::u 130AF ::num 30000 ::comment Egyptian Hieroglyph
+::u 130B0 ::num 40000 ::comment Egyptian Hieroglyph
+::u 130B1 ::num 50000 ::comment Egyptian Hieroglyph
+::u 130B2 ::num 60000 ::comment Egyptian Hieroglyph
+::u 130B3 ::num 70000 ::comment Egyptian Hieroglyph
+::u 130B4 ::num 80000 ::comment Egyptian Hieroglyph
+::u 130B5 ::num 90000 ::comment Egyptian Hieroglyph
+::u 130B6 ::num 50000 ::comment Egyptian Hieroglyph
+::u 130C0 ::r b ::comment Egyptian Hieroglyph ::pic foot
+::u 130ED ::r l ::comment Egyptian Hieroglyph [also rw] ::pic lion recumbent 
+::u 13121 ::r h ::comment Egyptian Hieroglyph (f-underscore) ::pic aninal's belly and udder 
+::u 1313F ::r a ::comment Egyptian Hieroglyph (alef) ::pic vulture
+::u 13153 ::r m ::comment Egyptian Hieroglyph ::pic owl
+::u 13171 ::r w ::comment Egyptian Hieroglyph ::pic quail chick
+::u 13187 ::r ::comment Egyptian Hieroglyph (determinative/son) H8 ::pic egg
+::u 13190 ::num 100000 ::comment Egyptian Hieroglyph
+::u 13191 ::r f ::comment Egyptian Hieroglyph ::pic horned viper
+::u 13193 ::r d ::comment Egyptian Hieroglyph (J) ::pic cobra
+::u 131BC ::num 1000 ::comment Egyptian Hieroglyph
+::u 131BD ::num 2000 ::comment Egyptian Hieroglyph
+::u 131BE ::num 3000 ::comment Egyptian Hieroglyph
+::u 131BF ::num 4000 ::comment Egyptian Hieroglyph
+::u 131C0 ::num 5000 ::comment Egyptian Hieroglyph
+::u 131C1 ::num 6000 ::comment Egyptian Hieroglyph
+::u 131C2 ::num 7000 ::comment Egyptian Hieroglyph
+::u 131C3 ::num 8000 ::comment Egyptian Hieroglyph
+::u 131C4 ::num 9000 ::comment Egyptian Hieroglyph
+::u 131CB ::r i ::comment Egyptian Hieroglyph (yod) ::pic single reed 
+::u 131CC ::r y ::comment Egyptian Hieroglyph ::pic double reed 
+::u 1320E ::r q ::comment Egyptian Hieroglyph (qaf) ::pic sandy slope
+::u 13209 ::comment Egyptian Hieroglyph ::pic desert hills
+::u 13216 ::r n ::comment Egyptian Hieroglyph ::pic ripple of water
+::u 13219 ::r sh ::comment Egyptian Hieroglyph (š) ::pic basin 
+::u 13254 ::r h ::comment Egyptian Hieroglyph ::pic reed shelter
+::u 13283 ::r z ::comment Egyptian Hieroglyph [also S?] ::pic door bolt 
+::u 132AA ::r p ::comment Egyptian Hieroglyph ::pic stool
+::u 132D4 ::r n ::comment Egyptian Hieroglyph ::pic red crown
+::u 132F4 ::r s ::comment Egyptian Hieroglyph [also Z?] ::pic folded cloth 
+::u 13319 ::comment Egyptian Hieroglyph ::pic throw stick
+::u 13362 ::num 100 ::comment Egyptian Hieroglyph
+::u 13363 ::num 200 ::comment Egyptian Hieroglyph
+::u 13364 ::num 300 ::comment Egyptian Hieroglyph
+::u 13365 ::num 400 ::comment Egyptian Hieroglyph
+::u 13366 ::num 500 ::comment Egyptian Hieroglyph
+::u 13367 ::num 600 ::comment Egyptian Hieroglyph
+::u 13368 ::num 700 ::comment Egyptian Hieroglyph
+::u 13369 ::num 800 ::comment Egyptian Hieroglyph
+::u 1336A ::num 900 ::comment Egyptian Hieroglyph
+::u 1336B ::num 500 ::comment Egyptian Hieroglyph
+::u 1336F ::r o ::comment Egyptian Hieroglyph ::pic lasso
+::u 1337F ::r t ::comment Egyptian Hieroglyph (ṯ) ::pic hobble 
+::u 13386 ::num 10 ::comment Egyptian Hieroglyph
+::u 13387 ::num 20 ::comment Egyptian Hieroglyph
+::u 13388 ::num 30 ::comment Egyptian Hieroglyph
+::u 13389 ::num 40 ::comment Egyptian Hieroglyph
+::u 1338A ::num 50 ::comment Egyptian Hieroglyph
+::u 1338B ::num 60 ::comment Egyptian Hieroglyph
+::u 1338C ::num 70 ::comment Egyptian Hieroglyph
+::u 1338D ::num 80 ::comment Egyptian Hieroglyph
+::u 1338E ::num 90 ::comment Egyptian Hieroglyph
+::u 1338F ::num 20 ::comment Egyptian Hieroglyph
+::u 13390 ::num 30 ::comment Egyptian Hieroglyph
+::u 13391 ::num 40 ::comment Egyptian Hieroglyph
+::u 13392 ::num 50 ::comment Egyptian Hieroglyph
+::u 1339B ::r h ::comment Egyptian Hieroglyph ::pic twisted flax
+::u 133A1 ::r k ::comment Egyptian Hieroglyph ::pic basket with handle
+::u 133A2 ::r k ::comment Egyptian Hieroglyph ::pic basket with handle, variant
+::u 133A4 ::r g ::comment Egyptian Hieroglyph ::pic bag
+::u 133BC ::r g ::comment Egyptian Hieroglyph ::pic stand
+::u 133CF ::r t ::comment Egyptian Hieroglyph ::pic loaf
+::u 133ED ::r y ::comment Egyptian Hieroglyph ::pic two strokes
+::u 133F2 ::r w ::comment Egyptian Hieroglyph ::pic quail chick, hieratic variant         
+::u 133FA ::num 1 ::comment Egyptian Hieroglyph
+::u 133FB ::num 2 ::comment Egyptian Hieroglyph
+::u 133FC ::num 3 ::comment Egyptian Hieroglyph
+::u 133FD ::num 4 ::comment Egyptian Hieroglyph
+::u 133FE ::num 5 ::comment Egyptian Hieroglyph
+::u 133FF ::num 6 ::comment Egyptian Hieroglyph
+::u 13400 ::num 7 ::comment Egyptian Hieroglyph
+::u 13401 ::num 8 ::comment Egyptian Hieroglyph
+::u 13402 ::num 9 ::comment Egyptian Hieroglyph
+::u 13403 ::num 5 ::comment Egyptian Hieroglyph
+::u 1340D ::r kh ::comment Egyptian Hieroglyph (ḫ, khah) ::pic placenta?
+::u 1341D ::r m ::comment Egyptian Hieroglyph (also jm)

uroman/data/romanization-table-arabic-block.txt

@@ -0,0 +1,179 @@
+::s ، ::t , ::comment ARABIC COMMA
+::s ؛ ::t ; ::comment ARABIC SEMICOLON
+::s ؟ ::t ? ::comment ARABIC QUESTION MARK
+::s ء ::t ' ::comment ARABIC LETTER HAMZA
+::s آ ::t a ::comment ARABIC LETTER ALEF WITH MADDA ABOVE
+::s أ ::t a ::comment ARABIC LETTER ALEF WITH HAMZA ABOVE
+::s ؤ ::t w ::comment ARABIC LETTER WAW WITH HAMZA ABOVE
+::s إ ::t i ::comment ARABIC LETTER ALEF WITH HAMZA BELOW
+::s ئ ::t ye ::comment ARABIC LETTER YEH WITH HAMZA ABOVE
+::s ا ::t a ::comment ARABIC LETTER ALEF
+::s ب ::t b ::comment ARABIC LETTER BEH
+::s ة ::t a ::comment ARABIC LETTER TEH MARBUTA
+::s ت ::t t ::comment ARABIC LETTER TEH
+::s ث ::t th ::comment ARABIC LETTER THEH
+::s ج ::t j ::comment ARABIC LETTER JEEM
+::s ح ::t h ::comment ARABIC LETTER HAH
+::s خ ::t kh ::comment ARABIC LETTER KHAH
+::s د ::t d ::comment ARABIC LETTER DAL
+::s ذ ::t th ::comment ARABIC LETTER THAL
+::s ر ::t r ::comment ARABIC LETTER REH
+::s ز ::t z ::comment ARABIC LETTER ZAIN
+::s س ::t s ::comment ARABIC LETTER SEEN
+::s ش ::t sh ::comment ARABIC LETTER SHEEN
+::s ص ::t s ::comment ARABIC LETTER SAD
+::s ض ::t d ::comment ARABIC LETTER DAD
+::s ط ::t t ::comment ARABIC LETTER TAH
+::s ظ ::t z ::comment ARABIC LETTER ZAH
+::s ع ::t ' ::comment ARABIC LETTER AIN
+::s غ ::t gh ::comment ARABIC LETTER GHAIN
+::s ـ ::t - ::comment ARABIC TATWEEL
+::s ف ::t f ::comment ARABIC LETTER FEH
+::s ق ::t q ::comment ARABIC LETTER QAF
+::s ك ::t k ::comment ARABIC LETTER KAF
+::s ل ::t l ::comment ARABIC LETTER LAM
+::s م ::t m ::comment ARABIC LETTER MEEM
+::s ن ::t n ::comment ARABIC LETTER NOON
+::s ه ::t h ::comment ARABIC LETTER HEH
+::s و ::t w ::comment ARABIC LETTER WAW
+::s ى ::t a ::comment ARABIC LETTER ALEF MAKSURA
+::s ي ::t y ::comment ARABIC LETTER YEH
+::s َ ::t a ::comment ARABIC FATHA
+::s ُ ::t u ::comment ARABIC DAMMA
+::s ِ ::t i ::comment ARABIC KASRA
+::s ْ ::t  ::comment ARABIC SUKUN
+::s ٔ ::t ' ::comment ARABIC HAMZA ABOVE
+::s ٕ ::t ' ::comment ARABIC HAMZA BELOW
+::s ٠ ::t 0 ::comment ARABIC-INDIC DIGIT ZERO
+::s ١ ::t 1 ::comment ARABIC-INDIC DIGIT ONE
+::s ٢ ::t 2 ::comment ARABIC-INDIC DIGIT TWO
+::s ٣ ::t 3 ::comment ARABIC-INDIC DIGIT THREE
+::s ٤ ::t 4 ::comment ARABIC-INDIC DIGIT FOUR
+::s ٥ ::t 5 ::comment ARABIC-INDIC DIGIT FIVE
+::s ٦ ::t 6 ::comment ARABIC-INDIC DIGIT SIX
+::s ٧ ::t 7 ::comment ARABIC-INDIC DIGIT SEVEN
+::s ٨ ::t 8 ::comment ARABIC-INDIC DIGIT EIGHT
+::s ٩ ::t 9 ::comment ARABIC-INDIC DIGIT NINE
+::s ٪ ::t % ::comment ARABIC PERCENT SIGN
+::s ٫ ::t , ::comment ARABIC DECIMAL SEPARATOR
+::s ٬ ::t , ::comment ARABIC THOUSANDS SEPARATOR
+::s ٮ ::t b ::comment ARABIC LETTER DOTLESS BEH
+::s ٯ ::t q ::comment ARABIC LETTER DOTLESS QAF
+::s ٰ ::t a ::comment ARABIC LETTER SUPERSCRIPT ALEF
+::s ٱ ::t a ::comment ARABIC LETTER ALEF WASLA
+::s ٲ ::t a ::comment ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE
+::s ٳ ::t a ::comment ARABIC LETTER ALEF WITH WAVY HAMZA BELOW
+::s ٷ ::t u ::comment ARABIC LETTER U WITH HAMZA ABOVE
+::s ٹ ::t tt ::comment ARABIC LETTER TTEH
+::s ٺ ::t tt ::comment ARABIC LETTER TTEHEH
+::s ٻ ::t b ::comment ARABIC LETTER BEEH
+::s ټ ::t t ::comment ARABIC LETTER TEH WITH RING
+::s ٽ ::t t ::comment ARABIC LETTER TEH WITH THREE DOTS ABOVE DOWNWARDS
+::s پ ::t p ::comment ARABIC LETTER PEH
+::s ٿ ::t t ::comment ARABIC LETTER TEHEH
+::s ڀ ::t b ::comment ARABIC LETTER BEHEH
+::s ځ ::t h ::comment ARABIC LETTER HAH WITH HAMZA ABOVE
+::s ڂ ::t h ::comment ARABIC LETTER HAH WITH TWO DOTS VERTICAL ABOVE
+::s ڃ ::t ny ::comment ARABIC LETTER NYEH
+::s ڄ ::t dy ::comment ARABIC LETTER DYEH
+::s څ ::t h ::comment ARABIC LETTER HAH WITH THREE DOTS ABOVE
+::s چ ::t tch ::comment ARABIC LETTER TCHEH
+::s ڇ ::t tch ::comment ARABIC LETTER TCHEHEH
+::s ڈ ::t dd ::comment ARABIC LETTER DDAL
+::s ډ ::t d ::comment ARABIC LETTER DAL WITH RING
+::s ڊ ::t d ::comment ARABIC LETTER DAL WITH DOT BELOW
+::s ڋ ::t d ::comment ARABIC LETTER DAL WITH DOT BELOW AND SMALL TAH
+::s ڌ ::t d ::comment ARABIC LETTER DAHAL
+::s ڍ ::t dd ::comment ARABIC LETTER DDAHAL
+::s ڎ ::t d ::comment ARABIC LETTER DUL
+::s ڏ ::t d ::comment ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS
+::s ڐ ::t d ::comment ARABIC LETTER DAL WITH FOUR DOTS ABOVE
+::s ڑ ::t rr ::comment ARABIC LETTER RREH
+::s ڒ ::t r ::comment ARABIC LETTER REH WITH SMALL V
+::s ړ ::t r ::comment ARABIC LETTER REH WITH RING
+::s ڔ ::t r ::comment ARABIC LETTER REH WITH DOT BELOW
+::s ڕ ::t r ::comment ARABIC LETTER REH WITH SMALL V BELOW
+::s ږ ::t r ::comment ARABIC LETTER REH WITH DOT BELOW AND DOT ABOVE
+::s ڗ ::t r ::comment ARABIC LETTER REH WITH TWO DOTS ABOVE
+::s ژ ::t j ::comment ARABIC LETTER JEH
+::s ڙ ::t r ::comment ARABIC LETTER REH WITH FOUR DOTS ABOVE
+::s ښ ::t s ::comment ARABIC LETTER SEEN WITH DOT BELOW AND DOT ABOVE
+::s ڛ ::t s ::comment ARABIC LETTER SEEN WITH THREE DOTS BELOW
+::s ڜ ::t s ::comment ARABIC LETTER SEEN WITH THREE DOTS BELOW AND THREE DOTS ABOVE
+::s ڝ ::t s ::comment ARABIC LETTER SAD WITH TWO DOTS BELOW
+::s ڞ ::t s ::comment ARABIC LETTER SAD WITH THREE DOTS ABOVE
+::s ڟ ::t t ::comment ARABIC LETTER TAH WITH THREE DOTS ABOVE
+::s ڠ ::t n ::comment ARABIC LETTER AIN WITH THREE DOTS ABOVE
+::s ڡ ::t f ::comment ARABIC LETTER DOTLESS FEH
+::s ڢ ::t f ::comment ARABIC LETTER FEH WITH DOT MOVED BELOW
+::s ڣ ::t f ::comment ARABIC LETTER FEH WITH DOT BELOW
+::s ڤ ::t v ::comment ARABIC LETTER VEH
+::s ڥ ::t f ::comment ARABIC LETTER FEH WITH THREE DOTS BELOW
+::s ڦ ::t p ::comment ARABIC LETTER PEHEH
+::s ڧ ::t q ::comment ARABIC LETTER QAF WITH DOT ABOVE
+::s ڨ ::t q ::comment ARABIC LETTER QAF WITH THREE DOTS ABOVE
+::s ک ::t k ::comment ARABIC LETTER KEHEH
+::s ڪ ::t k ::comment ARABIC LETTER SWASH KAF
+::s ګ ::t k ::comment ARABIC LETTER KAF WITH RING
+::s ڬ ::t k ::comment ARABIC LETTER KAF WITH DOT ABOVE
+::s ڭ ::t ng ::comment ARABIC LETTER NG
+::s ڮ ::t k ::comment ARABIC LETTER KAF WITH THREE DOTS BELOW
+::s گ ::t g ::comment ARABIC LETTER GAF
+::s ڰ ::t g ::comment ARABIC LETTER GAF WITH RING
+::s ڱ ::t ng ::comment ARABIC LETTER NGOEH
+::s ڲ ::t g ::comment ARABIC LETTER GAF WITH TWO DOTS BELOW
+::s ڳ ::t g ::comment ARABIC LETTER GUEH
+::s ڴ ::t g ::comment ARABIC LETTER GAF WITH THREE DOTS ABOVE
+::s ڵ ::t l ::comment ARABIC LETTER LAM WITH SMALL V
+::s ڶ ::t l ::comment ARABIC LETTER LAM WITH DOT ABOVE
+::s ڷ ::t l ::comment ARABIC LETTER LAM WITH THREE DOTS ABOVE
+::s ڸ ::t l ::comment ARABIC LETTER LAM WITH THREE DOTS BELOW
+::s ڹ ::t n ::comment ARABIC LETTER NOON WITH DOT BELOW
+::s ں ::t n ::comment ARABIC LETTER NOON GHUNNA
+::s ڻ ::t rn ::comment ARABIC LETTER RNOON
+::s ڼ ::t n ::comment ARABIC LETTER NOON WITH RING
+::s ڽ ::t n ::comment ARABIC LETTER NOON WITH THREE DOTS ABOVE
+::s ھ ::t h ::comment ARABIC LETTER HEH DOACHASHMEE
+::s ڿ ::t tch ::comment ARABIC LETTER TCHEH WITH DOT ABOVE
+::s ۀ ::t h ::comment ARABIC LETTER HEH WITH YEH ABOVE
+::s ہ ::t h ::comment ARABIC LETTER HEH GOAL
+::s ۂ ::t h ::comment ARABIC LETTER HEH GOAL WITH HAMZA ABOVE
+::s ۃ ::t a ::comment ARABIC LETTER TEH MARBUTA GOAL
+::s ۄ ::t w ::comment ARABIC LETTER WAW WITH RING
+::s ۅ ::t oe ::comment ARABIC LETTER KIRGHIZ OE
+::s ۆ ::t oe ::comment ARABIC LETTER OE
+::s ۇ ::t u ::comment ARABIC LETTER U
+::s ۈ ::t yu ::comment ARABIC LETTER YU
+::s ۉ ::t yu ::comment ARABIC LETTER KIRGHIZ YU
+::s ۊ ::t w ::comment ARABIC LETTER WAW WITH TWO DOTS ABOVE
+::s ۋ ::t v ::comment ARABIC LETTER VE
+::s ی ::t y ::comment ARABIC LETTER FARSI YEH
+::s ۍ ::t y ::comment ARABIC LETTER YEH WITH TAIL
+::s ێ ::t y ::comment ARABIC LETTER YEH WITH SMALL V
+::s ۏ ::t w ::comment ARABIC LETTER WAW WITH DOT ABOVE
+::s ې ::t e ::comment ARABIC LETTER E
+::s ۑ ::t y ::comment ARABIC LETTER YEH WITH THREE DOTS BELOW
+::s ے ::t y ::comment ARABIC LETTER YEH BARREE
+::s ۓ ::t y ::comment ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
+::s ۔ ::t . ::comment ARABIC FULL STOP
+::s ە ::t ae ::comment ARABIC LETTER AE
+::s ۮ ::t d ::comment ARABIC LETTER DAL WITH INVERTED V
+::s ۯ ::t r ::comment ARABIC LETTER REH WITH INVERTED V
+::s ۰ ::t 0 ::comment EXTENDED ARABIC-INDIC DIGIT ZERO
+::s ۱ ::t 1 ::comment EXTENDED ARABIC-INDIC DIGIT ONE
+::s ۲ ::t 2 ::comment EXTENDED ARABIC-INDIC DIGIT TWO
+::s ۳ ::t 3 ::comment EXTENDED ARABIC-INDIC DIGIT THREE
+::s ۴ ::t 4 ::comment EXTENDED ARABIC-INDIC DIGIT FOUR
+::s ۵ ::t 5 ::comment EXTENDED ARABIC-INDIC DIGIT FIVE
+::s ۶ ::t 6 ::comment EXTENDED ARABIC-INDIC DIGIT SIX
+::s ۷ ::t 7 ::comment EXTENDED ARABIC-INDIC DIGIT SEVEN
+::s ۸ ::t 8 ::comment EXTENDED ARABIC-INDIC DIGIT EIGHT
+::s ۹ ::t 9 ::comment EXTENDED ARABIC-INDIC DIGIT NINE
+::s ۺ ::t sh ::comment ARABIC LETTER SHEEN WITH DOT BELOW
+::s ۻ ::t d ::comment ARABIC LETTER DAD WITH DOT BELOW
+::s ۼ ::t gh ::comment ARABIC LETTER GHAIN WITH DOT BELOW
+::s ۽ ::t & ::comment ARABIC SIGN SINDHI AMPERSAND
+::s ﷲ ::t allah ::comment ARABIC LIGATURE ALLAH ISOLATED FORM
+
+::s ‌ ::t ::comment ZERO WIDTH NON-JOINER
+::s ‍ ::t ::comment ZERO WIDTH JOINER

uroman/data/romanization-table.txt

@@ -0,0 +1,2019 @@
+
+## European Latin extensions
+# Vowels
+::s Ä ::t Ae
+::s Ö ::t Oe
+::s Ü ::t Ue
+::s Å ::t Aa
+::s Æ ::t Ae
+::s Ø ::t oe
+::s Π::t Oe
+::s ä ::t ae
+::s ö ::t oe
+::s ü ::t ue
+::s å ::t aa
+::s æ ::t ae
+::s ø ::t oe
+::s œ ::t oe
+# Consonants
+::s Ç ::t S
+::s ç ::t s
+::s Ç ::t Ch ::lcode tur
+::s ç ::t ch ::lcode tur
+::s Ş ::t Sh
+::s ş ::t sh
+::s Ș ::t Sh
+::s ș ::t sh
+::s ß ::t ss
+::s Ț ::t Ts
+::s ț ::t ts
+
+# Digraphs
+# ::s ʣ ::t dz
+::s ʤ ::t dzh ::comment Latin small letter dezh digraph
+# ::s ʥ ::t dz
+# ::s ʦ ::t ts
+::s ʧ ::t tsh ::comment Latin small letter tesh digraph
+# ::s ʨ ::t tc
+
+# Miscellaneous
+::s ə ::t e
+
+# English
+::s chr ::t chr ::t-alt kr ::example chromosome, synchronize
+::s Chr ::t Chr ::t-alt Kr ::example Christmas, Chrysler
+::s eight ::t eight ::t-alt eit ::example eight, weight
+::s Eight ::t Eight ::t-alt Eit ::example Eighteen
+::s ight ::t ight ::t-alt ait ::example Knight
+::s gh ::t gh ::t-alt f, ph, "" ::example laugh, daughter
+::s high ::t high ::t-alt hai ::example highlight
+::s High ::t High ::t-alt Hai ::example High School
+::s Isle ::t Isle ::t-alt Ail ::use-only-for-whole-word ::example Isle
+::s Island ::t Island ::t-alt Ailand ::use-only-for-whole-word ::example Island
+::s kn ::t kn ::t-alt n ::use-only-at-start-of-word ::example knowledge
+::s Kn ::t Kn ::t-alt N ::use-only-at-start-of-word ::example Knight
+::s Mc ::t Mc ::t-alt Mac ::use-only-at-start-of-word ::example McNulty
+::s mc ::t mc ::t-alt mac ::use-only-at-start-of-word
+::s oo ::t oo ::t-alt u ::lcode eng ::example Brooklyn; Goose Bay
+::s ph ::t ph ::t-alt f ::example alpha
+::s Ph ::t Ph ::t-alt F ::example Philip
+::s Thom ::t Thom ::t-alt Tom ::use-only-at-start-of-word ::example Thomas, Thompson
+::s tion ::t tion ::t-alt shen ::example
+::s Sean ::t Sean ::t-alt Shawn ::use-only-for-whole-word
+::s ssion ::t ssion ::t-alt shen ::example Sessions
+::s St ::t St ::t-alt Saint ::use-only-for-whole-word
+::s St. ::t St. ::t-alt Saint ::use-only-for-whole-word
+::s Wr ::t Wr ::t-alt R ::example Wren
+::s wr ::t wr ::t-alt r ::example Cartwright
+::s x ::t x ::t-alt ks ::example Mexico
+::s x ::t x ::t-alt gz ::example example, anxiety, exhaust, exit
+
+# French
+::s â ::t a ::t-alt as ::example pâte/paste, pastry
+::s ê ::t e ::t-alt es ::example fête/feast
+::s î ::t i ::t-alt is ::example île/isle
+::s ô ::t o ::t-alt os ::example côte/coast
+::s û ::t u ::t-alt us ::example août/August
+::s eaux ::t eaux ::t-alt o ::example Bordeaux
+::s eau ::t eau ::t-alt o ::example Chateau
+::s auld ::t auld ::t-alt o ::use-only-at-end-of-word ::example Renauld
+::s ault ::t ault ::t-alt o ::use-only-at-end-of-word ::example Renault
+::s oux ::t oux ::t-alt u
+::s ois ::t ois ::t-alt oa ::use-only-at-end-of-word ::example Dubois
+
+# German
+::s Sch ::t Sch ::t-alt Sh
+::s sch ::t sch ::t-alt sh
+::s stein ::t stein ::t-alt shtain
+::s dt ::t dt ::t-alt tt ::use-only-at-end-of-word ::example Schmidt
+
+# Dutch
+::s ij ::t ij ::t-alt ai
+::s Ij ::t Ij ::t-alt Ai
+
+# Latvian
+::s Ā ::t A ::t-alt Aa ::lcode lav
+::s ā ::t a ::t-alt aa ::lcode lav
+::s Ē ::t E ::t-alt Ee ::lcode lav
+::s ē ::t e ::t-alt ee ::lcode lav
+::s Ī ::t I ::t-alt Ii ::lcode lav
+::s ī ::t i ::t-alt ii ::lcode lav
+::s Ū ::t U ::t-alt Uu ::lcode lav
+::s ū ::t u ::t-alt uu ::lcode lav
+::s Ģ ::t G ::t-alt Gj ::lcode lav
+::s ģ ::t g ::t-alt gj ::lcode lav
+::s Ķ ::t K ::t-alt Kj ::lcode lav
+::s ķ ::t k ::t-alt kj ::lcode lav
+::s Ļ ::t L ::t-alt Lj ::lcode lav
+::s ļ ::t l ::t-alt lj ::lcode lav
+::s Ņ ::t N ::t-alt Nj ::lcode lav
+::s ņ ::t n ::t-alt nj ::lcode lav
+::s C ::t C ::t-alt Ts ::lcode lav
+::s c ::t c ::t-alt ts ::lcode lav
+::s Č ::t C ::t-alt Tsh ::lcode lav
+::s č ::t c ::t-alt tsh ::lcode lav
+::s Š ::t Sh ::t-alt s ::lcode lav
+::s š ::t sh ::t-alt s ::lcode lav
+::s Ž ::t Z ::t-alt Zh ::lcode lav
+::s ž ::t z ::t-alt zh ::lcode lav
+
+# Lithuanian
+::s C ::t C ::t-alt Ts ::lcode lit
+::s c ::t c ::t-alt ts ::lcode lit
+::s Č ::t C ::t-alt Tsh ::lcode lit
+::s č ::t c ::t-alt tsh ::lcode lit
+::s Š ::t Sh ::t-alt s ::lcode lit
+::s š ::t sh ::t-alt s ::lcode lit
+::s Ž ::t Z ::t-alt Zh ::lcode lit
+::s ž ::t z ::t-alt zh ::lcode lit
+
+# International Greek (e.g. as used in chemical compounds)
+::s β ::t b
+::s Β ::t B
+::s ϐ ::t b
+
+# Ancient Greek
+::s β ::t b ::lcode grc
+::s Β ::t B ::lcode grc
+::s γγ ::t ng ::lcode grc
+::s γκ ::t nk ::lcode grc
+::s γξ ::t nx ::lcode grc
+::s γχ ::t nch ::lcode grc
+::s ϱ ::t r ::lcode grc
+
+# Pontic Greek
+::s β ::t v ::t-alt b ::lcode pnt
+::s Β ::t V ::t-alt B ::lcode pnt
+::s ϐ ::t v ::t-alt b ::lcode pnt
+
+# Modern Greek (generally the default)
+::s β ::t v ::t-alt b ::lcode ell
+::s Β ::t V ::t-alt B ::lcode ell
+::s ϐ ::t v ::t-alt b ::lcode ell
+::s Ι ::t I
+::s ι ::t i
+::s ί ::t i
+::s ἶ ::t i
+::s Υ ::t Y
+::s υ ::t y
+::s Ρ ::t R
+::s ρ ::t r
+::s ϱ ::t r
+::s Χ ::t Ch ::t-alt Kh
+::s χ ::t ch ::t-alt kh
+::s φ ::t f ::t-alt ph
+::s Φ ::t F ::t-alt Ph
+::s Ντ ::t D
+::s ντ ::t nd ::t-alt d, nt
+# ::s ντζ ::t ntz
+::s Μπ ::t B
+::s μπ ::t b ::use-only-at-start-of-word
+::s μπ ::t mb ::t-alt b, mp ::dont-use-at-start-of-word
+::s λμπ ::t lb
+::s νμπ ::t nb
+::s ρμπ ::t rb
+::s γγ ::t ng
+::s Γκ ::t G
+::s γκ ::t ng ::t-alt g ::dont-use-at-start-of-word
+::s γκ ::t g ::use-only-at-start-of-word
+::s γξ ::t nx ::lcode grc
+::s γχ ::t nch ::lcode grc
+::s ει ::t ei ::t-alt i
+::s Ει ::t Ei ::t-alt I
+::s ευ ::t eu ::t-alt ev ::comment donated by Constantine
+::s Ευ ::t Eu ::t-alt Ev ::comment donated by Constantine
+::s αυ ::t au ::t-alt av
+::s Αυ ::t Au ::t-alt Av
+::s ου ::t ou ::t-alt u
+::s Ου ::t Ou ::t-alt U
+::s ηυ ::t eu
+::s Ηυ ::t Eu
+::s υι ::t ui
+::s Υι ::t Ui
+::s ωυ ::t ou
+::s Ωυ ::t Ou
+::s ͺ ::t ::comment GREEK YPOGEGRAMMENI (U+037A)
+::s ϒ ::t Y ::comment GREEK UPSILON WITH HOOK SYMBOL (U+03D2)
+::s ϓ ::t Y ::comment GREEK UPSILON WITH ACUTE AND HOOK SYMBOL (U+03D3)
+::s ϔ ::t Y ::comment GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL (U+03D4)
+::s ι ::t ::comment GREEK PROSGEGRAMMENI (U+1FBE)
+::s ᾿ ::t ::comment GREEK PSILI (U+1FBF)
+::s ῀ ::t ::comment GREEK PERISPOMENI (U+1FC0)
+::s ` ::t ::comment GREEK VARIA (U+1FEF)
+::s ´ ::t ::comment GREEK OXIA (U+1FFD)
+
+# Glagolitic
+::s Ⰿ ::t M ::comment GLAGOLITIC CAPITAL LETTER MYSLITE (U+2C0F)
+::s Ⱞ ::t M ::comment GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE (U+2C2E)
+::s ⰿ ::t m ::comment GLAGOLITIC SMALL LETTER MYSLITE (U+2C3F)
+::s ⱞ ::t m ::comment GLAGOLITIC SMALL LETTER LATINATE MYSLITE (U+2C5E)
+::s 𞀏 ::t m ::comment COMBINING GLAGOLITIC LETTER MYSLITE (U+1E00F)
+
+# Cyrillic
+::s Г ::t G ::t-alt H ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt h ::comment Cyrillic small ghe
+::s Е ::t E ::t-alt Ye ::comment Cyrillic capital ie
+::s е ::t e ::t-alt ye ::comment Cyrillic small ie
+::s Ё ::t E ::t-alt Yo
+::s ё ::t e ::t-alt yo
+::s Х ::t Kh ::t-alt Ch, H ::comment Cyrillic capital ha
+::s х ::t kh ::t-alt ch, h ::comment Cyrillic small ha
+::s Щ ::t Shch ::t-alt Sh
+::s щ ::t shch ::t-alt sh
+::s Ъ ::t ::comment Cyrillic capital hard sign
+::s ъ ::t ::comment Cyrillic small hard sign
+::s ᲆ ::t ::comment CYRILLIC SMALL LETTER TALL HARD SIGN
+::s Ы ::t Y ::comment Cyrillic capital yeru
+::s ы ::t y ::comment Cyrillic small yeru
+::s Ь ::t ::comment Cyrillic capital soft sign
+::s ь ::t ::comment Cyrillic small soft sign
+::s Ж ::t Zh ::comment Cyrillic capital letter zhe
+::s Ш ::t Sh ::comment Cyrillic capital letter sha
+::s Ч ::t Ch ::comment Cyrillic capital letter che
+::s Џ ::t Dzh ::comment Cyrillic capital letter dzhe
+::s Є ::t Ie ::comment Cyrillic capital letter ie
+::s Ю ::t Yu ::comment Cyrillic capital letter yu
+::s Я ::t Ya ::comment  Cyrillic capital letter ya
+
+::s Ҥ ::t Ng ::comment Cyrillic capital ligature EN GHE
+::s ҥ ::t ng ::comment Cyrillic small ligature EN GHE
+::s Ә ::t e ::comment Cyrillic capital schwa
+::s ә ::t e ::comment Cyrillic small schwa
+::s Ӏ ::t ' ::comment Cyrillic palochka
+::s Ҵ ::t TS ::comment Cyrillic capital ligature te tse, used in Abkhasian
+::s ҵ ::t ts ::comment Cyrillic small ligature te tse, used in Abkhasian
+::s Ӕ ::t AE ::comment Cyrillic capital ligature a ie
+::s ӕ ::t ae ::comment Cyrillic small ligature a ie
+::s ʹ ::t "'" ::comment modifier letter prime
+::s ʺ ::t '"' ::comment modifier letter double prime
+::s ий ::t iy ::dont-use-at-end-of-word
+::s ий ::t y ::use-only-at-end-of-word
+
+::s ᲈ ::t u ::comment CYRILLIC SMALL LETTER UNBLENDED UK ligature ou
+
+# Russian
+::s Г ::t G ::t-alt _NONE_ ::lcode rus ::comment Cyrillic capital letter ghe 
+::s г ::t g ::t-alt _NONE_ ::lcode rus ::comment Cyrillic small letter ghe
+::s Й ::t Y ::t-alt I, J ::lcode rus ::comment Cyrillic capital letter short i 
+::s й ::t y ::t-alt i, j ::lcode rus ::comment Cyrillic small letter short i 
+::s Ц ::t Ts ::t-alt C ::lcode rus ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode rus ::comment Cyrillic small letter tse
+::s Щ ::t Shch ::t-alt _NONE_ ::lcode rus ::comment Cyrillic capital letter shcha 
+::s щ ::t shch ::t-alt _NONE_ ::lcode rus ::comment Cyrillic small letter shcha 
+::s Ѣ ::t E ::t-alt Ie ::lcode rus ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode rus ::comment archaic Cyrillic small letter yat
+::s Е ::t E ::t-alt Ye ::dont-use-at-start-of-word ::lcode rus ::comment Cyrillic capital ie
+::s Е ::t Ye ::t-alt E ::use-only-at-start-of-word ::lcode rus
+::s е ::t e ::t-alt ye ::dont-use-at-start-of-word ::lcode rus ::comment Cyrillic small ie
+::s е ::t ye ::t-alt e ::use-only-at-start-of-word ::lcode rus
+::s ае ::t aye ::lcode rus
+::s а́е ::t aye ::lcode rus
+::s ее ::t eye ::lcode rus
+::s е́е ::t eye ::lcode rus
+::s ие ::t iye ::lcode rus
+::s и́е ::t iye ::lcode rus
+::s ое ::t oye ::lcode rus
+::s о́е ::t oye ::lcode rus
+::s уе ::t uye ::lcode rus
+::s у́е ::t uye ::lcode rus
+::s ье ::t ye ::lcode rus
+::s ъе ::t ye ::lcode rus
+::s Ё ::t Yo ::t-alt E ::lcode rus ::comment Cyrillic capital io
+::s ё ::t yo ::t-alt e ::lcode rus
+::s аё ::t ayo ::lcode rus
+::s а́ё ::t ayo ::lcode rus
+::s её ::t eyo ::lcode rus
+::s е́ё ::t eyo ::lcode rus
+::s иё ::t iyo ::lcode rus
+::s и́ё ::t iyo ::lcode rus
+::s оё ::t oyo ::lcode rus
+::s о́ё ::t oyo ::lcode rus
+::s уё ::t uyo ::lcode rus
+::s у́ё ::t uyo ::lcode rus
+::s ьё ::t yo ::lcode rus
+::s ъё ::t yo ::lcode rus
+::s ий ::t y ::lcode rus
+
+# Ukranian
+::s Г ::t H ::lcode ukr ::comment Ukrainian capital letter he 
+::s г ::t h ::lcode ukr ::comment Ukrainian small letter he
+::s Ґ ::t G ::lcode ukr ::comment Ukrainian capital letter ghe
+::s ґ ::t g ::lcode ukr ::comment Ukrainian small letter ghe
+::s Е ::t E ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic small ie
+::s И ::t Y ::lcode ukr ::comment Ukrainian capital letter i
+::s и ::t y ::lcode ukr ::comment Ukrainian small letter i
+::s Ї ::t Yi ::lcode ukr ::comment Ukrainian capital letter yi
+::s ї ::t yi ::lcode ukr ::comment Ukrainian small letter yi
+::s Й ::t I ::t-alt Y ::lcode ukr ::comment Cyrillic capital letter short i 
+::s й ::t i ::t-alt y ::lcode ukr ::comment Cyrillic small letter short i 
+::s Ц ::t Ts ::t-alt C ::lcode ukr ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode ukr ::comment Cyrillic small letter tse
+::s Щ ::t Shch ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic capital letter shcha 
+::s щ ::t shch ::t-alt _NONE_ ::lcode ukr ::comment Cyrillic small letter shcha 
+::s Ѣ ::t E ::t-alt Ie ::lcode ukr ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode ukr ::comment archaic Cyrillic small letter yat
+::s Иї ::t Yi ::lcode ukr ::comment avoid Yyi
+::s иї ::t yi ::lcode ukr ::comment avoid yyi
+::s ій ::t iy ::lcode ukr
+::s і́й ::t iy ::lcode ukr
+::s ий ::t y ::lcode ukr ::comment Зеленський/Zelensky
+
+# Belarusian
+::s Г ::t H ::t-alt G ::lcode bel ::comment capital letter he 
+::s г ::t h ::t-alt g ::lcode bel ::comment small letter he
+::s Ґ ::t G ::lcode bel ::comment capital letter ghe
+::s ґ ::t g ::lcode bel ::comment small letter ghe
+::s Й ::t J ::t-alt Y ::lcode bel ::comment Cyrillic capital letter short i 
+::s й ::t j ::t-alt y ::lcode bel ::comment Cyrillic small letter short i
+::s Ц ::t Ts ::t-alt C ::lcode bel ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode bel ::comment Cyrillic small letter tse
+::s Щ ::t Shch ::t-alt _NONE_ ::lcode bel ::comment Cyrillic capital letter shcha 
+::s щ ::t shch ::t-alt _NONE_ ::lcode bel ::comment Cyrillic small letter shcha 
+::s Ѣ ::t E ::t-alt Ie ::lcode bel ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode bel ::comment archaic Cyrillic small letter yat
+::s 'я ::t ya ::lcode bel
+::s ’я ::t ya ::lcode bel
+::s 'і ::t i ::lcode bel
+::s ’і ::t i ::lcode bel
+::s Ё ::t Yo ::t-alt E ::lcode bel ::comment Cyrillic capital io
+::s ё ::t yo ::t-alt e ::lcode bel
+::s ёў ::t you ::lcode bel
+::s ий ::t y ::lcode bel
+
+# Serbian
+::s Г ::t G ::t-alt _NONE_ ::lcode srp ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode srp ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt _NONE_ ::lcode srp ::comment Cyrillic capital ha
+::s х ::t h ::t-alt _NONE_ ::lcode srp ::comment Cyrillic small ha
+::s Е ::t E ::t-alt _NONE_ ::lcode srp ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode srp ::comment Cyrillic small ie
+::s Ђ ::t Dj ::lcode srp ::comment Cyrillic capital dje
+::s Љ ::t Lj ::lcode srp ::comment Cyrillic capital lje
+::s Ћ ::t Tsh ::lcode srp ::comment Cyrillic capital tshe
+::s Ж ::t Zh ::lcode srp ::comment Cyrillic capital zhe
+::s Ц ::t C ::t-alt Ts ::lcode srp ::comment Cyrillic capital tse
+::s ц ::t c ::t-alt ts ::lcode srp ::comment Cyrillic capital tse 
+::s Đ ::t Dj ::lcode srp ::comment Latin capital d with stroke
+::s đ ::t dj ::lcode srp ::comment Latin small d with stroke
+::s Ž ::t Zh ::lcode srp ::comment Latin capital z with caron
+::s ž ::t zh ::lcode srp ::comment Latin small z with caron
+::s Ć ::t Tsh ::lcode srp ::comment Latin capital c with acute
+::s ć ::t tsh ::lcode srp ::comment Latin small c with acute
+::s Č ::t Ch ::lcode srp ::comment Latin capital c with caron
+::s č ::t ch ::lcode srp ::comment Latin small c with caron
+::s Š ::t Sh ::lcode srp ::comment Latin capital s with caron
+::s š ::t sh ::lcode srp ::comment Latin small s with caron
+
+::s Г ::t G ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital ha
+::s х ::t h ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small ha
+::s Ц ::t C ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small letter tse
+::s Ч ::t C ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital letter che
+::s ч ::t c ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small letter che
+::s Џ ::t Dz ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital letter dzhe
+::s џ ::t dz ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small letter dzhe
+::s Е ::t E ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small ie
+::s Ш ::t S ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital sha
+::s ш ::t s ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small sha
+::s Ж ::t Z ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital zhe
+::s ж ::t z ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small zhe
+::s Љ ::t Lj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital lje
+::s љ ::t lj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small lje
+::s Њ ::t Nj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital nje
+::s њ ::t nj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small nje
+::s Ђ ::t Dj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital dje
+::s ђ ::t dj ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small dje
+::s Ћ ::t C ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic capital tshe
+::s ћ ::t c ::t-alt _NONE_ ::lcode srp2 ::comment Cyrillic small tshe
+::s Đ ::t Dj ::lcode srp2 ::comment Latin capital d with stroke
+::s đ ::t dj ::lcode srp2 ::comment Latin small d with stroke
+
+# Montenegrin extension (controversial)
+::s З́ ::t Zj ::lcode srp ::comment Cyrillic capital zje
+::s з́ ::t zj ::lcode srp ::comment Cyrillic small zje
+::s С́ ::t Sj ::lcode srp ::comment Cyrillic capital sje
+::s с́ ::t sj ::lcode srp ::comment Cyrillic small sje
+::s Ź ::t Zj ::lcode srp ::comment Latin capital z with acute
+::s ź ::t zj ::lcode srp ::comment Latin small z with acute
+::s Ś ::t Sj ::lcode srp ::comment Latin capital s with acute
+::s ś ::t sj ::lcode srp ::comment Latin small s with acute
+
+::s З́ ::t Z ::lcode srp2 ::comment Cyrillic capital zje
+::s з́ ::t z ::lcode srp2 ::comment Cyrillic small zje
+::s С́ ::t S ::lcode srp2 ::comment Cyrillic capital sje
+::s с́ ::t s ::lcode srp2 ::comment Cyrillic small sje
+::s Ź ::t Z ::lcode srp2 ::comment Latin capital z with acute
+::s ź ::t z ::lcode srp2 ::comment Latin small z with acute
+::s Ś ::t S ::lcode srp2 ::comment Latin capital s with acute
+::s ś ::t s ::lcode srp2 ::comment Latin small s with acute
+
+# Bulgarian
+::s Г ::t G ::t-alt _NONE_ ::lcode bul ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode bul ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt Kh ::lcode bul ::comment Cyrillic capital letter ha
+::s х ::t h ::t-alt kh ::lcode bul ::comment Cyrillic small letter ha
+::s Ц ::t C ::t-alt Ts ::lcode bul ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt ts ::lcode bul ::comment Cyrillic small letter tse
+::s Щ ::t Sht ::t-alt _NONE_ ::lcode bul ::comment Cyrillic capital letter shcha
+::s щ ::t sht ::t-alt _NONE_ ::lcode bul ::comment Cyrillic small letter shcha
+::s Е ::t E ::t-alt _NONE_ ::lcode bul ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode bul ::comment Cyrillic small ie
+::s Ж ::t Zh ::t-alt Z, J ::lcode bul ::comment Cyrillic capital zhe
+::s ж ::t zh ::t-alt z, j ::lcode bul ::comment Cyrillic small zhe
+::s Й ::t I ::t-alt Y, J ::lcode bul ::comment Cyrillic capital letter short i
+::s й ::t i ::t-alt y, j ::lcode bul ::comment Cyrillic short letter short i
+::s Ю ::t Yu ::t-alt U, Ju, Iu ::lcode bul ::comment Cyrillic capital letter yu
+::s ю ::t yu ::t-alt u, ju, iu ::lcode bul ::comment Cyrillic small letter yu
+::s Ъ ::t U ::t-alt A ::lcode bul ::comment Cyrillic capital letter hard sign
+::s ъ ::t u ::t-alt a ::lcode bul ::comment Cyrillic capital letter hard sign
+::s Ѣ ::t E ::t-alt Ie ::lcode bul ::comment archaic Cyrillic capital letter yat
+::s ѣ ::t e ::t-alt ie ::lcode bul ::comment archaic Cyrillic small letter yat
+::s Ѫ ::t U ::lcode bul ::comment archaic Cyrillic capital letter yus
+::s ѫ ::t u ::lcode bul ::comment archaic Cyrillic small letter yus
+::s ИЯ ::t IA ::lcode bul ::use-only-at-end-of-word
+::s ия ::t ia ::lcode bul ::use-only-at-end-of-word
+
+::s Ž ::t Zh ::lcode bul ::comment Latin capital z with caron
+::s ž ::t zh ::lcode bul ::comment Latin small z with caron
+::s Č ::t Ch ::lcode bul ::comment Latin capital c with caron
+::s č ::t ch ::lcode bul ::comment Latin small c with caron
+::s Š ::t Sh ::lcode bul ::comment Latin capital s with caron
+::s š ::t sh ::lcode bul ::comment Latin small s with caron
+::s Ŝ ::t Sht ::lcode bul ::comment Latin capital s with circumflex
+::s ŝ ::t sht ::lcode bul ::comment Latin small s with circumflex
+::s Û ::t Yu ::t-alt U, Ju, Iu ::lcode bul ::comment Latin capital u with circumflex
+::s û ::t yu ::t-alt u, ju, iu ::lcode bul ::comment Latin small u with circumflex
+::s  ::t Ya ::t-alt _NONE_ ::lcode bul ::comment Latin capital a with circumflex
+::s â ::t ya ::t-alt _NONE_ ::lcode bul ::comment Latin small a with circumflex
+::s Ŭ ::t U ::t-alt A ::lcode bul ::comment Latin capital u with breve (for hard sign)
+::s ŭ ::t u ::t-alt a ::lcode bul ::comment Latin small u with breve (for hard sign)
+::s Ǎ ::t U ::t-alt A ::lcode bul ::comment Latin capital a with caron (for hard sign)
+::s ǎ ::t u ::t-alt a ::lcode bul ::comment Latin small a with caron (for hard sign)
+
+# Macedonian
+::s Г ::t G ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic small ghe
+::s Х ::t H ::lcode mkd ::comment Cyrillic capital ha
+::s х ::t h ::lcode mkd ::comment Cyrillic small ha
+::s Ц ::t C ::t-alt Ts ::lcode mkd ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt ts ::lcode mkd ::comment Cyrillic small letter tse
+::s Џ ::t Dzh ::t-alt Dj, Dz ::lcode mkd ::comment Cyrillic capital letter dzhe
+::s џ ::t dzh ::t-alt dj, dz ::lcode mkd ::comment Cyrillic small letter dzhe
+::s Е ::t E ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode mkd ::comment Cyrillic small ie
+::s Ž ::t Zh ::lcode mkd ::comment Latin capital z with caron
+::s ž ::t zh ::lcode mkd ::comment Latin small z with caron
+::s Č ::t Ch ::lcode mkd ::comment Latin capital c with caron
+::s č ::t ch ::lcode mkd ::comment Latin small c with caron
+::s Š ::t Sh ::lcode mkd ::comment Latin capital s with caron
+::s š ::t sh ::lcode mkd ::comment Latin small s with caron
+::s Ǵ ::t Gj ::lcode mkd
+::s ǵ ::t gj ::lcode mkd
+::s Đ ::t Gj ::lcode mkd
+::s đ ::t gj ::lcode mkd
+::s Ẑ ::t Dz ::lcode mkd
+::s ẑ ::t dz ::lcode mkd
+::s J̌ ::t J ::lcode mkd
+::s ǰ ::t j ::lcode mkd
+::s L̂ ::t Lj ::lcode mkd
+::s l̂ ::t lj ::lcode mkd
+::s N̂ ::t Nj ::lcode mkd
+::s n̂ ::t nj ::lcode mkd
+::s Ḱ ::t Kj ::lcode mkd
+::s ḱ ::t kj ::lcode mkd
+::s Ć ::t Kj ::lcode mkd
+::s ć ::t kj ::lcode mkd
+::s D̂ ::t Dzh ::lcode mkd
+::s d̂ ::t dzh ::lcode mkd
+
+::s Г ::t G ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small ghe
+::s Х ::t H ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital ha
+::s х ::t h ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small ha
+::s Ц ::t C ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital letter tse
+::s ц ::t c ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small letter tse
+::s Ч ::t C ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital letter che
+::s ч ::t c ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small letter che
+::s Џ ::t D ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital letter dzhe
+::s џ ::t d ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small letter dzhe
+::s Е ::t E ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small ie
+::s Ш ::t S ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital sha
+::s ш ::t s ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small sha
+::s Ѓ ::t G ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital gje
+::s ѓ ::t g ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small gje
+::s Ж ::t Z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital zhe
+::s ж ::t z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small zhe
+::s Ѕ ::t Z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital dze
+::s ѕ ::t z ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small dze
+::s Ќ ::t K ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital kje
+::s ќ ::t k ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small kje
+::s Љ ::t L ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital lje
+::s љ ::t l ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small lje
+::s Њ ::t N ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic capital nje
+::s њ ::t n ::t-alt _NONE_ ::lcode mkd2 ::comment Cyrillic small nje
+::s Ž ::t Z ::lcode mkd2 ::comment Latin capital z with caron
+::s ž ::t z ::lcode mkd2 ::comment Latin small z with caron
+::s Č ::t C ::lcode mkd2 ::comment Latin capital c with caron
+::s č ::t c ::lcode mkd2 ::comment Latin small c with caron
+::s Š ::t S ::lcode mkd2 ::comment Latin capital s with caron
+::s š ::t s ::lcode mkd2 ::comment Latin small s with caron
+::s Ǵ ::t G ::lcode mkd2
+::s ǵ ::t g ::lcode mkd2
+::s Đ ::t G ::lcode mkd2
+::s đ ::t g ::lcode mkd2
+::s Ẑ ::t D ::lcode mkd2
+::s ẑ ::t d ::lcode mkd2
+::s J̌ ::t J ::lcode mkd2
+::s ǰ ::t j ::lcode mkd2
+::s L̂ ::t L ::lcode mkd2
+::s l̂ ::t l ::lcode mkd2
+::s N̂ ::t N ::lcode mkd2
+::s n̂ ::t n ::lcode mkd2
+::s Ḱ ::t K ::lcode mkd2
+::s ḱ ::t k ::lcode mkd2
+::s Ć ::t K ::lcode mkd2
+::s ć ::t k ::lcode mkd2
+::s D̂ ::t D ::lcode mkd2
+::s d̂ ::t d ::lcode mkd2
+
+# Kazakh
+::s Ә ::t A ::lcode kaz
+::s ә ::t a ::lcode kaz
+::s Г ::t G ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small ghe
+::s Ғ ::t G ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital ghe with stroke
+::s ғ ::t g ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small ghe with stroke
+::s Е ::t E ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital ie
+::s е ::t e ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small ie
+::s Ё ::t Yo ::t-alt _NONE_ ::lcode kaz
+::s ё ::t yo ::t-alt _NONE_ ::lcode kaz
+::s Х ::t H ::t-alt X ::lcode kaz ::comment Cyrillic capital ha
+::s х ::t h ::t-alt x ::lcode kaz ::comment Cyrillic small ha
+::s Һ ::t H ::lcode kaz ::comment Cyrillic capital shha
+::s һ ::t h ::lcode kaz ::comment Cyrillic small shha
+::s Қ ::t Q ::t-alt K ::lcode kaz 
+::s қ ::t q ::t-alt k ::lcode kaz
+::s Ц ::t Ts ::t-alt C ::lcode kaz ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode kaz ::comment Cyrillic small letter tse
+::s Щ ::t Sh ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital letter shcha
+::s щ ::t sh ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small letter shcha
+::s У ::t U ::t-alt Y ::lcode kaz
+::s у ::t u ::t-alt y ::lcode kaz
+::s уы ::t wy ::lcode kaz
+::s Ж ::t J ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic capital zhe
+::s ж ::t j ::t-alt _NONE_ ::lcode kaz ::comment Cyrillic small zhe
+::s Ю ::t Yw ::t-alt Yuw, Yiw ::lcode kaz ::comment Cyrillic capital letter yu
+::s ю ::t yw ::t-alt yuw, yiw ::lcode kaz ::comment Cyrillic small letter yu
+
+# Kyrgyz
+::s Г ::t G ::t-alt _NONE_ ::lcode kir ::comment Cyrillic capital ghe
+::s г ::t g ::t-alt _NONE_ ::lcode kir ::comment Cyrillic small ghe
+::s Е ::t E ::t-alt Ye ::lcode kir ::comment Cyrillic capital ie
+::s е ::t e ::t-alt ye ::lcode kir ::comment Cyrillic small ie
+::s Ё ::t Yo ::t-alt _NONE_ ::lcode kir
+::s ё ::t yo ::t-alt _NONE_ ::lcode kir
+::s Х ::t Kh ::t-alt X, H ::lcode kir ::comment Cyrillic capital ha
+::s х ::t kh ::t-alt x, h ::lcode kir ::comment Cyrillic small ha
+::s Ж ::t Zh ::t-alt J ::lcode kir ::comment Cyrillic capital zhe
+::s ж ::t zh ::t-alt j ::lcode kir ::comment Cyrillic small zhe
+::s Й ::t Y ::t-alt I ::lcode kir ::comment Cyrillic capital letter short i 
+::s й ::t y ::t-alt i ::lcode kir ::comment Cyrillic small letter short i
+::s Ц ::t Ts ::t-alt C ::lcode kir ::comment Cyrillic capital letter tse
+::s ц ::t ts ::t-alt c ::lcode kir ::comment Cyrillic small letter tse
+::s Ң ::t Ng ::lcode kir
+::s ң  ::t ng ::lcode kir
+::s Ө ::t O ::t-alt Oe ::lcode kir
+::s ө ::t o ::t-alt oe ::lcode kir
+::s Ү ::t U ::t-alt Y, Ue ::lcode kir
+::s ү ::t u ::t-alt y, ue ::lcode kir
+::s Ы ::t I ::t-alt Y ::lcode kir
+::s ы ::t i ::t-alt y ::lcode kir
+::s йы ::t yi ::lcode kir
+::s ый ::t iy ::lcode kir
+
+# Ossetian
+::s ийы ::t iy ::lcode oss
+
+# Gothic
+::s 𐌴 ::t e ::comment Gothic letter aihvus
+::s 𐌹 ::t i ::comment Gothic letter eis
+::s 𐍇 ::t x ::comment Gothic letter iggws
+
+# Runic
+::s ᛫ ::t " " ::comment Runic single punctuation, used as word separator
+::s ᛬ ::t . ::comment Runic multiple punctuation, used as sentence separator
+
+# Ogham
+::s ᚁ ::t b ::comment Ogham letter Beith	
+::s ᚂ ::t l ::comment Ogham letter Luis	
+::s ᚃ ::t f ::comment Ogham letter Fearn	
+::s ᚄ ::t s ::comment Ogham letter Sail	
+::s ᚅ ::t n ::comment Ogham letter Nion	
+::s ᚋ ::t m ::comment Ogham letter Muin
+::s ᚌ ::t g ::comment Ogham letter Gort
+::s ᚍ ::t v ::t-alt ng ::comment Ogham letter nGéadal
+::s ᚎ ::t z ::comment Ogham letter Straif
+::s ᚏ ::t r ::comment Ogham letter Ruis
+::s ᚆ ::t h ::t-alt j ::comment Ogham letter Uath	
+::s ᚇ ::t d	::comment Ogham letter Dair	
+::s ᚈ ::t t ::comment Ogham letter Tinne	
+::s ᚉ ::t k ::comment Ogham letter Coll	
+::s ᚊ ::t q ::t-alt kw ::comment Ogham letter Ceirt
+::s ᚐ ::t a ::comment Ogham letter Ailm
+::s ᚑ ::t o	::comment Ogham letter Onn
+::s ᚒ ::t u ::comment Ogham letter Úr
+::s ᚓ ::t e ::comment Ogham letter Eadhadh
+::s ᚔ ::t i ::comment Ogham letter Iodhadh
+::s ᚚ ::t p ::comment Ogham letter Peith
+# Additional Ogham letters (outside standard alphabet)
+::s ᚕ ::t eo ::t-alt ea ::comment Ogham additional letter Éabhadh
+::s ᚖ ::t oi ::t-alt oe ::comment Ogham additional letter Ór
+::s ᚗ ::t ui ::t-alt ua ::comment Ogham additional letter Uilleann
+::s ᚘ ::t p ::t-alt io ::comment Ogham additional letter Ifín 
+::s ᚙ ::t ch ::t-alt x, ai ::comment Ogham additional letter Eamhancholl
+::s   ::t " " ::comment Ogham space mark
+::s ᚛ ::t "" ::comment Ogham feather mark
+::s ᚜ ::t "" ::comment Ogham feather mark
+
+# Georgian
+::s ა ::t a ::comment Georgian letter an
+::s ე ::t e ::comment Georgian letter en
+::s ი ::t i ::comment Georgian letter in
+::s ო ::t o ::comment Georgian letter on
+::s უ ::t u ::comment Georgian letter un
+::s ჱ ::t ey ::comment archaic Georgian letter he
+::s ჲ ::t i ::comment archaic Georgian letter hie
+::s ჳ :::t w ::comment archaic Georgian letter we
+::s ჴ ::t q ::comment archaic Georgian letter har
+::s ჵ ::t o ::comment archaic Georgian letter hoe
+::s ჶ ::t f ::comment Georgian letter fi (Greek phi)
+::s ჷ ::t e ::comment Georgian letter yn (schwa)
+::s ჸ ::t a ::comment Georgian letter elifi
+::s ჹ ::t g ::comment Georgian letter gan
+::s ჺ ::t ' ::comment Georgian letter ain
+::s ჼ ::t n ::comment Georgian letter nar
+::s ჽ ::t e ::comment Georgian letter aen
+::s ჾ ::t ::comment Georgian letter hard sign
+::s ჿ ::t w ::comment Georgian letter labial sign
+
+::s Ⴚ ::t TS ::comment GEORGIAN CAPITAL LETTER CAN
+::s ც ::t ts ::comment GEORGIAN LETTER CAN
+::s Ც ::t TS ::comment GEORGIAN MTAVRULI CAPITAL LETTER CAN
+::s ⴚ ::t ts ::comment GEORGIAN SMALL LETTER CAN
+::s Ⴜ ::t TS ::comment GEORGIAN CAPITAL LETTER CIL
+::s წ ::t ts ::comment GEORGIAN LETTER CIL
+::s Წ ::t TS ::comment GEORGIAN MTAVRULI CAPITAL LETTER CIL
+::s ⴜ ::t ts ::comment GEORGIAN SMALL LETTER CIL
+::s Ⴛ ::t DZ ::comment GEORGIAN CAPITAL LETTER JIL
+::s ძ ::t dz ::comment GEORGIAN LETTER JIL
+::s Ძ ::t DZ ::comment GEORGIAN MTAVRULI CAPITAL LETTER JIL
+::s ⴛ ::t dz ::comment GEORGIAN SMALL LETTER JIL
+::s Ⴟ ::t J ::comment GEORGIAN CAPITAL LETTER JHAN
+::s ჯ ::t j ::comment GEORGIAN LETTER JHAN
+::s Ჯ ::t J ::comment GEORGIAN MTAVRULI CAPITAL LETTER JHAN
+::s ⴟ ::t j ::comment GEORGIAN SMALL LETTER JHAN
+
+
+::s Ⴀ ::t A ::comment Georgian capital letter an
+::s Ⴄ ::t E ::comment Georgian capital letter en
+::s Ⴈ ::t I ::comment Georgian capital letter in
+::s Ⴍ ::t O ::comment Georgian capital letter on
+::s Ⴓ ::t U ::comment Georgian capital letter un
+::s Ⴡ ::t EY ::comment archaic Georgian capital letter he
+::s Ⴢ ::t I ::comment archaic Georgian capital letter hie
+::s Ⴣ :::t W ::comment archaic Georgian capitel letter we
+::s Ⴤ ::t Q ::comment archaic Georgian capital letter har
+::s Ⴥ ::t O ::comment archaic Georgian capital letter hoe
+::s Ⴧ ::t E ::comment archaic Georgian capital letter yn (schwa)
+::s Ⴭ ::t E ::comment archaic Georgian capital letter aen
+
+::s Ა ::t A ::comment Georgian Mtavruli capital letter an
+::s Ე ::t E ::comment Georgian Mtavruli capital letter en
+::s Ი ::t I ::comment Georgian Mtavruli capital letter in
+::s Ო ::t O ::comment Georgian Mtavruli capital letter on
+::s Უ ::t U ::comment Georgian Mtavruli capital letter un
+::s Ჱ ::t EY ::comment archaic Georgian Mtavruli capital letter he
+::s Ჲ ::t I ::comment archaic Georgian Mtavruli capital letter hie
+::s Ჳ :::t W ::comment archaic Georgian Mtavruli capital letter we
+::s Ჴ ::t Q ::comment archaic Georgian Mtavruli capital letter har
+::s Ჵ ::t O ::comment archaic Georgian Mtavruli capital letter hoe
+::s Ჶ ::t F ::comment Georgian Mtavruli capital letter fi (Greek phi)
+::s Ჷ ::t E ::comment Georgian Mtavruli capital letter yn (schwa)
+::s Ჸ ::t A ::comment Georgian Mtavruli capital letter elifi
+::s Ჹ ::t G ::comment Georgian Mtavruli capital letter gan
+::s Ჺ ::t ' ::comment Georgian Mtavruli capital letter ain
+::s Ჽ ::t E ::comment Georgian Mtavruli capital letter aen
+::s Ჾ ::t ::comment Georgian Mtavruli capital letter hard sign
+::s Ჿ ::t W ::comment Georgian Mtavruli capital letter labial sign
+
+::s ⴀ ::t a ::comment Georgian small letter an
+::s ⴄ ::t e ::comment Georgian small letter en
+::s ⴈ ::t i ::comment Georgian small letter in
+::s ⴍ ::t o ::comment Georgian small letter on
+::s ⴓ ::t u ::comment Georgian small letter un
+::s ⴡ ::t ey ::comment archaic Georgian small letter he
+::s ⴢ ::t i ::comment archaic Georgian small letter hie
+::s ⴣ :::t w ::comment archaic Georgian small letter we
+::s ⴤ ::t q ::comment archaic Georgian small letter har
+::s ⴥ ::t o ::comment archaic Georgian small letter hoe
+::s ⴧ ::t e ::comment Georgian small letter yn (schwa)
+::s ⴭ ::t e ::comment Georgian small letter aen
+
+# Armenian
+::s Ա ::t A ::comment Armenian capital letter ayb
+::s ա ::t a ::comment Armenian small letter ayb
+::s ՠ ::t a ::comment ARMENIAN SMALL LETTER TURNED AYB (CHECK)
+::s Ե ::t E ::comment Armenian capital letter ech ::dont-use-at-start-of-word
+::s ե ::t e ::comment Armenian small letter ech ::dont-use-at-start-of-word
+::s Ե ::t Ye ::comment Armenian capital letter ech ::use-only-at-start-of-word
+::s ե ::t ye ::comment Armenian small letter ech ::use-only-at-start-of-word
+::s Է ::t E ::comment Armenian capital letter eh
+::s է ::t e ::comment Armenian small letter eh
+::s Ը ::t E ::comment Armenian capital letter et
+::s ը ::t e ::comment Armenian small letter et
+::s Ի ::t I ::comment Armenian capital letter ini
+::s ի ::t i ::comment Armenian small letter ini
+::s Յ ::t Y ::comment Armenian capital letter yi
+::s յ ::t y ::comment Armenian small letter yi
+::s ֈ ::t y ::comment ARMENIAN SMALL LETTER YI WITH STROKE (CHECK)
+::s Ո ::t Vo ::comment Armenian capital letter vo ::use-only-at-start-of-word
+::s ո ::t vo ::comment Armenian small letter vo ::use-only-at-start-of-word
+::s Ո ::t O ::comment Armenian capital letter vo ::dont-use-at-start-of-word
+::s ո ::t o ::comment Armenian small letter vo ::dont-use-at-start-of-word
+::s Ւ ::t W ::comment Armenian capital letter yiwn
+::s ւ ::t w ::comment Armenian small letter yiwn
+::s Օ ::t O ::comment Armenian capital letter oh
+::s օ ::t o ::comment Armenian small letter oh
+::s Խ ::t Kh ::comment Armenian capital letter xeh
+::s խ ::t kh ::comment Armenian small letter xeh
+
+::s Ժ ::t Zh ::comment Armenian capital letter zhe
+::s Ղ ::t Gh ::comment Armenian capital letter ghad
+::s Ճ ::t Tch ::comment Armenian capital letter cheh
+::s ճ ::t tch ::comment Armenian small letter cheh
+::s Շ ::t Sh ::comment Armenian capital letter sha
+::s Չ ::t Ch ::comment Armenian capital letter cha
+::s Ջ ::t J ::comment Armenian capital letter jheh
+::s ջ ::t j ::comment Armenian small letter jheh
+::s Վ ::t V ::comment Armenian capital letter vew
+::s վ ::t v ::comment Armenian small letter vew
+::s Ձ ::t Dz ::comment Armenian capital letter ja
+::s ձ ::t dz ::comment Armenian small letter ja
+::s Ծ ::t Ts ::comment Armenian capital letter ca
+::s ծ ::t ts ::comment Armenian small letter ca
+::s Ք ::t K ::t-alt Q ::comment Armenian capital letter keh - sometimes romanized as K' or Q
+::s ք ::t k ::t-alt q ::comment Armenian small letter keh - sometimes romanized as k' or q
+
+::s են ::t en ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s եմ ::t em ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s ենք ::t enk ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s ես ::t es ::use-only-for-whole-word ::comment exception (auxiliary verb)
+::s եք ::t ek ::use-only-for-whole-word ::comment exception (auxiliary verb)
+
+::s և ::t ev ::comment Armenian small ligature ech yiwn
+::s ՈՒ ::t U ::comment Armenian capital vo+yiwn
+::s Ու ::t U ::comment Armenian capital/small vo+yiwn
+::s ու ::t u ::comment Armenian small vo+wywn
+
+::s իւ ::t yu 
+
+## Japanese
+# Katakana
+::s シ ::t shi
+::s チ ::t chi
+::s フ ::t fu
+::s ジ ::t ji
+::s ヂ ::t ji
+::s ヅ ::t zu
+::s シャ ::t sha
+::s シュ ::t shu
+::s ショ ::t sho
+::s チャ ::t cha
+::s チェ ::t che
+::s チュ ::t chu
+::s チョ ::t cho
+::s ジャ ::t ja
+::s ジュ ::t ju
+::s ジョ ::t jo
+::s ジェ ::t je
+::s ヂャ ::t ja
+::s ヂュ ::t ju
+::s ヂョ ::t jo
+::s フェ ::t fe
+::s ヴェ ::t ve
+::s フィ ::t fi
+::s ウィ ::t wi
+::s ヴィ ::t vi
+::s ティ ::t ti
+::s ディ ::t di
+::s ッ ::t (__SOKUON__) ::comment katakana double following consonant
+::s ー ::t (__CHOONPU__) ::comment katakana prolonged sound mark
+::s 𛅤 ::t i ::comment KATAKANA LETTER SMALL WI
+::s 𛅥 ::t e ::comment KATAKANA LETTER SMALL WE
+::s 𛅦 ::t o ::comment KATAKANA LETTER SMALL WO
+# Hiragana
+::s し ::t shi
+::s ち ::t chi
+::s つ ::t tsu
+::s ふ ::t fu
+::s を ::t o
+::s じ ::t ji
+::s ぢ ::t ji
+::s づ ::t zu
+::s しゃ ::t sha
+::s しゅ ::t shu
+::s しょ ::t sho
+::s ちゃ ::t cha
+::s ちゅ ::t chu
+::s ちょ ::t cho
+::s じゃ ::t ja
+::s じゅ ::t ju
+::s じょ ::t jo
+::s ぢゃ ::t ja
+::s ぢゅ ::t ju
+::s ぢょ ::t jo
+::s 𛅐 ::t i ::comment HIRAGANA LETTER SMALL WI
+::s 𛅑 ::t e ::comment HIRAGANA LETTER SMALL WE
+::s 𛅒 ::t o ::comment HIRAGANA LETTER SMALL WO
+::s っ ::t (__SOKUON__) ::comment hiragana double following consonant
+::s 々 ::t ² ::comment ideographic iteration mark ::annotation repetition-sign
+
+::s フ ::t fu ::t-alt f
+::s キ ::t ki ::t-alt k
+::s ク ::t ku ::t-alt k
+::s ラ ::t ra ::t-alt la
+::s リ ::t ri ::t-alt li
+::s ル ::t ru ::t-alt lu, l, r
+::s レ ::t re ::t-alt le
+::s ロ ::t ro ::t-alt lo
+::s ム ::t mu ::t-alt m ::example キム = Kim
+::s シ ::t shi ::t-alt si ::example メキシコ = meksiko (Mexico)
+::s ス ::t su ::t-alt s
+::s ト ::t to ::t-alt t
+::s ツ ::t tsu ::t-alt tu, ts ::example シュルツ = Schultz
+
+::s ㋿ ::t Reiwa ::comment SQUARE ERA NAME REIWA 
+
+# Chinese
+::s 邦 ::t bang ::t-alt bon, bum, bun, pon
+::s 鲍 ::t bao ::t-alt bow
+::s 堡 ::t bao ::t-alt berg, burg, bourg, burgh
+::s 贝 ::t bei ::t-alt ber
+::s 本 ::t ben ::t-alt bern, bon, bourn, burn
+::s 彼得 ::t bide ::t-alt peter, pet
+::s 伯 ::t bo ::t-alt ber
+::s 波 ::t bo ::t-alt po
+::s 布 ::t bu ::t-alt b
+::s 策 ::t ce ::t-alt tze, tzer
+::s 曾 ::t ceng ::t-alt tzen, zen
+::s 彻 ::t che ::t-alt tche
+::s 茨 ::t ci ::t-alt ts, tz, z
+::s 兹 ::t ci ::t-alt ds, dz, tz, z, zi
+::s 蒂 ::t di ::t-alt ti, tti
+::s 丁 ::t ding ::t-alt din, tin
+::s 顿 ::t dun ::t-alt ton
+::s 多 ::t duo ::t-alt do, dor, to
+::s 尔 ::t er ::t-alt l, le, ll, r
+::s 弗 ::t fu ::t-alt f, fer, pher, v, ver, vir
+::s 夫 ::t fu ::t-alt f, v, v
+::s 福 ::t fu ::t-alt faw, for, ford
+::s 哥 ::t ge ::t-alt go, co
+::s 戈 ::t ge ::t-alt go
+::s 各 ::t ge ::t-alt go, co
+::s 赫 ::t he ::t-alt ch, che, cher, ge
+::s 华 ::t hua ::t-alt ver, wa, war, wer ::example Washington
+::s 怀 ::t huai ::t-alt whi, wi, wy
+::s 惠 ::t hui ::t-alt wha, whea
+::s 基 ::t ji ::t-alt ki, chi
+::s 吉 ::t ji ::t-alt gi, gui
+::s 加 ::t jia ::t-alt ca, ga, ka ::example Canada
+::s 杰 ::t jie ::t-alt ger
+::s 金 ::t jin ::t-alt kin, gin
+::s 斤 ::t jin ::t-alt zin
+::s 康 ::t kang ::t-alt con, corn
+::s 考 ::t kao ::t-alt cow, cour
+::s 克 ::t ke ::t-alt k, che, cher
+::s 科 ::t ke ::t-alt ko
+::s 拉 ::t la ::t-alt ra ::example Tirana
+::s 朗 ::t lang ::t-alt lon, ron
+::s 赖 ::t lai ::t-alt ri
+::s 劳 ::t lao ::t-alt low
+::s 勒 ::t lei ::t-alt ler
+::s 伦 ::t lun ::t-alt lon, ran, ron
+::s 里 ::t li ::t-alt ri
+::s 利 ::t li ::t-alt ri ::example Ferrari
+::s 隆 ::t long ::t-alt lon, lum, lund
+::s 罗 ::t luo ::t-alt l, lo, lu, ro, row, ru
+::s 洛 ::t luo ::t-alt lo, low, ro
+::s 默 ::t mo ::t-alt mer
+::s 纳 ::t na ::t-alt ne, ner
+::s 珀 ::t po ::t-alt per
+::s 奇 ::t qi ::t-alt chi, dge, ge, tch
+::s 齐 ::t qi ::t-alt tsi, zi
+::s 乔 ::t qiao ::t-alt jo
+::s 青 ::t qing ::t-alt tsing
+::s 琼 ::t qiong ::t-alt jon, jum, jun
+::s 瑟 ::t se ::t-alt the
+::s 什 ::t shen ::t-alt sh
+::s 圣 ::t sheng ::t-alt san, sao, saint
+::s 斯 ::t si ::t-alt s, rth, th ::example Alaska
+::s 索 ::t suo ::t-alt tho
+::s 特 ::t te ::t-alt t
+::s 翁 ::t weng ::t-alt on
+::s 沃 ::t wo ::t-alt ver, vo, war, wer
+::s 乌 ::t wu ::t-alt ou, u
+::s 希 ::t xi ::t-alt chi, hi, shi
+::s 西 ::t xi ::t-alt s, si
+::s 锡 ::t xi ::t-alt ci, si, thi, zi
+::s 夏 ::t xia ::t-alt ha, cha, cia, sha, tia
+::s 香 ::t xiang ::t-alt chan, cham
+::s 歇 ::t xie ::t-alt she
+::s 谢 ::t xie ::t-alt che, she
+::s 辛 ::t xin ::t-alt cin, sen, sin, sing, sun, zen
+::s 欣 ::t xin ::t-alt hin, shin
+::s 休 ::t xiu ::t-alt hu, hue
+::s 修 ::t xiu ::t-alt ciu, siu, thew, tiu
+::s 许 ::t xu ::t-alt hue, schue
+::s 逊 ::t xun ::t-alt son
+::s 耶 ::t ye ::t-alt yer, ier
+::s 泽 ::t ze ::t-alt ser
+::s 扎 ::t zha ::t-alt za
+::s 詹 ::t zhan ::t-alt ja, jam, jan, jen, jon
+::s 治 ::t zhi ::t-alt ge ::example George
+
+## Numbers
+# Chinese and Japanese numbers
+::s 零 ::num 0
+::s 〇 ::num 0
+::s 一 ::num 1
+::s 二 ::num 2
+::s 三 ::num 3
+::s 四 ::num 4
+::s 五 ::num 5
+::s 六 ::num 6
+::s 七 ::num 7
+::s 八 ::num 8
+::s 九 ::num 9
+::s 十 ::num 10
+::s 百 ::num 100
+::s 千 ::num 1000
+::s 万 ::num 10000
+::s 萬 ::num 10000
+::s 亿 ::num 100000000
+::s 億 ::num 100000000
+::s 兆 ::num 1000000000000
+::s 京 ::num 10000000000000000
+
+# numbers in non-number words (to be exptended)
+::s 一贯 ::t yiguan ::comment consistent
+
+::s 红十字会 ::t hongshizihui ::comment Red Cross
+
+::s 百度 ::t baidu ::comment Baidu (company)
+::s 百分 ::t baifen ::comment percent
+::s 百合 ::t baihe ::comment lily
+::s 百货 ::t baihuo ::comment general merchandise
+::s 百科 ::t baike ::comment encyclopedia
+::s 百老汇 ::t bailaohui
+::s 百灵 ::t bailing
+::s 百慕大 ::t baimuda
+::s 百日咳 ::t bairike
+::s 百色市 ::t baiseshi
+::s 百事可乐 ::t baishikele ::comment Pepsi Cola
+::s 百無 ::t baiwu
+::s 百香 ::t baixiang
+::s 百姓 ::t baixing
+::s 百叶 ::t baiye
+::s 百色 ::t bose
+::s 杨百翰 ::t yangbaihan ::comment Brigham Young
+
+::s 北京 ::t beijing
+::s 京都 ::t jingdou
+::s 东京 ::t dongjing
+::s 京胡 ::t jinghu
+::s 南京 ::t nangjing
+::s 普京 ::t pujing ::comment Putin
+::s 東京 ::t dongjing ::comment Tokyo
+::s 京兆 ::t jingzhao
+
+::s ㎢ ::t km²
+::s ㎥ ::t m³
+::s ㎝ ::t cm
+
+## Indian
+# see mostly under UnicodeDataOverwrite.txt
+
+# Malayalam
+::s ൗ ::t au ::comment MALAYALAM AU LENGTH MARK
+
+# Tamil
+::s ட ::t d ::comment most commonly d, but t when word-initial or in a doubled consonant
+::s ஃப ::t f ::comment h+p=f
+::s ஃஜ ::t z ::comment h+j=z
+
+# Myanmar/Burmese
+# ::s ့ ::t ::comment dot below, denotes creaky tone 
+# ::s း ::t ::comment visarga, denotes high tone
+::s ၌ ::t -nai ::comment locative
+::s ၍ ::t -jwe ::comment completed
+::s ၎ ::t legau ::comment aforementioned
+::s ၏ ::t -i ::comment genetive
+
+# Lao
+::s ັ ::t a ::comment vowel sign mai kan
+::s ົ ::t o ::comment vowel sign mai kon
+::s ູ ::t uu ::comment vowel sign uu
+::s ຽ ::t y ::comment semivowel sign nyo
+::s ຼ ::t l ::comment semivowel sign lo
+::s ລ ::t l ::comment lo loot
+::s ຣ ::t l ::comment lo ling
+::s ໝ ::t m ::comment ho mo
+::s ໜ ::n ::comment ho no
+::s ຢ ::t y ::comment yo
+::s ໍ ::t oo ::comment niggahita (possibly also nasal -m in final position)
+::s ໆ ::t ² ::comment Lao ko la ::annotation repetition-sign
+::s ຯ ::t ...  ::comment Lao ellipsis
+
+# Thai
+::s ออ ::t o
+::s อั ::t a
+::s อิ ::t i
+::s ๆ ::t ² ::comment Thai character maiyamok ::annotation repetition-sign
+
+# Khmer
+::s ័ ::t "" ::comment Khmer samyok sannya: indicates deviation from the general rules of pronunciation
+::s ៏ ::t "" ::comment Khmer sign ahsda: denotes stressed intonation in some single-consonant words
+::s ៍ ::t "" ::comment Khmer sign toandakhiat: indicates that the base character is not pronounced
+::s ៌ ::t "" ::comment Khmer sign robat: a diacritic historically corresponding to the repha form of ra in Devanagari
+::s ប៉ ::t pa ::comment Khmer ba + musĕkâtônd -> pa
+::s ៗ ::t ² ::comment Khmer sign lek too ::annotation repetition-sign
+
+## Semitic languages
+# Arabic
+::s و ::t w  ::comment Arabic letter waw  ::t-alt o, u  ::lcode ara
+::s ء ::t ' ::comment hamza 
+::s ٔ ::t ' ::comment hamza above
+::s ٕ ::t ' ::comment hamza below
+::s ع ::t ' ::comment ain
+::s آ ::t a ::comment alef madda
+::s ٓا ::t a ::comment Arabic maddah above plus alef (presumably an ill-formed version of آ; found 1 instance in Urdu text)
+::s إ ::t i ::comment alef with hamza below
+::s ٱ ::t a ::comment alef wasla ::comment typically indicates liaison with preceding word 
+::s ة ::t a ::comment teh marbuta
+::s ۃ ::t a ::comment teh marbuta goal ::comment Used in Punjabi, Sindhi. Different from plain 'teh marbuta'?
+::s ي ::t y ::comment Arabic yeh
+::s ى ::t a ::comment alef maksura
+::s ﻯ ::t a ::comment alef maksura isolated form
+::s ﻰ ::t a ::comment alef maksura final form
+::s ﯨ ::t a ::comment Uighur Kazach Kirghiz alef maksura initial form
+::s ﯩ ::t a ::comment Uighur Kazach Kirghiz alef maksura medial form
+::s ٰ ::t a  ::comment Arabic letter superscript alef
+::s ـ ::t   ::comment tatweel (filler)
+::s َ ::t a ::comment fatha ("-a")
+::s ُ ::t u ::comment damma ("-u")
+::s ِ ::t i ::comment kasra ("-i")
+::s ْ ::t   ::comment sukun (no vowel)
+::s ۡ ::t  ::comment small high dotless head of khah; like sukun (no vowel); used in Kashmiri, Assamese
+::s ً ::t   ::comment fathatan ("-an")
+::s اً ::t an ::comment alef + fathatan
+::s ٌ ::t   ::comment dammatan ("-un")
+::s ٍ ::t   ::comment kasratan ("-in")
+::s ّ ::t   ::comment shadda (consonant doubler)
+::s ڃ ::t ny ::comment Arabic letter nyeh U+0683 (used in Sindhi (snd))
+::s ڄ ::t dy ::comment Arabic letter dyeh U+0684 (used in Sindhi (snd))
+::s ۾ ::t men ::comment Sindhi postposition men
+::s ؑ ::t alayhe wasallam ::comment "upon him be peace"
+::s ﷴ ::t mohammad ::comment "Mohammad"
+::s ﷸ ::t wasallam ::comment "and peace"
+::s ﷺ ::t sallallahou alayhe wasallam ::comment "prayer of God be upon him and his family and peace"
+
+::s ࣓ ::t waw ::comment ARABIC SMALL LOW WAW
+::s ࣔ ::t al-rub ::comment ARABIC SMALL HIGH WORD AR-RUB
+::s ࣕ ::t s ::comment ARABIC SMALL HIGH SAD
+::s ࣖ ::t ' ::comment ARABIC SMALL HIGH AIN
+::s ࣗ ::t q ::comment ARABIC SMALL HIGH QAF
+::s ࣘ ::t n ::comment ARABIC SMALL HIGH NOON WITH KASRA
+::s ࣙ ::t n ::comment ARABIC SMALL LOW NOON WITH KASRA
+::s ࣚ ::t al-thalatha ::comment ARABIC SMALL HIGH WORD ATH-THALATHA
+::s ࣛ ::t al-sajda ::comment ARABIC SMALL HIGH WORD AS-SAJDA
+::s ࣜ ::t al-nisf ::comment ARABIC SMALL HIGH WORD AN-NISF
+::s ࣝ ::t sakta ::comment ARABIC SMALL HIGH WORD SAKTA
+::s ࣞ ::t qif ::comment ARABIC SMALL HIGH WORD QIF
+::s ࣟ ::t waqfa ::comment ARABIC SMALL HIGH WORD WAQFA
+::s ࣠ ::t ::comment ARABIC SMALL HIGH FOOTNOTE MARKER (CHECK)
+::s ࣡ ::t ::comment ARABIC SMALL HIGH SIGN SAFHA (CHECK)
+::s ࣢ ::t ::comment ARABIC DISPUTED END OF AYAH (CHECK)
+
+# Farsi
+::s ی ::t i ::t-alt y ::comment Contributed by Nima
+::s ای ::t i ::t-alt ai ::use-only-at-start-of-word ::comment Contributed by Nima
+::s هٔ ::t eye ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s و ::t v ::t-alt o, u ::lcode fas ::comment Arabic letter waw
+::s ض ::t z ::t-alt d ::lcode fas ::comment Contributed by Marjan
+::s ث ::t s ::t-alt th ::lcode fas ::comment Contributed by Marjan
+::s ذ ::t z ::t-alt th ::lcode fas ::comment Contributed by Nima
+::s ع ::t a ::t-alt ' ::lcode fas ::comment Contributed by Nima
+::s عا ::t a ::lcode fas ::comment Contributed by Nima
+::s عی ::t i ::t-alt iy ::lcode fas ::comment Contributed by Nima
+::s عو ::t u ::t-alt o, av ::lcode fas ::comment Contributed by Nima
+::s چ ::t ch ::t-alt tch, tsh ::lcode fas ::comment Contributed by Nima
+::s ه ::t e ::t-alt h ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s ‌ ::t "" ::t-alt " " ::lcode fas ::comment source is character "zero-width non-joiner" (U+200C); Contributed by Nima
+::s غ ::t gh ::t-alt g ::lcode fas
+::s آئی ::t ai ::t-alt ae ::lcode fas
+::s ائی ::t ai ::t-alt ae ::lcode fas
+::s آئو ::t au ::t-alt ao ::lcode fas
+::s ائو ::t au ::t-alt ao ::lcode fas
+
+# Kashmiri (so far: educated guesses)
+::s ٖ ::t a ::comment Arabic subscript alef U+0656
+::s ٗ ::t u ::comment Arabic inverted damma U+0657
+::s ۚ ::t j ::comment Arabic small high jeem U+06DA
+::s ۪ ::t   ::comment Arabic emtpy centre low stop U+06EA
+::s ۬ ::t   ::comment Arabic rounded high stop with filled center U+06EC 
+
+# Pashto
+::s ٙ ::t e ::comment Arabic zwarakay
+::s ځ ::t z ::t-alt dz ::comment Pashto letter zim; Arabic letter "hah with hamza above"
+::s څ ::t ts ::t-alt c ::comment Pashto letter tsim; Arabic letter "h with three dots above"
+::s ګ ::t g ::comment Pashto letter gaf; Arabic letter "kaf with ring"
+::s ڼ ::t n ::comment Arabic letter "noon with ring"
+::s ږ ::t g ::t-alt z, zh, j ::comment pronunciation varies regionally
+::s ښ ::t kh ::t-alt sh ::comment pronunciation varies regionally
+::s ه ::t h ::t-alt a ::lcode pus
+::s ۀ ::t e ::lcode pus ::comment Arabic letter "heh with yeh above"
+::s و ::t w ::t-alt o, u ::lcode pus
+::s ی ::t ay ::t-alt y ::lcode pus
+::s وی ::t wy ::t-alt oy, uy ::lcode pus
+::s ای ::t ay ::lcode pus
+::s ۍ ::t ay ::lcode pus
+::s ئ ::t ay ::t-alt y ::lcode pus
+::s ژ ::t zh ::t-alt z ::lcode pus ::comment [ʒ]
+::s ض ::t z ::t-alt d ::lcode pus
+::s ث ::t s ::lcode pus ::t-alt th ::comment Arabic letter theh (unvoiced th/θ)
+::s ذ ::t z ::lcode pus ::t-alt th ::comment Arabic letter thal (voiced th/ð)
+
+# Hebrew
+::s ב ::t v ::comment Hebrew letter bet  ::t-alt b
+::s כ ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s ך ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s פ ::t f ::comment Hebrew letter pe  ::t-alt p
+::s ש ::t sh  ::comment Hebrew letter shin  ::t-alt s
+::s ו ::t v  ::comment Hebrew letter vav  ::t-alt o, u
+::s ח ::t ch ::comment Hebrew letter het  ::t-alt h ::use-alt-in-pointed
+::s ק ::t q ::t-alt k ::use-alt-in-pointed
+::s וֹ ::t o
+::s וּ ::t u
+::s קְוָ ::t qva ::t-alt kva  ::use-alt-in-pointed
+::s י ::t y
+::s יּ ::t y
+::s יָּ ::t ya
+::s ײ ::t yy ::comment Hebrew ligature Yiddish double Yod (CHECK)
+::s ׯ ::t yyy ::comment HEBREW YOD TRIANGLE (CHECK)
+::s ע ::t '
+::s ִי ::t i ::t-alt iy ::use-alt-in-pointed
+::s ֵי ::t e
+::s ִיּ ::t iy
+::s ִיָּ ::t iya
+::s ױ ::t oy
+::s א ::t a ::t-alt '
+::s אָ ::t a
+::s ֹא ::t o
+::s אַ ::t 'a
+::s אֲ ::t 'a
+::s אֶ ::t e
+::s אֱ ::t e
+::s פ ::t f
+::s פּ ::t p
+::s פַּ ::t pa
+::s פְּ ::t pe ::t-alt p ::use-alt-in-pointed 
+::s שׁ ::t sh
+::s שָׁ ::t sha
+::s שָּׁ ::t sha ::comment ?
+::s שְׁ ::t she ::t-alt sh ::use-alt-in-pointed
+::s שֶׁ ::t she
+::s שִׁ ::t shi
+::s שֻׁ ::t shu
+::s שׂ ::t s
+::s שָׂ ::t sa
+::s שְׂ ::t s ::t-alt se ::use-alt-in-pointed
+::s כּ ::t k
+::s כֶּ ::t ke
+::s כֹּ ::t ko
+::s בּ ::t b
+::s בַּ ::t ba
+::s בָּ ::t ba
+::s בְּ ::t be ::t-alt b ::use-alt-in-pointed
+::s בֶּ ::t be
+::s תּ ::t t
+::s תַּ ::t ta
+::s תֵּ ::t te
+::s תִּ ::t ti
+::s דָּ ::t da
+::s דְּ ::t de ::t-alt d ::use-alt-in-pointed
+::s גּ ::t g
+::s לֵּ ::t le
+::s ד׳ ::t dh
+::s ג׳ ::t j
+::s ת׳ ::t th
+::s ז׳ ::t zh
+::s חַ ::t ach ::comment furtive patah ::use-only-at-end-of-word
+::s עַ ::t a' ::comment furtive patah ::use-only-at-end-of-word
+::s הַּ ::t ah ::comment furtive patah ::use-only-at-end-of-word
+::s ַ ::t a ::comment Hebrew point patah
+::s ֲ ::t a ::comment Hebrew point hataf patah (hataf = reduced)
+::s ֳ ::t o ::comment Hebrew point hataf qamats
+::s ָ ::t a ::comment Hebrew point qamats ::t-alt o ::use-alt-in-pointed
+::s ֶ ::t e ::comment Hebrew point segol
+::s ֱ ::t e ::comment Hebrew point hataf segol (hataf = reduced)
+::s ְ ::t e ::comment Hebrew point sheva ::t-alt "" ::use-alt-in-pointed
+::s ֵ ::t e ::comment Hebrew point tsere
+::s ִ ::t i ::comment Hebrew point hiriq
+::s ֹ ::t o ::comment Hebrew point holam
+::s ֻ ::t u ::comment Hebrew point qubuts
+# ::s ּ ::t "" ::comment Hebrew point dagesh or mapiq
+
+# Yiddish
+::s א ::t a ::lcode yid ::comment called "silent" alef
+::s אי ::t y ::lcode yid
+::s איי ::t ey ::lcode yid
+::s או ::t u ::lcode yid
+::s אוי ::t oy ::lcode yid
+::s אַ ::t a ::lcode yid
+::s אָ ::t o ::lcode yid
+::s ב ::t b ::lcode yid
+::s בֿ ::t v ::lcode yid
+::s דזש ::t dzh ::lcode yid
+::s ו ::t u ::lcode yid
+::s וּ ::t u ::lcode yid
+::s וֹ ::t o ::lcode yid
+::s װ ::t v ::lcode yid
+::s ווא ::t wa ::lcode yid
+::s וואַ ::t wa ::lcode yid
+::s ווע ::t we ::lcode yid
+::s ווי ::t wi ::lcode yid
+::s וואוי ::t wo ::lcode yid
+::s וי ::t oy ::lcode yid
+::s זש ::t zh ::lcode yid
+::s ח ::t ch ::lcode yid
+::s טש ::t tsh ::lcode yid
+::s יִ::t i ::lcode yid
+::s יי ::t ey ::lcode yid ::comment maybe "yi" at beginning of word
+::s ײַ ::t ay ::lcode yid
+::s כּ ::t k ::lcode yid
+::s כ ::t ch ::lcode yid
+::s ך ::t ch ::lcode yid
+::s ע ::t e ::lcode yid
+::s פּ ::t p ::lcode yid
+::s פֿ ::t f ::lcode yid
+::s ף ::t f ::lcode yid ::comment sometimes p
+::s ק ::t k ::lcode yid
+::s ת ::t s ::lcode yid
+
+# Syriac/Aramaic (should be vetted by expert)
+::s ܰ ::t a  ::comment Syriac pthaha above
+::s ܲ ::t a  ::comment Syriac pthaha dotted
+::s ܳ ::t aa  ::comment Syriac zqapha above
+::s ܴ ::t aa  ::comment Syriac zqapha below
+::s ܵ ::t aa  ::comment Syriac zqapha dotted
+::s ܶ ::t e  ::comment Syriac rbasa above
+::s ܷ ::t e  ::comment Syriac rbasa below
+::s ܿ ::t o  ::comment Syriac rwaha
+::s ܸ ::t e  ::comment Syriac dotted zlama horizontal
+::s ܹ ::t e  ::comment Syriac dotted zlama angular
+::s ܺ ::t i  ::comment Syriac hbasa above
+::s ܝܺ ::t i  ::comment Syriac yudh + hbasa above
+::s ܼ ::t u  ::comment Syriac hbasa-esasa dotted
+::s ܽ ::t o  ::comment Syriac esasa above
+::s ܾ ::t u  ::comment Syriac esasa below
+::s ݇ ::t ""  ::comment Syriac oblique line above; indication of a silent letter
+
+::s ܖ ::t d  ::comment Syriac letter dotless dalath rish; ambiguous form for undifferentiated early dalath/rish
+::s ܜ ::t t  ::comment Syriac letter teth garshuni; used in Garshuni documents
+::s ܒ݂ ::t v  ::comment Syriac beth + rukkakha
+::s ܒ�� ::t v  ::comment Syriac beth + ring-below
+::s ܓ݂ ::t g  ::comment Syriac gammal + rukkakha [IPA: ɣ]
+::s ܓ̥ ::t g  ::comment Syriac gammal + ring-below [IPA: ɣ]
+::s ܕ݂ ::t d  ::comment Syriac dalath + rukkakha [IPA: ð]
+::s ܕ̥ ::t d  ::comment Syriac dalath + ring-below [IPA: ð]
+::s ܟ݂ ::t kh  ::comment Syriac kaph + rukkakha [IPA: x]
+::s ܟ̥ ::t kh  ::comment Syriac kaph + ring-below [IPA: x]
+::s ܦ݂ ::t f  ::comment Syriac pe + rukkakha
+::s ܦ̥ ::t f  ::comment Syriac pe + ring-below
+::s ܦ݁ ::t p  ::comment Syriac pe + qushshaya
+::s ܬ݂ ::t th  ::comment Syriac taw + rukkakha [IPA: θ]
+::s ܬ̥ ::t th  ::comment Syriac taw + ring-below [IPA: θ]
+
+::s ܄ ::t : ::comment Syriac sublinear colon; used at the end of verses of supplicationscolon skewed left
+::s ܆ ::t , ::comment Syriac colon skewed left; marks a dependent clause
+::s ܇ ::t , ::comment Syriac colon skewed right; marks the end of a subdivision of the apodosis, or latter part of a Biblical verse
+
+# Uzbek
+::s ʻ ::t ' ::comment modifies pronunciation of preceding "o" and "g"
+::s ʼ ::t ' ::comment glottal stop (tutuq belgisi)
+
+# Uyghur
+::s ئا ::t a ::lcode uig
+::s ە ::t e ::lcode uig
+::s ئې ::t e ::lcode uig ::latinplus ë
+::s ې ::t e ::lcode uig ::latinplus ë
+::s ئە ::t e ::lcode uig
+::s يە ::t e ::lcode uig
+::s ئى ::t i ::lcode uig
+::s ى ::t i ::lcode uig
+::s ئو ::t o ::lcode uig
+::s و ::t o ::lcode uig
+::s ئۇ ::t u ::lcode uig
+::s ۇ ::t u ::lcode uig
+::s چ ::t ch ::t-alt q ::lcode uig
+::s خ ::t x ::lcode uig
+::s ژ ::t zh ::lcode uig
+::s ئۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ئۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۋ ::t w ::lcode uig
+
+# Maldivian
+::s ް ::t ::comment thaana sukun
+::s ަ ::t a ::comment thaana abafili
+::s ާ ::t aa ::comment thaana aabaafili
+::s ި ::t i ::comment thaana ibifili
+::s ީ ::t ee ::comment thaana eebeefili
+::s ު ::t u ::comment thaana ubufili
+::s ޫ ::t oo ::comment thaana ooboofili
+::s ެ ::t e ::comment thaana ebefili
+::s ޭ ::t ey ::comment thaana eybeyfili
+::s ޮ ::t o ::comment thaana obofili
+::s ޯ ::t oa ::comment thaana oaboafili
+
+# Canadian syllabics (Inuktitut)
+::s ᑊ ::t p ::comment syllable final
+::s ᐟ ::t t ::comment syllable final
+::s ᐠ ::t k ::comment syllable final
+::s ᐨ ::t c ::comment syllable final
+::s ᒼ ::t m ::comment syllable final
+::s ᐣ ::t n ::comment syllable final
+::s ᐢ ::t s ::comment syllable final
+::s ᐧ ::t y ::comment syllable final
+::s ᐤ ::t w ::comment syllable final
+::s ᐦ ::t h ::comment syllable final
+::s ᕽ ::t hk ::comment syllable final
+::s ᓫ ::t l ::comment syllable final
+::s ᕑ ::t r ::comment syllable final
+
+# Mongolian
+::s ᢅ ::t ::comment MONGOLIAN LETTER ALI GALI BALUDA (CHECK) indicates assimilation
+::s ᢆ ::t ::comment MONGOLIAN LETTER ALI GALI THREE BALUDA (CHECK) indicates assimilation
+
+# Tibetan
+::s ྅ ::t ::comment TIBETAN MARK PALUTA (CHECK) indicates assimilation
+
+## Punctuation
+# delete
+::s ¿ ::t "" ::comment inverted question mark
+::s ¡ ::t "" ::comment inverted exclamation mark
+# decompose double-punctuation
+::s ‼ ::t !!
+::s ⁇ ::t ??
+::s ⁉ ::t !?
+::s ⁈ ::t ?!
+# preserve
+::s ′ ::t ′
+::s ∩ ::t ∩
+::s ‡ ::t ‡
+# Cyrillic
+::s ⁙ ::t . ::comment five dot punctuation
+# Amharic/Ethiopian
+::s ። ::t .
+::s ፣ ::t ,
+::s ፤ ::t ;
+::s ፥ ::t :
+::s ፧ ::t ? ::comment Ethiopic question mark
+::s ፡ ::t " " ::comment Ethiopic wordspace
+::s ፦ ::t :  ::comment Ethiopic preface colon
+# Ethiopic wordspace often appropriated for other purposes:
+::s ፡፡ ::t .
+::s ፡- ::t :
+::s "፡ " ::t ", "
+::s ቸ ::t cha ::comment Ethiopic syllable ca
+::s ቹ ::t chu ::comment Ethiopic syllable cu
+::s ቺ ::t chi ::comment Ethiopic syllable ci
+::s ቻ ::t chaa ::comment Ethiopic syllable caa
+::s ቼ ::t chee ::comment Ethiopic syllable cee
+::s ች ::t che ::comment Ethiopic syllable ce
+::s ቾ ::t cho ::comment Ethiopic syllable co
+::s ሠ ::t sa ::comment Ethiopic syllable sza
+::s ሡ ::t su ::comment Ethiopic syllable szu
+::s ሢ ::t si ::comment Ethiopic syllable szi
+::s ሣ ::t saa ::comment Ethiopic syllable szaa
+::s ሤ ::t see::comment Ethiopic syllable szee
+::s ሥ ::t se ::comment Ethiopic syllable sze
+::s ሦ ::t so ::comment Ethiopic syllable szo
+::s ጠ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጡ ::t tu ::comment Ethiopic syllable thu with ejective 't'
+::s ጢ ::t ti ::comment Ethiopic syllable thi with ejective 't'
+::s ጣ ::t taa ::comment Ethiopic syllable thaa with ejective 't'
+::s ጤ ::t tee ::comment Ethiopic syllable thee with ejective 't'
+::s ጥ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጦ ::t to ::comment Ethiopic syllable tho with ejective 't'
+
+# Devanagari (Hindi etc.)
+::s । ::t . ::comment danda
+::s ॥ ::t . ::comment double danda
+::s ৷ ::t . ::comment Bengali currency numerator four; used as danda
+::s ॰ ::t . ::comment Devanagari abbreviation sign
+# Bengali
+::s ৽ ::t . ::comment BENGALI ABBREVIATION SIGN
+::s ৾ ::t ::comment BENGALI SANDHI MARK (CHECK)
+# Gurmukhi
+::s ੶ ::t . ::comment GURMUKHI ABBREVIATION SIGN
+# Oriya/Odia (India)
+::s ୤ ::t . ::comment danda (deprecated, should use Devanagari danda ।)
+::s ୥ ::t . ::comment double danda (deprecated, should use Devanagari double danda ॥)
+# Tibetan
+::s ། ::t ,
+::s །: ::t :
+::s ༏ ::t ;
+::s ༎ ::t .
+::s ༑ ::t ,  ::comment Tibetan mark run chen spungs shad
+::s ༼ ::t (  ::comment Tibetan open roof punctuation
+::s ༽ ::t )  ::comment Tibetan close roof punctuation
+::s ༈ ::t ""  ::comment Tibetan mark srbul shad
+::s 【 ::t [  ::comment left black lenticular bracket
+::s 】 ::t ]  ::comment right black lenticular bracket
+::s ༄ ::t ""  ::comment Tibetan head mark
+::s ༄༅ ::t ""  ::comment Tibetan head mark
+::s ༆ ::t ""  ::comment Tibetan head mark
+# Myanmar/Burmese
+::s ၊ ::t ,
+::s ။ ::t .
+Khmer
+::s ៖ ::t ; ::comment Khmer sign camnuc pii kuuh
+::s ។ ::t . ::comment Khmer sign khan
+# Arabic
+::s ، ::t ,
+::s ؛ ::t ;
+::s ٬ ::t ,
+::s ۔ ::t .
+::s ؟ ::t ?
+::s ٪ ::t %
+::s ٫ ::t , ::comment Arabic decimal separator
+::s ۽ ::t & ::comment Arabic sign Sindhi ampersand
+# Aramaic
+::s ܀ ::t .
+::s ܂ ::t .
+# Hebrew
+::s ־ ::t - ::comment maqaf
+# Armenian
+::s ։ ::t .
+::s ՝ ::t , ::comment Armenian comma
+# Chinese
+::s ， ::t ", "
+::s 、 ::t ", "
+::s 。 ::t ". "
+::s ！ ::t "! "
+::s ？ ::t "? "
+::s 「 ::t ' "'
+::s 」 ::t '" '
+::s 《 ::t ' "'
+::s 》 ::t '" '
+::s （ ::t " ("
+::s ） ::t ") "
+::s ； ::t ;
+::s ： ::t ": "
+::s ︰ ::t ": "
+::s － ::t -
+::s ／ ::t /
+::s ＝ ::t =
+::s ～ ::t ~
+::s ＆ ::t &
+::s ＜ ::t <
+::s ＞ ::t >
+::s ％ ::t %
+::s ＿ ::t _ ::comment FULLWIDTH LOW LINE (U+FF3F)
+::s ｛ ::t { ::comment FULLWIDTH LEFT CURLY BRACKET (U+FF5B)
+::s ｝ ::t } ::comment FULLWIDTH RIGHT CURLY BRACKET (U+FF5D)
+::s 　 ::t " " ::comment ideographic space
+# Japanese
+::s 『 ::t ' "'
+::s 』 ::t '" '
+::s ・ ::t " "  ::comment Katakana middle dot; separates name elements such as first and last name
+# N'ko
+::s ߽ ::t . ::comment NKO DANTAYALAN used to abbreviate units of measure
+# Medefaidrin
+::s 𖺗 ::t , ::comment MEDEFAIDRIN COMMA
+::s 𖺘 ::t . ::comment MEDEFAIDRIN FULL STOP
+# Khitan
+::s 𖿤 ::t ::comment KHITAN SMALL SCRIPT FILLER
+
+# Symbols
+::s ∞ ::t ∞  ::comment infinity
+::s ­ ::t   ::comment soft hyphen; used to indicate preferred line breaks; remove
+::s ֊ ::t -  ::comment Armenian hyphen; map to regular hyphen-minus
+::s ᐩ ::t +  ::comment Canadian syllabics final plus; map to regular plus
+::s ﹐ ::t ,  ::comment small comma; map to regular comma
+::s ˚ ::t °  ::comment ring above; map to degree sign
+::s ⇒ ::t ⇒ ::comment rightwards double arrow
+::s † ::t †  ::comment dagger
+::s • ::t •  ::comment bullet
+::s ℃ ::t °C  ::comment degree Celsius; split into 2 characters
+::s ℉ ::t °F  ::comment degree Fahrenheit; split into 2 characters
+::s ― ::t ―  ::comment horizontal bar
+::s ˇ ::t ˇ  ::comment caron (sometimes apparently used for "Arabic vowel sign small v above" U+065A, e.g. in Gilaki language (glk))
+::s ″ ::t ″  ::comment double prime
+::s ﴾ ::t (  ::comment ornate left parenthesis
+::s ﴿ ::t )  ::comment ornate right parenthesis
+::s 〔 ::t [  ::comment left tortoise shell bracket
+::s 〕 ::t ]  ::comment right tortoise shell bracket
+::s ﹝ ::t (  ::comment small left tortoise shell bracket
+::s ﹞ ::t )  ::comment small left tortoise shell bracket
+::s ¦ ::t ¦ ::comment BROKEN BAR (U+00A6)
+::s ¨ ::t ::comment DIAERESIS (U+00A8)
+::s ¯ ::t ::comment MACRON (U+00AF)
+::s ¸ ::t ::comment CEDILLA (U+00B8)
+::s Ƿ ::t W ::comment LATIN CAPITAL LETTER WYNN (U+01F7)
+::s ˘ ::t ::comment BREVE (U+02D8)
+::s ˛ ::t ::comment OGONEK (U+02DB)
+::s ˜ ::t ~ ::comment SMALL TILDE (U+02DC)
+::s ̒ ::t ::comment COMBINING TURNED COMMA ABOVE (U+0312)
+::s ̔ ::t ::comment COMBINING REVERSED COMMA ABOVE (U+0314)
+::s ̜ ::t ::comment COMBINING LEFT HALF RING BELOW (U+031C)
+::s ̧ ::t ::comment COMBINING CEDILLA (U+0327)
+::s ̫ ::t ::comment COMBINING INVERTED DOUBLE ARCH BELOW (U+032B)
+::s ̲ ::t ::comment COMBINING LOW LINE (U+0332)
+::s ̳ ::t ::comment COMBINING DOUBLE LOW LINE (U+0333)
+::s ̹ ::t ::comment COMBINING RIGHT HALF RING BELOW (U+0339)
+::s ̺ ::t ::comment COMBINING INVERTED BRIDGE BELOW (U+033A)
+::s ̿ ::t ::comment COMBINING DOUBLE OVERLINE (U+033F)
+::s ͅ ::t ::comment COMBINING GREEK YPOGEGRAMMENI (U+0345)
+::s ͑ ::t ::comment COMBINING LEFT HALF RING ABOVE (U+0351)
+::s ͗ ::t ::comment COMBINING RIGHT HALF RING ABOVE (U+0357)
+::s ͚ ::t ::comment COMBINING DOUBLE RING BELOW (U+035A)
+::s ͜ ::t ::comment COMBINING DOUBLE BREVE BELOW (U+035C)
+::s ͝ ::t ::comment COMBINING DOUBLE BREVE (U+035D)
+::s ͞ ::t ::comment COMBINING DOUBLE MACRON (U+035E)
+::s ͟ ::t ::comment COMBINING DOUBLE MACRON BELOW (U+035F)
+::s ͠ ::t ::comment COMBINING DOUBLE TILDE (U+0360)
+
+::s ‐ ::t - ::comment HYPHEN (U+2010)
+::s ‗ ::t ‗ ::comment DOUBLE LOW LINE (U+2017)
+::s ‵ ::t ‵ ::comment REVERSED PRIME (U+2035)
+::s ‶ ::t ‶ ::comment REVERSED DOUBLE PRIME (U+2036)
+::s ‸ ::t ‸ ::comment CARET (U+2038)
+::s ‽ ::t ?! ::comment INTERROBANG (U+203D)
+::s ‾ ::t ‾ ::comment OVERLINE (U+203E)
+::s ‿ ::t ‿ ::comment UNDERTIE (U+203F)
+::s ⁂ ::t ⁂ ::comment ASTERISM (U+2042)
+::s ⁎ ::t * ::comment LOW ASTERISK (U+204E)
+::s ⁏ ::t ; ::comment REVERSED SEMICOLON (U+204F)
+::s ⁔ ::t ⁔ ::comment INVERTED UNDERTIE (U+2054)
+::s ⁝ ::t ⁝ ::comment TRICOLON (U+205D)
+::s   ::t " " ::comment MEDIUM MATHEMATICAL SPACE (U+205F)
+::s ₋ ::t - ::comment SUBSCRIPT MINUS (U+208B)
+::s ⃩ ::t ::comment COMBINING WIDE BRIDGE ABOVE (U+20E9)
+
+::s ﹔ ::t ; ::comment SMALL SEMICOLON (U+FE54)
+::s ﹕ ::t : ::comment SMALL COLON (U+FE55)
+::s ﹛ ::t { ::comment SMALL LEFT CURLY BRACKET (U+FE5B)
+::s ﹜ ::t } ::comment SMALL RIGHT CURLY BRACKET (U+FE5C)
+::s ﹠ ::t & ::comment SMALL AMPERSAND (U+FE60)
+::s ﹡ ::t * ::comment SMALL ASTERISK (U+FE61)
+::s ﹣ ::t - ::comment SMALL HYPHEN-MINUS (U+FE63)
+
+::s ℈ ::t ℈ ::comment SCRUPLE (U+2108)
+::s ℟ ::t ℟ ::comment RESPONSE (U+211F)
+::s ℣ ::t ℣ ::comment VERSICLE (U+2123)
+::s ℽ ::t ℽ ::comment DOUBLE-STRUCK SMALL GAMMA (U+213D)
+::s ℾ ::t ℾ ::comment DOUBLE-STRUCK CAPITAL GAMMA (U+213E)
+::s ⅋ ::t ⅋ ::comment TURNED AMPERSAND (U+214B)
+::s ⅍ ::t A/S::comment AKTIESELSKAB (U+214D)
+
+::s ⑃ ::t ⑃ ::comment OCR INVERTED FORK (U+2443)
+::s ⑊ ::t \\ ::comment OCR DOUBLE BACKSLASH (U+244A)
+::s ⟮ ::t ( ::comment MATHEMATICAL LEFT FLATTENED PARENTHESIS (U+27EE)
+::s ⟯ ::t ) ::comment MATHEMATICAL RIGHT FLATTENED PARENTHESIS (U+27EF)
+::s ⸨ ::t (( ::comment LEFT DOUBLE PARENTHESIS (U+2E28)
+::s ⸩ ::t )) ::comment RIGHT DOUBLE PARENTHESIS (U+2E29)
+
+# kavyka indicates alternative reading
+::s ᷶ ::t ::comment COMBINING KAVYKA ABOVE RIGHT (U+1DF6)
+::s ᷷ ::t ::comment COMBINING KAVYKA ABOVE LEFT (U+1DF7)
+::s ⹅ ::t ::comment INVERTED LOW KAVYKA (U+2E45)
+::s ⹆ ::t ::comment INVERTED LOW KAVYKA WITH KAVYKA ABOVE (U+2E46)
+::s ⹇ ::t ::comment LOW KAVYKA (U+2E47)
+::s ⹈ ::t ::comment LOW KAVYKA WITH DOT (U+2E48)
+::s ꙾ ::t ::comment CYRILLIC KAVYKA (U+A67E)
+
+# Braille
+::s ⠁ ::t a
+::s ⠃ ::t b
+::s ⠉ ::t c
+::s ⠙ ::t d
+::s ⠑ ::t e
+::s ⠋ ::t f
+::s ⠛ ::t g
+::s ⠓ ::t h
+::s ⠊ ::t i
+::s ⠚ ::t j
+::s ⠅ ::t k
+::s ⠇ ::t l
+::s ⠍ ::t m
+::s ⠝ ::t n
+::s ⠕ ::t o
+::s ⠏ ::t p
+::s ⠟ ::t q
+::s ⠗ ::t r
+::s ⠎ ::t s
+::s ⠞ ::t t
+::s ⠥ ::t u
+::s ⠧ ::t v
+::s ⠺ ::t w
+::s ⠭ ::t x
+::s ⠽ ::t y
+::s ⠵ ::t z
+
+::s ⠜ ::t ae
+::s ⠪ ::t oe
+::s ⠳ ::t ue
+::s ⠷ ::t a ::comment à
+::s ⠡ ::t a ::comment â
+::s ⠿ ::t e ::comment é
+::s ⠮ ::t e ::comment è
+::s ⠣ ::t e ::comment ê
+::s ⠫ ::t e ::comment ë
+::s ⠩ ::t i ::comment î
+::s ⠻ ::t i ::comment ï
+::s ⠹ ::t o ::comment ô
+::s ⠾ ::t u ::comment ù
+::s ⠱ ::t u ::comment û
+
+::s ⠡ ::t au ::lcode deu
+::s ⠌ ::t aeu ::lcode deu
+::s ⠹ ::t ch ::lcode deu
+::s ⠩ ::t ei ::lcode deu
+::s ⠣ ::t eu ::lcode deu
+::s ⠬ ::t ie ::lcode deu
+::s ⠱ ::t sch ::lcode deu
+::s ⠮ ::t ss ::lcode deu
+::s ⠾ ::t st ::lcode deu
+
+::s ⠠⠠ ::t "" ::comment start of word all-caps mode
+# ::s ⠠⠁ ::t A
+# ::s ⠠⠃ ::t B
+# ::s ⠠⠉ ::t C
+# ::s ⠠⠙ ::t D
+# ::s ⠠⠑ ::t E
+# ::s ⠠⠋ ::t F
+# ::s ⠠⠛ ::t G
+# ::s ⠠⠓ ::t H
+# ::s ⠠⠊ ::t I
+# ::s ⠠⠚ ::t J
+# ::s ⠠⠅ ::t K
+# ::s ⠠⠇ ::t L
+# ::s ⠠⠍ ::t M
+# ::s ⠠⠝ ::t N
+# ::s ⠠⠕ ::t O
+# ::s ⠠⠏ ::t P
+# ::s ⠠⠟ ::t Q
+# ::s ⠠⠗ ::t R
+# ::s ⠠⠎ ::t S
+# ::s ⠠⠞ ::t T
+# ::s ⠠⠥ ::t U
+# ::s ⠠⠧ ::t V
+# ::s ⠠⠺ ::t W
+# ::s ⠠⠭ ::t X
+# ::s ⠠⠽ ::t Y
+# ::s ⠠⠵ ::t Z
+
+::s ⠼⠁ ::t 1
+::s ⠼⠃ ::t 2
+::s ⠼⠉ ::t 3
+::s ⠼⠙ ::t 4
+::s ⠼⠑ ::t 5
+::s ⠼⠋ ::t 6
+::s ⠼⠛ ::t 7
+::s ⠼⠓ ::t 8
+::s ⠼⠊ ::t 9
+::s ⠼⠚ ::t 0
+
+::s ⠂ ::t ,
+::s ⠆ ::t ;
+::s ⠒ ::t : 
+::s ⠲ ::t .
+::s ⠦ ::t ?
+::s ⠖ ::t !
+::s ⠄ ::t '
+::s ⠤ ::t -
+::s ⠨⠤ ::t _
+
+::s ⠀ ::t " " ::comment blank
+# ::s ⠐ t " " ::comment blank in numeric mode
+::s ⠈ ::t "" ::comment accent
+# ::s ⠌ ::t / ::comment in numeric mode only
+# ::s ⠐ ::comment abbreviation sign
+# ::s ⠘ ::comment abbreviation sign
+# ::s ⠠ ::comment capital indicator
+::s ⠨ ::t . ::comment decimal point; emphasis
+::s ⠰ ::t "" ::comment letter indicator
+# ::s ⠴ ::t ”
+# ::s ⠶ ::t ()
+# ::s ⠸ ::comment abbreviation sign
+::s ⠼ ::t "" ::comment number indicator
+::s ⠘⠚ ::t ° ::word-external-punctuation
+::s ⠘⠚⠠⠉ ::t °C
+::s ⠘⠚⠉ ::t °C
+::s ⠘⠚⠠⠋ ::t °F
+::s ⠘⠚⠋ ::t °F
+
+::s ⠠⠶ ::t " ::word-external-punctuation
+::s ⠘⠦ ::t “ ::word-external-punctuation
+::s ⠘⠴ ::t ” ::word-external-punctuation
+::s ⠄⠦ ::t ‘
+::s ⠄⠴ ::t ’
+::s ⠠⠴ ::t ���
+::s ⠐⠣ ::t ( ::word-external-punctuation
+::s ⠐⠜ ::t ) ::word-external-punctuation
+::s ⠨⠣ ::t [ ::word-external-punctuation
+::s ⠨⠜ ::t ] ::word-external-punctuation
+::s ⠸⠣ ::t { ::word-external-punctuation
+::s ⠸⠜ ::t } ::word-external-punctuation
+::s ⠈⠣ ::t < ::word-external-punctuation
+::s ⠈⠜ ::t > ::word-external-punctuation
+::s ⠸⠌ ::t / ::word-external-punctuation
+::s ⠸⠡ ::t \ ::word-external-punctuation
+::s ⠠⠤ ::t – ::word-external-punctuation
+::s ⠐⠠⠤ ::t — ::word-external-punctuation
+::s ⠈⠯ ::t & ::word-external-punctuation
+::s ⠐⠔ ::t * ::word-external-punctuation
+::s ⠨⠦ ::t ∩ ::word-external-punctuation
+::s ⠨⠴ ::t % ::word-external-punctuation
+::s ⠐⠖ ::t + ::word-external-punctuation
+::s ⠐⠤ ::t − ::word-external-punctuation
+::s ⠐⠶ ::t = ::word-external-punctuation
+::s ⠈⠎ ::t $ ::word-external-punctuation
+::s ⠈⠉ ::t ¢ ::word-external-punctuation
+::s ⠈⠇ ::t £ ::word-external-punctuation
+::s ⠈⠽ ::t ¥ ::word-external-punctuation
+::s ⠈⠁ ::t @ ::word-external-punctuation
+::s ⠸⠹ ::t # ::word-external-punctuation
+::s ⠸⠲ ::t • ::word-external-punctuation
+::s ⠈⠢ ::t ^ ::word-external-punctuation
+::s ⠈⠔ ::t ~ ::word-external-punctuation
+::s ⠘⠉ ::t © ::word-external-punctuation
+::s ⠐⠌ ::t ÷ ::word-external-punctuation
+::s ⠐⠦ ::t × ::word-external-punctuation
+::s ⠈⠠⠹ ::t † ::word-external-punctuation
+::s ⠈⠠⠻ ::t ‡ ::word-external-punctuation
+::s ⠘⠏ ::t ¶ ::word-external-punctuation
+::s ⠘⠎ ::t § ::word-external-punctuation
+::s ⠘⠗ ::t ® ::word-external-punctuation
+::s ⠘⠞ ::t ™ ::word-external-punctuation
+
+# English Braille
+::s ⠁⠃ ::t about ::lcode eng ::use-only-for-whole-word
+::s ⠁⠃⠧ ::t above ::lcode eng ::use-only-for-whole-word
+::s ⠁⠉ ::t according ::lcode eng ::use-only-for-whole-word
+::s ⠁⠉⠗ ::t across ::lcode eng ::use-only-for-whole-word
+::s ⠁⠋ ::t after ::lcode eng ::use-only-for-whole-word
+::s ⠁⠋⠝ ::t afternoon ::lcode eng ::use-only-for-whole-word
+::s ⠁⠋⠺ ::t afterward ::lcode eng ::use-only-for-whole-word
+::s ⠁⠛ ::t again ::lcode eng ::use-only-for-whole-word
+::s ⠁⠛⠌ ::t against ::lcode eng ::use-only-for-whole-word
+::s ⠠⠽ ::t ally ::lcode eng ::use-only-at-end-of-word ::use-only-in-lower-case-environment
+::s ⠁⠇⠍ ::t almost ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠗ ::t already ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇ ::t also ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠹ ::t although ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠞ ::t altogether ::lcode eng ::use-only-for-whole-word
+::s ⠁⠇⠺ ::t always ::lcode eng ::use-only-for-whole-word
+::s ⠨⠑ ::t ance ::lcode eng
+::s ⠯ ::t and ::lcode eng
+::s ⠜ ::t ar ::lcode eng
+::s ⠵ ::t as ::lcode eng ::use-only-for-whole-word
+::s ⠠⠝ ::t ation ::lcode eng ::use-only-at-end-of-word ::use-only-in-lower-case-environment
+::s ⠃ ::t b ::lcode eng
+::s ⠆ ::t bb ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠆ ::t be ::lcode eng ::use-only-at-start-of-word
+::s ⠆⠉ ::t because ::lcode eng ::use-only-for-whole-word
+::s ⠆⠋ ::t before ::lcode eng ::use-only-for-whole-word
+::s ⠆⠓ ::t behind ::lcode eng ::use-only-for-whole-word
+::s ⠆⠇ ::t below ::lcode eng ::use-only-for-whole-word
+::s ⠆⠝ ::t beneath ::lcode eng ::use-only-for-whole-word
+::s ⠆⠎ ::t beside ::lcode eng ::use-only-for-whole-word
+::s ⠆⠞ ::t between ::lcode eng ::use-only-for-whole-word
+::s ⠆⠽ ::t beyond ::lcode eng ::use-only-for-whole-word
+::s ⠃⠇ ::t blind ::lcode eng ::use-only-for-whole-word
+::s ⠃⠗⠇ ::t Braille ::lcode eng ::use-only-for-whole-word
+::s ⠃ ::t but ::lcode eng ::use-only-for-whole-word
+::s ⠉ ::t c ::lcode eng
+::s ⠉ ::t can ::lcode eng ::use-only-for-whole-word
+::s ⠸⠉ ::t cannot ::lcode eng
+::s ⠒ ::t cc ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠉⠧ ::t ceive ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠧⠙ ::t ceived ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠧⠎ ::t ceives ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠧⠛ ::t ceiving ::lcode eng
+::s ⠡ ::t ch ::lcode eng
+::s ⠐⠡ ::t character ::lcode eng
+::s ⠡ ::t child ::lcode eng ::use-only-for-whole-word
+::s ⠡⠝ ::t children ::lcode eng ::use-only-for-whole-word
+::s ⠒ ::t con ::lcode eng ::use-only-at-start-of-word
+::s ⠒ ::t : ::lcode eng ::use-only-at-end-of-word
+::s ⠉⠙ ::t could ::lcode eng ::use-only-for-whole-word
+::s ⠙ ::t d ::lcode eng
+::s ⠙ ::t do ::lcode eng ::use-only-for-whole-word
+::s ⠐⠙ ::t day ::lcode eng
+# ::s ⠲ ::t dd ::t-alt . ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word ::comment abolished; interferes with period in abbrevisations such as U.S.
+::s ⠙⠉⠇ ::t declare ::lcode eng
+::s ⠙⠉⠇⠛ ::t declaring ::lcode eng
+::s ⠲ ::t dis ::lcode eng ::use-only-at-start-of-word
+::s ⠲ ::t .  ::lcode eng ::dont-use-at-start-of-word
+::s ⠑ ::t e ::lcode eng
+::s ⠂ ::t ea ::lcode eng ::dont-use-at-end-of-word
+::s ⠂ ::t , ::lcode eng ::use-only-at-end-of-word
+::s ⠫ ::t ed ::lcode eng
+::s ⠑⠊ ::t either ::lcode eng ::use-only-for-whole-word
+::s ⠢ ::t en ::lcode eng
+::s ⠰⠑ ::t ence ::lcode eng ::dont-use-at-start-of-word
+::s ⠢ ::t enough ::lcode eng ::use-only-for-whole-word
+::s ⠻ ::t er ::lcode eng
+::s ⠐⠑ ::t ever ::lcode eng
+::s ⠑ ::t every ::lcode eng ::use-only-for-whole-word
+::s ⠋ ::t f ::lcode eng
+::s ⠐⠋ ::t father ::lcode eng
+::s ⠖ ::t ff ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠋⠌ ::t first ::lcode eng
+::s ⠿ ::t for ::lcode eng
+::s ⠋⠗ ::t friend ::lcode eng ::use-only-for-whole-word
+::s ⠋⠗⠎ ::t friends ::lcode eng ::use-only-for-whole-word
+::s ⠋ ::t from ::lcode eng ::use-only-for-whole-word
+::s ⠰⠇ ::t ful ::lcode eng ::dont-use-at-start-of-word
+::s ⠛ ::t g ::lcode eng
+::s ⠶ ::t gg ::lcode eng ::dont-use-at-start-of-word ::dont-use-at-end-of-word
+::s ⠣ ::t gh ::lcode eng
+::s ⠛ ::t go ::lcode eng ::use-only-for-whole-word
+::s ⠛⠙ ::t good ::lcode eng ::use-only-at-start-of-word
+::s ⠛⠗⠞ ::t great ::lcode eng
+::s ⠓ ::t h ::lcode eng
+::s ⠸⠓ ::t had ::lcode eng
+::s ⠓ ::t have ::lcode eng ::use-only-for-whole-word
+::s ⠐⠓ ::t here ::lcode eng
+::s ⠓⠻⠋ ::t herself ::lcode eng ::use-only-for-whole-word
+::s ⠓⠍ ::t him ::lcode eng ::use-only-for-whole-word
+::s ⠓⠍⠋ ::t himself ::lcode eng ::use-only-for-whole-word
+::s ⠦ ::t ? ::lcode eng
+::s ⠦ ::t his ::lcode eng ::use-only-for-whole-word
+::s ⠊⠍⠍ ::t immediate ::lcode eng ::use-only-for-whole-word
+::s ⠊⠍⠍⠇⠽ ::t immediately ::lcode eng ::use-only-for-whole-word
+::s ⠔ ::t in ::lcode eng
+::s ⠔⠒ ::t incon ::lcode eng ::use-only-at-start-of-word
+::s ⠬ ::t ing ::lcode eng
+::s ⠭ ::t it ::lcode eng ::use-only-for-whole-word
+::s ⠭⠎ ::t its ::lcode eng ::use-only-for-whole-word
+::s ⠭⠋ ::t itself ::lcode eng ::use-only-for-whole-word
+::s ⠰⠽ ::t ity ::lcode eng ::dont-use-at-start-of-word
+::s ⠚ ::t j ::lcode eng
+::s ⠚ ::t just ::lcode eng ::use-only-for-whole-word
+::s ⠅ ::t k ::lcode eng
+::s ⠐⠅ ::t know ::lcode eng
+::s ⠅ ::t knowledge ::lcode eng ::use-only-for-whole-word
+::s ⠇ ::t l ::lcode eng
+::s ⠨⠎ ::t less ::lcode eng ::dont-use-at-start-of-word
+::s ⠇⠗ ::t letter ::lcode eng ::use-only-for-whole-word
+::s ⠇⠗⠎ ::t letters ::lcode eng ::use-only-for-whole-word
+::s ⠇ ::t like ::lcode eng ::use-only-for-whole-word
+::s ⠇⠇ ::t little ::lcode eng ::use-only-for-whole-word
+::s ⠐⠇ ::t lord ::lcode eng
+::s ⠍ ::t m ::lcode eng
+::s ⠸⠍ ::t many ::lcode eng
+::s ⠰⠞ ::t ment ::lcode eng ::dont-use-at-start-of-word
+::s ⠍ ::t more ::lcode eng ::use-only-for-whole-word
+::s ⠐⠍ ::t mother ::lcode eng
+::s ⠍⠡ ::t much ::lcode eng ::use-only-for-whole-word
+::s ⠍⠌ ::t must ::lcode eng ::use-only-for-whole-word
+::s ⠍⠽⠋ ::t myself ::lcode eng ::use-only-for-whole-word
+::s ⠝  ::t n ::lcode eng
+::s ⠐⠝ ::t name ::lcode eng
+::s ⠝⠑⠉ ::t necessary ::lcode eng ::use-only-for-whole-word
+::s ⠝⠑⠊ ::t neither ::lcode eng ::use-only-for-whole-word
+::s ⠰⠎ ::t ness ::lcode eng ::dont-use-at-start-of-word
+::s ⠝  ::t not ::lcode eng ::use-only-for-whole-word
+::s ⠕⠄⠉ ::t o'clock ::lcode eng ::use-only-for-whole-word
+::s ⠷ ::t of ::lcode eng
+::s ⠐⠕ ::t one ::lcode eng
+::s ⠰⠛ ::t ong ::lcode eng ::dont-use-at-start-of-word
+::s ⠳ ::t ou ::lcode eng
+::s ⠨⠙ ::t ound ::lcode eng
+::s ⠨⠞ ::t ount ::lcode eng
+::s ⠐⠳ ::t ought ::lcode eng
+::s ⠳⠗⠧⠎ ::t ourselves ::lcode eng ::use-only-for-whole-word
+::s ⠳ ::t out ::lcode eng ::use-only-for-whole-word
+::s ⠪ ::t ow ::lcode eng
+::s ⠏ ::t p ::lcode eng
+::s ⠏⠙ ::t paid ::lcode eng ::use-only-for-whole-word
+::s ⠐⠏ ::t part ::lcode eng
+::s ⠏ ::t people ::lcode eng ::use-only-for-whole-word
+::s ⠏⠻⠓ ::t perhaps ::lcode eng ::use-only-for-whole-word
+::s ⠟ ::t q ::lcode eng
+::s ⠐⠟ ::t question ::lcode eng
+::s ⠟⠅ ::t quick ::lcode eng ::use-only-for-whole-word
+::s ⠟⠅⠻ ::t quicker ::lcode eng ::use-only-for-whole-word
+::s ⠟⠅⠑⠌ ::t quickest ::lcode eng ::use-only-for-whole-word
+::s ⠟ ::t quite ::lcode eng ::use-only-for-whole-word
+::s ⠗ ::t r ::lcode eng
+::s ⠗ ::t rather ::lcode eng ::use-only-for-whole-word
+::s ⠐⠗ ::t right ::lcode eng
+::s ⠗⠚⠉ ::t rejoice ::lcode eng
+::s ⠗⠚⠉⠛ ::t rejoicing ::lcode eng
+::s ⠎ ::t s ::lcode eng
+::s ⠎⠙ ::t said ::lcode eng ::use-only-for-whole-word
+::s ⠩ ::t sh ::lcode eng
+::s ⠩ ::t shall ::lcode eng ::use-only-for-whole-word
+::s ⠩⠙ ::t should ::lcode eng ::use-only-for-whole-word
+::s ⠨⠝ ::t sion ::lcode eng
+::s ⠎ ::t so ::lcode eng ::use-only-for-whole-word
+::s ⠐⠎ ::t some ::lcode eng
+::s ⠸⠎ ::t spirit ::lcode eng
+::s ⠌ ::t st ::lcode eng
+::s ⠌ ::t still ::lcode eng ::use-only-for-whole-word
+::s ⠎⠡ ::t such ::lcode eng ::use-only-for-whole-word
+::s ⠞ ::t t ::lcode eng
+::s ⠹ ::t th ::lcode eng
+::s ⠞ ::t that ::lcode eng ::use-only-for-whole-word
+::s ⠹ ::t this ::lcode eng ::use-only-for-whole-word
+::s ⠮ ::t the ::lcode eng
+::s ⠸⠮ ::t their ::lcode eng
+::s ⠮⠍⠧⠎ ::t themselves ::lcode eng ::use-only-for-whole-word
+::s ⠐⠮ ::t there ::lcode eng
+::s ⠘⠮ ::t these ::lcode eng
+::s ⠘⠹ ::t those ::lcode eng
+::s ⠐⠹ ::t through ::lcode eng
+::s ⠐⠞ ::t time ::lcode eng
+::s ⠰⠝ ::t tion ::lcode eng ::dont-use-at-start-of-word
+::s ⠖ ::t to ::lcode eng ::use-only-for-whole-word
+::s ⠞⠙ ::t today ::lcode eng ::use-only-for-whole-word
+::s ⠞⠛⠗ ::t together ::lcode eng ::use-only-for-whole-word
+::s ⠞⠍ ::t tomorrow ::lcode eng ::use-only-for-whole-word
+::s ⠞⠝ ::t tonight ::lcode eng ::use-only-for-whole-word
+::s ⠥ ::t u ::lcode eng
+::s ⠥⠝⠒ ::t uncon ::lcode eng ::use-only-at-start-of-word
+::s ⠥ ::t us ::lcode eng ::use-only-for-whole-word
+::s ⠠⠥⠲⠎⠲ ::t U.S. ::lcode eng
+::s ⠐⠥ ::t under ::lcode eng
+::s ⠘⠥ ::t upon ::lcode eng
+::s ⠧ ::t v ::lcode eng
+::s ⠧ ::t very ::lcode eng ::use-only-for-whole-word
+::s ⠺ ::t w ::lcode eng
+::s ⠴ ::t " ::lcode eng
+::s ⠴ ::t was ::lcode eng ::use-only-for-whole-word
+::s ⠶ ::t were ::lcode eng ::use-only-for-whole-word
+::s ⠱ ::t wh ::lcode eng
+::s ⠐⠱ ::t where ::lcode eng
+::s ⠱ ::t which ::lcode eng ::use-only-for-whole-word
+::s ⠘⠱ ::t whose ::lcode eng
+::s ⠺ ::t will ::lcode eng ::use-only-for-whole-word
+::s ⠾ ::t with ::lcode eng
+::s ⠘⠺ ::t word ::lcode eng
+::s ⠐⠺ ::t work ::lcode eng
+::s ⠸⠺ ::t world ::lcode eng
+::s ⠺⠙ ::t would ::lcode eng ::use-only-for-whole-word
+::s ⠭ ::t x ::lcode eng
+::s ⠽ ::t y ::lcode eng
+::s ⠽ ::t you ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗ ::t your ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗⠎ ::t yours ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗⠋ ::t yourself ::lcode eng ::use-only-for-whole-word
+::s ⠽⠗⠧⠎ ::t yourselves ::lcode eng ::use-only-for-whole-word
+::s ⠐⠽ ::t young ::lcode eng
+::s ⠵ ::t z ::lcode eng
+::s ⠠⠴ ::t ’ ::lcode eng
+
+::preserve ::from U+2190 ::to U+21FF ::comments Arrows
+::preserve ::from U+2200 ::to U+22FF ::comment Mathematical Operators
+::preserve ::from U+2300 ::to U+23FF ::comment Miscellaneous Technical
+::preserve ::from U+2500 ::to U+257F ::comment Box Drawing
+::preserve ::from U+2580 ::to U+259F ::comment Block Elements
+::preserve ::from U+25A0 ::to U+25FF ::comment Geometric Shapes
+::preserve ::from U+2600 ::to U+26FF ::comment Miscellaneous Symbols
+::preserve ::from U+27C0 ::to U+27ED ::comment Miscellaneous Mathematical Symbols-A
+::preserve ::from U+27F0 ::to U+27FF ::comment Supplemental Arrows-A
+::preserve ::from U+2900 ::to U+297F ::comment Supplemental Arrows-B
+::preserve ::from U+2980 ::to U+29FF ::comment Miscellaneous Mathematical Symbols-B
+::preserve ::from U+2A00 ::to U+2AFF ::comment Supplemental Mathematical Operators
+::preserve ::from U+2B00 ::to U+2BFF ::comment Miscellaneous Symbols and Arrows
+::preserve ::from U+2E00 ::to U+2E27 ::comment Supplemental Punctuation (excluding ⸨⸩)
+::preserve ::from U+2E2A ::to U+2E7F ::comment Supplemental Punctuation (cont'd)
+::preserve ::from U+18B00 ::to U+18CD5 ::comment Khitan Small Script
+::preserve ::from U+1D100 ::to U+1D1FF ::comment Musical Symbols
+::preserve ::from U+1D6A8 ::to U+1D7CB ::comment Mathematical Alphanumeric Symbols (Greek)
+::preserve ::from U+1D800 ::to U+1DAAF ::comment Sutton SignWriting
+::preserve ::from U+1F800 ::to U+1F8FF ::comment Supplemental Arrows-C
+::preserve ::from U+1FA00 ::to U+1FA6F ::comment Chess Symbols
+::preserve ::from U+1FB00 ::to U+1FBCF ::comment Symbols for Legacy Computing
+::preserve ::from U+1FA70 ::to U+1FAFF ::comment Symbols and Pictographs Extended-A

uroman/data/romanization-table.v1.2.1.txt

@@ -0,0 +1,814 @@
+
+## European Latin extensions
+# Vowels
+::s Ä ::t Ae
+::s Ö ::t Oe
+::s Ü ::t Ue
+::s Å ::t Aa
+::s Æ ::t Ae
+::s Ø ::t oe
+::s Π::t Oe
+::s ä ::t ae
+::s ö ::t oe
+::s ü ::t ue
+::s å ::t aa
+::s æ ::t ae
+::s ø ::t oe
+::s œ ::t oe
+# Consonants
+::s Ç ::t S
+::s ç ::t s
+::s Ç ::t Ch ::lcode tur
+::s ç ::t ch ::lcode tur
+::s Ş ::t Sh
+::s ş ::t sh
+::s Ș ::t Sh
+::s ș ::t sh
+::s ß ::t ss
+::s Ț ::t Ts
+::s ț ::t ts
+
+# Miscellaneous
+::s ə ::t e
+
+# English
+::s chr ::t chr ::t-alt kr ::example chromosome, synchronize
+::s Chr ::t Chr ::t-alt Kr ::example Christmas, Chrysler
+::s eight ::t eight ::t-alt eit ::example eight, weight
+::s Eight ::t Eight ::t-alt Eit ::example Eighteen
+::s ight ::t ight ::t-alt ait ::example Knight
+::s gh ::t gh ::t-alt f, ph, "" ::example laugh, daughter
+::s high ::t high ::t-alt hai ::example highlight
+::s High ::t High ::t-alt Hai ::example High School
+::s Isle ::t Isle ::t-alt Ail ::use-only-at-start-of-word ::use-only-at-end-of-word ::example Isle
+::s Island ::t Island ::t-alt Ailand ::use-only-at-start-of-word ::use-only-at-end-of-word ::example Island
+::s kn ::t kn ::t-alt n ::use-only-at-start-of-word ::example knowledge
+::s Kn ::t Kn ::t-alt N ::use-only-at-start-of-word ::example Knight
+::s Mc ::t Mc ::t-alt Mac ::use-only-at-start-of-word ::example McNulty
+::s mc ::t mc ::t-alt mac ::use-only-at-start-of-word
+::s oo ::t oo ::t-alt u ::lcode eng ::example Brooklyn; Goose Bay
+::s ph ::t ph ::t-alt f ::example alpha
+::s Ph ::t Ph ::t-alt F ::example Philip
+::s Thom ::t Thom ::t-alt Tom ::use-only-at-start-of-word ::example Thomas, Thompson
+::s tion ::t tion ::t-alt shen ::example
+::s Sean ::t Sean ::t-alt Shawn ::use-only-at-start-of-word ::use-only-at-end-of-word
+::s ssion ::t ssion ::t-alt shen ::example Sessions
+::s St  ::t St ::t-alt Saint ::use-only-at-start-of-word ::use-only-at-end-of-word
+::s St.  ::t St. ::t-alt Saint ::use-only-at-start-of-word ::use-only-at-end-of-word
+::s Wr ::t Wr ::t-alt R ::example Wren
+::s wr ::t wr ::t-alt r ::example Cartwright
+::s x ::t x ::t-alt ks ::example Mexico
+::s x ::t x ::t-alt gz ::example example, anxiety, exhaust, exit
+
+# French
+::s â ::t a ::t-alt as ::example pâte/paste, pastry
+::s ê ::t e ::t-alt es ::example fête/feast
+::s î ::t i ::t-alt is ::example île/isle
+::s ô ::t o ::t-alt os ::example côte/coast
+::s û ::t u ::t-alt us ::example août/August
+::s eaux ::t eaux ::t-alt o ::example Bordeaux
+::s eau ::t eau ::t-alt o ::example Chateau
+::s auld ::t auld ::t-alt o ::use-only-at-end-of-word ::example Renauld
+::s ault ::t ault ::t-alt o ::use-only-at-end-of-word ::example Renault
+::s oux ::t oux ::t-alt u
+::s ois ::t ois ::t-alt oa ::use-only-at-end-of-word ::example Dubois
+
+# German
+::s Sch ::t Sch ::t-alt Sh
+::s sch ::t sch ::t-alt sh
+::s stein ::t stein ::t-alt shtain
+::s dt ::t dt ::t-alt tt ::use-only-at-end-of-word ::example Schmidt
+
+# Dutch
+::s ij ::t ij ::t-alt ai
+::s Ij ::t Ij ::t-alt Ai
+
+# Greek
+::s Ι ::t I
+::s ι ::t i
+::s ί ::t i
+::s ἶ ::t i
+::s Υ ::t Y
+::s υ ::t y
+::s Ρ ::t R
+::s ρ ::t r
+::s Ντ ::t D
+::s ντ ::t nd ::t-alt d
+# ::s ντζ ::t ntz
+::s Μπ ::t B
+::s μπ ::t mb ::t-alt b
+::s γγ ::t ng
+::s γκ ::t ng ::t-alt g
+::s ει ::t ei ::t-alt i
+::s ου ::t ou ::t-alt u
+::s χ ::t ch ::t-alt kh
+
+# Cyrillic
+::s Г ::t G ::t-alt H
+::s г ::t g ::t-alt h
+::s Е ::t E ::t-alt Ye
+::s е ::t e ::t-alt ye
+::s Ё ::t E ::t-alt Yo
+::s ё ::t e ::t-alt yo
+::s Х ::t Kh ::t-alt Ch, H ::comment Cyrillic capital ha
+::s х ::t kh ::t-alt ch, h ::comment Cyrillic small ha
+::s Щ ::t Shch ::t-alt Sh
+::s щ ::t shch ::t-alt sh
+::s Ъ ::t ::comment Cyrillic capital hard sign
+::s ъ ::t ::comment Cyrillic small hard sign
+::s Ы ::t Y ::comment Cyrillic capital yeru
+::s ы ::t y ::comment Cyrillic small yeru
+::s Ь ::t ::comment Cyrillic capital soft sign
+::s ь ::t ::comment Cyrillic small soft sign
+
+::s Ҥ ::t Ng ::comment Cyrillic capital ligature EN GHE
+::s ҥ ::t ng ::comment Cyrillic small ligature EN GHE
+::s Ә ::t e ::comment Cyrillic capital schwa
+::s ә ::t e ::comment Cyrillic small schwa
+::s Ӏ ::t ' ::comment Cyrillic palochka
+::s Ҵ ::t TS ::comment Cyrillic capital ligature te tse, used in Abkhasian
+::s ҵ ::t ts ::comment Cyrillic small ligature te tse, used in Abkhasian
+::s Ӕ ::t AE ::comment Cyrillic capital ligature a ie
+::s ӕ ::t ae ::comment Cyrillic small ligature a ie
+::s Г ::t H ::lcode ukr ::comment Ukrainian capital letter he 
+::s г ::t h ::lcode ukr ::comment Ukrainian small letter he
+::s Ґ ::t G ::lcode ukr ::comment Ukrainian capital letter ghe
+::s ґ ::t g ::lcode ukr ::comment Ukrainian small letter ghe
+
+# Gothic
+::s 𐌴 ::t e ::comment Gothic letter aihvus
+::s 𐌹 ::t i ::comment Gothic letter eis
+::s 𐍇 ::t x ::comment Gothic letter iggws
+
+# Georgian
+::s ა ::t a ::comment Georgian letter an
+::s ე ::t e ::comment Georgian letter en
+::s ი ::t i ::comment Georgian letter in
+::s ო ::t o ::comment Georgian letter on
+::s უ ::t u ::comment Georgian letter un
+
+# Armenian
+::s Ա ::t a ::comment Armenian capital letter ayb
+::s ա ::t a ::comment Armenian small letter ayb
+::s Ե ::t e ::comment Armenian capital letter ech
+::s ե ::t e ::comment Armenian small letter ech
+::s և ::t ev ::comment Armenian small ligature ech yiwn
+::s Է ::t e ::comment Armenian capital letter eh
+::s է ::t e ::comment Armenian small letter eh
+::s Ի ::t i ::comment Armenian capital letter ini
+::s ի ::t i ::comment Armenian small letter ini
+::s Օ ::t o ::comment Armenian capital letter oh
+::s օ ::t o ::comment Armenian small letter oh
+
+## Japanese
+# Katakana
+::s シ ::t shi
+::s チ ::t chi
+::s フ ::t fu
+::s ジ ::t ji
+::s ヂ ::t ji
+::s ヅ ::t zu
+::s シャ ::t sha
+::s シュ ::t shu
+::s ショ ::t sho
+::s チャ ::t cha
+::s チェ ::t che
+::s チュ ::t chu
+::s チョ ::t cho
+::s ジャ ::t ja
+::s ジュ ::t ju
+::s ジョ ::t jo
+::s ジェ ::t je
+::s ヂャ ::t ja
+::s ヂュ ::t ju
+::s ヂョ ::t jo
+::s フェ ::t fe
+::s ヴェ ::t ve
+::s フィ ::t fi
+::s ウィ ::t wi
+::s ヴィ ::t vi
+::s ティ ::t ti
+::s ディ ::t di
+::s ッ ::t (__SOKUON__) ::comment katakana double following consonant
+::s ー ::t (__CHOONPU__) ::comment katakana prolonged sound mark
+# Hiragana
+::s し ::t shi
+::s ち ::t chi
+::s つ ::t tsu
+::s ふ ::t fu
+::s を ::t o
+::s じ ::t ji
+::s ぢ ::t ji
+::s づ ::t zu
+::s しゃ ::t sha
+::s しゅ ::t shu
+::s しょ ::t sho
+::s ちゃ ::t cha
+::s ちゅ ::t chu
+::s ちょ ::t cho
+::s じゃ ::t ja
+::s じゅ ::t ju
+::s じょ ::t jo
+::s ぢゃ ::t ja
+::s ぢゅ ::t ju
+::s ぢょ ::t jo
+::s っ ::t (__SOKUON__) ::comment hiragana double following consonant
+::s 々 ::t ² ::comment ideographic iteration mark ::annotation repetition-sign
+
+::s フ ::t fu ::t-alt f
+::s キ ::t ki ::t-alt k
+::s ク ::t ku ::t-alt k
+::s ラ ::t ra ::t-alt la
+::s リ ::t ri ::t-alt li
+::s ル ::t ru ::t-alt lu, l, r
+::s レ ::t re ::t-alt le
+::s ロ ::t ro ::t-alt lo
+::s ム ::t mu ::t-alt m ::example キム = Kim
+::s シ ::t shi ::t-alt si ::example メキシコ = meksiko (Mexico)
+::s ス ::t su ::t-alt s
+::s ト ::t to ::t-alt t
+::s ツ ::t tsu ::t-alt tu, ts ::example シュルツ = Schultz
+
+# Chinese
+::s 邦 ::t bang ::t-alt bon, bum, bun, pon
+::s 鲍 ::t bao ::t-alt bow
+::s 堡 ::t bao ::t-alt berg, burg, bourg, burgh
+::s 贝 ::t bei ::t-alt ber
+::s 本 ::t ben ::t-alt bern, bon, bourn, burn
+::s 彼得 ::t bide ::t-alt peter, pet
+::s 伯 ::t bo ::t-alt ber
+::s 波 ::t bo ::t-alt po
+::s 布 ::t bu ::t-alt b
+::s 策 ::t ce ::t-alt tze, tzer
+::s 曾 ::t ceng ::t-alt tzen, zen
+::s 彻 ::t che ::t-alt tche
+::s 茨 ::t ci ::t-alt ts, tz, z
+::s 兹 ::t ci ::t-alt ds, dz, tz, z, zi
+::s 蒂 ::t di ::t-alt ti, tti
+::s 丁 ::t ding ::t-alt din, tin
+::s 顿 ::t dun ::t-alt ton
+::s 多 ::t duo ::t-alt do, dor, to
+::s 尔 ::t er ::t-alt l, le, ll, r
+::s 弗 ::t fu ::t-alt f, fer, pher, v, ver, vir
+::s 夫 ::t fu ::t-alt f, v, v
+::s 福 ::t fu ::t-alt faw, for, ford
+::s 哥 ::t ge ::t-alt go, co
+::s 戈 ::t ge ::t-alt go
+::s 各 ::t ge ::t-alt go, co
+::s 赫 ::t he ::t-alt ch, che, cher, ge
+::s 华 ::t hua ::t-alt ver, wa, war, wer ::example Washington
+::s 怀 ::t huai ::t-alt whi, wi, wy
+::s 惠 ::t hui ::t-alt wha, whea
+::s 基 ::t ji ::t-alt ki, chi
+::s 吉 ::t ji ::t-alt gi, gui
+::s 加 ::t jia ::t-alt ca, ga, ka ::example Canada
+::s 杰 ::t jie ::t-alt ger
+::s 金 ::t jin ::t-alt kin, gin
+::s 斤 ::t jin ::t-alt zin
+::s 康 ::t kang ::t-alt con, corn
+::s 考 ::t kao ::t-alt cow, cour
+::s 克 ::t ke ::t-alt k, che, cher
+::s 科 ::t ke ::t-alt ko
+::s 拉 ::t la ::t-alt ra ::example Tirana
+::s 朗 ::t lang ::t-alt lon, ron
+::s 赖 ::t lai ::t-alt ri
+::s 劳 ::t lao ::t-alt low
+::s 勒 ::t lei ::t-alt ler
+::s 伦 ::t lun ::t-alt lon, ran, ron
+::s 里 ::t li ::t-alt ri
+::s 利 ::t li ::t-alt ri ::example Ferrari
+::s 隆 ::t long ::t-alt lon, lum, lund
+::s 罗 ::t luo ::t-alt l, lo, lu, ro, row, ru
+::s 洛 ::t luo ::t-alt lo, low, ro
+::s 默 ::t mo ::t-alt mer
+::s 纳 ::t na ::t-alt ne, ner
+::s 珀 ::t po ::t-alt per
+::s 奇 ::t qi ::t-alt chi, dge, ge, tch
+::s 齐 ::t qi ::t-alt tsi, zi
+::s 乔 ::t qiao ::t-alt jo
+::s 青 ::t qing ::t-alt tsing
+::s 琼 ::t qiong ::t-alt jon, jum, jun
+::s 瑟 ::t se ::t-alt the
+::s 什 ::t shen ::t-alt sh
+::s 圣 ::t sheng ::t-alt san, sao, saint
+::s 斯 ::t si ::t-alt s, rth, th ::example Alaska
+::s 索 ::t suo ::t-alt tho
+::s 特 ::t te ::t-alt t
+::s 翁 ::t weng ::t-alt on
+::s 沃 ::t wo ::t-alt ver, vo, war, wer
+::s 乌 ::t wu ::t-alt ou, u
+::s 希 ::t xi ::t-alt chi, hi, shi
+::s 西 ::t xi ::t-alt s, si
+::s 锡 ::t xi ::t-alt ci, si, thi, zi
+::s 夏 ::t xia ::t-alt ha, cha, cia, sha, tia
+::s 香 ::t xiang ::t-alt chan, cham
+::s 歇 ::t xie ::t-alt she
+::s 谢 ::t xie ::t-alt che, she
+::s 辛 ::t xin ::t-alt cin, sen, sin, sing, sun, zen
+::s 欣 ::t xin ::t-alt hin, shin
+::s 休 ::t xiu ::t-alt hu, hue
+::s 修 ::t xiu ::t-alt ciu, siu, thew, tiu
+::s 许 ::t xu ::t-alt hue, schue
+::s 逊 ::t xun ::t-alt son
+::s 耶 ::t ye ::t-alt yer, ier
+::s 泽 ::t ze ::t-alt ser
+::s 扎 ::t zha ::t-alt za
+::s 詹 ::t zhan ::t-alt ja, jam, jan, jen, jon
+::s 治 ::t zhi ::t-alt ge ::example George
+
+## Numbers
+# Chinese and Japanese numbers
+::s 零 ::num 0
+::s 〇 ::num 0
+::s 一 ::num 1
+::s 二 ::num 2
+::s 三 ::num 3
+::s 四 ::num 4
+::s 五 ::num 5
+::s 六 ::num 6
+::s 七 ::num 7
+::s 八 ::num 8
+::s 九 ::num 9
+::s 十 ::num 10
+::s 百 ::num 100
+::s 千 ::num 1000
+::s 万 ::num 10000
+::s 萬 ::num 10000
+::s 亿 ::num 100000000
+::s 億 ::num 100000000
+::s 兆 ::num 1000000000000
+::s 京 ::num 10000000000000000
+
+::s 北京 ::t beijing
+::s 京都 ::t jingdou
+::s 东京 ::t dongjing
+::s 京胡 ::t jinghu
+::s 南京 ::t nangjing
+::s 普京 ::t pujing ::comment Putin
+::s 東京 ::t dongjing ::comment Tokyo
+::s 京兆 ::t jingzhao
+
+::s ㎢ ::t km²
+::s ㎥ ::t m³
+::s ㎝ ::t cm
+
+## Indian
+# see mostly under UnicodeDataOverwrite.txt
+
+# Malayalam
+::s ൗ ::t au ::comment MALAYALAM AU LENGTH MARK
+
+# Tamil
+::s ட ::t d ::comment most commonly d, but t when word-initial or in a doubled consonant
+::s ஃப ::t f ::comment h+p=f
+::s ஃஜ ::t z ::comment h+j=z
+
+# Myanmar/Burmese
+# ::s ့ ::t ::comment dot below, denotes creaky tone 
+# ::s း ::t ::comment visarga, denotes high tone
+::s ၌ ::t -nai ::comment locative
+::s ၍ ::t -jwe ::comment completed
+::s ၎ ::t legau ::comment aforementioned
+::s ၏ ::t -i ::comment genetive
+
+# Lao
+::s ັ ::t a ::comment vowel sign mai kan
+::s ົ ::t o ::comment vowel sign mai kon
+::s ູ ::t uu ::comment vowel sign uu
+::s ຽ ::t y ::comment semivowel sign nyo
+::s ຼ ::t l ::comment semivowel sign lo
+::s ລ ::t l ::comment lo loot
+::s ຣ ::t l ::comment lo ling
+::s ໝ ::t m ::comment ho mo
+::s ໜ ::n ::comment ho no
+::s ຢ ::t y ::comment yo
+::s ໍ ::t oo ::comment niggahita (possibly also nasal -m in final position)
+::s ໆ ::t ² ::comment Lao ko la ::annotation repetition-sign
+::s ຯ ::t ...  ::comment Lao ellipsis
+
+# Thai
+::s ออ ::t o
+::s อั ::t a
+::s อิ ::t i
+::s ๆ ::t ² ::comment Thai character maiyamok ::annotation repetition-sign
+
+# Khmer
+::s ័ ::t "" ::comment Khmer samyok sannya: indicates deviation from the general rules of pronunciation
+::s ៏ ::t "" ::comment Khmer sign ahsda: denotes stressed intonation in some single-consonant words
+::s ៍ ::t "" ::comment Khmer sign toandakhiat: indicates that the base character is not pronounced
+::s ៌ ::t "" ::comment Khmer sign robat: a diacritic historically corresponding to the repha form of ra in Devanagari
+::s ប៉ ::t pa ::comment Khmer ba + musĕkâtônd -> pa
+::s ៗ ::t ² ::comment Khmer sign lek too ::annotation repetition-sign
+
+## Semitic languages
+# Arabic
+::s و ::t w  ::comment Arabic letter waw  ::t-alt o, u  ::lcode ara
+::s ء ::t ' ::comment hamza 
+::s ٔ ::t ' ::comment hamza above
+::s ٕ ::t ' ::comment hamza below
+::s ع ::t ' ::comment ain
+::s آ ::t a ::comment alef madda
+::s ٓا ::t a ::comment Arabic maddah above plus alef (presumably an ill-formed version of آ; found 1 instance in Urdu text)
+::s إ ::t i ::comment alef with hamza below
+::s ٱ ::t a ::comment alef wasla ::comment typically indicates liaison with preceding word 
+::s ة ::t a ::comment teh marbuta
+::s ۃ ::t a ::comment teh marbuta goal ::comment Used in Punjabi, Sindhi. Different from plain 'teh marbuta'?
+::s ي ::t y ::comment Arabic yeh
+::s ى ::t a ::comment alef maksura
+::s ﻯ ::t a ::comment alef maksura isolated form
+::s ﻰ ::t a ::comment alef maksura final form
+::s ﯨ ::t a ::comment Uighur Kazach Kirghiz alef maksura initial form
+::s ﯩ ::t a ::comment Uighur Kazach Kirghiz alef maksura medial form
+::s ٰ ::t a  ::comment Arabic letter superscript alef
+::s ـ ::t   ::comment tatweel (filler)
+::s َ ::t a ::comment fatha ("-a")
+::s ُ ::t u ::comment damma ("-u")
+::s ِ ::t i ::comment kasra ("-i")
+::s ْ ::t   ::comment sukun (no vowel)
+::s ۡ ::t  ::comment small high dotless head of khah; like sukun (no vowel); used in Kashmiri, Assamese
+::s ً ::t   ::comment fathatan ("-an")
+::s اً ::t an ::comment alef + fathatan
+::s ٌ ::t   ::comment dammatan ("-un")
+::s ٍ ::t   ::comment kasratan ("-in")
+::s ّ ::t   ::comment shadda (consonant doubler)
+::s ڃ ::t ny ::comment Arabic letter nyeh U+0683 (used in Sindhi (snd))
+::s ڄ ::t dy ::comment Arabic letter dyeh U+0684 (used in Sindhi (snd))
+::s ۾ ::t men ::comment Sindhi postposition men
+::s ؑ ::t alayhe wasallam ::comment "upon him be peace"
+::s ﷴ ::t mohammad ::comment "Mohammad"
+::s ﷸ ::t wasallam ::comment "and peace"
+::s ﷺ ::t sallallahou alayhe wasallam ::comment "prayer of God be upon him and his family and peace"
+
+# Farsi
+::s ی ::t i ::t-alt y ::comment Contributed by Nima
+::s ای ::t i ::t-alt ai ::use-only-at-start-of-word ::comment Contributed by Nima
+::s هٔ ::t eye ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s و ::t v ::t-alt o, u ::lcode fas ::comment Arabic letter waw
+::s ض ::t z ::t-alt d ::lcode fas ::comment Contributed by Marjan
+::s ث ::t s ::t-alt th ::lcode fas ::comment Contributed by Marjan
+::s ذ ::t z ::t-alt th ::lcode fas ::comment Contributed by Nima
+::s ع ::t a ::t-alt ' ::lcode fas ::comment Contributed by Nima
+::s عا ::t a ::lcode fas ::comment Contributed by Nima
+::s عی ::t i ::t-alt iy ::lcode fas ::comment Contributed by Nima
+::s عو ::t u ::t-alt o, av ::lcode fas ::comment Contributed by Nima
+::s چ ::t ch ::t-alt tch, tsh ::lcode fas ::comment Contributed by Nima
+::s ه ::t e ::t-alt h ::use-only-at-end-of-word ::lcode fas ::comment Contributed by Nima
+::s ‌ ::t "" ::t-alt " " ::lcode fas ::comment source is character "zero-width non-joiner" (U+200C); Contributed by Nima
+::s غ ::t gh ::t-alt g ::lcode fas
+::s آئی ::t ai ::t-alt ae ::lcode fas
+::s ائی ::t ai ::t-alt ae ::lcode fas
+::s آئو ::t au ::t-alt ao ::lcode fas
+::s ائو ::t au ::t-alt ao ::lcode fas
+
+# Kashmiri (so far: educated guesses)
+::s ٖ ::t a ::comment Arabic subscript alef U+0656
+::s ٗ ::t u ::comment Arabic inverted damma U+0657
+::s ۚ ::t j ::comment Arabic small high jeem U+06DA
+::s ۪ ::t   ::comment Arabic emtpy centre low stop U+06EA
+::s ۬ ::t   ::comment Arabic rounded high stop with filled center U+06EC 
+
+# Pashto
+::s ٙ ::t e
+
+# Hebrew
+::s ב ::t v ::comment Hebrew letter bet  ::t-alt b
+::s כ ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s ך ::t k ::comment Hebrew letter kaf  ::t-alt kh
+::s פ ::t f ::comment Hebrew letter pe  ::t-alt p
+::s ש ::t sh  ::comment Hebrew letter shin  ::t-alt s
+::s ו ::t v  ::comment Hebrew letter vav  ::t-alt o, u
+::s ח ::t ch ::comment Hebrew letter het  ::t-alt h ::use-alt-in-pointed
+::s ק ::t q ::t-alt k ::use-alt-in-pointed
+::s וֹ ::t o
+::s וּ ::t u
+::s קְוָ ::t qva ::t-alt kva  ::use-alt-in-pointed
+::s י ::t y
+::s יּ ::t y
+::s יָּ ::t ya
+::s ע ::t '
+::s ִי ::t i ::t-alt iy ::use-alt-in-pointed
+::s ֵי ::t e
+::s ִיּ ::t iy
+::s ִיָּ ::t iya
+::s ױ ::t oy
+::s א ::t a ::t-alt '
+::s אָ ::t a
+::s ֹא ::t o
+::s אַ ::t 'a
+::s אֲ ::t 'a
+::s אֶ ::t e
+::s אֱ ::t e
+::s פ ::t f
+::s פּ ::t p
+::s פַּ ::t pa
+::s פְּ ::t pe ::t-alt p ::use-alt-in-pointed 
+::s שׁ ::t sh
+::s שָׁ ::t sha
+::s שָּׁ ::t sha ::comment ?
+::s שְׁ ::t she ::t-alt sh ::use-alt-in-pointed
+::s שֶׁ ::t she
+::s שִׁ ::t shi
+::s שֻׁ ::t shu
+::s שׂ ::t s
+::s שָׂ ::t sa
+::s שְׂ ::t s ::t-alt se ::use-alt-in-pointed
+::s כּ ::t k
+::s כֶּ ::t ke
+::s כֹּ ::t ko
+::s בּ ::t b
+::s בַּ ::t ba
+::s בָּ ::t ba
+::s בְּ ::t be ::t-alt b ::use-alt-in-pointed
+::s בֶּ ::t be
+::s תּ ::t t
+::s תַּ ::t ta
+::s תֵּ ::t te
+::s תִּ ::t ti
+::s דָּ ::t da
+::s דְּ ::t de ::t-alt d ::use-alt-in-pointed
+::s גּ ::t g
+::s לֵּ ::t le
+::s ד׳ ::t dh
+::s ג׳ ::t j
+::s ת׳ ::t th
+::s ז׳ ::t zh
+::s חַ ::t ach ::comment furtive patah ::use-only-at-end-of-word
+::s עַ ::t a' ::comment furtive patah ::use-only-at-end-of-word
+::s הַּ ::t ah ::comment furtive patah ::use-only-at-end-of-word
+::s ַ ::t a ::comment Hebrew point patah
+::s ֲ ::t a ::comment Hebrew point hataf patah (hataf = reduced)
+::s ֳ ::t o ::comment Hebrew point hataf qamats
+::s ָ ::t a ::comment Hebrew point qamats ::t-alt o ::use-alt-in-pointed
+::s ֶ ::t e ::comment Hebrew point segol
+::s ֱ ::t e ::comment Hebrew point hataf segol (hataf = reduced)
+::s ְ ::t e ::comment Hebrew point sheva ::t-alt "" ::use-alt-in-pointed
+::s ֵ ::t e ::comment Hebrew point tsere
+::s ִ ::t i ::comment Hebrew point hiriq
+::s ֹ ::t o ::comment Hebrew point holam
+::s ֻ ::t u ::comment Hebrew point qubuts
+# ::s ּ ::t "" ::comment Hebrew point dagesh or mapiq
+
+# Yiddish
+::s א ::t a ::lcode yid ::comment called "silent" alef
+::s אי ::t y ::lcode yid
+::s איי ::t ey ::lcode yid
+::s או ::t u ::lcode yid
+::s אוי ::t oy ::lcode yid
+::s אַ ::t a ::lcode yid
+::s אָ ::t o ::lcode yid
+::s ב ::t b ::lcode yid
+::s בֿ ::t v ::lcode yid
+::s דזש ::t dzh ::lcode yid
+::s ו ::t u ::lcode yid
+::s וּ ::t u ::lcode yid
+::s וֹ ::t o ::lcode yid
+::s װ ::t v ::lcode yid
+::s ווא ::t wa ::lcode yid
+::s וואַ ::t wa ::lcode yid
+::s ווע ::t we ::lcode yid
+::s ווי ::t wi ::lcode yid
+::s וואוי ::t wo ::lcode yid
+::s וי ::t oy ::lcode yid
+::s זש ::t zh ::lcode yid
+::s ח ::t ch ::lcode yid
+::s טש ::t tsh ::lcode yid
+::s יִ::t i ::lcode yid
+::s יי ::t ey ::lcode yid ::comment maybe "yi" at beginning of word
+::s ײַ ::t ay ::lcode yid
+::s כּ ::t k ::lcode yid
+::s כ ::t ch ::lcode yid
+::s ך ::t ch ::lcode yid
+::s ע ::t e ::lcode yid
+::s פּ ::t p ::lcode yid
+::s פֿ ::t f ::lcode yid
+::s ף ::t f ::lcode yid ::comment sometimes p
+::s ק ::t k ::lcode yid
+::s ת ::t s ::lcode yid
+
+# Syriac/Aramaic (should be vetted by expert)
+::s ܰ ::t a  ::comment Syriac pthaha above
+::s ܲ ::t a  ::comment Syriac pthaha dotted
+::s ܳ ::t aa  ::comment Syriac zqapha above
+::s ܴ ::t aa  ::comment Syriac zqapha below
+::s ܵ ::t aa  ::comment Syriac zqapha dotted
+::s ܶ ::t e  ::comment Syriac rbasa above
+::s ܷ ::t e  ::comment Syriac rbasa below
+::s ܿ ::t o  ::comment Syriac rwaha
+::s ܸ ::t e  ::comment Syriac dotted zlama horizontal
+::s ܹ ::t e  ::comment Syriac dotted zlama angular
+::s ܺ ::t i  ::comment Syriac hbasa above
+::s ܝܺ ::t i  ::comment Syriac yudh + hbasa above
+::s ܼ ::t u  ::comment Syriac hbasa-esasa dotted
+::s ܽ ::t o  ::comment Syriac esasa above
+::s ܾ ::t u  ::comment Syriac esasa below
+::s ݇ ::t ""  ::comment Syriac oblique line above; indication of a silent letter
+
+::s ܖ ::t d  ::comment Syriac letter dotless dalath rish; ambiguous form for undifferentiated early dalath/rish
+::s ܜ ::t t  ::comment Syriac letter teth garshuni; used in Garshuni documents
+::s ܒ݂ ::t v  ::comment Syriac beth + rukkakha
+::s ܒ̥ ::t v  ::comment Syriac beth + ring-below
+::s ܓ݂ ::t g  ::comment Syriac gammal + rukkakha [IPA: ɣ]
+::s ܓ̥ ::t g  ::comment Syriac gammal + ring-below [IPA: ɣ]
+::s ܕ݂ ::t d  ::comment Syriac dalath + rukkakha [IPA: ð]
+::s ܕ̥ ::t d  ::comment Syriac dalath + ring-below [IPA: ð]
+::s ܟ݂ ::t kh  ::comment Syriac kaph + rukkakha [IPA: x]
+::s ܟ̥ ::t kh  ::comment Syriac kaph + ring-below [IPA: x]
+::s ܦ݂ ::t f  ::comment Syriac pe + rukkakha
+::s ܦ̥ ::t f  ::comment Syriac pe + ring-below
+::s ܦ݁ ::t p  ::comment Syriac pe + qushshaya
+::s ܬ݂ ::t th  ::comment Syriac taw + rukkakha [IPA: θ]
+::s ܬ̥ ::t th  ::comment Syriac taw + ring-below [IPA: θ]
+
+::s ܄ ::t : ::comment Syriac sublinear colon; used at the end of verses of supplicationscolon skewed left
+::s ܆ ::t , ::comment Syriac colon skewed left; marks a dependent clause
+::s ܇ ::t , ::comment Syriac colon skewed right; marks the end of a subdivision of the apodosis, or latter part of a Biblical verse
+
+# Uzbek
+::s ʻ ::t ' ::comment modifies pronunciation of preceding "o" and "g"
+::s ʼ ::t ' ::comment glottal stop (tutuq belgisi)
+
+# Uyghur
+::s ئا ::t a ::lcode uig
+::s ە ::t e ::lcode uig
+::s ئې ::t e ::lcode uig ::latinplus ë
+::s ې ::t e ::lcode uig ::latinplus ë
+::s ئە ::t e ::lcode uig
+::s يە ::t e ::lcode uig
+::s ئى ::t i ::lcode uig
+::s ى ::t i ::lcode uig
+::s ئو ::t o ::lcode uig
+::s و ::t o ::lcode uig
+::s ئۇ ::t u ::lcode uig
+::s ۇ ::t u ::lcode uig
+::s چ ::t ch ::t-alt q ::lcode uig
+::s خ ::t x ::lcode uig
+::s ژ ::t zh ::lcode uig
+::s ئۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ۆ ::t oe ::t-alt o ::lcode uig ::latinplus ö
+::s ئۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۈ ::t ue ::t-alt u ::lcode uig ::latinplus ü
+::s ۋ ::t w ::lcode uig
+
+# Maldivian
+::s ް ::t ::comment thaana sukun
+::s ަ ::t a ::comment thaana abafili
+::s ާ ::t aa ::comment thaana aabaafili
+::s ި ::t i ::comment thaana ibifili
+::s ީ ::t ee ::comment thaana eebeefili
+::s ު ::t u ::comment thaana ubufili
+::s ޫ ::t oo ::comment thaana ooboofili
+::s ެ ::t e ::comment thaana ebefili
+::s ޭ ::t ey ::comment thaana eybeyfili
+::s ޮ ::t o ::comment thaana obofili
+::s ޯ ::t oa ::comment thaana oaboafili
+
+# Canadian syllabics (Inuktitut)
+::s ᑊ ::t p ::comment syllable final
+::s ᐟ ::t t ::comment syllable final
+::s ᐠ ::t k ::comment syllable final
+::s ᐨ ::t c ::comment syllable final
+::s ᒼ ::t m ::comment syllable final
+::s ᐣ ::t n ::comment syllable final
+::s ᐢ ::t s ::comment syllable final
+::s ᐧ ::t y ::comment syllable final
+::s ᐤ ::t w ::comment syllable final
+::s ᐦ ::t h ::comment syllable final
+::s ᕽ ::t hk ::comment syllable final
+::s ᓫ ::t l ::comment syllable final
+::s ᕑ ::t r ::comment syllable final
+
+## Punctuation
+# delete
+::s ¿ ::t "" ::comment inverted question mark
+::s ¡ ::t "" ::comment inverted exclamation mark
+# preserve
+::s ′ ::t ′
+# Cyrillic
+::s ⁙ ::t . ::comment five dot punctuation
+# Amharic/Ethiopian
+::s ። ::t .
+::s ፣ ::t ,
+::s ፤ ::t ;
+::s ፥ ::t :
+::s ፡ ::t " " ::comment Ethiopic wordspace
+::s ፦ ::t :  ::comment Ethiopic preface colon
+::s ቸ ::t cha ::comment Ethiopic syllable ca
+::s ቹ ::t chu ::comment Ethiopic syllable cu
+::s ቺ ::t chi ::comment Ethiopic syllable ci
+::s ቻ ::t chaa ::comment Ethiopic syllable caa
+::s ቼ ::t chee ::comment Ethiopic syllable cee
+::s ች ::t che ::comment Ethiopic syllable ce
+::s ቾ ::t cho ::comment Ethiopic syllable co
+::s ሠ ::t sa ::comment Ethiopic syllable sza
+::s ሡ ::t su ::comment Ethiopic syllable szu
+::s ሢ ::t si ::comment Ethiopic syllable szi
+::s ሣ ::t saa ::comment Ethiopic syllable szaa
+::s ሤ ::t see::comment Ethiopic syllable szee
+::s ሥ ::t se ::comment Ethiopic syllable sze
+::s ሦ ::t so ::comment Ethiopic syllable szo
+::s ጠ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጡ ::t tu ::comment Ethiopic syllable thu with ejective 't'
+::s ጢ ::t ti ::comment Ethiopic syllable thi with ejective 't'
+::s ጣ ::t taa ::comment Ethiopic syllable thaa with ejective 't'
+::s ጤ ::t tee ::comment Ethiopic syllable thee with ejective 't'
+::s ጥ ::t te ::comment Ethiopic syllable the with ejective 't'
+::s ጦ ::t to ::comment Ethiopic syllable tho with ejective 't'
+
+# Devanagari (Hindi etc.)
+::s । ::t . ::comment danda
+::s ॥ ::t . ::comment double danda
+::s ৷ ::t . ::comment Bengali currency numerator four; used as danda
+::s ॰ ::t . ::comment Devanagari abbreviation sign
+# Oriya/Odia (India)
+::s ୤ ::t . ::comment danda (deprecated, should use Devanagari danda ।)
+::s ୥ ::t . ::comment double danda (deprecated, should use Devanagari double danda ॥)
+# Tibetan
+::s ། ::t ,
+::s །: ::t :
+::s ༏ ::t ;
+::s ༎ ::t .
+::s ༑ ::t ,  ::comment Tibetan mark run chen spungs shad
+::s ༼ ::t (  ::comment Tibetan open roof punctuation
+::s ༽ ::t )  ::comment Tibetan close roof punctuation
+::s ༈ ::t ""  ::comment Tibetan mark srbul shad
+::s 【 ::t [  ::comment left black lenticular bracket
+::s 】 ::t ]  ::comment right black lenticular bracket
+::s ༄ ::t ""  ::comment Tibetan head mark
+::s ༄༅ ::t ""  ::comment Tibetan head mark
+::s ༆ ::t ""  ::comment Tibetan head mark
+# Myanmar/Burmese
+::s ၊ ::t ,
+::s ။ ::t .
+Khmer
+::s ៖ ::t ; ::comment Khmer sign camnuc pii kuuh
+::s ។ ::t . ::comment Khmer sign khan
+# Arabic
+::s ، ::t ,
+::s ؛ ::t ;
+::s ٬ ::t ,
+::s ۔ ::t .
+::s ؟ ::t ?
+::s ٪ ::t %
+::s ٫ ::t , ::comment Arabic decimal separator
+::s ۽ ::t & ::comment Arabic sign Sindhi ampersand
+# Aramaic
+::s ܀ ::t .
+::s ܂ ::t .
+# Hebrew
+::s ־ ::t - ::comment maqaf
+# Armenian
+::s ։ ::t .
+::s ՝ ::t , ::comment Armenian comma
+# Chinese
+::s ， ::t ", "
+::s 、 ::t ", "
+::s 。 ::t ". "
+::s ！ ::t "! "
+::s ？ ::t "? "
+::s 「 ::t ' "'
+::s 」 ::t '" '
+::s 《 ::t ' "'
+::s 》 ::t '" '
+::s （ ::t " ("
+::s ） ::t ") "
+::s ； ::t ;
+::s ： ::t ": "
+::s ︰ ::t ": "
+::s － ::t -
+::s ／ ::t /
+::s ＝ ::t =
+::s ～ ::t ~
+::s ＆ ::t &
+::s ＜ ::t <
+::s ＞ ::t >
+::s ％ ::t %
+::s 　 ::t " " ::comment ideographic space
+# Japanese
+::s 『 ::t ' "'
+::s 』 ::t '" '
+::s ・ ::t " "  ::comment Katakana middle dot; separates name elements such as first and last name
+
+# Symbols
+::s ∞ ::t ∞  ::comment infinity
+::s ­ ::t   ::comment soft hyphen; used to indicate preferred line breaks; remove
+::s ֊ ::t -  ::comment Armenian hyphen; map to regular hyphen-minus
+::s ᐩ ::t +  ::comment Canadian syllabics final plus; map to regular plus
+::s ﹐ ::t ,  ::comment small comma; map to regular comma
+::s ˚ ::t °  ::comment ring above; map to degree sign
+::s ⇒ ::t ⇒ ::comment rightwards double arrow
+::s † ::t †  ::comment dagger
+::s • ::t •  ::comment bullet
+::s ℃ ::t °C  ::comment degree Celsius; split into 2 characters
+::s ℉ ::t °F  ::comment degree Fahrenheit; split into 2 characters
+::s ― ::t ―  ::comment horizontal bar
+::s ˇ ::t ˇ  ::comment caron (sometimes apparently used for "Arabic vowel sign small v above" U+065A, e.g. in Gilaki language (glk))
+::s ″ ::t ″  ::comment double prime
+::s ﴾ ::t (  ::comment ornate left parenthesis
+::s ﴿ ::t )  ::comment ornate right parenthesis
+::s 〔 ::t [  ::comment left tortoise shell bracket
+::s 〕 ::t ]  ::comment right tortoise shell bracket
+::s ﹝ ::t (  ::comment small left tortoise shell bracket
+::s ﹞ ::t )  ::comment small left tortoise shell bracket
+::s ♄ ::t ♄  ::comment Saturn
+::s ♆ ::t ♆  ::comment Neptune
+::s ♋ ::t ♋  ::comment Cancer

uroman/data/string-distance-cost-rules.txt

@@ -0,0 +1,896 @@
+# String distance
+
+::s1 a ::s2 ::cost 0.1
+::s1 b ::s2 ::cost 1
+::s1 b ::s2 ::cost 0.2 ::left1 /[aou]m$/ ::right1 [e] ::lc1 eng ::lc2 zho ::example Balcombe
+::s1 c ::s2 ::cost 1
+::s1 c ::s2 ::cost 0.2 ::left1 /[aeou]$/ ::right1 [cgkq] ::lc2 zho
+::s1 c ::s2 ::cost 0.5 ::left1 /[aeou][lnr]?$/ ::right1 [h] ::lc2 zho
+::s1 d ::s2 ::cost 1
+::s1 d ::s2 ::cost 0.5 ::left1 /[aeiou][lnr]$/ ::right1 [-,$ ]
+::s1 d ::s2 ::cost 0.4 ::lc1 eng ::lc2 zho ::right1 [bcfgklmnpqrstvwxz]
+::s1 e ::s2 ::cost 0.1
+::s1 é ::s2 ::cost 0.1
+::s1 e ::s2 ::cost 0.02 ::lc2 fas
+::s1 e ::s2 ::cost 0.02 ::lc1 amh ::lc2 eng
+::s1 f ::s2 ::cost 1
+::s1 g ::s2 ::cost 1
+::s1 g ::s2 ::cost 0.4 ::right1 [bcdfghklmnpqrstvwxz] ::lc2 zho
+::s1 g ::s2 ::cost 0.2 ::right1 [k] ::lc2 zho
+::s1 h ::s2 ::cost 0.5
+::s1 h ::s2 ::cost 0.1 ::left1 /[aeiouy]$/ ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 h ::s2 ::cost 0.2 ::left1 /[bdlnr]$/ ::right1 [-,$ aeiouy] ::example Delhi, Minh, Riyadh
+::s1 i ::s2 ::cost 0.1
+::s1 j ::s2 ::cost 0.5
+::s1 k ::s2 ::cost 1
+::s1 l ::s2 ::cost 1
+::s1 l ::s2 ::cost 0.3 ::left1 /eui$/ ::right1 [-,$ ] ::example Argenteuil
+::s1 l ::s2 ::cost 0.3 ::left1 /a$/ ::right1 [km] ::comment walk, palm
+::s1 l ::s2 ::cost 0.3 ::left1 /[aeiou]$/ ::right1 [bdfgkmpstvwz] ::lc2 zho
+::s1 m ::s2 ::cost 1
+::s1 n ::s2 ::cost 1
+::s1 n ::s2 ::cost 0.7 ::right1 [-,$ ]
+::s1 o ::s2 ::cost 0.1
+::s1 p ::s2 ::cost 1
+::s1 q ::s2 ::cost 1
+::s1 r ::s2 ::cost 1
+::s1 r ::s2 ::cost 0.5 ::left1 /[aou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ]
+::s1 r ::s2 ::cost 0.3 ::left1 /[aeiou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ] ::lc2 zho
+::s1 re ::s2 ::cost 0.4 ::left1 /[ou]$/ ::right1 [-,$ ] ::lc2 zho
+::s1 re ::s2 ::cost 0.5 ::left1 /[aeiou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ] ::lc2 zho
+::s1 rr ::s2 ::cost 0.5 ::left1 /[aeiou]$/ ::right1 [-,bcdfghjklmnpqrstvwxz$ ] ::lc2 zho
+::s1 s ::s2 ::cost 1
+::s1 s ::s2 ::cost 0.6 ::right1 [-,$ ]
+::s1 t ::s2 ::cost 1
+::s1 t ::s2 ::cost 0.5 ::left1 /[aeiou][lnr]?$/ ::right1 [-,$ ]
+::s1 t ::s2 ::cost 0.6 ::left1 /[bcdfghklmnpqrstvwxz]$/ ::right1 [bcdfghklmnpqrstvwxz]
+::s1 u ::s2 ::cost 0.1
+::s1 v ::s2 ::cost 1
+::s1 w ::s2 ::cost 1
+::s1 w ::s2 ::cost 0.4 ::lc1 eng ::right1 [i][c][hk][-,$ ] ::example Greenwich, Alnwick
+::s1 x ::s2 ::cost 1
+::s1 y ::s2 ::cost 0.3
+::s1 z ::s2 ::cost 1
+::s1 ı ::s2 ::cost 0.3
+::s1 0 ::s2 ::cost 1
+::s1 1 ::s2 ::cost 1
+::s1 2 ::s2 ::cost 1
+::s1 3 ::s2 ::cost 1
+::s1 4 ::s2 ::cost 1
+::s1 5 ::s2 ::cost 1
+::s1 6 ::s2 ::cost 1
+::s1 7 ::s2 ::cost 1
+::s1 8 ::s2 ::cost 1
+::s1 9 ::s2 ::cost 1
+::s1 ' ::s2 ::cost 0.1
+::s1 ` ::s2 ::cost 0.1
+::s1 ( ::s2 ::cost 0.1
+::s1 ) ::s2 ::cost 0.1
+::s1 , ::s2 ::cost 0.1
+::s1 ; ::s2 ::cost 0.1
+::s1 - ::s2 ::cost 0.1
+::s1 . ::s2 ::cost 0.1
+::s1 .. ::s2 ::cost 0.12
+::s1 ... ::s2 ::cost 0.14
+::s1 ? ::s2 ::cost 0.2
+::s1 ! ::s2 ::cost 0.2
+::s1 ‼ ::s2 ::cost 0.2
+::s1 ‼ ::s2 !! ::cost 0.02
+::s1 ‼ ::s2 ! ::cost 0.1
+::s1 / ::s2 ::cost 0.1
+::s1 : ::s2 ::cost 0.1
+::s1 ː ::s2 ::cost 0.1
+::s1 ː ::s2 : ::cost 0.1
+::s1 « ::s2 ::cost 0.1
+::s1 » ::s2 ::cost 0.1
+::s1 – ::s2 ::cost 0.1
+::s1 – ::s2 - ::cost 0.05
+::s1 — ::s2 ::cost 0.15
+::s1 — ::s2 - ::cost 0.1
+::s1 — ::s2 – ::cost 0.05
+::s1 ─ ::s2 ::cost 0.2
+::s1 ─ ::s2 - ::cost 0.15
+::s1 ─ ::s2 – ::cost 0.1
+::s1 ─ ::s2 — ::cost 0.05
+::s1 ’ ::s2 ::cost 0.1
+::s1 ʼ ::s2 ::cost 0.1
+::s1 " " ::s2 ::cost 0.1
+::s1 “ ::s2 ::cost 0.1
+::s1 ” ::s2 ::cost 0.1
+::s1 ″ ::s2 ::cost 0.1
+::s1 # ::s2 ::cost 0.3
+::s1 + ::s2 ::cost 0.3
+::s1 * ::s2 ::cost 0.3
+::s1 = ::s2 ::cost 0.3
+::s1 < ::s2 ::cost 0.3
+::s1 > ::s2 ::cost 0.3
+::s1 [ ::s2 ::cost 0.3
+::s1 ] ::s2 ::cost 0.3
+::s1 { ::s2 ::cost 0.3
+::s1 } ::s2 ::cost 0.3
+::s1 | ::s2 ::cost 0.3
+::s1 & ::s2 ::cost 0.3
+::s1 _ ::s2 ::cost 0.3
+::s1 • ::s2 ::cost 0.1
+::s1 · ::s2 ::cost 0.1
+::s1 ◦ ::s2 ::cost 0.1
+::s1 ° ::s2 ::cost 0.1
+::s1 … ::s2 ::cost 0.1
+::s1 … ::s2 ... ::cost 0
+::s1 @ ::s2 ::cost 0.3
+::s1 © ::s2 ::cost 0.3
+::s1 © ::s2 (c) ::cost 0.1
+
+
+::s1 a ::s2 aa ::cost 0.02
+::s1 a ::s2 aaa ::cost 0.03
+::s1 a ::s2 aaaa ::cost 0.03
+::s1 a ::s2 aaaaa ::cost 0.03
+::s1 a ::s2 aaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaaaa ::cost 0.04
+::s1 a ::s2 aaaaaaaaaaaaaaaa ::cost 0.04
+::s1 b ::s2 bb ::cost 0.02
+::s1 b ::s2 bbb ::cost 0.03
+::s1 b ::s2 bbbb ::cost 0.03
+::s1 b ::s2 bbbbb ::cost 0.03
+::s1 c ::s2 cc ::cost 0.02
+::s1 c ::s2 ccc ::cost 0.03
+::s1 c ::s2 cccc ::cost 0.03
+::s1 c ::s2 ccccc ::cost 0.03
+::s1 d ::s2 dd ::cost 0.02
+::s1 d ::s2 ddd ::cost 0.03
+::s1 d ::s2 dddd ::cost 0.03
+::s1 d ::s2 ddddd ::cost 0.03
+::s1 e ::s2 ee ::cost 0.02
+::s1 e ::s2 eee ::cost 0.03
+::s1 e ::s2 eeee ::cost 0.03
+::s1 e ::s2 eeeee ::cost 0.03
+::s1 e ::s2 eeeeee ::cost 0.04
+::s1 e ::s2 eeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeeeee ::cost 0.04
+::s1 e ::s2 eeeeeeeeeeeeeeee ::cost 0.04
+::s1 f ::s2 ff ::cost 0.02
+::s1 f ::s2 fff ::cost 0.03
+::s1 f ::s2 ffff ::cost 0.03
+::s1 f ::s2 fffff ::cost 0.03
+::s1 g ::s2 gg ::cost 0.02
+::s1 g ::s2 ggg ::cost 0.03
+::s1 g ::s2 gggg ::cost 0.03
+::s1 g ::s2 ggggg ::cost 0.03
+::s1 h ::s2 hh ::cost 0.02
+::s1 h ::s2 hhh ::cost 0.03
+::s1 h ::s2 hhhh ::cost 0.03
+::s1 h ::s2 hhhhh ::cost 0.03
+::s1 i ::s2 ii ::cost 0.02
+::s1 i ::s2 iii ::cost 0.03
+::s1 i ::s2 iiii ::cost 0.03
+::s1 i ::s2 iiiii ::cost 0.03
+::s1 i ::s2 iiiiii ::cost 0.04
+::s1 i ::s2 iiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiiiii ::cost 0.04
+::s1 i ::s2 iiiiiiiiiiiiiiii ::cost 0.04
+::s1 j ::s2 jj ::cost 0.02
+::s1 j ::s2 jjj ::cost 0.03
+::s1 j ::s2 jjjj ::cost 0.03
+::s1 j ::s2 jjjjj ::cost 0.03
+::s1 k ::s2 kk ::cost 0.02
+::s1 k ::s2 kkk ::cost 0.03
+::s1 k ::s2 kkkk ::cost 0.03
+::s1 k ::s2 kkkkk ::cost 0.03
+::s1 l ::s2 ll ::cost 0.02
+::s1 l ::s2 lll ::cost 0.03
+::s1 l ::s2 llll ::cost 0.03
+::s1 l ::s2 lllll ::cost 0.03
+::s1 m ::s2 mm ::cost 0.02
+::s1 m ::s2 mmm ::cost 0.03
+::s1 m ::s2 mmmm ::cost 0.03
+::s1 m ::s2 mmmmm ::cost 0.03
+::s1 n ::s2 nn ::cost 0.02
+::s1 n ::s2 nnn ::cost 0.03
+::s1 n ::s2 nnnn ::cost 0.03
+::s1 n ::s2 nnnnn ::cost 0.03
+::s1 o ::s2 oo ::cost 0.02
+::s1 o ::s2 ooo ::cost 0.03
+::s1 o ::s2 oooo ::cost 0.03
+::s1 o ::s2 ooooo ::cost 0.03
+::s1 o ::s2 oooooo ::cost 0.04
+::s1 o ::s2 ooooooo ::cost 0.04
+::s1 o ::s2 oooooooo ::cost 0.04
+::s1 o ::s2 ooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooo ::cost 0.04
+::s1 o ::s2 ooooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooooo ::cost 0.04
+::s1 o ::s2 ooooooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooooooo ::cost 0.04
+::s1 o ::s2 ooooooooooooooo ::cost 0.04
+::s1 o ::s2 oooooooooooooooo ::cost 0.04
+::s1 p ::s2 pp ::cost 0.02
+::s1 p ::s2 ppp ::cost 0.03
+::s1 p ::s2 pppp ::cost 0.03
+::s1 p ::s2 ppppp ::cost 0.03
+::s1 q ::s2 qq ::cost 0.02
+::s1 q ::s2 qqq ::cost 0.03
+::s1 q ::s2 qqqq ::cost 0.03
+::s1 q ::s2 qqqqq ::cost 0.03
+::s1 r ::s2 rr ::cost 0.02
+::s1 r ::s2 rrr ::cost 0.03
+::s1 r ::s2 rrrr ::cost 0.03
+::s1 r ::s2 rrrrr ::cost 0.03
+::s1 s ::s2 ss ::cost 0.02
+::s1 s ::s2 sss ::cost 0.03
+::s1 s ::s2 ssss ::cost 0.03
+::s1 s ::s2 sssss ::cost 0.03
+::s1 t ::s2 tt ::cost 0.02
+::s1 t ::s2 ttt ::cost 0.03
+::s1 t ::s2 tttt ::cost 0.03
+::s1 t ::s2 ttttt ::cost 0.03
+::s1 u ::s2 uu ::cost 0.02
+::s1 u ::s2 uuu ::cost 0.03
+::s1 u ::s2 uuuu ::cost 0.03
+::s1 u ::s2 uuuuu ::cost 0.03
+::s1 u ::s2 uuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuuuu ::cost 0.04
+::s1 u ::s2 uuuuuuuuuuuuuuuu ::cost 0.04
+::s1 v ::s2 vv ::cost 0.02
+::s1 v ::s2 vvv ::cost 0.03
+::s1 v ::s2 vvvv ::cost 0.03
+::s1 v ::s2 vvvvv ::cost 0.03
+::s1 w ::s2 ww ::cost 0.02
+::s1 w ::s2 www ::cost 0.03
+::s1 w ::s2 wwww ::cost 0.03
+::s1 w ::s2 wwwww ::cost 0.03
+::s1 x ::s2 xx ::cost 0.02
+::s1 x ::s2 xxx ::cost 0.03
+::s1 x ::s2 xxxx ::cost 0.03
+::s1 x ::s2 xxxxx ::cost 0.03
+::s1 y ::s2 yy ::cost 0.02
+::s1 y ::s2 yyy ::cost 0.03
+::s1 y ::s2 yyyy ::cost 0.03
+::s1 y ::s2 yyyyy ::cost 0.03
+::s1 z ::s2 zz ::cost 0.02
+::s1 z ::s2 zzz ::cost 0.03
+::s1 z ::s2 zzzz ::cost 0.03
+::s1 z ::s2 zzzzz ::cost 0.03
+::s1 " " ::s2 "  " ::cost 0
+::s1 . ::s2 ::left1 /\./ ::left2 /\./  ::cost 0.02
+::s1 … ::s2 ::left1 /…/ ::left2 /…/  ::cost 0.01
+::s1 _ ::s2 ::left1 /_/ ::left2 /_/ ::cost 0.01
+::s1 = ::s2 ::left1 /=/ ::left2 /=/ ::cost 0.01
+::s1 ! ::s2 ::left1 /!/ ::left2 /!/ ::cost 0.02
+::s1 ? ::s2 ::left1 /\?/ ::left2 /\?/ ::cost 0.02
+::s1 aa ::s2 aː ::cost 0.02
+::s1 ee ::s2 eː ::cost 0.02
+::s1 ii ::s2 iː ::cost 0.02
+::s1 oo ::s2 oː ::cost 0.02
+::s1 uu ::s2 uː ::cost 0.02
+
+::s1 a ::s2 e ::cost 0.1
+::s1 au ::s2 o ::cost 0.1 ::lc1 eng
+::s1 aw ::s2 o ::cost 0.3 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 aw ::s2 o ::cost 0.1 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 aw ::s2 a ::cost 0.2 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 ay ::s2 i ::cost 0.02 ::lc1 fas ::lc2 eng
+::s1 aye ::s2 ae ::cost 0.05 ::lc1 fas
+::s1 é ::s2 e ::cost 0.05
+::s1 e ::s2 i ::cost 0.15
+::s1 e ::s2 i ::cost 0.1 ::lc1 uig ::lc2 uig
+::s1 e ::s2 y ::cost 0.15
+::s1 ew ::s2 u ::cost 0.3 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ew ::s2 u ::cost 0.1 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 ew ::s2 u ::cost 0.3 ::right1 [aei][lgnrst] ::lc1 eng
+::s1 ew ::s2 e ::cost 0.3 ::right1 [-,bcdfghklmnpqrstvwxz$ ] ::lc1 eng
+::s1 i ::s2 a ::cost 0.1 ::right1 [-,$ ] ::lc1 fas
+::s1 i ::s2 ea ::cost 0.03 ::lc2 eng
+::s1 i ::s2 ee ::cost 0.03 ::lc2 eng
+::s1 i ::s2 ei ::cost 0.05 ::lc2 eng
+::s1 i ::s2 ie ::cost 0.03 ::lc2 eng
+::s1 i ::s2 ı ::cost 0.05 
+::s1 i ::s2 e ::cost 0.1 ::lc2 eng
+::s1 i ::s2 y ::cost 0.15
+::s1 i ::s2 y ::cost 0.1 ::right2 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ie ::s2 ei ::cost 0.15
+::s1 ie ::s2 y ::cost 0.15
+::s1 ij ::s2 ai ::cost 0.15
+::s1 o ::s2 u ::cost 0.1
+::s1 oo ::s2 u ::cost 0.1
+::s1 ow ::s2 au ::cost 0.2 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ow ::s2 o ::cost 0.2 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ow ::s2 o ::cost 0.2 ::lc1 eng ::lc2 zho ::right1 [e]
+::s1 ow ::s2 o ::cost 0.4 ::lc1 eng ::lc2 zho ::right1 [iy]
+::s1 u ::s2 a ::cost 0.1 ::lc1 eng ::right1 [-,bcdfghklmnpqrstvwxz][bcdfghklmnpqrstvwxz$ ]
+::s1 u ::s2 ou ::cost 0.05
+::s1 u ::s2 yu ::cost 0.05 ::left1 /^(.*[- ])?$/
+::s1 yeo ::s2 eo ::cost 0.1 ::lc1 fas
+
+# Amharic
+::s1 a ::s2 e ::cost 0.05 ::lc1 amh
+::s1 aa ::s2 o ::cost 0.15 ::lc1 amh
+::s1 aawe ::s2 au ::cost 0.05 ::lc1 amh
+::s1 aawe ::s2 ao ::cost 0.1 ::lc1 amh
+::s1 aawe ::s2 ou ::cost 0.1 ::lc1 amh
+::s1 aawo ::s2 ao ::cost 0.05 ::lc1 amh
+::s1 aaye ::s2 ai ::cost 0.05 ::lc1 amh
+::s1 aaye ::s2 i ::cost 0.1 ::lc1 amh
+::s1 aaye ::s2 ei ::cost 0.1 ::lc1 amh
+::s1 awe ::s2 au ::cost 0.05 ::lc1 amh
+::s1 awe ::s2 ao ::cost 0.1 ::lc1 amh
+::s1 awe ::s2 ou ::cost 0.1 ::lc1 amh
+::s1 ee ::s2 ai ::cost 0.1 ::lc1 amh
+::s1 eewo ::s2 eo ::cost 0.05 ::lc1 amh
+::s1 eeyaa ::s2 ea ::cost 0.1 ::lc1 amh
+::s1 eeye ::s2 ai ::cost 0.1 ::lc1 amh
+::s1 ewee ::s2 ue ::cost 0.1 ::lc1 amh
+::s1 gwaa ::s2 gua ::cost 0.05 ::lc1 amh
+::s1 iya ::s2 ie  ::cost 0.05 ::lc1 amh
+::s1 iyaa ::s2 ia ::cost 0.05 ::lc1 amh
+::s1 iyo ::s2 io ::cost 0.05 ::lc1 amh
+::s1 kxaa ::s2 kha ::cost 0.05 ::lc1 amh
+::s1 liyaa ::s2 llia ::cost 0.05 ::lc1 amh
+::s2 qaa ::s2 cca ::cost 0.05 ::lc1 amh
+::s1 uwaa ::s2 ua ::cost 0.05 ::lc1 amh
+::s1 uwee ::s2 ue ::cost 0.05 ::lc1 amh
+::s1 uwi ::s2 oui ::cost 0.05 ::lc1 amh
+::s1 uwi ::s2 ui ::cost 0.05 ::lc1 amh
+::s1 xaaye ::s2 hai ::cost 0.1 ::lc1 amh
+::s1 xwaa ::s2 jua ::cost 0.1 ::lc1 amh
+::s1 ziyaa ::s1 sia ::cost 0.05 ::lc1 amh
+::s1 w ::s2 ::cost 0.3 ::lc1 amh ::left1 /[aeiou]$/ ::right1 [aeiou]
+::s1 y ::s2 ::cost 0.1 ::lc1 amh ::left1 /[aeiou]$/ ::right1 [aeiou]
+# abbreviations
+::s1 ee. ::s2 a ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 si. ::s2 c ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 di. ::s2 d ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 eefe. ::s2 f ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+::s1 are. ::s2 r ::cost 0.02 ::lc1 amh ::left1 /^(.*[- ])?$/
+
+# Arabic
+::s1 ::s2 a ::cost 0.02 ::lc1 ara
+::s1 ::s2 e ::cost 0.02 ::lc1 ara
+::s1 ::s2 i ::cost 0.05 ::lc1 ara
+::s1 ::s2 o ::cost 0.05 ::lc1 ara
+::s1 ::s2 p ::cost 0.15 ::lc1 ara ::left2 /m$/ ::right2 [dfgklmnpqrstvwz]
+::s1 ::s2 u ::cost 0.05 ::lc1 ara
+::s1 y ::s2 a ::cost 0.15 ::lc1 ara
+::s1 y ::s2 e ::cost 0.05 ::lc1 ara
+::s1 y ::s2 ea ::cost 0.02 ::lc1 ara
+::s1 y ::s2 ee ::cost 0.02 ::lc1 ara
+::s1 y ::s2 i ::cost 0.02 ::lc1 ara
+::s1 y ::s2 ie ::cost 0.02 ::lc1 ara
+::s1 b ::s2 p ::cost 0.02 ::lc1 ara
+::s1 b ::s2 pp ::cost 0.03 ::lc1 ara
+::s1 f ::s2 v ::cost 0.02 ::lc1 ara
+::s1 fyl ::s2 ville ::right2 [-,$ ] ::cost 0.05 ::lc1 ara
+::s1 gh ::s2 g ::right2 [abcdfgklmnopqrstuvwz] ::cost 0.05 ::lc1 ara
+::s1 ghz ::s2 gs ::cost 0.05 ::lc1 ara
+::s1 j ::s2 g ::cost 0.2 ::lc1 ara
+::s1 kh ::s2 g ::cost 0.3 ::lc1 ara ::right2 [eiy] 
+::s1 q ::s2 g ::cost 0.2 ::lc1 ara ::right2 [arouz]
+::s1 q ::s2 gg ::cost 0.2 ::lc1 ara ::right2 [arouz]
+::s1 th ::s2 z ::cost 0.4 ::lc1 ara ::right2 [aou] ::comment Spanish
+::s1 " (" ::s2 ", " ::cost 0.02 ::lc1 ara
+::s1 ) ::s2 ::right2 [-,$ ] ::cost 0.02 ::lc1 ara
+
+# Bengali
+::s1 aoyaa ::s2 wa ::cost 0.1 ::lc1 ben
+::s1 aoye ::s2 way ::cost 0.1 ::lc1 ben
+::s1 bhaa ::s2 ve ::cost 0.1 ::lc1 ben
+::s1 bh ::s2 v ::cost 0.2 ::lc1 ben
+::s1 bh ::s2 w ::cost 0.2 ::lc1 ben
+::s1 b ::s2 v ::cost 0.3 ::lc1 ben
+::s1 b ::s2 w ::cost 0.3 ::lc1 ben
+::s1 dda ::s2 rh ::right2 [-,$ ] ::cost 0.2 ::lc1 ben
+::s1 dd ::s2 r ::cost 0.4 ::lc1 ben
+::s1 gk ::s2 k ::cost 0.05 ::lc1 ben
+::s1 h ::s2 g ::right2 [eiy] ::cost 0.4 ::lc1 ben
+::s1 h ::s2 j ::cost 0.4 ::lc1 ben
+::s1 hoyaai ::s2 whi ::cost 0.05 ::lc1 ben
+::s1 j ::s2 z ::cost 0.1 ::lc1 ben
+::s1 j ::s2 s ::cost 0.3 ::lc1 ben
+::s1 myaaka ::s2 mc ::cost 0.1 ::lc1 ben
+::s1 myaaka ::s2 mac ::cost 0.1 ::lc1 ben
+::s1 oyaa ::s2 wa ::cost 0.02 ::lc1 ben
+::s1 oyaa ::s2 wo ::cost 0.1 ::lc1 ben
+::s1 oyena ::s2 owen ::cost 0.1 ::lc1 ben
+::s1 ph ::s2 v ::cost 0.1 ::lc1 ben
+::s1 phana ::s2 von ::cost 0.1 ::lc1 ben
+::s1 rhio ::s2 gio ::cost 0.2 ::lc1 ben
+::s1 sh ::s2 s ::cost 0.4 ::lc1 ben
+::s1 ss ::s2 sh ::left1 /[k]$/ ::cost 0.15 ::lc1 ben
+::s1 ss ::s2 sh ::cost 0.3 ::lc1 ben
+::s1 o ::s2 wo ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 oye ::s2 we ::cost 0.2 ::lc1 ben
+::s1 tta ::s2 tho ::cost 0.3 ::lc1 ben
+::s1 tthaa ::s2 ta ::cost 0.3 ::lc1 ben
+::s1 u ::s2 wo ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 u ::s2 woo ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 u ::s2 wu ::cost 0.2 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 ui ::s2 wi ::cost 0.02 ::lc1 ben ::left1 /^(.*[-, ]?)$/
+::s1 yaa ::s2 wa ::cost 0.3 ::lc1 ben
+::s1 ye ::s2 we ::cost 0.3 ::lc1 ben
+
+# Russian
+::s1 ::s2 os ::cost 0.4 ::left2 /[bcdfghilmnprstvx]$/ ::right2 [-,$ ] ::lc1 rus
+::s1 ::s2 us ::cost 0.4 ::left2 /[bcdfghilmnprstvx]$/ ::right2 [-,$ ] ::lc1 rus
+::s1 av ::s2 au ::cost 0.05 ::lc1 rus
+::s1 ch ::s2 cz ::cost 0.1 ::lc1 rus ::comment Polish
+::s1 chch ::s2 cci ::right2 [aou] ::cost 0.1 ::lc1 rus
+::s1 chch ::s2 cc ::right2 [eiy] ::cost 0.1 ::lc1 rus
+::s1 chzh ::s2 zh ::cost 0.1 ::lc1 rus
+::s1 dz ::s2 zz ::cost 0.1 ::lc1 rus ::right2 [aeiouy]
+::s1 dz ::s2 j ::cost 0.3 ::lc1 rus ::right2 [aeiouy] ::comment Japanese
+::s1 dzh ::s2 g ::cost 0.05 ::lc1 rus ::right2 [eiy]
+::s1 dzh ::s2 gg ::cost 0.05 ::lc1 rus ::right2 [eiy]
+::s1 dzh ::s2 j ::cost 0.05 ::lc1 rus
+::s1 ev ::s2 eu ::cost 0.1 ::lc1 rus
+::s1 f ::s2 th ::cost 0.6 ::lc1 rus
+::s1 ievye ::s2 iaceae ::cost 0.02 ::right1 [-,$ ] ::lc1 rus ::comment scientific names for families of species 
+::s1 ii ::s2 ius ::cost 0.2 ::right1 [-,$ ] ::lc1 rus
+::s1 i ::s2 j ::cost 0.2 ::lc1 rus
+::s1 naya ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::suffix adjective
+::s1 nyi ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::suffix adjective
+::s1 ovye ::s2 aceae ::cost 0.02 ::right1 [-,$ ] ::lc1 rus ::comment scientific names for families of species
+::s1 shsh ::s2 sh ::cost 0 ::lc1 rus
+::s1 skaya ::s2 ian ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix possessive
+::s1 skaya ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix possessive
+::s1 skii ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix possessive
+::s1 skii ::s2 ian ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::suffix adjective
+::s1 tsian ::s2 tian ::cost 0.05 ::lc1 rus
+::s1 tsion ::s2 tion ::cost 0.05 ::lc1 rus
+::s1 ts ::s2 c ::cost 0.3 ::lc1 rus
+::s1 ts ::s2 c ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 tsz ::s2 z ::cost 0.1 ::lc1 rus
+::s1 itsa ::s2 ica ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 etski ::s2 ecky ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 tsiya ::s2 tion ::cost 0.02 ::right1 [-,$ ] ::lc1 rus
+::s1 tsi ::s2 qi ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 tsy ::s2 qi ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 tszi ::s2 ji ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 tszy ::s2 ji ::cost 0.15 ::lc1 rus ::comment Chinese names
+::s1 u ::s2 w ::right2 [aeio] ::cost 0.05 ::lc1 rus
+::s1 u ::s2 w ::cost 0.2 ::lc1 rus
+::s1 uo ::s2 wa ::cost 0.2 ::lc1 rus ::right2 [lnrst]
+::s1 v ::s2 u ::cost 0.05 ::lc1 rus ::left1 /[bcdfghjklmnpqrstvwxz]$/ ::right1 [aeiou] 
+::s1 gva ::s2 gua ::cost 0.02 ::lc1 rus
+::s1 gvi ::s2 gui ::cost 0.02 ::lc1 rus
+::s1 x ::s2 sh ::cost 0.2 ::left2 /[aeiou]$/ ::right2 [-,aouct$-] ::lc1 rus
+::s1 y ::s2 s ::cost 0.4 ::right2 [-,$-] ::lc1 rus
+::s1 zh ::s2 rz ::cost 0.1 ::lc1 rus ::comment Polish rz
+
+# Russian case endings
+::s1 em ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ey ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 om ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 oy ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 oyu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 y ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ya ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ye ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 yem ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ym ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ymi ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 yu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ii ::s2 iya ::cost 0.1 ::right1 [-,$ ] ::right2 [-,$ ] ::lc1 rus ::lc2 rus ::comment Russian case endings
+::s1 ii ::s2 iye ::cost 0.1 ::right1 [-,$ ] ::right2 [-,$ ] ::lc1 rus ::lc2 rus ::comment Russian case endings
+
+::s1 am ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ami ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 em ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ev ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 eri ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 eryu ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 om ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ov ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 akh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+::s1 ykh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 rus ::comment Russian case ending
+
+# Ukrainian case endings
+::s1 eyu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 oyu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 ya ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yi ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yu ::s2 ::cost 0.1 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+
+::s1 am ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 amy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 em ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 evy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 iv ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 om ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 ovy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yam ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yamy ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yiv ::s2 ::cost 0.2 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 akh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+::s1 yakh ::s2 ::cost 0.3 ::right1 [-,$ ] ::lc1 ukr ::comment Ukrainian case ending
+
+# Uyghur
+::s1 aw ::s2 ao ::cost 0.05 ::lc1 uig
+::s1 aw ::s2 au ::cost 0.05 ::lc1 uig
+::s1 gwi ::s2 gui ::cost 0.05 ::lc1 uig
+::s1 iye ::s2 ia ::cost 0.05 ::lc1 uig
+::s1 istan ::s2 ia ::cost 0.1 ::right1 [-,$ ] ::lc1 uig
+::s1 j ::s2 c ::cost 0.4 ::lc1 uig
+::s1 q ::s2 h ::cost 0.2 ::lc1 uig
+::s1 sey ::s2 cai ::cost 0.2 ::lc1 uig
+::s1 sh ::s2 x ::cost 0.2 ::lc1 uig
+
+::s1 b ::s2 p ::cost 0.3
+::s1 b ::s2 v ::cost 0.5 ::left2 /^(.*[- ])?$/
+::s1 b ::s2 v ::cost 0.7
+::s1 c ::s2 ch ::cost 0.25 ::right1 [eiy]
+::s1 c ::s2 ck ::cost 0.02 ::right1 [-,abcdfghklmnpoqrstuvwxz$ ]
+::s1 c ::s2 k ::cost 0.4
+::s1 c ::s2 k ::cost 0.05 ::left1 /^(.* )?ma?$/ ::comment MacIntyre
+::s1 c ::s2 k ::cost 0.02 ::right1 [-,abcdfghklmnpoqrstuvwxz$ ]
+::s1 c ::s2 kk ::cost 0.02 ::right1 [-,abcdfghklmnpoqrstuvwxz$ ]
+::s1 c ::s2 s ::cost 0.7
+::s1 c ::s2 s ::cost 0.1 ::right1 [eiy]
+::s1 c ::s2 ts ::cost 0.15 ::right1 [eiy]
+::s1 c ::s2 z ::cost 0.3
+::s1 ch ::s2 ck ::cost 0.2
+::s1 ch ::s2 g ::cost 0.3 ::right1 [eiy] ::right2 [eiy]
+::s1 ch ::s2 k ::cost 0.2
+::s1 ch ::s2 kk ::cost 0.2
+::s1 ch ::s2 sh ::cost 0.3
+::s1 ch ::s2 sh ::cost 0.2 ::left1 /eiy$/ ::right1 [$ ]
+::s1 ch ::s2 tch ::cost 0.1
+::s1 ch ::s2 tsh ::cost 0.1
+::s1 ch ::s2 z ::cost 0.5
+::s1 ck ::s2 kk ::cost 0.02
+::s1 cz ::s2 ch ::cost 0.2 ::left1 /i$/
+::s1 d ::s2 t ::cost 0.3
+::s1 de ::s2 dre ::cost 0.3 ::lc1 zho ::right2 [-,$ ]
+::s1 dg ::s2 j ::cost 0.6 ::lc1 eng ::comment Cambridge
+::s1 dg ::s2 j ::cost 0.3 ::right1 [eiy] ::lc1 eng
+::s1 dg ::s2 j ::cost 0.1 ::right1 [eiy] ::lc1 eng ::lc2 fas, jpn
+::s1 dt ::s2 d ::cost 0.3
+::s1 dt ::s2 t ::cost 0.03
+::s1 dt ::s2 tt ::cost 0.03
+::s1 f ::s2 p ::cost 0.8
+::s1 f ::s2 ph ::cost 0.01
+::s1 ff ::s2 ph ::cost 0.02
+::s1 f ::s2 pf ::cost 0.1
+::s1 f ::s2 v ::cost 0.3
+::s1 f ::s2 v ::cost 0.1  ::right1 [-,$ ]
+::s1 ef ::s2 ev ::cost 0.1 ::right1 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 f ::s2 w ::cost 0.3
+::s1 g ::s2 j ::cost 0.6
+::s1 g ::s2 j ::cost 0.3 ::right1 [eiy]
+::s1 g ::s2 j ::cost 0.1 ::right1 [eiy] ::lc2 amh, ara, fas, jpn, som
+::s1 g ::s2 k ::cost 0.3
+::s1 g ::s2 gh ::cost 0.3
+::s1 g ::s2 ch ::cost 0.4 ::left1 /[eiy]$/ ::right1 [-,$ ] ::comment German: Ludwig, Braunschweig
+::s1 gh ::s2 f ::cost 0.2 ::lc1 eng ::comment laughter
+::s1 gh ::s2 "" ::cost 0.2 ::lc1 eng ::comment daughter
+::s1 gh ::s2 g ::cost 0.2 ::lc1 eng ::comment Afghanistan
+::s1 gl ::s2 l ::cost 0.2 ::lc1 eng ::right1 [i]
+::s1 gn ::s2 n ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 gn ::s2 n ::cost 0.2 ::lc1 eng
+::s1 gz ::s2 ks ::cost 0.2
+::s1 h ::s2 e ::cost 0.4 ::lc1 fas
+::s1 ise ::s2 ize ::cost 0.1
+::s1 j ::s2 y ::cost 0.2
+::s1 j ::s2 dj ::cost 0.2
+::s1 j ::s2 h ::cost 0.4 ::right2 [aeiou] ::lc2 amh ::example Jose
+::s1 j ::s2 hh ::cost 0.4 ::right2 [aeiou] ::lc2 amh ::example Tardajos
+::s1 j ::s2 zh ::cost 0.2
+::s1 k ::s2 cc ::cost 0.02 ::right2 [aour]
+::s1 k ::s2 cc ::cost 0.3
+::s1 k ::s2 cch ::cost 0.15
+::s1 k ::s2 ck ::cost 0.02
+::s1 k ::s2 cq ::cost 0.05
+::s1 k ::s2 cqu ::cost 0.05
+::s1 k ::s2 cque ::cost 0.1
+::s1 k ::s2 cque ::cost 0.05 ::right2 [-,$ ]
+::s1 k ::s2 cques ::cost 0.05 ::right2 [-,$ ]
+::s1 k ::s2 q ::cost 0.05
+::s1 k ::s2 qu ::cost 0.05
+::s1 k ::s2 que ::cost 0.1
+::s1 k ::s2 que ::cost 0.05 ::right2 [-,$ ]
+::s1 k ::s2 ques ::cost 0.1 ::right2 [-,$ ]
+::s1 kh ::s2 j ::cost 0.2
+::s1 kh ::s2 q ::cost 0.2
+::s1 kh ::s2 k ::cost 0.25 ::right1 [aeiouy]
+::s1 kh ::s2 k ::cost 0.1 ::right1 [aeiouys] ::lc2 amh
+::s1 kn ::s2 n ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 kj ::s2 sh ::cost 0.2 ::comment Swedish
+::s1 l ::s2 r ::cost 0.1 ::lc1 zho
+::s1 aib ::s2 alb ::cost 0.1 ::lc1 zho
+::s1 al ::s2 ::cost 0.5 ::left1 /^(.* )?$/
+::s1 al- ::s2 ::cost 0.3 ::left1 /^(.* )?$/
+::s1 el ::s2 ::cost 0.5 ::left1 /^(.* )?$/
+::s1 el- ::s2 ::cost 0.3 ::left1 /^(.* )?$/
+::s1 ll ::s2 y ::cost 0.1 ::left1 /[aeiouy]$/ ::right1 [aeiouy] ::comment Guillermo, Guillaume 
+::s1 mb ::s2 m ::cost 0.2 ::right1 [-,bcdfghklmnpqstvwxz$ ] ::lc1 eng ::comment bomb
+::s1 n ::s2 m ::cost 0.5 ::left1 /[aeiou]$/ ::left2 /[aeiou]$/ ::right1 [bcdfghklmnpqrstvwxz$ ] ::right2 [-,bcdfghklmnpqrstvwxz$ ]
+::s1 ng ::s2 n ::cost 0.1 ::left1 /[aeiou]$/ ::lc1 zho
+::s1 ng ::s2 m ::cost 0.25 ::left1 /[aeiou]$/ ::lc1 zho
+::s1 ng ::s2 n ::cost 0.1 ::left2 /[aeiou]$/ ::lc2 ara, ben, rus, zho
+::s1 nm ::s2 m ::cost 0.25 ::lc1 zho ::left1 
+::s1 pn ::s2 n ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 ph ::s2 p ::cost 0.3 ::lc1 amh
+::s1 q ::s2 c ::cost 0.15
+::s1 q ::s2 ch ::cost 0.2 ::right2 [eiy]
+::s1 q ::s2 ck ::cost 0.2
+::s1 q ::s2 kk ::cost 0.2
+::s1 q ::s2 gh ::cost 0.2 ::lc1 fas ::right2 [aeiouy]
+::s1 qi ::s2 ch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 cci ::cost 0.1 ::lc1 zho
+::s1 qi ::s2 chi ::cost 0.1 ::lc1 zho
+::s1 qi ::s2 tch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 ts ::cost 0.4 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 tsch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 tzsch ::cost 0.2 ::lc1 zho ::right1 [aeou]
+::s1 qi ::s2 czy ::cost 0.2 ::lc1 zho
+::s1 qu ::s2 kw ::cost 0.15
+::s1 qu ::s2 kv ::cost 0.15
+::s1 e ::s2 er ::cost 0.25 ::left1 /[bcdfghklmnpqrstvwxz]$/ ::lc1 zho
+::s1 re ::s2 er ::cost 0.1
+::s1 rh ::s2 r ::cost 0.05 ::left1 /^(.*[- ])?$/ ::example Rhine
+::s1 s ::s2 sh ::cost 0.03 ::right2 [aeiou] ::lc2 amh
+::s1 s ::s2 sz ::cost 0.3 ::lc2 eng ::example Liszt (Hungarian)
+::s1 s ::s2 ts ::cost 0.4 ::lc1 amh, zho
+::s1 s ::s2 z ::cost 0.4
+::s1 s ::s2 z ::cost 0.1 ::left1 /[aeiouy]$/ ::right1 [aeiouy] ::lc1 eng
+::s1 s ::s2 z ::cost 0.1 ::left1 /[aeiouy][bdglmnrvw]?$/ ::right1 [-,$ ] ::lc1 eng
+::s1 s ::s2 z ::cost 0.2 ::lc2 fas
+::s1 sc ::s2 s ::cost 0.2 ::right1 [i] ::example Nascimento
+::s1 sci ::s2 sh ::cost 0.2 ::example Brescia
+::s1 sch ::s2 sh ::cost 0.1
+::s1 sh ::s2 sz ::cost 0.2 ::example Mariusz (Polish) ::lc2 eng
+::s1 si ::s2 j ::cost 0.1 ::right2 [a] ::lc1 eng
+::s1 ss ::s2 z ::cost 0.5
+# ::s1 smith ::s2 mith ::cost 0.75 ::lc2 zho ::comment weird, but several different Xinhua examples
+::s1 tch ::s2 c ::cost 0.2 ::left2 /[aeiou]$/ ::right2 [-,e$ ]
+::s1 te ::s2 tre ::cost 0.3 ::lc1 zho ::right2 [-,$ ]
+::s1 th ::s2 t ::cost 0.2 ::lc2 amh, fas, uig
+::s1 th ::s2 s ::cost 0.4 ::lc2 zho
+::s1 th ::s2 sth ::cost 0.4 ::lc1 zho
+::s1 th ::s2 ths ::cost 0.4 ::lc1 zho
+::s1 th ::s2 z ::cost 0.3 ::lc2 amh ::right2 [-,$ aeot]
+::s1 v ::s2 w ::cost 0.02
+::s1 v ::s2 wh ::cost 0.02 ::left1 /^(.* )?$/
+::s1 vv ::s2 w ::cost 0.02
+::s1 w ::s2 u ::cost 0.1 ::lc2 uig
+::s1 wa ::s2 ua ::cost 0.05
+::s1 wh ::s2 w ::cost 0.05 ::left1 /^(.* )?$/
+::s1 wr ::s2 r ::cost 0.05 ::left1 /^(.* )?$/ ::lc1 eng
+::s1 x ::s2 ks ::cost 0.05
+::s1 x ::s2 s ::cost 0.2 ::left1 /^(.* )?$/
+::s1 x ::s2 sh ::cost 0.2 ::lc1 uig ::left1 /^(.* )?$/ ::right1 [aeiou]
+::s1 x ::s2 z ::cost 0.2 ::left1 /^(.* )?$/ ::right1 [aeiouy]
+::s1 x ::s2 h ::cost 0.3 ::lc1 uig
+::s1 x ::s2 h ::cost 0.05 ::lc1 uig ::left1 /^(.* )?$/ ::right1 [aeiou]
+::s1 x ::s2 kh ::cost 0.1 ::lc1 uig
+::s1 xi ::s2 sch ::cost 0.2 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 sh ::cost 0.2 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 ch ::cost 0.4 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 sci ::cost 0.4 ::right1 [aeou] ::lc1 zho
+::s1 xi ::s2 s ::cost 0.6 ::right1 [aeou] ::lc1 zho
+::s1 z ::s2 dz ::cost 0.1 ::left1 /^(.*[ aeiouy])?[lnr]?$/
+::s1 z ::s2 ts ::cost 0.15
+::s1 z ::s2 tz ::cost 0.15
+::s1 zh ::s2 g ::cost 0.2 ::right2 [eiy]
+::s1 zh ::s2 g ::cost 0.1 ::right2 [eiy] ::lc2 amh
+::s1 zz ::s2 ts ::cost 0.15
+::s1 zz ::s2 tz ::cost 0.1
+
+# Oromo
+::s1 nb ::s2 mb ::cost 0.4 ::lc1 orm ::lc2 orm ::left1 /[aeiou]$/ ::left2 /[aeiou]$/
+::s1 np ::s2 mp ::cost 0.4 ::lc1 orm ::lc2 orm ::left1 /[aeiou]$/ ::left2 /[aeiou]$/
+::s1 ph ::s2 p ::cost 0.3 ::lc1 orm ::lc2 orm
+
+# Tigrinya
+::s1 aaye ::s2 a ::cost 0.4 ::lc1 tir ::lc2 tir ::left1 /[bcdfghklmnpqrstvwxz]$/ ::right1 [bcdfghklmnpqrstvwxz] ::comment internal plural
+::s1 aaye ::s2 i ::cost 0.4 ::lc1 tir ::lc2 tir ::left1 /[bcdfghklmnpqrstvwxz]$/ ::right1 [bcdfghklmnpqrstvwxz] ::comment internal plural
+
+# Somali
+::s1 ay ::s2 ey ::cost 0.1 ::lc1 som ::lc2 som
+::s1 ay ::s2 eey ::cost 0.15 ::lc1 som ::lc2 som
+::s1 aha ::s2 ihii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aha ::s2 ihi ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aha ::s2 uhu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ihii ::s2 uhu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ihi ::s2 uhu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ha ::s2 hii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ha ::s2 hi ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ha ::s2 hu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 hii ::s2 hu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 hi ::s2 hu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aka ::s2 ikii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aka ::s2 iki ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aka ::s2 uku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ikii ::s2 uku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 iki ::s2 uku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ka ::s2 kii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ka ::s2 ki ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ka ::s2 ku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 kii ::s2 ku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ki ::s2 ku ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 aga ::s2 ugu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ga ::s2 gu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 itii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 iti ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 utu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 itii ::s2 utu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 iti ::s2 utu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ta ::s2 tii ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ta ::s2 ti ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ta ::s2 tu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 tii ::s2 tu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ti ::s2 tu ::cost 0.15 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [-,$ ]
+::s1 ata ::s2 ete ::cost 0.15 ::lc1 som ::lc2 som
+::s1 ata ::s2 iti ::cost 0.2 ::lc1 som ::lc2 som
+::s1 ete ::s2 iti ::cost 0.15 ::lc1 som ::lc2 som
+::s1 g ::s2 k ::cost 0.2 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [aeiou]
+::s1 g ::s2 k ::cost 0.25 ::lc1 som ::lc2 som
+::s1 g ::s2 kh ::cost 0.25 ::lc1 som ::lc2 som
+::s1 gh ::s2 kh ::cost 0.1 ::lc1 som ::lc2 som
+::s1 gh ::s2 k ::cost 0.2 ::lc1 som ::lc2 som
+::s1 g ::s2 q ::cost 0.25 ::lc1 som ::lc2 som
+::s1 g ::s2 q ::cost 0.2 ::lc1 som ::lc2 som ::right1 [aou] ::right2 [aou]
+::s1 ga ::s2 q ::cost 0.2 ::lc1 som ::lc2 som ::left1 /^(.*[aeiou])?$/ ::left2 /^(.*[aeiou])?$/ ::right1 [bcdfghklmnpqrstvwxz] ::right2 [bcdfghklmnpqrstvwxz]
+::s1 g ::s2 j ::cost 0.25 ::lc1 som ::lc2 som
+::s1 g ::s2 j ::cost 0.15 ::lc1 som ::lc2 som ::right1 [ei] ::right2 [ei]
+::s1 gi ::s2 j ::cost 0.15 ::lc1 som ::lc2 som ::right2 [ei]
+::s1 n ::s2 m ::cost 0.2 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [aeiou]
+::s1 n ::s2 mm ::cost 0.2 ::lc1 som ::lc2 som ::right1 [-,$ ] ::right2 [aeiou]
+::s1 n ::s2 m ::cost 0.25 ::lc1 som ::lc2 som ::right2 [aeiko]
+::s1 n ::s2 mm ::cost 0.25 ::lc1 som ::lc2 som ::right2 [aeiko]
+::s1 ii ::s2 a ::cost 0.15 ::lc1 som ::lc2 som
+::s1 y ::s2 dj ::cost 0.2 ::lc2 som
+::s1 ca ::s2 a ::cost 0.15 ::left1 /^(.*[-, ])?$/ ::lc1 som
+::s1 c ::s2 ::cost 0.25 ::left1 /^(.*[-, ])?$/ ::lc1 som
+::s1 x ::s2 h ::cost 0.25 ::lc1 som
+::s1 x ::s2 h ::cost 0.05 ::lc1 som ::left1 /^(.* )?$/ ::right1 [aeiou]
+::s1 x ::s2 h ::cost 0.1 ::lc1 som ::left1 /[aeiou]$/
+::s1 b ::s2 p ::cost 0.1 ::lc1 som
+::s1 majm ::s2 mahm ::cost 0.1 ::lc1 som
+::s1 chalim ::s2 halim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 chalim ::s2 jalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 chalim ::s2 kalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 halim ::s2 jalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 halim ::s2 kalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 jalim ::s2 kalim ::cost 0.1 ::lc1 som ::lc2 som
+::s1 dh ::s2 r ::cost 0.25 ::lc1 som ::lc2 som ::left1 /[aeiou]$/
+::s1 j ::s2 ch ::cost 0.25 ::lc1 som ::lc2 som
+::s1 j ::s2 kh ::cost 0.25 ::lc1 som ::lc2 som
+::s1 ch ::s2 sh ::cost 0.2 ::lc1 som ::lc2 som
+
+# French
+::s1 aud ::s2 o ::cost 0.3 ::right1 [-,$ ] ::lc1 eng, fra
+::s1 aux ::s2 o ::cost 0.05 ::right1 [-,$ ]
+::s1 eaux ::s2 o ::cost 0.05 ::right1 [-,$ ]
+::s1 eux ::s2 o ::cost 0.05 ::right1 [-,$ ]
+::s1 eux ::s2 e ::cost 0.15 ::right1 [-,$ ]
+
+::s1 - ::s2 " " ::cost 0.1
+::s1 : ::s2 , ::cost 0.1 ::lc1 amh
+
+# mini dictionary Amharic-English
+::s1 dabube ::s2 south ::cost 0 ::lc1 amh ::lc2 eng
+::s1 daseete ::s2 island ::cost 0 ::lc1 amh ::lc2 eng
+::s1 daseetoche ::s2 islands ::cost 0 ::lc1 amh ::lc2 eng
+::s1 kaaweneti ::s2 county ::cost 0 ::lc1 amh ::lc2 eng
+::s1 katamaa ::s2 city ::cost 0 ::lc1 amh ::lc2 eng
+::s1 kelele ::s2 region ::cost 0 ::lc1 amh ::lc2 eng
+::s1 meseraaqe ::s2 east ::cost 0 ::lc1 amh ::lc2 eng
+::s1 sameene ::s2 north ::cost 0 ::lc1 amh ::lc2 eng
+::s1 setaadiyame ::s2 stadium ::cost 0 ::lc1 amh ::lc2 eng
+::s1 waneze ::s2 river ::cost 0 ::lc1 amh ::lc2 eng
+
+# mini dictionary Arabic-English
+::s1 " " ::s2 " of " ::cost 0 ::lc1 ara ::lc2 eng
+::s1 " alawl" ::s2 " i" ::cost 0 ::lc1 ara ::lc2 eng ::right2 [-,$ ]
+
+# mini dictionary Bengali-English
+::s1 anychala ::s2 zone ::cost 0 ::lc1 ben ::lc2 eng
+::s1 pradesha ::s2 province ::cost 0 ::lc1 ben ::lc2 eng
+::s1 saamraajya ::s2 empire ::cost 0 ::lc1 ben ::lc2 eng
+::s1 upajelaa ::s2 upazila ::cost 0 ::lc1 ben ::lc2 eng
+::s1 uttara ::s2 north ::cost 0 ::lc1 ben ::lc2 eng
+::s1 "dya " ::s2 "the " ::left1 /^(.*[-, ])?$/ ::cost 0.2 ::lc1 ben ::lc2 eng
+::s1 " aba " ::s2 " of " ::cost 0 ::lc1 ben ::lc2 eng
+
+# mini dictionary Russian-English
+::s1 akademiya ::s2 academy ::cost 0 ::lc1 rus ::lc2 eng
+::s1 eparkhiya ::s2 diocese ::cost 0 ::lc1 rus ::lc2 eng
+::s1 gorod ::s2 city ::cost 0 ::lc1 rus ::lc2 eng
+::s1 gosudarstvennyi ::s2 state ::cost 0 ::lc1 rus ::lc2 eng
+::s1 gubernator ::s2 governor ::cost 0 ::lc1 rus ::lc2 eng
+::s1 guberniya ::s2 governate ::cost 0 ::lc1 rus ::lc2 eng
+::s1 imperator ::s2 emperor ::cost 0 ::lc1 rus ::lc2 eng
+::s1 komitet ::s2 committee ::cost 0 ::lc1 rus ::lc2 eng
+::s1 korolevstvo ::s2 kingdom ::cost 0 ::lc1 rus ::lc2 eng
+::s1 koroli ::s2 king ::cost 0 ::lc1 rus ::lc2 eng
+::s1 mezhdunarodnaya ::s2 international ::cost 0 ::lc1 rus ::lc2 eng
+::s1 natsionalnyi ::s2 national ::cost 0 ::lc1 rus ::lc2 eng
+::s1 novyi ::s2 new ::cost 0 ::lc1 rus ::lc2 eng
+::s1 oblast ::s2 province ::cost 0 ::lc1 rus ::lc2 eng
+::s1 oblast ::s2 region ::cost 0 ::lc1 rus ::lc2 eng
+::s1 obshchestvo ::s2 society ::cost 0 ::lc1 rus ::lc2 eng
+::s1 okrug ::s2 district ::cost 0 ::lc1 rus ::lc2 eng
+::s1 okrug ::s2 region ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ostrova ::s2 island ::cost 0 ::lc1 rus ::lc2 eng
+::s1 partiya ::s2 party ::cost 0 ::lc1 rus ::lc2 eng
+::s1 raion ::s2 district ::cost 0 ::lc1 rus ::lc2 eng
+::s1 respublika ::s2 republic ::cost 0 ::lc1 rus ::lc2 eng
+::s1 respublik ::s2 republic ::cost 0 ::lc1 rus ::lc2 eng
+::s1 sbornaya ::s2 team ::cost 0 ::lc1 rus ::lc2 eng
+::s1 severnaya ::s2 north ::cost 0 ::lc1 rus ::lc2 eng
+::s1 sovet council ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 alliance ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 association ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 league ::cost 0 ::lc1 rus ::lc2 eng
+::s1 soyuz ::s2 union ::cost 0 ::lc1 rus ::lc2 eng
+::s1 svyataya ::s2 saint ::cost 0 ::lc1 rus ::lc2 eng
+::s1 svobodnyi ::s2 free ::cost 0 ::lc1 rus ::lc2 eng
+::s1 tserkov ::s2 church ::cost 0 ::lc1 rus ::lc2 eng
+::s1 uezd ::s2 county ::cost 0 ::lc1 rus ::lc2 eng
+::s1 universitet ::s2 university ::cost 0 ::lc1 rus ::lc2 eng
+::s1 vostochnaya ::s2 east ::cost 0 ::lc1 rus ::lc2 eng
+::s1 vostochnaya ::s2 eastern ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnaya ::s2 south ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnaya ::s2 southern ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnoi ::s2 south ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnoi ::s2 southern ::cost 0 ::lc1 rus ::lc2 eng
+::s1 yuzhnyi ::s2 south ::cost 0 ::lc1 rus ::lc2 eng
+# often dropped in Russian name
+::s1 ::s2 county ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 island ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 pope ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 river ::cost 0 ::lc1 rus ::lc2 eng
+::s1 ::s2 "the " ::cost 0 ::lc1 rus ::lc2 eng ::left2 /^(.*[- ])?$/
+::s1 " " ::s2 " of " ::cost 0 ::lc1 rus ::lc2 eng
+
+
+# mini dictionary Uyghur-English
+::s1 aptonom ::s2 automomous ::cost 0 ::lc1 uig ::lc2 eng
+::s1 aralliri ::s2 islands ::cost 0 ::lc1 uig ::lc2 eng
+::s1 aralliri ::s2           ::cost 0 ::lc1 uig ::lc2 eng
+::s1 arili ::s2 island ::cost 0 ::lc1 uig ::lc2 eng
+::s1 arili ::s2          ::cost 0 ::lc1 uig ::lc2 eng
+::s1 nahiyisi ::s2 county ::cost 0 ::lc1 uig ::lc2 eng
+::s1 oelkisi ::s2 province ::cost 0 ::lc1 uig ::lc2 eng
+::s1 oelkisi ::s2             ::cost 0 ::lc1 uig ::lc2 eng
+::s1 ottura ::s2 central ::cost 0 ::lc1 uig ::lc2 eng
+::s1 rayoni ::s2 region ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shehiri ::s2 city ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shehiri ::s2       ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shitati ::s2 state ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shitati ::s2         ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shtati ::s2 state ::cost 0 ::lc1 uig ::lc2 eng
+::s1 shtati ::s2         ::cost 0 ::lc1 uig ::lc2 eng
+::s1 uniwersiteti ::s2 university ::cost 0 ::lc1 uig ::lc2 eng
+::s1 yengi ::s2 new ::cost 0 ::lc1 uig ::lc2 eng
+

uroman/lib/JSON.pm

@@ -0,0 +1,2317 @@
+package JSON;
+
+
+use strict;
+use Carp ();
+use base qw(Exporter);
+@JSON::EXPORT = qw(from_json to_json jsonToObj objToJson encode_json decode_json);
+
+BEGIN {
+    $JSON::VERSION = '2.90';
+    $JSON::DEBUG   = 0 unless (defined $JSON::DEBUG);
+    $JSON::DEBUG   = $ENV{ PERL_JSON_DEBUG } if exists $ENV{ PERL_JSON_DEBUG };
+}
+
+my $Module_XS  = 'JSON::XS';
+my $Module_PP  = 'JSON::PP';
+my $Module_bp  = 'JSON::backportPP'; # included in JSON distribution
+my $PP_Version = '2.27203';
+my $XS_Version = '2.34';
+
+
+# XS and PP common methods
+
+my @PublicMethods = qw/
+    ascii latin1 utf8 pretty indent space_before space_after relaxed canonical allow_nonref 
+    allow_blessed convert_blessed filter_json_object filter_json_single_key_object 
+    shrink max_depth max_size encode decode decode_prefix allow_unknown
+/;
+
+my @Properties = qw/
+    ascii latin1 utf8 indent space_before space_after relaxed canonical allow_nonref
+    allow_blessed convert_blessed shrink max_depth max_size allow_unknown
+/;
+
+my @XSOnlyMethods = qw/allow_tags/; # Currently nothing
+
+my @PPOnlyMethods = qw/
+    indent_length sort_by
+    allow_singlequote allow_bignum loose allow_barekey escape_slash as_nonblessed
+/; # JSON::PP specific
+
+
+# used in _load_xs and _load_pp ($INSTALL_ONLY is not used currently)
+my $_INSTALL_DONT_DIE  = 1; # When _load_xs fails to load XS, don't die.
+my $_INSTALL_ONLY      = 2; # Don't call _set_methods()
+my $_ALLOW_UNSUPPORTED = 0;
+my $_UNIV_CONV_BLESSED = 0;
+my $_USSING_bpPP       = 0;
+
+
+# Check the environment variable to decide worker module. 
+
+unless ($JSON::Backend) {
+    $JSON::DEBUG and  Carp::carp("Check used worker module...");
+
+    my $backend = exists $ENV{PERL_JSON_BACKEND} ? $ENV{PERL_JSON_BACKEND} : 1;
+
+    if ($backend eq '1' or $backend =~ /JSON::XS\s*,\s*JSON::PP/) {
+        _load_xs($_INSTALL_DONT_DIE) or _load_pp();
+    }
+    elsif ($backend eq '0' or $backend eq 'JSON::PP') {
+        _load_pp();
+    }
+    elsif ($backend eq '2' or $backend eq 'JSON::XS') {
+        _load_xs();
+    }
+    elsif ($backend eq 'JSON::backportPP') {
+        $_USSING_bpPP = 1;
+        _load_pp();
+    }
+    else {
+        Carp::croak "The value of environmental variable 'PERL_JSON_BACKEND' is invalid.";
+    }
+}
+
+
+sub import {
+    my $pkg = shift;
+    my @what_to_export;
+    my $no_export;
+
+    for my $tag (@_) {
+        if ($tag eq '-support_by_pp') {
+            if (!$_ALLOW_UNSUPPORTED++) {
+                JSON::Backend::XS
+                    ->support_by_pp(@PPOnlyMethods) if ($JSON::Backend eq $Module_XS);
+            }
+            next;
+        }
+        elsif ($tag eq '-no_export') {
+            $no_export++, next;
+        }
+        elsif ( $tag eq '-convert_blessed_universally' ) {
+            eval q|
+                require B;
+                *UNIVERSAL::TO_JSON = sub {
+                    my $b_obj = B::svref_2object( $_[0] );
+                    return    $b_obj->isa('B::HV') ? { %{ $_[0] } }
+                            : $b_obj->isa('B::AV') ? [ @{ $_[0] } ]
+                            : undef
+                            ;
+                }
+            | if ( !$_UNIV_CONV_BLESSED++ );
+            next;
+        }
+        push @what_to_export, $tag;
+    }
+
+    return if ($no_export);
+
+    __PACKAGE__->export_to_level(1, $pkg, @what_to_export);
+}
+
+
+# OBSOLETED
+
+sub jsonToObj {
+    my $alternative = 'from_json';
+    if (defined $_[0] and UNIVERSAL::isa($_[0], 'JSON')) {
+        shift @_; $alternative = 'decode';
+    }
+    Carp::carp "'jsonToObj' will be obsoleted. Please use '$alternative' instead.";
+    return JSON::from_json(@_);
+};
+
+sub objToJson {
+    my $alternative = 'to_json';
+    if (defined $_[0] and UNIVERSAL::isa($_[0], 'JSON')) {
+        shift @_; $alternative = 'encode';
+    }
+    Carp::carp "'objToJson' will be obsoleted. Please use '$alternative' instead.";
+    JSON::to_json(@_);
+};
+
+
+# INTERFACES
+
+sub to_json ($@) {
+    if (
+        ref($_[0]) eq 'JSON'
+        or (@_ > 2 and $_[0] eq 'JSON')
+    ) {
+        Carp::croak "to_json should not be called as a method.";
+    }
+    my $json = JSON->new;
+
+    if (@_ == 2 and ref $_[1] eq 'HASH') {
+        my $opt  = $_[1];
+        for my $method (keys %$opt) {
+            $json->$method( $opt->{$method} );
+        }
+    }
+
+    $json->encode($_[0]);
+}
+
+
+sub from_json ($@) {
+    if ( ref($_[0]) eq 'JSON' or $_[0] eq 'JSON' ) {
+        Carp::croak "from_json should not be called as a method.";
+    }
+    my $json = JSON->new;
+
+    if (@_ == 2 and ref $_[1] eq 'HASH') {
+        my $opt  = $_[1];
+        for my $method (keys %$opt) {
+            $json->$method( $opt->{$method} );
+        }
+    }
+
+    return $json->decode( $_[0] );
+}
+
+
+
+sub true  { $JSON::true  }
+
+sub false { $JSON::false }
+
+sub null  { undef; }
+
+
+sub require_xs_version { $XS_Version; }
+
+sub backend {
+    my $proto = shift;
+    $JSON::Backend;
+}
+
+#*module = *backend;
+
+
+sub is_xs {
+    return $_[0]->backend eq $Module_XS;
+}
+
+
+sub is_pp {
+    return not $_[0]->is_xs;
+}
+
+
+sub pureperl_only_methods { @PPOnlyMethods; }
+
+
+sub property {
+    my ($self, $name, $value) = @_;
+
+    if (@_ == 1) {
+        my %props;
+        for $name (@Properties) {
+            my $method = 'get_' . $name;
+            if ($name eq 'max_size') {
+                my $value = $self->$method();
+                $props{$name} = $value == 1 ? 0 : $value;
+                next;
+            }
+            $props{$name} = $self->$method();
+        }
+        return \%props;
+    }
+    elsif (@_ > 3) {
+        Carp::croak('property() can take only the option within 2 arguments.');
+    }
+    elsif (@_ == 2) {
+        if ( my $method = $self->can('get_' . $name) ) {
+            if ($name eq 'max_size') {
+                my $value = $self->$method();
+                return $value == 1 ? 0 : $value;
+            }
+            $self->$method();
+        }
+    }
+    else {
+        $self->$name($value);
+    }
+
+}
+
+
+
+# INTERNAL
+
+sub _load_xs {
+    my $opt = shift;
+
+    $JSON::DEBUG and Carp::carp "Load $Module_XS.";
+
+    # if called after install module, overload is disable.... why?
+    JSON::Boolean::_overrride_overload($Module_XS);
+    JSON::Boolean::_overrride_overload($Module_PP);
+
+    eval qq|
+        use $Module_XS $XS_Version ();
+    |;
+
+    if ($@) {
+        if (defined $opt and $opt & $_INSTALL_DONT_DIE) {
+            $JSON::DEBUG and Carp::carp "Can't load $Module_XS...($@)";
+            return 0;
+        }
+        Carp::croak $@;
+    }
+
+    unless (defined $opt and $opt & $_INSTALL_ONLY) {
+        _set_module( $JSON::Backend = $Module_XS );
+        my $data = join("", <DATA>); # this code is from Jcode 2.xx.
+        close(DATA);
+        eval $data;
+        JSON::Backend::XS->init;
+    }
+
+    return 1;
+};
+
+
+sub _load_pp {
+    my $opt = shift;
+    my $backend = $_USSING_bpPP ? $Module_bp : $Module_PP;
+
+    $JSON::DEBUG and Carp::carp "Load $backend.";
+
+    # if called after install module, overload is disable.... why?
+    JSON::Boolean::_overrride_overload($Module_XS);
+    JSON::Boolean::_overrride_overload($backend);
+
+    if ( $_USSING_bpPP ) {
+        eval qq| require $backend |;
+    }
+    else {
+        eval qq| use $backend $PP_Version () |;
+    }
+
+    if ($@) {
+        if ( $backend eq $Module_PP ) {
+            $JSON::DEBUG and Carp::carp "Can't load $Module_PP ($@), so try to load $Module_bp";
+            $_USSING_bpPP++;
+            $backend = $Module_bp;
+            JSON::Boolean::_overrride_overload($backend);
+            local $^W; # if PP installed but invalid version, backportPP redefines methods.
+            eval qq| require $Module_bp |;
+        }
+        Carp::croak $@ if $@;
+    }
+
+    unless (defined $opt and $opt & $_INSTALL_ONLY) {
+        _set_module( $JSON::Backend = $Module_PP ); # even if backportPP, set $Backend with 'JSON::PP'
+        JSON::Backend::PP->init;
+    }
+};
+
+
+sub _set_module {
+    return if defined $JSON::true;
+
+    my $module = shift;
+
+    local $^W;
+    no strict qw(refs);
+
+    $JSON::true  = ${"$module\::true"};
+    $JSON::false = ${"$module\::false"};
+
+    push @JSON::ISA, $module;
+    if ( JSON->is_xs and JSON->backend->VERSION < 3 ) {
+        eval 'package JSON::PP::Boolean';
+        push @{"$module\::Boolean::ISA"}, qw(JSON::PP::Boolean);
+    }
+
+    *{"JSON::is_bool"} = \&{"$module\::is_bool"};
+
+    for my $method ($module eq $Module_XS ? @PPOnlyMethods : @XSOnlyMethods) {
+        *{"JSON::$method"} = sub {
+            Carp::carp("$method is not supported in $module.");
+            $_[0];
+        };
+    }
+
+    return 1;
+}
+
+
+
+#
+# JSON Boolean
+#
+
+package JSON::Boolean;
+
+my %Installed;
+
+sub _overrride_overload {
+    return; # this function is currently disable.
+    return if ($Installed{ $_[0] }++);
+
+    my $boolean = $_[0] . '::Boolean';
+
+    eval sprintf(q|
+        package %s;
+        use overload (
+            '""' => sub { ${$_[0]} == 1 ? 'true' : 'false' },
+            'eq' => sub {
+                my ($obj, $op) = ref ($_[0]) ? ($_[0], $_[1]) : ($_[1], $_[0]);
+                if ($op eq 'true' or $op eq 'false') {
+                    return "$obj" eq 'true' ? 'true' eq $op : 'false' eq $op;
+                }
+                else {
+                    return $obj ? 1 == $op : 0 == $op;
+                }
+            },
+        );
+    |, $boolean);
+
+    if ($@) { Carp::croak $@; }
+
+    if ( exists $INC{'JSON/XS.pm'} and $boolean eq 'JSON::XS::Boolean' ) {
+        local $^W;
+        my $true  = do { bless \(my $dummy = 1), $boolean };
+        my $false = do { bless \(my $dummy = 0), $boolean };
+        *JSON::XS::true  = sub () { $true };
+        *JSON::XS::false = sub () { $false };
+    }
+    elsif ( exists $INC{'JSON/PP.pm'} and $boolean eq 'JSON::PP::Boolean' ) {
+        local $^W;
+        my $true  = do { bless \(my $dummy = 1), $boolean };
+        my $false = do { bless \(my $dummy = 0), $boolean };
+        *JSON::PP::true  = sub { $true };
+        *JSON::PP::false = sub { $false };
+    }
+
+    return 1;
+}
+
+
+#
+# Helper classes for Backend Module (PP)
+#
+
+package JSON::Backend::PP;
+
+sub init {
+    local $^W;
+    no strict qw(refs); # this routine may be called after JSON::Backend::XS init was called.
+    *{"JSON::decode_json"} = \&{"JSON::PP::decode_json"};
+    *{"JSON::encode_json"} = \&{"JSON::PP::encode_json"};
+    *{"JSON::PP::is_xs"}  = sub { 0 };
+    *{"JSON::PP::is_pp"}  = sub { 1 };
+    return 1;
+}
+
+#
+# To save memory, the below lines are read only when XS backend is used.
+#
+
+package JSON;
+
+1;
+__DATA__
+
+
+#
+# Helper classes for Backend Module (XS)
+#
+
+package JSON::Backend::XS;
+
+use constant INDENT_LENGTH_FLAG => 15 << 12;
+
+use constant UNSUPPORTED_ENCODE_FLAG => {
+    ESCAPE_SLASH      => 0x00000010,
+    ALLOW_BIGNUM      => 0x00000020,
+    AS_NONBLESSED     => 0x00000040,
+    EXPANDED          => 0x10000000, # for developer's
+};
+
+use constant UNSUPPORTED_DECODE_FLAG => {
+    LOOSE             => 0x00000001,
+    ALLOW_BIGNUM      => 0x00000002,
+    ALLOW_BAREKEY     => 0x00000004,
+    ALLOW_SINGLEQUOTE => 0x00000008,
+    EXPANDED          => 0x20000000, # for developer's
+};
+
+
+sub init {
+    local $^W;
+    no strict qw(refs);
+    *{"JSON::decode_json"} = \&{"JSON::XS::decode_json"};
+    *{"JSON::encode_json"} = \&{"JSON::XS::encode_json"};
+    *{"JSON::XS::is_xs"}  = sub { 1 };
+    *{"JSON::XS::is_pp"}  = sub { 0 };
+    return 1;
+}
+
+
+sub support_by_pp {
+    my ($class, @methods) = @_;
+
+    local $^W;
+    no strict qw(refs);
+
+    my $JSON_XS_encode_orignal     = \&JSON::XS::encode;
+    my $JSON_XS_decode_orignal     = \&JSON::XS::decode;
+    my $JSON_XS_incr_parse_orignal = \&JSON::XS::incr_parse;
+
+    *JSON::XS::decode     = \&JSON::Backend::XS::Supportable::_decode;
+    *JSON::XS::encode     = \&JSON::Backend::XS::Supportable::_encode;
+    *JSON::XS::incr_parse = \&JSON::Backend::XS::Supportable::_incr_parse;
+
+    *{JSON::XS::_original_decode}     = $JSON_XS_decode_orignal;
+    *{JSON::XS::_original_encode}     = $JSON_XS_encode_orignal;
+    *{JSON::XS::_original_incr_parse} = $JSON_XS_incr_parse_orignal;
+
+    push @JSON::Backend::XS::Supportable::ISA, 'JSON';
+
+    my $pkg = 'JSON::Backend::XS::Supportable';
+
+    *{JSON::new} = sub {
+        my $proto = JSON::XS->new; $$proto = 0;
+        bless  $proto, $pkg;
+    };
+
+
+    for my $method (@methods) {
+        my $flag = uc($method);
+        my $type |= (UNSUPPORTED_ENCODE_FLAG->{$flag} || 0);
+           $type |= (UNSUPPORTED_DECODE_FLAG->{$flag} || 0);
+
+        next unless($type);
+
+        $pkg->_make_unsupported_method($method => $type);
+    }
+
+#    push @{"JSON::XS::Boolean::ISA"}, qw(JSON::PP::Boolean);
+#    push @{"JSON::PP::Boolean::ISA"}, qw(JSON::Boolean);
+
+    $JSON::DEBUG and Carp::carp("set -support_by_pp mode.");
+
+    return 1;
+}
+
+
+
+
+#
+# Helper classes for XS
+#
+
+package JSON::Backend::XS::Supportable;
+
+$Carp::Internal{'JSON::Backend::XS::Supportable'} = 1;
+
+sub _make_unsupported_method {
+    my ($pkg, $method, $type) = @_;
+
+    local $^W;
+    no strict qw(refs);
+
+    *{"$pkg\::$method"} = sub {
+        local $^W;
+        if (defined $_[1] ? $_[1] : 1) {
+            ${$_[0]} |= $type;
+        }
+        else {
+            ${$_[0]} &= ~$type;
+        }
+        $_[0];
+    };
+
+    *{"$pkg\::get_$method"} = sub {
+        ${$_[0]} & $type ? 1 : '';
+    };
+
+}
+
+
+sub _set_for_pp {
+    JSON::_load_pp( $_INSTALL_ONLY );
+
+    my $type  = shift;
+    my $pp    = JSON::PP->new;
+    my $prop = $_[0]->property;
+
+    for my $name (keys %$prop) {
+        $pp->$name( $prop->{$name} ? $prop->{$name} : 0 );
+    }
+
+    my $unsupported = $type eq 'encode' ? JSON::Backend::XS::UNSUPPORTED_ENCODE_FLAG
+                                        : JSON::Backend::XS::UNSUPPORTED_DECODE_FLAG;
+    my $flags       = ${$_[0]} || 0;
+
+    for my $name (keys %$unsupported) {
+        next if ($name eq 'EXPANDED'); # for developer's
+        my $enable = ($flags & $unsupported->{$name}) ? 1 : 0;
+        my $method = lc $name;
+        $pp->$method($enable);
+    }
+
+    $pp->indent_length( $_[0]->get_indent_length );
+
+    return $pp;
+}
+
+sub _encode { # using with PP encode
+    if (${$_[0]}) {
+        _set_for_pp('encode' => @_)->encode($_[1]);
+    }
+    else {
+        $_[0]->_original_encode( $_[1] );
+    }
+}
+
+
+sub _decode { # if unsupported-flag is set, use PP
+    if (${$_[0]}) {
+        _set_for_pp('decode' => @_)->decode($_[1]);
+    }
+    else {
+        $_[0]->_original_decode( $_[1] );
+    }
+}
+
+
+sub decode_prefix { # if unsupported-flag is set, use PP
+    _set_for_pp('decode' => @_)->decode_prefix($_[1]);
+}
+
+
+sub _incr_parse {
+    if (${$_[0]}) {
+        _set_for_pp('decode' => @_)->incr_parse($_[1]);
+    }
+    else {
+        $_[0]->_original_incr_parse( $_[1] );
+    }
+}
+
+
+sub get_indent_length {
+    ${$_[0]} << 4 >> 16;
+}
+
+
+sub indent_length {
+    my $length = $_[1];
+
+    if (!defined $length or $length > 15 or $length < 0) {
+        Carp::carp "The acceptable range of indent_length() is 0 to 15.";
+    }
+    else {
+        local $^W;
+        $length <<= 12;
+        ${$_[0]} &= ~ JSON::Backend::XS::INDENT_LENGTH_FLAG;
+        ${$_[0]} |= $length;
+        *JSON::XS::encode = \&JSON::Backend::XS::Supportable::_encode;
+    }
+
+    $_[0];
+}
+
+
+1;
+__END__
+
+=head1 NAME
+
+JSON - JSON (JavaScript Object Notation) encoder/decoder
+
+=head1 SYNOPSIS
+
+ use JSON; # imports encode_json, decode_json, to_json and from_json.
+ 
+ # simple and fast interfaces (expect/generate UTF-8)
+ 
+ $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref;
+ $perl_hash_or_arrayref  = decode_json $utf8_encoded_json_text;
+ 
+ # OO-interface
+ 
+ $json = JSON->new->allow_nonref;
+ 
+ $json_text   = $json->encode( $perl_scalar );
+ $perl_scalar = $json->decode( $json_text );
+ 
+ $pretty_printed = $json->pretty->encode( $perl_scalar ); # pretty-printing
+ 
+ # If you want to use PP only support features, call with '-support_by_pp'
+ # When XS unsupported feature is enable, using PP (de|en)code instead of XS ones.
+ 
+ use JSON -support_by_pp;
+ 
+ # option-acceptable interfaces (expect/generate UNICODE by default)
+ 
+ $json_text   = to_json( $perl_scalar, { ascii => 1, pretty => 1 } );
+ $perl_scalar = from_json( $json_text, { utf8  => 1 } );
+ 
+ # Between (en|de)code_json and (to|from)_json, if you want to write
+ # a code which communicates to an outer world (encoded in UTF-8),
+ # recommend to use (en|de)code_json.
+ 
+=head1 VERSION
+
+    2.90
+
+This version is compatible with JSON::XS B<2.34> and later.
+(Not yet compatble to JSON::XS B<3.0x>.)
+
+
+=head1 NOTE
+
+JSON::PP was earlier included in the C<JSON> distribution, but
+has since Perl 5.14 been a core module. For this reason,
+L<JSON::PP> was removed from the JSON distribution and can now
+be found also in the Perl5 repository at
+
+=over
+
+=item * L<http://perl5.git.perl.org/perl.git>
+
+=back
+
+(The newest JSON::PP version still exists in CPAN.)
+
+Instead, the C<JSON> distribution will include JSON::backportPP
+for backwards computability. JSON.pm should thus work as it did
+before.
+
+=head1 DESCRIPTION
+
+ *************************** CAUTION **************************************
+ *                                                                        *
+ * INCOMPATIBLE CHANGE (JSON::XS version 2.90)                            *
+ *                                                                        *
+ * JSON.pm had patched JSON::XS::Boolean and JSON::PP::Boolean internally *
+ * on loading time for making these modules inherit JSON::Boolean.        *
+ * But since JSON::XS v3.0 it use Types::Serialiser as boolean class.     *
+ * Then now JSON.pm breaks boolean classe overload features and           *
+ * -support_by_pp if JSON::XS v3.0 or later is installed.                 *
+ *                                                                        *
+ * JSON::true and JSON::false returned JSON::Boolean objects.             *
+ * For workaround, they return JSON::PP::Boolean objects in this version. *
+ *                                                                        *
+ *     isa_ok(JSON::true, 'JSON::PP::Boolean');                           *
+ *                                                                        *
+ * And it discards a feature:                                             *
+ *                                                                        *
+ *     ok(JSON::true eq 'true');                                          *
+ *                                                                        *
+ * In other word, JSON::PP::Boolean overload numeric only.                *
+ *                                                                        *
+ *     ok( JSON::true == 1 );                                             *
+ *                                                                        *
+ **************************************************************************
+
+ ************************** CAUTION ********************************
+ * This is 'JSON module version 2' and there are many differences  *
+ * to version 1.xx                                                 *
+ * Please check your applications using old version.              *
+ *   See to 'INCOMPATIBLE CHANGES TO OLD VERSION'                  *
+ *******************************************************************
+
+JSON (JavaScript Object Notation) is a simple data format.
+See to L<http://www.json.org/> and C<RFC4627>(L<http://www.ietf.org/rfc/rfc4627.txt>).
+
+This module converts Perl data structures to JSON and vice versa using either
+L<JSON::XS> or L<JSON::PP>.
+
+JSON::XS is the fastest and most proper JSON module on CPAN which must be
+compiled and installed in your environment.
+JSON::PP is a pure-Perl module which is bundled in this distribution and
+has a strong compatibility to JSON::XS.
+
+This module try to use JSON::XS by default and fail to it, use JSON::PP instead.
+So its features completely depend on JSON::XS or JSON::PP.
+
+See to L<BACKEND MODULE DECISION>.
+
+To distinguish the module name 'JSON' and the format type JSON,
+the former is quoted by CE<lt>E<gt> (its results vary with your using media),
+and the latter is left just as it is.
+
+Module name : C<JSON>
+
+Format type : JSON
+
+=head2 FEATURES
+
+=over
+
+=item * correct unicode handling
+
+This module (i.e. backend modules) knows how to handle Unicode, documents
+how and when it does so, and even documents what "correct" means.
+
+Even though there are limitations, this feature is available since Perl version 5.6.
+
+JSON::XS requires Perl 5.8.2 (but works correctly in 5.8.8 or later), so in older versions
+C<JSON> should call JSON::PP as the backend which can be used since Perl 5.005.
+
+With Perl 5.8.x JSON::PP works, but from 5.8.0 to 5.8.2, because of a Perl side problem,
+JSON::PP works slower in the versions. And in 5.005, the Unicode handling is not available.
+See to L<JSON::PP/UNICODE HANDLING ON PERLS> for more information.
+
+See also to L<JSON::XS/A FEW NOTES ON UNICODE AND PERL>
+and L<JSON::XS/ENCODING/CODESET_FLAG_NOTES>.
+
+
+=item * round-trip integrity
+
+When you serialise a perl data structure using only data types supported
+by JSON and Perl, the deserialised data structure is identical on the Perl
+level. (e.g. the string "2.0" doesn't suddenly become "2" just because
+it looks like a number). There I<are> minor exceptions to this, read the
+L</MAPPING> section below to learn about those.
+
+
+=item * strict checking of JSON correctness
+
+There is no guessing, no generating of illegal JSON texts by default,
+and only JSON is accepted as input by default (the latter is a security
+feature).
+
+See to L<JSON::XS/FEATURES> and L<JSON::PP/FEATURES>.
+
+=item * fast
+
+This module returns a JSON::XS object itself if available.
+Compared to other JSON modules and other serialisers such as Storable,
+JSON::XS usually compares favorably in terms of speed, too.
+
+If not available, C<JSON> returns a JSON::PP object instead of JSON::XS and
+it is very slow as pure-Perl.
+
+=item * simple to use
+
+This module has both a simple functional interface as well as an
+object oriented interface interface.
+
+=item * reasonably versatile output formats
+
+You can choose between the most compact guaranteed-single-line format possible
+(nice for simple line-based protocols), a pure-ASCII format (for when your transport
+is not 8-bit clean, still supports the whole Unicode range), or a pretty-printed
+format (for when you want to read that stuff). Or you can combine those features
+in whatever way you like.
+
+=back
+
+=head1 FUNCTIONAL INTERFACE
+
+Some documents are copied and modified from L<JSON::XS/FUNCTIONAL INTERFACE>.
+C<to_json> and C<from_json> are additional functions.
+
+=head2 encode_json
+
+    $json_text = encode_json $perl_scalar
+
+Converts the given Perl data structure to a UTF-8 encoded, binary string.
+
+This function call is functionally identical to:
+
+    $json_text = JSON->new->utf8->encode($perl_scalar)
+
+=head2 decode_json
+
+    $perl_scalar = decode_json $json_text
+
+The opposite of C<encode_json>: expects an UTF-8 (binary) string and tries
+to parse that as an UTF-8 encoded JSON text, returning the resulting
+reference.
+
+This function call is functionally identical to:
+
+    $perl_scalar = JSON->new->utf8->decode($json_text)
+
+
+=head2 to_json
+
+   $json_text = to_json($perl_scalar)
+
+Converts the given Perl data structure to a json string.
+
+This function call is functionally identical to:
+
+   $json_text = JSON->new->encode($perl_scalar)
+
+Takes a hash reference as the second.
+
+   $json_text = to_json($perl_scalar, $flag_hashref)
+
+So,
+
+   $json_text = to_json($perl_scalar, {utf8 => 1, pretty => 1})
+
+equivalent to:
+
+   $json_text = JSON->new->utf8(1)->pretty(1)->encode($perl_scalar)
+
+If you want to write a modern perl code which communicates to outer world,
+you should use C<encode_json> (supposed that JSON data are encoded in UTF-8).
+
+=head2 from_json
+
+   $perl_scalar = from_json($json_text)
+
+The opposite of C<to_json>: expects a json string and tries
+to parse it, returning the resulting reference.
+
+This function call is functionally identical to:
+
+    $perl_scalar = JSON->decode($json_text)
+
+Takes a hash reference as the second.
+
+    $perl_scalar = from_json($json_text, $flag_hashref)
+
+So,
+
+    $perl_scalar = from_json($json_text, {utf8 => 1})
+
+equivalent to:
+
+    $perl_scalar = JSON->new->utf8(1)->decode($json_text)
+
+If you want to write a modern perl code which communicates to outer world,
+you should use C<decode_json> (supposed that JSON data are encoded in UTF-8).
+
+=head2 JSON::is_bool
+
+    $is_boolean = JSON::is_bool($scalar)
+
+Returns true if the passed scalar represents either JSON::true or
+JSON::false, two constants that act like C<1> and C<0> respectively
+and are also used to represent JSON C<true> and C<false> in Perl strings.
+
+=head2 JSON::true
+
+Returns JSON true value which is blessed object.
+It C<isa> JSON::Boolean object.
+
+=head2 JSON::false
+
+Returns JSON false value which is blessed object.
+It C<isa> JSON::Boolean object.
+
+=head2 JSON::null
+
+Returns C<undef>.
+
+See L<MAPPING>, below, for more information on how JSON values are mapped to
+Perl.
+
+=head1 HOW DO I DECODE A DATA FROM OUTER AND ENCODE TO OUTER
+
+This section supposes that your perl version is 5.8 or later.
+
+If you know a JSON text from an outer world - a network, a file content, and so on,
+is encoded in UTF-8, you should use C<decode_json> or C<JSON> module object
+with C<utf8> enable. And the decoded result will contain UNICODE characters.
+
+  # from network
+  my $json        = JSON->new->utf8;
+  my $json_text   = CGI->new->param( 'json_data' );
+  my $perl_scalar = $json->decode( $json_text );
+  
+  # from file content
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  $json_text   = <$fh>;
+  $perl_scalar = decode_json( $json_text );
+
+If an outer data is not encoded in UTF-8, firstly you should C<decode> it.
+
+  use Encode;
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  my $encoding = 'cp932';
+  my $unicode_json_text = decode( $encoding, <$fh> ); # UNICODE
+  
+  # or you can write the below code.
+  #
+  # open( my $fh, "<:encoding($encoding)", 'json.data' );
+  # $unicode_json_text = <$fh>;
+
+In this case, C<$unicode_json_text> is of course UNICODE string.
+So you B<cannot> use C<decode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable or C<from_json>.
+
+  $perl_scalar = $json->utf8(0)->decode( $unicode_json_text );
+  # or
+  $perl_scalar = from_json( $unicode_json_text );
+
+Or C<encode 'utf8'> and C<decode_json>:
+
+  $perl_scalar = decode_json( encode( 'utf8', $unicode_json_text ) );
+  # this way is not efficient.
+
+And now, you want to convert your C<$perl_scalar> into JSON data and
+send it to an outer world - a network or a file content, and so on.
+
+Your data usually contains UNICODE strings and you want the converted data to be encoded
+in UTF-8, you should use C<encode_json> or C<JSON> module object with C<utf8> enable.
+
+  print encode_json( $perl_scalar ); # to a network? file? or display?
+  # or
+  print $json->utf8->encode( $perl_scalar );
+
+If C<$perl_scalar> does not contain UNICODE but C<$encoding>-encoded strings
+for some reason, then its characters are regarded as B<latin1> for perl
+(because it does not concern with your $encoding).
+You B<cannot> use C<encode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable or C<to_json>.
+Note that the resulted text is a UNICODE string but no problem to print it.
+
+  # $perl_scalar contains $encoding encoded string values
+  $unicode_json_text = $json->utf8(0)->encode( $perl_scalar );
+  # or 
+  $unicode_json_text = to_json( $perl_scalar );
+  # $unicode_json_text consists of characters less than 0x100
+  print $unicode_json_text;
+
+Or C<decode $encoding> all string values and C<encode_json>:
+
+  $perl_scalar->{ foo } = decode( $encoding, $perl_scalar->{ foo } );
+  # ... do it to each string values, then encode_json
+  $json_text = encode_json( $perl_scalar );
+
+This method is a proper way but probably not efficient.
+
+See to L<Encode>, L<perluniintro>.
+
+
+=head1 COMMON OBJECT-ORIENTED INTERFACE
+
+=head2 new
+
+    $json = JSON->new
+
+Returns a new C<JSON> object inherited from either JSON::XS or JSON::PP
+that can be used to de/encode JSON strings.
+
+All boolean flags described below are by default I<disabled>.
+
+The mutators for flags all return the JSON object again and thus calls can
+be chained:
+
+   my $json = JSON->new->utf8->space_after->encode({a => [1,2]})
+   => {"a": [1, 2]}
+
+=head2 ascii
+
+    $json = $json->ascii([$enable])
+    
+    $enabled = $json->get_ascii
+
+If $enable is true (or missing), then the encode method will not generate characters outside
+the code range 0..127. Any Unicode characters outside that range will be escaped using either
+a single \uXXXX or a double \uHHHH\uLLLLL escape sequence, as per RFC4627.
+
+If $enable is false, then the encode method will not escape Unicode characters unless
+required by the JSON syntax or other flags. This results in a faster and more compact format.
+
+This feature depends on the used Perl version and environment.
+
+See to L<JSON::PP/UNICODE HANDLING ON PERLS> if the backend is PP.
+
+  JSON->new->ascii(1)->encode([chr 0x10401])
+  => ["\ud801\udc01"]
+
+=head2 latin1
+
+    $json = $json->latin1([$enable])
+    
+    $enabled = $json->get_latin1
+
+If $enable is true (or missing), then the encode method will encode the resulting JSON
+text as latin1 (or iso-8859-1), escaping any characters outside the code range 0..255.
+
+If $enable is false, then the encode method will not escape Unicode characters
+unless required by the JSON syntax or other flags.
+
+  JSON->new->latin1->encode (["\x{89}\x{abc}"]
+  => ["\x{89}\\u0abc"]    # (perl syntax, U+abc escaped, U+89 not)
+
+=head2 utf8
+
+    $json = $json->utf8([$enable])
+    
+    $enabled = $json->get_utf8
+
+If $enable is true (or missing), then the encode method will encode the JSON result
+into UTF-8, as required by many protocols, while the decode method expects to be handled
+an UTF-8-encoded string. Please note that UTF-8-encoded strings do not contain any
+characters outside the range 0..255, they are thus useful for bytewise/binary I/O.
+
+In future versions, enabling this option might enable autodetection of the UTF-16 and UTF-32
+encoding families, as described in RFC4627.
+
+If $enable is false, then the encode method will return the JSON string as a (non-encoded)
+Unicode string, while decode expects thus a Unicode string. Any decoding or encoding
+(e.g. to UTF-8 or UTF-16) needs to be done yourself, e.g. using the Encode module.
+
+
+Example, output UTF-16BE-encoded JSON:
+
+  use Encode;
+  $jsontext = encode "UTF-16BE", JSON::XS->new->encode ($object);
+
+Example, decode UTF-32LE-encoded JSON:
+
+  use Encode;
+  $object = JSON::XS->new->decode (decode "UTF-32LE", $jsontext);
+
+See to L<JSON::PP/UNICODE HANDLING ON PERLS> if the backend is PP.
+
+
+=head2 pretty
+
+    $json = $json->pretty([$enable])
+
+This enables (or disables) all of the C<indent>, C<space_before> and
+C<space_after> (and in the future possibly more) flags in one call to
+generate the most readable (or most compact) form possible.
+
+Equivalent to:
+
+   $json->indent->space_before->space_after
+
+The indent space length is three and JSON::XS cannot change the indent
+space length.
+
+=head2 indent
+
+    $json = $json->indent([$enable])
+    
+    $enabled = $json->get_indent
+
+If C<$enable> is true (or missing), then the C<encode> method will use a multiline
+format as output, putting every array member or object/hash key-value pair
+into its own line, identifying them properly.
+
+If C<$enable> is false, no newlines or indenting will be produced, and the
+resulting JSON text is guaranteed not to contain any C<newlines>.
+
+This setting has no effect when decoding JSON texts.
+
+The indent space length is three.
+With JSON::PP, you can also access C<indent_length> to change indent space length.
+
+
+=head2 space_before
+
+    $json = $json->space_before([$enable])
+    
+    $enabled = $json->get_space_before
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space before the C<:> separating keys from values in JSON objects.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before enabled, space_after and indent disabled:
+
+   {"key" :"value"}
+
+
+=head2 space_after
+
+    $json = $json->space_after([$enable])
+    
+    $enabled = $json->get_space_after
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space after the C<:> separating keys from values in JSON objects
+and extra whitespace after the C<,> separating key-value pairs and array
+members.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before and indent disabled, space_after enabled:
+
+   {"key": "value"}
+
+
+=head2 relaxed
+
+    $json = $json->relaxed([$enable])
+    
+    $enabled = $json->get_relaxed
+
+If C<$enable> is true (or missing), then C<decode> will accept some
+extensions to normal JSON syntax (see below). C<encode> will not be
+affected in anyway. I<Be aware that this option makes you accept invalid
+JSON texts as if they were valid!>. I suggest only to use this option to
+parse application-specific files written by humans (configuration files,
+resource files etc.)
+
+If C<$enable> is false (the default), then C<decode> will only accept
+valid JSON texts.
+
+Currently accepted extensions are:
+
+=over 4
+
+=item * list items can have an end-comma
+
+JSON I<separates> array elements and key-value pairs with commas. This
+can be annoying if you write JSON texts manually and want to be able to
+quickly append elements, so this extension accepts comma at the end of
+such items not just between them:
+
+   [
+      1,
+      2, <- this comma not normally allowed
+   ]
+   {
+      "k1": "v1",
+      "k2": "v2", <- this comma not normally allowed
+   }
+
+=item * shell-style '#'-comments
+
+Whenever JSON allows whitespace, shell-style comments are additionally
+allowed. They are terminated by the first carriage-return or line-feed
+character, after which more white-space and comments are allowed.
+
+  [
+     1, # this comment not allowed in JSON
+        # neither this one...
+  ]
+
+=back
+
+
+=head2 canonical
+
+    $json = $json->canonical([$enable])
+    
+    $enabled = $json->get_canonical
+
+If C<$enable> is true (or missing), then the C<encode> method will output JSON objects
+by sorting their keys. This is adding a comparatively high overhead.
+
+If C<$enable> is false, then the C<encode> method will output key-value
+pairs in the order Perl stores them (which will likely change between runs
+of the same script).
+
+This option is useful if you want the same data structure to be encoded as
+the same JSON text (given the same overall settings). If it is disabled,
+the same hash might be encoded differently even if contains the same data,
+as key-value pairs have no inherent ordering in Perl.
+
+This setting has no effect when decoding JSON texts.
+
+=head2 allow_nonref
+
+    $json = $json->allow_nonref([$enable])
+    
+    $enabled = $json->get_allow_nonref
+
+If C<$enable> is true (or missing), then the C<encode> method can convert a
+non-reference into its corresponding string, number or null JSON value,
+which is an extension to RFC4627. Likewise, C<decode> will accept those JSON
+values instead of croaking.
+
+If C<$enable> is false, then the C<encode> method will croak if it isn't
+passed an arrayref or hashref, as JSON texts must either be an object
+or array. Likewise, C<decode> will croak if given something that is not a
+JSON object or array.
+
+   JSON->new->allow_nonref->encode ("Hello, World!")
+   => "Hello, World!"
+
+=head2 allow_unknown
+
+    $json = $json->allow_unknown ([$enable])
+    
+    $enabled = $json->get_allow_unknown
+
+If $enable is true (or missing), then "encode" will *not* throw an
+exception when it encounters values it cannot represent in JSON (for
+example, filehandles) but instead will encode a JSON "null" value.
+Note that blessed objects are not included here and are handled
+separately by c<allow_nonref>.
+
+If $enable is false (the default), then "encode" will throw an
+exception when it encounters anything it cannot encode as JSON.
+
+This option does not affect "decode" in any way, and it is
+recommended to leave it off unless you know your communications
+partner.
+
+=head2 allow_blessed
+
+    $json = $json->allow_blessed([$enable])
+    
+    $enabled = $json->get_allow_blessed
+
+If C<$enable> is true (or missing), then the C<encode> method will not
+barf when it encounters a blessed reference. Instead, the value of the
+B<convert_blessed> option will decide whether C<null> (C<convert_blessed>
+disabled or no C<TO_JSON> method found) or a representation of the
+object (C<convert_blessed> enabled and C<TO_JSON> method found) is being
+encoded. Has no effect on C<decode>.
+
+If C<$enable> is false (the default), then C<encode> will throw an
+exception when it encounters a blessed object.
+
+
+=head2 convert_blessed
+
+    $json = $json->convert_blessed([$enable])
+    
+    $enabled = $json->get_convert_blessed
+
+If C<$enable> is true (or missing), then C<encode>, upon encountering a
+blessed object, will check for the availability of the C<TO_JSON> method
+on the object's class. If found, it will be called in scalar context
+and the resulting scalar will be encoded instead of the object. If no
+C<TO_JSON> method is found, the value of C<allow_blessed> will decide what
+to do.
+
+The C<TO_JSON> method may safely call die if it wants. If C<TO_JSON>
+returns other blessed objects, those will be handled in the same
+way. C<TO_JSON> must take care of not causing an endless recursion cycle
+(== crash) in this case. The name of C<TO_JSON> was chosen because other
+methods called by the Perl core (== not by the user of the object) are
+usually in upper case letters and to avoid collisions with the C<to_json>
+function or method.
+
+This setting does not yet influence C<decode> in any way.
+
+If C<$enable> is false, then the C<allow_blessed> setting will decide what
+to do when a blessed object is found.
+
+=over
+
+=item convert_blessed_universally mode
+
+If use C<JSON> with C<-convert_blessed_universally>, the C<UNIVERSAL::TO_JSON>
+subroutine is defined as the below code:
+
+   *UNIVERSAL::TO_JSON = sub {
+       my $b_obj = B::svref_2object( $_[0] );
+       return    $b_obj->isa('B::HV') ? { %{ $_[0] } }
+               : $b_obj->isa('B::AV') ? [ @{ $_[0] } ]
+               : undef
+               ;
+   }
+
+This will cause that C<encode> method converts simple blessed objects into
+JSON objects as non-blessed object.
+
+   JSON -convert_blessed_universally;
+   $json->allow_blessed->convert_blessed->encode( $blessed_object )
+
+This feature is experimental and may be removed in the future.
+
+=back
+
+=head2 filter_json_object
+
+    $json = $json->filter_json_object([$coderef])
+
+When C<$coderef> is specified, it will be called from C<decode> each
+time it decodes a JSON object. The only argument passed to the coderef
+is a reference to the newly-created hash. If the code references returns
+a single scalar (which need not be a reference), this value
+(i.e. a copy of that scalar to avoid aliasing) is inserted into the
+deserialised data structure. If it returns an empty list
+(NOTE: I<not> C<undef>, which is a valid scalar), the original deserialised
+hash will be inserted. This setting can slow down decoding considerably.
+
+When C<$coderef> is omitted or undefined, any existing callback will
+be removed and C<decode> will not change the deserialised hash in any
+way.
+
+Example, convert all JSON objects into the integer 5:
+
+   my $js = JSON->new->filter_json_object (sub { 5 });
+   # returns [5]
+   $js->decode ('[{}]'); # the given subroutine takes a hash reference.
+   # throw an exception because allow_nonref is not enabled
+   # so a lone 5 is not allowed.
+   $js->decode ('{"a":1, "b":2}');
+
+
+=head2 filter_json_single_key_object
+
+    $json = $json->filter_json_single_key_object($key [=> $coderef])
+
+Works remotely similar to C<filter_json_object>, but is only called for
+JSON objects having a single key named C<$key>.
+
+This C<$coderef> is called before the one specified via
+C<filter_json_object>, if any. It gets passed the single value in the JSON
+object. If it returns a single value, it will be inserted into the data
+structure. If it returns nothing (not even C<undef> but the empty list),
+the callback from C<filter_json_object> will be called next, as if no
+single-key callback were specified.
+
+If C<$coderef> is omitted or undefined, the corresponding callback will be
+disabled. There can only ever be one callback for a given key.
+
+As this callback gets called less often then the C<filter_json_object>
+one, decoding speed will not usually suffer as much. Therefore, single-key
+objects make excellent targets to serialise Perl objects into, especially
+as single-key JSON objects are as close to the type-tagged value concept
+as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not
+support this in any way, so you need to make sure your data never looks
+like a serialised Perl hash.
+
+Typical names for the single object key are C<__class_whatever__>, or
+C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even
+things like C<__class_md5sum(classname)__>, to reduce the risk of clashing
+with real hashes.
+
+Example, decode JSON objects of the form C<< { "__widget__" => <id> } >>
+into the corresponding C<< $WIDGET{<id>} >> object:
+
+   # return whatever is in $WIDGET{5}:
+   JSON
+      ->new
+      ->filter_json_single_key_object (__widget__ => sub {
+            $WIDGET{ $_[0] }
+         })
+      ->decode ('{"__widget__": 5')
+
+   # this can be used with a TO_JSON method in some "widget" class
+   # for serialisation to json:
+   sub WidgetBase::TO_JSON {
+      my ($self) = @_;
+
+      unless ($self->{id}) {
+         $self->{id} = ..get..some..id..;
+         $WIDGET{$self->{id}} = $self;
+      }
+
+      { __widget__ => $self->{id} }
+   }
+
+
+=head2 shrink
+
+    $json = $json->shrink([$enable])
+    
+    $enabled = $json->get_shrink
+
+With JSON::XS, this flag resizes strings generated by either
+C<encode> or C<decode> to their minimum size possible. This can save
+memory when your JSON texts are either very very long or you have many
+short strings. It will also try to downgrade any strings to octet-form
+if possible: perl stores strings internally either in an encoding called
+UTF-X or in octet-form. The latter cannot store everything but uses less
+space in general (and some buggy Perl or C code might even rely on that
+internal representation being used).
+
+With JSON::PP, it is noop about resizing strings but tries
+C<utf8::downgrade> to the returned string by C<encode>. See to L<utf8>.
+
+See to L<JSON::XS/OBJECT-ORIENTED INTERFACE> and L<JSON::PP/METHODS>.
+
+=head2 max_depth
+
+    $json = $json->max_depth([$maximum_nesting_depth])
+    
+    $max_depth = $json->get_max_depth
+
+Sets the maximum nesting level (default C<512>) accepted while encoding
+or decoding. If a higher nesting level is detected in JSON text or a Perl
+data structure, then the encoder and decoder will stop and croak at that
+point.
+
+Nesting level is defined by number of hash- or arrayrefs that the encoder
+needs to traverse to reach a given point or the number of C<{> or C<[>
+characters without their matching closing parenthesis crossed to reach a
+given character in a string.
+
+If no argument is given, the highest possible setting will be used, which
+is rarely useful.
+
+Note that nesting is implemented by recursion in C. The default value has
+been chosen to be as large as typical operating systems allow without
+crashing. (JSON::XS)
+
+With JSON::PP as the backend, when a large value (100 or more) was set and
+it de/encodes a deep nested object/text, it may raise a warning
+'Deep recursion on subroutine' at the perl runtime phase.
+
+See L<JSON::XS/SECURITY CONSIDERATIONS> for more info on why this is useful.
+
+=head2 max_size
+
+    $json = $json->max_size([$maximum_string_size])
+    
+    $max_size = $json->get_max_size
+
+Set the maximum length a JSON text may have (in bytes) where decoding is
+being attempted. The default is C<0>, meaning no limit. When C<decode>
+is called on a string that is longer then this many bytes, it will not
+attempt to decode the string but throw an exception. This setting has no
+effect on C<encode> (yet).
+
+If no argument is given, the limit check will be deactivated (same as when
+C<0> is specified).
+
+See L<JSON::XS/SECURITY CONSIDERATIONS>, below, for more info on why this is useful.
+
+=head2 encode
+
+    $json_text = $json->encode($perl_scalar)
+
+Converts the given Perl data structure (a simple scalar or a reference
+to a hash or array) to its JSON representation. Simple scalars will be
+converted into JSON string or number sequences, while references to arrays
+become JSON arrays and references to hashes become JSON objects. Undefined
+Perl values (e.g. C<undef>) become JSON C<null> values.
+References to the integers C<0> and C<1> are converted into C<true> and C<false>.
+
+=head2 decode
+
+    $perl_scalar = $json->decode($json_text)
+
+The opposite of C<encode>: expects a JSON text and tries to parse it,
+returning the resulting simple scalar or reference. Croaks on error.
+
+JSON numbers and strings become simple Perl scalars. JSON arrays become
+Perl arrayrefs and JSON objects become Perl hashrefs. C<true> becomes
+C<1> (C<JSON::true>), C<false> becomes C<0> (C<JSON::false>) and
+C<null> becomes C<undef>.
+
+=head2 decode_prefix
+
+    ($perl_scalar, $characters) = $json->decode_prefix($json_text)
+
+This works like the C<decode> method, but instead of raising an exception
+when there is trailing garbage after the first JSON object, it will
+silently stop parsing there and return the number of characters consumed
+so far.
+
+   JSON->new->decode_prefix ("[1] the tail")
+   => ([], 3)
+
+See to L<JSON::XS/OBJECT-ORIENTED INTERFACE>
+
+=head2 property
+
+    $boolean = $json->property($property_name)
+
+Returns a boolean value about above some properties.
+
+The available properties are C<ascii>, C<latin1>, C<utf8>,
+C<indent>,C<space_before>, C<space_after>, C<relaxed>, C<canonical>,
+C<allow_nonref>, C<allow_unknown>, C<allow_blessed>, C<convert_blessed>,
+C<shrink>, C<max_depth> and C<max_size>.
+
+   $boolean = $json->property('utf8');
+    => 0
+   $json->utf8;
+   $boolean = $json->property('utf8');
+    => 1
+
+Sets the property with a given boolean value.
+
+    $json = $json->property($property_name => $boolean);
+
+With no argument, it returns all the above properties as a hash reference.
+
+    $flag_hashref = $json->property();
+
+=head1 INCREMENTAL PARSING
+
+Most of this section are copied and modified from L<JSON::XS/INCREMENTAL PARSING>.
+
+In some cases, there is the need for incremental parsing of JSON texts.
+This module does allow you to parse a JSON stream incrementally.
+It does so by accumulating text until it has a full JSON object, which
+it then can decode. This process is similar to using C<decode_prefix>
+to see if a full JSON object is available, but is much more efficient
+(and can be implemented with a minimum of method calls).
+
+The backend module will only attempt to parse the JSON text once it is sure it
+has enough text to get a decisive result, using a very simple but
+truly incremental parser. This means that it sometimes won't stop as
+early as the full parser, for example, it doesn't detect parenthesis
+mismatches. The only thing it guarantees is that it starts decoding as
+soon as a syntactically valid JSON text has been seen. This means you need
+to set resource limits (e.g. C<max_size>) to ensure the parser will stop
+parsing in the presence if syntax errors.
+
+The following methods implement this incremental parser.
+
+=head2 incr_parse
+
+    $json->incr_parse( [$string] ) # void context
+    
+    $obj_or_undef = $json->incr_parse( [$string] ) # scalar context
+    
+    @obj_or_empty = $json->incr_parse( [$string] ) # list context
+
+This is the central parsing function. It can both append new text and
+extract objects from the stream accumulated so far (both of these
+functions are optional).
+
+If C<$string> is given, then this string is appended to the already
+existing JSON fragment stored in the C<$json> object.
+
+After that, if the function is called in void context, it will simply
+return without doing anything further. This can be used to add more text
+in as many chunks as you want.
+
+If the method is called in scalar context, then it will try to extract
+exactly I<one> JSON object. If that is successful, it will return this
+object, otherwise it will return C<undef>. If there is a parse error,
+this method will croak just as C<decode> would do (one can then use
+C<incr_skip> to skip the erroneous part). This is the most common way of
+using the method.
+
+And finally, in list context, it will try to extract as many objects
+from the stream as it can find and return them, or the empty list
+otherwise. For this to work, there must be no separators between the JSON
+objects or arrays, instead they must be concatenated back-to-back. If
+an error occurs, an exception will be raised as in the scalar context
+case. Note that in this case, any previously-parsed JSON texts will be
+lost.
+
+Example: Parse some JSON arrays/objects in a given string and return them.
+
+    my @objs = JSON->new->incr_parse ("[5][7][1,2]");
+
+=head2 incr_text
+
+    $lvalue_string = $json->incr_text
+
+This method returns the currently stored JSON fragment as an lvalue, that
+is, you can manipulate it. This I<only> works when a preceding call to
+C<incr_parse> in I<scalar context> successfully returned an object. Under
+all other circumstances you must not call this function (I mean it.
+although in simple tests it might actually work, it I<will> fail under
+real world conditions). As a special exception, you can also call this
+method before having parsed anything.
+
+This function is useful in two cases: a) finding the trailing text after a
+JSON object or b) parsing multiple JSON objects separated by non-JSON text
+(such as commas).
+
+    $json->incr_text =~ s/\s*,\s*//;
+
+In Perl 5.005, C<lvalue> attribute is not available.
+You must write codes like the below:
+
+    $string = $json->incr_text;
+    $string =~ s/\s*,\s*//;
+    $json->incr_text( $string );
+
+=head2 incr_skip
+
+    $json->incr_skip
+
+This will reset the state of the incremental parser and will remove the
+parsed text from the input buffer. This is useful after C<incr_parse>
+died, in which case the input buffer and incremental parser state is left
+unchanged, to skip the text parsed so far and to reset the parse state.
+
+=head2 incr_reset
+
+    $json->incr_reset
+
+This completely resets the incremental parser, that is, after this call,
+it will be as if the parser had never parsed anything.
+
+This is useful if you want to repeatedly parse JSON objects and want to
+ignore any trailing data, which means you have to reset the parser after
+each successful decode.
+
+See to L<JSON::XS/INCREMENTAL PARSING> for examples.
+
+
+=head1 JSON::PP SUPPORT METHODS
+
+The below methods are JSON::PP own methods, so when C<JSON> works
+with JSON::PP (i.e. the created object is a JSON::PP object), available.
+See to L<JSON::PP/JSON::PP OWN METHODS> in detail.
+
+If you use C<JSON> with additional C<-support_by_pp>, some methods
+are available even with JSON::XS. See to L<USE PP FEATURES EVEN THOUGH XS BACKEND>.
+
+   BEING { $ENV{PERL_JSON_BACKEND} = 'JSON::XS' }
+   
+   use JSON -support_by_pp;
+   
+   my $json = JSON->new;
+   $json->allow_nonref->escape_slash->encode("/");
+
+   # functional interfaces too.
+   print to_json(["/"], {escape_slash => 1});
+   print from_json('["foo"]', {utf8 => 1});
+
+If you do not want to all functions but C<-support_by_pp>,
+use C<-no_export>.
+
+   use JSON -support_by_pp, -no_export;
+   # functional interfaces are not exported.
+
+=head2 allow_singlequote
+
+    $json = $json->allow_singlequote([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+any JSON strings quoted by single quotations that are invalid JSON
+format.
+
+    $json->allow_singlequote->decode({"foo":'bar'});
+    $json->allow_singlequote->decode({'foo':"bar"});
+    $json->allow_singlequote->decode({'foo':'bar'});
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+=head2 allow_barekey
+
+    $json = $json->allow_barekey([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+bare keys of JSON object that are invalid JSON format.
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+    $json->allow_barekey->decode('{foo:"bar"}');
+
+=head2 allow_bignum
+
+    $json = $json->allow_bignum([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will convert
+the big integer Perl cannot handle as integer into a L<Math::BigInt>
+object and convert a floating number (any) into a L<Math::BigFloat>.
+
+On the contrary, C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers with C<allow_blessed> enable.
+
+   $json->allow_nonref->allow_blessed->allow_bignum;
+   $bigfloat = $json->decode('2.000000000000000000000000001');
+   print $json->encode($bigfloat);
+   # => 2.000000000000000000000000001
+
+See to L<MAPPING> about the conversion of JSON number.
+
+=head2 loose
+
+    $json = $json->loose([$enable])
+
+The unescaped [\x00-\x1f\x22\x2f\x5c] strings are invalid in JSON strings
+and the module doesn't allow to C<decode> to these (except for \x2f).
+If C<$enable> is true (or missing), then C<decode>  will accept these
+unescaped strings.
+
+    $json->loose->decode(qq|["abc
+                                   def"]|);
+
+See to L<JSON::PP/JSON::PP OWN METHODS>.
+
+=head2 escape_slash
+
+    $json = $json->escape_slash([$enable])
+
+According to JSON Grammar, I<slash> (U+002F) is escaped. But by default
+JSON backend modules encode strings without escaping slash.
+
+If C<$enable> is true (or missing), then C<encode> will escape slashes.
+
+=head2 indent_length
+
+    $json = $json->indent_length($length)
+
+With JSON::XS, The indent space length is 3 and cannot be changed.
+With JSON::PP, it sets the indent space length with the given $length.
+The default is 3. The acceptable range is 0 to 15.
+
+=head2 sort_by
+
+    $json = $json->sort_by($function_name)
+    $json = $json->sort_by($subroutine_ref)
+
+If $function_name or $subroutine_ref are set, its sort routine are used.
+
+   $js = $pc->sort_by(sub { $JSON::PP::a cmp $JSON::PP::b })->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   $js = $pc->sort_by('own_sort')->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   sub JSON::PP::own_sort { $JSON::PP::a cmp $JSON::PP::b }
+
+As the sorting routine runs in the JSON::PP scope, the given
+subroutine name and the special variables C<$a>, C<$b> will begin
+with 'JSON::PP::'.
+
+If $integer is set, then the effect is same as C<canonical> on.
+
+See to L<JSON::PP/JSON::PP OWN METHODS>.
+
+=head1 MAPPING
+
+This section is copied from JSON::XS and modified to C<JSON>.
+JSON::XS and JSON::PP mapping mechanisms are almost equivalent.
+
+See to L<JSON::XS/MAPPING>.
+
+=head2 JSON -> PERL
+
+=over 4
+
+=item object
+
+A JSON object becomes a reference to a hash in Perl. No ordering of object
+keys is preserved (JSON does not preserver object key ordering itself).
+
+=item array
+
+A JSON array becomes a reference to an array in Perl.
+
+=item string
+
+A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON
+are represented by the same codepoints in the Perl string, so no manual
+decoding is necessary.
+
+=item number
+
+A JSON number becomes either an integer, numeric (floating point) or
+string scalar in perl, depending on its range and any fractional parts. On
+the Perl level, there is no difference between those as Perl handles all
+the conversion details, but an integer may take slightly less memory and
+might represent more values exactly than floating point numbers.
+
+If the number consists of digits only, C<JSON> will try to represent
+it as an integer value. If that fails, it will try to represent it as
+a numeric (floating point) value if that is possible without loss of
+precision. Otherwise it will preserve the number as a string value (in
+which case you lose roundtripping ability, as the JSON number will be
+re-encoded to a JSON string).
+
+Numbers containing a fractional or exponential part will always be
+represented as numeric (floating point) values, possibly at a loss of
+precision (in which case you might lose perfect roundtripping ability, but
+the JSON number will still be re-encoded as a JSON number).
+
+Note that precision is not accuracy - binary floating point values cannot
+represent most decimal fractions exactly, and when converting from and to
+floating point, C<JSON> only guarantees precision up to but not including
+the least significant bit.
+
+If the backend is JSON::PP and C<allow_bignum> is enable, the big integers 
+and the numeric can be optionally converted into L<Math::BigInt> and
+L<Math::BigFloat> objects.
+
+=item true, false
+
+These JSON atoms become C<JSON::true> and C<JSON::false>,
+respectively. They are overloaded to act almost exactly like the numbers
+C<1> and C<0>. You can check whether a scalar is a JSON boolean by using
+the C<JSON::is_bool> function.
+
+   print JSON::true + 1;
+    => 1
+
+   ok(JSON::true eq  '1');
+   ok(JSON::true == 1);
+
+C<JSON> will install these missing overloading features to the backend modules.
+
+
+=item null
+
+A JSON null atom becomes C<undef> in Perl.
+
+C<JSON::null> returns C<undef>.
+
+=back
+
+
+=head2 PERL -> JSON
+
+The mapping from Perl to JSON is slightly more difficult, as Perl is a
+truly typeless language, so we can only guess which JSON type is meant by
+a Perl value.
+
+=over 4
+
+=item hash references
+
+Perl hash references become JSON objects. As there is no inherent ordering
+in hash keys (or JSON objects), they will usually be encoded in a
+pseudo-random order that can change between runs of the same program but
+stays generally the same within a single run of a program. C<JSON>
+optionally sort the hash keys (determined by the I<canonical> flag), so
+the same data structure will serialise to the same JSON text (given same
+settings and version of JSON::XS), but this incurs a runtime overhead
+and is only rarely useful, e.g. when you want to compare some JSON text
+against another for equality.
+
+In future, the ordered object feature will be added to JSON::PP using C<tie> mechanism.
+
+
+=item array references
+
+Perl array references become JSON arrays.
+
+=item other references
+
+Other unblessed references are generally not allowed and will cause an
+exception to be thrown, except for references to the integers C<0> and
+C<1>, which get turned into C<false> and C<true> atoms in JSON. You can
+also use C<JSON::false> and C<JSON::true> to improve readability.
+
+   to_json [\0,JSON::true]      # yields [false,true]
+
+=item JSON::true, JSON::false, JSON::null
+
+These special values become JSON true and JSON false values,
+respectively. You can also use C<\1> and C<\0> directly if you want.
+
+JSON::null returns C<undef>.
+
+=item blessed objects
+
+Blessed objects are not directly representable in JSON. See the
+C<allow_blessed> and C<convert_blessed> methods on various options on
+how to deal with this: basically, you can choose between throwing an
+exception, encoding the reference as if it weren't blessed, or provide
+your own serialiser method.
+
+With C<convert_blessed_universally> mode,  C<encode> converts blessed
+hash references or blessed array references (contains other blessed references)
+into JSON members and arrays.
+
+   use JSON -convert_blessed_universally;
+   JSON->new->allow_blessed->convert_blessed->encode( $blessed_object );
+
+See to L<convert_blessed>.
+
+=item simple scalars
+
+Simple Perl scalars (any scalar that is not a reference) are the most
+difficult objects to encode: JSON::XS and JSON::PP will encode undefined scalars as
+JSON C<null> values, scalars that have last been used in a string context
+before encoding as JSON strings, and anything else as number value:
+
+   # dump as number
+   encode_json [2]                      # yields [2]
+   encode_json [-3.0e17]                # yields [-3e+17]
+   my $value = 5; encode_json [$value]  # yields [5]
+
+   # used as string, so dump as string
+   print $value;
+   encode_json [$value]                 # yields ["5"]
+
+   # undef becomes null
+   encode_json [undef]                  # yields [null]
+
+You can force the type to be a string by stringifying it:
+
+   my $x = 3.1; # some variable containing a number
+   "$x";        # stringified
+   $x .= "";    # another, more awkward way to stringify
+   print $x;    # perl does it for you, too, quite often
+
+You can force the type to be a number by numifying it:
+
+   my $x = "3"; # some variable containing a string
+   $x += 0;     # numify it, ensuring it will be dumped as a number
+   $x *= 1;     # same thing, the choice is yours.
+
+You can not currently force the type in other, less obscure, ways.
+
+Note that numerical precision has the same meaning as under Perl (so
+binary to decimal conversion follows the same rules as in Perl, which
+can differ to other languages). Also, your perl interpreter might expose
+extensions to the floating point numbers of your platform, such as
+infinities or NaN's - these cannot be represented in JSON, and it is an
+error to pass those in.
+
+=item Big Number
+
+If the backend is JSON::PP and C<allow_bignum> is enable, 
+C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers.
+
+
+=back
+
+=head1 JSON and ECMAscript
+
+See to L<JSON::XS/JSON and ECMAscript>.
+
+=head1 JSON and YAML
+
+JSON is not a subset of YAML.
+See to L<JSON::XS/JSON and YAML>.
+
+
+=head1 BACKEND MODULE DECISION
+
+When you use C<JSON>, C<JSON> tries to C<use> JSON::XS. If this call failed, it will
+C<uses> JSON::PP. The required JSON::XS version is I<2.2> or later.
+
+The C<JSON> constructor method returns an object inherited from the backend module,
+and JSON::XS object is a blessed scalar reference while JSON::PP is a blessed hash
+reference.
+
+So, your program should not depend on the backend module, especially
+returned objects should not be modified.
+
+ my $json = JSON->new; # XS or PP?
+ $json->{stash} = 'this is xs object'; # this code may raise an error!
+
+To check the backend module, there are some methods - C<backend>, C<is_pp> and C<is_xs>.
+
+  JSON->backend; # 'JSON::XS' or 'JSON::PP'
+  
+  JSON->backend->is_pp: # 0 or 1
+  
+  JSON->backend->is_xs: # 1 or 0
+  
+  $json->is_xs; # 1 or 0
+  
+  $json->is_pp; # 0 or 1
+
+
+If you set an environment variable C<PERL_JSON_BACKEND>, the calling action will be changed.
+
+=over
+
+=item PERL_JSON_BACKEND = 0 or PERL_JSON_BACKEND = 'JSON::PP'
+
+Always use JSON::PP
+
+=item PERL_JSON_BACKEND == 1 or PERL_JSON_BACKEND = 'JSON::XS,JSON::PP'
+
+(The default) Use compiled JSON::XS if it is properly compiled & installed,
+otherwise use JSON::PP.
+
+=item PERL_JSON_BACKEND == 2 or PERL_JSON_BACKEND = 'JSON::XS'
+
+Always use compiled JSON::XS, die if it isn't properly compiled & installed.
+
+=item PERL_JSON_BACKEND = 'JSON::backportPP'
+
+Always use JSON::backportPP.
+JSON::backportPP is JSON::PP back port module.
+C<JSON> includes JSON::backportPP instead of JSON::PP.
+
+=back
+
+These ideas come from L<DBI::PurePerl> mechanism.
+
+example:
+
+ BEGIN { $ENV{PERL_JSON_BACKEND} = 'JSON::PP' }
+ use JSON; # always uses JSON::PP
+
+In future, it may be able to specify another module.
+
+=head1 USE PP FEATURES EVEN THOUGH XS BACKEND
+
+Many methods are available with either JSON::XS or JSON::PP and
+when the backend module is JSON::XS, if any JSON::PP specific (i.e. JSON::XS unsupported)
+method is called, it will C<warn> and be noop.
+
+But If you C<use> C<JSON> passing the optional string C<-support_by_pp>,
+it makes a part of those unsupported methods available.
+This feature is achieved by using JSON::PP in C<de/encode>.
+
+   BEGIN { $ENV{PERL_JSON_BACKEND} = 2 } # with JSON::XS
+   use JSON -support_by_pp;
+   my $json = JSON->new;
+   $json->allow_nonref->escape_slash->encode("/");
+
+At this time, the returned object is a C<JSON::Backend::XS::Supportable>
+object (re-blessed XS object), and  by checking JSON::XS unsupported flags
+in de/encoding, can support some unsupported methods - C<loose>, C<allow_bignum>,
+C<allow_barekey>, C<allow_singlequote>, C<escape_slash> and C<indent_length>.
+
+When any unsupported methods are not enable, C<XS de/encode> will be
+used as is. The switch is achieved by changing the symbolic tables.
+
+C<-support_by_pp> is effective only when the backend module is JSON::XS
+and it makes the de/encoding speed down a bit.
+
+See to L<JSON::PP SUPPORT METHODS>.
+
+=head1 INCOMPATIBLE CHANGES TO OLD VERSION
+
+There are big incompatibility between new version (2.00) and old (1.xx).
+If you use old C<JSON> 1.xx in your code, please check it.
+
+See to L<Transition ways from 1.xx to 2.xx.>
+
+=over
+
+=item jsonToObj and objToJson are obsoleted.
+
+Non Perl-style name C<jsonToObj> and C<objToJson> are obsoleted
+(but not yet deleted from the source).
+If you use these functions in your code, please replace them
+with C<from_json> and C<to_json>.
+
+
+=item Global variables are no longer available.
+
+C<JSON> class variables - C<$JSON::AUTOCONVERT>, C<$JSON::BareKey>, etc...
+- are not available any longer.
+Instead, various features can be used through object methods.
+
+
+=item Package JSON::Converter and JSON::Parser are deleted.
+
+Now C<JSON> bundles with JSON::PP which can handle JSON more properly than them.
+
+=item Package JSON::NotString is deleted.
+
+There was C<JSON::NotString> class which represents JSON value C<true>, C<false>, C<null>
+and numbers. It was deleted and replaced by C<JSON::Boolean>.
+
+C<JSON::Boolean> represents C<true> and C<false>.
+
+C<JSON::Boolean> does not represent C<null>.
+
+C<JSON::null> returns C<undef>.
+
+C<JSON> makes L<JSON::XS::Boolean> and L<JSON::PP::Boolean> is-a relation
+to L<JSON::Boolean>.
+
+=item function JSON::Number is obsoleted.
+
+C<JSON::Number> is now needless because JSON::XS and JSON::PP have
+round-trip integrity.
+
+=item JSONRPC modules are deleted.
+
+Perl implementation of JSON-RPC protocol - C<JSONRPC >, C<JSONRPC::Transport::HTTP>
+and C<Apache::JSONRPC > are deleted in this distribution.
+Instead of them, there is L<JSON::RPC> which supports JSON-RPC protocol version 1.1.
+
+=back
+
+=head2 Transition ways from 1.xx to 2.xx.
+
+You should set C<suport_by_pp> mode firstly, because
+it is always successful for the below codes even with JSON::XS.
+
+    use JSON -support_by_pp;
+
+=over
+
+=item Exported jsonToObj (simple)
+
+  from_json($json_text);
+
+=item Exported objToJson (simple)
+
+  to_json($perl_scalar);
+
+=item Exported jsonToObj (advanced)
+
+  $flags = {allow_barekey => 1, allow_singlequote => 1};
+  from_json($json_text, $flags);
+
+equivalent to:
+
+  $JSON::BareKey = 1;
+  $JSON::QuotApos = 1;
+  jsonToObj($json_text);
+
+=item Exported objToJson (advanced)
+
+  $flags = {allow_blessed => 1, allow_barekey => 1};
+  to_json($perl_scalar, $flags);
+
+equivalent to:
+
+  $JSON::BareKey = 1;
+  objToJson($perl_scalar);
+
+=item jsonToObj as object method
+
+  $json->decode($json_text);
+
+=item objToJson as object method
+
+  $json->encode($perl_scalar);
+
+=item new method with parameters
+
+The C<new> method in 2.x takes any parameters no longer.
+You can set parameters instead;
+
+   $json = JSON->new->pretty;
+
+=item $JSON::Pretty, $JSON::Indent, $JSON::Delimiter
+
+If C<indent> is enable, that means C<$JSON::Pretty> flag set. And
+C<$JSON::Delimiter> was substituted by C<space_before> and C<space_after>.
+In conclusion:
+
+   $json->indent->space_before->space_after;
+
+Equivalent to:
+
+  $json->pretty;
+
+To change indent length, use C<indent_length>.
+
+(Only with JSON::PP, if C<-support_by_pp> is not used.)
+
+  $json->pretty->indent_length(2)->encode($perl_scalar);
+
+=item $JSON::BareKey
+
+(Only with JSON::PP, if C<-support_by_pp> is not used.)
+
+  $json->allow_barekey->decode($json_text)
+
+=item $JSON::ConvBlessed
+
+use C<-convert_blessed_universally>. See to L<convert_blessed>.
+
+=item $JSON::QuotApos
+
+(Only with JSON::PP, if C<-support_by_pp> is not used.)
+
+  $json->allow_singlequote->decode($json_text)
+
+=item $JSON::SingleQuote
+
+Disable. C<JSON> does not make such a invalid JSON string any longer.
+
+=item $JSON::KeySort
+
+  $json->canonical->encode($perl_scalar)
+
+This is the ascii sort.
+
+If you want to use with your own sort routine, check the C<sort_by> method.
+
+(Only with JSON::PP, even if C<-support_by_pp> is used currently.)
+
+  $json->sort_by($sort_routine_ref)->encode($perl_scalar)
+ 
+  $json->sort_by(sub { $JSON::PP::a <=> $JSON::PP::b })->encode($perl_scalar)
+
+Can't access C<$a> and C<$b> but C<$JSON::PP::a> and C<$JSON::PP::b>.
+
+=item $JSON::SkipInvalid
+
+  $json->allow_unknown
+
+=item $JSON::AUTOCONVERT
+
+Needless. C<JSON> backend modules have the round-trip integrity.
+
+=item $JSON::UTF8
+
+Needless because C<JSON> (JSON::XS/JSON::PP) sets
+the UTF8 flag on properly.
+
+    # With UTF8-flagged strings
+
+    $json->allow_nonref;
+    $str = chr(1000); # UTF8-flagged
+
+    $json_text  = $json->utf8(0)->encode($str);
+    utf8::is_utf8($json_text);
+    # true
+    $json_text  = $json->utf8(1)->encode($str);
+    utf8::is_utf8($json_text);
+    # false
+
+    $str = '"' . chr(1000) . '"'; # UTF8-flagged
+
+    $perl_scalar  = $json->utf8(0)->decode($str);
+    utf8::is_utf8($perl_scalar);
+    # true
+    $perl_scalar  = $json->utf8(1)->decode($str);
+    # died because of 'Wide character in subroutine'
+
+See to L<JSON::XS/A FEW NOTES ON UNICODE AND PERL>.
+
+=item $JSON::UnMapping
+
+Disable. See to L<MAPPING>.
+
+=item $JSON::SelfConvert
+
+This option was deleted.
+Instead of it, if a given blessed object has the C<TO_JSON> method,
+C<TO_JSON> will be executed with C<convert_blessed>.
+
+  $json->convert_blessed->encode($blessed_hashref_or_arrayref)
+  # if need, call allow_blessed
+
+Note that it was C<toJson> in old version, but now not C<toJson> but C<TO_JSON>.
+
+=back
+
+=head1 TODO
+
+=over
+
+=item example programs
+
+=back
+
+=head1 THREADS
+
+No test with JSON::PP. If with JSON::XS, See to L<JSON::XS/THREADS>.
+
+
+=head1 BUGS
+
+Please report bugs relevant to C<JSON> to E<lt>makamaka[at]cpan.orgE<gt>.
+
+
+=head1 SEE ALSO
+
+Most of the document is copied and modified from JSON::XS doc.
+
+L<JSON::XS>, L<JSON::PP>
+
+C<RFC4627>(L<http://www.ietf.org/rfc/rfc4627.txt>)
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+JSON::XS was written by  Marc Lehmann <schmorp[at]schmorp.de>
+
+The release of this new version owes to the courtesy of Marc Lehmann.
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2005-2013 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut
+

uroman/lib/JSON/backportPP.pm

@@ -0,0 +1,2806 @@
+package # This is JSON::backportPP
+    JSON::PP;
+
+# JSON-2.0
+
+use 5.005;
+use strict;
+use base qw(Exporter);
+use overload ();
+
+use Carp ();
+use B ();
+#use Devel::Peek;
+
+use vars qw($VERSION);
+$VERSION = '2.27204';
+
+@JSON::PP::EXPORT = qw(encode_json decode_json from_json to_json);
+
+# instead of hash-access, i tried index-access for speed.
+# but this method is not faster than what i expected. so it will be changed.
+
+use constant P_ASCII                => 0;
+use constant P_LATIN1               => 1;
+use constant P_UTF8                 => 2;
+use constant P_INDENT               => 3;
+use constant P_CANONICAL            => 4;
+use constant P_SPACE_BEFORE         => 5;
+use constant P_SPACE_AFTER          => 6;
+use constant P_ALLOW_NONREF         => 7;
+use constant P_SHRINK               => 8;
+use constant P_ALLOW_BLESSED        => 9;
+use constant P_CONVERT_BLESSED      => 10;
+use constant P_RELAXED              => 11;
+
+use constant P_LOOSE                => 12;
+use constant P_ALLOW_BIGNUM         => 13;
+use constant P_ALLOW_BAREKEY        => 14;
+use constant P_ALLOW_SINGLEQUOTE    => 15;
+use constant P_ESCAPE_SLASH         => 16;
+use constant P_AS_NONBLESSED        => 17;
+
+use constant P_ALLOW_UNKNOWN        => 18;
+
+use constant OLD_PERL => $] < 5.008 ? 1 : 0;
+
+BEGIN {
+    my @xs_compati_bit_properties = qw(
+            latin1 ascii utf8 indent canonical space_before space_after allow_nonref shrink
+            allow_blessed convert_blessed relaxed allow_unknown
+    );
+    my @pp_bit_properties = qw(
+            allow_singlequote allow_bignum loose
+            allow_barekey escape_slash as_nonblessed
+    );
+
+    # Perl version check, Unicode handling is enable?
+    # Helper module sets @JSON::PP::_properties.
+    if ($] < 5.008 ) {
+        my $helper = $] >= 5.006 ? 'JSON::backportPP::Compat5006' : 'JSON::backportPP::Compat5005';
+        eval qq| require $helper |;
+        if ($@) { Carp::croak $@; }
+    }
+
+    for my $name (@xs_compati_bit_properties, @pp_bit_properties) {
+        my $flag_name = 'P_' . uc($name);
+
+        eval qq/
+            sub $name {
+                my \$enable = defined \$_[1] ? \$_[1] : 1;
+
+                if (\$enable) {
+                    \$_[0]->{PROPS}->[$flag_name] = 1;
+                }
+                else {
+                    \$_[0]->{PROPS}->[$flag_name] = 0;
+                }
+
+                \$_[0];
+            }
+
+            sub get_$name {
+                \$_[0]->{PROPS}->[$flag_name] ? 1 : '';
+            }
+        /;
+    }
+
+}
+
+
+
+# Functions
+
+my %encode_allow_method
+     = map {($_ => 1)} qw/utf8 pretty allow_nonref latin1 self_encode escape_slash
+                          allow_blessed convert_blessed indent indent_length allow_bignum
+                          as_nonblessed
+                        /;
+my %decode_allow_method
+     = map {($_ => 1)} qw/utf8 allow_nonref loose allow_singlequote allow_bignum
+                          allow_barekey max_size relaxed/;
+
+
+my $JSON; # cache
+
+sub encode_json ($) { # encode
+    ($JSON ||= __PACKAGE__->new->utf8)->encode(@_);
+}
+
+
+sub decode_json { # decode
+    ($JSON ||= __PACKAGE__->new->utf8)->decode(@_);
+}
+
+# Obsoleted
+
+sub to_json($) {
+   Carp::croak ("JSON::PP::to_json has been renamed to encode_json.");
+}
+
+
+sub from_json($) {
+   Carp::croak ("JSON::PP::from_json has been renamed to decode_json.");
+}
+
+
+# Methods
+
+sub new {
+    my $class = shift;
+    my $self  = {
+        max_depth   => 512,
+        max_size    => 0,
+        indent      => 0,
+        FLAGS       => 0,
+        fallback      => sub { encode_error('Invalid value. JSON can only reference.') },
+        indent_length => 3,
+    };
+
+    bless $self, $class;
+}
+
+
+sub encode {
+    return $_[0]->PP_encode_json($_[1]);
+}
+
+
+sub decode {
+    return $_[0]->PP_decode_json($_[1], 0x00000000);
+}
+
+
+sub decode_prefix {
+    return $_[0]->PP_decode_json($_[1], 0x00000001);
+}
+
+
+# accessor
+
+
+# pretty printing
+
+sub pretty {
+    my ($self, $v) = @_;
+    my $enable = defined $v ? $v : 1;
+
+    if ($enable) { # indent_length(3) for JSON::XS compatibility
+        $self->indent(1)->indent_length(3)->space_before(1)->space_after(1);
+    }
+    else {
+        $self->indent(0)->space_before(0)->space_after(0);
+    }
+
+    $self;
+}
+
+# etc
+
+sub max_depth {
+    my $max  = defined $_[1] ? $_[1] : 0x80000000;
+    $_[0]->{max_depth} = $max;
+    $_[0];
+}
+
+
+sub get_max_depth { $_[0]->{max_depth}; }
+
+
+sub max_size {
+    my $max  = defined $_[1] ? $_[1] : 0;
+    $_[0]->{max_size} = $max;
+    $_[0];
+}
+
+
+sub get_max_size { $_[0]->{max_size}; }
+
+
+sub filter_json_object {
+    $_[0]->{cb_object} = defined $_[1] ? $_[1] : 0;
+    $_[0]->{F_HOOK} = ($_[0]->{cb_object} or $_[0]->{cb_sk_object}) ? 1 : 0;
+    $_[0];
+}
+
+sub filter_json_single_key_object {
+    if (@_ > 1) {
+        $_[0]->{cb_sk_object}->{$_[1]} = $_[2];
+    }
+    $_[0]->{F_HOOK} = ($_[0]->{cb_object} or $_[0]->{cb_sk_object}) ? 1 : 0;
+    $_[0];
+}
+
+sub indent_length {
+    if (!defined $_[1] or $_[1] > 15 or $_[1] < 0) {
+        Carp::carp "The acceptable range of indent_length() is 0 to 15.";
+    }
+    else {
+        $_[0]->{indent_length} = $_[1];
+    }
+    $_[0];
+}
+
+sub get_indent_length {
+    $_[0]->{indent_length};
+}
+
+sub sort_by {
+    $_[0]->{sort_by} = defined $_[1] ? $_[1] : 1;
+    $_[0];
+}
+
+sub allow_bigint {
+    Carp::carp("allow_bigint() is obsoleted. use allow_bignum() insted.");
+}
+
+###############################
+
+###
+### Perl => JSON
+###
+
+
+{ # Convert
+
+    my $max_depth;
+    my $indent;
+    my $ascii;
+    my $latin1;
+    my $utf8;
+    my $space_before;
+    my $space_after;
+    my $canonical;
+    my $allow_blessed;
+    my $convert_blessed;
+
+    my $indent_length;
+    my $escape_slash;
+    my $bignum;
+    my $as_nonblessed;
+
+    my $depth;
+    my $indent_count;
+    my $keysort;
+
+
+    sub PP_encode_json {
+        my $self = shift;
+        my $obj  = shift;
+
+        $indent_count = 0;
+        $depth        = 0;
+
+        my $idx = $self->{PROPS};
+
+        ($ascii, $latin1, $utf8, $indent, $canonical, $space_before, $space_after, $allow_blessed,
+            $convert_blessed, $escape_slash, $bignum, $as_nonblessed)
+         = @{$idx}[P_ASCII .. P_SPACE_AFTER, P_ALLOW_BLESSED, P_CONVERT_BLESSED,
+                    P_ESCAPE_SLASH, P_ALLOW_BIGNUM, P_AS_NONBLESSED];
+
+        ($max_depth, $indent_length) = @{$self}{qw/max_depth indent_length/};
+
+        $keysort = $canonical ? sub { $a cmp $b } : undef;
+
+        if ($self->{sort_by}) {
+            $keysort = ref($self->{sort_by}) eq 'CODE' ? $self->{sort_by}
+                     : $self->{sort_by} =~ /\D+/       ? $self->{sort_by}
+                     : sub { $a cmp $b };
+        }
+
+        encode_error("hash- or arrayref expected (not a simple scalar, use allow_nonref to allow this)")
+             if(!ref $obj and !$idx->[ P_ALLOW_NONREF ]);
+
+        my $str  = $self->object_to_json($obj);
+
+        $str .= "\n" if ( $indent ); # JSON::XS 2.26 compatible
+
+        unless ($ascii or $latin1 or $utf8) {
+            utf8::upgrade($str);
+        }
+
+        if ($idx->[ P_SHRINK ]) {
+            utf8::downgrade($str, 1);
+        }
+
+        return $str;
+    }
+
+
+    sub object_to_json {
+        my ($self, $obj) = @_;
+        my $type = ref($obj);
+
+        if($type eq 'HASH'){
+            return $self->hash_to_json($obj);
+        }
+        elsif($type eq 'ARRAY'){
+            return $self->array_to_json($obj);
+        }
+        elsif ($type) { # blessed object?
+            if (blessed($obj)) {
+
+                return $self->value_to_json($obj) if ( $obj->isa('JSON::PP::Boolean') );
+
+                if ( $convert_blessed and $obj->can('TO_JSON') ) {
+                    my $result = $obj->TO_JSON();
+                    if ( defined $result and ref( $result ) ) {
+                        if ( refaddr( $obj ) eq refaddr( $result ) ) {
+                            encode_error( sprintf(
+                                "%s::TO_JSON method returned same object as was passed instead of a new one",
+                                ref $obj
+                            ) );
+                        }
+                    }
+
+                    return $self->object_to_json( $result );
+                }
+
+                return "$obj" if ( $bignum and _is_bignum($obj) );
+                return $self->blessed_to_json($obj) if ($allow_blessed and $as_nonblessed); # will be removed.
+
+                encode_error( sprintf("encountered object '%s', but neither allow_blessed "
+                    . "nor convert_blessed settings are enabled", $obj)
+                ) unless ($allow_blessed);
+
+                return 'null';
+            }
+            else {
+                return $self->value_to_json($obj);
+            }
+        }
+        else{
+            return $self->value_to_json($obj);
+        }
+    }
+
+
+    sub hash_to_json {
+        my ($self, $obj) = @_;
+        my @res;
+
+        encode_error("json text or perl structure exceeds maximum nesting level (max_depth set too low?)")
+                                         if (++$depth > $max_depth);
+
+        my ($pre, $post) = $indent ? $self->_up_indent() : ('', '');
+        my $del = ($space_before ? ' ' : '') . ':' . ($space_after ? ' ' : '');
+
+        for my $k ( _sort( $obj ) ) {
+            if ( OLD_PERL ) { utf8::decode($k) } # key for Perl 5.6 / be optimized
+            push @res, string_to_json( $self, $k )
+                          .  $del
+                          . ( $self->object_to_json( $obj->{$k} ) || $self->value_to_json( $obj->{$k} ) );
+        }
+
+        --$depth;
+        $self->_down_indent() if ($indent);
+
+        return   '{' . ( @res ? $pre : '' ) . ( @res ? join( ",$pre", @res ) . $post : '' )  . '}';
+    }
+
+
+    sub array_to_json {
+        my ($self, $obj) = @_;
+        my @res;
+
+        encode_error("json text or perl structure exceeds maximum nesting level (max_depth set too low?)")
+                                         if (++$depth > $max_depth);
+
+        my ($pre, $post) = $indent ? $self->_up_indent() : ('', '');
+
+        for my $v (@$obj){
+            push @res, $self->object_to_json($v) || $self->value_to_json($v);
+        }
+
+        --$depth;
+        $self->_down_indent() if ($indent);
+
+        return '[' . ( @res ? $pre : '' ) . ( @res ? join( ",$pre", @res ) . $post : '' ) . ']';
+    }
+
+
+    sub value_to_json {
+        my ($self, $value) = @_;
+
+        return 'null' if(!defined $value);
+
+        my $b_obj = B::svref_2object(\$value);  # for round trip problem
+        my $flags = $b_obj->FLAGS;
+
+        return $value # as is 
+            if $flags & ( B::SVp_IOK | B::SVp_NOK ) and !( $flags & B::SVp_POK ); # SvTYPE is IV or NV?
+
+        my $type = ref($value);
+
+        if(!$type){
+            return string_to_json($self, $value);
+        }
+        elsif( blessed($value) and  $value->isa('JSON::PP::Boolean') ){
+            return $$value == 1 ? 'true' : 'false';
+        }
+        elsif ($type) {
+            if ((overload::StrVal($value) =~ /=(\w+)/)[0]) {
+                return $self->value_to_json("$value");
+            }
+
+            if ($type eq 'SCALAR' and defined $$value) {
+                return   $$value eq '1' ? 'true'
+                       : $$value eq '0' ? 'false'
+                       : $self->{PROPS}->[ P_ALLOW_UNKNOWN ] ? 'null'
+                       : encode_error("cannot encode reference to scalar");
+            }
+
+             if ( $self->{PROPS}->[ P_ALLOW_UNKNOWN ] ) {
+                 return 'null';
+             }
+             else {
+                 if ( $type eq 'SCALAR' or $type eq 'REF' ) {
+                    encode_error("cannot encode reference to scalar");
+                 }
+                 else {
+                    encode_error("encountered $value, but JSON can only represent references to arrays or hashes");
+                 }
+             }
+
+        }
+        else {
+            return $self->{fallback}->($value)
+                 if ($self->{fallback} and ref($self->{fallback}) eq 'CODE');
+            return 'null';
+        }
+
+    }
+
+
+    my %esc = (
+        "\n" => '\n',
+        "\r" => '\r',
+        "\t" => '\t',
+        "\f" => '\f',
+        "\b" => '\b',
+        "\"" => '\"',
+        "\\" => '\\\\',
+        "\'" => '\\\'',
+    );
+
+
+    sub string_to_json {
+        my ($self, $arg) = @_;
+
+        $arg =~ s/([\x22\x5c\n\r\t\f\b])/$esc{$1}/g;
+        $arg =~ s/\//\\\//g if ($escape_slash);
+        $arg =~ s/([\x00-\x08\x0b\x0e-\x1f])/'\\u00' . unpack('H2', $1)/eg;
+
+        if ($ascii) {
+            $arg = JSON_PP_encode_ascii($arg);
+        }
+
+        if ($latin1) {
+            $arg = JSON_PP_encode_latin1($arg);
+        }
+
+        if ($utf8) {
+            utf8::encode($arg);
+        }
+
+        return '"' . $arg . '"';
+    }
+
+
+    sub blessed_to_json {
+        my $reftype = reftype($_[1]) || '';
+        if ($reftype eq 'HASH') {
+            return $_[0]->hash_to_json($_[1]);
+        }
+        elsif ($reftype eq 'ARRAY') {
+            return $_[0]->array_to_json($_[1]);
+        }
+        else {
+            return 'null';
+        }
+    }
+
+
+    sub encode_error {
+        my $error  = shift;
+        Carp::croak "$error";
+    }
+
+
+    sub _sort {
+        defined $keysort ? (sort $keysort (keys %{$_[0]})) : keys %{$_[0]};
+    }
+
+
+    sub _up_indent {
+        my $self  = shift;
+        my $space = ' ' x $indent_length;
+
+        my ($pre,$post) = ('','');
+
+        $post = "\n" . $space x $indent_count;
+
+        $indent_count++;
+
+        $pre = "\n" . $space x $indent_count;
+
+        return ($pre,$post);
+    }
+
+
+    sub _down_indent { $indent_count--; }
+
+
+    sub PP_encode_box {
+        {
+            depth        => $depth,
+            indent_count => $indent_count,
+        };
+    }
+
+} # Convert
+
+
+sub _encode_ascii {
+    join('',
+        map {
+            $_ <= 127 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', _encode_surrogates($_));
+        } unpack('U*', $_[0])
+    );
+}
+
+
+sub _encode_latin1 {
+    join('',
+        map {
+            $_ <= 255 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', _encode_surrogates($_));
+        } unpack('U*', $_[0])
+    );
+}
+
+
+sub _encode_surrogates { # from perlunicode
+    my $uni = $_[0] - 0x10000;
+    return ($uni / 0x400 + 0xD800, $uni % 0x400 + 0xDC00);
+}
+
+
+sub _is_bignum {
+    $_[0]->isa('Math::BigInt') or $_[0]->isa('Math::BigFloat');
+}
+
+
+
+#
+# JSON => Perl
+#
+
+my $max_intsize;
+
+BEGIN {
+    my $checkint = 1111;
+    for my $d (5..64) {
+        $checkint .= 1;
+        my $int   = eval qq| $checkint |;
+        if ($int =~ /[eE]/) {
+            $max_intsize = $d - 1;
+            last;
+        }
+    }
+}
+
+{ # PARSE 
+
+    my %escapes = ( #  by Jeremy Muhlich <jmuhlich [at] bitflood.org>
+        b    => "\x8",
+        t    => "\x9",
+        n    => "\xA",
+        f    => "\xC",
+        r    => "\xD",
+        '\\' => '\\',
+        '"'  => '"',
+        '/'  => '/',
+    );
+
+    my $text; # json data
+    my $at;   # offset
+    my $ch;   # 1chracter
+    my $len;  # text length (changed according to UTF8 or NON UTF8)
+    # INTERNAL
+    my $depth;          # nest counter
+    my $encoding;       # json text encoding
+    my $is_valid_utf8;  # temp variable
+    my $utf8_len;       # utf8 byte length
+    # FLAGS
+    my $utf8;           # must be utf8
+    my $max_depth;      # max nest number of objects and arrays
+    my $max_size;
+    my $relaxed;
+    my $cb_object;
+    my $cb_sk_object;
+
+    my $F_HOOK;
+
+    my $allow_bigint;   # using Math::BigInt
+    my $singlequote;    # loosely quoting
+    my $loose;          # 
+    my $allow_barekey;  # bareKey
+
+    # $opt flag
+    # 0x00000001 .... decode_prefix
+    # 0x10000000 .... incr_parse
+
+    sub PP_decode_json {
+        my ($self, $opt); # $opt is an effective flag during this decode_json.
+
+        ($self, $text, $opt) = @_;
+
+        ($at, $ch, $depth) = (0, '', 0);
+
+        if ( !defined $text or ref $text ) {
+            decode_error("malformed JSON string, neither array, object, number, string or atom");
+        }
+
+        my $idx = $self->{PROPS};
+
+        ($utf8, $relaxed, $loose, $allow_bigint, $allow_barekey, $singlequote)
+            = @{$idx}[P_UTF8, P_RELAXED, P_LOOSE .. P_ALLOW_SINGLEQUOTE];
+
+        if ( $utf8 ) {
+            utf8::downgrade( $text, 1 ) or Carp::croak("Wide character in subroutine entry");
+        }
+        else {
+            utf8::upgrade( $text );
+        }
+
+        $len = length $text;
+
+        ($max_depth, $max_size, $cb_object, $cb_sk_object, $F_HOOK)
+             = @{$self}{qw/max_depth  max_size cb_object cb_sk_object F_HOOK/};
+
+        if ($max_size > 1) {
+            use bytes;
+            my $bytes = length $text;
+            decode_error(
+                sprintf("attempted decode of JSON text of %s bytes size, but max_size is set to %s"
+                    , $bytes, $max_size), 1
+            ) if ($bytes > $max_size);
+        }
+
+        # Currently no effect
+        # should use regexp
+        my @octets = unpack('C4', $text);
+        $encoding =   ( $octets[0] and  $octets[1]) ? 'UTF-8'
+                    : (!$octets[0] and  $octets[1]) ? 'UTF-16BE'
+                    : (!$octets[0] and !$octets[1]) ? 'UTF-32BE'
+                    : ( $octets[2]                ) ? 'UTF-16LE'
+                    : (!$octets[2]                ) ? 'UTF-32LE'
+                    : 'unknown';
+
+        white(); # remove head white space
+
+        my $valid_start = defined $ch; # Is there a first character for JSON structure?
+
+        my $result = value();
+
+        return undef if ( !$result && ( $opt & 0x10000000 ) ); # for incr_parse
+
+        decode_error("malformed JSON string, neither array, object, number, string or atom") unless $valid_start;
+
+        if ( !$idx->[ P_ALLOW_NONREF ] and !ref $result ) {
+                decode_error(
+                'JSON text must be an object or array (but found number, string, true, false or null,'
+                       . ' use allow_nonref to allow this)', 1);
+        }
+
+        Carp::croak('something wrong.') if $len < $at; # we won't arrive here.
+
+        my $consumed = defined $ch ? $at - 1 : $at; # consumed JSON text length
+
+        white(); # remove tail white space
+
+        if ( $ch ) {
+            return ( $result, $consumed ) if ($opt & 0x00000001); # all right if decode_prefix
+            decode_error("garbage after JSON object");
+        }
+
+        ( $opt & 0x00000001 ) ? ( $result, $consumed ) : $result;
+    }
+
+
+    sub next_chr {
+        return $ch = undef if($at >= $len);
+        $ch = substr($text, $at++, 1);
+    }
+
+
+    sub value {
+        white();
+        return          if(!defined $ch);
+        return object() if($ch eq '{');
+        return array()  if($ch eq '[');
+        return string() if($ch eq '"' or ($singlequote and $ch eq "'"));
+        return number() if($ch =~ /[0-9]/ or $ch eq '-');
+        return word();
+    }
+
+    sub string {
+        my ($i, $s, $t, $u);
+        my $utf16;
+        my $is_utf8;
+
+        ($is_valid_utf8, $utf8_len) = ('', 0);
+
+        $s = ''; # basically UTF8 flag on
+
+        if($ch eq '"' or ($singlequote and $ch eq "'")){
+            my $boundChar = $ch;
+
+            OUTER: while( defined(next_chr()) ){
+
+                if($ch eq $boundChar){
+                    next_chr();
+
+                    if ($utf16) {
+                        decode_error("missing low surrogate character in surrogate pair");
+                    }
+
+                    utf8::decode($s) if($is_utf8);
+
+                    return $s;
+                }
+                elsif($ch eq '\\'){
+                    next_chr();
+                    if(exists $escapes{$ch}){
+                        $s .= $escapes{$ch};
+                    }
+                    elsif($ch eq 'u'){ # UNICODE handling
+                        my $u = '';
+
+                        for(1..4){
+                            $ch = next_chr();
+                            last OUTER if($ch !~ /[0-9a-fA-F]/);
+                            $u .= $ch;
+                        }
+
+                        # U+D800 - U+DBFF
+                        if ($u =~ /^[dD][89abAB][0-9a-fA-F]{2}/) { # UTF-16 high surrogate?
+                            $utf16 = $u;
+                        }
+                        # U+DC00 - U+DFFF
+                        elsif ($u =~ /^[dD][c-fC-F][0-9a-fA-F]{2}/) { # UTF-16 low surrogate?
+                            unless (defined $utf16) {
+                                decode_error("missing high surrogate character in surrogate pair");
+                            }
+                            $is_utf8 = 1;
+                            $s .= JSON_PP_decode_surrogates($utf16, $u) || next;
+                            $utf16 = undef;
+                        }
+                        else {
+                            if (defined $utf16) {
+                                decode_error("surrogate pair expected");
+                            }
+
+                            if ( ( my $hex = hex( $u ) ) > 127 ) {
+                                $is_utf8 = 1;
+                                $s .= JSON_PP_decode_unicode($u) || next;
+                            }
+                            else {
+                                $s .= chr $hex;
+                            }
+                        }
+
+                    }
+                    else{
+                        unless ($loose) {
+                            $at -= 2;
+                            decode_error('illegal backslash escape sequence in string');
+                        }
+                        $s .= $ch;
+                    }
+                }
+                else{
+
+                    if ( ord $ch  > 127 ) {
+                        if ( $utf8 ) {
+                            unless( $ch = is_valid_utf8($ch) ) {
+                                $at -= 1;
+                                decode_error("malformed UTF-8 character in JSON string");
+                            }
+                            else {
+                                $at += $utf8_len - 1;
+                            }
+                        }
+                        else {
+                            utf8::encode( $ch );
+                        }
+
+                        $is_utf8 = 1;
+                    }
+
+                    if (!$loose) {
+                        if ($ch =~ /[\x00-\x1f\x22\x5c]/)  { # '/' ok
+                            $at--;
+                            decode_error('invalid character encountered while parsing JSON string');
+                        }
+                    }
+
+                    $s .= $ch;
+                }
+            }
+        }
+
+        decode_error("unexpected end of string while parsing JSON string");
+    }
+
+
+    sub white {
+        while( defined $ch  ){
+            if($ch le ' '){
+                next_chr();
+            }
+            elsif($ch eq '/'){
+                next_chr();
+                if(defined $ch and $ch eq '/'){
+                    1 while(defined(next_chr()) and $ch ne "\n" and $ch ne "\r");
+                }
+                elsif(defined $ch and $ch eq '*'){
+                    next_chr();
+                    while(1){
+                        if(defined $ch){
+                            if($ch eq '*'){
+                                if(defined(next_chr()) and $ch eq '/'){
+                                    next_chr();
+                                    last;
+                                }
+                            }
+                            else{
+                                next_chr();
+                            }
+                        }
+                        else{
+                            decode_error("Unterminated comment");
+                        }
+                    }
+                    next;
+                }
+                else{
+                    $at--;
+                    decode_error("malformed JSON string, neither array, object, number, string or atom");
+                }
+            }
+            else{
+                if ($relaxed and $ch eq '#') { # correctly?
+                    pos($text) = $at;
+                    $text =~ /\G([^\n]*(?:\r\n|\r|\n|$))/g;
+                    $at = pos($text);
+                    next_chr;
+                    next;
+                }
+
+                last;
+            }
+        }
+    }
+
+
+    sub array {
+        my $a  = $_[0] || []; # you can use this code to use another array ref object.
+
+        decode_error('json text or perl structure exceeds maximum nesting level (max_depth set too low?)')
+                                                    if (++$depth > $max_depth);
+
+        next_chr();
+        white();
+
+        if(defined $ch and $ch eq ']'){
+            --$depth;
+            next_chr();
+            return $a;
+        }
+        else {
+            while(defined($ch)){
+                push @$a, value();
+
+                white();
+
+                if (!defined $ch) {
+                    last;
+                }
+
+                if($ch eq ']'){
+                    --$depth;
+                    next_chr();
+                    return $a;
+                }
+
+                if($ch ne ','){
+                    last;
+                }
+
+                next_chr();
+                white();
+
+                if ($relaxed and $ch eq ']') {
+                    --$depth;
+                    next_chr();
+                    return $a;
+                }
+
+            }
+        }
+
+        decode_error(", or ] expected while parsing array");
+    }
+
+
+    sub object {
+        my $o = $_[0] || {}; # you can use this code to use another hash ref object.
+        my $k;
+
+        decode_error('json text or perl structure exceeds maximum nesting level (max_depth set too low?)')
+                                                if (++$depth > $max_depth);
+        next_chr();
+        white();
+
+        if(defined $ch and $ch eq '}'){
+            --$depth;
+            next_chr();
+            if ($F_HOOK) {
+                return _json_object_hook($o);
+            }
+            return $o;
+        }
+        else {
+            while (defined $ch) {
+                $k = ($allow_barekey and $ch ne '"' and $ch ne "'") ? bareKey() : string();
+                white();
+
+                if(!defined $ch or $ch ne ':'){
+                    $at--;
+                    decode_error("':' expected");
+                }
+
+                next_chr();
+                $o->{$k} = value();
+                white();
+
+                last if (!defined $ch);
+
+                if($ch eq '}'){
+                    --$depth;
+                    next_chr();
+                    if ($F_HOOK) {
+                        return _json_object_hook($o);
+                    }
+                    return $o;
+                }
+
+                if($ch ne ','){
+                    last;
+                }
+
+                next_chr();
+                white();
+
+                if ($relaxed and $ch eq '}') {
+                    --$depth;
+                    next_chr();
+                    if ($F_HOOK) {
+                        return _json_object_hook($o);
+                    }
+                    return $o;
+                }
+
+            }
+
+        }
+
+        $at--;
+        decode_error(", or } expected while parsing object/hash");
+    }
+
+
+    sub bareKey { # doesn't strictly follow Standard ECMA-262 3rd Edition
+        my $key;
+        while($ch =~ /[^\x00-\x23\x25-\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\x7F]/){
+            $key .= $ch;
+            next_chr();
+        }
+        return $key;
+    }
+
+
+    sub word {
+        my $word =  substr($text,$at-1,4);
+
+        if($word eq 'true'){
+            $at += 3;
+            next_chr;
+            return $JSON::PP::true;
+        }
+        elsif($word eq 'null'){
+            $at += 3;
+            next_chr;
+            return undef;
+        }
+        elsif($word eq 'fals'){
+            $at += 3;
+            if(substr($text,$at,1) eq 'e'){
+                $at++;
+                next_chr;
+                return $JSON::PP::false;
+            }
+        }
+
+        $at--; # for decode_error report
+
+        decode_error("'null' expected")  if ($word =~ /^n/);
+        decode_error("'true' expected")  if ($word =~ /^t/);
+        decode_error("'false' expected") if ($word =~ /^f/);
+        decode_error("malformed JSON string, neither array, object, number, string or atom");
+    }
+
+
+    sub number {
+        my $n    = '';
+        my $v;
+
+        # According to RFC4627, hex or oct digits are invalid.
+        if($ch eq '0'){
+            my $peek = substr($text,$at,1);
+            my $hex  = $peek =~ /[xX]/; # 0 or 1
+
+            if($hex){
+                decode_error("malformed number (leading zero must not be followed by another digit)");
+                ($n) = ( substr($text, $at+1) =~ /^([0-9a-fA-F]+)/);
+            }
+            else{ # oct
+                ($n) = ( substr($text, $at) =~ /^([0-7]+)/);
+                if (defined $n and length $n > 1) {
+                    decode_error("malformed number (leading zero must not be followed by another digit)");
+                }
+            }
+
+            if(defined $n and length($n)){
+                if (!$hex and length($n) == 1) {
+                   decode_error("malformed number (leading zero must not be followed by another digit)");
+                }
+                $at += length($n) + $hex;
+                next_chr;
+                return $hex ? hex($n) : oct($n);
+            }
+        }
+
+        if($ch eq '-'){
+            $n = '-';
+            next_chr;
+            if (!defined $ch or $ch !~ /\d/) {
+                decode_error("malformed number (no digits after initial minus)");
+            }
+        }
+
+        while(defined $ch and $ch =~ /\d/){
+            $n .= $ch;
+            next_chr;
+        }
+
+        if(defined $ch and $ch eq '.'){
+            $n .= '.';
+
+            next_chr;
+            if (!defined $ch or $ch !~ /\d/) {
+                decode_error("malformed number (no digits after decimal point)");
+            }
+            else {
+                $n .= $ch;
+            }
+
+            while(defined(next_chr) and $ch =~ /\d/){
+                $n .= $ch;
+            }
+        }
+
+        if(defined $ch and ($ch eq 'e' or $ch eq 'E')){
+            $n .= $ch;
+            next_chr;
+
+            if(defined($ch) and ($ch eq '+' or $ch eq '-')){
+                $n .= $ch;
+                next_chr;
+                if (!defined $ch or $ch =~ /\D/) {
+                    decode_error("malformed number (no digits after exp sign)");
+                }
+                $n .= $ch;
+            }
+            elsif(defined($ch) and $ch =~ /\d/){
+                $n .= $ch;
+            }
+            else {
+                decode_error("malformed number (no digits after exp sign)");
+            }
+
+            while(defined(next_chr) and $ch =~ /\d/){
+                $n .= $ch;
+            }
+
+        }
+
+        $v .= $n;
+
+        if ($v !~ /[.eE]/ and length $v > $max_intsize) {
+            if ($allow_bigint) { # from Adam Sussman
+                require Math::BigInt;
+                return Math::BigInt->new($v);
+            }
+            else {
+                return "$v";
+            }
+        }
+        elsif ($allow_bigint) {
+            require Math::BigFloat;
+            return Math::BigFloat->new($v);
+        }
+
+        return 0+$v;
+    }
+
+
+    sub is_valid_utf8 {
+
+        $utf8_len = $_[0] =~ /[\x00-\x7F]/  ? 1
+                  : $_[0] =~ /[\xC2-\xDF]/  ? 2
+                  : $_[0] =~ /[\xE0-\xEF]/  ? 3
+                  : $_[0] =~ /[\xF0-\xF4]/  ? 4
+                  : 0
+                  ;
+
+        return unless $utf8_len;
+
+        my $is_valid_utf8 = substr($text, $at - 1, $utf8_len);
+
+        return ( $is_valid_utf8 =~ /^(?:
+             [\x00-\x7F]
+            |[\xC2-\xDF][\x80-\xBF]
+            |[\xE0][\xA0-\xBF][\x80-\xBF]
+            |[\xE1-\xEC][\x80-\xBF][\x80-\xBF]
+            |[\xED][\x80-\x9F][\x80-\xBF]
+            |[\xEE-\xEF][\x80-\xBF][\x80-\xBF]
+            |[\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF]
+        )$/x )  ? $is_valid_utf8 : '';
+    }
+
+
+    sub decode_error {
+        my $error  = shift;
+        my $no_rep = shift;
+        my $str    = defined $text ? substr($text, $at) : '';
+        my $mess   = '';
+        my $type   = $] >= 5.008           ? 'U*'
+                   : $] <  5.006           ? 'C*'
+                   : utf8::is_utf8( $str ) ? 'U*' # 5.6
+                   : 'C*'
+                   ;
+
+        for my $c ( unpack( $type, $str ) ) { # emulate pv_uni_display() ?
+            $mess .=  $c == 0x07 ? '\a'
+                    : $c == 0x09 ? '\t'
+                    : $c == 0x0a ? '\n'
+                    : $c == 0x0d ? '\r'
+                    : $c == 0x0c ? '\f'
+                    : $c <  0x20 ? sprintf('\x{%x}', $c)
+                    : $c == 0x5c ? '\\\\'
+                    : $c <  0x80 ? chr($c)
+                    : sprintf('\x{%x}', $c)
+                    ;
+            if ( length $mess >= 20 ) {
+                $mess .= '...';
+                last;
+            }
+        }
+
+        unless ( length $mess ) {
+            $mess = '(end of string)';
+        }
+
+        Carp::croak (
+            $no_rep ? "$error" : "$error, at character offset $at (before \"$mess\")"
+        );
+
+    }
+
+
+    sub _json_object_hook {
+        my $o    = $_[0];
+        my @ks = keys %{$o};
+
+        if ( $cb_sk_object and @ks == 1 and exists $cb_sk_object->{ $ks[0] } and ref $cb_sk_object->{ $ks[0] } ) {
+            my @val = $cb_sk_object->{ $ks[0] }->( $o->{$ks[0]} );
+            if (@val == 1) {
+                return $val[0];
+            }
+        }
+
+        my @val = $cb_object->($o) if ($cb_object);
+        if (@val == 0 or @val > 1) {
+            return $o;
+        }
+        else {
+            return $val[0];
+        }
+    }
+
+
+    sub PP_decode_box {
+        {
+            text    => $text,
+            at      => $at,
+            ch      => $ch,
+            len     => $len,
+            depth   => $depth,
+            encoding      => $encoding,
+            is_valid_utf8 => $is_valid_utf8,
+        };
+    }
+
+} # PARSE
+
+
+sub _decode_surrogates { # from perlunicode
+    my $uni = 0x10000 + (hex($_[0]) - 0xD800) * 0x400 + (hex($_[1]) - 0xDC00);
+    my $un  = pack('U*', $uni);
+    utf8::encode( $un );
+    return $un;
+}
+
+
+sub _decode_unicode {
+    my $un = pack('U', hex shift);
+    utf8::encode( $un );
+    return $un;
+}
+
+#
+# Setup for various Perl versions (the code from JSON::PP58)
+#
+
+BEGIN {
+
+    unless ( defined &utf8::is_utf8 ) {
+       require Encode;
+       *utf8::is_utf8 = *Encode::is_utf8;
+    }
+
+    if ( $] >= 5.008 ) {
+        *JSON::PP::JSON_PP_encode_ascii      = \&_encode_ascii;
+        *JSON::PP::JSON_PP_encode_latin1     = \&_encode_latin1;
+        *JSON::PP::JSON_PP_decode_surrogates = \&_decode_surrogates;
+        *JSON::PP::JSON_PP_decode_unicode    = \&_decode_unicode;
+    }
+
+    if ($] >= 5.008 and $] < 5.008003) { # join() in 5.8.0 - 5.8.2 is broken.
+        package # hide from PAUSE
+          JSON::PP;
+        require subs;
+        subs->import('join');
+        eval q|
+            sub join {
+                return '' if (@_ < 2);
+                my $j   = shift;
+                my $str = shift;
+                for (@_) { $str .= $j . $_; }
+                return $str;
+            }
+        |;
+    }
+
+
+    sub JSON::PP::incr_parse {
+        local $Carp::CarpLevel = 1;
+        ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_parse( @_ );
+    }
+
+
+    sub JSON::PP::incr_skip {
+        ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_skip;
+    }
+
+
+    sub JSON::PP::incr_reset {
+        ( $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new )->incr_reset;
+    }
+
+    eval q{
+        sub JSON::PP::incr_text : lvalue {
+            $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new;
+
+            if ( $_[0]->{_incr_parser}->{incr_parsing} ) {
+                Carp::croak("incr_text can not be called when the incremental parser already started parsing");
+            }
+            $_[0]->{_incr_parser}->{incr_text};
+        }
+    } if ( $] >= 5.006 );
+
+} # Setup for various Perl versions (the code from JSON::PP58)
+
+
+###############################
+# Utilities
+#
+
+BEGIN {
+    eval 'require Scalar::Util';
+    unless($@){
+        *JSON::PP::blessed = \&Scalar::Util::blessed;
+        *JSON::PP::reftype = \&Scalar::Util::reftype;
+        *JSON::PP::refaddr = \&Scalar::Util::refaddr;
+    }
+    else{ # This code is from Scalar::Util.
+        # warn $@;
+        eval 'sub UNIVERSAL::a_sub_not_likely_to_be_here { ref($_[0]) }';
+        *JSON::PP::blessed = sub {
+            local($@, $SIG{__DIE__}, $SIG{__WARN__});
+            ref($_[0]) ? eval { $_[0]->a_sub_not_likely_to_be_here } : undef;
+        };
+        my %tmap = qw(
+            B::NULL   SCALAR
+            B::HV     HASH
+            B::AV     ARRAY
+            B::CV     CODE
+            B::IO     IO
+            B::GV     GLOB
+            B::REGEXP REGEXP
+        );
+        *JSON::PP::reftype = sub {
+            my $r = shift;
+
+            return undef unless length(ref($r));
+
+            my $t = ref(B::svref_2object($r));
+
+            return
+                exists $tmap{$t} ? $tmap{$t}
+              : length(ref($$r)) ? 'REF'
+              :                    'SCALAR';
+        };
+        *JSON::PP::refaddr = sub {
+          return undef unless length(ref($_[0]));
+
+          my $addr;
+          if(defined(my $pkg = blessed($_[0]))) {
+            $addr .= bless $_[0], 'Scalar::Util::Fake';
+            bless $_[0], $pkg;
+          }
+          else {
+            $addr .= $_[0]
+          }
+
+          $addr =~ /0x(\w+)/;
+          local $^W;
+          #no warnings 'portable';
+          hex($1);
+        }
+    }
+}
+
+
+# shamelessly copied and modified from JSON::XS code.
+
+unless ( $INC{'JSON/PP.pm'} ) {
+    eval q|
+        package
+            JSON::PP::Boolean;
+
+        use overload (
+            "0+"     => sub { ${$_[0]} },
+            "++"     => sub { $_[0] = ${$_[0]} + 1 },
+            "--"     => sub { $_[0] = ${$_[0]} - 1 },
+            fallback => 1,
+        );
+    |;
+}
+
+$JSON::PP::true  = do { bless \(my $dummy = 1), "JSON::PP::Boolean" };
+$JSON::PP::false = do { bless \(my $dummy = 0), "JSON::PP::Boolean" };
+
+sub is_bool { defined $_[0] and UNIVERSAL::isa($_[0], "JSON::PP::Boolean"); }
+
+sub true  { $JSON::PP::true  }
+sub false { $JSON::PP::false }
+sub null  { undef; }
+
+###############################
+
+###############################
+
+package # hide from PAUSE
+  JSON::PP::IncrParser;
+
+use strict;
+
+use constant INCR_M_WS   => 0; # initial whitespace skipping
+use constant INCR_M_STR  => 1; # inside string
+use constant INCR_M_BS   => 2; # inside backslash
+use constant INCR_M_JSON => 3; # outside anything, count nesting
+use constant INCR_M_C0   => 4;
+use constant INCR_M_C1   => 5;
+
+use vars qw($VERSION);
+$VERSION = '1.01';
+
+my $unpack_format = $] < 5.006 ? 'C*' : 'U*';
+
+sub new {
+    my ( $class ) = @_;
+
+    bless {
+        incr_nest    => 0,
+        incr_text    => undef,
+        incr_parsing => 0,
+        incr_p       => 0,
+    }, $class;
+}
+
+
+sub incr_parse {
+    my ( $self, $coder, $text ) = @_;
+
+    $self->{incr_text} = '' unless ( defined $self->{incr_text} );
+
+    if ( defined $text ) {
+        if ( utf8::is_utf8( $text ) and !utf8::is_utf8( $self->{incr_text} ) ) {
+            utf8::upgrade( $self->{incr_text} ) ;
+            utf8::decode( $self->{incr_text} ) ;
+        }
+        $self->{incr_text} .= $text;
+    }
+
+
+    my $max_size = $coder->get_max_size;
+
+    if ( defined wantarray ) {
+
+        $self->{incr_mode} = INCR_M_WS unless defined $self->{incr_mode};
+
+        if ( wantarray ) {
+            my @ret;
+
+            $self->{incr_parsing} = 1;
+
+            do {
+                push @ret, $self->_incr_parse( $coder, $self->{incr_text} );
+
+                unless ( !$self->{incr_nest} and $self->{incr_mode} == INCR_M_JSON ) {
+                    $self->{incr_mode} = INCR_M_WS if $self->{incr_mode} != INCR_M_STR;
+                }
+
+            } until ( length $self->{incr_text} >= $self->{incr_p} );
+
+            $self->{incr_parsing} = 0;
+
+            return @ret;
+        }
+        else { # in scalar context
+            $self->{incr_parsing} = 1;
+            my $obj = $self->_incr_parse( $coder, $self->{incr_text} );
+            $self->{incr_parsing} = 0 if defined $obj; # pointed by Martin J. Evans
+            return $obj ? $obj : undef; # $obj is an empty string, parsing was completed.
+        }
+
+    }
+
+}
+
+
+sub _incr_parse {
+    my ( $self, $coder, $text, $skip ) = @_;
+    my $p = $self->{incr_p};
+    my $restore = $p;
+
+    my @obj;
+    my $len = length $text;
+
+    if ( $self->{incr_mode} == INCR_M_WS ) {
+        while ( $len > $p ) {
+            my $s = substr( $text, $p, 1 );
+            $p++ and next if ( 0x20 >= unpack($unpack_format, $s) );
+            $self->{incr_mode} = INCR_M_JSON;
+            last;
+       }
+    }
+
+    while ( $len > $p ) {
+        my $s = substr( $text, $p++, 1 );
+
+        if ( $s eq '"' ) {
+            if (substr( $text, $p - 2, 1 ) eq '\\' ) {
+                next;
+            }
+
+            if ( $self->{incr_mode} != INCR_M_STR  ) {
+                $self->{incr_mode} = INCR_M_STR;
+            }
+            else {
+                $self->{incr_mode} = INCR_M_JSON;
+                unless ( $self->{incr_nest} ) {
+                    last;
+                }
+            }
+        }
+
+        if ( $self->{incr_mode} == INCR_M_JSON ) {
+
+            if ( $s eq '[' or $s eq '{' ) {
+                if ( ++$self->{incr_nest} > $coder->get_max_depth ) {
+                    Carp::croak('json text or perl structure exceeds maximum nesting level (max_depth set too low?)');
+                }
+            }
+            elsif ( $s eq ']' or $s eq '}' ) {
+                last if ( --$self->{incr_nest} <= 0 );
+            }
+            elsif ( $s eq '#' ) {
+                while ( $len > $p ) {
+                    last if substr( $text, $p++, 1 ) eq "\n";
+                }
+            }
+
+        }
+
+    }
+
+    $self->{incr_p} = $p;
+
+    return if ( $self->{incr_mode} == INCR_M_STR and not $self->{incr_nest} );
+    return if ( $self->{incr_mode} == INCR_M_JSON and $self->{incr_nest} > 0 );
+
+    return '' unless ( length substr( $self->{incr_text}, 0, $p ) );
+
+    local $Carp::CarpLevel = 2;
+
+    $self->{incr_p} = $restore;
+    $self->{incr_c} = $p;
+
+    my ( $obj, $tail ) = $coder->PP_decode_json( substr( $self->{incr_text}, 0, $p ), 0x10000001 );
+
+    $self->{incr_text} = substr( $self->{incr_text}, $p );
+    $self->{incr_p} = 0;
+
+    return $obj || '';
+}
+
+
+sub incr_text {
+    if ( $_[0]->{incr_parsing} ) {
+        Carp::croak("incr_text can not be called when the incremental parser already started parsing");
+    }
+    $_[0]->{incr_text};
+}
+
+
+sub incr_skip {
+    my $self  = shift;
+    $self->{incr_text} = substr( $self->{incr_text}, $self->{incr_c} );
+    $self->{incr_p} = 0;
+}
+
+
+sub incr_reset {
+    my $self = shift;
+    $self->{incr_text}    = undef;
+    $self->{incr_p}       = 0;
+    $self->{incr_mode}    = 0;
+    $self->{incr_nest}    = 0;
+    $self->{incr_parsing} = 0;
+}
+
+###############################
+
+
+1;
+__END__
+=pod
+
+=head1 NAME
+
+JSON::PP - JSON::XS compatible pure-Perl module.
+
+=head1 SYNOPSIS
+
+ use JSON::PP;
+
+ # exported functions, they croak on error
+ # and expect/generate UTF-8
+
+ $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref;
+ $perl_hash_or_arrayref  = decode_json $utf8_encoded_json_text;
+
+ # OO-interface
+
+ $coder = JSON::PP->new->ascii->pretty->allow_nonref;
+ 
+ $json_text   = $json->encode( $perl_scalar );
+ $perl_scalar = $json->decode( $json_text );
+ 
+ $pretty_printed = $json->pretty->encode( $perl_scalar ); # pretty-printing
+ 
+ # Note that JSON version 2.0 and above will automatically use
+ # JSON::XS or JSON::PP, so you should be able to just:
+ 
+ use JSON;
+
+
+=head1 VERSION
+
+    2.27200
+
+L<JSON::XS> 2.27 (~2.30) compatible.
+
+=head1 DESCRIPTION
+
+This module is L<JSON::XS> compatible pure Perl module.
+(Perl 5.8 or later is recommended)
+
+JSON::XS is the fastest and most proper JSON module on CPAN.
+It is written by Marc Lehmann in C, so must be compiled and
+installed in the used environment.
+
+JSON::PP is a pure-Perl module and has compatibility to JSON::XS.
+
+
+=head2 FEATURES
+
+=over
+
+=item * correct unicode handling
+
+This module knows how to handle Unicode (depending on Perl version).
+
+See to L<JSON::XS/A FEW NOTES ON UNICODE AND PERL> and
+L<UNICODE HANDLING ON PERLS>.
+
+
+=item * round-trip integrity
+
+When you serialise a perl data structure using only data types
+supported by JSON and Perl, the deserialised data structure is
+identical on the Perl level. (e.g. the string "2.0" doesn't suddenly
+become "2" just because it looks like a number). There I<are> minor
+exceptions to this, read the MAPPING section below to learn about
+those.
+
+
+=item * strict checking of JSON correctness
+
+There is no guessing, no generating of illegal JSON texts by default,
+and only JSON is accepted as input by default (the latter is a
+security feature). But when some options are set, loose checking
+features are available.
+
+=back
+
+=head1 FUNCTIONAL INTERFACE
+
+Some documents are copied and modified from L<JSON::XS/FUNCTIONAL INTERFACE>.
+
+=head2 encode_json
+
+    $json_text = encode_json $perl_scalar
+
+Converts the given Perl data structure to a UTF-8 encoded, binary string.
+
+This function call is functionally identical to:
+
+    $json_text = JSON::PP->new->utf8->encode($perl_scalar)
+
+=head2 decode_json
+
+    $perl_scalar = decode_json $json_text
+
+The opposite of C<encode_json>: expects an UTF-8 (binary) string and tries
+to parse that as an UTF-8 encoded JSON text, returning the resulting
+reference.
+
+This function call is functionally identical to:
+
+    $perl_scalar = JSON::PP->new->utf8->decode($json_text)
+
+=head2 JSON::PP::is_bool
+
+    $is_boolean = JSON::PP::is_bool($scalar)
+
+Returns true if the passed scalar represents either JSON::PP::true or
+JSON::PP::false, two constants that act like C<1> and C<0> respectively
+and are also used to represent JSON C<true> and C<false> in Perl strings.
+
+=head2 JSON::PP::true
+
+Returns JSON true value which is blessed object.
+It C<isa> JSON::PP::Boolean object.
+
+=head2 JSON::PP::false
+
+Returns JSON false value which is blessed object.
+It C<isa> JSON::PP::Boolean object.
+
+=head2 JSON::PP::null
+
+Returns C<undef>.
+
+See L<MAPPING>, below, for more information on how JSON values are mapped to
+Perl.
+
+
+=head1 HOW DO I DECODE A DATA FROM OUTER AND ENCODE TO OUTER
+
+This section supposes that your perl version is 5.8 or later.
+
+If you know a JSON text from an outer world - a network, a file content, and so on,
+is encoded in UTF-8, you should use C<decode_json> or C<JSON> module object
+with C<utf8> enable. And the decoded result will contain UNICODE characters.
+
+  # from network
+  my $json        = JSON::PP->new->utf8;
+  my $json_text   = CGI->new->param( 'json_data' );
+  my $perl_scalar = $json->decode( $json_text );
+  
+  # from file content
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  $json_text   = <$fh>;
+  $perl_scalar = decode_json( $json_text );
+
+If an outer data is not encoded in UTF-8, firstly you should C<decode> it.
+
+  use Encode;
+  local $/;
+  open( my $fh, '<', 'json.data' );
+  my $encoding = 'cp932';
+  my $unicode_json_text = decode( $encoding, <$fh> ); # UNICODE
+  
+  # or you can write the below code.
+  #
+  # open( my $fh, "<:encoding($encoding)", 'json.data' );
+  # $unicode_json_text = <$fh>;
+
+In this case, C<$unicode_json_text> is of course UNICODE string.
+So you B<cannot> use C<decode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable.
+
+  $perl_scalar = $json->utf8(0)->decode( $unicode_json_text );
+
+Or C<encode 'utf8'> and C<decode_json>:
+
+  $perl_scalar = decode_json( encode( 'utf8', $unicode_json_text ) );
+  # this way is not efficient.
+
+And now, you want to convert your C<$perl_scalar> into JSON data and
+send it to an outer world - a network or a file content, and so on.
+
+Your data usually contains UNICODE strings and you want the converted data to be encoded
+in UTF-8, you should use C<encode_json> or C<JSON> module object with C<utf8> enable.
+
+  print encode_json( $perl_scalar ); # to a network? file? or display?
+  # or
+  print $json->utf8->encode( $perl_scalar );
+
+If C<$perl_scalar> does not contain UNICODE but C<$encoding>-encoded strings
+for some reason, then its characters are regarded as B<latin1> for perl
+(because it does not concern with your $encoding).
+You B<cannot> use C<encode_json> nor C<JSON> module object with C<utf8> enable.
+Instead of them, you use C<JSON> module object with C<utf8> disable.
+Note that the resulted text is a UNICODE string but no problem to print it.
+
+  # $perl_scalar contains $encoding encoded string values
+  $unicode_json_text = $json->utf8(0)->encode( $perl_scalar );
+  # $unicode_json_text consists of characters less than 0x100
+  print $unicode_json_text;
+
+Or C<decode $encoding> all string values and C<encode_json>:
+
+  $perl_scalar->{ foo } = decode( $encoding, $perl_scalar->{ foo } );
+  # ... do it to each string values, then encode_json
+  $json_text = encode_json( $perl_scalar );
+
+This method is a proper way but probably not efficient.
+
+See to L<Encode>, L<perluniintro>.
+
+
+=head1 METHODS
+
+Basically, check to L<JSON> or L<JSON::XS>.
+
+=head2 new
+
+    $json = JSON::PP->new
+
+Returns a new JSON::PP object that can be used to de/encode JSON
+strings.
+
+All boolean flags described below are by default I<disabled>.
+
+The mutators for flags all return the JSON object again and thus calls can
+be chained:
+
+   my $json = JSON::PP->new->utf8->space_after->encode({a => [1,2]})
+   => {"a": [1, 2]}
+
+=head2 ascii
+
+    $json = $json->ascii([$enable])
+    
+    $enabled = $json->get_ascii
+
+If $enable is true (or missing), then the encode method will not generate characters outside
+the code range 0..127. Any Unicode characters outside that range will be escaped using either
+a single \uXXXX or a double \uHHHH\uLLLLL escape sequence, as per RFC4627.
+(See to L<JSON::XS/OBJECT-ORIENTED INTERFACE>).
+
+In Perl 5.005, there is no character having high value (more than 255).
+See to L<UNICODE HANDLING ON PERLS>.
+
+If $enable is false, then the encode method will not escape Unicode characters unless
+required by the JSON syntax or other flags. This results in a faster and more compact format.
+
+  JSON::PP->new->ascii(1)->encode([chr 0x10401])
+  => ["\ud801\udc01"]
+
+=head2 latin1
+
+    $json = $json->latin1([$enable])
+    
+    $enabled = $json->get_latin1
+
+If $enable is true (or missing), then the encode method will encode the resulting JSON
+text as latin1 (or iso-8859-1), escaping any characters outside the code range 0..255.
+
+If $enable is false, then the encode method will not escape Unicode characters
+unless required by the JSON syntax or other flags.
+
+  JSON::XS->new->latin1->encode (["\x{89}\x{abc}"]
+  => ["\x{89}\\u0abc"]    # (perl syntax, U+abc escaped, U+89 not)
+
+See to L<UNICODE HANDLING ON PERLS>.
+
+=head2 utf8
+
+    $json = $json->utf8([$enable])
+    
+    $enabled = $json->get_utf8
+
+If $enable is true (or missing), then the encode method will encode the JSON result
+into UTF-8, as required by many protocols, while the decode method expects to be handled
+an UTF-8-encoded string. Please note that UTF-8-encoded strings do not contain any
+characters outside the range 0..255, they are thus useful for bytewise/binary I/O.
+
+(In Perl 5.005, any character outside the range 0..255 does not exist.
+See to L<UNICODE HANDLING ON PERLS>.)
+
+In future versions, enabling this option might enable autodetection of the UTF-16 and UTF-32
+encoding families, as described in RFC4627.
+
+If $enable is false, then the encode method will return the JSON string as a (non-encoded)
+Unicode string, while decode expects thus a Unicode string. Any decoding or encoding
+(e.g. to UTF-8 or UTF-16) needs to be done yourself, e.g. using the Encode module.
+
+Example, output UTF-16BE-encoded JSON:
+
+  use Encode;
+  $jsontext = encode "UTF-16BE", JSON::PP->new->encode ($object);
+
+Example, decode UTF-32LE-encoded JSON:
+
+  use Encode;
+  $object = JSON::PP->new->decode (decode "UTF-32LE", $jsontext);
+
+
+=head2 pretty
+
+    $json = $json->pretty([$enable])
+
+This enables (or disables) all of the C<indent>, C<space_before> and
+C<space_after> flags in one call to generate the most readable
+(or most compact) form possible.
+
+Equivalent to:
+
+   $json->indent->space_before->space_after
+
+=head2 indent
+
+    $json = $json->indent([$enable])
+    
+    $enabled = $json->get_indent
+
+The default indent space length is three.
+You can use C<indent_length> to change the length.
+
+=head2 space_before
+
+    $json = $json->space_before([$enable])
+    
+    $enabled = $json->get_space_before
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space before the C<:> separating keys from values in JSON objects.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before enabled, space_after and indent disabled:
+
+   {"key" :"value"}
+
+=head2 space_after
+
+    $json = $json->space_after([$enable])
+    
+    $enabled = $json->get_space_after
+
+If C<$enable> is true (or missing), then the C<encode> method will add an extra
+optional space after the C<:> separating keys from values in JSON objects
+and extra whitespace after the C<,> separating key-value pairs and array
+members.
+
+If C<$enable> is false, then the C<encode> method will not add any extra
+space at those places.
+
+This setting has no effect when decoding JSON texts.
+
+Example, space_before and indent disabled, space_after enabled:
+
+   {"key": "value"}
+
+=head2 relaxed
+
+    $json = $json->relaxed([$enable])
+    
+    $enabled = $json->get_relaxed
+
+If C<$enable> is true (or missing), then C<decode> will accept some
+extensions to normal JSON syntax (see below). C<encode> will not be
+affected in anyway. I<Be aware that this option makes you accept invalid
+JSON texts as if they were valid!>. I suggest only to use this option to
+parse application-specific files written by humans (configuration files,
+resource files etc.)
+
+If C<$enable> is false (the default), then C<decode> will only accept
+valid JSON texts.
+
+Currently accepted extensions are:
+
+=over 4
+
+=item * list items can have an end-comma
+
+JSON I<separates> array elements and key-value pairs with commas. This
+can be annoying if you write JSON texts manually and want to be able to
+quickly append elements, so this extension accepts comma at the end of
+such items not just between them:
+
+   [
+      1,
+      2, <- this comma not normally allowed
+   ]
+   {
+      "k1": "v1",
+      "k2": "v2", <- this comma not normally allowed
+   }
+
+=item * shell-style '#'-comments
+
+Whenever JSON allows whitespace, shell-style comments are additionally
+allowed. They are terminated by the first carriage-return or line-feed
+character, after which more white-space and comments are allowed.
+
+  [
+     1, # this comment not allowed in JSON
+        # neither this one...
+  ]
+
+=back
+
+=head2 canonical
+
+    $json = $json->canonical([$enable])
+    
+    $enabled = $json->get_canonical
+
+If C<$enable> is true (or missing), then the C<encode> method will output JSON objects
+by sorting their keys. This is adding a comparatively high overhead.
+
+If C<$enable> is false, then the C<encode> method will output key-value
+pairs in the order Perl stores them (which will likely change between runs
+of the same script).
+
+This option is useful if you want the same data structure to be encoded as
+the same JSON text (given the same overall settings). If it is disabled,
+the same hash might be encoded differently even if contains the same data,
+as key-value pairs have no inherent ordering in Perl.
+
+This setting has no effect when decoding JSON texts.
+
+If you want your own sorting routine, you can give a code reference
+or a subroutine name to C<sort_by>. See to C<JSON::PP OWN METHODS>.
+
+=head2 allow_nonref
+
+    $json = $json->allow_nonref([$enable])
+    
+    $enabled = $json->get_allow_nonref
+
+If C<$enable> is true (or missing), then the C<encode> method can convert a
+non-reference into its corresponding string, number or null JSON value,
+which is an extension to RFC4627. Likewise, C<decode> will accept those JSON
+values instead of croaking.
+
+If C<$enable> is false, then the C<encode> method will croak if it isn't
+passed an arrayref or hashref, as JSON texts must either be an object
+or array. Likewise, C<decode> will croak if given something that is not a
+JSON object or array.
+
+   JSON::PP->new->allow_nonref->encode ("Hello, World!")
+   => "Hello, World!"
+
+=head2 allow_unknown
+
+    $json = $json->allow_unknown ([$enable])
+    
+    $enabled = $json->get_allow_unknown
+
+If $enable is true (or missing), then "encode" will *not* throw an
+exception when it encounters values it cannot represent in JSON (for
+example, filehandles) but instead will encode a JSON "null" value.
+Note that blessed objects are not included here and are handled
+separately by c<allow_nonref>.
+
+If $enable is false (the default), then "encode" will throw an
+exception when it encounters anything it cannot encode as JSON.
+
+This option does not affect "decode" in any way, and it is
+recommended to leave it off unless you know your communications
+partner.
+
+=head2 allow_blessed
+
+    $json = $json->allow_blessed([$enable])
+    
+    $enabled = $json->get_allow_blessed
+
+If C<$enable> is true (or missing), then the C<encode> method will not
+barf when it encounters a blessed reference. Instead, the value of the
+B<convert_blessed> option will decide whether C<null> (C<convert_blessed>
+disabled or no C<TO_JSON> method found) or a representation of the
+object (C<convert_blessed> enabled and C<TO_JSON> method found) is being
+encoded. Has no effect on C<decode>.
+
+If C<$enable> is false (the default), then C<encode> will throw an
+exception when it encounters a blessed object.
+
+=head2 convert_blessed
+
+    $json = $json->convert_blessed([$enable])
+    
+    $enabled = $json->get_convert_blessed
+
+If C<$enable> is true (or missing), then C<encode>, upon encountering a
+blessed object, will check for the availability of the C<TO_JSON> method
+on the object's class. If found, it will be called in scalar context
+and the resulting scalar will be encoded instead of the object. If no
+C<TO_JSON> method is found, the value of C<allow_blessed> will decide what
+to do.
+
+The C<TO_JSON> method may safely call die if it wants. If C<TO_JSON>
+returns other blessed objects, those will be handled in the same
+way. C<TO_JSON> must take care of not causing an endless recursion cycle
+(== crash) in this case. The name of C<TO_JSON> was chosen because other
+methods called by the Perl core (== not by the user of the object) are
+usually in upper case letters and to avoid collisions with the C<to_json>
+function or method.
+
+This setting does not yet influence C<decode> in any way.
+
+If C<$enable> is false, then the C<allow_blessed> setting will decide what
+to do when a blessed object is found.
+
+=head2 filter_json_object
+
+    $json = $json->filter_json_object([$coderef])
+
+When C<$coderef> is specified, it will be called from C<decode> each
+time it decodes a JSON object. The only argument passed to the coderef
+is a reference to the newly-created hash. If the code references returns
+a single scalar (which need not be a reference), this value
+(i.e. a copy of that scalar to avoid aliasing) is inserted into the
+deserialised data structure. If it returns an empty list
+(NOTE: I<not> C<undef>, which is a valid scalar), the original deserialised
+hash will be inserted. This setting can slow down decoding considerably.
+
+When C<$coderef> is omitted or undefined, any existing callback will
+be removed and C<decode> will not change the deserialised hash in any
+way.
+
+Example, convert all JSON objects into the integer 5:
+
+   my $js = JSON::PP->new->filter_json_object (sub { 5 });
+   # returns [5]
+   $js->decode ('[{}]'); # the given subroutine takes a hash reference.
+   # throw an exception because allow_nonref is not enabled
+   # so a lone 5 is not allowed.
+   $js->decode ('{"a":1, "b":2}');
+
+=head2 filter_json_single_key_object
+
+    $json = $json->filter_json_single_key_object($key [=> $coderef])
+
+Works remotely similar to C<filter_json_object>, but is only called for
+JSON objects having a single key named C<$key>.
+
+This C<$coderef> is called before the one specified via
+C<filter_json_object>, if any. It gets passed the single value in the JSON
+object. If it returns a single value, it will be inserted into the data
+structure. If it returns nothing (not even C<undef> but the empty list),
+the callback from C<filter_json_object> will be called next, as if no
+single-key callback were specified.
+
+If C<$coderef> is omitted or undefined, the corresponding callback will be
+disabled. There can only ever be one callback for a given key.
+
+As this callback gets called less often then the C<filter_json_object>
+one, decoding speed will not usually suffer as much. Therefore, single-key
+objects make excellent targets to serialise Perl objects into, especially
+as single-key JSON objects are as close to the type-tagged value concept
+as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not
+support this in any way, so you need to make sure your data never looks
+like a serialised Perl hash.
+
+Typical names for the single object key are C<__class_whatever__>, or
+C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even
+things like C<__class_md5sum(classname)__>, to reduce the risk of clashing
+with real hashes.
+
+Example, decode JSON objects of the form C<< { "__widget__" => <id> } >>
+into the corresponding C<< $WIDGET{<id>} >> object:
+
+   # return whatever is in $WIDGET{5}:
+   JSON::PP
+      ->new
+      ->filter_json_single_key_object (__widget__ => sub {
+            $WIDGET{ $_[0] }
+         })
+      ->decode ('{"__widget__": 5')
+
+   # this can be used with a TO_JSON method in some "widget" class
+   # for serialisation to json:
+   sub WidgetBase::TO_JSON {
+      my ($self) = @_;
+
+      unless ($self->{id}) {
+         $self->{id} = ..get..some..id..;
+         $WIDGET{$self->{id}} = $self;
+      }
+
+      { __widget__ => $self->{id} }
+   }
+
+=head2 shrink
+
+    $json = $json->shrink([$enable])
+    
+    $enabled = $json->get_shrink
+
+In JSON::XS, this flag resizes strings generated by either
+C<encode> or C<decode> to their minimum size possible.
+It will also try to downgrade any strings to octet-form if possible.
+
+In JSON::PP, it is noop about resizing strings but tries
+C<utf8::downgrade> to the returned string by C<encode>.
+See to L<utf8>.
+
+See to L<JSON::XS/OBJECT-ORIENTED INTERFACE>
+
+=head2 max_depth
+
+    $json = $json->max_depth([$maximum_nesting_depth])
+    
+    $max_depth = $json->get_max_depth
+
+Sets the maximum nesting level (default C<512>) accepted while encoding
+or decoding. If a higher nesting level is detected in JSON text or a Perl
+data structure, then the encoder and decoder will stop and croak at that
+point.
+
+Nesting level is defined by number of hash- or arrayrefs that the encoder
+needs to traverse to reach a given point or the number of C<{> or C<[>
+characters without their matching closing parenthesis crossed to reach a
+given character in a string.
+
+If no argument is given, the highest possible setting will be used, which
+is rarely useful.
+
+See L<JSON::XS/SSECURITY CONSIDERATIONS> for more info on why this is useful.
+
+When a large value (100 or more) was set and it de/encodes a deep nested object/text,
+it may raise a warning 'Deep recursion on subroutine' at the perl runtime phase.
+
+=head2 max_size
+
+    $json = $json->max_size([$maximum_string_size])
+    
+    $max_size = $json->get_max_size
+
+Set the maximum length a JSON text may have (in bytes) where decoding is
+being attempted. The default is C<0>, meaning no limit. When C<decode>
+is called on a string that is longer then this many bytes, it will not
+attempt to decode the string but throw an exception. This setting has no
+effect on C<encode> (yet).
+
+If no argument is given, the limit check will be deactivated (same as when
+C<0> is specified).
+
+See L<JSON::XS/SECURITY CONSIDERATIONS> for more info on why this is useful.
+
+=head2 encode
+
+    $json_text = $json->encode($perl_scalar)
+
+Converts the given Perl data structure (a simple scalar or a reference
+to a hash or array) to its JSON representation. Simple scalars will be
+converted into JSON string or number sequences, while references to arrays
+become JSON arrays and references to hashes become JSON objects. Undefined
+Perl values (e.g. C<undef>) become JSON C<null> values.
+References to the integers C<0> and C<1> are converted into C<true> and C<false>.
+
+=head2 decode
+
+    $perl_scalar = $json->decode($json_text)
+
+The opposite of C<encode>: expects a JSON text and tries to parse it,
+returning the resulting simple scalar or reference. Croaks on error.
+
+JSON numbers and strings become simple Perl scalars. JSON arrays become
+Perl arrayrefs and JSON objects become Perl hashrefs. C<true> becomes
+C<1> (C<JSON::true>), C<false> becomes C<0> (C<JSON::false>) and
+C<null> becomes C<undef>.
+
+=head2 decode_prefix
+
+    ($perl_scalar, $characters) = $json->decode_prefix($json_text)
+
+This works like the C<decode> method, but instead of raising an exception
+when there is trailing garbage after the first JSON object, it will
+silently stop parsing there and return the number of characters consumed
+so far.
+
+   JSON->new->decode_prefix ("[1] the tail")
+   => ([], 3)
+
+=head1 INCREMENTAL PARSING
+
+Most of this section are copied and modified from L<JSON::XS/INCREMENTAL PARSING>.
+
+In some cases, there is the need for incremental parsing of JSON texts.
+This module does allow you to parse a JSON stream incrementally.
+It does so by accumulating text until it has a full JSON object, which
+it then can decode. This process is similar to using C<decode_prefix>
+to see if a full JSON object is available, but is much more efficient
+(and can be implemented with a minimum of method calls).
+
+This module will only attempt to parse the JSON text once it is sure it
+has enough text to get a decisive result, using a very simple but
+truly incremental parser. This means that it sometimes won't stop as
+early as the full parser, for example, it doesn't detect parenthesis
+mismatches. The only thing it guarantees is that it starts decoding as
+soon as a syntactically valid JSON text has been seen. This means you need
+to set resource limits (e.g. C<max_size>) to ensure the parser will stop
+parsing in the presence if syntax errors.
+
+The following methods implement this incremental parser.
+
+=head2 incr_parse
+
+    $json->incr_parse( [$string] ) # void context
+    
+    $obj_or_undef = $json->incr_parse( [$string] ) # scalar context
+    
+    @obj_or_empty = $json->incr_parse( [$string] ) # list context
+
+This is the central parsing function. It can both append new text and
+extract objects from the stream accumulated so far (both of these
+functions are optional).
+
+If C<$string> is given, then this string is appended to the already
+existing JSON fragment stored in the C<$json> object.
+
+After that, if the function is called in void context, it will simply
+return without doing anything further. This can be used to add more text
+in as many chunks as you want.
+
+If the method is called in scalar context, then it will try to extract
+exactly I<one> JSON object. If that is successful, it will return this
+object, otherwise it will return C<undef>. If there is a parse error,
+this method will croak just as C<decode> would do (one can then use
+C<incr_skip> to skip the erroneous part). This is the most common way of
+using the method.
+
+And finally, in list context, it will try to extract as many objects
+from the stream as it can find and return them, or the empty list
+otherwise. For this to work, there must be no separators between the JSON
+objects or arrays, instead they must be concatenated back-to-back. If
+an error occurs, an exception will be raised as in the scalar context
+case. Note that in this case, any previously-parsed JSON texts will be
+lost.
+
+Example: Parse some JSON arrays/objects in a given string and return them.
+
+    my @objs = JSON->new->incr_parse ("[5][7][1,2]");
+
+=head2 incr_text
+
+    $lvalue_string = $json->incr_text
+
+This method returns the currently stored JSON fragment as an lvalue, that
+is, you can manipulate it. This I<only> works when a preceding call to
+C<incr_parse> in I<scalar context> successfully returned an object. Under
+all other circumstances you must not call this function (I mean it.
+although in simple tests it might actually work, it I<will> fail under
+real world conditions). As a special exception, you can also call this
+method before having parsed anything.
+
+This function is useful in two cases: a) finding the trailing text after a
+JSON object or b) parsing multiple JSON objects separated by non-JSON text
+(such as commas).
+
+    $json->incr_text =~ s/\s*,\s*//;
+
+In Perl 5.005, C<lvalue> attribute is not available.
+You must write codes like the below:
+
+    $string = $json->incr_text;
+    $string =~ s/\s*,\s*//;
+    $json->incr_text( $string );
+
+=head2 incr_skip
+
+    $json->incr_skip
+
+This will reset the state of the incremental parser and will remove the
+parsed text from the input buffer. This is useful after C<incr_parse>
+died, in which case the input buffer and incremental parser state is left
+unchanged, to skip the text parsed so far and to reset the parse state.
+
+=head2 incr_reset
+
+    $json->incr_reset
+
+This completely resets the incremental parser, that is, after this call,
+it will be as if the parser had never parsed anything.
+
+This is useful if you want to repeatedly parse JSON objects and want to
+ignore any trailing data, which means you have to reset the parser after
+each successful decode.
+
+See to L<JSON::XS/INCREMENTAL PARSING> for examples.
+
+
+=head1 JSON::PP OWN METHODS
+
+=head2 allow_singlequote
+
+    $json = $json->allow_singlequote([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+JSON strings quoted by single quotations that are invalid JSON
+format.
+
+    $json->allow_singlequote->decode({"foo":'bar'});
+    $json->allow_singlequote->decode({'foo':"bar"});
+    $json->allow_singlequote->decode({'foo':'bar'});
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+
+=head2 allow_barekey
+
+    $json = $json->allow_barekey([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will accept
+bare keys of JSON object that are invalid JSON format.
+
+As same as the C<relaxed> option, this option may be used to parse
+application-specific files written by humans.
+
+    $json->allow_barekey->decode('{foo:"bar"}');
+
+=head2 allow_bignum
+
+    $json = $json->allow_bignum([$enable])
+
+If C<$enable> is true (or missing), then C<decode> will convert
+the big integer Perl cannot handle as integer into a L<Math::BigInt>
+object and convert a floating number (any) into a L<Math::BigFloat>.
+
+On the contrary, C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers with C<allow_blessed> enable.
+
+   $json->allow_nonref->allow_blessed->allow_bignum;
+   $bigfloat = $json->decode('2.000000000000000000000000001');
+   print $json->encode($bigfloat);
+   # => 2.000000000000000000000000001
+
+See to L<JSON::XS/MAPPING> about the normal conversion of JSON number.
+
+=head2 loose
+
+    $json = $json->loose([$enable])
+
+The unescaped [\x00-\x1f\x22\x2f\x5c] strings are invalid in JSON strings
+and the module doesn't allow to C<decode> to these (except for \x2f).
+If C<$enable> is true (or missing), then C<decode>  will accept these
+unescaped strings.
+
+    $json->loose->decode(qq|["abc
+                                   def"]|);
+
+See L<JSON::XS/SSECURITY CONSIDERATIONS>.
+
+=head2 escape_slash
+
+    $json = $json->escape_slash([$enable])
+
+According to JSON Grammar, I<slash> (U+002F) is escaped. But default
+JSON::PP (as same as JSON::XS) encodes strings without escaping slash.
+
+If C<$enable> is true (or missing), then C<encode> will escape slashes.
+
+=head2 indent_length
+
+    $json = $json->indent_length($length)
+
+JSON::XS indent space length is 3 and cannot be changed.
+JSON::PP set the indent space length with the given $length.
+The default is 3. The acceptable range is 0 to 15.
+
+=head2 sort_by
+
+    $json = $json->sort_by($function_name)
+    $json = $json->sort_by($subroutine_ref)
+
+If $function_name or $subroutine_ref are set, its sort routine are used
+in encoding JSON objects.
+
+   $js = $pc->sort_by(sub { $JSON::PP::a cmp $JSON::PP::b })->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   $js = $pc->sort_by('own_sort')->encode($obj);
+   # is($js, q|{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8,"i":9}|);
+
+   sub JSON::PP::own_sort { $JSON::PP::a cmp $JSON::PP::b }
+
+As the sorting routine runs in the JSON::PP scope, the given
+subroutine name and the special variables C<$a>, C<$b> will begin
+'JSON::PP::'.
+
+If $integer is set, then the effect is same as C<canonical> on.
+
+=head1 INTERNAL
+
+For developers.
+
+=over
+
+=item PP_encode_box
+
+Returns
+
+        {
+            depth        => $depth,
+            indent_count => $indent_count,
+        }
+
+
+=item PP_decode_box
+
+Returns
+
+        {
+            text    => $text,
+            at      => $at,
+            ch      => $ch,
+            len     => $len,
+            depth   => $depth,
+            encoding      => $encoding,
+            is_valid_utf8 => $is_valid_utf8,
+        };
+
+=back
+
+=head1 MAPPING
+
+This section is copied from JSON::XS and modified to C<JSON::PP>.
+JSON::XS and JSON::PP mapping mechanisms are almost equivalent.
+
+See to L<JSON::XS/MAPPING>.
+
+=head2 JSON -> PERL
+
+=over 4
+
+=item object
+
+A JSON object becomes a reference to a hash in Perl. No ordering of object
+keys is preserved (JSON does not preserver object key ordering itself).
+
+=item array
+
+A JSON array becomes a reference to an array in Perl.
+
+=item string
+
+A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON
+are represented by the same codepoints in the Perl string, so no manual
+decoding is necessary.
+
+=item number
+
+A JSON number becomes either an integer, numeric (floating point) or
+string scalar in perl, depending on its range and any fractional parts. On
+the Perl level, there is no difference between those as Perl handles all
+the conversion details, but an integer may take slightly less memory and
+might represent more values exactly than floating point numbers.
+
+If the number consists of digits only, C<JSON> will try to represent
+it as an integer value. If that fails, it will try to represent it as
+a numeric (floating point) value if that is possible without loss of
+precision. Otherwise it will preserve the number as a string value (in
+which case you lose roundtripping ability, as the JSON number will be
+re-encoded to a JSON string).
+
+Numbers containing a fractional or exponential part will always be
+represented as numeric (floating point) values, possibly at a loss of
+precision (in which case you might lose perfect roundtripping ability, but
+the JSON number will still be re-encoded as a JSON number).
+
+Note that precision is not accuracy - binary floating point values cannot
+represent most decimal fractions exactly, and when converting from and to
+floating point, C<JSON> only guarantees precision up to but not including
+the least significant bit.
+
+When C<allow_bignum> is enable, the big integers 
+and the numeric can be optionally converted into L<Math::BigInt> and
+L<Math::BigFloat> objects.
+
+=item true, false
+
+These JSON atoms become C<JSON::PP::true> and C<JSON::PP::false>,
+respectively. They are overloaded to act almost exactly like the numbers
+C<1> and C<0>. You can check whether a scalar is a JSON boolean by using
+the C<JSON::is_bool> function.
+
+   print JSON::PP::true . "\n";
+    => true
+   print JSON::PP::true + 1;
+    => 1
+
+   ok(JSON::true eq  '1');
+   ok(JSON::true == 1);
+
+C<JSON> will install these missing overloading features to the backend modules.
+
+
+=item null
+
+A JSON null atom becomes C<undef> in Perl.
+
+C<JSON::PP::null> returns C<undef>.
+
+=back
+
+
+=head2 PERL -> JSON
+
+The mapping from Perl to JSON is slightly more difficult, as Perl is a
+truly typeless language, so we can only guess which JSON type is meant by
+a Perl value.
+
+=over 4
+
+=item hash references
+
+Perl hash references become JSON objects. As there is no inherent ordering
+in hash keys (or JSON objects), they will usually be encoded in a
+pseudo-random order that can change between runs of the same program but
+stays generally the same within a single run of a program. C<JSON>
+optionally sort the hash keys (determined by the I<canonical> flag), so
+the same data structure will serialise to the same JSON text (given same
+settings and version of JSON::XS), but this incurs a runtime overhead
+and is only rarely useful, e.g. when you want to compare some JSON text
+against another for equality.
+
+
+=item array references
+
+Perl array references become JSON arrays.
+
+=item other references
+
+Other unblessed references are generally not allowed and will cause an
+exception to be thrown, except for references to the integers C<0> and
+C<1>, which get turned into C<false> and C<true> atoms in JSON. You can
+also use C<JSON::false> and C<JSON::true> to improve readability.
+
+   to_json [\0,JSON::PP::true]      # yields [false,true]
+
+=item JSON::PP::true, JSON::PP::false, JSON::PP::null
+
+These special values become JSON true and JSON false values,
+respectively. You can also use C<\1> and C<\0> directly if you want.
+
+JSON::PP::null returns C<undef>.
+
+=item blessed objects
+
+Blessed objects are not directly representable in JSON. See the
+C<allow_blessed> and C<convert_blessed> methods on various options on
+how to deal with this: basically, you can choose between throwing an
+exception, encoding the reference as if it weren't blessed, or provide
+your own serialiser method.
+
+See to L<convert_blessed>.
+
+=item simple scalars
+
+Simple Perl scalars (any scalar that is not a reference) are the most
+difficult objects to encode: JSON::XS and JSON::PP will encode undefined scalars as
+JSON C<null> values, scalars that have last been used in a string context
+before encoding as JSON strings, and anything else as number value:
+
+   # dump as number
+   encode_json [2]                      # yields [2]
+   encode_json [-3.0e17]                # yields [-3e+17]
+   my $value = 5; encode_json [$value]  # yields [5]
+
+   # used as string, so dump as string
+   print $value;
+   encode_json [$value]                 # yields ["5"]
+
+   # undef becomes null
+   encode_json [undef]                  # yields [null]
+
+You can force the type to be a string by stringifying it:
+
+   my $x = 3.1; # some variable containing a number
+   "$x";        # stringified
+   $x .= "";    # another, more awkward way to stringify
+   print $x;    # perl does it for you, too, quite often
+
+You can force the type to be a number by numifying it:
+
+   my $x = "3"; # some variable containing a string
+   $x += 0;     # numify it, ensuring it will be dumped as a number
+   $x *= 1;     # same thing, the choice is yours.
+
+You can not currently force the type in other, less obscure, ways.
+
+Note that numerical precision has the same meaning as under Perl (so
+binary to decimal conversion follows the same rules as in Perl, which
+can differ to other languages). Also, your perl interpreter might expose
+extensions to the floating point numbers of your platform, such as
+infinities or NaN's - these cannot be represented in JSON, and it is an
+error to pass those in.
+
+=item Big Number
+
+When C<allow_bignum> is enable, 
+C<encode> converts C<Math::BigInt> objects and C<Math::BigFloat>
+objects into JSON numbers.
+
+
+=back
+
+=head1 UNICODE HANDLING ON PERLS
+
+If you do not know about Unicode on Perl well,
+please check L<JSON::XS/A FEW NOTES ON UNICODE AND PERL>.
+
+=head2 Perl 5.8 and later
+
+Perl can handle Unicode and the JSON::PP de/encode methods also work properly.
+
+    $json->allow_nonref->encode(chr hex 3042);
+    $json->allow_nonref->encode(chr hex 12345);
+
+Returns C<"\u3042"> and C<"\ud808\udf45"> respectively.
+
+    $json->allow_nonref->decode('"\u3042"');
+    $json->allow_nonref->decode('"\ud808\udf45"');
+
+Returns UTF-8 encoded strings with UTF8 flag, regarded as C<U+3042> and C<U+12345>.
+
+Note that the versions from Perl 5.8.0 to 5.8.2, Perl built-in C<join> was broken,
+so JSON::PP wraps the C<join> with a subroutine. Thus JSON::PP works slow in the versions.
+
+
+=head2 Perl 5.6
+
+Perl can handle Unicode and the JSON::PP de/encode methods also work.
+
+=head2 Perl 5.005
+
+Perl 5.005 is a byte semantics world -- all strings are sequences of bytes.
+That means the unicode handling is not available.
+
+In encoding,
+
+    $json->allow_nonref->encode(chr hex 3042);  # hex 3042 is 12354.
+    $json->allow_nonref->encode(chr hex 12345); # hex 12345 is 74565.
+
+Returns C<B> and C<E>, as C<chr> takes a value more than 255, it treats
+as C<$value % 256>, so the above codes are equivalent to :
+
+    $json->allow_nonref->encode(chr 66);
+    $json->allow_nonref->encode(chr 69);
+
+In decoding,
+
+    $json->decode('"\u00e3\u0081\u0082"');
+
+The returned is a byte sequence C<0xE3 0x81 0x82> for UTF-8 encoded
+japanese character (C<HIRAGANA LETTER A>).
+And if it is represented in Unicode code point, C<U+3042>.
+
+Next, 
+
+    $json->decode('"\u3042"');
+
+We ordinary expect the returned value is a Unicode character C<U+3042>.
+But here is 5.005 world. This is C<0xE3 0x81 0x82>.
+
+    $json->decode('"\ud808\udf45"');
+
+This is not a character C<U+12345> but bytes - C<0xf0 0x92 0x8d 0x85>.
+
+
+=head1 TODO
+
+=over
+
+=item speed
+
+=item memory saving
+
+=back
+
+
+=head1 SEE ALSO
+
+Most of the document are copied and modified from JSON::XS doc.
+
+L<JSON::XS>
+
+RFC4627 (L<http://www.ietf.org/rfc/rfc4627.txt>)
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2007-2012 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut

uroman/lib/JSON/backportPP/Boolean.pm

@@ -0,0 +1,27 @@
+=head1 NAME
+
+JSON::PP::Boolean - dummy module providing JSON::PP::Boolean
+
+=head1 SYNOPSIS
+
+ # do not "use" yourself
+
+=head1 DESCRIPTION
+
+This module exists only to provide overload resolution for Storable
+and similar modules. See L<JSON::PP> for more info about this class.
+
+=cut
+
+use JSON::backportPP ();
+use strict;
+
+1;
+
+=head1 AUTHOR
+
+This idea is from L<JSON::XS::Boolean> written by
+Marc Lehmann <schmorp[at]schmorp.de>
+
+=cut
+

uroman/lib/JSON/backportPP/Compat5005.pm

@@ -0,0 +1,131 @@
+package # This is JSON::backportPP
+    JSON::backportPP5005;
+
+use 5.005;
+use strict;
+
+my @properties;
+
+$JSON::PP5005::VERSION = '1.10';
+
+BEGIN {
+
+    sub utf8::is_utf8 {
+        0; # It is considered that UTF8 flag off for Perl 5.005.
+    }
+
+    sub utf8::upgrade {
+    }
+
+    sub utf8::downgrade {
+        1; # must always return true.
+    }
+
+    sub utf8::encode  {
+    }
+
+    sub utf8::decode {
+    }
+
+    *JSON::PP::JSON_PP_encode_ascii      = \&_encode_ascii;
+    *JSON::PP::JSON_PP_encode_latin1     = \&_encode_latin1;
+    *JSON::PP::JSON_PP_decode_surrogates = \&_decode_surrogates;
+    *JSON::PP::JSON_PP_decode_unicode    = \&_decode_unicode;
+
+    # missing in B module.
+    sub B::SVp_IOK () { 0x01000000; }
+    sub B::SVp_NOK () { 0x02000000; }
+    sub B::SVp_POK () { 0x04000000; }
+
+    $INC{'bytes.pm'} = 1; # dummy
+}
+
+
+
+sub _encode_ascii {
+    join('', map { $_ <= 127 ? chr($_) : sprintf('\u%04x', $_) } unpack('C*', $_[0]) );
+}
+
+
+sub _encode_latin1 {
+    join('', map { chr($_) } unpack('C*', $_[0]) );
+}
+
+
+sub _decode_surrogates { # from http://homepage1.nifty.com/nomenclator/unicode/ucs_utf.htm
+    my $uni = 0x10000 + (hex($_[0]) - 0xD800) * 0x400 + (hex($_[1]) - 0xDC00); # from perlunicode
+    my $bit = unpack('B32', pack('N', $uni));
+
+    if ( $bit =~ /^00000000000(...)(......)(......)(......)$/ ) {
+        my ($w, $x, $y, $z) = ($1, $2, $3, $4);
+        return pack('B*', sprintf('11110%s10%s10%s10%s', $w, $x, $y, $z));
+    }
+    else {
+        Carp::croak("Invalid surrogate pair");
+    }
+}
+
+
+sub _decode_unicode {
+    my ($u) = @_;
+    my ($utf8bit);
+
+    if ( $u =~ /^00([89a-f][0-9a-f])$/i ) { # 0x80-0xff
+         return pack( 'H2', $1 );
+    }
+
+    my $bit = unpack("B*", pack("H*", $u));
+
+    if ( $bit =~ /^00000(.....)(......)$/ ) {
+        $utf8bit = sprintf('110%s10%s', $1, $2);
+    }
+    elsif ( $bit =~ /^(....)(......)(......)$/ ) {
+        $utf8bit = sprintf('1110%s10%s10%s', $1, $2, $3);
+    }
+    else {
+        Carp::croak("Invalid escaped unicode");
+    }
+
+    return pack('B*', $utf8bit);
+}
+
+
+sub JSON::PP::incr_text {
+    $_[0]->{_incr_parser} ||= JSON::PP::IncrParser->new;
+
+    if ( $_[0]->{_incr_parser}->{incr_parsing} ) {
+        Carp::croak("incr_text can not be called when the incremental parser already started parsing");
+    }
+
+    $_[0]->{_incr_parser}->{incr_text} = $_[1] if ( @_ > 1 );
+    $_[0]->{_incr_parser}->{incr_text};
+}
+
+
+1;
+__END__
+
+=pod
+
+=head1 NAME
+
+JSON::PP5005 - Helper module in using JSON::PP in Perl 5.005
+
+=head1 DESCRIPTION
+
+JSON::PP calls internally.
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2007-2012 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut
+

uroman/lib/JSON/backportPP/Compat5006.pm

@@ -0,0 +1,173 @@
+package # This is JSON::backportPP
+    JSON::backportPP56;
+
+use 5.006;
+use strict;
+
+my @properties;
+
+$JSON::PP56::VERSION = '1.08';
+
+BEGIN {
+
+    sub utf8::is_utf8 {
+        my $len =  length $_[0]; # char length
+        {
+            use bytes; #  byte length;
+            return $len != length $_[0]; # if !=, UTF8-flagged on.
+        }
+    }
+
+
+    sub utf8::upgrade {
+        ; # noop;
+    }
+
+
+    sub utf8::downgrade ($;$) {
+        return 1 unless ( utf8::is_utf8( $_[0] ) );
+
+        if ( _is_valid_utf8( $_[0] ) ) {
+            my $downgrade;
+            for my $c ( unpack( "U*", $_[0] ) ) {
+                if ( $c < 256 ) {
+                    $downgrade .= pack("C", $c);
+                }
+                else {
+                    $downgrade .= pack("U", $c);
+                }
+            }
+            $_[0] = $downgrade;
+            return 1;
+        }
+        else {
+            Carp::croak("Wide character in subroutine entry") unless ( $_[1] );
+            0;
+        }
+    }
+
+
+    sub utf8::encode ($) { # UTF8 flag off
+        if ( utf8::is_utf8( $_[0] ) ) {
+            $_[0] = pack( "C*", unpack( "C*", $_[0] ) );
+        }
+        else {
+            $_[0] = pack( "U*", unpack( "C*", $_[0] ) );
+            $_[0] = pack( "C*", unpack( "C*", $_[0] ) );
+        }
+    }
+
+
+    sub utf8::decode ($) { # UTF8 flag on
+        if ( _is_valid_utf8( $_[0] ) ) {
+            utf8::downgrade( $_[0] );
+            $_[0] = pack( "U*", unpack( "U*", $_[0] ) );
+        }
+    }
+
+
+    *JSON::PP::JSON_PP_encode_ascii      = \&_encode_ascii;
+    *JSON::PP::JSON_PP_encode_latin1     = \&_encode_latin1;
+    *JSON::PP::JSON_PP_decode_surrogates = \&JSON::PP::_decode_surrogates;
+    *JSON::PP::JSON_PP_decode_unicode    = \&JSON::PP::_decode_unicode;
+
+    unless ( defined &B::SVp_NOK ) { # missing in B module.
+        eval q{ sub B::SVp_NOK () { 0x02000000; } };
+    }
+
+}
+
+
+
+sub _encode_ascii {
+    join('',
+        map {
+            $_ <= 127 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', JSON::PP::_encode_surrogates($_));
+        } _unpack_emu($_[0])
+    );
+}
+
+
+sub _encode_latin1 {
+    join('',
+        map {
+            $_ <= 255 ?
+                chr($_) :
+            $_ <= 65535 ?
+                sprintf('\u%04x', $_) : sprintf('\u%x\u%x', JSON::PP::_encode_surrogates($_));
+        } _unpack_emu($_[0])
+    );
+}
+
+
+sub _unpack_emu { # for Perl 5.6 unpack warnings
+    return   !utf8::is_utf8($_[0]) ? unpack('C*', $_[0]) 
+           : _is_valid_utf8($_[0]) ? unpack('U*', $_[0])
+           : unpack('C*', $_[0]);
+}
+
+
+sub _is_valid_utf8 {
+    my $str = $_[0];
+    my $is_utf8;
+
+    while ($str =~ /(?:
+          (
+             [\x00-\x7F]
+            |[\xC2-\xDF][\x80-\xBF]
+            |[\xE0][\xA0-\xBF][\x80-\xBF]
+            |[\xE1-\xEC][\x80-\xBF][\x80-\xBF]
+            |[\xED][\x80-\x9F][\x80-\xBF]
+            |[\xEE-\xEF][\x80-\xBF][\x80-\xBF]
+            |[\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF]
+            |[\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF]
+          )
+        | (.)
+    )/xg)
+    {
+        if (defined $1) {
+            $is_utf8 = 1 if (!defined $is_utf8);
+        }
+        else {
+            $is_utf8 = 0 if (!defined $is_utf8);
+            if ($is_utf8) { # eventually, not utf8
+                return;
+            }
+        }
+    }
+
+    return $is_utf8;
+}
+
+
+1;
+__END__
+
+=pod
+
+=head1 NAME
+
+JSON::PP56 - Helper module in using JSON::PP in Perl 5.6
+
+=head1 DESCRIPTION
+
+JSON::PP calls internally.
+
+=head1 AUTHOR
+
+Makamaka Hannyaharamitu, E<lt>makamaka[at]cpan.orgE<gt>
+
+
+=head1 COPYRIGHT AND LICENSE
+
+Copyright 2007-2012 by Makamaka Hannyaharamitu
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut
+

uroman/lib/NLP/Chinese.pm

@@ -0,0 +1,239 @@
+################################################################
+#                                                              #
+# Chinese                                                      #
+#                                                              #
+################################################################
+
+package NLP::Chinese;
+
+$utf8 = NLP::UTF8;
+%empty_ht = ();
+
+sub read_chinese_tonal_pinyin_files {
+   local($caller, *ht, @filenames) = @_;
+
+   $n_kHanyuPinlu = 0;
+   $n_kXHC1983 = 0;
+   $n_kHanyuPinyin = 0;
+   $n_kMandarin = 0;
+   $n_cedict = 0;
+   $n_simple_pinyin = 0;
+
+   foreach $filename (@filenames) {
+      if ($filename =~ /unihan/i) {
+	 my $line_number = 0;
+         if (open(IN, $filename)) {
+            while (<IN>) {
+	       $line_number++;
+               next if /^#/;
+               s/\s*$//;
+               if (($u, $type, $value) = split(/\t/, $_)) {
+                  if ($type =~ /^(kHanyuPinlu|kXHC1983|kHanyuPinyin|kMandarin)$/) {
+	             $u = $util->trim($u);
+	             $type = $util->trim($type);
+	             $value = $util->trim($value);
+                     $f = $utf8->unicode_string2string($u);
+
+                     if ($type eq "kHanyuPinlu") {
+	                $value =~ s/\(.*?\)//g;
+			$value = $util->trim($value);
+                        $translit = $caller->number_to_accent_tone($value);
+	                $ht{"kHanyuPinlu"}->{$f} = $translit;
+	                $n_kHanyuPinlu++;
+                     } elsif ($type eq "kXHC1983") {
+	                @translits = ($value =~ /:(\S+)/g);
+	                $translit = join(" ", @translits);
+	                $ht{"kXHC1983"}->{$f} = $translit;
+	                $n_kXHC1983++;
+                     } elsif ($type eq "kHanyuPinyin") {
+	                $value =~ s/^.*://;
+	                $value =~ s/,/ /g;
+	                $ht{"kHanyuPinyin"}->{$f} = $value;
+	                $n_kHanyuPinyin++;
+                     } elsif ($type eq "kMandarin") {
+			$ht{"kMandarin"}->{$f} = $value;
+			$n_kMandarin++;
+                     }
+                  }
+               }
+            }
+            close(IN);
+            print "Read in $n_kHanyuPinlu kHanyuPinlu, $n_kXHC1983 n_kXHC1983, $n_kHanyuPinyin n_kHanyuPinyin $n_kMandarin n_kMandarin\n";
+         } else {
+	    print STDERR "Can't open $filename\n";
+	 }
+      } elsif ($filename =~ /cedict/i) {
+         if (open(IN, $filename)) {
+	    my $line_number = 0;
+            while (<IN>) {
+	       $line_number++;
+               next if /^#/;
+               s/\s*$//;
+               if (($f, $translit) = ($_ =~ /^\S+\s+(\S+)\s+\[([^\[\]]+)\]/)) {
+                  $translit = $utf8->extended_lower_case($translit);
+                  $translit = $caller->number_to_accent_tone($translit);
+                  $translit =~ s/\s//g;
+		  if ($old_translit = $ht{"cedict"}->{$f}) {
+                     # $ht{CONFLICT}->{("DUPLICATE " . $f)} = "CEDICT($f): $old_translit\nCEDICT($f): $translit (duplicate)\n" unless $translit eq $old_translit;
+                     $ht{"cedicts"}->{$f} = join(" ", $ht{"cedicts"}->{$f}, $translit) unless $old_translit eq $translit;
+		  } else {
+                     $ht{"cedict"}->{$f} = $translit;
+                     $ht{"cedicts"}->{$f} = $translit;
+		  }
+                  $n_cedict++;
+	       }
+            }
+            close(IN);
+            # print "Read in $n_cedict n_cedict\n";
+         } else {
+	    print STDERR "Can't open $filename";
+         }
+      } elsif ($filename =~ /chinese_to_pinyin/i) {
+	 if (open(IN, $filename)) {
+	    my $line_number = 0;
+	    while (<IN>) {
+	       $line_number++;
+	       next if /^#/;
+	       if (($f, $translit) = ($_ =~ /^(\S+)\t(\S+)\s*$/)) {
+		  $ht{"simple_pinyin"}->{$f} = $translit; 
+		  $n_simple_pinyin++;
+	       }
+	    }
+	    close(IN);
+	    # print "Read in $n_simple_pinyin n_simple_pinyin\n";
+	 } else {
+	    print STDERR "Can't open $filename";
+	 }
+      } else {
+	 print STDERR "Don't know what to do with file $filename (in read_chinese_tonal_pinyin_files)\n";
+      }
+   }
+}
+
+sub tonal_pinyin {
+   local($caller, $s, *ht, $gloss) = @_;
+
+   return $result if defined($result = $ht{COMBINED}->{$s});
+
+   $cedict_pinyin = $ht{"cedict"}->{$s} || "";
+   $cedicts_pinyin = $ht{"cedicts"}->{$s} || "";
+   $unihan_pinyin = "";
+   @characters = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   foreach $c (@characters) {
+      if ($pinyin = $ht{"simple_pinyin"}->{$c}) {
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"kHanyuPinlu"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"kXHC1983"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"kHanyuPinyin"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      } elsif ($pinyin = $ht{"cedicts"}->{$c}) {
+	 $pinyin =~ s/^(\S+)\s.*$/$1/;
+	 $unihan_pinyin .= $pinyin;
+      # middle dot, katakana middle dot, multiplication sign
+      } elsif ($c =~ /^(\xC2\xB7|\xE3\x83\xBB|\xC3\x97)$/) {
+	 $unihan_pinyin .= $c;
+      # ASCII
+      } elsif ($c =~ /^([\x21-\x7E])$/) {
+	 $unihan_pinyin .= $c;
+      } else {
+	 $unihan_pinyin .= "?";
+	 $hex = $utf8->utf8_to_hex($c);
+	 $unicode = uc $utf8->utf8_to_4hex_unicode($c);
+	 # print STDERR "Tonal pinyin: Unknown character $c ($hex/U+$unicode) -> ?\n";
+      }
+   }
+   $pinyin_title = "";
+   if (($#characters >= 1) && $cedicts_pinyin) {
+      foreach $pinyin (split(/\s+/, $cedicts_pinyin)) {
+	 $pinyin_title .= "$s $pinyin (CEDICT)\n";
+      }
+      $pinyin_title .= "\n";
+   }
+   foreach $c (@characters) {
+      my %local_ht = ();
+      @pinyins = ();
+      foreach $type (("kHanyuPinlu", "kXHC1983", "kHanyuPinyin", "cedicts")) {
+	 if ($pinyin_s = $ht{$type}->{$c}) {
+	    foreach $pinyin (split(/\s+/, $pinyin_s)) {
+	       push(@pinyins, $pinyin) unless $util->member($pinyin, @pinyins);
+	       $type2 = ($type eq "cedicts") ? "CEDICT" : $type;
+	       $local_ht{$pinyin} = ($local_ht{$pinyin}) ? join(", ", $local_ht{$pinyin}, $type2) : $type2;
+	    }
+	 }
+      }
+      foreach $pinyin (@pinyins) {
+	 $type_s = $local_ht{$pinyin};
+	 $pinyin_title .= "$c $pinyin ($type_s)\n";
+      }
+   }
+   $pinyin_title =~ s/\n$//;
+   $pinyin_title =~ s/\n/&#xA;/g;
+   $unihan_pinyin = "" if $unihan_pinyin =~ /^\?+$/;
+   if (($#characters >= 1) && $cedict_pinyin && $unihan_pinyin && ($unihan_pinyin ne $cedict_pinyin)) {
+      $log = "Gloss($s): $gloss\nCEdict($s): $cedicts_pinyin\nUnihan($s): $unihan_pinyin\n";
+      foreach $type (("kHanyuPinlu", "kXHC1983", "kHanyuPinyin")) {
+	 $log_line = "$type($s): ";
+	 foreach $c (@characters) {
+	    $pinyin = $ht{$type}->{$c} || "";
+	    if ($pinyin =~ / /) {
+	       $log_line .= "($pinyin)";
+	    } elsif ($pinyin) {
+	       $log_line .= $pinyin;
+	    } else {
+	       $log_line .= "?";
+	    }
+	 }
+	 $log .= "$log_line\n";
+      }
+      $ht{CONFLICT}->{$s} = $log;
+   }
+   $result = $unihan_pinyin || $cedict_pinyin;
+   $result = $cedict_pinyin if ($#characters > 0) && $cedict_pinyin;
+   $ht{COMBINED}->{$s} = $result;
+   $ht{PINYIN_TITLE}->{$s} = $pinyin_title;
+   return $result;
+}
+
+%number_to_accent_tone_ht = (
+   "a1", "\xC4\x81", "a2", "\xC3\xA1", "a3", "\xC7\x8E", "a4", "\xC3\xA0",
+   "e1", "\xC4\x93", "e2", "\xC3\xA9", "e3", "\xC4\x9B", "e4", "\xC3\xA8",
+   "i1", "\xC4\xAB", "i2", "\xC3\xAD", "i3", "\xC7\x90", "i4", "\xC3\xAC",
+   "o1", "\xC5\x8D", "o2", "\xC3\xB3", "o3", "\xC7\x92", "o4", "\xC3\xB2",
+   "u1", "\xC5\xAB", "u2", "\xC3\xBA", "u3", "\xC7\x94", "u4", "\xC3\xB9",
+   "u:1","\xC7\x96", "u:2","\xC7\x98", "u:3","\xC7\x9A", "u:4","\xC7\x9C",
+   "\xC3\xBC1","\xC7\x96","\xC3\xBC2","\xC7\x98","\xC3\xBC3","\xC7\x9A","\xC3\xBC4","\xC7\x9C"
+);
+
+sub number_to_accent_tone {
+   local($caller, $s) = @_;
+
+   my $result = "";
+   while (($pre,$alpha,$tone_number,$rest) = ($s =~ /^(.*?)((?:[a-z]|u:|\xC3\xBC)+)([1-5])(.*)$/i)) {
+      if ($tone_number eq "5") {
+         $result .= "$pre$alpha";
+      } elsif ((($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)([ae])(.*)$/))
+       || (($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)(o)(u.*)$/))
+       || (($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)(u:|[iou]|\xC3\xBC)([^aeiou]*)$/))) {
+         $result .= "$pre$pre_acc" . ($number_to_accent_tone_ht{($acc_letter . $tone_number)} || ($acc_letter . $tone_number)) . $post_acc;
+      } else {
+         $result .= "$pre$alpha$tone_number";
+      }
+      $s = $rest;
+   }
+   $result .= $s;
+   $result =~ s/u:/\xC3\xBC/g;
+   return $result;
+}
+
+sub string_contains_utf8_cjk_unified_ideograph_p {
+   local($caller, $s) = @_;
+
+   return ($s =~ /([\xE4-\xE9]|\xE3[\x90-\xBF]|\xF0[\xA0-\xAC])/);
+}
+
+1;

uroman/lib/NLP/Romanizer.pm

@@ -0,0 +1,2020 @@
+################################################################
+#                                                              #
+# Romanizer                                                    #
+#                                                              #
+################################################################
+
+package NLP::Romanizer;
+
+use NLP::Chinese;
+use NLP::UTF8;
+use NLP::utilities;
+use JSON;
+$utf8 = NLP::UTF8;
+$util = NLP::utilities;
+$chinesePM = NLP::Chinese;
+
+my $verbosePM = 0;
+%empty_ht = ();
+
+my $braille_capital_letter_indicator = "\xE2\xA0\xA0";
+my $braille_number_indicator = "\xE2\xA0\xBC";
+my $braille_decimal_point = "\xE2\xA0\xA8";
+my $braille_comma = "\xE2\xA0\x82";
+my $braille_solidus = "\xE2\xA0\x8C";
+my $braille_numeric_space = "\xE2\xA0\x90";
+my $braille_letter_indicator = "\xE2\xA0\xB0";
+my $braille_period = "\xE2\xA0\xB2";
+
+sub new {
+   local($caller) = @_;
+
+   my $object = {};
+   my $class = ref( $caller ) || $caller;
+   bless($object, $class);
+   return $object;
+}
+
+sub load_unicode_data {
+   local($this, *ht, $filename) = @_;
+   # ../../data/UnicodeData.txt
+
+   $n = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+	 if (($unicode_value, $char_name, $general_category, $canon_comb_classes, $bidir_category, $char_decomp_mapping, $decimal_digit_value, $digit_value, $numeric_value, $mirrored, $unicode_1_0_name, $comment_field, $uc_mapping, $lc_mapping, $title_case_mapping) = split(";", $_)) {
+            $utf8_code = $utf8->unicode_hex_string2string($unicode_value);
+	    $ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name;
+	    $ht{UTF_NAME_TO_UNICODE}->{$char_name} = $unicode_value;
+	    $ht{UTF_NAME_TO_CODE}->{$char_name} = $utf8_code;
+	    $ht{UTF_TO_CAT}->{$utf8_code} = $general_category;
+	    $ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric_value unless $numeric_value eq "";
+	    $n++;
+	 }
+      }
+      close(IN);
+      # print STDERR "Loaded $n entries from $filename\n";
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub load_unicode_overwrite_romanization {
+   local($this, *ht, $filename) = @_;
+   # ../../data/UnicodeDataOverwrite.txt
+
+   $n = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+	 next if /^#/;
+         $unicode_value = $util->slot_value_in_double_colon_del_list($_, "u");
+         $romanization = $util->slot_value_in_double_colon_del_list($_, "r");
+         $numeric = $util->slot_value_in_double_colon_del_list($_, "num");
+         $picture = $util->slot_value_in_double_colon_del_list($_, "pic");
+	 $syllable_info = $util->slot_value_in_double_colon_del_list($_, "syllable-info");
+	 $tone_mark = $util->slot_value_in_double_colon_del_list($_, "tone-mark");
+	 $char_name = $util->slot_value_in_double_colon_del_list($_, "name");
+	 $entry_processed_p = 0;
+         $utf8_code = $utf8->unicode_hex_string2string($unicode_value);
+	 if ($unicode_value) {
+	    $ht{UTF_TO_CHAR_ROMANIZATION}->{$utf8_code} = $romanization if $romanization;
+	    $ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric if defined($numeric) && ($numeric ne "");
+	    $ht{UTF_TO_PICTURE_DESCR}->{$utf8_code} = $picture if $picture;
+	    $ht{UTF_TO_SYLLABLE_INFO}->{$utf8_code} = $syllable_info if $syllable_info;
+	    $ht{UTF_TO_TONE_MARK}->{$utf8_code} = $tone_mark if $tone_mark;
+	    $ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name if $char_name;
+	    $entry_processed_p = 1 if $romanization || $numeric || $picture || $syllable_info || $tone_mark;
+	 }
+	 $n++ if $entry_processed_p;
+      }
+      close(IN);
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub load_script_data {
+   local($this, *ht, $filename) = @_;
+   # ../../data/Scripts.txt
+
+   $n = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+         next unless $script_name = $util->slot_value_in_double_colon_del_list($_, "script-name");
+         $abugida_default_vowel_s = $util->slot_value_in_double_colon_del_list($_, "abugida-default-vowel");
+         $alt_script_name_s = $util->slot_value_in_double_colon_del_list($_, "alt-script-name");
+         $language_s = $util->slot_value_in_double_colon_del_list($_, "language");
+         $direction = $util->slot_value_in_double_colon_del_list($_, "direction"); # right-to-left
+         $font_family_s = $util->slot_value_in_double_colon_del_list($_, "font-family");
+         $ht{SCRIPT_P}->{$script_name} = 1;
+	 $ht{SCRIPT_NORM}->{(uc $script_name)} = $script_name;
+         $ht{DIRECTION}->{$script_name} = $direction if $direction;
+	 foreach $language (split(/,\s*/, $language_s)) {
+	    $ht{SCRIPT_LANGUAGE}->{$script_name}->{$language} = 1;
+	    $ht{LANGUAGE_SCRIPT}->{$language}->{$script_name} = 1;
+	 }
+	 foreach $alt_script_name (split(/,\s*/, $alt_script_name_s)) {
+	    $ht{SCRIPT_NORM}->{$alt_script_name} = $script_name;
+	    $ht{SCRIPT_NORM}->{(uc $alt_script_name)} = $script_name;
+	 }
+	 foreach $abugida_default_vowel (split(/,\s*/, $abugida_default_vowel_s)) {
+	    $ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$script_name}->{$abugida_default_vowel} = 1 if $abugida_default_vowel;
+	 }
+	 foreach $font_family (split(/,\s*/, $font_family_s)) {
+	    $ht{SCRIPT_FONT}->{$script_name}->{$font_family} = 1 if $font_family;
+	 }
+	 $n++;
+      }
+      close(IN);
+      # print STDERR "Loaded $n entries from $filename\n";
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub unicode_hangul_romanization {
+   local($this, $s, $pass_through_p) = @_;
+
+   $pass_through_p = 0 unless defined($pass_through_p);
+   @leads = split(/\s+/, "g gg n d dd r m b bb s ss - j jj c k t p h");
+   # @vowels = split(/\s+/, "a ae ya yai e ei ye yei o oa oai oi yo u ue uei ui yu w wi i");
+   @vowels = split(/\s+/, "a ae ya yae eo e yeo ye o wa wai oe yo u weo we wi yu eu yi i");
+   @tails = split(/\s+/, "- g gg gs n nj nh d l lg lm lb ls lt lp lh m b bs s ss ng j c k t p h");
+   $result = "";
+   @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   foreach $char (@chars) {
+      $unicode = $utf8->utf8_to_unicode($char);
+      if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
+	 $code = $unicode - 0xAC00;
+	 $lead_index = int($code / (28*21));
+	 $vowel_index = int($code/28) % 21;
+	 $tail_index = $code % 28;
+	 $rom = $leads[$lead_index] . $vowels[$vowel_index] . $tails[$tail_index];
+	 $rom =~ s/-//g;
+	 $result .= $rom;
+      } elsif ($pass_through_p) {
+	 $result .= $char;
+      }
+   }
+   return $result;
+}
+
+sub listify_comma_sep_string {
+   local($this, $s) = @_;
+
+   @result_list = ();
+   return @result_list unless $s =~ /\S/;
+   $s = $util->trim2($s);
+   my $elem;
+
+   while (($elem, $rest) = ($s =~ /^("(?:\\"|[^"])*"|'(?:\\'|[^'])*'|[^"', ]+),\s*(.*)$/)) {
+      push(@result_list, $util->dequote_string($elem));
+      $s = $rest;
+   }
+   push(@result_list, $util->dequote_string($s)) if $s =~ /\S/;
+
+   return @result_list;
+}
+
+sub braille_string_p {
+   local($this, $s) = @_;
+
+   return ($s =~ /^(\xE2[\xA0-\xA3][\x80-\xBF])+$/);
+}
+
+sub register_word_boundary_info {
+   local($this, *ht, $lang_code, $utf8_source_string, $utf8_target_string, $use_only_for_whole_word_p, 
+	 $use_only_at_start_of_word_p, $use_only_at_end_of_word_p, 
+	 $dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p) = @_;
+
+   if ($use_only_for_whole_word_p) {
+      if ($lang_code) {
+         $ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{USE_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($use_only_at_start_of_word_p) {
+      if ($lang_code) {
+         $ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{USE_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($use_only_at_end_of_word_p) {
+      if ($lang_code) {
+         $ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{USE_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($dont_use_at_start_of_word_p) {
+      if ($lang_code) {
+         $ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{DONT_USE_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+   if ($dont_use_at_end_of_word_p) {
+      if ($lang_code) {
+         $ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      } else {
+         $ht{DONT_USE_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
+      }
+   }
+}
+
+sub load_romanization_table {
+   local($this, *ht, $filename) = @_;
+   # ../../data/romanization-table.txt
+
+   $n = 0;
+   $line_number = 0;
+   if (open(IN, $filename)) {
+      while (<IN>) {
+         $line_number++;
+	 next if /^#/;
+	 if ($_ =~ /^::preserve\s/) {
+	    $from_unicode = $util->slot_value_in_double_colon_del_list($_, "from");
+	    $to_unicode = $util->slot_value_in_double_colon_del_list($_, "to");
+	    if ($from_unicode =~ /^(?:U\+|\\u)[0-9A-F]{4,}$/i) {
+	       $from_unicode =~ s/^(?:U\+|\\u)//;
+	       $from_code_point = hex($from_unicode);
+	    } else {
+	       $from_code_point = "";
+	    }
+	    if ($to_unicode =~ /^(?:U\+|\\u)[0-9A-F]{4,}$/i) {
+	       $to_unicode =~ s/^(?:U\+|\\u)//;
+	       $to_code_point = hex($to_unicode);
+	    } else {
+	       $to_code_point = $from_code_point;
+	    }
+	    if ($from_code_point ne "") {
+	       # print STDERR "Preserve code-points $from_unicode--$to_unicode = $from_code_point--$to_code_point\n";
+	       foreach $code_point (($from_code_point .. $to_code_point)) {
+	          $utf8_string = $utf8->unicode2string($code_point);
+                  $ht{UTF_CHAR_MAPPING}->{$utf8_string}->{$utf8_string} = 1;
+	       }
+	       $n++;
+	    }
+	    next;
+	 }
+         $utf8_source_string = $util->slot_value_in_double_colon_del_list($_, "s");
+         $utf8_target_string = $util->slot_value_in_double_colon_del_list($_, "t");
+         $utf8_alt_target_string_s = $util->slot_value_in_double_colon_del_list($_, "t-alt");
+         $use_alt_in_pointed_p = ($_ =~ /::use-alt-in-pointed\b/);
+         $use_only_for_whole_word_p = ($_ =~ /::use-only-for-whole-word\b/);
+         $use_only_at_start_of_word_p = ($_ =~ /::use-only-at-start-of-word\b/);
+         $use_only_at_end_of_word_p = ($_ =~ /::use-only-at-end-of-word\b/);
+         $dont_use_at_start_of_word_p = ($_ =~ /::dont-use-at-start-of-word\b/);
+         $dont_use_at_end_of_word_p = ($_ =~ /::dont-use-at-end-of-word\b/);
+	 $use_only_in_lower_case_enviroment_p = ($_ =~ /::use-only-in-lower-case-enviroment\b/);
+	 $word_external_punctuation_p = ($_ =~ /::word-external-punctuation\b/);
+	 $utf8_source_string =~ s/\s*$//;
+	 $utf8_target_string =~ s/\s*$//;
+	 $utf8_alt_target_string_s =~ s/\s*$//;
+	 $utf8_target_string =~ s/^"(.*)"$/$1/;
+	 $utf8_target_string =~ s/^'(.*)'$/$1/;
+	 @utf8_alt_targets = $this->listify_comma_sep_string($utf8_alt_target_string_s);
+         $numeric = $util->slot_value_in_double_colon_del_list($_, "num");
+	 $numeric =~ s/\s*$//;
+         $annotation = $util->slot_value_in_double_colon_del_list($_, "annotation");
+	 $annotation =~ s/\s*$//;
+         $lang_code = $util->slot_value_in_double_colon_del_list($_, "lcode");
+         $prob = $util->slot_value_in_double_colon_del_list($_, "p") || 1;
+	 unless (($utf8_target_string eq "") && ($numeric =~ /\d/)) {
+	    if ($lang_code) {
+               $ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	    } else {
+               $ht{UTF_CHAR_MAPPING}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	    }
+	    if ($word_external_punctuation_p) {
+	       if ($lang_code) {
+                  $ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	       } else {
+                  $ht{WORD_EXTERNAL_PUNCTUATION}->{$utf8_source_string}->{$utf8_target_string} = $prob;
+	       }
+	    }
+            if ($this->braille_string_p($utf8_source_string)) {
+	       if (($utf8_target_string =~ /^[a-z]+$/)
+	        && (! ($utf8_source_string =~ /^$braille_capital_letter_indicator/))) {
+	          my $uc_utf8_source_string = "$braille_capital_letter_indicator$utf8_source_string";
+	          my $uc_utf8_target_string = ucfirst $utf8_target_string;
+	          if ($lang_code) {
+                     $ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
+	          } else {
+                     $ht{UTF_CHAR_MAPPING}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
+	          }
+                  $this->register_word_boundary_info(*ht, $lang_code, $uc_utf8_source_string, $uc_utf8_target_string,
+                            $use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p, 
+	                    $dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
+	       }
+	       if (($utf8_target_string =~ /^[0-9]$/)
+	        && ($utf8_source_string =~ /^$braille_number_indicator./)) {
+	          my $core_number_char = $utf8_source_string;
+		  $core_number_char =~ s/$braille_number_indicator//;
+	          $ht{BRAILLE_TO_DIGIT}->{$core_number_char} = $utf8_target_string;
+	       }
+	    }
+	 }
+	 if ($use_only_in_lower_case_enviroment_p) {
+	    if ($lang_code) {
+	       $ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
+	    } else {
+	       $ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT}->{$utf8_source_string}->{$utf8_target_string} = 1;
+	    }
+	 }
+         $this->register_word_boundary_info(*ht, $lang_code, $utf8_source_string, $utf8_target_string, 
+                   $use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p, 
+	           $dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
+	 foreach $utf8_alt_target (@utf8_alt_targets) {
+	    if ($lang_code) {
+               $ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
+	       $ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
+	    } else {
+               $ht{UTF_CHAR_ALT_MAPPING}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
+	       $ht{USE_ALT_IN_POINTED}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
+	    }
+	    if ($use_only_for_whole_word_p) {
+	       if ($lang_code) {
+	          $ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       } else {
+	          $ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       }
+	    }
+	    if ($use_only_at_start_of_word_p) {
+	       if ($lang_code) {
+	          $ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       } else {
+	          $ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       }
+	    }
+	    if ($use_only_at_end_of_word_p) {
+	       if ($lang_code) {
+	          $ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       } else {
+	          $ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
+	       }
+	    }
+	 }
+	 if ($numeric =~ /\d/) {
+	    $ht{UTF_TO_NUMERIC}->{$utf8_source_string} = $numeric;
+	 }
+	 if ($annotation =~ /\S/) {
+	    $ht{UTF_ANNOTATION}->{$utf8_source_string} = $annotation;
+	 }
+         $n++;
+      }
+      close(IN);
+      # print STDERR "Loaded $n entries from $filename\n";
+   } else {
+      print STDERR "Can't open $filename\n";
+   }
+}
+
+sub char_name_to_script {
+   local($this, $char_name, *ht) = @_;
+
+   return $cached_result if $cached_result = $ht{CHAR_NAME_TO_SCRIPT}->{$char_name};
+   $orig_char_name = $char_name;
+   $char_name =~ s/\s+(CONSONANT|LETTER|LIGATURE|SIGN|SYLLABLE|SYLLABICS|VOWEL)\b.*$//;
+   my $script_name;
+   while ($char_name) {
+      last if $script_name = $ht{SCRIPT_NORM}->{(uc $char_name)};
+      $char_name =~ s/\s*\S+\s*$//;
+   }
+   $script_name = "" unless defined($script_name);
+   $ht{CHAR_NAME_TO_SCRIPT}->{$char_name} = $script_name;
+   return $script_name;
+}
+
+sub letter_plus_char_p {
+   local($this, $char_name) = @_;
+
+   return $cached_result if $cached_result = $ht{CHAR_NAME_LETTER_PLUS}->{$char_name};
+   my $letter_plus_p = ($char_name =~ /\b(?:LETTER|VOWEL SIGN|AU LENGTH MARK|CONSONANT SIGN|SIGN VIRAMA|SIGN PAMAAEH|SIGN COENG|SIGN AL-LAKUNA|SIGN ASAT|SIGN ANUSVARA|SIGN ANUSVARAYA|SIGN BINDI|TIPPI|SIGN NIKAHIT|SIGN CANDRABINDU|SIGN VISARGA|SIGN REAHMUK|SIGN NUKTA|SIGN DOT BELOW|HEBREW POINT)\b/) ? 1 : 0;
+   $ht{CHAR_NAME_LETTER_PLUS}->{$char_name} = $letter_plus_p;
+   return $letter_plus_p;
+}
+
+sub subjoined_char_p {
+   local($this, $char_name) = @_;
+
+   return $cached_result if $cached_result = $ht{CHAR_NAME_SUBJOINED}->{$char_name};
+   my $subjoined_p = (($char_name =~ /\b(?:SUBJOINED LETTER|VOWEL SIGN|AU LENGTH MARK|EMPHASIS MARK|CONSONANT SIGN|SIGN VIRAMA|SIGN PAMAAEH|SIGN COENG|SIGN ASAT|SIGN ANUSVARA|SIGN ANUSVARAYA|SIGN BINDI|TIPPI|SIGN NIKAHIT|SIGN CANDRABINDU|SIGN VISARGA|SIGN REAHMUK|SIGN DOT BELOW|HEBREW (POINT|PUNCTUATION GERESH)|ARABIC (?:DAMMA|DAMMATAN|FATHA|FATHATAN|HAMZA|KASRA|KASRATAN|MADDAH|SHADDA|SUKUN))\b/)) ? 1 : 0;
+   $ht{CHAR_NAME_SUBJOINED}->{$char_name} = $subjoined_p;
+   return $subjoined_p;
+}
+
+sub new_node_id {
+   local($this, *chart_ht) = @_;
+
+   my $n_nodes = $chart_ht{N_NODES};
+   $n_nodes++;
+   $chart_ht{N_NODES} = $n_nodes;
+   return $n_nodes;
+}
+
+sub add_node {
+   local($this, $s, $start, $end, *chart_ht, $type, $comment) = @_;
+
+   my $node_id = $this->new_node_id(*chart_ht);
+   # print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if $comment =~ /number/;
+   # print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if ($start >= 0) && ($start < 50);
+   $chart_ht{NODE_START}->{$node_id} = $start;
+   $chart_ht{NODE_END}->{$node_id} = $end;
+   $chart_ht{NODES_STARTING_AT}->{$start}->{$node_id} = 1;
+   $chart_ht{NODES_ENDING_AT}->{$end}->{$node_id} = 1;
+   $chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}->{$node_id} = 1;
+   $chart_ht{NODE_TYPE}->{$node_id} = $type;
+   $chart_ht{NODE_COMMENT}->{$node_id} = $comment;
+   $chart_ht{NODE_ROMAN}->{$node_id} = $s;
+   return $node_id;
+}
+
+sub get_node_for_span {
+   local($this, $start, $end, *chart_ht) = @_;
+
+   return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+
+   return (@node_ids) ? $node_ids[0] : "";
+}
+
+sub get_node_for_span_and_type {
+   local($this, $start, $end, *chart_ht, $type) = @_;
+
+   return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+
+   foreach $node_id (@node_ids) {
+      return $node_id if $chart_ht{NODE_TYPE}->{$node_id} eq $type;
+   }
+   return "";
+}
+
+sub get_node_roman {
+   local($this, $node_id, *chart_id, $default) = @_;
+
+   $default = "" unless defined($default);
+   my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
+   return (defined($roman)) ? $roman : $default;
+}
+
+sub set_node_id_slot_value {
+   local($this, $node_id, $slot, $value, *chart_id) = @_;
+ 
+   $chart_ht{NODE_SLOT}->{$node_id}->{$slot} = $value;
+}
+
+sub copy_slot_values {
+   local($this, $old_node_id, $new_node_id, *chart_id, @slots) = @_;
+
+   if (@slots) {
+      foreach $slot (keys %{$chart_ht{NODE_SLOT}->{$old_node_id}}) {
+         if (($slots[0] eq "all") || $util->member($slot, @slots)) {
+	    my $value = $chart_ht{NODE_SLOT}->{$old_node_id}->{$slot};
+	    $chart_ht{NODE_SLOT}->{$new_node_id}->{$slot} = $value if defined($value);
+	 }
+      }
+   }
+}
+
+sub get_node_id_slot_value {
+   local($this, $node_id, $slot, *chart_id, $default) = @_;
+ 
+   $default = "" unless defined($default);
+   my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
+   return (defined($value)) ? $value : $default;
+}
+
+sub get_node_for_span_with_slot_value {
+   local($this, $start, $end, $slot, *chart_id, $default) = @_;
+
+   $default = "" unless defined($default);
+   return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+   foreach $node_id (@node_ids) {
+      my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
+      return $value if defined($value);
+   }
+   return $default;
+}
+
+sub get_node_for_span_with_slot {
+   local($this, $start, $end, $slot, *chart_id, $default) = @_;
+
+   $default = "" unless defined($default);
+   return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
+   my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+   foreach $node_id (@node_ids) {
+      my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
+      return $node_id if defined($value);
+   }
+   return $default;
+}
+
+sub register_new_complex_number_span_segment {
+   local($this, $start, $mid, $end, *chart_id, $line_number) = @_;
+   # e.g. 4 10 (= 40); 20 5 (= 25)
+   # might become part of larger complex number span, e.g. 4 1000 3 100 20 1
+
+   # print STDERR "register_new_complex_number_span_segment $start-$mid-$end\n" if $line_number == 43;
+   if (defined($old_start = $chart_ht{COMPLEX_NUMERIC_END_START}->{$mid})) {
+      undef($chart_ht{COMPLEX_NUMERIC_END_START}->{$mid});
+      $chart_ht{COMPLEX_NUMERIC_START_END}->{$old_start} = $end;
+      $chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $old_start;
+   } else {
+      $chart_ht{COMPLEX_NUMERIC_START_END}->{$start} = $end;
+      $chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $start;
+   }
+}
+
+sub romanize_by_token_with_caching {
+   local($this, $s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number) = @_;
+
+   $control = "" unless defined($control);
+   my $return_chart_p = ($control =~ /return chart/i);
+   my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
+   return $this->romanize($s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number)
+     if $return_chart_p || $return_offset_mappings_p;
+   my $result = "";
+   my @separators = ();
+   my @tokens = ();
+   $s =~ s/\n$//; # Added May 2, 2019 as bug-fix (duplicate empty lines)
+   while (($sep, $token, $rest) = ($s =~ /^(\s*)(\S+)(.*)$/)) {
+      push(@separators, $sep);
+      push(@tokens, $token);
+      $s = $rest;
+   }
+   push(@separators, $s);
+   while (@tokens) {
+      my $sep = shift @separators;
+      my $token = shift @tokens;
+      $result .= $sep;
+      if ($token =~ /^[\x00-\x7F]*$/) { # all ASCII
+         $result .= $token;
+      } else {
+         my $rom_token = $ht{CACHED_ROMANIZATION}->{$lang_code}->{$token};
+         unless (defined($rom_token)) {
+            $rom_token = $this->romanize($token, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number);
+	    $ht{CACHED_ROMANIZATION}->{$lang_code}->{$token} = $rom_token if defined($rom_token);
+         }
+         $result .= $rom_token;
+      }
+   }
+   my $sep = shift @separators;
+   $result .= $sep if defined($sep);
+
+   return $result;
+}
+
+sub romanize {
+   local($this, $s, $lang_code, $output_style, *ht, *pinyin_ht, $initial_char_offset, $control, $line_number, $initial_rom_char_offset) = @_;
+
+   my $orig_lang_code = $lang_code;
+   # Check whether the text (to be romanized) starts with a language code directive.
+   if (($line_lang_code) = ($s =~ /^::lcode\s+([a-z][a-z][a-z])\s/)) {
+      $lang_code = $line_lang_code;
+   }
+   $initial_char_offset = 0 unless defined($initial_char_offset);
+   $initial_rom_char_offset = 0 unless defined($initial_rom_char_offset);
+   $control = "" unless defined($control);
+   my $return_chart_p = ($control =~ /return chart/i);
+   my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
+   $line_number = "" unless defined($line_number);
+   my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   my $n_characters = $#chars + 1;
+   %chart_ht = ();
+   $chart_ht{N_CHARS} = $n_characters;
+   $chart_ht{N_NODES} = 0;
+   my $char = "";
+   my $char_name = "";
+   my $prev_script = "";
+   my $current_script = "";
+   my $script_start = 0;
+   my $script_end = 0;
+   my $prev_letter_plus_script = "";
+   my $current_letter_plus_script = "";
+   my $letter_plus_script_start = 0;
+   my $letter_plus_script_end = 0;
+   my $log ="";
+   my $n_right_to_left_chars = 0;
+   my $n_left_to_right_chars = 0;
+   my $hebrew_word_start = ""; # used to identify Hebrew words with points
+   my $hebrew_word_contains_point = 0;
+   my $current_word_start = "";
+   my $current_word_script = "";
+   my $braille_all_caps_p = 0;
+
+   # prep
+   foreach $i ((0 .. ($#chars + 1))) {
+      if ($i <= $#chars) {
+         $char = $chars[$i];
+         $chart_ht{ORIG_CHAR}->{$i} = $char;
+         $char_name = $ht{UTF_TO_CHAR_NAME}->{$char} || "";
+         $chart_ht{CHAR_NAME}->{$i} = $char_name;
+         $current_script = $this->char_name_to_script($char_name, *ht);
+	 $current_script_direction = $ht{DIRECTION}->{$current_script} || '';
+	 if ($current_script_direction eq 'right-to-left') {
+	    $n_right_to_left_chars++;
+	 } elsif (($char =~ /^[a-z]$/i) || ! ($char =~ /^[\x00-\x7F]$/)) {
+	    $n_left_to_right_chars++;
+	 }
+         $chart_ht{CHAR_SCRIPT}->{$i} = $current_script;
+         $chart_ht{SCRIPT_SEGMENT_START}->{$i} = ""; # default value, to be updated later
+         $chart_ht{SCRIPT_SEGMENT_END}->{$i} = "";   # default value, to be updated later
+         $chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = ""; # default value, to be updated later
+         $chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = "";   # default value, to be updated later
+	 $subjoined_char_p = $this->subjoined_char_p($char_name);
+	 $chart_ht{CHAR_SUBJOINED}->{$i} = $subjoined_char_p;
+	 $letter_plus_char_p = $this->letter_plus_char_p($char_name);
+	 $chart_ht{CHAR_LETTER_PLUS}->{$i} = $letter_plus_char_p;
+	 $current_letter_plus_script = ($letter_plus_char_p) ? $current_script : "";
+         $numeric_value = $ht{UTF_TO_NUMERIC}->{$char};
+         $numeric_value = "" unless defined($numeric_value);
+         $annotation = $ht{UTF_ANNOTATION}->{$char};
+         $annotation = "" unless defined($annotation);
+	 $chart_ht{CHAR_NUMERIC_VALUE}->{$i} = $numeric_value;
+	 $chart_ht{CHAR_ANNOTATION}->{$i} = $annotation;
+         $syllable_info = $ht{UTF_TO_SYLLABLE_INFO}->{$char} || "";
+	 $chart_ht{CHAR_SYLLABLE_INFO}->{$i} = $syllable_info;
+         $tone_mark = $ht{UTF_TO_TONE_MARK}->{$char} || "";
+	 $chart_ht{CHAR_TONE_MARK}->{$i} = $tone_mark;
+      } else {
+	 $char = "";
+         $char_name = "";
+	 $current_script = "";
+	 $current_letter_plus_script = "";
+      }
+      if ($char_name =~ /^HEBREW (LETTER|POINT|PUNCTUATION GERESH) /) {
+	 $hebrew_word_start = $i if $hebrew_word_start eq "";
+	 $hebrew_word_contains_point = 1 if $char_name =~ /^HEBREW POINT /;
+      } elsif ($hebrew_word_start ne "") {
+	 if ($hebrew_word_contains_point) {
+	    foreach $j (($hebrew_word_start .. ($i-1))) {
+	       $chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$j} = 1;
+	    }
+	    $chart_ht{CHAR_START_OF_WORD}->{$hebrew_word_start} = 1;
+	    $chart_ht{CHAR_END_OF_WORD}->{($i-1)} = 1;
+	 }
+	 $hebrew_word_start = "";
+	 $hebrew_word_contains_point = 0;
+      }
+      my $part_of_word_p = $current_script
+                        && ($this->letter_plus_char_p($char_name)
+                         || $this->subjoined_char_p($char_name)
+			 || ($char_name =~ /\b(LETTER|SYLLABLE|SYLLABICS|LIGATURE)\b/));
+
+      # Braille punctuation
+      my $end_offset = 0;
+      if ($char_name =~ /^Braille\b/i) {
+         if (($char =~ /^\s*$/) || ($char_name =~ /BLANK/)) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+	 } elsif ($chart_ht{NOT_PART_OF_WORD_P}->{$i}) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+	 } elsif ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$char}})
+	       || (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$char}})) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+	 } elsif (($i+1 <= $#chars) 
+	       && ($s1 = $char . $chars[$i+1])
+	       && ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s1}})
+	        || (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s1}}))) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+            $chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
+	 } elsif (($i+2 <= $#chars)
+	       && ($s2 = $char . $chars[$i+1] . $chars[$i+2])
+	       && ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s2}})
+	        || (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s2}}))) {
+	    $part_of_word_p = 0;
+            $braille_all_caps_p = 0;
+            $chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
+            $chart_ht{NOT_PART_OF_WORD_P}->{($i+2)} = 1;
+	 } elsif (($i+1 <= $#chars)
+	       && ($char eq $braille_capital_letter_indicator)
+	       && ($chars[$i+1] eq $braille_capital_letter_indicator)) {
+	    $braille_all_caps_p = 1;
+	 } else {
+	    $part_of_word_p = 1;
+	 }
+	 # last period in Braille text is also not part_of_word_p
+	 if (($char eq $braille_period)
+	  && (($i == $#chars)
+	   || (($i < $#chars)
+	    && (! $this->braille_string_p($chars[$i+1]))))) {
+	    $part_of_word_p = 0;
+	 }
+	 # period before other word-external punctuation is also not part_of_word_p
+	 if (($i > 0)
+	  && ($chars[$i-1] eq $braille_period)
+	  && (! $part_of_word_p)
+	  && ($current_word_start ne "")) {
+	    $end_offset = -1;
+	 }
+      } else {
+         $braille_all_caps_p = 0;
+      }
+      $chart_ht{BRAILLE_ALL_CAPS_P}->{$i} = $braille_all_caps_p;
+
+      if (($current_word_start ne "")
+       && ((! $part_of_word_p)
+        || ($current_script ne $current_word_script))) {
+         # END OF WORD
+	 $chart_ht{CHAR_START_OF_WORD}->{$current_word_start} = 1;
+	 $chart_ht{CHAR_END_OF_WORD}->{($i-1+$end_offset)} = 1;
+	 my $word = join("", @chars[$current_word_start .. ($i-1+$end_offset)]);
+	 $chart_ht{WORD_START_END}->{$current_word_start}->{$i} = $word;
+	 $chart_ht{WORD_END_START}->{$i+$end_offset}->{$current_word_start} = $word;
+	 # print STDERR "Word ($current_word_start-$i+$end_offset): $word ($current_word_script)\n";
+	 $current_word_start = "";
+	 $current_word_script = "";
+      }
+      if ($part_of_word_p && ($current_word_start eq "")) {
+         # START OF WORD
+	 $current_word_start = $i;
+	 $current_word_script = $current_script;
+      }
+      # print STDERR "$i char: $char ($current_script)\n";
+      unless ($current_script eq $prev_script) {
+	 if ($prev_script && ($i-1 >= $script_start)) {
+	    my $script_end = $i;
+            $chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start} = $script_end;
+            $chart_ht{SCRIPT_SEGMENT_END_TO_START}->{$script_end} = $script_start;
+	    foreach $i (($script_start .. $script_end)) {
+               $chart_ht{SCRIPT_SEGMENT_START}->{$i} = $script_start;
+               $chart_ht{SCRIPT_SEGMENT_END}->{$i} = $script_end;
+	    }
+	    # print STDERR "Script segment $script_start-$script_end: $prev_script\n";
+	 }
+	 $script_start = $i;
+      }
+      unless ($current_letter_plus_script eq $prev_letter_plus_script) {
+	 if ($prev_letter_plus_script && ($i-1 >= $letter_plus_script_start)) {
+	    my $letter_plus_script_end = $i;
+            $chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$letter_plus_script_start} = $letter_plus_script_end;
+            $chart_ht{LETTER_TOKEN_SEGMENT_END_TO_START}->{$letter_plus_script_end} = $letter_plus_script_start;
+	    foreach $i (($letter_plus_script_start .. $letter_plus_script_end)) {
+               $chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = $letter_plus_script_start;
+               $chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = $letter_plus_script_end;
+	    }
+	    # print STDERR "Script token segment $letter_plus_script_start-$letter_plus_script_end: $prev_letter_plus_script\n";
+	 }
+	 $letter_plus_script_start = $i;
+      }
+      $prev_script = $current_script;
+      $prev_letter_plus_script = $current_letter_plus_script;
+   }
+   $ht{STRING_IS_DOMINANTLY_RIGHT_TO_LEFT}->{$s} = 1 if $n_right_to_left_chars > $n_left_to_right_chars;
+
+   # main
+   my $i = 0;
+   while ($i <= $#chars) {
+      my $char = $chart_ht{ORIG_CHAR}->{$i};
+      my $current_script = $chart_ht{CHAR_SCRIPT}->{$i};
+      $chart_ht{CHART_CONTAINS_SCRIPT}->{$current_script} = 1;
+      my $script_segment_start = $chart_ht{SCRIPT_SEGMENT_START}->{$i};
+      my $script_segment_end = $chart_ht{SCRIPT_SEGMENT_END}->{$i};
+      my $char_name = $chart_ht{CHAR_NAME}->{$i};
+      my $subjoined_char_p = $chart_ht{CHAR_SUBJOINED}->{$i};
+      my $letter_plus_char_p = $chart_ht{CHAR_LETTER_PLUS}->{$i};
+      my $numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{$i};
+      my $annotation = $chart_ht{CHAR_ANNOTATION}->{$i};
+      # print STDERR "  $char_name annotation: $annotation\n" if $annotation;
+      my $tone_mark = $chart_ht{CHAR_TONE_MARK}->{$i};
+      my $found_char_mapping_p = 0;
+      my $prev_char_name = ($i >= 1) ? $chart_ht{CHAR_NAME}->{($i-1)} : "";
+      my $prev2_script = ($i >= 2) ? $chart_ht{CHAR_SCRIPT}->{($i-2)} : "";
+      my $prev_script = ($i >= 1) ? $chart_ht{CHAR_SCRIPT}->{($i-1)} : "";
+      my $next_script = ($i < $#chars) ? $chart_ht{CHAR_SCRIPT}->{($i+1)} : "";
+      my $next_char = ($i < $#chars) ? $chart_ht{ORIG_CHAR}->{($i+1)} : "";
+      my $next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char} || "";
+      my $prev2_letter_plus_char_p = ($i >= 2) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-2)} : 0;
+      my $prev_letter_plus_char_p = ($i >= 1) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-1)} : 0;
+      my $next_letter_plus_char_p = ($i < $#chars) ? $chart_ht{CHAR_LETTER_PLUS}->{($i+1)} : 0;
+      my $next_index = $i + 1;
+
+      # Braille numeric mode
+      if ($char eq $braille_number_indicator) {
+	 my $offset = 0;
+	 my $numeric_value = "";
+	 my $digit;
+         while ($i+$offset < $#chars) {
+	    $offset++;
+	    my $offset_char = $chart_ht{ORIG_CHAR}->{$i+$offset};
+            if (defined($digit = $ht{BRAILLE_TO_DIGIT}->{$offset_char})) {
+	       $numeric_value .= $digit;
+	    } elsif (($offset_char eq $braille_decimal_point)
+                  || ($ht{UTF_CHAR_MAPPING}->{$offset_char}->{"."})) {
+	       $numeric_value .= ".";
+	    } elsif ($offset_char eq $braille_comma) {
+	       $numeric_value .= ",";
+	    } elsif ($offset_char eq $braille_numeric_space) {
+	       $numeric_value .= " ";
+	    } elsif ($offset_char eq $braille_solidus) {
+	       $numeric_value .= "/";
+	    } elsif ($offset_char eq $braille_number_indicator) {
+	        # stay in Braille numeric mode
+	    } elsif ($offset_char eq $braille_letter_indicator) {
+	       # consider as part of number, but without contributing to numeric_value
+	       last;
+	    } else {
+	       $offset--; 
+	       last;
+	    }
+	 }
+	 if ($offset) {
+            $next_index = $i + $offset + 1;
+	    $node_id = $this->add_node($numeric_value, $i, $next_index, *chart_ht, "", "braille number");
+	    $found_char_mapping_p = 1;
+	 }
+      }
+
+      unless ($found_char_mapping_p) {
+        foreach $string_length (reverse(1 .. 6)) {
+	  next if ($i + $string_length-1) > $#chars;
+	  my $start_of_word_p = $chart_ht{CHAR_START_OF_WORD}->{$i} || 0;
+	  my $end_of_word_p = $chart_ht{CHAR_END_OF_WORD}->{($i+$string_length-1)} || 0;
+	  my $multi_char_substring = join("", @chars[$i..($i+$string_length-1)]);
+	  my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
+	  @mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
+	  my @mappings_whole = ();
+	  my @mappings_start_or_end = ();
+	  my @mappings_other = ();
+	  foreach $mapping (@mappings) {
+	    next if $mapping =~ /\(__.*__\)/;
+	    if ($ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	     || $ht{USE_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$mapping}) {
+	       push(@mappings_whole, $mapping) if $start_of_word_p && $end_of_word_p;
+	    } elsif ($ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	          || $ht{USE_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       push(@mappings_start_or_end, $mapping) if $start_of_word_p;
+	    } elsif ($ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	          || $ht{USE_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       push(@mappings_start_or_end, $mapping) if $end_of_word_p;
+	    } else {
+	       push(@mappings_other, $mapping);
+	    }
+	  }
+	  @mappings = @mappings_whole;
+	  @mappings = @mappings_start_or_end unless @mappings;
+	  @mappings = @mappings_other unless @mappings;
+	  foreach $mapping (@mappings) {
+	    next if $mapping =~ /\(__.*__\)/;
+	    if ($ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	     || $ht{DONT_USE_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       next if $start_of_word_p;
+	    }
+	    if ($ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
+	     || $ht{DONT_USE_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
+	       next if $end_of_word_p;
+	    }
+	    my $mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $mapping) : $mapping;
+	    $node_id = $this->add_node($mapping2, $i, $i+$string_length, *chart_ht, "", "multi-char-mapping");
+	    $next_index = $i + $string_length;
+	    $found_char_mapping_p = 1;
+	    if ($annotation) {
+	       @annotation_elems = split(/,\s*/, $annotation);
+	       foreach $annotation_elem (@annotation_elems) {
+	          if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
+	             $this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
+		  } else {
+	             $this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
+		  }
+	       }
+	    }
+	  }
+	  my @alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
+	  @alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING}->{$multi_char_substring}} unless @alt_mappings;
+	  @alt_mappings = () if ($#alt_mappings == 0) && ($alt_mappings[0] eq "_NONE_");
+	  foreach $alt_mapping (@alt_mappings) {
+	    if ($chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$i}) {
+	       next unless
+	          $ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	       || $ht{USE_ALT_IN_POINTED}->{$multi_char_substring}->{$alt_mapping};
+	    }
+	    if ($ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	     || $ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$alt_mapping}) {
+	       next unless $start_of_word_p && $end_of_word_p;
+	    }
+	    if ($ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	     || $ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
+	       next unless $start_of_word_p;
+	    }
+	    if ($ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
+	     || $ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
+	       next unless $end_of_word_p;
+	    }
+	    my $alt_mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $alt_mapping) : $alt_mapping;
+	    $node_id = $this->add_node($alt_mapping2, $i, $i+$string_length, *chart_ht, "alt", "multi-char-mapping");
+	    if ($annotation) {
+	       @annotation_elems = split(/,\s*/, $annotation);
+	       foreach $annotation_elem (@annotation_elems) {
+	          if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
+	             $this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
+	          } else {
+	             $this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
+	          }
+	       }
+	    }
+	  }
+        }
+      }
+      unless ($found_char_mapping_p) {
+	 my $prev_node_id = $this->get_node_for_span($i-4, $i, *chart_ht)
+			 || $this->get_node_for_span($i-3, $i, *chart_ht)
+			 || $this->get_node_for_span($i-2, $i, *chart_ht)
+			 || $this->get_node_for_span($i-1, $i, *chart_ht);
+	 my $prev_char_roman = ($prev_node_id) ? $this->get_node_roman($prev_node_id, *chart_id) : "";
+	 my $prev_node_start = ($prev_node_id) ? $chart_ht{NODE_START}->{$prev_node_id} : "";
+
+	 # Number
+         if (($numeric_value =~ /\d/)
+	       && (! ($char_name =~ /SUPERSCRIPT/))) {
+	    my $prev_numeric_value = $this->get_node_for_span_with_slot_value($i-1, $i, "numeric-value", *chart_id);
+	    my $sep = "";
+	    $sep = " " if ($char_name =~ /^vulgar fraction /i) && ($prev_numeric_value =~ /\d/);
+	    $node_id = $this->add_node("$sep$numeric_value", $i, $i+1, *chart_ht, "", "number");
+	    $this->set_node_id_slot_value($node_id, "numeric-value", $numeric_value, *chart_ht);
+	    if ((($prev_numeric_value =~ /\d/) && ($numeric_value =~ /\d\d/))
+	     || (($prev_numeric_value =~ /\d\d/) && ($numeric_value =~ /\d/))) {
+	       # pull in any other parts of single digits
+	       my $j = 1;
+	       # pull in any single digits adjoining on left
+	       if ($prev_numeric_value =~ /^\d$/) {
+		  while (1) {
+	             if (($i-$j-1 >= 0)
+		      && defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-1, $i-$j, "numeric-value", *chart_id))
+		      && ($digit_value =~ /^\d$/)) {
+		        $j++;
+		     } elsif (($i-$j-2 >= 0)
+                           && ($chart_ht{ORIG_CHAR}->{($i-$j-1)} =~ /^[.,]$/)
+		           && defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-2, $i-$j-1, "numeric-value", *chart_id))
+		           && ($digit_value =~ /^\d$/)) {
+		        $j += 2;
+		     } else {
+		        last;
+		     }
+		  }
+	       }
+	       # pull in any single digits adjoining on right
+	       my $k = 0;
+	       if ($numeric_value =~ /^\d$/) {
+	          while (1) {
+	             if (defined($next_numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{($i+$k+1)})
+		      && ($next_numeric_value =~ /^\d$/)) {
+		        $k++;
+		     } else {
+		        last;
+		     }
+		  }
+	       }
+	       $this->register_new_complex_number_span_segment($i-$j, $i, $i+$k+1, *chart_ht, $line_number);
+	    }
+	    if ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
+	     && ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
+	       $de_accented_translit = $util->de_accent_string($tonal_translit);
+	       if ($numeric_value =~ /^(10000|1000000000000|10000000000000000)$/) {
+                  $chart_ht{NODE_TYPE}->{$node_id} = "alt"; # keep, but demote
+	          $alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
+	       } else {
+	          $alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "alt", "CJK");
+	       }
+            }
+
+	 # ASCII
+	 } elsif ($char =~ /^[\x00-\x7F]$/) {
+	    $this->add_node($char, $i, $i+1, *chart_ht, "", "ASCII"); # ASCII character, incl. control characters
+
+	 # Emoji, dingbats, pictographs
+	 } elsif ($char =~ /^(\xE2[\x98-\x9E]|\xF0\x9F[\x8C-\xA7])/) {
+	    $this->add_node($char, $i, $i+1, *chart_ht, "", "pictograph");
+
+         # Hangul (Korean)
+         } elsif (($char =~ /^[\xEA-\xED]/)
+	       && ($romanized_char = $this->unicode_hangul_romanization($char))) {
+	    $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "Hangul");
+
+         # CJK (Chinese, Japanese, Korean)
+	 } elsif ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
+	       && ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
+	    $de_accented_translit = $util->de_accent_string($tonal_translit);
+	    $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
+
+	 # Virama (cancel preceding vowel in Abudiga scripts)
+	 } elsif ($char_name =~ /\bSIGN (?:VIRAMA|AL-LAKUNA|ASAT|COENG|PAMAAEH)\b/) {
+	    # VIRAMA: cancel preceding default vowel (in Abudiga scripts)
+	    if (($prev_script eq $current_script)
+	     && (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
+	     && ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
+	       $this->add_node($prev_char_roman_consonant, $prev_node_start, $i+1, *chart_ht, "", "virama");
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-virama");
+	    }
+
+	 # Nukta (special (typically foreign) variant)
+	 } elsif ($char_name =~ /\bSIGN (?:NUKTA)\b/) {
+	    # NUKTA (dot): indicates special (typically foreign) variant; normally covered by multi-mappings
+	    if ($prev_script eq $current_script) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "nukta");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	       $this->set_node_id_slot_value($node_id, "nukta", 1, *chart_ht);
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-nukta");
+	    }
+
+	 # Zero-width character, incl. zero width space/non-joiner/joiner, left-to-right/right-to-left mark
+	 } elsif ($char =~ /^\xE2\x80[\x8B-\x8F\xAA-\xAE]$/) {
+	    if ($prev_node_id) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "zero-width-char");
+	    }
+	 } elsif (($char =~ /^\xEF\xBB\xBF$/) && $prev_node_id) { # OK to leave byte-order-mark at beginning of line
+	    my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
+            $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+
+	 # Tone mark
+	 } elsif ($tone_mark) {
+	    if ($prev_script eq $current_script) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "tone-mark");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	       $this->set_node_id_slot_value($node_id, "tone-mark", $tone_mark, *chart_ht);
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-tone-mark");
+	    }
+
+	 # Diacritic
+	 } elsif (($char_name =~ /\b(ACCENT|TONE|COMBINING DIAERESIS|COMBINING DIAERESIS BELOW|COMBINING MACRON|COMBINING VERTICAL LINE ABOVE|COMBINING DOT ABOVE RIGHT|COMBINING TILDE|COMBINING CYRILLIC|MUUSIKATOAN|TRIISAP)\b/) && ($ht{UTF_TO_CAT}->{$char} =~ /^Mn/)) {
+	    if ($prev_script eq $current_script) {
+	       my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "diacritic");
+               $this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
+	       $diacritic = lc $char_name;
+	       $diacritic =~ s/^.*(?:COMBINING CYRILLIC|COMBINING|SIGN)\s+//i;
+	       $diacritic =~ s/^.*(ACCENT|TONE)/$1/i;
+	       $diacritic =~ s/^\s*//;
+	       $this->set_node_id_slot_value($node_id, "diacritic", $diacritic, *chart_ht);
+	       # print STDERR "diacritic: $diacritic\n";
+	    } else {
+	       $this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-diacritic");
+	    }
+
+	 # Romanize to find out more
+	 } elsif ($char_name) {
+	    if (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht))) {
+	       # print STDERR "ROM l.$line_number/$i: $romanized_char\n" if $line_number =~ /^[12]$/;
+	       print STDOUT "ROM l.$line_number/$i: $romanized_char\n" if $verbosePM;
+
+	       # Empty string mapping
+	       if ($romanized_char eq "\"\"") {
+	          $this->add_node("", $i, $i+1, *chart_ht, "", "empty-string-mapping");
+               # consider adding something for implausible romanizations of length 6+
+
+	       # keep original character (instead of romanized_char lengthener, character-18b00 etc.)
+	       } elsif (($romanized_char =~ /^(character|lengthener|modifier)/)) {
+	          $this->add_node($char, $i, $i+1, *chart_ht, "", "nevermind-keep-original");
+
+	       # Syllabic suffix in Abudiga languages, e.g. -m, -ng
+               } elsif (($romanized_char =~ /^\+(H|M|N|NG)$/i)
+		     && ($prev_script eq $current_script)
+		     && ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{"a"})) {
+		  my $core_suffix = $romanized_char;
+		  $core_suffix =~ s/^\+//;
+		  if ($prev_char_roman =~ /[aeiou]$/i) {
+	             $this->add_node($core_suffix, $i, $i+1, *chart_ht, "", "syllable-end-consonant");
+		  } else {
+	             $this->add_node(join("", $prev_char_roman, "a", $core_suffix), $prev_node_start, $i+1, *chart_ht, "", "syllable-end-consonant-with-added-a");
+	             $this->add_node(join("", "a", $core_suffix), $i, $i+1, *chart_ht, "backup", "syllable-end-consonant");
+		  }
+
+	       # Japanese special cases
+	       } elsif ($char_name =~ /(?:HIRAGANA|KATAKANA) LETTER SMALL Y/) {
+		  if (($prev_script eq $current_script)
+		   && (($prev_char_roman_consonant) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])i$/i))) {
+                     unless ($this->get_node_for_span_and_type($prev_node_start, $i+1, *chart_ht, "")) {
+	                $this->add_node("$prev_char_roman_consonant$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "japanese-contraction");
+		     }
+		  } else {
+	             $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "unexpected-japanese-contraction-character");
+		  }
+	       } elsif (($prev_script =~ /^(HIRAGANA|KATAKANA)$/i)
+		     && ($char_name eq "KATAKANA-HIRAGANA PROLONGED SOUND MARK") # Choonpu
+		     && (($prev_char_roman_vowel) = ($prev_char_roman =~ /([aeiou])$/i))) {
+	          $this->add_node("$prev_char_roman$prev_char_roman_vowel", $prev_node_start, $i+1, *chart_ht, "", "japanese-vowel-lengthening");
+	       } elsif (($current_script =~ /^(Hiragana|Katakana)$/i)
+	             && ($char_name =~ /^(HIRAGANA|KATAKANA) LETTER SMALL TU$/i) # Sokuon/Sukun
+		     && ($next_script eq $current_script)
+	             && ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, *ht, *chart_ht))
+		     && (($doubled_consonant) = ($romanized_next_char =~ /^(ch|[bcdfghjklmnpqrstwz])/i))) {
+		  # Note: $romanized_next_char could be part of a multi-character mapping
+		  # print STDERR "current_script: $current_script char_name: $char_name next_script: $next_script romanized_next_char: $romanized_next_char doubled_consonant: $doubled_consonant\n";
+		  $doubled_consonant = "t" if $doubled_consonant eq "ch";
+	          $this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "japanese-consonant-doubling");
+ 
+               # Greek small letter mu to micro-sign (instead of to "m") as used in abbreviations for microgram/micrometer/microliter/microsecond/micromolar/microfarad etc.
+               } elsif (($char_name eq "GREEK SMALL LETTER MU")
+	             && (! ($prev_script =~ /^GREEK$/))
+		     && ($i < $#chars)
+		     && ($chart_ht{ORIG_CHAR}->{($i+1)} =~ /^[cfgjlmstv]$/i)) {
+	          $this->add_node("\xC2\xB5", $i, $i+1, *chart_ht, "", "greek-mu-to-micro-sign");
+
+	       # Gurmukhi addak (doubles following consonant)
+               } elsif (($current_script eq "Gurmukhi")
+		     && ($char_name eq "GURMUKHI ADDAK")) {
+                  if (($next_script eq $current_script)
+		   && ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, *ht, *chart_ht))
+	           && (($doubled_consonant) = ($romanized_next_char =~ /^([bcdfghjklmnpqrstvwxz])/i))) {
+	             $this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "gurmukhi-consonant-doubling");
+		  } else {
+	             $this->add_node("'", $i, $i+1, *chart_ht, "", "gurmukhi-unexpected-addak");
+		  }
+
+	       # Subjoined character
+               } elsif ($subjoined_char_p
+		     && ($prev_script eq $current_script)
+	             && (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
+	             && ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
+		  my $new_roman = "$prev_char_roman_consonant$romanized_char";
+	          $this->add_node($new_roman, $prev_node_start, $i+1, *chart_ht, "", "subjoined-character");
+	          # print STDERR "  Subjoin l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;
+
+	       # Thai special case: written-pre-consonant-spoken-post-consonant
+	       } elsif (($char_name =~ /THAI CHARACTER/)
+		     && ($prev_script eq $current_script)
+		     && ($chart_ht{CHAR_SYLLABLE_INFO}->{($i-1)} =~ /written-pre-consonant-spoken-post-consonant/i)
+		     && ($prev_char_roman =~ /^[aeiou]+$/i)
+		     && ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]/)) {
+	          $this->add_node("$romanized_char$prev_char_roman", $prev_node_start, $i+1, *chart_ht, "", "thai-vowel-consonant-swap");
+
+	       # Thai special case: THAI CHARACTER O ANG (U+0E2D "\xE0\xB8\xAD")
+	       } elsif ($char_name eq "THAI CHARACTER O ANG") {
+		  if ($prev_script ne $current_script) {
+	             $this->add_node("", $i, $i+1, *chart_ht, "", "thai-initial-o-ang-drop");
+		  } elsif ($next_script ne $current_script) {
+	             $this->add_node("", $i, $i+1, *chart_ht, "", "thai-final-o-ang-drop");
+		  } else {
+	             my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
+		     my $romanized_prev2_char = $this->romanize_char_at_position($i-2, $lang_code, $output_style, *ht, *chart_ht);
+		     if (($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
+		      && ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
+	                $this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonants
+		     } elsif (($prev2_script eq $current_script)
+			   && 0
+		           && ($prev_char_name =~ /^THAI CHARACTER MAI [A-Z]+$/) # Thai tone
+			   && ($romanized_prev2_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
+			   && ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
+	                $this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonant+tone-mark and consonant
+		     } else {
+	                $this->add_node("", $i, $i+1, *chart_ht, "", "thai-middle-o-ang-drop"); # drop next to vowel
+		     }
+		  }
+
+	       # Romanization with space
+	       } elsif ($romanized_char =~ /\s/) {
+	          $this->add_node($char, $i, $i+1, *chart_ht, "", "space");
+
+	       # Tibetan special cases
+	       } elsif ($current_script eq "Tibetan") {
+
+                  if ($subjoined_char_p
+		   && ($prev_script eq $current_script)
+		   && $prev_letter_plus_char_p
+	           && ($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
+	             $this->add_node("$prev_char_roman$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "subjoined-tibetan-character");
+		  } elsif ($romanized_char =~ /^-A$/i) {
+	             my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
+		     if (! $prev_letter_plus_char_p) {
+	                $this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-frontal-dash-a");
+		     } elsif (($prev_script eq $current_script)
+		           && ($next_script eq $current_script)
+			   && ($prev_char_roman =~ /[bcdfghjklmnpqrstvwxyz]$/)
+			   && ($romanized_next_char =~ /^[aeiou]/)) {
+			$this->add_node("a'", $i, $i+1, *chart_ht, "", "tibetan-medial-dash-a");
+		     } elsif (($prev_script eq $current_script)
+		           && ($next_script eq $current_script)
+			   && ($prev_char_roman =~ /[aeiou]$/)
+			   && ($romanized_next_char =~ /[aeiou]/)) {
+			$this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-reduced-medial-dash-a");
+		     } elsif (($prev_script eq $current_script)
+		           && (! ($prev_char_roman =~ /[aeiou]/))
+			   && (! $next_letter_plus_char_p)) {
+			$this->add_node("a", $i, $i+1, *chart_ht, "", "tibetan-final-dash-a");
+		     } else {
+			$this->add_node("a", $i, $i+1, *chart_ht, "", "unexpected-tibetan-dash-a");
+		     }
+		  } elsif (($romanized_char =~ /^[AEIOU]/i)
+			&& ($prev_script eq $current_script)
+		        && ($prev_char_roman =~ /^A$/i)
+			&& (! $prev2_letter_plus_char_p)) {
+	             $this->add_node($romanized_char, $prev_node_start, $i+1, *chart_ht, "", "tibetan-dropped-word-initial-a");
+		  } else {
+	             $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
+		  }
+
+               # Khmer (for MUUSIKATOAN etc. see under "Diacritic" above)
+	       } elsif (($current_script eq "Khmer")
+	             && (($char_roman_consonant, $char_roman_vowel) = ($romanized_char =~ /^(.*[bcdfghjklmnpqrstvwxyz])([ao]+)-$/i))) {
+	           my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, *ht, *chart_ht);
+		   if (($next_script eq $current_script)
+		    && ($romanized_next_char =~ /^[aeiouy]/i)) {
+                      $this->add_node($char_roman_consonant, $i, $i+1, *chart_ht, "", "khmer-vowel-drop");
+		   } else {
+                      $this->add_node("$char_roman_consonant$char_roman_vowel", $i, $i+1, *chart_ht, "", "khmer-standard-unicode-based-romanization");
+		   }
+
+	       # Abudiga add default vowel
+	       } elsif ((@abudiga_default_vowels = sort keys %{$ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}})
+		     && ($abudiga_default_vowel = $abudiga_default_vowels[0])
+	             && ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
+		  my $new_roman = join("", $romanized_char, $abudiga_default_vowel);
+	          $this->add_node($new_roman, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization-plus-abudiga-default-vowel");
+	          # print STDERR "  Abudiga add default vowel l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;
+
+	       # Standard romanization
+	       } else {
+	          $node_id = $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
+	       }
+	    } else {
+	       $this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original");
+	    }
+	 } elsif (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht))
+	       && ((length($romanized_char) <= 2)
+	        || ($ht{UTF_TO_CHAR_ROMANIZATION}->{$char}))) { # or from unicode_overwrite_romanization table
+	    $romanized_char =~ s/^""$//;
+	    $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "romanized-without-character-name");
+	 } else {
+	    $this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original-without-character-name");
+	 }
+      }
+      $i = $next_index;
+   }
+
+   $this->schwa_deletion(0, $n_characters, *chart_ht, $lang_code);
+   $this->default_vowelize_tibetan(0, $n_characters, *chart_ht, $lang_code, $line_number) if $chart_ht{CHART_CONTAINS_SCRIPT}->{"Tibetan"};
+   $this->assemble_numbers_in_chart(*chart_ht, $line_number);
+
+   if ($return_chart_p) {
+   } elsif ($return_offset_mappings_p) {
+      ($result, $offset_mappings, $new_char_offset, $new_rom_char_offset) = $this->best_romanized_string(0, $n_characters, *chart_ht, $control, $initial_char_offset, $initial_rom_char_offset);
+   } else {
+      $result = $this->best_romanized_string(0, $n_characters, *chart_ht) unless $return_chart_p;
+   }
+
+   if ($verbosePM) {
+      my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-log.txt";
+      $util->append_to_file($logfile, $log) if $log && (-r $logfile);
+   }
+
+   return ($result, $offset_mappings) if $return_offset_mappings_p;
+   return *chart_ht if $return_chart_p;
+   return $result;
+}
+
+sub string_to_json_string {
+   local($this, $s) = @_;
+
+   utf8::decode($s);
+   my $j = JSON->new->utf8->encode([$s]);
+   $j =~ s/^\[(.*)\]$/$1/;
+   return $j;
+}
+
+sub chart_to_json_romanization_elements {
+   local($this, $chart_start, $chart_end, *chart_ht, $line_number) = @_;
+
+   my $result = "";
+   my $start = $chart_start;
+   my $end;
+   while ($start < $chart_end) {
+      $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my @best_romanizations;
+      if (($end && ($start < $end))
+       && (@best_romanizations = $this->best_romanizations($start, $end, *chart_ht))) {
+         $orig_segment = $this->orig_string_at_span($start, $end, *chart_ht);
+         $next_start = $end;
+      } else {
+         $orig_segment = $chart_ht{ORIG_CHAR}->{$start};
+         @best_romanizations = ($orig);
+	 $next_start = $start + 1;
+      }
+      $exclusive_end = $end - 1;
+      # $guarded_orig = $util->string_guard($orig_segment);
+      $guarded_orig = $this->string_to_json_string($orig_segment);
+      $result .= "  { \"line\": $line_number, \"start\": $start, \"end\": $exclusive_end, \"orig\": $guarded_orig, \"roms\": [";
+      foreach $i ((0 .. $#best_romanizations)) {
+         my $rom = $best_romanizations[$i];
+	 # my $guarded_rom = $util->string_guard($rom);
+	 my $guarded_rom = $this->string_to_json_string($rom);
+	 $result .= " { \"rom\": $guarded_rom";
+	 # $result .= ", \"alt\": true" if $i >= 1;
+	 $result .= " }";
+	 $result .= "," if $i < $#best_romanizations;
+      }
+      $result .= " ] },\n";
+      $start = $next_start;
+   }
+   return $result;
+}
+
+sub default_vowelize_tibetan {
+   local($this, $chart_start, $chart_end, *chart_ht, $lang_code, $line_number) = @_;
+
+   # my $verbose = ($line_number == 103);
+   # print STDERR "\nStart default_vowelize_tibetan l.$line_number $chart_start-$chart_end\n" if $verbose;
+   my $token_start = $chart_start;
+   my $next_token_start = $chart_start;
+   while (($token_start = $next_token_start) < $chart_end) {
+      $next_token_start = $token_start + 1;
+
+      next unless $chart_ht{CHAR_LETTER_PLUS}->{$token_start};
+      my $current_script = $chart_ht{CHAR_SCRIPT}->{$token_start};
+         next unless ($current_script eq "Tibetan");
+      my $token_end = $chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$token_start};
+	 next unless $token_end;
+	 next unless $token_end > $token_start;
+      $next_token_start = $token_end;
+
+      my $start = $token_start;
+      my $end;
+      my @node_ids = ();
+      while ($start < $token_end) {
+         $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+	 last unless $end && ($end > $start);
+         my @alt_node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
+         last unless @alt_node_ids;
+         push(@node_ids, $alt_node_ids[0]);
+	 $start = $end;
+      }
+      my $contains_vowel_p = 0;
+      my @romanizations = ();
+      foreach $node_id (@node_ids) {
+         my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
+	 $roman = "" unless defined($roman);
+	 push(@romanizations, $roman);
+	 $contains_vowel_p = 1 if $roman =~ /[aeiou]/i;
+      }
+      # print STDERR "   old: $token_start-$token_end @romanizations\n" if $verbose;
+      unless ($contains_vowel_p) {
+	 my $default_vowel_target_index;
+	 if ($#node_ids <= 1) {
+	    $default_vowel_target_index = 0;
+	 } elsif ($romanizations[$#romanizations] eq "s") {
+	    if ($romanizations[($#romanizations-1)] eq "y") {
+	       $default_vowel_target_index = $#romanizations-1;
+	    } else {
+	       $default_vowel_target_index = $#romanizations-2;
+	    }
+	 } else {
+	    $default_vowel_target_index = $#romanizations-1;
+	 }
+	 $romanizations[$default_vowel_target_index] .= "a";
+	 my $old_node_id = $node_ids[$default_vowel_target_index];
+         my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+         my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
+	 my $new_roman = $old_roman . "a";
+	 my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-default-vowel");
+         $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+         $chart_ht{NODE_TYPE}->{$old_node_id} = "backup"; # keep, but demote
+      }
+      if (($romanizations[0] eq "'")
+       && ($#romanizations >= 1)
+       && ($romanizations[1] =~ /^[o]$/)) {
+	 my $old_node_id = $node_ids[0];
+         my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+         my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $new_node_id = $this->add_node("", $old_start, $old_end, *chart_ht, "", "tibetan-delete-apostrophe");
+	 $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+	 $chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
+      }
+      if (($#node_ids >= 1)
+       && ($romanizations[$#romanizations] =~ /^[bcdfghjklmnpqrstvwxz]+y$/)) {
+	 my $old_node_id = $node_ids[$#romanizations];
+         my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+         my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
+	 my $new_roman = $old_roman . "a";
+	 my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-final-vowel");
+	 $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+	 $chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
+      }
+      foreach $old_node_id (@node_ids) {
+         my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
+	 next unless $old_roman =~ /-a/;
+	 my $old_start = $chart_ht{NODE_START}->{$old_node_id};
+	 my $old_end = $chart_ht{NODE_END}->{$old_node_id};
+	 my $new_roman = $old_roman;
+	 $new_roman =~ s/-a/a/;
+	 my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-delete-dash");
+	 $this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
+	 $chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
+      }
+   }
+}
+
+sub schwa_deletion {
+   local($this, $chart_start, $chart_end, *chart_ht, $lang_code) = @_;
+   # delete word-final simple "a" in Devanagari (e.g. nepaala -> nepaal)
+   # see Wikipedia article "Schwa deletion in Indo-Aryan languages"
+
+   if ($chart_ht{CHART_CONTAINS_SCRIPT}->{"Devanagari"}) {
+      my $script_start = $chart_start;
+      my $next_script_start = $chart_start;
+      while (($script_start = $next_script_start) < $chart_end) {
+         $next_script_start = $script_start + 1;
+
+         my $current_script = $chart_ht{CHAR_SCRIPT}->{$script_start};
+	    next unless ($current_script eq "Devanagari");
+	 my $script_end = $chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start};
+	    next unless $script_end;
+	    next unless $script_end - $script_start >= 2;
+	 $next_script_start = $script_end;
+	 my $end_node_id = $this->get_node_for_span($script_end-1, $script_end, *chart_ht);
+	    next unless $end_node_id;
+         my $end_roman = $chart_ht{NODE_ROMAN}->{$end_node_id};
+	 next unless ($end_consonant) = ($end_roman =~ /^([bcdfghjklmnpqrstvwxz]+)a$/i);
+	 my $prev_node_id = $this->get_node_for_span($script_end-4, $script_end-1, *chart_ht)
+	                 || $this->get_node_for_span($script_end-3, $script_end-1, *chart_ht)
+	                 || $this->get_node_for_span($script_end-2, $script_end-1, *chart_ht);
+	    next unless $prev_node_id;
+         my $prev_roman = $chart_ht{NODE_ROMAN}->{$prev_node_id};
+	 next unless $prev_roman =~ /[aeiou]/i;
+	 # TO DO: check further back for vowel (e.g. if $prev_roman eq "r" due to vowel cancelation)
+	 
+         $chart_ht{NODE_TYPE}->{$end_node_id} = "alt"; # keep, but demote
+	 # print STDERR "* Schwa deletion " . ($script_end-1) . "-$script_end $end_roman->$end_consonant\n";
+	 $this->add_node($end_consonant, $script_end-1, $script_end, *chart_ht, "", "devanagari-with-deleted-final-schwa");
+      }
+   }
+}
+
+sub best_romanized_string {
+   local($this, $chart_start, $chart_end, *chart_ht, $control, $orig_char_offset, $rom_char_offset) = @_;
+
+   $control = "" unless defined($control);
+   my $current_orig_char_offset = $orig_char_offset || 0;
+   my $current_rom_char_offset = $rom_char_offset || 0;
+   my $return_offset_mappings_p = ($control =~ /\breturn offset mappings\b/);
+   my $result = "";
+   my $start = $chart_start;
+   my $end;
+   my @char_offsets = ("$current_orig_char_offset:$current_rom_char_offset");
+   while ($start < $chart_end) {
+      $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my $n_orig_chars_in_segment = 0;
+      my $n_rom_chars_in_segment = 0;
+      if ($end && ($start < $end)) {
+         my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
+	 my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
+	 if (defined($best_romanization)) {
+            $result .= $best_romanization;
+	    if ($return_offset_mappings_p) {
+	       $n_orig_chars_in_segment = $end-$start;
+	       $n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
+	    }
+	    $start = $end;
+	 } else {
+	    my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
+            $result .= $best_romanization;
+	    $start++;
+	    if ($return_offset_mappings_p) {
+	       $n_orig_chars_in_segment = 1;
+	       $n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
+	    }
+	 }
+      } else {
+	 my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
+         $result .= $best_romanization;
+	 $start++;
+	 if ($return_offset_mappings_p) {
+	    $n_orig_chars_in_segment = 1;
+	    $n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
+	 }
+      }
+      if ($return_offset_mappings_p) {
+         my $new_orig_char_offset = $current_orig_char_offset + $n_orig_chars_in_segment;
+         my $new_rom_char_offset = $current_rom_char_offset + $n_rom_chars_in_segment;
+         my $offset_mapping = "$new_orig_char_offset:$new_rom_char_offset";
+         push(@char_offsets, $offset_mapping);
+         $current_orig_char_offset = $new_orig_char_offset;
+         $current_rom_char_offset = $new_rom_char_offset;
+      }
+   }
+   return ($result, join(",", @char_offsets), $current_orig_char_offset, $current_rom_char_offset) if $return_offset_mappings_p;
+   return $result;
+}
+
+sub orig_string_at_span {
+   local($this, $start, $end, *chart_ht) = @_;
+
+   my $result = "";
+   foreach $i (($start .. ($end-1))) {
+      $result .= $chart_ht{ORIG_CHAR}->{$i};
+   }
+   return $result;
+}
+
+sub find_end_of_rom_segment {
+   local($this, $start, $chart_end, *chart_ht) = @_;
+
+   my @ends = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}};
+   my $end_index = $#ends;
+   while (($end_index >= 0) && ($ends[$end_index] > $chart_end)) {
+      $end_index--;
+   }
+   if (($end_index >= 0)
+    && defined($end = $ends[$end_index])
+    && ($start < $end)) {
+      return $end;
+   } else {
+      return "";
+   }
+}
+
+sub best_romanizations {
+   local($this, $start, $end, *chart_ht) = @_;
+
+   @regular_romanizations = ();
+   @alt_romanizations = ();
+   @backup_romanizations = ();
+
+   foreach $node_id (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}}) {
+      my $type = $chart_ht{NODE_TYPE}->{$node_id};
+      my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
+      if (! defined($roman)) {
+         # ignore
+      } elsif (($type eq "backup") && ! defined($backup_romanization)) {
+         push(@backup_romanizations, $roman) unless $util->member($roman, @backup_romanizations);
+      } elsif (($type eq "alt") && ! defined($alt_romanization)) {
+         push(@alt_romanizations, $roman) unless $util->member($roman, @alt_romanizations);
+      } else {
+         push(@regular_romanizations, $roman) unless $util->member($roman, @regular_romanizations);
+      }
+   }
+   @regular_alt_romanizations = sort @regular_romanizations;
+   foreach $alt_romanization (sort @alt_romanizations) {
+      push(@regular_alt_romanizations, $alt_romanization) unless $util->member($alt_romanization, @regular_alt_romanizations);
+   }
+   return @regular_alt_romanizations if @regular_alt_romanizations;
+   return sort @backup_romanizations;
+}
+
+sub join_alt_romanizations_for_viz {
+   local($this, @list) = @_;
+
+   my @viz_romanizations = ();
+
+   foreach $alt_rom (@list) {
+      if ($alt_rom eq "") {
+         push(@viz_romanizations, "-");
+      } else {
+         push(@viz_romanizations, $alt_rom);
+      }
+   }
+   return join(", ", @viz_romanizations);
+}
+
+sub markup_orig_rom_strings {
+   local($this, $chart_start, $chart_end, *ht, *chart_ht, *pinyin_ht, $last_group_id_index) = @_;
+
+   my $marked_up_rom = "";
+   my $marked_up_orig = "";
+   my $start = $chart_start;
+   my $end;
+   while ($start < $chart_end) {
+      my $segment_start = $start;
+      my $segment_end = $start+1;
+      my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my $rom_segment = "";
+      my $orig_segment = "";
+      my $rom_title = "";
+      my $orig_title = "";
+      my $contains_alt_romanizations = 0;
+      if ($end) {
+	 $segment_end = $end;
+         my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
+	 my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
+	 if (defined($best_romanization)) {
+            $rom_segment .= $best_romanization;
+            $orig_segment .= $this->orig_string_at_span($start, $end, *chart_ht);
+	    $segment_end = $end;
+	    if ($#best_romanizations >= 1) {
+	       $rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
+	       $contains_alt_romanizations = 1;
+	    }
+	 } else {
+	    my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
+            $rom_segment .= $segment;
+            $orig_segment .= $segment;
+	    $segment_end = $start+1;
+	 }
+	 $start = $segment_end;
+      } else {
+         $rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
+         $orig_segment .= $this->orig_string_at_span($start, $start+1, *chart_ht);
+	 $segment_end = $start+1;
+	 $start = $segment_end;
+      }
+      my $next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
+      my $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+      while ($next_char_is_combining_p
+          && ($segment_end < $chart_end)
+	  && ($end = $this->find_end_of_rom_segment($segment_end, $chart_end, *chart_ht))
+	  && ($end > $segment_end)
+	  && (@best_romanizations = $this->best_romanizations($segment_end, $end, *chart_ht))
+	  && defined($best_romanization = $best_romanizations[0])) {
+         $orig_segment .= $this->orig_string_at_span($segment_end, $end, *chart_ht);
+	 $rom_segment .= $best_romanization;
+	 if ($#best_romanizations >= 1) {
+	    $rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
+	    $contains_alt_romanizations = 1;
+	 }
+	 $segment_end = $end;
+	 $start = $segment_end;
+	 $next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
+	 $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+      }
+      foreach $i (($segment_start .. ($segment_end-1))) {
+	 $orig_title .= "+&#x200E; &#x200E;" unless $orig_title eq "";
+         my $char = $chart_ht{ORIG_CHAR}->{$i};
+	 my $numeric = $ht{UTF_TO_NUMERIC}->{$char};
+	 $numeric = "" unless defined($numeric);
+	 my $pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
+	 $pic_descr = "" unless defined($pic_descr);
+	 if ($char =~ /^\xE4\xB7[\x80-\xBF]$/) {
+	    $orig_title .= "$char_name\n";
+	 } elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
+	    my $unicode = $utf8->utf8_to_unicode($char);
+	    $orig_title .= "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode)) . "\n";
+	    $orig_title .= "Chinese: $tonal_translit\n" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
+	    $orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
+	 } elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
+	    $orig_title .= "$char_name\n";
+	    $orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
+	    $orig_title .= "Picture: $pic_descr\n" if $pic_descr =~ /\S/;
+	 } else {
+	    my $unicode = $utf8->utf8_to_unicode($char);
+	    if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
+	       $orig_title .= "Hangul syllable U+" . (uc sprintf("%04x", $unicode)) . "\n";
+	    } else {
+	       $orig_title .= "Unicode character U+" . (uc sprintf("%04x", $unicode)) . "\n";
+	    }
+	 }
+      }
+      (@non_ascii_roms) = ($rom_segment =~ /([\xC0-\xFF][\x80-\xBF]*)/g);
+      foreach $char (@non_ascii_roms) {
+	 my $char_name = $ht{UTF_TO_CHAR_NAME}->{$char};
+         my $unicode = $utf8->utf8_to_unicode($char);
+	 my $unicode_s = "U+" . (uc sprintf("%04x", $unicode));
+	 if ($char_name) {
+	    $rom_title .= "$char_name\n";
+	 } else {
+	    $rom_title .= "$unicode_s\n";
+	 }
+      }
+      $last_group_id_index++;
+      $rom_title =~ s/\s*$//;
+      $rom_title =~ s/\n/&#xA;/g;
+      $orig_title =~ s/\s*$//;
+      $orig_title =~ s/\n/&#xA;&#x200E;/g;
+      $orig_title = "&#x202D;" . $orig_title . "&#x202C;";
+      my $rom_title_clause  = ($rom_title  eq "") ? "" : " title=\"$rom_title\"";
+      my $orig_title_clause = ($orig_title eq "") ? "" : " title=\"$orig_title\"";
+      my $alt_rom_clause = ($contains_alt_romanizations) ? "border-bottom:1px dotted;" : "";
+      $marked_up_rom .= "<span id=\"span-$last_group_id_index-1\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\" style=\"color:#00BB00;$alt_rom_clause\"$rom_title_clause>" . $util->guard_html($rom_segment) . "<\/span>";
+      $marked_up_orig .= "<span id=\"span-$last_group_id_index-2\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\"$orig_title_clause>" . $util->guard_html($orig_segment) . "<\/span>";
+      if (($last_char = $chart_ht{ORIG_CHAR}->{($segment_end-1)})
+       && ($last_char_name = $ht{UTF_TO_CHAR_NAME}->{$last_char})
+       && ($last_char_name =~ /^(FULLWIDTH COLON|FULLWIDTH COMMA|FULLWIDTH RIGHT PARENTHESIS|IDEOGRAPHIC COMMA|IDEOGRAPHIC FULL STOP|RIGHT CORNER BRACKET|BRAILLE PATTERN BLANK|TIBETAN MARK .*)$/)) {
+         $marked_up_orig .= "<wbr>";
+         $marked_up_rom .= "<wbr>";
+      }
+   }
+   return ($marked_up_rom, $marked_up_orig, $last_group_id_index);
+}
+
+sub romanizations_with_alternatives {
+   local($this, *ht, *chart_ht, *pinyin_ht, $chart_start, $chart_end) = @_;
+
+   $chart_start = 0 unless defined($chart_start);
+   $chart_end = $chart_ht{N_CHARS} unless defined($chart_end);
+   my $result = "";
+   my $start = $chart_start;
+   my $end;
+   # print STDOUT "romanizations_with_alternatives $chart_start-$chart_end\n";
+   while ($start < $chart_end) {
+      my $segment_start = $start;
+      my $segment_end = $start+1;
+      my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
+      my $rom_segment = "";
+      # print STDOUT "  $start-$end\n";
+      if ($end) {
+	 $segment_end = $end;
+         my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
+         # print STDOUT "  $start-$end @best_romanizations\n";
+	 if (@best_romanizations) {
+	    if ($#best_romanizations == 0) {
+	       $rom_segment .= $best_romanizations[0];
+	    } else {
+	       $rom_segment .= "{" . join("|", @best_romanizations) . "}";
+	    }
+	    $segment_end = $end;
+	 } else {
+	    my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
+            $rom_segment .= $segment;
+	    $segment_end = $start+1;
+	 }
+	 $start = $segment_end;
+      } else {
+         $rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
+	 $segment_end = $start+1;
+	 $start = $segment_end;
+      }
+      # print STDOUT "  $start-$end ** $rom_segment\n";
+      $result .= $rom_segment;
+   }
+   return $result;
+}
+
+sub quick_romanize {
+   local($this, $s, $lang_code, *ht) = @_;
+
+   my $result = "";
+   my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   while (@chars) {
+      my $found_match_in_table_p = 0;
+      foreach $string_length (reverse(1..4)) {
+	 next if ($string_length-1) > $#chars;
+	 $multi_char_substring = join("", @chars[0..($string_length-1)]);
+	 my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
+	 @mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
+	 if (@mappings) {
+	    my $mapping = $mappings[0];
+	    $result .= $mapping;
+            foreach $_ ((1 .. $string_length)) {
+	       shift @chars;
+	    }
+	    $found_match_in_table_p = 1;
+	    last;
+	 }
+      }
+      unless ($found_match_in_table_p) {
+	 $result .= $chars[0];
+	 shift @chars;
+      }
+   }
+   return $result;
+}
+
+sub char_is_combining_char {
+   local($this, $c, *ht) = @_;
+
+   return 0 unless $c;
+   my $category = $ht{UTF_TO_CAT}->{$c};
+   return 0 unless $category;
+   return $category =~ /^M/;
+}
+
+sub mark_up_string_for_mouse_over {
+   local($this, $s, *ht, $control, *pinyin_ht) = @_;
+
+   $control = "" unless defined($control);
+   $no_ascii_p = ($control =~ /NO-ASCII/);
+   my $result = "";
+   @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
+   while (@chars) {
+      $char = shift @chars;
+      $numeric = $ht{UTF_TO_NUMERIC}->{$char};
+      $numeric = "" unless defined($numeric);
+      $pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
+      $pic_descr = "" unless defined($pic_descr);
+      $next_char = ($#chars >= 0) ? $chars[0] : "";
+      $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+      if ($no_ascii_p
+       && ($char =~ /^[\x00-\x7F]*$/)
+       && ! $next_char_is_combining_p) {
+	 $result .= $util->guard_html($char);
+      } elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
+	 $unicode = $utf8->utf8_to_unicode($char);
+	 $title = "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode));
+	 $title .= "&#xA;Chinese: $tonal_translit" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
+	 $title .= "&#xA;Number: $numeric" if $numeric =~ /\d/;
+	 $result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
+      } elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
+	 $title = $char_name;
+	 $title .= "&#xA;Number: $numeric" if $numeric =~ /\d/;
+	 $title .= "&#xA;Picture: $pic_descr" if $pic_descr =~ /\S/;
+	 $char_plus = $char;
+	 while ($next_char_is_combining_p) {
+	    # combining marks (Mc:non-spacing, Mc:spacing combining, Me: enclosing)
+	    $next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char};
+	    $title .= "&#xA;+ $next_char_name";
+	    $char = shift @chars;
+	    $char_plus .= $char;
+	    $next_char = ($#chars >= 0) ? $chars[0] : "";
+	    $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
+	 }
+	 $result .= "<span title=\"$title\">" . $util->guard_html($char_plus) . "<\/span>";
+	 $result .= "<wbr>" if $char_name =~ /^(FULLWIDTH COLON|FULLWIDTH COMMA|FULLWIDTH RIGHT PARENTHESIS|IDEOGRAPHIC COMMA|IDEOGRAPHIC FULL STOP|RIGHT CORNER BRACKET)$/;
+      } elsif (($unicode = $utf8->utf8_to_unicode($char))
+	    && ($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
+	 $title = "Hangul syllable U+" . (uc sprintf("%04x", $unicode));
+	 $result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
+      } else {
+	 $result .= $util->guard_html($char);
+      }
+   }
+   return $result;
+}
+
+sub romanize_char_at_position_incl_multi {
+   local($this, $i, $lang_code, $output_style, *ht, *chart_ht) = @_;
+
+   my $char = $chart_ht{ORIG_CHAR}->{$i};
+   return "" unless defined($char);
+   my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$char}}; 
+   return $mappings[0] if @mappings;
+   @mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$char}};
+   return $mappings[0] if @mappings;
+   return $this->romanize_char_at_position($i, $lang_code, $output_style, *ht, *chart_ht);
+}
+
+sub romanize_char_at_position {
+   local($this, $i, $lang_code, $output_style, *ht, *chart_ht) = @_;
+
+   my $char = $chart_ht{ORIG_CHAR}->{$i};
+   return "" unless defined($char);
+   return $char if $char =~ /^[\x00-\x7F]$/; # ASCII
+   my $romanization = $ht{UTF_TO_CHAR_ROMANIZATION}->{$char};
+   return $romanization if $romanization;
+   my $char_name = $chart_ht{CHAR_NAME}->{$i};
+   $romanization = $this->romanize_charname($char_name, $lang_code, $output_style, *ht, $char);
+   $ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization}
+      = ($ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization} || 0) + 1
+      unless (length($romanization) < 4) 
+          || ($romanization =~ /\s/)
+          || ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,3}[aeiou]-$/) # Khmer ngo-/nyo-/pho- OK
+          || ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,2}[aeiougw][aeiou]{1,2}$/) # Canadian, Ethiopic syllable OK
+	  || ($romanization =~ /^(allah|bbux|nyaa|nnya|quuv|rrep|shch|shur|syrx)$/i)  # Arabic; Yi; Ethiopic syllable nyaa; Cyrillic letter shcha
+          || (($char_name =~ /^(YI SYLLABLE|VAI SYLLABLE|ETHIOPIC SYLLABLE|CANADIAN SYLLABICS|CANADIAN SYLLABICS CARRIER)\s+(\S+)$/) && (length($romanization) <= 5));
+   # print STDERR "romanize_char_at_position $i $char_name :: $romanization\n" if $char_name =~ /middle/i;
+   return $romanization;
+}
+
+sub romanize_charname {
+   local($this, $char_name, $lang_code, $output_style, *ht, $char) = @_;
+
+   my $cached_result = $ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style};
+   # print STDERR "(C) romanize_charname($char_name): $cached_result\n" if $cached_result && ($char_name =~ /middle/i);
+   return $cached_result if defined($cashed_result);
+   $orig_char_name = $char_name;
+   $char_name =~ s/^.* LETTER\s+([A-Z]+)-\d+$/$1/; # HENTAIGANA LETTER A-3
+   $char_name =~ s/^.* LETTER\s+//;
+   $char_name =~ s/^.* SYLLABLE\s+B\d\d\d\s+//; # Linear B syllables
+   $char_name =~ s/^.* SYLLABLE\s+//;
+   $char_name =~ s/^.* SYLLABICS\s+//;
+   $char_name =~ s/^.* LIGATURE\s+//;
+   $char_name =~ s/^.* VOWEL SIGN\s+//;
+   $char_name =~ s/^.* CONSONANT SIGN\s+//;
+   $char_name =~ s/^.* CONSONANT\s+//;
+   $char_name =~ s/^.* VOWEL\s+//;
+   $char_name =~ s/ WITH .*$//;
+   $char_name =~ s/ WITHOUT .*$//;
+   $char_name =~ s/\s+(ABOVE|AGUNG|BAR|BARREE|BELOW|CEDILLA|CEREK|DIGRAPH|DOACHASHMEE|FINAL FORM|GHUNNA|GOAL|INITIAL FORM|ISOLATED FORM|KAWI|LELET|LELET RASWADI|LONSUM|MAHAPRANA|MEDIAL FORM|MURDA|MURDA MAHAPRANA|REVERSED|ROTUNDA|SASAK|SUNG|TAM|TEDUNG|TYPE ONE|TYPE TWO|WOLOSO)\s*$//;
+   $char_name =~ s/^([A-Z]+)\d+$/$1/; # Linear B syllables etc.
+   foreach $_ ((1 .. 3)) {
+      $char_name =~ s/^.*\b(?:ABKHASIAN|ACADEMY|AFRICAN|AIVILIK|AITON|AKHMIMIC|ALEUT|ALI GALI|ALPAPRAANA|ALTERNATE|ALTERNATIVE|AMBA|ARABIC|ARCHAIC|ASPIRATED|ATHAPASCAN|BASELINE|BLACKLETTER|BARRED|BASHKIR|BERBER|BHATTIPROLU|BIBLE-CREE|BIG|BINOCULAR|BLACKFOOT|BLENDED|BOTTOM|BROAD|BROKEN|CANDRA|CAPITAL|CARRIER|CHILLU|CLOSE|CLOSED|COPTIC|CROSSED|CRYPTOGRAMMIC|CURLED|CURLY|CYRILLIC|DANTAJA|DENTAL|DIALECT-P|DIAERESIZED|DOTLESS|DOUBLE|DOUBLE-STRUCK|EASTERN PWO KAREN|EGYPTOLOGICAL|FARSI|FINAL|FLATTENED|GLOTTAL|GREAT|GREEK|HALF|HIGH|INITIAL|INSULAR|INVERTED|IOTIFIED|JONA|KANTAJA|KASHMIRI|KHAKASSIAN|KHAMTI|KHANDA|KINNA|KIRGHIZ|KOMI|L-SHAPED|LATINATE|LITTLE|LONG|LONG-LEGGED|LOOPED|LOW|MAHAAPRAANA|MALAYALAM|MANCHU|MANDAILING|MATHEMATICAL|MEDIAL|MIDDLE-WELSH|MON|MONOCULAR|MOOSE-CREE|MULTIOCULAR|MUURDHAJA|N-CREE|NARROW|NASKAPI|NDOLE|NEUTRAL|NIKOLSBURG|NORTHERN|NUBIAN|NUNAVIK|NUNAVUT|OJIBWAY|OLD|OPEN|ORKHON|OVERLONG|PALI|PERSIAN|PHARYNGEAL|PRISHTHAMATRA|R-CREE|REDUPLICATION|REVERSED|ROMANIAN|ROUND|ROUNDED|RUDIMENTA|RUMAI PALAUNG|SANSKRIT|SANYAKA|SARA|SAYISI|SCRIPT|SEBATBEIT|SEMISOFT|SGAW KAREN|SHAN|SHARP|SHWE PALAUNG|SHORT|SIBE|SIDEWAYS|SIMALUNGUN|SMALL|SOGDIAN|SOFT|SOUTH-SLAVEY|SOUTHERN|SPIDERY|STIRRUP|STRAIGHT|STRETCHED|SUBSCRIPT|SWASH|TAI LAING|TAILED|TAILLESS|TAALUJA|TH-CREE|TALL|THREE-LEGGED|TURNED|TODO|TOP|TROKUTASTI|TUAREG|UKRAINIAN|UNBLENDED|VISIGOTHIC|VOCALIC|VOICED|VOICELESS|VOLAPUK|WAVY|WESTERN PWO KAREN|WEST-CREE|WESTERN|WIDE|WOODS-CREE|Y-CREE|YENISEI|YIDDISH)\s+//;
+   }
+   $char_name =~ s/\s+(ABOVE|AGUNG|BAR|BARREE|BELOW|CEDILLA|CEREK|DIGRAPH|DOACHASHMEE|FINAL FORM|GHUNNA|GOAL|INITIAL FORM|ISOLATED FORM|KAWI|LELET|LELET RASWADI|LONSUM|MAHAPRANA|MEDIAL FORM|MURDA|MURDA MAHAPRANA|REVERSED|ROTUNDA|SASAK|SUNG|TAM|TEDUNG|TYPE ONE|TYPE TWO|WOLOSO)\s*$//;
+   if ($char_name =~ /THAI CHARACTER/) {
+      $char_name =~ s/^THAI CHARACTER\s+//;
+      if ($char =~ /^\xE0\xB8[\x81-\xAE]/) {
+	 # Thai consonants
+	 $char_name =~ s/^([^AEIOU]*).*/$1/i;
+      } elsif ($char_name =~ /^SARA [AEIOU]/) {
+	 # Thai vowels
+	 $char_name =~ s/^SARA\s+//;
+      } else {
+	 $char_name = $char;
+      }
+   }
+   if ($orig_char_name =~ /(HIRAGANA LETTER|KATAKANA LETTER|SYLLABLE|LIGATURE)/) {
+      $char_name = lc $char_name;
+   } elsif ($char_name =~ /\b(ANUSVARA|ANUSVARAYA|NIKAHIT|SIGN BINDI|TIPPI)\b/) {
+      $char_name = "+m";
+   } elsif ($char_name =~ /\bSCHWA\b/) {
+      $char_name = "e";
+   } elsif ($char_name =~ /\bIOTA\b/) {
+      $char_name = "i";
+   } elsif ($char_name =~ /\s/) {
+   } elsif ($orig_char_name =~ /KHMER LETTER/) {
+      $char_name .= "-";
+   } elsif ($orig_char_name =~ /CHEROKEE LETTER/) {
+      # use whole letter as is
+   } elsif ($orig_char_name =~ /KHMER INDEPENDENT VOWEL/) {
+      $char_name =~ s/q//;
+   } elsif ($orig_char_name =~ /LETTER/) {
+      $char_name =~ s/^[AEIOU]+([^AEIOU]+)$/$1/i;
+      $char_name =~ s/^([^-AEIOUY]+)[AEIOU].*/$1/i;
+      $char_name =~ s/^(Y)[AEIOU].*/$1/i if $orig_char_name =~ /\b(?:BENGALI|DEVANAGARI|GURMUKHI|GUJARATI|KANNADA|MALAYALAM|MODI|MYANMAR|ORIYA|TAMIL|TELUGU|TIBETAN)\b.*\bLETTER YA\b/;
+      $char_name =~ s/^(Y[AEIOU]+)[^AEIOU].*$/$1/i;
+      $char_name =~ s/^([AEIOU]+)[^AEIOU]+[AEIOU].*/$1/i;
+   }
+
+   my $result = ($orig_char_name =~ /\bCAPITAL\b/) ? (uc $char_name) : (lc $char_name);
+   # print STDERR "(R) romanize_charname($orig_char_name): $result\n" if $orig_char_name =~ /middle/i;
+   $ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style} = $result;
+   return $result;
+}
+
+sub assemble_numbers_in_chart {
+   local($this, *chart_ht, $line_number) = @_;
+
+   foreach $start (sort { $a <=> $b } keys %{$chart_ht{COMPLEX_NUMERIC_START_END}}) {
+      my $end = $chart_ht{COMPLEX_NUMERIC_START_END}->{$start};
+      my @numbers = ();
+      foreach $i (($start .. ($end-1))) {
+         my $orig_char = $chart_ht{ORIG_CHAR}->{$i};
+         my $node_id = $this->get_node_for_span_with_slot($i, $i+1, "numeric-value", *chart_id);
+	 if (defined($node_id)) {
+	    my $number = $chart_ht{NODE_ROMAN}->{$node_id};
+	    if (defined($number)) {
+               push(@numbers, $number);
+	    } elsif ($orig_char =~ /^[.,]$/) { # decimal point, comma separator
+	       push(@numbers, $orig_char);
+	    } else {
+	       print STDERR "Found no romanization for node_id $node_id ($i-" . ($i+1) . ") in assemble_numbers_in_chart\n" if $verbosePM;
+	    }
+	 } else {
+	    print STDERR "Found no node_id for span $i-" . ($i+1) . " in assemble_numbers_in_chart\n" if $verbosePM;
+	 }
+      }
+      my $complex_number = $this->assemble_number(join("\xC2\xB7", @numbers), $line_number);
+      # print STDERR "assemble_numbers_in_chart l.$line_number $start-$end $complex_number (@numbers)\n";
+      $this->add_node($complex_number, $start, $end, *chart_ht, "", "complex-number");
+   }
+}
+
+sub assemble_number {
+   local($this, $s, $line_number) = @_;
+   # e.g. 10 9 100 7 10 8 = 1978
+
+   my $middot = "\xC2\xB7";
+   my @tokens = split(/$middot/, $s); # middle dot U+00B7
+   my $i = 0;
+   my @orig_tokens = @tokens;
+
+   # assemble single digit numbers, e.g. 1 7 5 -> 175
+   while ($i < $#tokens) {
+      if ($tokens[$i] =~ /^\d$/) {
+         my $j = $i+1;
+	 while (($j <= $#tokens) && ($tokens[$j] =~ /^[0-9.,]$/)) {
+	    $j++;
+	 }
+	 $j--;
+	 if ($j>$i) {
+	    my $new_token = join("", @tokens[$i .. $j]);
+	    $new_token =~ s/,//g;
+	    splice(@tokens, $i, $j-$i+1, $new_token);
+	 }
+      }
+      $i++;
+   }
+
+   foreach $power ((10, 100, 1000, 10000, 100000, 1000000, 100000000, 1000000000, 1000000000000)) {
+      for (my $i=0; $i <= $#tokens; $i++) {
+	 if ($tokens[$i] == $power) {
+            if (($i > 0) && ($tokens[($i-1)] < $power)) {
+	       splice(@tokens, $i-1, 2, ($tokens[($i-1)] * $tokens[$i]));
+	       $i--;
+               if (($i < $#tokens) && ($tokens[($i+1)] < $power)) {
+	          splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
+	          $i--;
+	       }
+	    }
+	 } 
+	 # 400 30 (e.g. Egyptian)
+	 my $gen_pattern = $power;
+         $gen_pattern =~ s/^1/\[1-9\]/;
+         if (($tokens[$i] =~ /^$gen_pattern$/) && ($i < $#tokens) && ($tokens[($i+1)] < $power)) {
+	    splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
+	    $i--;
+	 }
+      }
+      last if $#tokens == 0;
+   }
+   my $result = join($middot, @tokens);
+   if ($verbosePM) {
+      my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-number-log.txt";
+      $util->append_to_file($logfile, "$s -> $result\n") if -r $logfile;
+      # print STDERR "  assemble number l.$line_number @orig_tokens -> $result\n" if $line_number == 43;
+   }
+   return $result;
+}
+
+1;
+

uroman/lib/NLP/UTF8.pm

@@ -0,0 +1,1404 @@
+################################################################
+#                                                              #
+# UTF8                                                         #
+#                                                              #
+################################################################
+
+package NLP::UTF8;
+
+use NLP::utilities;
+$util = NLP::utilities;
+
+%empty_ht = ();
+
+sub new {
+   local($caller) = @_;
+
+   my $object = {};
+   my $class = ref( $caller ) || $caller;
+   bless($object, $class);
+   return $object;
+}
+
+sub unicode_string2string {
+# input: string that might contain unicode sequences such as "U+0627"
+# output: string in pure utf-8
+   local($caller,$s) = @_;
+
+   my $pre; 
+   my $unicode; 
+   my $post; 
+   my $r1; 
+   my $r2; 
+   my $r3;
+
+   ($pre,$unicode,$post) = ($s =~ /^(.*)(?:U\+|\\u)([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])(.*)$/);
+   return $s unless defined($post);
+   $r1 = $caller->unicode_string2string($pre);
+   $r2 = $caller->unicode_hex_string2string($unicode);
+   $r3 = $caller->unicode_string2string($post);
+   $result = $r1 . $r2 . $r3;
+   return $result;
+}
+
+sub unicode_hex_string2string {
+# input: "0627" (interpreted as hex code)
+# output: utf-8 string for Arabic letter alef
+   local($caller,$unicode) = @_;
+   return "" unless defined($unicode);
+   my $d = hex($unicode);
+   return $caller->unicode2string($d);
+}
+
+sub unicode2string {
+# input: non-neg integer, e.g. 0x627
+# output: utf-8 string for Arabic letter alef
+   local($caller,$d) = @_;
+   return "" unless defined($d) && $d >= 0;
+   return sprintf("%c",$d) if $d <= 0x7F;
+
+   my $lastbyte1 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c",$d | 0xC0, $lastbyte1) if $d <= 0x1F;
+
+   my $lastbyte2 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c",$d | 0xE0, $lastbyte2, $lastbyte1) if $d <= 0xF;
+
+   my $lastbyte3 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c%c",$d | 0xF0, $lastbyte3, $lastbyte2, $lastbyte1) if $d <= 0x7;
+
+   my $lastbyte4 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c%c%c",$d | 0xF8, $lastbyte4, $lastbyte3, $lastbyte2, $lastbyte1) if $d <= 0x3;
+
+   my $lastbyte5 = ($d & 0x3F) | 0x80;
+   $d >>= 6;
+   return sprintf("%c%c%c%c%c%c",$d | 0xFC, $lastbyte5, $lastbyte4, $lastbyte3, $lastbyte2, $lastbyte1) if $d <= 0x1;
+   return ""; # bad input
+}
+
+sub html2utf8 {
+   local($caller, $string) = @_;
+
+   return $string unless $string =~ /\&\#\d{3,5};/;
+
+   my $prev = "";
+   my $s = $string;
+   while ($s ne $prev) {
+      $prev = $s;
+      ($pre,$d,$post) = ($s =~ /^(.*)\&\#(\d+);(.*)$/);
+      if (defined($d) && ((($d >= 160) && ($d <= 255))
+                       || (($d >= 1500) && ($d <= 1699))
+                       || (($d >= 19968) && ($d <= 40879)))) {
+         $html_code = "\&\#" . $d . ";";
+         $utf8_code = $caller->unicode2string($d);
+         $s =~ s/$html_code/$utf8_code/;
+      }
+   }
+   return $s;
+}
+
+sub xhtml2utf8 {
+   local($caller, $string) = @_;
+
+   return $string unless $string =~ /\&\#x[0-9a-fA-F]{2,5};/;
+
+   my $prev = "";
+   my $s = $string;
+   while ($s ne $prev) {
+      $prev = $s;
+      if (($pre, $html_code, $x, $post) = ($s =~ /^(.*)(\&\#x([0-9a-fA-F]{2,5});)(.*)$/)) {
+         $utf8_code = $caller->unicode_hex_string2string($x);
+         $s =~ s/$html_code/$utf8_code/;
+      }
+   }
+   return $s;
+}
+
+sub utf8_marker {
+   return sprintf("%c%c%c\n", 0xEF, 0xBB, 0xBF);
+}
+
+sub enforcer {
+# input: string that might not conform to utf-8
+# output: string in pure utf-8, with a few "smart replacements" and possibly "?"
+   local($caller,$s,$no_repair) = @_;
+
+   my $ascii;
+   my $utf8;
+   my $rest;
+
+   return $s if $s =~ /^[\x00-\x7F]*$/;
+
+   $no_repair = 0 unless defined($no_repair);
+   $orig = $s;
+   $result = "";
+
+   while ($s ne "") {
+      ($ascii,$rest) = ($s =~ /^([\x00-\x7F]+)(.*)$/);
+      if (defined($ascii)) {
+	 $result .= $ascii;
+	 $s = $rest;
+	 next;
+      }
+      ($utf8,$rest) = ($s =~ /^([\xC0-\xDF][\x80-\xBF])(.*)$/);
+      ($utf8,$rest) = ($s =~ /^([\xE0-\xEF][\x80-\xBF][\x80-\xBF])(.*)$/) 
+	 unless defined($rest);
+      ($utf8,$rest) = ($s =~ /^([\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF])(.*)$/)
+	 unless defined($rest);
+      ($utf8,$rest) = ($s =~ /^([\xF8-\xFB][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF])(.*)$/) 
+	 unless defined($rest);
+      if (defined($utf8)) {
+	 $result .= $utf8;
+	 $s = $rest;
+	 next;
+      }
+      ($c,$rest) = ($s =~ /^(.)(.*)$/);
+      if (defined($c)) {
+	 if    ($no_repair)   { $result .= "?"; }
+	 elsif ($c =~ /\x85/) { $result .= "..."; }
+	 elsif ($c =~ /\x91/) { $result .= "'"; }
+	 elsif ($c =~ /\x92/) { $result .= "'"; }
+	 elsif ($c =~ /\x93/) { $result .= $caller->unicode2string(0x201C); }
+	 elsif ($c =~ /\x94/) { $result .= $caller->unicode2string(0x201D); }
+	 elsif ($c =~ /[\xC0-\xFF]/) {
+	    $c2 = $c;
+	    $c2 =~ tr/[\xC0-\xFF]/[\x80-\xBF]/;
+	    $result .= "\xC3$c2";
+	 } else {
+	    $result .= "?";
+	 }
+	 $s = $rest;
+	 next;
+      }
+      $s = "";
+   }
+   $result .= "\n" if ($orig =~ /\n$/) && ! ($result =~ /\n$/);
+   return $result;
+}
+
+sub split_into_utf8_characters {
+# input: utf8 string
+# output: list of sub-strings, each representing a utf8 character
+   local($caller,$string,$group_control, *ht) = @_;
+
+   @characters = ();
+   $end_of_token_p_string = "";
+   $skipped_bytes = "";
+   $group_control = "" unless defined($group_control);
+   $group_ascii_numbers = ($group_control =~ /ASCII numbers/);
+   $group_ascii_spaces = ($group_control =~ /ASCII spaces/);
+   $group_ascii_punct = ($group_control =~ /ASCII punct/);
+   $group_ascii_chars = ($group_control =~ /ASCII chars/);
+   $group_xml_chars = ($group_control =~ /XML chars/);
+   $group_xml_tags = ($group_control =~ /XML tags/);
+   $return_only_chars = ($group_control =~ /return only chars/);
+   $return_trailing_whitespaces = ($group_control =~ /return trailing whitespaces/);
+   if ($group_control =~ /ASCII all/) {
+      $group_ascii_numbers = 1;
+      $group_ascii_spaces = 1;
+      $group_ascii_chars = 1;
+      $group_ascii_punct = 1;
+   }
+   if ($group_control =~ /(XML chars and tags|XML tags and chars)/) {
+      $group_xml_chars = 1;
+      $group_xml_tags = 1;
+   }
+   $orig_string = $string;
+   $string .= " ";
+   while ($string =~ /\S/) {
+      # one-character UTF-8 = ASCII
+      if ($string =~ /^[\x00-\x7F]/) {
+	 if ($group_xml_chars
+	  && (($dec_unicode, $rest) = ($string =~ /^&#(\d+);(.*)$/s))
+	  && ($utf8_char = $caller->unicode2string($dec_unicode))) {
+	    push(@characters, $utf8_char);
+	    $string = $rest;
+	 } elsif ($group_xml_chars
+	  && (($hex_unicode, $rest) = ($string =~ /^&#x([0-9a-f]{1,6});(.*)$/is))
+	  && ($utf8_char = $caller->unicode_hex_string2string($hex_unicode))) {
+	    push(@characters, $utf8_char);
+	    $string = $rest;
+	 } elsif ($group_xml_chars
+	  && (($html_entity_name, $rest) = ($string =~ /^&([a-z]{1,6});(.*)$/is))
+	  && ($dec_unicode = $ht{HTML_ENTITY_NAME_TO_DECUNICODE}->{$html_entity_name})
+	  && ($utf8_char = $caller->unicode2string($dec_unicode))
+	  ) {
+	    push(@characters, $utf8_char);
+	    $string = $rest;
+	 } elsif ($group_xml_tags
+	       && (($tag, $rest) = ($string =~ /^(<\/?[a-zA-Z][-_:a-zA-Z0-9]*(\s+[a-zA-Z][-_:a-zA-Z0-9]*=\"[^"]*\")*\s*\/?>)(.*)$/s))) {
+            push(@characters, $tag);
+	    $string = $rest;
+	 } elsif ($group_ascii_numbers && ($string =~ /^[12]\d\d\d\.[01]?\d.[0-3]?\d([^0-9].*)?$/)) {
+	    ($date) = ($string =~ /^(\d\d\d\d\.\d?\d.\d?\d)([^0-9].*)?$/);
+	    push(@characters,$date);
+	    $string = substr($string, length($date));
+	 } elsif ($group_ascii_numbers && ($string =~ /^\d/)) {
+	    ($number) = ($string =~ /^(\d+(,\d\d\d)*(\.\d+)?)/);
+	    push(@characters,$number);
+	    $string = substr($string, length($number));
+	 } elsif ($group_ascii_spaces && ($string =~ /^(\s+)/)) {
+	    ($space) = ($string =~ /^(\s+)/);
+	    $string = substr($string, length($space));
+	 } elsif ($group_ascii_punct && (($punct_seq) = ($string =~ /^(-+|\.+|[:,%()"])/))) {
+	    push(@characters,$punct_seq);
+	    $string = substr($string, length($punct_seq));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^(\$[A-Z]*|[A-Z]{1,3}\$)/))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && (($abbrev) = ($string =~ /^((?:Jan|Feb|Febr|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec|Mr|Mrs|Dr|a.m|p.m)\.)/))) {
+	    push(@characters,$abbrev);
+	    $string = substr($string, length($abbrev));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^(second|minute|hour|day|week|month|year|inch|foot|yard|meter|kilometer|mile)-(?:long|old)/i))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^(zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|trillion)-/i))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && (($word) = ($string =~ /^([a-zA-Z]+)(?:[ ,;%?|()"]|'s |' |\. |\d+[:hms][0-9 ])/))) {
+	    push(@characters,$word);
+	    $string = substr($string, length($word));
+	 } elsif ($group_ascii_chars && ($string =~ /^([\x21-\x27\x2A-\x7E]+)/)) { # exclude ()
+	    ($ascii) = ($string =~ /^([\x21-\x27\x2A-\x7E]+)/);  # ASCII black-characters
+	    push(@characters,$ascii);
+	    $string = substr($string, length($ascii));
+	 } elsif ($group_ascii_chars && ($string =~ /^([\x21-\x7E]+)/)) {
+	    ($ascii) = ($string =~ /^([\x21-\x7E]+)/);  # ASCII black-characters
+	    push(@characters,$ascii);
+	    $string = substr($string, length($ascii));
+	 } elsif ($group_ascii_chars && ($string =~ /^([\x00-\x7F]+)/)) {
+	    ($ascii) = ($string =~ /^([\x00-\x7F]+)/);
+	    push(@characters,$ascii);
+	    $string = substr($string, length($ascii));
+	 } else {
+	    push(@characters,substr($string, 0, 1));
+	    $string = substr($string, 1);
+	 }
+
+      # two-character UTF-8
+      } elsif ($string =~ /^[\xC0-\xDF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 2));
+	 $string = substr($string, 2);
+
+      # three-character UTF-8
+      } elsif ($string =~ /^[\xE0-\xEF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 3));
+	 $string = substr($string, 3);
+
+      # four-character UTF-8
+      } elsif ($string =~ /^[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 4));
+	 $string = substr($string, 4);
+
+      # five-character UTF-8
+      } elsif ($string =~ /^[\xF8-\xFB][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 5));
+	 $string = substr($string, 5);
+
+      # six-character UTF-8
+      } elsif ($string =~ /^[\xFC-\xFD][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF]/) {
+	 push(@characters,substr($string, 0, 6));
+	 $string = substr($string, 6);
+
+      # not a UTF-8 character
+      } else {
+         $skipped_bytes .= substr($string, 0, 1);
+	 $string = substr($string, 1);
+      }
+      
+      $end_of_token_p_string .= ($string =~ /^\S/) ? "0" : "1" 
+	 if $#characters >= length($end_of_token_p_string);
+   }
+   $string =~ s/ $//; # remove previously added space, but keep original spaces
+   if ($return_trailing_whitespaces) {
+      while ($string =~ /^[ \t]/) {
+         push(@characters,substr($string, 0, 1));
+         $string = substr($string, 1);
+      }
+      push(@characters, "\n") if $orig_string =~ /\n$/;
+   }
+   return ($return_only_chars) ? @characters : ($skipped_bytes, $end_of_token_p_string, @characters);
+}
+
+sub max_substring_info {
+   local($caller,$s1,$s2,$info_type) = @_;
+
+   ($skipped_bytes1, $end_of_token_p_string1, @char_list1) = $caller->split_into_utf8_characters($s1, "", *empty_ht);
+   ($skipped_bytes2, $end_of_token_p_string2, @char_list2) = $caller->split_into_utf8_characters($s2, "", *empty_ht);
+   return 0 if $skipped_bytes1 || $skipped_bytes2;
+
+   $best_substring_start1 = 0;
+   $best_substring_start2 = 0;
+   $best_substring_length = 0;
+
+   foreach $start_pos2 ((0 .. $#char_list2)) {
+      last if $start_pos2 + $best_substring_length > $#char_list2;
+      foreach $start_pos1 ((0 .. $#char_list1)) {
+         last if $start_pos1 + $best_substring_length > $#char_list1;
+         $matching_length = 0;
+         while (($start_pos1 + $matching_length <= $#char_list1)
+	     && ($start_pos2 + $matching_length <= $#char_list2)
+	     && ($char_list1[$start_pos1+$matching_length] eq $char_list2[$start_pos2+$matching_length])) {
+	    $matching_length++;
+	 }
+	 if ($matching_length > $best_substring_length) {
+	    $best_substring_length = $matching_length;
+	    $best_substring_start1 = $start_pos1;
+	    $best_substring_start2 = $start_pos2;
+	 }
+      }
+   }
+   if ($info_type =~ /^max-ratio1$/) {
+      $length1 = $#char_list1 + 1;
+      return ($length1 > 0) ? ($best_substring_length / $length1) : 0;
+   } elsif ($info_type =~ /^max-ratio2$/) {
+      $length2 = $#char_list2 + 1;
+      return ($length2 > 0) ? ($best_substring_length / $length2) : 0;
+   } elsif ($info_type =~ /^substring$/) {
+      return join("", @char_list1[$best_substring_start1 .. $best_substring_start1+$best_substring_length-1]);
+   } else {
+      $length1 = $#char_list1 + 1;
+      $length2 = $#char_list2 + 1;
+      $info = "s1=$s1;s2=$s2";
+      $info .= ";best_substring_length=$best_substring_length";
+      $info .= ";best_substring_start1=$best_substring_start1";
+      $info .= ";best_substring_start2=$best_substring_start2";
+      $info .= ";length1=$length1";
+      $info .= ";length2=$length2";
+      return $info;
+   }
+}
+
+sub n_shared_chars_at_start {
+   local($caller,$s1,$s2) = @_;
+
+   my $n = 0;
+   while (($s1 ne "") && ($s2 ne "")) {
+      ($c1, $rest1) = ($s1 =~ /^(.[\x80-\xBF]*)(.*)$/);
+      ($c2, $rest2) = ($s2 =~ /^(.[\x80-\xBF]*)(.*)$/);
+      if ($c1 eq $c2) {
+	 $n++;
+	 $s1 = $rest1;
+	 $s2 = $rest2;
+      } else {
+	 last;
+      }
+   }
+   return $n;
+}
+
+sub char_length {
+   local($caller,$string,$byte_offset) = @_;
+
+   my $char = ($byte_offset) ? substr($string, $byte_offset) : $string;
+   return 1 if $char =~ /^[\x00-\x7F]/;
+   return 2 if $char =~ /^[\xC0-\xDF]/;
+   return 3 if $char =~ /^[\xE0-\xEF]/;
+   return 4 if $char =~ /^[\xF0-\xF7]/;
+   return 5 if $char =~ /^[\xF8-\xFB]/;
+   return 6 if $char =~ /^[\xFC-\xFD]/;
+   return 0;
+}
+
+sub length_in_utf8_chars {
+   local($caller,$s) = @_;
+
+   $s =~ s/[\x80-\xBF]//g;
+   $s =~ s/[\x00-\x7F\xC0-\xFF]/c/g;
+   return length($s);
+}
+
+sub byte_length_of_n_chars {
+   local($caller,$char_length,$string,$byte_offset,$undef_return_value) = @_;
+
+   $byte_offset = 0 unless defined($byte_offset);
+   $undef_return_value = -1 unless defined($undef_return_value);
+   my $result = 0;
+   my $len;
+   foreach $i ((1 .. $char_length)) {
+      $len = $caller->char_length($string,($byte_offset+$result));
+      return $undef_return_value unless $len;
+      $result += $len;
+   }
+   return $result;
+}
+
+sub replace_non_ASCII_bytes {
+   local($caller,$string,$replacement) = @_;
+
+   $replacement = "HEX" unless defined($replacement);
+   if ($replacement =~ /^(Unicode|U\+4|\\u|HEX)$/) {
+      $new_string = "";
+      while (($pre,$utf8_char, $post) = ($string =~ /^([\x09\x0A\x20-\x7E]*)([\x00-\x08\x0B-\x1F\x7F]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF]|[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF]|[\xF8-\xFF][\x80-\xBF]+|[\x80-\xBF])(.*)$/s)) {
+	 if ($replacement =~ /Unicode/) {
+	    $new_string .= $pre . "<U" . (uc $caller->utf8_to_unicode($utf8_char)) . ">";
+	 } elsif ($replacement =~ /\\u/) {
+	    $new_string .= $pre . "\\u" . (uc sprintf("%04x", $caller->utf8_to_unicode($utf8_char)));
+	 } elsif ($replacement =~ /U\+4/) {
+	    $new_string .= $pre . "<U+" . (uc $caller->utf8_to_4hex_unicode($utf8_char)) . ">";
+	 } else {
+	    $new_string .= $pre . "<HEX-" . $caller->utf8_to_hex($utf8_char) . ">";
+	 }
+	 $string = $post;
+      }
+      $new_string .= $string;
+   } else {
+      $new_string = $string;
+      $new_string =~ s/[\x80-\xFF]/$replacement/g;
+   }
+   return $new_string;
+}
+
+sub valid_utf8_string_p {
+   local($caller,$string) = @_;
+
+   return $string =~ /^(?:[\x09\x0A\x20-\x7E]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF]|[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF])*$/;
+}
+
+sub valid_utf8_string_incl_ascii_control_p {
+   local($caller,$string) = @_;
+
+   return $string =~ /^(?:[\x00-\x7F]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF]|[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF])*$/;
+}
+
+sub utf8_to_hex {
+   local($caller,$s) = @_;
+
+   $hex = "";
+   foreach $i ((0 .. length($s)-1)) {
+      $hex .= uc sprintf("%2.2x",ord(substr($s, $i, 1)));
+   }
+   return $hex;
+}
+
+sub hex_to_utf8 {
+   local($caller,$s) = @_;
+   # surface string \xE2\x80\xBA to UTF8
+
+   my $utf8 = "";
+   while (($hex, $rest) = ($s =~ /^(?:\\x)?([0-9A-Fa-f]{2,2})(.*)$/)) {
+      $utf8 .= sprintf("%c", hex($hex));
+      $s = $rest;
+   }
+   return $utf8;
+}
+
+sub utf8_to_4hex_unicode {
+   local($caller,$s) = @_;
+
+   return sprintf("%4.4x", $caller->utf8_to_unicode($s));
+}
+
+sub utf8_to_unicode {
+   local($caller,$s) = @_;
+
+   $unicode = 0;
+   foreach $i ((0 .. length($s)-1)) {
+      $c = substr($s, $i, 1);
+      if ($c =~ /^[\x80-\xBF]$/) {
+	 $unicode = $unicode * 64 + (ord($c) & 0x3F);
+      } elsif ($c =~ /^[\xC0-\xDF]$/) {
+	 $unicode = $unicode * 32 + (ord($c) & 0x1F);
+      } elsif ($c =~ /^[\xE0-\xEF]$/) {
+	 $unicode = $unicode * 16 + (ord($c) & 0x0F);
+      } elsif ($c =~ /^[\xF0-\xF7]$/) {
+	 $unicode = $unicode * 8 + (ord($c) & 0x07);
+      } elsif ($c =~ /^[\xF8-\xFB]$/) {
+	 $unicode = $unicode * 4 + (ord($c) & 0x03);
+      } elsif ($c =~ /^[\xFC-\xFD]$/) {
+	 $unicode = $unicode * 2 + (ord($c) & 0x01);
+      }
+   }
+   return $unicode;
+}
+
+sub charhex {
+   local($caller,$string) = @_;
+
+   my $result = "";
+   while ($string ne "") {
+      $char = substr($string, 0, 1);
+      $string = substr($string, 1);
+      if ($char =~ /^[ -~]$/) {
+         $result .= $char;
+      } else {
+	 $hex = sprintf("%2.2x",ord($char));
+	 $hex =~ tr/a-f/A-F/;
+         $result .= "<HEX-$hex>";
+      }
+   }
+   return $result;
+}
+
+sub windows1252_to_utf8 {
+   local($caller,$s, $norm_to_ascii_p, $preserve_potential_utf8s_p) = @_;
+
+   return $s if $s =~ /^[\x00-\x7F]*$/; # all ASCII
+
+   $norm_to_ascii_p = 1 unless defined($norm_to_ascii_p);
+   $preserve_potential_utf8s_p = 1 unless defined($preserve_potential_utf8s_p);
+   my $result = "";
+   my $c = "";
+   while ($s ne "") {
+      $n_bytes = 1;
+      if ($s =~ /^[\x00-\x7F]/) {
+	 $result .= substr($s, 0, 1);  # ASCII
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xC0-\xDF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 2);  # valid 2-byte UTF8
+         $n_bytes = 2;
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xE0-\xEF][\x80-\xBF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 3);  # valid 3-byte UTF8
+         $n_bytes = 3;
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xF0-\xF7][\x80-\xBF][\x80-\xBF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 4);  # valid 4-byte UTF8
+         $n_bytes = 4;
+      } elsif ($preserve_potential_utf8s_p && ($s =~ /^[\xF8-\xFB][\x80-\xBF][\x80-\xBF][\x80-\xBF][\x80-\xBF]/)) {
+	 $result .= substr($s, 0, 5);  # valid 5-byte UTF8
+         $n_bytes = 5;
+      } elsif ($s =~ /^[\xA0-\xBF]/) {
+	 $c = substr($s, 0, 1);
+	 $result .= "\xC2$c";
+      } elsif ($s =~ /^[\xC0-\xFF]/) {
+	 $c = substr($s, 0, 1);
+	 $c =~ tr/[\xC0-\xFF]/[\x80-\xBF]/;
+	 $result .= "\xC3$c";
+      } elsif ($s =~ /^\x80/) {
+	 $result .= "\xE2\x82\xAC";  # Euro sign
+      } elsif ($s =~ /^\x82/) {
+	 $result .= "\xE2\x80\x9A";  # single low quotation mark
+      } elsif ($s =~ /^\x83/) {
+	 $result .= "\xC6\x92";      # Latin small letter f with hook
+      } elsif ($s =~ /^\x84/) {
+	 $result .= "\xE2\x80\x9E";  # double low quotation mark
+      } elsif ($s =~ /^\x85/) {
+	 $result .= ($norm_to_ascii_p) ? "..." : "\xE2\x80\xA6";  # horizontal ellipsis (three dots)
+      } elsif ($s =~ /^\x86/) {
+	 $result .= "\xE2\x80\xA0";  # dagger
+      } elsif ($s =~ /^\x87/) {
+	 $result .= "\xE2\x80\xA1";  # double dagger
+      } elsif ($s =~ /^\x88/) {
+	 $result .= "\xCB\x86";      # circumflex
+      } elsif ($s =~ /^\x89/) {
+	 $result .= "\xE2\x80\xB0";  # per mille sign
+      } elsif ($s =~ /^\x8A/) {
+	 $result .= "\xC5\xA0";      # Latin capital letter S with caron
+      } elsif ($s =~ /^\x8B/) {
+	 $result .= "\xE2\x80\xB9";  # single left-pointing angle quotation mark
+      } elsif ($s =~ /^\x8C/) {
+	 $result .= "\xC5\x92";      # OE ligature
+      } elsif ($s =~ /^\x8E/) {
+	 $result .= "\xC5\xBD";      # Latin capital letter Z with caron
+      } elsif ($s =~ /^\x91/) {
+	 $result .= ($norm_to_ascii_p) ? "`" : "\xE2\x80\x98";  # left single quotation mark
+      } elsif ($s =~ /^\x92/) {
+	 $result .= ($norm_to_ascii_p) ? "'" : "\xE2\x80\x99";  # right single quotation mark
+      } elsif ($s =~ /^\x93/) {
+	 $result .= "\xE2\x80\x9C";  # left double quotation mark
+      } elsif ($s =~ /^\x94/) {
+	 $result .= "\xE2\x80\x9D";  # right double quotation mark
+      } elsif ($s =~ /^\x95/) {
+	 $result .= "\xE2\x80\xA2";  # bullet
+      } elsif ($s =~ /^\x96/) {
+	 $result .= ($norm_to_ascii_p) ? "-" : "\xE2\x80\x93";  # n dash
+      } elsif ($s =~ /^\x97/) {
+	 $result .= ($norm_to_ascii_p) ? "-" : "\xE2\x80\x94";  # m dash
+      } elsif ($s =~ /^\x98/) {
+	 $result .= ($norm_to_ascii_p) ? "~" : "\xCB\x9C";      # small tilde
+      } elsif ($s =~ /^\x99/) {
+	 $result .= "\xE2\x84\xA2";  # trade mark sign
+      } elsif ($s =~ /^\x9A/) {
+	 $result .= "\xC5\xA1";      # Latin small letter s with caron
+      } elsif ($s =~ /^\x9B/) {
+	 $result .= "\xE2\x80\xBA";  # single right-pointing angle quotation mark
+      } elsif ($s =~ /^\x9C/) {
+	 $result .= "\xC5\x93";      # oe ligature
+      } elsif ($s =~ /^\x9E/) {
+	 $result .= "\xC5\xBE";      # Latin small letter z with caron
+      } elsif ($s =~ /^\x9F/) {
+	 $result .= "\xC5\xB8";      # Latin capital letter Y with diaeresis
+      } else {
+	 $result .= "?";
+      }
+      $s = substr($s, $n_bytes);
+   }
+   return $result;
+}
+
+sub delete_weird_stuff {
+   local($caller, $s) = @_;
+
+   # delete control chacters (except tab and linefeed), zero-width characters, byte order mark,
+   # directional marks, join marks, variation selectors, Arabic tatweel
+   $s =~ s/([\x00-\x08\x0B-\x1F\x7F]|\xC2[\x80-\x9F]|\xD9\x80|\xE2\x80[\x8B-\x8F]|\xEF\xB8[\x80-\x8F]|\xEF\xBB\xBF|\xF3\xA0[\x84-\x87][\x80-\xBF])//g;
+   return $s;
+}
+
+sub number_of_utf8_character {
+   local($caller, $s) = @_;
+
+   $s2 = $s;
+   $s2 =~ s/[\x80-\xBF]//g;
+   return length($s2);
+}
+
+sub cap_letter_reg_exp {
+   # includes A-Z and other Latin-based capital letters with accents, umlauts and other decorations etc.
+   return "[A-Z]|\xC3[\x80-\x96\x98-\x9E]|\xC4[\x80\x82\x84\x86\x88\x8A\x8C\x8E\x90\x94\x964\x98\x9A\x9C\x9E\xA0\xA2\xA4\xA6\xA8\xAA\xAC\xAE\xB0\xB2\xB4\xB6\xB9\xBB\xBD\xBF]|\xC5[\x81\x83\x85\x87\x8A\x8C\x8E\x90\x92\x96\x98\x9A\x9C\x9E\xA0\xA2\xA4\xA6\xA8\xAA\xAC\xB0\xB2\xB4\xB6\xB8\xB9\xBB\xBD]";
+}
+
+sub regex_extended_case_expansion {
+   local($caller, $s) = @_;
+
+   if ($s =~ /\xC3/) {
+   $s =~ s/\xC3\xA0/\xC3\[\x80\xA0\]/g;
+   $s =~ s/\xC3\xA1/\xC3\[\x81\xA1\]/g;
+   $s =~ s/\xC3\xA2/\xC3\[\x82\xA2\]/g;
+   $s =~ s/\xC3\xA3/\xC3\[\x83\xA3\]/g;
+   $s =~ s/\xC3\xA4/\xC3\[\x84\xA4\]/g;
+   $s =~ s/\xC3\xA5/\xC3\[\x85\xA5\]/g;
+   $s =~ s/\xC3\xA6/\xC3\[\x86\xA6\]/g;
+   $s =~ s/\xC3\xA7/\xC3\[\x87\xA7\]/g;
+   $s =~ s/\xC3\xA8/\xC3\[\x88\xA8\]/g;
+   $s =~ s/\xC3\xA9/\xC3\[\x89\xA9\]/g;
+   $s =~ s/\xC3\xAA/\xC3\[\x8A\xAA\]/g;
+   $s =~ s/\xC3\xAB/\xC3\[\x8B\xAB\]/g;
+   $s =~ s/\xC3\xAC/\xC3\[\x8C\xAC\]/g;
+   $s =~ s/\xC3\xAD/\xC3\[\x8D\xAD\]/g;
+   $s =~ s/\xC3\xAE/\xC3\[\x8E\xAE\]/g;
+   $s =~ s/\xC3\xAF/\xC3\[\x8F\xAF\]/g;
+   $s =~ s/\xC3\xB0/\xC3\[\x90\xB0\]/g;
+   $s =~ s/\xC3\xB1/\xC3\[\x91\xB1\]/g;
+   $s =~ s/\xC3\xB2/\xC3\[\x92\xB2\]/g;
+   $s =~ s/\xC3\xB3/\xC3\[\x93\xB3\]/g;
+   $s =~ s/\xC3\xB4/\xC3\[\x94\xB4\]/g;
+   $s =~ s/\xC3\xB5/\xC3\[\x95\xB5\]/g;
+   $s =~ s/\xC3\xB6/\xC3\[\x96\xB6\]/g;
+   $s =~ s/\xC3\xB8/\xC3\[\x98\xB8\]/g;
+   $s =~ s/\xC3\xB9/\xC3\[\x99\xB9\]/g;
+   $s =~ s/\xC3\xBA/\xC3\[\x9A\xBA\]/g;
+   $s =~ s/\xC3\xBB/\xC3\[\x9B\xBB\]/g;
+   $s =~ s/\xC3\xBC/\xC3\[\x9C\xBC\]/g;
+   $s =~ s/\xC3\xBD/\xC3\[\x9D\xBD\]/g;
+   $s =~ s/\xC3\xBE/\xC3\[\x9E\xBE\]/g;
+   }
+   if ($s =~ /\xC5/) {
+   $s =~ s/\xC5\x91/\xC5\[\x90\x91\]/g;
+   $s =~ s/\xC5\xA1/\xC5\[\xA0\xA1\]/g;
+   $s =~ s/\xC5\xB1/\xC5\[\xB0\xB1\]/g;
+   }
+
+   return $s;
+}
+
+sub extended_lower_case {
+   local($caller, $s) = @_;
+
+   $s =~ tr/A-Z/a-z/;
+
+      # Latin-1
+      if ($s =~ /\xC3[\x80-\x9F]/) {
+         $s =~ s/À/à/g;
+         $s =~ s/Á/á/g;
+         $s =~ s/Â/â/g;
+         $s =~ s/Ã/ã/g;
+         $s =~ s/Ä/ä/g;
+         $s =~ s/Å/å/g;
+         $s =~ s/Æ/æ/g;
+         $s =~ s/Ç/ç/g;
+         $s =~ s/È/è/g;
+         $s =~ s/É/é/g;
+         $s =~ s/Ê/ê/g;
+         $s =~ s/Ë/ë/g;
+         $s =~ s/Ì/ì/g;
+         $s =~ s/Í/í/g;
+         $s =~ s/Î/î/g;
+         $s =~ s/Ï/ï/g;
+         $s =~ s/Ð/ð/g;
+         $s =~ s/Ñ/ñ/g;
+         $s =~ s/Ò/ò/g;
+         $s =~ s/Ó/ó/g;
+         $s =~ s/Ô/ô/g;
+         $s =~ s/Õ/õ/g;
+         $s =~ s/Ö/ö/g;
+         $s =~ s/Ø/ø/g;
+         $s =~ s/Ù/ù/g;
+         $s =~ s/Ú/ú/g;
+         $s =~ s/Û/û/g;
+         $s =~ s/Ü/ü/g;
+         $s =~ s/Ý/ý/g;
+         $s =~ s/Þ/þ/g;
+      }
+      # Latin Extended-A
+      if ($s =~ /[\xC4-\xC5][\x80-\xBF]/) {
+         $s =~ s/Ā/ā/g;
+         $s =~ s/Ă/ă/g;
+         $s =~ s/Ą/ą/g;
+         $s =~ s/Ć/ć/g;
+         $s =~ s/Ĉ/ĉ/g;
+         $s =~ s/Ċ/ċ/g;
+         $s =~ s/Č/č/g;
+         $s =~ s/Ď/ď/g;
+         $s =~ s/Đ/đ/g;
+         $s =~ s/Ē/ē/g;
+         $s =~ s/Ĕ/ĕ/g;
+         $s =~ s/Ė/ė/g;
+         $s =~ s/Ę/ę/g;
+         $s =~ s/Ě/ě/g;
+         $s =~ s/Ĝ/ĝ/g;
+         $s =~ s/Ğ/ğ/g;
+         $s =~ s/Ġ/ġ/g;
+         $s =~ s/Ģ/ģ/g;
+         $s =~ s/Ĥ/ĥ/g;
+         $s =~ s/Ħ/ħ/g;
+         $s =~ s/Ĩ/ĩ/g;
+         $s =~ s/Ī/ī/g;
+         $s =~ s/Ĭ/ĭ/g;
+         $s =~ s/Į/į/g;
+         $s =~ s/İ/ı/g;
+         $s =~ s/IJ/ij/g;
+         $s =~ s/Ĵ/ĵ/g;
+         $s =~ s/Ķ/ķ/g;
+         $s =~ s/Ĺ/ĺ/g;
+         $s =~ s/Ļ/ļ/g;
+         $s =~ s/Ľ/ľ/g;
+         $s =~ s/Ŀ/ŀ/g;
+         $s =~ s/Ł/ł/g;
+         $s =~ s/Ń/ń/g;
+         $s =~ s/Ņ/ņ/g;
+         $s =~ s/Ň/ň/g;
+         $s =~ s/Ŋ/ŋ/g;
+         $s =~ s/Ō/ō/g;
+         $s =~ s/Ŏ/ŏ/g;
+         $s =~ s/Ő/ő/g;
+         $s =~ s/Œ/œ/g;
+         $s =~ s/Ŕ/ŕ/g;
+         $s =~ s/Ŗ/ŗ/g;
+         $s =~ s/Ř/ř/g;
+         $s =~ s/Ś/ś/g;
+         $s =~ s/Ŝ/ŝ/g;
+         $s =~ s/Ş/ş/g;
+         $s =~ s/Š/š/g;
+         $s =~ s/Ţ/ţ/g;
+         $s =~ s/Ť/ť/g;
+         $s =~ s/Ŧ/ŧ/g;
+         $s =~ s/Ũ/ũ/g;
+         $s =~ s/Ū/ū/g;
+         $s =~ s/Ŭ/ŭ/g;
+         $s =~ s/Ů/ů/g;
+         $s =~ s/Ű/ű/g;
+         $s =~ s/Ų/ų/g;
+         $s =~ s/Ŵ/ŵ/g;
+         $s =~ s/Ŷ/ŷ/g;
+         $s =~ s/Ź/ź/g;
+         $s =~ s/Ż/ż/g;
+         $s =~ s/Ž/ž/g;
+      }
+      # Greek letters
+      if ($s =~ /\xCE[\x86-\xAB]/) {
+         $s =~ s/Α/α/g;
+         $s =~ s/Β/β/g;
+         $s =~ s/Γ/γ/g;
+         $s =~ s/Δ/δ/g;
+         $s =~ s/Ε/ε/g;
+         $s =~ s/Ζ/ζ/g;
+         $s =~ s/Η/η/g;
+         $s =~ s/Θ/θ/g;
+         $s =~ s/Ι/ι/g;
+         $s =~ s/Κ/κ/g;
+         $s =~ s/Λ/λ/g;
+         $s =~ s/Μ/μ/g;
+         $s =~ s/Ν/ν/g;
+         $s =~ s/Ξ/ξ/g;
+         $s =~ s/Ο/ο/g;
+         $s =~ s/Π/π/g;
+         $s =~ s/Ρ/ρ/g;
+         $s =~ s/Σ/σ/g;
+         $s =~ s/Τ/τ/g;
+         $s =~ s/Υ/υ/g;
+         $s =~ s/Φ/φ/g;
+         $s =~ s/Χ/χ/g;
+         $s =~ s/Ψ/ψ/g;
+         $s =~ s/Ω/ω/g;
+         $s =~ s/Ϊ/ϊ/g;
+         $s =~ s/Ϋ/ϋ/g;
+         $s =~ s/Ά/ά/g;
+         $s =~ s/Έ/έ/g;
+         $s =~ s/Ή/ή/g;
+         $s =~ s/Ί/ί/g;
+         $s =~ s/Ό/ό/g;
+         $s =~ s/Ύ/ύ/g;
+         $s =~ s/Ώ/ώ/g;
+      }
+      # Cyrillic letters
+      if ($s =~ /\xD0[\x80-\xAF]/) {
+         $s =~ s/А/а/g;
+         $s =~ s/Б/б/g;
+         $s =~ s/В/в/g;
+         $s =~ s/Г/г/g;
+         $s =~ s/Д/д/g;
+         $s =~ s/Е/е/g;
+         $s =~ s/Ж/ж/g;
+         $s =~ s/З/з/g;
+         $s =~ s/И/и/g;
+         $s =~ s/Й/й/g;
+         $s =~ s/К/к/g;
+         $s =~ s/Л/л/g;
+         $s =~ s/М/м/g;
+         $s =~ s/Н/н/g;
+         $s =~ s/О/о/g;
+         $s =~ s/П/п/g;
+         $s =~ s/Р/р/g;
+         $s =~ s/С/с/g;
+         $s =~ s/Т/т/g;
+         $s =~ s/У/у/g;
+         $s =~ s/Ф/ф/g;
+         $s =~ s/Х/х/g;
+         $s =~ s/Ц/ц/g;
+         $s =~ s/Ч/ч/g;
+         $s =~ s/Ш/ш/g;
+         $s =~ s/Щ/щ/g;
+         $s =~ s/Ъ/ъ/g;
+         $s =~ s/Ы/ы/g;
+         $s =~ s/Ь/ь/g;
+         $s =~ s/Э/э/g;
+         $s =~ s/Ю/ю/g;
+         $s =~ s/Я/я/g;
+         $s =~ s/Ѐ/ѐ/g;
+         $s =~ s/Ё/ё/g;
+         $s =~ s/Ђ/ђ/g;
+         $s =~ s/Ѓ/ѓ/g;
+         $s =~ s/Є/є/g;
+         $s =~ s/Ѕ/ѕ/g;
+         $s =~ s/І/і/g;
+         $s =~ s/Ї/ї/g;
+         $s =~ s/Ј/ј/g;
+         $s =~ s/Љ/љ/g;
+         $s =~ s/Њ/њ/g;
+         $s =~ s/Ћ/ћ/g;
+         $s =~ s/Ќ/ќ/g;
+         $s =~ s/Ѝ/ѝ/g;
+         $s =~ s/Ў/ў/g;
+         $s =~ s/Џ/џ/g;
+      }
+      # Fullwidth A-Z
+      if ($s =~ /\xEF\xBC[\xA1-\xBA]/) {
+         $s =~ s/Ａ/ａ/g;
+         $s =~ s/Ｂ/ｂ/g;
+         $s =~ s/Ｃ/ｃ/g;
+         $s =~ s/Ｄ/ｄ/g;
+         $s =~ s/Ｅ/ｅ/g;
+         $s =~ s/Ｆ/ｆ/g;
+         $s =~ s/Ｇ/ｇ/g;
+         $s =~ s/Ｈ/ｈ/g;
+         $s =~ s/Ｉ/ｉ/g;
+         $s =~ s/Ｊ/ｊ/g;
+         $s =~ s/Ｋ/ｋ/g;
+         $s =~ s/Ｌ/ｌ/g;
+         $s =~ s/Ｍ/ｍ/g;
+         $s =~ s/Ｎ/ｎ/g;
+         $s =~ s/Ｏ/ｏ/g;
+         $s =~ s/Ｐ/ｐ/g;
+         $s =~ s/Ｑ/ｑ/g;
+         $s =~ s/Ｒ/ｒ/g;
+         $s =~ s/Ｓ/ｓ/g;
+         $s =~ s/Ｔ/ｔ/g;
+         $s =~ s/Ｕ/ｕ/g;
+         $s =~ s/Ｖ/ｖ/g;
+         $s =~ s/Ｗ/ｗ/g;
+         $s =~ s/Ｘ/ｘ/g;
+         $s =~ s/Ｙ/ｙ/g;
+         $s =~ s/Ｚ/ｚ/g;
+      }
+
+   return $s;
+}
+
+sub extended_upper_case {
+   local($caller, $s) = @_;
+
+   $s =~ tr/a-z/A-Z/;
+   return $s unless $s =~ /[\xC3-\xC5][\x80-\xBF]/;
+
+   $s =~ s/\xC3\xA0/\xC3\x80/g;
+   $s =~ s/\xC3\xA1/\xC3\x81/g;
+   $s =~ s/\xC3\xA2/\xC3\x82/g;
+   $s =~ s/\xC3\xA3/\xC3\x83/g;
+   $s =~ s/\xC3\xA4/\xC3\x84/g;
+   $s =~ s/\xC3\xA5/\xC3\x85/g;
+   $s =~ s/\xC3\xA6/\xC3\x86/g;
+   $s =~ s/\xC3\xA7/\xC3\x87/g;
+   $s =~ s/\xC3\xA8/\xC3\x88/g;
+   $s =~ s/\xC3\xA9/\xC3\x89/g;
+   $s =~ s/\xC3\xAA/\xC3\x8A/g;
+   $s =~ s/\xC3\xAB/\xC3\x8B/g;
+   $s =~ s/\xC3\xAC/\xC3\x8C/g;
+   $s =~ s/\xC3\xAD/\xC3\x8D/g;
+   $s =~ s/\xC3\xAE/\xC3\x8E/g;
+   $s =~ s/\xC3\xAF/\xC3\x8F/g;
+   $s =~ s/\xC3\xB0/\xC3\x90/g;
+   $s =~ s/\xC3\xB1/\xC3\x91/g;
+   $s =~ s/\xC3\xB2/\xC3\x92/g;
+   $s =~ s/\xC3\xB3/\xC3\x93/g;
+   $s =~ s/\xC3\xB4/\xC3\x94/g;
+   $s =~ s/\xC3\xB5/\xC3\x95/g;
+   $s =~ s/\xC3\xB6/\xC3\x96/g;
+   $s =~ s/\xC3\xB8/\xC3\x98/g;
+   $s =~ s/\xC3\xB9/\xC3\x99/g;
+   $s =~ s/\xC3\xBA/\xC3\x9A/g;
+   $s =~ s/\xC3\xBB/\xC3\x9B/g;
+   $s =~ s/\xC3\xBC/\xC3\x9C/g;
+   $s =~ s/\xC3\xBD/\xC3\x9D/g;
+   $s =~ s/\xC3\xBE/\xC3\x9E/g;
+
+   $s =~ s/\xC5\x91/\xC5\x90/g;
+   $s =~ s/\xC5\xA1/\xC5\xA0/g;
+   $s =~ s/\xC5\xB1/\xC5\xB0/g;
+   return $s unless $s =~ /[\xC3-\xC5][\x80-\xBF]/;
+
+   return $s;
+}
+
+sub extended_first_upper_case {
+   local($caller, $s) = @_;
+
+   if (($first_char, $rest) = ($s =~ /^([\x00-\x7F]|[\xC0-\xDF][\x80-\xBF]|[\xE0-\xEF][\x80-\xBF][\x80-\xBF])(.*)$/)) {
+      return $caller->extended_upper_case($first_char) . $rest;
+   } else {
+      return $s;
+   }
+}
+
+sub repair_doubly_converted_utf8_strings {
+   local($caller, $s) = @_;
+
+   if ($s =~ /\xC3[\x82-\x85]\xC2[\x80-\xBF]/) {
+      $s =~ s/\xC3\x82\xC2([\x80-\xBF])/\xC2$1/g;
+      $s =~ s/\xC3\x83\xC2([\x80-\xBF])/\xC3$1/g;
+      $s =~ s/\xC3\x84\xC2([\x80-\xBF])/\xC4$1/g;
+      $s =~ s/\xC3\x85\xC2([\x80-\xBF])/\xC5$1/g;
+   }
+   return $s;
+}
+
+sub repair_misconverted_windows_to_utf8_strings {
+   local($caller, $s) = @_;
+
+   # correcting conversions of UTF8 using Latin1-to-UTF converter
+   if ($s =~ /\xC3\xA2\xC2\x80\xC2[\x90-\xEF]/) {
+      my $result = "";
+      while (($pre,$last_c,$post) = ($s =~ /^(.*?)\xC3\xA2\xC2\x80\xC2([\x90-\xEF])(.*)$/s)) {
+         $result .= "$pre\xE2\x80$last_c";
+         $s = $post;
+      }
+      $result .= $s;
+      $s = $result;
+   }
+   # correcting conversions of Windows1252-to-UTF8 using Latin1-to-UTF converter
+   if ($s =~ /\xC2[\x80-\x9F]/) {
+      my $result = "";
+      while (($pre,$c_windows,$post) = ($s =~ /^(.*?)\xC2([\x80-\x9F])(.*)$/s)) {
+	 $c_utf8 = $caller->windows1252_to_utf8($c_windows, 0);
+         $result .= ($c_utf8 eq "?") ? ($pre . "\xC2" . $c_windows) : "$pre$c_utf8";
+         $s = $post;
+      }
+      $result .= $s;
+      $s = $result;
+   }
+   if ($s =~ /\xC3/) {
+      $s =~ s/\xC3\xA2\xE2\x80\x9A\xC2\xAC/\xE2\x82\xAC/g;     # x80 -> Euro sign
+                                                               # x81 codepoint undefined in Windows 1252
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC5\xA1/\xE2\x80\x9A/g;     # x82 -> single low-9 quotation mark
+      $s =~ s/\xC3\x86\xE2\x80\x99/\xC6\x92/g;                 # x83 -> Latin small letter f with hook
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC5\xBE/\xE2\x80\x9E/g;     # x84 -> double low-9 quotation mark
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA6/\xE2\x80\xA6/g;     # x85 -> horizontal ellipsis
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA0/\xE2\x80\xA0/g;     # x86 -> dagger
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA1/\xE2\x80\xA1/g;     # x87 -> double dagger
+      $s =~ s/\xC3\x8B\xE2\x80\xA0/\xCB\x86/g;                 # x88 -> modifier letter circumflex accent
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xB0/\xE2\x80\xB0/g;     # x89 -> per mille sign
+      $s =~ s/\xC3\x85\xC2\xA0/\xC5\xA0/g;                     # x8A -> Latin capital letter S with caron
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xB9/\xE2\x80\xB9/g;     # x8B -> single left-pointing angle quotation mark
+      $s =~ s/\xC3\x85\xE2\x80\x99/\xC5\x92/g;                 # x8C -> Latin capital ligature OE
+                                                               # x8D codepoint undefined in Windows 1252
+      $s =~ s/\xC3\x85\xC2\xBD/\xC5\xBD/g;                     # x8E -> Latin capital letter Z with caron
+                                                               # x8F codepoint undefined in Windows 1252
+                                                               # x90 codepoint undefined in Windows 1252
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xCB\x9C/\xE2\x80\x98/g;     # x91 a-circumflex+euro+small tilde -> left single quotation mark
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xE2\x84\xA2/\xE2\x80\x99/g; # x92 a-circumflex+euro+trademark -> right single quotation mark
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC5\x93/\xE2\x80\x9C/g;     # x93 a-circumflex+euro+Latin small ligature oe -> left double quotation mark
+							       # x94 maps through undefined intermediate code point
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xA2/\xE2\x80\xA2/g;     # x95 a-circumflex+euro+cent sign -> bullet
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xE2\x80\x9C/\xE2\x80\x93/g; # x96 a-circumflex+euro+left double quotation mark -> en dash
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xE2\x80\x9D/\xE2\x80\x94/g; # x97 a-circumflex+euro+right double quotation mark -> em dash
+      $s =~ s/\xC3\x8B\xC5\x93/\xCB\x9C/g;                     # x98 Latin capital e diaeresis+Latin small ligature oe -> small tilde
+      $s =~ s/\xC3\xA2\xE2\x80\x9E\xC2\xA2/\xE2\x84\xA2/g;     # x99 -> trade mark sign
+      $s =~ s/\xC3\x85\xC2\xA1/\xC5\xA1/g;                     # x9A -> Latin small letter s with caron
+      $s =~ s/\xC3\xA2\xE2\x82\xAC\xC2\xBA/\xE2\x80\xBA/g;     # x9B -> single right-pointing angle quotation mark
+      $s =~ s/\xC3\x85\xE2\x80\x9C/\xC5\x93/g;                 # x9C -> Latin small ligature oe
+							       # x9D codepoint undefined in Windows 1252
+      $s =~ s/\xC3\x85\xC2\xBE/\xC5\xBE/g;                     # x9E -> Latin small letter z with caron
+      $s =~ s/\xC3\x85\xC2\xB8/\xC5\xB8/g;                     # x9F -> Latin capital letter Y with diaeresis 
+      $s =~ s/\xC3\xAF\xC2\xBF\xC2\xBD/\xEF\xBF\xBD/g;         # replacement character
+   }
+
+   return $s;
+}
+
+sub latin1_to_utf {
+   local($caller, $s) = @_;
+
+   my $result = "";
+   while (($pre,$c,$post) = ($s =~ /^(.*?)([\x80-\xFF])(.*)$/s)) {
+      $result .= $pre;
+      if ($c =~ /^[\x80-\xBF]$/) {
+         $result .= "\xC2$c";
+      } elsif ($c =~ /^[\xC0-\xFF]$/) {
+         $c =~ tr/[\xC0-\xFF]/[\x80-\xBF]/;
+         $result .= "\xC3$c";
+      }
+      $s = $post;
+   }
+   $result .= $s;
+   return $result;
+}
+
+sub character_type_is_letter_type {
+   local($caller, $char_type) = @_;
+
+   return ($char_type =~ /\b((CJK|hiragana|kana|katakana)\s+character|diacritic|letter|syllable)\b/);
+}
+
+sub character_type {
+   local($caller, $c) = @_;
+
+   if ($c =~ /^[\x00-\x7F]/) {
+      return "XML tag" if $c =~ /^<.*>$/;
+      return "ASCII Latin letter" if $c =~ /^[a-z]$/i;
+      return "ASCII digit" if $c =~ /^[0-9]$/i;
+      return "ASCII whitespace" if $c =~ /^[\x09-\x0D\x20]$/;
+      return "ASCII control-character" if $c =~ /^[\x00-\x1F\x7F]$/;
+      return "ASCII currency" if $c eq "\$";
+      return "ASCII punctuation";
+   } elsif ($c =~ /^[\xC0-\xDF]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /^[\xC0-\xDF][\x80-\xBF]$/;
+      return "non-shortest-UTF8 (invalid)" if $c =~ /[\xC0-\xC1]/;
+      return "non-ASCII control-character" if $c =~ /\xC2[\x80-\x9F]/;
+      return "non-ASCII whitespace" if $c =~ /\xC2\xA0/;
+      return "non-ASCII currency" if $c =~ /\xC2[\xA2-\xA5]/;
+      return "fraction" if $c =~ /\xC2[\xBC-\xBE]/; # NEW
+      return "superscript digit"  if $c =~ /\xC2[\xB2\xB3\xB9]/;
+      return "non-ASCII Latin letter" if $c =~ /\xC2\xB5/; # micro sign
+      return "non-ASCII punctuation" if $c =~ /\xC2[\xA0-\xBF]/;
+      return "non-ASCII punctuation" if $c =~ /\xC3[\x97\xB7]/;
+      return "non-ASCII Latin letter" if $c =~ /\xC3[\x80-\xBF]/;
+      return "Latin ligature letter" if $c =~ /\xC4[\xB2\xB3]/;
+      return "Latin ligature letter" if $c =~ /\xC5[\x92\x93]/;
+      return "non-ASCII Latin letter" if $c =~ /[\xC4-\xC8]/;
+      return "non-ASCII Latin letter" if $c =~ /\xC9[\x80-\x8F]/;
+      return "IPA" if $c =~ /\xC9[\x90-\xBF]/;
+      return "IPA" if $c =~ /\xCA[\x80-\xBF]/;
+      return "IPA" if $c =~ /\xCB[\x80-\xBF]/;
+      return "combining-diacritic" if $c =~ /\xCC[\x80-\xBF]/;
+      return "combining-diacritic" if $c =~ /\xCD[\x80-\xAF]/;
+      return "Greek punctuation" if $c =~ /\xCD[\xBE]/; # Greek question mark
+      return "Greek punctuation" if $c =~ /\xCE[\x87]/; # Greek semicolon
+      return "Greek letter" if $c =~ /\xCD[\xB0-\xBF]/;
+      return "Greek letter" if $c =~ /\xCE/;
+      return "Greek letter" if $c =~ /\xCF[\x80-\xA1\xB3\xB7\xB8\xBA\xBB]/;
+      return "Coptic letter" if $c =~ /\xCF[\xA2-\xAF]/;
+      return "Cyrillic letter" if $c =~ /[\xD0-\xD3]/;
+      return "Cyrillic letter" if $c =~ /\xD4[\x80-\xAF]/;
+      return "Armenian punctuation" if $c =~ /\xD5[\x9A-\x9F]/;
+      return "Armenian punctuation" if $c =~ /\xD6[\x89-\x8F]/;
+      return "Armenian letter" if $c =~ /\xD4[\xB0-\xBF]/;
+      return "Armenian letter" if $c =~ /\xD5/;
+      return "Armenian letter" if $c =~ /\xD6[\x80-\x8F]/;
+      return "Hebrew accent" if $c =~ /\xD6[\x91-\xAE]/;
+      return "Hebrew punctuation" if $c =~ /\xD6\xBE/;
+      return "Hebrew punctuation" if $c =~ /\xD7[\x80\x83\x86\xB3\xB4]/;
+      return "Hebrew point" if $c =~ /\xD6[\xB0-\xBF]/;
+      return "Hebrew point" if $c =~ /\xD7[\x81\x82\x87]/;
+      return "Hebrew letter" if $c =~ /\xD7[\x90-\xB2]/;
+      return "other Hebrew" if $c =~ /\xD6[\x90-\xBF]/;
+      return "other Hebrew" if $c =~ /\xD7/;
+      return "Arabic currency" if $c =~ /\xD8\x8B/; # Afghani sign
+      return "Arabic punctuation" if $c =~ /\xD8[\x89-\x8D\x9B\x9E\x9F]/;
+      return "Arabic punctuation" if $c =~ /\xD9[\xAA-\xAD]/;
+      return "Arabic punctuation" if $c =~ /\xDB[\x94]/;
+      return "Arabic tatweel" if $c =~ /\xD9\x80/;
+      return "Arabic letter"  if $c =~ /\xD8[\xA0-\xBF]/;
+      return "Arabic letter"  if $c =~ /\xD9[\x81-\x9F]/;
+      return "Arabic letter"  if $c =~ /\xD9[\xAE-\xBF]/;
+      return "Arabic letter"  if $c =~ /\xDA[\x80-\xBF]/;
+      return "Arabic letter"  if $c =~ /\xDB[\x80-\x95]/;
+      return "Arabic Indic digit" if $c =~ /\xD9[\xA0-\xA9]/;
+      return "Arabic Indic digit" if $c =~ /\xDB[\xB0-\xB9]/;
+      return "other Arabic" if $c =~ /[\xD8-\xDB]/;
+      return "Syriac punctuation" if $c =~ /\xDC[\x80-\x8F]/;
+      return "Syriac letter" if $c =~ /\xDC[\x90-\xAF]/;
+      return "Syriac diacritic" if $c =~ /\xDC[\xB0-\xBF]/;
+      return "Syriac diacritic" if $c =~ /\xDD[\x80-\x8A]/;
+      return "Thaana letter" if $c =~ /\xDE/;
+   } elsif ($c =~ /^[\xE0-\xEF]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /^[\xE0-\xEF][\x80-\xBF]{2,2}$/;
+      return "non-shortest-UTF8 (invalid)" if $c =~ /\xE0[\x80-\x9F]/;
+      return "Arabic letter"     if $c =~ /\xE0\xA2[\xA0-\xBF]/; # extended letters
+      return "other Arabic"      if $c =~ /\xE0\xA3/; # extended characters
+      return "Devanagari punctuation" if $c =~ /\xE0\xA5[\xA4\xA5]/; # danda, double danda
+      return "Devanagari digit" if $c =~ /\xE0\xA5[\xA6-\xAF]/;
+      return "Devanagari letter" if $c =~ /\xE0[\xA4-\xA5]/;
+      return "Bengali digit" if $c =~ /\xE0\xA7[\xA6-\xAF]/;
+      return "Bengali currency" if $c =~ /\xE0\xA7[\xB2-\xB9]/;
+      return "Bengali letter" if $c =~ /\xE0[\xA6-\xA7]/;
+      return "Gurmukhi digit" if $c =~ /\xE0\xA9[\xA6-\xAF]/;
+      return "Gurmukhi letter" if $c =~ /\xE0[\xA8-\xA9]/;
+      return "Gujarati digit" if $c =~ /\xE0\xAB[\xA6-\xAF]/;
+      return "Gujarati letter" if $c =~ /\xE0[\xAA-\xAB]/;
+      return "Oriya digit" if $c =~ /\xE0\xAD[\xA6-\xAF]/;
+      return "Oriya fraction" if $c =~ /\xE0\xAD[\xB2-\xB7]/;
+      return "Oriya letter" if $c =~ /\xE0[\xAC-\xAD]/;
+      return "Tamil digit" if $c =~ /\xE0\xAF[\xA6-\xAF]/;
+      return "Tamil number" if $c =~ /\xE0\xAF[\xB0-\xB2]/; # number (10, 100, 1000)
+      return "Tamil letter" if $c =~ /\xE0[\xAE-\xAF]/;
+      return "Telegu digit" if $c =~ /\xE0\xB1[\xA6-\xAF]/;
+      return "Telegu fraction" if $c =~ /\xE0\xB1[\xB8-\xBE]/;
+      return "Telegu letter" if $c =~ /\xE0[\xB0-\xB1]/;
+      return "Kannada digit" if $c =~ /\xE0\xB3[\xA6-\xAF]/;
+      return "Kannada letter" if $c =~ /\xE0[\xB2-\xB3]/;
+      return "Malayalam digit" if $c =~ /\xE0\xB5[\x98-\x9E\xA6-\xB8]/;
+      return "Malayalam punctuation" if $c =~ /\xE0\xB5\xB9/; # date mark
+      return "Malayalam letter" if $c =~ /\xE0[\xB4-\xB5]/;
+      return "Sinhala digit" if $c =~ /\xE0\xB7[\xA6-\xAF]/;
+      return "Sinhala punctuation" if $c =~ /\xE0\xB7\xB4/;
+      return "Sinhala letter" if $c =~ /\xE0[\xB6-\xB7]/;
+      return "Thai currency" if $c =~ /\xE0\xB8\xBF/;
+      return "Thai digit" if $c =~ /\xE0\xB9[\x90-\x99]/;
+      return "Thai character" if $c =~ /\xE0[\xB8-\xB9]/;
+      return "Lao punctuation" if $c =~ /\xE0\xBA\xAF/; # Lao ellipsis
+      return "Lao digit" if $c =~ /\xE0\xBB[\x90-\x99]/;
+      return "Lao character" if $c =~ /\xE0[\xBA-\xBB]/;
+      return "Tibetan punctuation" if $c =~ /\xE0\xBC[\x81-\x94]/;
+      return "Tibetan sign"        if $c =~ /\xE0\xBC[\x95-\x9F]/;
+      return "Tibetan digit"       if $c =~ /\xE0\xBC[\xA0-\xB3]/;
+      return "Tibetan punctuation" if $c =~ /\xE0\xBC[\xB4-\xBD]/;
+      return "Tibetan letter" if $c =~ /\xE0[\xBC-\xBF]/;
+      return "Myanmar digit" if $c =~ /\xE1\x81[\x80-\x89]/;
+      return "Myanmar digit" if $c =~ /\xE1\x82[\x90-\x99]/; # Myanmar Shan digits
+      return "Myanmar punctuation" if $c =~ /\xE1\x81[\x8A-\x8B]/;
+      return "Myanmar letter" if $c =~ /\xE1[\x80-\x81]/;
+      return "Myanmar letter" if $c =~ /\xE1\x82[\x80-\x9F]/;
+      return "Georgian punctuation" if $c =~ /\xE1\x83\xBB/;
+      return "Georgian letter" if $c =~ /\xE1\x82[\xA0-\xBF]/;
+      return "Georgian letter" if $c =~ /\xE1\x83/;
+      return "Georgian letter" if $c =~ /\xE1\xB2[\x90-\xBF]/; # Georgian Mtavruli capital letters
+      return "Georgian letter" if $c =~ /\xE2\xB4[\x80-\xAF]/; # Georgian small letters (Khutsuri)
+      return "Korean Hangul letter" if $c =~ /\xE1[\x84-\x87]/;
+      return "Ethiopic punctuation" if $c =~ /\xE1\x8D[\xA0-\xA8]/;
+      return "Ethiopic digit" if $c =~ /\xE1\x8D[\xA9-\xB1]/;
+      return "Ethiopic number" if $c =~ /\xE1\x8D[\xB2-\xBC]/;
+      return "Ethiopic syllable" if $c =~ /\xE1[\x88-\x8D]/;
+      return "Cherokee letter" if $c =~ /\xE1\x8E[\xA0-\xBF]/;
+      return "Cherokee letter" if $c =~ /\xE1\x8F/;
+      return "Canadian punctuation" if $c =~ /\xE1\x90\x80/; # Canadian Syllabics hyphen
+      return "Canadian punctuation" if $c =~ /\xE1\x99\xAE/; # Canadian Syllabics full stop
+      return "Canadian syllable" if $c =~ /\xE1[\x90-\x99]/;
+      return "Canadian syllable" if $c =~ /\xE1\xA2[\xB0-\xBF]/;
+      return "Canadian syllable" if $c =~ /\xE1\xA3/;
+      return "Ogham whitespace" if $c =~ /\xE1\x9A\x80/;
+      return "Ogham letter" if $c =~ /\xE1\x9A[\x81-\x9A]/;
+      return "Ogham punctuation" if $c =~ /\xE1\x9A[\x9B-\x9C]/;
+      return "Runic punctuation" if $c =~ /\xE1\x9B[\xAB-\xAD]/;
+      return "Runic letter" if $c =~ /\xE1\x9A[\xA0-\xBF]/;
+      return "Runic letter" if $c =~ /\xE1\x9B/;
+      return "Khmer currency" if $c =~ /\xE1\x9F\x9B/;
+      return "Khmer digit" if $c =~ /\xE1\x9F[\xA0-\xA9]/;
+      return "Khmer letter" if $c =~ /\xE1[\x9E-\x9F]/;
+      return "Mongolian punctuation" if $c =~ /\xE1\xA0[\x80-\x8A]/;
+      return "Mongolian digit"       if $c =~ /\xE1\xA0[\x90-\x99]/;
+      return "Mongolian letter" if $c =~ /\xE1[\xA0-\xA1]/;
+      return "Mongolian letter" if $c =~ /\xE1\xA2[\x80-\xAF]/;
+      return "Buginese letter" if $c =~ /\xE1\xA8[\x80-\x9B]/;
+      return "Buginese punctuation" if $c =~ /\xE1\xA8[\x9E-\x9F]/;
+      return "Balinese letter" if $c =~ /\xE1\xAC/;
+      return "Balinese letter" if $c =~ /\xE1\xAD[\x80-\x8F]/;
+      return "Balinese digit" if $c =~ /\xE1\xAD[\x90-\x99]/;
+      return "Balinese puncutation" if $c =~ /\xE1\xAD[\x9A-\xA0]/;
+      return "Balinese symbol" if $c =~ /\xE1\xAD[\xA1-\xBF]/;
+      return "Sundanese digit" if $c =~ /\xE1\xAE[\xB0-\xB9]/;
+      return "Sundanese letter" if $c =~ /\xE1\xAE/;
+      return "Cyrillic letter" if $c =~ /\xE1\xB2[\x80-\x8F]/;
+      return "Sundanese punctuation" if $c =~ /\xE1\xB3[\x80-\x8F]/;
+      return "IPA" if $c =~ /\xE1[\xB4-\xB6]/;
+      return "non-ASCII Latin letter" if $c =~ /\xE1[\xB8-\xBB]/;
+      return "Greek letter" if $c =~ /\xE1[\xBC-\xBF]/;
+      return "non-ASCII whitespace"  if $c =~ /\xE2\x80[\x80-\x8A\xAF]/;
+      return "zero-width space"      if $c =~ /\xE2\x80\x8B/;
+      return "zero-width non-space"  if $c =~ /\xE2\x80\x8C/;
+      return "zero-width joiner"     if $c =~ /\xE2\x80\x8D/;
+      return "directional mark"      if $c =~ /\xE2\x80[\x8E-\x8F\xAA-\xAE]/;
+      return "non-ASCII punctuation" if $c =~ /\xE2\x80[\x90-\xBF]/;
+      return "non-ASCII punctuation" if $c =~ /\xE2\x81[\x80-\x9E]/;
+      return "superscript letter"    if $c =~ /\xE2\x81[\xB1\xBF]/;
+      return "superscript digit"     if $c =~ /\xE2\x81[\xB0-\xB9]/;
+      return "superscript punctuation" if $c =~ /\xE2\x81[\xBA-\xBE]/;
+      return "subscript digit"       if $c =~ /\xE2\x82[\x80-\x89]/;
+      return "subscript punctuation" if $c =~ /\xE2\x82[\x8A-\x8E]/;
+      return "non-ASCII currency"    if $c =~ /\xE2\x82[\xA0-\xBF]/;
+      return "letterlike symbol"     if $c =~ /\xE2\x84/;
+      return "letterlike symbol"     if $c =~ /\xE2\x85[\x80-\x8F]/;
+      return "fraction"              if $c =~ /\xE2\x85[\x90-\x9E]/; # NEW
+      return "Roman number"          if $c =~ /\xE2\x85[\xA0-\xBF]/; # NEW
+      return "arrow symbol"          if $c =~ /\xE2\x86[\x90-\xBF]/;
+      return "arrow symbol"          if $c =~ /\xE2\x87/;
+      return "mathematical operator" if $c =~ /\xE2[\x88-\x8B]/;
+      return "technical symbol"      if $c =~ /\xE2[\x8C-\x8F]/;
+      return "enclosed alphanumeric" if $c =~ /\xE2\x91[\xA0-\xBF]/;
+      return "enclosed alphanumeric" if $c =~ /\xE2[\x92-\x93]/;
+      return "box drawing" if $c =~ /\xE2[\x94-\x95]/;
+      return "geometric shape" if $c =~ /\xE2\x96[\xA0-\xBF]/;
+      return "geometric shape" if $c =~ /\xE2\x97/;
+      return "pictograph" if $c =~ /\xE2[\x98-\x9E]/;
+      return "arrow symbol"          if $c =~ /\xE2\xAC[\x80-\x91\xB0-\xBF]/;
+      return "geometric shape"       if $c =~ /\xE2\xAC[\x92-\xAF]/;         
+      return "arrow symbol"          if $c =~ /\xE2\xAD[\x80-\x8F\x9A-\xBF]/;
+      return "geometric shape"       if $c =~ /\xE2\xAD[\x90-\x99]/;        
+      return "arrow symbol"          if $c =~ /\xE2\xAE[\x80-\xB9]/;       
+      return "geometric shape"       if $c =~ /\xE2\xAE[\xBA-\xBF]/;      
+      return "geometric shape"       if $c =~ /\xE2\xAF[\x80-\x88\x8A-\x8F]/;
+      return "symbol"                if $c =~ /\xE2[\xAC-\xAF]/;
+      return "Coptic fraction" if $c =~ /\xE2\xB3\xBD/;
+      return "Coptic punctuation" if $c =~ /\xE2\xB3[\xB9-\xBF]/;
+      return "Coptic letter" if $c =~ /\xE2[\xB2-\xB3]/;
+      return "Georgian letter" if $c =~ /\xE2\xB4[\x80-\xAF]/;
+      return "Tifinagh punctuation" if $c =~ /\xE2\xB5\xB0/;
+      return "Tifinagh letter" if $c =~ /\xE2\xB4[\xB0-\xBF]/;
+      return "Tifinagh letter" if $c =~ /\xE2\xB5/;
+      return "Ethiopic syllable" if $c =~ /\xE2\xB6/;
+      return "Ethiopic syllable" if $c =~ /\xE2\xB7[\x80-\x9F]/;
+      return "non-ASCII punctuation" if $c =~ /\xE3\x80[\x80-\x91\x94-\x9F\xB0\xBB-\xBD]/;
+      return "symbol" if $c =~ /\xE3\x80[\x91\x92\xA0\xB6\xB7]/;
+      return "Japanese hiragana character" if $c =~ /\xE3\x81/;
+      return "Japanese hiragana character" if $c =~ /\xE3\x82[\x80-\x9F]/;
+      return "Japanese katakana character" if $c =~ /\xE3\x82[\xA0-\xBF]/;
+      return "Japanese katakana character" if $c =~ /\xE3\x83/;
+      return "Bopomofo letter" if $c =~ /\xE3\x84[\x80-\xAF]/;
+      return "Korean Hangul letter" if $c =~ /\xE3\x84[\xB0-\xBF]/;
+      return "Korean Hangul letter" if $c =~ /\xE3\x85/;
+      return "Korean Hangul letter" if $c =~ /\xE3\x86[\x80-\x8F]/;
+      return "Bopomofo letter" if $c =~ /\xE3\x86[\xA0-\xBF]/;
+      return "CJK stroke" if $c =~ /\xE3\x87[\x80-\xAF]/;
+      return "Japanese kana character" if $c =~ /\xE3\x87[\xB0-\xBF]/;
+      return "CJK symbol" if $c =~ /\xE3[\x88-\x8B]/;
+      return "CJK square Latin abbreviation" if $c =~ /\xE3\x8D[\xB1-\xBA]/;
+      return "CJK square Latin abbreviation" if $c =~ /\xE3\x8E/;
+      return "CJK square Latin abbreviation" if $c =~ /\xE3\x8F[\x80-\x9F\xBF]/;
+      return "CJK character" if $c =~ /\xE4[\xB8-\xBF]/;
+      return "CJK character" if $c =~ /[\xE5-\xE9]/;
+      return "Yi syllable" if $c =~ /\xEA[\x80-\x92]/;
+      return "Lisu letter"      if $c =~ /\xEA\x93[\x90-\xBD]/;
+      return "Lisu punctuation" if $c =~ /\xEA\x93[\xBE-\xBF]/;
+      return "Cyrillic letter" if $c =~ /\xEA\x99/;
+      return "Cyrillic letter" if $c =~ /\xEA\x9A[\x80-\x9F]/;
+      return "modifier tone" if $c =~ /\xEA\x9C[\x80-\xA1]/;
+      return "Javanese punctuation" if $c =~ /\xEA\xA7[\x81-\x8D\x9E-\x9F]/;
+      return "Javanese digit" if $c =~ /\xEA\xA7[\x90-\x99]/;
+      return "Javanese letter" if $c =~ /\xEA\xA6/;
+      return "Javanese letter" if $c =~ /\xEA\xA7[\x80-\x9F]/;
+      return "Ethiopic syllable" if $c =~ /\xEA\xAC[\x80-\xAF]/;
+      return "Cherokee letter" if $c =~ /\xEA\xAD[\xB0-\xBF]/;
+      return "Cherokee letter" if $c =~ /\xEA\xAE/;
+      return "Meetai Mayek digit" if $c =~ /\xEA\xAF[\xB0-\xB9]/;
+      return "Meetai Mayek letter" if $c =~ /\xEA\xAF/;
+      return "Korean Hangul syllable" if $c =~ /\xEA[\xB0-\xBF]/;
+      return "Korean Hangul syllable" if $c =~ /[\xEB-\xEC]/;
+      return "Korean Hangul syllable" if $c =~ /\xED[\x80-\x9E]/;
+      return "Klingon letter" if $c =~ /\xEF\xA3[\x90-\xA9]/;
+      return "Klingon digit" if $c =~ /\xEF\xA3[\xB0-\xB9]/;
+      return "Klingon punctuation" if $c =~ /\xEF\xA3[\xBD-\xBE]/;
+      return "Klingon symbol" if $c =~ /\xEF\xA3\xBF/;
+      return "private use character" if $c =~ /\xEE/;
+      return "Latin typographic ligature" if $c =~ /\xEF\xAC[\x80-\x86]/;
+      return "Hebrew presentation letter" if $c =~ /\xEF\xAC[\x9D-\xBF]/;
+      return "Hebrew presentation letter" if $c =~ /\xEF\xAD[\x80-\x8F]/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xAD[\x90-\xBF]/;
+      return "Arabic presentation letter" if $c =~ /\xEF[\xAE-\xB7]/;
+      return "non-ASCII punctuation" if $c =~ /\xEF\xB8[\x90-\x99]/;
+      return "non-ASCII punctuation" if $c =~ /\xEF\xB8[\xB0-\xBF]/;
+      return "non-ASCII punctuation" if $c =~ /\xEF\xB9[\x80-\xAB]/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xB9[\xB0-\xBF]/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xBA/;
+      return "Arabic presentation letter" if $c =~ /\xEF\xBB[\x80-\xBC]/;
+      return "byte-order mark/zero-width no-break space" if $c eq "\xEF\xBB\xBF";
+      return "fullwidth currency" if $c =~ /\xEF\xBC\x84/;
+      return "fullwidth digit" if $c =~ /\xEF\xBC[\x90-\x99]/;
+      return "fullwidth Latin letter" if $c =~ /\xEF\xBC[\xA1-\xBA]/;
+      return "fullwidth Latin letter" if $c =~ /\xEF\xBD[\x81-\x9A]/;
+      return "fullwidth punctuation" if $c =~ /\xEF\xBC/;
+      return "fullwidth punctuation" if $c =~ /\xEF\xBD[\x9B-\xA4]/;
+      return "halfwidth Japanese punctuation" if $c =~ /\xEF\xBD[\xA1-\xA4]/;
+      return "halfwidth Japanese katakana character" if $c =~ /\xEF\xBD[\xA5-\xBF]/;
+      return "halfwidth Japanese katakana character" if $c =~ /\xEF\xBE[\x80-\x9F]/;
+      return "fullwidth currency" if $c =~ /\xEF\xBF[\xA0-\xA6]/;
+      return "replacement character" if $c eq "\xEF\xBF\xBD";
+   } elsif ($c =~ /[\xF0-\xF7]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /[\xF0-\xF7][\x80-\xBF]{3,3}$/;
+      return "non-shortest-UTF8 (invalid)" if $c =~ /\xF0[\x80-\x8F]/;
+      return "Linear B syllable" if $c =~ /\xF0\x90\x80/;
+      return "Linear B syllable" if $c =~ /\xF0\x90\x81[\x80-\x8F]/;
+      return "Linear B symbol"   if $c =~ /\xF0\x90\x81[\x90-\x9F]/;
+      return "Linear B ideogram" if $c =~ /\xF0\x90[\x82-\x83]/;
+      return "Gothic letter" if $c =~ /\xF0\x90\x8C[\xB0-\xBF]/;
+      return "Gothic letter" if $c =~ /\xF0\x90\x8D[\x80-\x8F]/;
+      return "Phoenician letter" if $c =~ /\xF0\x90\xA4[\x80-\x95]/;
+      return "Phoenician number" if $c =~ /\xF0\x90\xA4[\x96-\x9B]/;
+      return "Phoenician punctuation" if $c =~ /\xF0\x90\xA4\x9F/; # word separator
+      return "Old Hungarian number" if $c =~ /\xF0\x90\xB3[\xBA-\xBF]/;
+      return "Old Hungarian letter" if $c =~ /\xF0\x90[\xB2-\xB3]/;
+      return "Cuneiform digit" if $c =~ /\xF0\x92\x90/; # numberic sign
+      return "Cuneiform digit" if $c =~ /\xF0\x92\x91[\x80-\xAF]/; # numberic sign
+      return "Cuneiform punctuation" if $c =~ /\xF0\x92\x91[\xB0-\xBF]/;
+      return "Cuneiform sign" if $c =~ /\xF0\x92[\x80-\x95]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x81\xA8/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x82[\xAD-\xB6]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x86[\x90\xBC-\xBF]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x87[\x80-\x84]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x8D[\xA2-\xAB]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x8E[\x86-\x92]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x8F[\xBA-\xBF]/;
+      return "Egyptian hieroglyph number" if $c =~ /\xF0\x93\x90[\x80-\x83]/;
+      return "Egyptian hieroglyph" if $c =~ /\xF0\x93[\x80-\x90]/;
+      return "enclosed alphanumeric" if $c =~ /\xF0\x9F[\x84-\x87]/;
+      return "Mahjong symbol" if $c =~ /\xF0\x9F\x80[\x80-\xAF]/;
+      return "Domino symbol" if $c =~ /\xF0\x9F\x80[\xB0-\xBF]/;
+      return "Domino symbol" if $c =~ /\xF0\x9F\x81/;
+      return "Domino symbol" if $c =~ /\xF0\x9F\x82[\x80-\x9F]/;
+      return "Playing card symbol" if $c =~ /\xF0\x9F\x82[\xA0-\xBF]/;
+      return "Playing card symbol" if $c =~ /\xF0\x9F\x83/;
+      return "CJK symbol" if $c =~ /\xF0\x9F[\x88-\x8B]/;
+      return "pictograph" if $c =~ /\xF0\x9F[\x8C-\x9B]/;
+      return "geometric shape" if $c =~ /\xF0\x9F[\x9E-\x9F]/;
+      return "non-ASCII punctuation" if $c =~ /\xF0\x9F[\xA0-\xA3]/;
+      return "pictograph" if $c =~ /\xF0\x9F[\xA4-\xAB]/;
+      return "CJK character" if $c =~ /\xF0[\xA0-\xAF]/;
+      return "tag" if $c =~ /\xF3\xA0[\x80-\x81]/;
+      return "variation selector" if $c =~ /\xF3\xA0[\x84-\x87]/;
+      return "private use character" if $c =~ /\xF3[\xB0-\xBF]/;
+      return "private use character" if $c =~ /\xF4[\x80-\x8F]/;
+      # ...
+   } elsif ($c =~ /[\xF8-\xFB]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /[\xF8-\xFB][\x80-\xBF]{4,4}$/;
+   } elsif ($c =~ /[\xFC-\xFD]/) {
+      return "non-UTF8 (invalid)" unless $c =~ /[\xFC-\xFD][\x80-\xBF]{5,5}$/;
+   } elsif ($c =~ /\xFE/) {
+      return "non-UTF8 (invalid)" unless $c =~ /\xFE][\x80-\xBF]{6,6}$/;
+   } else {
+      return "non-UTF8 (invalid)";
+   }
+   return "other character";
+}
+
+1;
+
+

uroman/lib/NLP/stringDistance.pm

@@ -0,0 +1,724 @@
+################################################################
+#                                                              #
+# stringDistance                                               #
+#                                                              #
+################################################################
+
+package NLP::stringDistance;
+
+use List::Util qw(min max);
+$utf8 = NLP::UTF8;
+$util = NLP::utilities;
+$romanizer = NLP::Romanizer;
+
+%dummy_ht = ();
+
+sub rule_string_expansion {
+   local($this, *ht, $s, $lang_code) = @_;
+   
+   my @characters = $utf8->split_into_utf8_characters($s, "return only chars, return trailing whitespaces", *dummy_ht);
+   foreach $sub_len ((0 .. ($#characters-1))) {
+      my $sub = join("", @characters[0 .. $sub_len]);
+      foreach $super_len ((($sub_len + 1) .. $#characters)) {
+         my $super = join("", @characters[0 .. $super_len]);
+	 # print STDERR "  $sub -> $super\n" unless $ht{RULE_STRING_EXPANSION}->{$lang_code}->{$sub}->{$super};
+	 $ht{RULE_STRING_EXPANSION}->{$lang_code}->{$sub}->{$super} = 1;
+	 $ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$sub} = 1;
+	 # print STDERR "  RULE_STRING_HAS_EXPANSION $lang_code $sub\n";
+      }
+   }
+}
+
+sub load_string_distance_data {
+   local($this, $filename, *ht, $verbose) = @_;
+
+   $verbose = 0 unless defined($verbose);
+   open(IN,$filename) || die "Could not open $filename";
+   my $line_number = 0;
+   my $n_cost_rules = 0;
+   while (<IN>) {
+      $line_number++;
+      my $line = $_;
+      $line =~ s/^\xEF\xBB\xBF//;
+      $line =~ s/\s*$//;
+      next if $line =~ /^\s*(\#.*)?$/;
+      print STDERR "** Warning: line $line_number contains suspicious control character: $line\n" if $line =~ /[\x00-\x1F]/;
+      my $s1 = $util->slot_value_in_double_colon_del_list($line, "s1");
+      my $s2 = $util->slot_value_in_double_colon_del_list($line, "s2");
+      $s1 = $util->dequote_string($s1); # 'can\'t' => can't
+      $s2 = $util->dequote_string($s2);
+      my $cost = $util->slot_value_in_double_colon_del_list($line, "cost");
+      if (($s1 eq "") && ($s2 eq "")) {
+	 print STDERR "Ignoring bad line $line_number in $filename, because both s1 and s2 are empty strings\n";
+         next;
+      }
+      unless ($cost =~ /^\d+(\.\d+)?$/) {
+	 if ($cost eq "") {
+	    print STDERR "Ignoring bad line $line_number in $filename, because of missing cost\n";
+	 } else {
+	    print STDERR "Ignoring bad line $line_number in $filename, because of ill-formed cost $cost\n";
+	 }
+         next;
+      }
+      my $lang_code1_s = $util->slot_value_in_double_colon_del_list($line, "lc1");
+      my $lang_code2_s = $util->slot_value_in_double_colon_del_list($line, "lc2");
+      my @lang_codes_1 = ($lang_code1_s eq "") ? ("") : split(/,\s*/, $lang_code1_s);
+      my @lang_codes_2 = ($lang_code2_s eq "") ? ("") : split(/,\s*/, $lang_code2_s);
+      my $left_context1 = $util->slot_value_in_double_colon_del_list($line, "left1");
+      my $left_context2 = $util->slot_value_in_double_colon_del_list($line, "left2");
+      my $right_context1 = $util->slot_value_in_double_colon_del_list($line, "right1");
+      my $right_context2 = $util->slot_value_in_double_colon_del_list($line, "right2");
+      my $bad_left = $util->slot_value_in_double_colon_del_list($line, "left");
+      if ($bad_left) {
+	 print STDERR "** Warning: slot '::left $bad_left' in line $line_number\n";
+         next;
+      }
+      my $bad_right = $util->slot_value_in_double_colon_del_list($line, "right");
+      if ($bad_right) {
+	 print STDERR "** Warning: slot '::right $bad_right' in line $line_number\n";
+         next;
+      }
+      my $in_lang_codes1 = $util->slot_value_in_double_colon_del_list($line, "in-lc1");
+      my $in_lang_codes2 = $util->slot_value_in_double_colon_del_list($line, "in-lc2");
+      my $out_lang_codes1 = $util->slot_value_in_double_colon_del_list($line, "out-lc1");
+      my $out_lang_codes2 = $util->slot_value_in_double_colon_del_list($line, "out-lc2");
+      if ($left_context1) {
+         if ($left_context1 =~ /^\/.*\/$/) {
+            $left_context1 =~ s/^\///;
+            $left_context1 =~ s/\/$//;
+         } else {
+            print STDERR "Ignoring unrecognized non-regular-express ::left1 $left_context1 in $line_number of $filename\n";
+            $left_context1 = "";
+         }
+      }
+      if ($left_context2) {
+         if ($left_context2 =~ /^\/.*\/$/) {
+            $left_context2 =~ s/^\///;
+            $left_context2 =~ s/\/$//;
+         } else {
+            $left_context2 = "";
+            print STDERR "Ignoring unrecognized non-regular-express ::left2 $left_context2 in $line_number of $filename\n";
+         }
+      }
+      if ($right_context1) {
+         unless ($right_context1 =~ /^(\[[^\[\]]*\])+$/) {
+	    $right_context1 = "";
+	    print STDERR "Ignoring unrecognized right-context ::right1 $right_context1 in $line_number of $filename\n";
+	 }
+      }
+      if ($right_context2) {
+         unless ($right_context2 =~ /^(\[[^\[\]]*\])+$/) {
+	    $right_context2 = "";
+	    print STDERR "Ignoring unrecognized right-context ::right2 $right_context2 in $line_number of $filename\n";
+	 }
+      }
+      foreach $lang_code1 (@lang_codes_1) {
+         foreach $lang_code2 (@lang_codes_2) {
+            $n_cost_rules++;
+            my $cost_rule_id = $n_cost_rules;
+            $ht{COST}->{$lang_code1}->{$lang_code2}->{$s1}->{$s2}->{$cost_rule_id} = $cost;
+            $ht{RULE_STRING}->{$lang_code1}->{$s1} = 1;
+            $ht{RULE_STRING}->{$lang_code2}->{$s2} = 1;
+            $ht{LEFT1}->{$cost_rule_id} = $left_context1;
+            $ht{LEFT2}->{$cost_rule_id} = $left_context2;
+            $ht{RIGHT1}->{$cost_rule_id} = $right_context1;
+            $ht{RIGHT2}->{$cost_rule_id} = $right_context2;
+            $ht{INLC1}->{$cost_rule_id} = $in_lang_codes1;
+            $ht{INLC2}->{$cost_rule_id} = $in_lang_codes2;
+            $ht{OUTLC1}->{$cost_rule_id} = $out_lang_codes1;
+            $ht{OUTLC2}->{$cost_rule_id} = $out_lang_codes2;
+            unless (($s1 eq $s2)
+	         && ($lang_code1 eq $lang_code2)
+	         && ($left_context1 eq $left_context2)
+	         && ($right_context1 eq $right_context2)
+	         && ($in_lang_codes1 eq $in_lang_codes2)
+	         && ($out_lang_codes1 eq $out_lang_codes2)) {
+               $n_cost_rules++;
+               $cost_rule_id = $n_cost_rules;
+               $ht{COST}->{$lang_code2}->{$lang_code1}->{$s2}->{$s1}->{$cost_rule_id} = $cost;
+               $ht{LEFT1}->{$cost_rule_id} = $left_context2;
+               $ht{LEFT2}->{$cost_rule_id} = $left_context1;
+               $ht{RIGHT1}->{$cost_rule_id} = $right_context2;
+               $ht{RIGHT2}->{$cost_rule_id} = $right_context1;
+               $ht{INLC1}->{$cost_rule_id} = $in_lang_codes2;
+               $ht{INLC2}->{$cost_rule_id} = $in_lang_codes1;
+               $ht{OUTLC1}->{$cost_rule_id} = $out_lang_codes2;
+               $ht{OUTLC2}->{$cost_rule_id} = $out_lang_codes1;
+	       # print STDERR "  Flip rule in line $line: $line\n";
+            }
+            $this->rule_string_expansion(*ht, $s1, $lang_code1);
+            $this->rule_string_expansion(*ht, $s2, $lang_code2);
+	 }
+      }
+   }
+   close(IN);
+   print STDERR "Read in $n_cost_rules rules from $line_number lines in $filename\n" if $verbose;
+}
+
+sub romanized_string_to_simple_chart {
+   local($this, $s, *chart_ht) = @_;
+
+   my @characters = $utf8->split_into_utf8_characters($s, "return only chars, return trailing whitespaces", *dummy_ht);
+   $chart_ht{N_CHARS} = $#characters + 1;
+   $chart_ht{N_NODES} = 0;
+   foreach $i ((0 .. $#characters)) {
+      $romanizer->add_node($characters[$i], $i, ($i+1), *chart_ht, "", "");
+   }
+}
+
+sub linearize_chart_points {
+   local($this, *chart_ht, $chart_id, *sd_ht, $verbose) = @_;
+
+   $verbose = 0 unless defined($verbose);
+   print STDERR "Linearize $chart_id\n" if $verbose;
+   my $current_chart_pos = 0;
+   my $current_linear_chart_pos = 0;
+   $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos} = $current_linear_chart_pos;
+   $sd_ht{LINPOS2POS}->{$chart_id}->{$current_linear_chart_pos} = $current_chart_pos;
+   print STDERR "  LINPOS2POS.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos\n" if $verbose;
+   my @end_chart_positions = keys %{$chart_ht{NODES_ENDING_AT}};
+   my $end_chart_pos = (@end_chart_positions) ? max(@end_chart_positions) : 0;
+   $sd_ht{MAXPOS}->{$chart_id} = $end_chart_pos;
+   print STDERR "  Chart span: $current_chart_pos-$end_chart_pos\n" if $verbose;
+   while ($current_chart_pos < $end_chart_pos) {
+      my @node_ids = keys %{$chart_ht{NODES_STARTING_AT}->{$current_chart_pos}};
+      foreach $node_id (@node_ids) {
+	 my $roman_s = $chart_ht{NODE_ROMAN}->{$node_id};
+         my @roman_chars = $utf8->split_into_utf8_characters($roman_s, "return only chars, return trailing whitespaces", *dummy_ht);
+         print STDERR "  $current_chart_pos/$current_linear_chart_pos node: $node_id $roman_s (@roman_chars)\n" if $verbose;
+	 if ($#roman_chars >= 1) {
+	    foreach $i ((1 .. $#roman_chars)) {
+	       $current_linear_chart_pos++;
+	       $sd_ht{SPLITPOS2LINPOS}->{$chart_id}->{$current_chart_pos}->{$node_id}->{$i} = $current_linear_chart_pos;
+	       $sd_ht{LINPOS2SPLITPOS}->{$chart_id}->{$current_linear_chart_pos}->{$current_chart_pos}->{$node_id}->{$i} = 1;
+	       print STDERR "  LINPOS2SPLITPOS.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos NODE: $node_id I: $i\n" if $verbose;
+	    }
+	 }
+      }
+      $current_chart_pos++;
+      if ($util->member($current_chart_pos, @end_chart_positions)) {
+         $current_linear_chart_pos++;
+         $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos} = $current_linear_chart_pos;
+         $sd_ht{LINPOS2POS}->{$chart_id}->{$current_linear_chart_pos} = $current_chart_pos;
+         print STDERR "  LINPOS2POS.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos\n" if $verbose;
+      }
+   }
+   $current_chart_pos = 0;
+   while ($current_chart_pos <= $end_chart_pos) {
+      my $current_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos};
+      $current_linear_chart_pos = "?" unless defined($current_linear_chart_pos);
+      my @node_ids = keys %{$chart_ht{NODES_STARTING_AT}->{$current_chart_pos}};
+      # print STDERR "  LINROM.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos NODES: @node_ids\n" if $verbose;
+      foreach $node_id (@node_ids) {
+	 my $end_pos = $chart_ht{NODE_END}->{$node_id};
+	 my $end_linpos = $sd_ht{POS2LINPOS}->{$chart_id}->{$end_pos};
+	 my $roman_s = $chart_ht{NODE_ROMAN}->{$node_id};
+         my @roman_chars = $utf8->split_into_utf8_characters($roman_s, "return only chars, return trailing whitespaces", *dummy_ht);
+         print STDERR "  LINROM.$chart_id LIN: $current_linear_chart_pos POS: $current_chart_pos NODE: $node_id CHARS: @roman_chars\n" if $verbose;
+	 if (@roman_chars) {
+            foreach $i ((0 .. $#roman_chars)) {
+	       my $from_linear_chart_pos 
+		  = (($i == 0) 
+		     ? $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos}
+	             : $sd_ht{SPLITPOS2LINPOS}->{$chart_id}->{$current_chart_pos}->{$node_id}->{$i});
+	       print STDERR "  FROM.$chart_id I: $i POS: $current_chart_pos NODE: $node_id FROM: $from_linear_chart_pos\n" if $verbose;
+	       my $to_linear_chart_pos
+		  = (($i == $#roman_chars)
+		     ? $end_linpos
+		     : $sd_ht{SPLITPOS2LINPOS}->{$chart_id}->{$current_chart_pos}->{$node_id}->{($i+1)});
+	       print STDERR "  TO.$chart_id I: $i POS: $current_chart_pos NODE: $node_id FROM: $to_linear_chart_pos\n" if $verbose;
+	       my $roman_char = $roman_chars[$i];
+	       $sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}->{$to_linear_chart_pos}->{$roman_char} = 1;
+	    }
+	 } else {
+	    my $from_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{$current_chart_pos};
+	    my $to_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{($current_chart_pos+1)};
+	    # HHERE check this out
+	    my $i = 1;
+	    while (! (defined($to_linear_chart_pos))) {
+	       $i++;
+	       $to_linear_chart_pos = $sd_ht{POS2LINPOS}->{$chart_id}->{($current_chart_pos+$i)};
+	    }
+	    if (defined($from_linear_chart_pos) && defined($to_linear_chart_pos)) {
+	       $sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}->{$to_linear_chart_pos}->{""} = 1
+	    } else {
+	       print STDERR "  UNDEF.$chart_id from: " 
+	                  . ((defined($from_linear_chart_pos)) ? $from_linear_chart_pos : "?")
+			  . " to: "
+	                  . ((defined($to_linear_chart_pos))   ? $to_linear_chart_pos   : "?")
+			  . "\n";
+	    }
+	 }
+      }
+      $current_chart_pos++;
+   }
+   $sd_ht{MAXLINPOS}->{$chart_id} = $sd_ht{POS2LINPOS}->{$chart_id}->{$end_chart_pos};
+}
+
+sub expand_lin_ij_roman {
+   local($this, *sd_ht, $chart_id, $lang_code, *ht) = @_;
+
+   foreach $start (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}}) {
+      foreach $end (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start}}) {
+	 foreach $roman (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start}->{$end}}) {
+	    if ($ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$roman}
+	     || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$roman}) {
+	       $this->expand_lin_ij_roman_rec(*sd_ht, $chart_id, $start, $end, $roman, $lang_code, *ht);
+	    }
+	 }
+      }
+   }
+}
+
+sub expand_lin_ij_roman_rec {
+   local($this, *sd_ht, $chart_id, $start, $end, $roman, $lang_code, *ht) = @_;
+
+   # print STDERR "  expand_lin_ij_roman_rec.$chart_id $start-$end $lang_code $roman\n";
+   return unless $ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$roman}
+              || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$roman};
+   foreach $new_end (keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$end}}) {
+      foreach $next_roman (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$end}->{$new_end}}) {
+         my $exp_roman = join("", $roman, $next_roman);
+         if ($ht{RULE_STRING}->{$lang_code}->{$exp_roman}
+          || $ht{RULE_STRING}->{""}->{$exp_roman}) {
+            $sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start}->{$new_end}->{$exp_roman} = 1;
+	    # print STDERR "  Expansion ($start-$new_end) $exp_roman\n";
+         }
+         if ($ht{RULE_STRING_HAS_EXPANSION}->{$lang_code}->{$exp_roman}
+          || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$exp_roman}) {
+	    $this->expand_lin_ij_roman_rec(*sd_ht, $chart_id, $start, $new_end, $exp_roman, $lang_code, *ht);
+         }
+      }
+   }
+}
+
+sub trace_string_distance {
+   local($this, *sd_ht, $chart1_id, $chart2_id, $control, $line_number, $cost) = @_;
+
+   my $chart_comb_id = join("/", $chart1_id, $chart2_id);
+   return "mismatch" if $sd_ht{MISMATCH}->{$chart_comb_id};
+   my $chart1_end = $sd_ht{MAXLINPOS}->{$chart1_id};
+   my $chart2_end = $sd_ht{MAXLINPOS}->{$chart2_id};
+   my $verbose = ($control =~ /verbose/);
+   my $chunks_p = ($control =~ /chunks/);
+   my @traces = ();
+   my @s1_s = ();
+   my @s2_s = ();
+   my @e1_s = ();
+   my @e2_s = ();
+   my @r1_s = ();
+   my @r2_s = ();
+   my @ic_s = ();
+
+   # print STDERR "trace_string_distance $chart1_id $chart2_id $line_number\n";
+   while ($chart1_end || $chart2_end) {
+      my $incr_cost = $sd_ht{INCR_COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+      my $prec_i = $sd_ht{PREC_I}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+      my $prec_j = $sd_ht{PREC_J}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+      if ($incr_cost || $verbose || $chunks_p) {
+         my $roman1 = $sd_ht{ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+         my $roman2 = $sd_ht{ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+         if ($verbose) {
+	    push(@traces, "$prec_i-$chart1_end/$prec_j-$chart2_end:$roman1/$roman2:$incr_cost");
+	 } else {
+	    if (defined($roman1)) {
+	       push(@traces, "$roman1/$roman2:$incr_cost");
+	    } else {
+	       $print_prec_i = (defined($prec_i)) ? $prec_i : "?";
+	       $print_prec_j = (defined($prec_j)) ? $prec_j : "?";
+	       print STDERR "  $prec_i-$chart1_end, $prec_j-$chart2_end\n";
+	    }
+	 }
+         if ($chunks_p) {
+            push(@s1_s, $prec_i);
+            push(@s2_s, $prec_j);
+            push(@e1_s, $chart1_end);
+            push(@e2_s, $chart2_end);
+            push(@r1_s, $roman1);
+            push(@r2_s, $roman2);
+            push(@ic_s, $incr_cost);
+         }
+      }
+      $chart1_end = $prec_i;
+      $chart2_end = $prec_j;
+   }
+   if ($chunks_p) {
+      my $r1 = "";
+      my $r2 = "";
+      my $tc = 0;
+      my $in_chunk = 0;
+      foreach $i ((0 .. $#ic_s)) {
+	 if ($ic_s[$i]) {
+	    $r1 = $r1_s[$i] . $r1;
+	    $r2 = $r2_s[$i] . $r2;
+	    $tc += $ic_s[$i];
+	    $in_chunk = 1;
+	 } elsif ($in_chunk) {
+	    $chunk = "$r1/$r2/$tc";
+	    $chunk .= "*" if $cost > 5;
+	    $sd_ht{N_COST_CHUNK}->{$chunk} = ($sd_ht{N_COST_CHUNK}->{$chunk} || 0) + 1;
+	    $sd_ht{EX_COST_CHUNK}->{$chunk}->{$line_number} = 1;
+            $r1 = "";
+            $r2 = "";
+            $tc = 0;
+	    $in_chunk = 0;
+	 }
+      }
+      if ($in_chunk) {
+	 $chunk = "$r1/$r2/$tc";
+	 $chunk .= "*" if $cost > 5;
+	 $sd_ht{N_COST_CHUNK}->{$chunk} = ($sd_ht{N_COST_CHUNK}->{$chunk} || 0) + 1;
+	 $sd_ht{EX_COST_CHUNK}->{$chunk}->{$line_number} = 1;
+      }
+   } else {
+      return join(" ", reverse @traces);
+   }
+}
+
+sub right_context_match {
+   local($this, $right_context_rule, *sd_ht, $chart_id, $start_pos) = @_;
+   
+   return 1 if $right_context_rule eq "";
+   if (($right_context_item, $right_context_rest) = ($right_context_rule =~ /^\[([^\[\]]*)\]*(.*)$/)) {
+      my $guarded_right_context_item = $right_context_item;
+      $guarded_right_context_item =~ s/\$/\\\$/g;
+      my @end_positions = keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start_pos}};
+      return 1 if ($#end_positions == -1)
+               && (($right_context_item eq "") 
+	        || ($right_context_item =~ /\$/));
+      foreach $end_pos (@end_positions) {
+	 my @romans = keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$start_pos}->{$end_pos}};
+         foreach $roman (@romans) {
+	    if ($roman =~ /^[$guarded_right_context_item]/) {
+	       return $this->right_context_match($right_context_rest, *sd_ht, $chart_id, $end_pos);
+	    }
+	 }
+      }
+   }
+   return 0;
+}
+
+sub string_distance {
+   local($this, *sd_ht, $chart1_id, $chart2_id, $lang_code1, $lang_code2, *ht, $control) = @_;
+
+   my $verbose = ($control =~ /verbose/i);
+   my $chart_comb_id = join("/", $chart1_id, $chart2_id);
+
+   my $chart1_end_pos = $sd_ht{MAXLINPOS}->{$chart1_id};
+   my $chart2_end_pos = $sd_ht{MAXLINPOS}->{$chart2_id};
+   print STDERR "string_distance.$chart_comb_id $chart1_end_pos/$chart2_end_pos\n" if $verbose;
+   $sd_ht{COST_IJ}->{$chart_comb_id}->{0}->{0} = 0;
+   $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{0}->{0} = "";
+   $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{0}->{0} = "";
+   # HHERE
+   foreach $chart1_start ((0 .. $chart1_end_pos)) {
+      # print STDERR "  C1 $chart1_start- ($chart1_start .. $chart1_end_pos)\n";
+      my $prev_further_expansion_possible = 0;
+      my @chart1_ends = sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart1_id}->{$chart1_start}};
+      my $max_chart1_ends = (@chart1_ends) ? $chart1_ends[$#chart1_ends] : -1;
+      foreach $chart1_end (($chart1_start .. $chart1_end_pos)) {
+	 my $further_expansion_possible = ($chart1_start == $chart1_end)
+				       || defined($sd_ht{LINPOS2SPLITPOS}->{$chart1_id}->{$chart1_start})
+				       || ($chart1_end < $max_chart1_ends);
+	 my @romans1 = (($chart1_start == $chart1_end)
+			? ("")
+			: (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart1_id}->{$chart1_start}->{$chart1_end}}));
+	 if ($#romans1 == -1) {
+	    $further_expansion_possible = 1 if $prev_further_expansion_possible;
+	 } else {
+	    $prev_further_expansion_possible = 0;
+	 }
+         # print STDERR "  C1 $chart1_start-$chart1_end romans1: @romans1 {$further_expansion_possible} *l*\n";
+	 foreach $roman1 (@romans1) {
+            # print STDERR "  C1 $chart1_start-$chart1_end $roman1 {$further_expansion_possible} *?*\n";
+	    next unless $ht{RULE_STRING}->{$lang_code1}->{$roman1}
+		     || $ht{RULE_STRING}->{""}->{$roman1};
+            # print STDERR "  C1 $chart1_start-$chart1_end $roman1 {$further_expansion_possible} ***\n";
+	    foreach $lang_code1o (($lang_code1, "")) {
+	       foreach $lang_code2o (($lang_code2, "")) {
+		  my @chart2_starts = (sort { $a <=> $b } keys %{$sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_start}});
+	          foreach $chart2_start (@chart2_starts) {
+                     # print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start- (@chart2_starts)\n";
+		     foreach $chart2_end (($chart2_start .. $chart2_end_pos)) {
+                        print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start-$chart2_end\n";
+			my @romans2 = (($chart2_start == $chart2_end)
+				      ? ("")
+				      : (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart2_id}->{$chart2_start}->{$chart2_end}}));
+			foreach $roman2 (@romans2) {
+			   if ($roman1 eq $roman2) {
+                              print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start-$chart2_end $roman2 (IDENTITY)\n";
+			      my $cost = 0;
+			      my $preceding_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+			      my $combined_cost = $preceding_cost + $cost;
+			      my $old_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+			      if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+			         $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $combined_cost;
+			         push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+			         $sd_ht{PREC_I}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart1_start;
+			         $sd_ht{PREC_J}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart2_start;
+			         $sd_ht{ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman1;
+			         $sd_ht{ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman2;
+			         $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			            = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman1;
+			         $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			            = $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman2;
+				 $comb_left_roman1 = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+			         $sd_ht{INCR_COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $cost;
+			         $sd_ht{COST_RULE}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = "IDENTITY";
+			         print STDERR "  New cost $chart1_end/$chart2_end: $combined_cost (+$cost from $chart1_start/$chart2_start $roman1/$roman2)\n" if $verbose;
+			      }
+			   } else {
+	                      next unless $ht{RULE_STRING}->{$lang_code2o}->{$roman2};
+                              print STDERR "  C1 $chart1_start-$chart1_end $roman1 C2 $chart2_start-$chart2_end $roman2\n";
+			      next unless defined($ht{COST}->{$lang_code1o}->{$lang_code2o}->{$roman1}->{$roman2});
+		              my @cost_rule_ids = keys %{$ht{COST}->{$lang_code1o}->{$lang_code2o}->{$roman1}->{$roman2}};
+		              foreach $cost_rule_id (@cost_rule_ids) {
+			         ## check whether any context requirements are satisfied
+			         # left context rules are regular expressions
+                                 my $left_context_rule1 = $ht{LEFT1}->{$cost_rule_id};
+			         if ($left_context_rule1) {
+				    my $comb_left_roman1 = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+				    if (defined($comb_left_roman1)) {
+				       next unless $comb_left_roman1 =~ /$left_context_rule1/;
+			            } else {
+				       print STDERR "  No comb_left_roman1 value for $chart_comb_id $chart1_start,$chart2_start\n";
+				    }
+			         }
+                                 my $left_context_rule2 = $ht{LEFT2}->{$cost_rule_id};
+			         if ($left_context_rule2) {
+				    my $comb_left_roman2 = $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+				    if (defined($comb_left_roman2)) {
+				       next unless $comb_left_roman2 =~ /$left_context_rule2/;
+				    } else {
+				       print STDERR "  No comb_left_roman2 value for $chart_comb_id $chart1_start,$chart2_start\n";
+				    }
+			         }
+                                 my $right_context_rule1 = $ht{RIGHT1}->{$cost_rule_id};
+			         if ($right_context_rule1) {
+			            my $match_p = $this->right_context_match($right_context_rule1, *sd_ht, $chart1_id, $chart1_end);
+				    # print STDERR "  Match?($right_context_rule1, 1, $chart1_end) = $match_p\n";
+			            next unless $match_p;
+			         }
+                                 my $right_context_rule2 = $ht{RIGHT2}->{$cost_rule_id};
+			         if ($right_context_rule2) {
+			            my $match_p = $this->right_context_match($right_context_rule2, *sd_ht, $chart2_id, $chart2_end);
+				    # print STDERR "  Match?($right_context_rule2, 2, $chart2_end) = $match_p\n";
+			            next unless $match_p;
+			         }
+			         my $cost = $ht{COST}->{$lang_code1o}->{$lang_code2o}->{$roman1}->{$roman2}->{$cost_rule_id};
+			         my $preceding_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_start}->{$chart2_start};
+			         my $combined_cost = $preceding_cost + $cost;
+			         my $old_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+			         if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+			            $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $combined_cost;
+				    push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+			            $sd_ht{PREC_I}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart1_start;
+			            $sd_ht{PREC_J}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $chart2_start;
+			            $sd_ht{ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman1;
+			            $sd_ht{ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $roman2;
+			            $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			               = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman1;
+			            $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_end}->{$chart2_end}
+			               = $sd_ht{COMB_LEFT_ROMAN2}->{$chart_comb_id}->{$chart1_start}->{$chart2_start} . $roman2;
+				    $comb_left_roman1 = $sd_ht{COMB_LEFT_ROMAN1}->{$chart_comb_id}->{$chart1_end}->{$chart2_end};
+				    # print STDERR "  Comb-left-roman1($chart_comb_id,$chart1_end,$chart2_end) = $comb_left_roman1\n";
+			            $sd_ht{INCR_COST_IJ}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $cost;
+			            $sd_ht{COST_RULE}->{$chart_comb_id}->{$chart1_end}->{$chart2_end} = $cost_rule_id;
+			            print STDERR "  New cost $chart1_end/$chart2_end: $combined_cost (+$cost from $chart1_start/$chart2_start $roman1/$roman2)\n" if $verbose;
+			         }
+			      }
+			   }
+			}
+		     }
+		  }
+	       }
+	    }
+	    $further_expansion_possible = 1
+	       if $ht{RULE_STRING_HAS_EXPANSION}->{$lang_code1}->{$roman1}
+	       || $ht{RULE_STRING_HAS_EXPANSION}->{""}->{$roman1};
+	    # print STDERR "  further_expansion_possible: $further_expansion_possible (lc: $lang_code1 r1: $roman1) ***\n";
+	 }
+         # print STDERR "  last C1 $chart1_start-$chart1_end (@romans1)\n" unless $further_expansion_possible;
+	 last unless $further_expansion_possible;
+         $prev_further_expansion_possible = 1 if $further_expansion_possible;
+      }
+   }
+   my $total_cost = $sd_ht{COST_IJ}->{$chart_comb_id}->{$chart1_end_pos}->{$chart2_end_pos};
+   unless (defined($total_cost)) {
+      $total_cost = 99.9999;
+      $sd_ht{MISMATCH}->{$chart_comb_id} = 1;
+   }
+   return $total_cost;
+}
+
+sub print_sd_ht {
+   local($this, *sd_ht, $chart1_id, $chart2_id, *OUT) = @_;
+
+   print OUT "string-distance chart:\n";
+   foreach $chart_id (($chart1_id, $chart2_id)) {
+      print OUT "SD chart $chart_id:\n";
+      foreach $from_linear_chart_pos (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}}) {
+	 foreach $to_linear_chart_pos (sort { $a <=> $b } keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}}) {
+	    foreach $roman_char (sort keys %{$sd_ht{LIN_IJ_ROMAN}->{$chart_id}->{$from_linear_chart_pos}->{$to_linear_chart_pos}}) {
+	       print OUT "  Lnode($from_linear_chart_pos-$to_linear_chart_pos): $roman_char\n";
+	    }
+	 }
+      }
+   }
+}
+
+sub print_chart_ht {
+   local($this, *chart_ht, *OUT) = @_;
+
+   print OUT "uroman chart:\n";
+   foreach $start (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AT}}) {
+      foreach $end (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}}) {
+	 foreach $node_id (keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}}) {
+	    $roman_s = $chart_ht{NODE_ROMAN}->{$node_id};
+	    print OUT "  Node $node_id ($start-$end): $roman_s\n";
+	 }
+      }
+   }
+}
+
+sub normalize_string {
+   local($this, $s) = @_;
+
+#  $s =~ s/(\xE2\x80\x8C)//g; # delete zero width non-joiner
+   $s =~ s/(\xE2\x80[\x93-\x94])/-/g; # en-dash, em-dash
+   $s =~ s/([\x00-\x7F\xC0-\xFE][\x80-\xBF]*)\1+/$1$1/g; # shorten 3 or more occurrences of same character in a row to 2
+   $s =~ s/[ \t]+/ /g;
+
+   return $s;
+}
+
+my $string_distance_chart_id = 0;
+sub string_distance_by_chart {
+   local($this, $s1, $s2, $lang_code1, $lang_code2, *ht, *pinyin_ht, $control) = @_;
+
+   $control = "" unless defined($control);
+   %sd_ht = ();
+
+   $s1 = $this->normalize_string($s1);
+   my $lc_s1 = $utf8->extended_lower_case($s1);
+   $string_distance_chart_id++;
+   my $chart1_id = $string_distance_chart_id;
+   *chart_ht = $romanizer->romanize($lc_s1, $lang_code1, "", *ht, *pinyin_ht, 0, "return chart", $chart1_id);
+   $this->linearize_chart_points(*chart_ht, $chart1_id, *sd_ht);
+   $this->expand_lin_ij_roman(*sd_ht, $chart1_id, $lang_code1, *ht);
+
+   $s2 = $this->normalize_string($s2);
+   my $lc_s2 = $utf8->extended_lower_case($s2);
+   $string_distance_chart_id++;
+   my $chart2_id = $string_distance_chart_id;
+   *chart_ht = $romanizer->romanize($lc_s2, $lang_code2, "", *ht, *pinyin_ht, 0, "return chart", $chart2_id);
+   $this->linearize_chart_points(*chart_ht, $chart2_id, *sd_ht);
+   $this->expand_lin_ij_roman(*sd_ht, $chart2_id, $lang_code2, *ht);
+
+   my $cost = $this->string_distance(*sd_ht, $chart1_id, $chart2_id, $lang_code1, $lang_code2, *ht, $control);
+   return $cost;
+}
+
+my $n_quick_romanized_string_distance = 0;
+sub quick_romanized_string_distance_by_chart {
+   local($this, $s1, $s2, *ht, $control, $lang_code1, $lang_code2) = @_;
+
+   # my $verbose = ($s1 eq "apit") && ($s2 eq "apet");
+   # print STDERR "Start quick_romanized_string_distance_by_chart\n";
+   $s1 = lc $s1;
+   $s2 = lc $s2;
+   $control = "" unless defined($control);
+   $lang_code1 = "" unless defined($lang_code1);
+   $lang_code2 = "" unless defined($lang_code2);
+   my $cache_p = ($control =~ /cache/);
+   my $total_cost;
+   if ($cache_p) {
+      $total_cost = $ht{CACHED_QRSD}->{$s1}->{$s2};
+      if (defined($total_cost)) {
+         return $total_cost;
+      }
+   }
+   my @lang_codes1 = ($lang_code1 eq "") ? ("") : ($lang_code1, "");
+   my @lang_codes2 = ($lang_code2 eq "") ? ("") : ($lang_code2, "");
+   my $chart1_end_pos = length($s1);
+   my $chart2_end_pos = length($s2);
+   my %sd_ht = ();
+   $sd_ht{COST_IJ}->{0}->{0} = 0;
+   foreach $chart1_start ((0 .. $chart1_end_pos)) {
+      foreach $chart1_end (($chart1_start .. $chart1_end_pos)) {
+	 my $substr1 = substr($s1, $chart1_start, ($chart1_end-$chart1_start));
+	 foreach $lang_code1o (@lang_codes1) {
+	    foreach $lang_code2o (@lang_codes2) {
+	       # next unless defined($ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1});
+            }
+	 }
+	 my @chart2_starts = (sort { $a <=> $b } keys %{$sd_ht{COST_IJ}->{$chart1_start}});
+	 foreach $chart2_start (@chart2_starts) {
+	    foreach $chart2_end (($chart2_start .. $chart2_end_pos)) {
+	       my $substr2 = substr($s2, $chart2_start, ($chart2_end-$chart2_start));
+	       foreach $lang_code1o (@lang_codes1) {
+	          foreach $lang_code2o (@lang_codes2) {
+		     if ($substr1 eq $substr2) {
+			my $cost = 0;
+	                my $preceding_cost = $sd_ht{COST_IJ}->{$chart1_start}->{$chart2_start};
+			if (defined($preceding_cost)) {
+		           my $combined_cost = $preceding_cost + $cost;
+	                   my $old_cost = $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end};
+	                   if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+		              $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end} = $combined_cost;
+		              push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+		           }
+			}
+		     } else {
+	                next unless defined($ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1}->{$substr2});
+	                my @cost_rule_ids = keys %{$ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1}->{$substr2}};
+	                my $best_cost = 99.99;
+	                foreach $cost_rule_id (@cost_rule_ids) {
+		           my $cost = $ht{COST}->{$lang_code1o}->{$lang_code2o}->{$substr1}->{$substr2}->{$cost_rule_id};
+                           my $left_context_rule1 = $ht{LEFT1}->{$cost_rule_id};
+			   next if $left_context_rule1
+			        && (! (substr($s1, 0, $chart1_start) =~ /$left_context_rule1/));
+                           my $left_context_rule2 = $ht{LEFT2}->{$cost_rule_id};
+			   next if $left_context_rule2
+			        && (! (substr($s2, 0, $chart2_start) =~ /$left_context_rule2/));
+                           my $right_context_rule1 = $ht{RIGHT1}->{$cost_rule_id};
+			   my $right_context1 = substr($s1, $chart1_end);
+			   next if $right_context_rule1
+			        && (! (($right_context1 =~ /^$right_context_rule1/)
+			            || (($right_context_rule1 =~ /^\[[^\[\]]*\$/)
+			             && ($right_context1 eq ""))));
+                           my $right_context_rule2 = $ht{RIGHT2}->{$cost_rule_id};
+			   my $right_context2 = substr($s2, $chart2_end);
+			   next if $right_context_rule2
+			        && (! (($right_context2 =~ /^$right_context_rule2/)
+			            || (($right_context_rule2 =~ /^\[[^\[\]]*\$/)
+			             && ($right_context2 eq ""))));
+		           $best_cost = $cost if $cost < $best_cost;
+	                   my $preceding_cost = $sd_ht{COST_IJ}->{$chart1_start}->{$chart2_start};
+		           my $combined_cost = $preceding_cost + $cost;
+	                   my $old_cost = $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end};
+	                   if ((! defined($old_cost)) || ($combined_cost < $old_cost)) {
+		              $sd_ht{COST_IJ}->{$chart1_end}->{$chart2_end} = $combined_cost;
+		              push(@chart2_starts, $chart2_end) unless $util->member($chart2_end, @chart2_starts);
+		           }
+		        }
+		     }
+		  }
+	       }
+	    }
+	 }
+      }
+   }
+   $total_cost = $sd_ht{COST_IJ}->{$chart1_end_pos}->{$chart2_end_pos};
+   $total_cost = 99.99 unless defined($total_cost);
+   $ht{CACHED_QRSD}->{$s1}->{$s2} = $total_cost if $cache_p;
+   $n_quick_romanized_string_distance++;
+   return $total_cost;
+}
+
+sub get_n_quick_romanized_string_distance {
+   return $n_quick_romanized_string_distance;
+}
+
+1;
+

uroman/tarballs/uroman-v1.0.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:912655beef069e5abb43c8fc4c3c4428fd0af6f4a1697accc98277933d3e1ee5
+size 440252

uroman/tarballs/uroman-v1.1.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df990f6096a10e093ac5f28c2b86d5ef9e9098ef7472855843f9a841bb3b963d
+size 507234

uroman/tarballs/uroman-v1.2.4.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77d707f3c17d5c45869b80fe71caee6023d1d9949ccffb446626f374605a25e2
+size 503690

uroman/tarballs/uroman-v1.2.5.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2e9044afff8b4483f43a99b1fb1279889336760d76245ee93f300e660a46660
+size 575581

uroman/tarballs/uroman-v1.2.6.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02f6f73b067b972a8f7d408da2f9b22741629af67f55b2ea768d11710fbf40a4
+size 567522

uroman/tarballs/uroman-v1.2.7.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbb51506ed3ea6dcb902c824e62bea39b3741f6526564ba05d6e0083d8d876e5
+size 566800

uroman/tarballs/uroman-v1.2.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c69e56d9c5eea9416ae00ca4dd859a1ef5129c1867778b66ad2f811f0fd33c9
+size 494625

uroman/test/multi-script.txt

@@ -0,0 +1,32 @@
+::lcode deu Grüße aus Bordeaux
+::lcode tur İstanbul, Türkiye'de yer alan şehir ve ülkenin 81 ilinden biri.
+::lcode eng ⠠⠺⠑⠀⠓⠕⠇⠙⠀⠘⠮⠀⠞⠗⠥⠹⠎⠀⠞⠕⠀⠆⠀⠎⠑⠇⠋⠤⠑⠧⠊⠙⠢⠞⠂⠀⠞⠀⠁⠇⠇⠀⠍⠑⠝⠀⠜⠑⠀⠉⠗⠂⠞⠫⠀⠑⠟⠥⠁⠇⠂⠀⠞⠀⠮⠽⠀⠜⠑⠀⠑⠝⠙⠪⠫⠀⠃⠽⠀⠸⠮⠀⠠⠉⠗⠑⠁⠞⠕⠗⠀⠾⠀⠉⠻⠞⠁⠔⠀⠥⠝⠁⠇⠊⠑⠝⠁⠃⠇⠑⠀⠠⠐⠗⠎⠂⠀⠞⠀⠁⠍⠰⠛⠀⠘⠮⠀⠜⠑⠀⠠⠇⠊⠋⠑⠂⠀⠠⠇⠊⠃⠻⠞⠽⠀⠯⠀⠮⠀⠏⠥⠗⠎⠥⠊⠞⠀⠷⠀⠠⠓⠁⠏⠏⠊⠰⠎⠲
+::lcode ell Το Λος Άντζελες (στα ισπανικά Los Angeles = Οι Άγγελοι) ή στην Αμερικανική αργκό L.A., ελ έι) είναι η δεύτερη μεγαλύτερη πόλη των Ηνωμένων Πολιτειών από άποψη πληθυσμού, καθώς και ένα από τα σημαντικότερα οικονομικά, πολιτιστικά επιστημονικά και ψυχαγωγικά κέντρα του κόσμου.
+::lcode rus Герма́ния (нем. Deutschland), официальное название — Федерати́вная Респу́блика Герма́ния (нем. Bundesrepublik Deutschland), ФРГ (нем. BRD) — государство в Западной Европе. Площадь территории — 357 021 км². Численность населения по переписи 2011 года — более 80 миллионов человек. [2][6].
+::lcode ukr Володи́мир Олекса́ндрович Зеле́нський (нар. 25 січня 1978, Кривий Ріг) — український державний діяч, політик, шоумен, актор, комік, режисер, продюсер та сценарист, шостий Президент України з 20 травня 2019 року.
+::lcode srp Сва људска бића рађају се слободна и једнака у достојанству и правима. Она су обдарена разумом и свешћу и треба једни према другима да поступају у духу братства.
+::lcode ara كندا (بالإنجليزية: Canada) هي دولة في أمريكا الشمالية تتألف من 10 مقاطعات وثلاثة أقاليم. تقع في القسم الشمالي من القارة وتمتد من المحيط الأطلسي في الشرق إلى المحيط الهادئ في الغرب وتمتد شمالاً في المحيط المتجمد الشمالي. كندا هي البلد الثاني عالمياً من حيث المساحة الكلية. كما أن حدود كندا المشتركة مع الولايات المتحدة من الجنوب والشمال الغربي هي الأطول في العالم.
+::lcode fas کالیفرنیا (به انگلیسی: California) ایالتی در غرب آمریکا بر کرانهٔ اقیانوس آرام است. مرکز آن ساکرامنتو و شهرهای مهم آن لس‌آنجلس، سن دیگو، سن خوزه و سان‌فرانسیسکو هستند.همچنین این ایالت پر جمعیت ترین ایالت امریکا است.
+::lcode uig ئامېرىكا قوشما شتاتلىرى بولسا شىمالىي ئامېرىكاغا جايلاشقان بىر دۆلەت. ئۇنىڭ پايتەختى بولسا ۋاشىنگتون، ئەڭ چوڭ شەھىرى بولسا نيۇيورك شەھىرى. دۆلەت تىلى بولسا ئېنگلىزتىلى. ھازىرقى زۇڭتۇڭ باراك ئوباما. بۇ دۆلەت ئەسلىدە ئەنگىلىيەنىڭ مۇستەملىكىسى بولۇپ ۋاشىنگىتوننىڭ رەھپەرلىكىدە 1776 يىلى 7 ئاينىڭ 4 كۇنى مۇستەقىل بولغان، يەر مەيدانى 9 مىلىيون 826 مىڭ 630 كۋادىرات كلومېتىر، نوپۇسى 306 مىللىيون 142 مىڭ، بۇلارنىڭ ئاسساسلىق دىنى خرىستىئان دىنى.
+::lcode amh ኢትዮጵያ ከዓለም ሶስቱ ትልቅ የአብርሃም ሀይማኖቶች ጋር ታሪካዊ ግንኙነት አላት።
+::lcode hin कैलिफ़ोर्निया शब्द का पहला अर्थ था जो क्षेत्र जहाँ आज बाहा कैलिफ़ोर्निया प्रायद्वीप, नेवाडा, यूटा और एरिज़ोना, नया मेक्सिको, और वायोमिंग के कई विभाग स्थित हैं।
+::lcode mar लंडन (इंग्लिश: London ) हे इंग्लंडचे व युनायटेड किंग्डमचे राजधानीचे व सर्वात मोठे शहर तसेच युरोपियन संघामधील सर्वात मोठे महान���र क्षेत्र आहे.
+::lcode nep यसको उचाइ समुन्द्र सतहबाट ८,८४८ मीटर (२९,०२८ फीट) छ। यो नेपालको सोलुखुम्बु जिल्लाको खुम्जुङ्ग गा. वि. स. मा पर्छ ।
+::lcode tam தமிழ்நாடு (Tamil Nadu) இந்தியாவின் 29 மாநிலங்களில் ஒன்றாகும். தமிழ்நாடு, தமிழகம் என்றும் பரவலாக அழைக்கப்படுகிறது.
+::lcode mal ഇന്ത്യയുടെ തെക്കുപടിഞ്ഞാറെ അറ്റത്തുള്ള സംസ്ഥാനമാണ് കേരളം.
+::lcode ori ଓଡ଼ିଶା ଭାରତର ପୂର୍ବ ଉପକୂଳରେ ଥିବା ଏକ ପ୍ରଶାସନିକ ରାଜ୍ୟ । ଏହାର ଉତ୍ତର-ପୂର୍ବରେ ପଶ୍ଚିମବଙ୍ଗ, ଉତ୍ତରରେ ଝାଡ଼ଖଣ୍ଡ, ପଶ୍ଚିମ ଓ ଉତ୍ତର-ପଶ୍ଚିମରେ ଛତିଶଗଡ଼, ଦକ୍ଷିଣ ଓ ଦକ୍ଷିଣ-ପଶ୍ଚିମରେ ଆନ୍ଧ୍ରପ୍ରଦେଶ ଅବସ୍ଥିତ । ଏହା ଆୟତନ ହିସାବରେ ନବମ ଓ ଜନସଂଖ୍ୟା ହିସାବରେ ଏଗାରତମ ରାଜ୍ୟ । ଓଡ଼ିଆ ଭାଷା ରାଜ୍ୟର ସରକାରୀ ଭାଷା । ୨୦୦୧ ଜନଗଣନା ଅନୁସାରେ ରାଜ୍ୟର ପ୍ରାୟ ୩୩.୨ ନିୟୁତ ଲୋକ ଓଡ଼ିଆ ଭାଷା ବ୍ୟବହାର କରନ୍ତି । 
+::lcode zho 加拿大在一万四千年前即有原住民在此生活。
+::lcode heb כֹּל עוֹד בַּלֵּבָב פְּנִימָה נֶפֶשׁ יְהוּדִי הוֹמִיָּה וּלְפַאֲתֵי מִזְרָח, קָדִימָה, עַיִן לְצִיּוֹן צוֹפִיָּה, עוֹד לֹא אָבְדָה תִּקְוָתֵנוּ, הַתִּקְוָה בַּת שְׁנוֹת אַלְפַּיִם לִהְיוֹת עַם חָפְשִׁי בְּאַרְצֵנוּ, אֶרֶץ צִיּוֹן וִירוּשָׁלַיִם.
+::lcode yid דווקא איז אן העברעישער זשורנאל וואס באשרייבט די יידיש־שפראכיקע קולטור. עס איז דערשינען געווארן תמוז ה'תשס"ז (יולי 2006).
+::lcode hye Տալնոեի շրջան (ուկր.՝ Тальнівський район), շրջան Ուկրաինայի Չերկասիի մարզում։ Ստեղծվել է 1923 թվականին։ Վարչական կենտրոնը՝ Տալնոե։ Աշխարհագրությունը Շրջանի տարածքի մակերեսը կազմում է 917 կմ²։ Բնակչություն
+::lcode tai มีประเทศอิสระ 2 ประเทศ คือ ซานมารีโนและนครรัฐวาติกัน เป็นดินแดนที่ล้อมรอบไปด้วยพื้นที่ของอิตาลี ในขณะที่เมืองกัมปีโอเนดีตาเลีย เป็นดินแดนส่วนแยกของอิตาลีที่ถูกล้อมรอบด้วยพื้นที่ประเทศสวิตเซอร์แลนด์
+북쪽에는 인도네시아와 동티모르, 파푸아 뉴기니, 북동쪽에는 솔로몬 제도와 바누아투, 누벨칼레도니, 그리고 남동쪽에는 뉴질랜드가 있다.
+ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸು ಇಂದೆನ್ನ ಹೃದಯದಲಿ ನಿತ್ಯವೂ ಅವತರಿಪ ಸತ್ಯಾವತಾರ ಮಣ್ಣಾಗಿ ಮರವಾಗಿ ಮಿಗವಾಗಿ ಕಗವಾಗೀ... ಮಣ್ಣಾಗಿ ಮರವಾಗಿ ಮಿಗವಾಗಿ ಕಗವಾಗಿ ಭವ ಭವದಿ ಭತಿಸಿಹೇ ಭವತಿ ದೂರ ನಿತ್ಯವೂ ಅವತರಿಪ ಸತ್ಯಾವತಾರ || ಬಾ ಇಲ್ಲಿ ||
+ვეპხის ტყაოსანი შოთა რუსთაველი ღმერთსი შემვედრე, ნუთუ კვლა დამხსნას სოფლისა შრომასა, ცეცხლს, წყალსა და მიწასა, ჰაერთა თანა მრომასა; მომცნეს ფრთენი და აღვფრინდე, მივჰხვდე მას ჩემსა ნდომასა, დღისით და ღამით ვჰხედვიდე მზისა ელვათა კრთომაასა.
+᚛ᚐᚅᚋ ᚋᚖᚂᚓᚌᚖᚋᚏᚔᚇ ᚋᚐᚉᚔ ᚍᚓᚉᚒᚋᚓᚅ᚜
+ᛁᚳ᛫ᛗᚨᚷ᛫ᚷᛚᚨᛋ᛫ᛖᚩᛏᚪᚾ᛫ᚩᚾᛞ᛫ᚻᛁᛏ᛫ᚾᛖ᛫ᚻᛖᚪᚱᛗᛁᚪᚧ᛫ᛗᛖ᛬
+𓊪𓏏𓍯𓃭𓐝𓇌𓋴
+チェコスロバキア
+ལྷ་ས་གྲ���ང་ཁྱེར
+ᓵᓕ ᓴᕕᐊᕐᔪᒃ ᐃᒻᒥᓂᒃ ᓂᓪᓕᕈᑎᖃᓲᖑᕗᖅ ᑕᐃᑦᓱᒪᓂᑕᑦᓴᔭᐅᓂᕋᕐᓱᓂ. ᐃᒻᒥᓂᓪᓗᑕᐅᖅ ᓂᓪᓕᕈᑎᖃᓱᖑᒻᒥᓱᓂ ᐅᓪᓗᒥᓂᑕᑦᓴᔭᐅᓂᕋᕐᓱᓂ.
+ⴰⵎⴰⴳⵔⴰⴷ 1 ⴰⵔ ⴷ ⵜⵜⵍⴰⵍⴰⵏ ⵎⵉⴷⴷⵏ ⴳⴰⵏ ⵉⵍⴻⵍⵍⵉⵜⵏ ⵎⴳⴰⴷⴷⴰⵏ ⵖ ⵡⴰⴷⴷⵓⵔ ⴷ ⵉⵣⵔⴼⴰⵏ, ⵢⵉⵍⵉ ⴰⴽⵯ ⴷⴰⵔⵙⵏ ⵓⵏⵍⵍⵉ ⴷ ⵓⴼⵔⴰⴽ, ⵉⵍⵍⴰ ⴼⵍⵍⴰ ⵙⵏ ⴰⴷ ⵜⵜⵎⵢⴰⵡⴰⵙⵏ ⵏⴳⵔⴰⵜⵙⵏ ⵙ ⵜⴰⴳⵎⴰⵜ.

uroman/test/multi-script.uroman-ref.txt

@@ -0,0 +1,32 @@
+::lcode deu Gruesse aus Bordeaux
+::lcode tur Istanbul, Tuerkiye'de yer alan shehir ve uelkenin 81 ilinden biri.
+::lcode eng We hold ⠘e truos to ; self-evid⠢t, t all men aee cr,te equal, t ey aee endoee by ⠸e Creator u cita⠔ unalienable ⠠⠐rs, t amg ⠘e aee Life, Libity ⠯ e pursuit a Happis.
+::lcode ell To Los Andzeles (sta ispanika Los Angeles = Oi Angeloi) e sten Amerikanike arngo L.A., el ei) einai e deutere megalutere pole ton Enomenon Politeion apo apopse plethysmou, kathos kai ena apo ta semandikotera oikonomika, politistika epistemonika kai psychagogika kendra tou kosmou.
+::lcode rus Germaniya (nem. Deutschland), ofitsialnoe nazvanie — Federativnaya Respublika Germaniya (nem. Bundesrepublik Deutschland), FRG (nem. BRD) — gosudarstvo v Zapadnoi Evrope. Ploshchad territorii — 357 021 km². Chislennost naseleniya po perepisi 2011 goda — bolee 80 millionov chelovek. [2][6].
+::lcode ukr Volodimir Oleksandrovich Zelensky (nar. 25 sichnya 1978, Krivy Rig) — ukrayinsky derzhavny diyach, politik, shoumen, aktor, komik, rezhiser, prodyuser ta stsenarist, shosty Prezident Ukrayini z 20 travnya 2019 roku.
+::lcode srp Sva ljudska bitsha radjaju se slobodna i jednaka u dostojanstvu i pravima. Ona su obdarena razumom i sveshtshu i treba jedni prema drugima da postupaju u dukhu bratstva.
+::lcode ara knda (balinjlyzya: Canada) hy dwla fy amryka alshmalya ttalf mn 10 mqat'at wthlatha aqalym. tq' fy alqsm alshmaly mn alqara wtmtd mn almhyt alatlsy fy alshrq ila almhyt alhadye fy alghrb wtmtd shmalan fy almhyt almtjmd alshmaly. knda hy albld althany 'almyan mn hyth almsaha alklya. kma an hdwd knda almshtrka m' alwlayat almthda mn aljnwb walshmal alghrby hy alatwl fy al'alm.
+::lcode fas kalifrnia (bh anglisi: California) ialti dr ghrb amrika br kranh' aqianws aram ast. mrkz an sakramntw w shhrhai mhm an lsanjls, sn digw, sn khwzh w sanfransiskw hstnd.hmtchnin in ialt pr jm'it trin ialt amrika ast.
+::lcode uig yeameraka qwshma shtatlara bwlsa shamalay yeamerakagha jaylashqan bar doelaet. yeunang paytaekhta bwlsa vashangtwn, yeaeng tchwng shaehara bwlsa nyuywrk shaehara. doelaet tala bwlsa yeenglaztala. hazarqa zungtung barak yewbama. bu doelaet yeaesladae yeaengalayaenang mustaemlakasa bwlup vashangatwnnang raehpaerlakadae 1776 yala 7 yeaynang 4 kuna mustaeqal bwlghan, yaer maeydana 9 malaywn 826 mang 630 kvadarat klwmetar, nwpusa 306 mallaywn 142 mang, bularnang yeassaslaq dana khrastayean dana.
+::lcode amh iteyopheyaa kaaalame sosetu teleqe yaaberehaame hayemaanotoche gaare taarikaawi genenyunate alaate.
+::lcode hin kailiphorniyaa shabda kaa pahalaa artha thaa jo kssetra jahaam aaj baahaa kailiphorniyaa praayadviip, nevaaddaa, yuuttaa aur erijonaa, nayaa meksiko, aur vaayomimga ke kaii vibhaag sthit haim.
+::lcode mar lamddan (imglish: London ) he imglamddace va yunaayattedd kimgddamace raajadhaaniice va sarvaat motthe shahar tasec yuropiyan samghaamadhiil sarvaat motthe mahaanagar kssetra aahe.
+::lcode nep yasako ucaai samundra satahabaatt 8,848 miittar (29,028 phiitt) cha. yo nepaalako solukhumbu jillaako khumjungga gaa. vi. sa. maa parcha .
+::lcode tam tamilnaadu (Tamil Nadu) intiyaavin 29 maanilangkalil onraakum. tamilnaadu, tamilakam enrum paravalaaka alaikkappadukiratu.
+::lcode mal intyayutte tekkupattinynyaarre arrrrattulllla samsthaanamaann keerallam.
+::lcode ori oddishaa bhaaratara puurba upakuullare thibaa eka prashaasanika raajya . ehaara uttara-puurbare pashcimabangga, uttarare jhaaddakhanndda, pashcima o uttara-pashcimare chatishagadda, dakssinna o dakssinna-pashcimare aandhrapradesha abasthita . ehaa aayatana hisaabare nabama o janasamkhyaa hisaabare egaaratama raajya . oddiaa bhaassaa raajyara sarakaarii bhaassaa . 2001 janagannanaa anusaare raajyara praaya 33.2 niyuta loka oddiaa bhaassaa byabahaara karanti . 
+::lcode zho jianadazai14000nianqianjiyouyuanzhuminzaicishenghuo. 
+::lcode heb kol 'od balevav penimah nefesh yehudi homiyah ulefa'ate mizerach, qadimah, 'ayin letsiyon tsofiyah, 'od lo avedah tiqvatenu, hatiqvah bat shenot 'alepayim liheyot 'am chafeshiy be'aretsenu, erets tsiyon virushalayim.
+::lcode yid dvvqa ayz an h'vr'ysh'r zshvrnal vvas vashryyvt dy yydysh-shfrakyq' qvltvr. 's ayz d'rshyn'n g'vvarn tmvz h'tshs"z (yvly 2006).
+::lcode hye Talnoei shrjan (ukr., Talnivsky raion), shrjan Ukrainayi Cherkasii marzum. Steghtsvel e 1923 tvakanin. Varchakan kentrone, Talnoe. Ashkharhagrutyune Shrjani taratski makerese kazmum e 917 km². Bnakchutyun
+::lcode tai miipratesisra 2 prates kuee saanmaariinolaeankrratwaatikan peondindaentiilomrobpaidwypueentiikongitaalii naiknatiimeueengkampiionediitaaleiiy peondindaenswnyaekkongitaaliitiituuklomrobdwypueentiipratesswitserlaend
+bugjjogeneun indonesiawa dongtimoreu, papua nyugini, bugdongjjogeneun solromon jedowa banuatu, nubelkalredoni, geurigo namdongjjogeneun nyujilraendeuga issda.
+baa illi sambhavisu imdenna hrdayadali nityavuu avataripa satyaavataara mannnnaagi maravaagi migavaagi kagavaagii... mannnnaagi maravaagi migavaagi kagavaagi bhava bhavadi bhatisihee bhavati duura nityavuu avataripa satyaavataara || baa illi ||
+vepxis tqaosani shota rustaveli ghmertsi shemvedre, nutu kvla damxsnas sophlisa shromasa, tsetsxls, tsqalsa da mitsasa, haerta tana mromasa; momtsnes phrteni da aghvphrinde, mivhxvde mas chemsa ndomasa, dghisit da ghamit vhxedvide mzisa elvata krtomaasa.
+anm moilegoimrid maki vekumen
+ic mag glas eotan ond hit ne hearmiath me.
+ptolmys
+chekosurobakia
+lha·sa·grong·khyer
+saali safiaryok imminik nillirotiqasoongofoq taitsomanitatsayaonirarsoni. imminillotaoq nillirotiqasongommisoni ollominitatsayaonirarsoni.
+amagrad 1 ar d ttlalan middn gan ilellitn mgaddan gh waddur d izrfan, yili ak darsn unlli d ufrak, illa flla sn ad ttmyawasn ngratsn s tagmat.

uroman/test/string-similarity-test-input.txt

@@ -0,0 +1,7 @@
+trap	strap
+colour	color
+labeling	labelling
+organisation	organization
+Philadelphia	Filadelfia
+Vladimir	Volodymyr
+Moskva	Moskvoy

uroman/test/string-similarity-test-output-ref.txt

@@ -0,0 +1,8 @@
+# Lang-code-1: eng Lang-code-2: eng
+trap	strap	1
+colour	color	0.1
+labeling	labelling	0.02
+organisation	organization	0.1
+Philadelphia	Filadelfia	0.02
+Vladimir	Volodymyr	0.5
+Moskva	Moskvoy	0.5

uroman/text/amh.txt

@@ -0,0 +1,7 @@
+ኢትዮጵያ ከዓለም ሶስቱ ትልቅ የአብርሃም ሀይማኖቶች ጋር ታሪካዊ ግንኙነት አላት። 
+ክርስትናን በአራተኛው ምዕተ-ዓመት ተቀብላለች። 
+ከሕዝቡ አንድ ሶስተኛው እስላም ነው። 
+የመጀመሪያው የእስላም ሂጅራ ወደ ኢትዮጵያ ነው የተከናወነው። 
+ነጋሽ በአፍሪካ የመጀመሪያው የእስላም መቀመጫ ናት። 
+እስከ ፲፱፻፸ ዎቹ ድረስ ብዙ ቤተ-እስራኤሎች በኢትዮጵያ ይኖሩ ነበር። 
+የራስ ተፈሪ እንቅስቃሴ ኢትዮጵያን በትልቅ ክብር ነው የሚያያት።

uroman/text/ara.txt

@@ -0,0 +1,3 @@
+كندا (بالإنجليزية: Canada) هي دولة في أمريكا الشمالية تتألف من 10 مقاطعات وثلاثة أقاليم. تقع في القسم الشمالي من القارة وتمتد من المحيط الأطلسي في الشرق إلى المحيط الهادئ في الغرب وتمتد شمالاً في المحيط المتجمد الشمالي. كندا هي البلد الثاني عالمياً من حيث المساحة الكلية. كما أن حدود كندا المشتركة مع الولايات المتحدة من الجنوب والشمال الغربي هي الأطول في العالم.
+أراضي كندا مأهولة منذ آلاف السنين من قبل مجموعات مختلفة من السكان الأصليين. مع حلول أواخر القرن الخامس عشر بدأت الحملات البريطانية والفرنسية استكشاف المنطقة ومن ثم استوطنتها على طول ساحل المحيط الأطلسي. تنازلت فرنسا عن ما يقرب من جميع مستعمراتها في أمريكا الشمالية في عام 1763 بعد حرب السنوات السبع. في عام 1867، مع اتحاد ثلاثة مستعمرات بريطانية في أمريكا الشمالية عبر كونفدرالية تشكلت كندا باعتبارها كيانًا فدراليًا ذا سيادة يضم أربع مقاطعات. بدأ ذلك عملية اتسعت فيها مساحة كندا وتوسع حكمها الذاتي عن المملكة المتحدة. تجلت هذه الاستقلالية من خلال تشريع وستمنستر عام 1931 وبلغت ذروتها في صورة قانون كندا عام 1982 والذي قطع الاعتماد القانوني لكندا على البرلمان البريطاني.
+كندا دولة فيدرالية يحكمها نظام ديمقراطي تمثيلي وملكية دستورية حيث الملكة إليزابيث الثانية قائدة للدولة. الأمة الكندية أمة ثنائية اللغة حيث الإنكليزية والفرنسية لغتان رسميتان على المستوى الاتحادي. تعد كندا واحدة من أكثر دول العالم تطوراً، حيث تمتلك اقتصاداً متنوعاً وتعتمد على مواردها الطبيعية الوفيرة، وعلى التجارة وبخاصة مع الولايات المتحدة اللتان تربطهما علاقة طويلة ومعقدة. كندا عضو في مجموعة الدول الصناعية السبع ومجموعة الثماني ومجموعة العشرين وحلف شمال الأطلسي ومنظمة التعاون والتنمية الاقتصادية ومنظمة التجارة العالمية ودول الكومنولث والفرنكوفونية ومنظمة الدول الأمريكية والإبيك والأمم المتحدة. تمتلك كندا واحداً من أعلى مستويات المعيشة في العالم حيث مؤشر التنمية البشرية يضعها في المرتبة الثامنة عالمياً.

uroman/text/ben.txt

@@ -0,0 +1,8 @@
+বার্লিন (জার্মান: Berlin বেয়ালিন্‌') জার্মানির রাজধানী, এবং ইউরোপ মহাদেশের একটি ঐতিহাসিক শহর। বার্লিন শহরে ৩৪ লক্ষেরও বেশি লোক বাস করেন। শহরটি একাধারে একটি শহর এবং জার্মানির একটি রাজ্য। বার্লিনের আয়তন ৩৪৩ বর্গমাইল; এটির আয়তন প্যারিস শহরের প্রায় ৯ গুণ।
+বার্লিন একটি বহুসাংস্কৃতিক শহর। বিশ্বের ১৮৪টি দেশ থেকে আগত প্রায় ৪ লক্ষ ৩০ হাজার অভিবাসী বার্লিনে বাস করে। এদের মধ্যে তুরস্ক থেকে আগত অভিবাসীরা সংখ্যা সবচেয়ে বেশি; বার্লিনে প্রায় ১ লক্ষ ১৯ হাজার তুর্কি অভিবাসী বাস করে। তুরস্কের বাইরে বার্লিনেই ইউরোপে তুর্কিদের সবচেয়ে বড় সম্প্রদায় অবস্থিত।
+১৯৪৯ সাল থেকে ১৯৯০ পর্যন্ত বার্লিন পূর্ব বার্লিন ও পশ্চিম বার্লিন---এই দুই ভাগে বিভক্ত ছিল। ১৯৬১ সালে পূর্ব জার্মান সরকার সেখানকার নাগরিকদের পশ্চিম বার্লিনে পালিয়ে যাওয়া ঠেকাতে দুই বার্লিনের মাঝে একটি দেয়াল তুলে দেয়। দেয়ালটি ১৯৬১ সাল থেকে ১৯৮৯ সাল পর্যন্ত টিকে ছিল। ঐ সময় ৫ হাজারেরও বেশি ব্যক্তি দেয়ালটি টপকানোর চেষ্টা করে; এদের মধ্যে ৩২০০ জনকে গ্রেফতার করা হয় এবং ১৯১ জন নিহত হয়।
+১৯৮৯ সালে দেয়ালটি ভেঙে ফেলার পর বার্লিনের ব্রান্ডেনবুর্গ ফটক পূর্ব ও পশ্চিম বার্লিনের পুনঃএকত্রীকরণের প্রতীক হিসেবে দাঁড়িয়ে আছে।
+বার্লিনের স্থানীয় ফুটবল দলের নাম হের্টা বে এস ৎসে বের্লিন। তারা ঘরোয়া ম্যাচগুলি বার্লিনের "অলিম্পিয়াষ্টাডিয়ন" নামের স্টেডিয়ামে খেলে থাকে। এই স্টেডিয়ামেই ১৯৩৬ সালের গ্রীষ্মকালীন অলিম্পিক্‌স অনুষ্ঠিত হয়।
+বার্লিনে কুকুর পোষা খুবই ব্যয়বহুল একটি কাজ। কুকুরের মালিককে প্রতি বছর দেড়শ ইউরো কর দিতে হয়।
+বার্লিনের কাউফ্‌হাউস ডেস ভেস্টেন্‌স (Kaufhaus des Westens, সংক্ষেপে KaDeWe, কাডেভে) ইউরোপের বৃহত্তম ডিপার্টমেন্ট স্টোর। এর আট তলাবিশিষ্ট ভবনে প্রায় ৪ লক্ষ জিনিস বেচা কেনা হয়।
+মার্কিন যুক্তরাষ্ট্রের লস অ্যাঞ্জেলেস বার্লিনের ভগ্নী শহর।

uroman/text/bod.txt

@@ -0,0 +1,3 @@
+ཁྲིན་ཀོན་ཆུས
+ལྷ་ས་གྲོང་ཁྱེར
+[[ཁྲིན་ཀོན་ཆུས་‎ཞེས་པ་ནི་རྒྱ་ནག་གཞུང་གིས་བཙན་འཛུལ་བྱས་རྗེས་བཏགས་པའི་མིང་ཞིག་ཡིན་པ་དང། དེ་ནི་ད་ལྟའི་ཆར་ལྷ་ས་གྲོང་ཁྱེར་གྱི་ཁོངས་གཏོགས་རྫོང་ཁག་བདུན་པོ་ཕུད་པའི་གྲོང་ཁྱེར་ནང་ཁུལ་གྱི་ས་ཁུལ་ཁག་བསྡུས་པའི་གནས་དེར་ཁྲེང་ཀོན་ཆུས་ཞེས་པའི་ཁོངས་སུ་གཏོགས་པར་བཤད་ཡོད་ཅིང། ནུབ་ཏུ་སྟོད་ལུང་ས་འབྲེལ་འབྲས་སྤུངས་དན་བག་ཡན་དང་ཤར་དུ་གཤོངས་ཀ་གླིང་ཡན་ཙམ་དུ་ཡིན་ཚོད་འདུག]]

uroman/text/egy.txt

@@ -0,0 +1,5 @@
+𓈎𓃭𓇋𓍯𓊪𓄿𓆓𓂋𓄿𓏏𓆇
+𓊪𓏏𓍯𓃭𓐝𓇌𓋴
+𓆿𓍧𓎇𓏻
+𓇌𓊪𓏲𓌙𓈉
+

uroman/text/ell.txt

@@ -0,0 +1,8 @@
+Το Λος Άντζελες (στα ισπανικά Los Angeles = Οι Άγγελοι) ή στην Αμερικανική αργκό L.A., ελ έι) είναι η δεύτερη μεγαλύτερη πόλη των Ηνωμένων Πολιτειών από άποψη πληθυσμού, καθώς και ένα από τα σημαντικότερα οικονομικά, πολιτιστικά επιστημονικά και ψυχαγωγικά κέντρα του κόσμου. 
+Βρίσκεται στη δυτική ακτή των Η.Π.Α., στην πολιτεία της Καλιφόρνιας. 
+Έχει 3,85 εκατομμύρια κατοίκους σύμφωνα με εκτίμηση του 2006 και έκταση 1.214,9 τετραγωνικών χιλιομέτρων. 
+Η αχανής μητροπολιτική περιοχή του Λος Άντζελες εκτιμάται ότι αριθμεί περίπου 13 εκατομμύρια κατοίκους, οι οποίοι αποκαλούνται Angelenos. 
+Η πόλη αποτελεί ένα από τα πιο κοσμοπολίτικα μέρη στον κόσμο, καθώς κατοικούν άνθρωποι προερχόμενοι από κάθε γωνιά της γης, που προσελκύονται από το ευχάριστο κλίμα, τον έντονο και γεμάτο ενέργεια τρόπο ζωής αλλά και την υπόσχεση του αμερικανικού ονείρου.
+
+Γερούν Ντάισελμπλουμ
+Γιώργος Κωνσταντινίδης

uroman/text/fas.txt

@@ -0,0 +1,6 @@
+کالیفرنیا (به انگلیسی: California) ایالتی در غرب آمریکا بر کرانهٔ اقیانوس آرام است. مرکز آن ساکرامنتو و شهرهای مهم آن لس‌آنجلس، سن دیگو، سن خوزه و سان‌فرانسیسکو هستند.همچنین این ایالت پر جمعیت ترین ایالت امریکا است.
+نام این ایالت از زبان اسپانیولی و به خصوص از رمانی به نام Las sergas de Esplandián گرفته شده، و متعلق به شخصیتی از این داستان است به نام ملکه Califia که احتمالاً از واژهٔ عربی «خلیفه» گرفته شده.[۲]
+کالیفرنیا پرجمعیت‌ترین ایالت ایالات متحده آمریکاست و نیز بزرگ‌ترین جمعیت ایرانی تبار خارج از خاور میانه را در خود جای داده است[۳] به طوری که چندین تن از اعضای شورای شهر بورلی هیلز ایرانی‌الاصل هستند.
+کالیفرنیا نهمین اقتصاد جهان است. در سال ۲۰۱۲، این ایالت تولید ناخالص داخلی برابر با ۱٬۹۵۸٬۹۰۴تریلیون دلار داشت، که نزدیک به تولید ناخالص داخلی کشور ایتالیا (۲٬۰۱۳٬۳۷۵ میلیون دلار) بود.[۴]
+کالیفرنیا صنعت فناوری اطلاعات و رایانه‌ای بسیار پیشرفته‌ای دارد به طوری که شرکتهای اوراکل، سیسکو سیستمز، اینتل، گوگل، یاهو، شرکت ای‌ام‌دی، سان مایکروسیستمز، و نیز شرکت رایانه‌ای اپل و شِوران نیز در این ایالت مرکزیت دارند.
+علاوه بر این، دو عدد از آزمایشگاه‌های فدرال بزرگ آمریکا در این ایالت قرار دارند از جمله آزمایشگاه ملی لارنس لیورمور و آزمایشگاه ملی لارنس برکلی.

uroman/text/heb.txt

@@ -0,0 +1,15 @@
+כֹּל עוֹד בַּלֵּבָב פְּנִימָה
+נֶפֶשׁ יְהוּדִי הוֹמִיָּה
+וּלְפַאֲתֵי מִזְרָח, קָדִימָה,
+עַיִן לְצִיּוֹן צוֹפִיָּה,
+עוֹד לֹא אָבְדָה תִּקְוָתֵנוּ,
+הַתִּקְוָה בַּת שְׁנוֹת אַלְפַּיִם
+לִהְיוֹת עַם חָפְשִׁי בְּאַרְצֵנוּ,
+אֶרֶץ צִיּוֹן וִירוּשָׁלַיִם.
+
+ניוּ יוֹרק (באנגלית: New York City, בקיצור NYC, או New York בלבד) היא העיר המאוכלסת ביותר בארצות הברית. שטחה העירוני הוא אחד מן הגדולים בעולם. כינויה של העיר הוא "התפוח הגדול". כבר למעלה ממאה שנים מהווה ניו יורק מרכז מסחרי וכלכלי מהגדולים בעולם. העיר מדורגת כעיר עולם מסוג אלפא (היא ולונדון הן היחידות המוגדרות כאלפא ++), בשל השפעתה העולמית על התקשורת, הפוליטיקה, החינוך, הבידור, התרבות והאופנה. מרכז התרבות האמנותי של העיר הוא בעל השפעה רבה בכל רחבי ארצות הברית ומחוצה לה. העיר היא מרכז לפעילות בינלאומית, ושוכן בה מטהו של ארגון האומות המאוחדות. ראש העיר החל מתחילת 2014 הוא ביל דה בלאזיו.
+העיר ניו יורק מורכבת מחמישה רבעים: הברונקס, ברוקלין, מנהטן, קווינס וסטטן איילנד. אוכלוסייתה מונה מעל 8.2 מיליון תושבים, המשתכנים בשטח של 1,214 קילומטר רבוע, והיא הצפופה מבין ערי ארצות הברית.
+רבות משכונות העיר ומאתריה ידועים ברחבי העולם. פסל החירות קידם את פניהם של מיליוני המהגרים בבואם לאמריקה בשלהי המאה ה-19 ובתחילת המאה ה-20. וול סטריט, השוכן במנהטן תחתית, מהווה מרכז פיננסי עולמי מאז סוף מלחמת העולם השנייה ומשכנה של הבורסה לניירות ערך. בעיר ממוקמים כמה מהבניינים הגבוהים בעולם, כמו בניין האמפייר סטייט, וכן שכנו בה מגדלי התאומים שנהרסו בפיגועי 11 בספטמבר.
+ניו יורק היא מקום היווסדן של תנועות תרבות אמריקאיות רבות, כמו תנועת הספרות והאמנות הידועה בשם "הרנסאנס של הארלם", סגנון הציור האקספרסיוניסטי המופשט (שידוע גם כאסכולת ניו יורק), והסגנונות המוזיקליים היפ הופ, פאנק וסלסה. 36 אחוזים מאוכלוסייתה נולדו מחוץ לארצות הברית, ובשנת 2005 דוברו בה כמעט 170 שפות שונות. הרכבת התחתית של ניו יורק פועלת 24 שעות ביממה, רחובותיה סואנים בהולכי רגל ובמכוניות והיא מכונה "העיר שאינה ישנה לעולם".
+
+אֱלוֹהַּ יֵשׁוּעַ ר֫וּחַ