|
|
|
|
|
|
|
|
|
|
|
|
| package NLP::Chinese;
|
|
|
| $utf8 = NLP::UTF8;
|
| %empty_ht = ();
|
|
|
| sub read_chinese_tonal_pinyin_files {
|
| local($caller, *ht, @filenames) = @_;
|
|
|
| $n_kHanyuPinlu = 0;
|
| $n_kXHC1983 = 0;
|
| $n_kHanyuPinyin = 0;
|
| $n_kMandarin = 0;
|
| $n_cedict = 0;
|
| $n_simple_pinyin = 0;
|
|
|
| foreach $filename (@filenames) {
|
| if ($filename =~ /unihan/i) {
|
| my $line_number = 0;
|
| if (open(IN, $filename)) {
|
| while (<IN>) {
|
| $line_number++;
|
| next if /^
|
| s/\s*$//;
|
| if (($u, $type, $value) = split(/\t/, $_)) {
|
| if ($type =~ /^(kHanyuPinlu|kXHC1983|kHanyuPinyin|kMandarin)$/) {
|
| $u = $util->trim($u);
|
| $type = $util->trim($type);
|
| $value = $util->trim($value);
|
| $f = $utf8->unicode_string2string($u);
|
|
|
| if ($type eq "kHanyuPinlu") {
|
| $value =~ s/\(.*?\)//g;
|
| $value = $util->trim($value);
|
| $translit = $caller->number_to_accent_tone($value);
|
| $ht{"kHanyuPinlu"}->{$f} = $translit;
|
| $n_kHanyuPinlu++;
|
| } elsif ($type eq "kXHC1983") {
|
| @translits = ($value =~ /:(\S+)/g);
|
| $translit = join(" ", @translits);
|
| $ht{"kXHC1983"}->{$f} = $translit;
|
| $n_kXHC1983++;
|
| } elsif ($type eq "kHanyuPinyin") {
|
| $value =~ s/^.*://;
|
| $value =~ s/,/ /g;
|
| $ht{"kHanyuPinyin"}->{$f} = $value;
|
| $n_kHanyuPinyin++;
|
| } elsif ($type eq "kMandarin") {
|
| $ht{"kMandarin"}->{$f} = $value;
|
| $n_kMandarin++;
|
| }
|
| }
|
| }
|
| }
|
| close(IN);
|
| print "Read in $n_kHanyuPinlu kHanyuPinlu, $n_kXHC1983 n_kXHC1983, $n_kHanyuPinyin n_kHanyuPinyin $n_kMandarin n_kMandarin\n";
|
| } else {
|
| print STDERR "Can't open $filename\n";
|
| }
|
| } elsif ($filename =~ /cedict/i) {
|
| if (open(IN, $filename)) {
|
| my $line_number = 0;
|
| while (<IN>) {
|
| $line_number++;
|
| next if /^
|
| s/\s*$//;
|
| if (($f, $translit) = ($_ =~ /^\S+\s+(\S+)\s+\[([^\[\]]+)\]/)) {
|
| $translit = $utf8->extended_lower_case($translit);
|
| $translit = $caller->number_to_accent_tone($translit);
|
| $translit =~ s/\s//g;
|
| if ($old_translit = $ht{"cedict"}->{$f}) {
|
|
|
| $ht{"cedicts"}->{$f} = join(" ", $ht{"cedicts"}->{$f}, $translit) unless $old_translit eq $translit;
|
| } else {
|
| $ht{"cedict"}->{$f} = $translit;
|
| $ht{"cedicts"}->{$f} = $translit;
|
| }
|
| $n_cedict++;
|
| }
|
| }
|
| close(IN);
|
|
|
| } else {
|
| print STDERR "Can't open $filename";
|
| }
|
| } elsif ($filename =~ /chinese_to_pinyin/i) {
|
| if (open(IN, $filename)) {
|
| my $line_number = 0;
|
| while (<IN>) {
|
| $line_number++;
|
| next if /^
|
| if (($f, $translit) = ($_ =~ /^(\S+)\t(\S+)\s*$/)) {
|
| $ht{"simple_pinyin"}->{$f} = $translit;
|
| $n_simple_pinyin++;
|
| }
|
| }
|
| close(IN);
|
|
|
| } else {
|
| print STDERR "Can't open $filename";
|
| }
|
| } else {
|
| print STDERR "Don't know what to do with file $filename (in read_chinese_tonal_pinyin_files)\n";
|
| }
|
| }
|
| }
|
|
|
| sub tonal_pinyin {
|
| local($caller, $s, *ht, $gloss) = @_;
|
|
|
| return $result if defined($result = $ht{COMBINED}->{$s});
|
|
|
| $cedict_pinyin = $ht{"cedict"}->{$s} || "";
|
| $cedicts_pinyin = $ht{"cedicts"}->{$s} || "";
|
| $unihan_pinyin = "";
|
| @characters = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
|
| foreach $c (@characters) {
|
| if ($pinyin = $ht{"simple_pinyin"}->{$c}) {
|
| $unihan_pinyin .= $pinyin;
|
| } elsif ($pinyin = $ht{"kHanyuPinlu"}->{$c}) {
|
| $pinyin =~ s/^(\S+)\s.*$/$1/;
|
| $unihan_pinyin .= $pinyin;
|
| } elsif ($pinyin = $ht{"kXHC1983"}->{$c}) {
|
| $pinyin =~ s/^(\S+)\s.*$/$1/;
|
| $unihan_pinyin .= $pinyin;
|
| } elsif ($pinyin = $ht{"kHanyuPinyin"}->{$c}) {
|
| $pinyin =~ s/^(\S+)\s.*$/$1/;
|
| $unihan_pinyin .= $pinyin;
|
| } elsif ($pinyin = $ht{"cedicts"}->{$c}) {
|
| $pinyin =~ s/^(\S+)\s.*$/$1/;
|
| $unihan_pinyin .= $pinyin;
|
|
|
| } elsif ($c =~ /^(\xC2\xB7|\xE3\x83\xBB|\xC3\x97)$/) {
|
| $unihan_pinyin .= $c;
|
|
|
| } elsif ($c =~ /^([\x21-\x7E])$/) {
|
| $unihan_pinyin .= $c;
|
| } else {
|
| $unihan_pinyin .= "?";
|
| $hex = $utf8->utf8_to_hex($c);
|
| $unicode = uc $utf8->utf8_to_4hex_unicode($c);
|
|
|
| }
|
| }
|
| $pinyin_title = "";
|
| if (($#characters >= 1) && $cedicts_pinyin) {
|
| foreach $pinyin (split(/\s+/, $cedicts_pinyin)) {
|
| $pinyin_title .= "$s $pinyin (CEDICT)\n";
|
| }
|
| $pinyin_title .= "\n";
|
| }
|
| foreach $c (@characters) {
|
| my %local_ht = ();
|
| @pinyins = ();
|
| foreach $type (("kHanyuPinlu", "kXHC1983", "kHanyuPinyin", "cedicts")) {
|
| if ($pinyin_s = $ht{$type}->{$c}) {
|
| foreach $pinyin (split(/\s+/, $pinyin_s)) {
|
| push(@pinyins, $pinyin) unless $util->member($pinyin, @pinyins);
|
| $type2 = ($type eq "cedicts") ? "CEDICT" : $type;
|
| $local_ht{$pinyin} = ($local_ht{$pinyin}) ? join(", ", $local_ht{$pinyin}, $type2) : $type2;
|
| }
|
| }
|
| }
|
| foreach $pinyin (@pinyins) {
|
| $type_s = $local_ht{$pinyin};
|
| $pinyin_title .= "$c $pinyin ($type_s)\n";
|
| }
|
| }
|
| $pinyin_title =~ s/\n$//;
|
| $pinyin_title =~ s/\n/
/g;
|
| $unihan_pinyin = "" if $unihan_pinyin =~ /^\?+$/;
|
| if (($#characters >= 1) && $cedict_pinyin && $unihan_pinyin && ($unihan_pinyin ne $cedict_pinyin)) {
|
| $log = "Gloss($s): $gloss\nCEdict($s): $cedicts_pinyin\nUnihan($s): $unihan_pinyin\n";
|
| foreach $type (("kHanyuPinlu", "kXHC1983", "kHanyuPinyin")) {
|
| $log_line = "$type($s): ";
|
| foreach $c (@characters) {
|
| $pinyin = $ht{$type}->{$c} || "";
|
| if ($pinyin =~ / /) {
|
| $log_line .= "($pinyin)";
|
| } elsif ($pinyin) {
|
| $log_line .= $pinyin;
|
| } else {
|
| $log_line .= "?";
|
| }
|
| }
|
| $log .= "$log_line\n";
|
| }
|
| $ht{CONFLICT}->{$s} = $log;
|
| }
|
| $result = $unihan_pinyin || $cedict_pinyin;
|
| $result = $cedict_pinyin if ($#characters > 0) && $cedict_pinyin;
|
| $ht{COMBINED}->{$s} = $result;
|
| $ht{PINYIN_TITLE}->{$s} = $pinyin_title;
|
| return $result;
|
| }
|
|
|
| %number_to_accent_tone_ht = (
|
| "a1", "\xC4\x81", "a2", "\xC3\xA1", "a3", "\xC7\x8E", "a4", "\xC3\xA0",
|
| "e1", "\xC4\x93", "e2", "\xC3\xA9", "e3", "\xC4\x9B", "e4", "\xC3\xA8",
|
| "i1", "\xC4\xAB", "i2", "\xC3\xAD", "i3", "\xC7\x90", "i4", "\xC3\xAC",
|
| "o1", "\xC5\x8D", "o2", "\xC3\xB3", "o3", "\xC7\x92", "o4", "\xC3\xB2",
|
| "u1", "\xC5\xAB", "u2", "\xC3\xBA", "u3", "\xC7\x94", "u4", "\xC3\xB9",
|
| "u:1","\xC7\x96", "u:2","\xC7\x98", "u:3","\xC7\x9A", "u:4","\xC7\x9C",
|
| "\xC3\xBC1","\xC7\x96","\xC3\xBC2","\xC7\x98","\xC3\xBC3","\xC7\x9A","\xC3\xBC4","\xC7\x9C"
|
| );
|
|
|
| sub number_to_accent_tone {
|
| local($caller, $s) = @_;
|
|
|
| my $result = "";
|
| while (($pre,$alpha,$tone_number,$rest) = ($s =~ /^(.*?)((?:[a-z]|u:|\xC3\xBC)+)([1-5])(.*)$/i)) {
|
| if ($tone_number eq "5") {
|
| $result .= "$pre$alpha";
|
| } elsif ((($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)([ae])(.*)$/))
|
| || (($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)(o)(u.*)$/))
|
| || (($pre_acc,$acc_letter,$post_acc) = ($alpha =~ /^(.*)(u:|[iou]|\xC3\xBC)([^aeiou]*)$/))) {
|
| $result .= "$pre$pre_acc" . ($number_to_accent_tone_ht{($acc_letter . $tone_number)} || ($acc_letter . $tone_number)) . $post_acc;
|
| } else {
|
| $result .= "$pre$alpha$tone_number";
|
| }
|
| $s = $rest;
|
| }
|
| $result .= $s;
|
| $result =~ s/u:/\xC3\xBC/g;
|
| return $result;
|
| }
|
|
|
| sub string_contains_utf8_cjk_unified_ideograph_p {
|
| local($caller, $s) = @_;
|
|
|
| return ($s =~ /([\xE4-\xE9]|\xE3[\x90-\xBF]|\xF0[\xA0-\xAC])/);
|
| }
|
|
|
| 1;
|
|
|
