Upload 2 files

detokenizer.perl

@@ -0,0 +1,242 @@
+#!/usr/bin/env perl
+
+# ProLNat Tokenizer (provided with Sentence Identifier)
+# autor: Grupo ProLNat@GE, CiTIUS
+# Universidade de Santiago de Compostela
+
+
+# Script que integra 2 funçoes perl: sentences e tokens
+package Tokens;
+
+#<ignore-block>
+use strict; 
+binmode STDIN, ':utf8';
+binmode STDOUT, ':utf8';
+use utf8;
+#<ignore-block>
+
+# Pipe
+my $pipe = !defined (caller);#<ignore-line> 
+
+# Absolute path 
+use File::Basename;#<ignore-line>
+my $abs_path = ".";#<string>
+$abs_path = dirname(__FILE__);#<ignore-line>
+
+##variaveis globais
+##para sentences e tokens:
+my $UpperCase = "[A-ZÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÑÇÜ]";#<string>
+my $LowerCase = "[a-záéíóúàèìòùâêîôûñçü]";#<string>
+#my $Punct =  qr/[\.\,\;\«\»\“\”\'\"\‘\’\&\$\#\=\(\)\<\>\!\¡\?\¿\\\[\]\{\}\|\^\*\€\·\¬\…]/;#<string>
+my $Punct =  qr/[\:\,\;\»\”\’\&\$\=\)\>\!\?\]\}\|\^\€\·\¬\…]/;#<string>
+my $Punct_urls = qr/[\:\/\~]/;#<string>
+my $Punct_open =  qr/[\‘\«\“\(\<\¡\¿\\\[\{\#\*]/;#<string>
+
+##para splitter:
+##########INFORMAÇAO DEPENDENTE DA LINGUA###################
+#my $pron = "(me|te|se|le|les|la|lo|las|los|nos|os)";
+# Formas que não se separam do 's (e sim os nomes próprios) so ingles
+my $contr = "([Hh]e|[Hh]ere|[Hh]ow|[Ii]t|[Ss]he|[Tt]hat|[Tt]here|[Ww]hat|[Ww]hen|[Ww]here|[Ww]ho|[Ww]hy)";#<string>
+###########################################################
+my $w = "[A-ZÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÑÇÜa-záéíóúàèìòùâêîôûñçü]";#<string>
+
+sub tokens {
+	
+	my ($sentences) = @_;#<ref><list><string>
+
+	###puntuaçoes compostas
+	my $susp = "3SUSP012";#<string>
+	my $duplo1 = "2DOBR111";#<string>
+	my $duplo2 = "2DOBR222";#<string>
+	my $duplo3 = "2DOBR333";#<string>
+	my $duplo4 = "2DOBR444";#<string>
+
+	##pontos e virgulas entre numeros
+	my $dot_quant = "44DOTQUANT77";#<string>
+	my $comma_quant = "44COMMQUANT77";#<string>
+	my $quote_quant = "44QUOTQUANT77";#<string>
+
+	#(my @sentences) = split ('\n', $texto);
+
+	my @saida = ();#<list><string>
+	
+	foreach my $sentence (@{$sentences}) {
+	
+		chomp $sentence;
+		#substituir puntuaçoes 
+
+		$sentence =~ s/[ ]*$//;
+		$sentence =~ s/ \.\.\./$susp /g ;
+		$sentence =~ s/ \<\</$duplo1 /g ;
+		$sentence =~ s/ \>\>/$duplo2 /g ;
+		$sentence =~ s/ \'\'/$duplo3 /g ;
+		$sentence =~ s/ \`\`/$duplo4 /g ;
+
+		$sentence =~ s/([0-9]+)\.([0-9]+)/${1}$dot_quant$2/g ;
+		$sentence =~ s/([0-9]+)\,([0-9]+)/${1}$comma_quant$2/g ;
+		$sentence =~ s/([0-9]+)\'([0-9]+)/${1}$quote_quant$2/g ;
+
+		#print STDERR "1#$sentence#\n";
+
+
+                ##casos para o inglês: I'm he's, he'd
+		$sentence =~ s/I ([\'\’]) (m|ve|d)(?:\s|$)/I$1$2 /g;
+		$sentence =~ s/([yY]ou|[tT]hey|[wW]e) ([\'\’]) (re|ve|d)(?:\s|$)/$1$2$3 /g;
+		$sentence =~ s/([sS]he|[hH]e) ([\'\’]) (s|ve|d)(?:\s|$)/$1$2$3 /g;
+		$sentence =~ s/ ([\'\’]) s(?:\s|$|\.)/$1s /g;
+		$sentence =~ s/ ([\'\’]) t(?:\s|$|\.)/$1t /g;
+		$sentence =~ s/ ([\'\’]) ll(?:\s|$|\.)/$1ll /g;
+		#print STDERR "#$sentence#\n";
+		
+	        $sentence =~ s/\" ([^\"]+) \"/\"$1\"/g;
+		$sentence =~ s/\' ([^\']+) \'/\'$1\'/g;
+                $sentence =~ s/\' ([^\’]+) \’/\'$1\’/g;
+		$sentence =~ s/\* ([^\*]+) \*/\*$1\*/g;
+
+		#$sentence =~ s/ ($Punct_open) ($Punct_open) / $1$2/g ;
+		$sentence =~ s/ ($Punct_open) / $1/g ;
+		$sentence =~ s/^($Punct_open) /$1/g ;
+		#$sentence =~ s/($Punct_open) /$1/g ;
+                $sentence =~ s/($Punct_open)($Punct_open) /$1$2/g ;
+		$sentence =~ s/ ($Punct_urls)(?:[\s\n]|$) /$1/g  ;
+		$sentence =~ s/ ($Punct) /$1 /g ;
+		$sentence =~ s/ ($Punct)$/$1/g ;
+
+		##hypen - no fim de palavra ou no principio:
+		$sentence =~ s/(\w) - /$1- /g  ;
+		$sentence =~ s/ - (\w)/ -$1/g  ;
+		$sentence =~ s/(\w) -$/$1-/g  ;
+		$sentence =~ s/^- (\w)/-$1/g  ;
+
+
+		$sentence =~ s/[\s]*\./\./g  ; ##ponto 
+
+		my @tokens = split (" ", $sentence);#<array><string> 
+
+		foreach my $token (@tokens) {
+
+			$token =~ s/^[\s]*//;
+			$token =~ s/[\s]*$//;
+			$token =~ s/$susp/\.\.\./;
+			$token =~ s/$duplo1/\<\</;
+			$token =~ s/$duplo2/\>\>/;
+			$token =~ s/$duplo3/\'\'/;
+			$token =~ s/$duplo4/\`\`/;
+			$token =~ s/$dot_quant/\./;
+			$token =~ s/$comma_quant/\,/;
+			$token =~ s/$quote_quant/\'/;
+
+			if($pipe){#<ignore-line>
+				print "$token ";#<ignore-line>
+			}else{#<ignore-line>
+				push (@saida, $token);
+			}#<ignore-line>
+		}
+	
+		if($pipe){#<ignore-line>
+			print "\n";#<ignore-line>
+		}else{#<ignore-line>
+			push (@saida, "");
+		}#<ignore-line>
+	}
+	
+	return \@saida;
+}
+
+#<ignore-block>
+if($pipe){
+	my @tokens=<STDIN>;
+	tokens(\@tokens);
+}
+#<ignore-block>
+
+###OUTRAS FUNÇOES
+
+sub punct {
+	my ($p) = @_ ;#<string>
+	my $result ="";#<string>
+
+	if ($p eq "\.") {
+		$result = "Fp"; 
+	}
+	elsif ($p eq "\,") {
+		$result = "Fc"; 
+	}
+	elsif ($p eq "\:") {
+		$result = "Fd"; 
+	}
+	elsif ($p eq "\;") {
+		$result = "Fx"; 
+	}
+	elsif ($p =~ /^(\-|\-\-)$/) {
+		$result = "Fg"; 
+	} 
+	elsif ($p =~ /^(\'|\"|\`\`|\'\')$/) {
+		$result = "Fe"; 
+	}
+	elsif ($p eq "\.\.\.") {
+		$result = "Fs"; 
+	}
+	elsif ($p =~ /^(\<\<|«)/) {
+		$result = "Fra"; 
+	}
+	elsif ($p =~ /^(\>\>|»)/) {
+		$result = "Frc"; 
+	}
+	elsif ($p eq "\%") {
+		$result = "Ft"; 
+	}
+	elsif ($p =~ /^(\/|\\)$/) {
+		$result = "Fh"; 
+	}
+	elsif ($p eq "\(") {
+		$result = "Fpa"; 
+	}
+	elsif ($p eq "\)") {
+		$result = "Fpt"; 
+	}
+	elsif ($p eq "\¿") {
+		$result = "Fia"; 
+	} 
+	elsif ($p eq "\?") {
+		$result = "Fit"; 
+	}
+	elsif ($p eq "\¡") {
+		$result = "Faa"; 
+	}
+	elsif ($p eq "\!") {
+		$result = "Fat"; 
+	}
+	elsif ($p eq "\[") {
+		$result = "Fca"; 
+	} 
+	elsif ($p eq "\]") {
+		$result = "Fct"; 
+	}
+	elsif ($p eq "\{") {
+		$result = "Fla"; 
+	} 
+	elsif ($p eq "\}") {
+		$result = "Flt"; 
+	}
+	return $result;
+}
+
+
+sub lowercase {
+  my ($x) = @_ ;#<string>
+  $x = lc ($x);
+  $x =~  tr/ÁÉÍÓÚÇÑ/áéíóúçñ/;
+
+  return $x;
+} 
+
+sub Trim {
+  my ($x) = @_ ;#<string>
+
+  $x =~ s/^[\s]*//;  
+  $x =~ s/[\s]$//;  
+
+  return $x;
+}
+

tokenizer.perl

@@ -0,0 +1,218 @@
+#!/usr/bin/env perl
+
+# ProLNat Tokenizer (provided with Sentence Identifier)
+# autor: Grupo ProLNat@GE, CiTIUS
+# Universidade de Santiago de Compostela
+
+
+# Script que integra 2 funçoes perl: sentences e tokens
+package Tokens;
+
+#<ignore-block>
+use strict; 
+binmode STDIN, ':utf8';
+binmode STDOUT, ':utf8';
+use utf8;
+#<ignore-block>
+
+# Pipe
+my $pipe = !defined (caller);#<ignore-line> 
+
+# Absolute path 
+use File::Basename;#<ignore-line>
+my $abs_path = ".";#<string>
+$abs_path = dirname(__FILE__);#<ignore-line>
+
+##variaveis globais
+##para sentences e tokens:
+my $UpperCase = "[A-ZÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÑÇÜ]";#<string>
+my $LowerCase = "[a-záéíóúàèìòùâêîôûñçü]";#<string>
+my $Punct =  qr/[\,\;\«\»\“\”\'\"\&\$\#\=\(\)\<\>\!\¡\?\¿\\\[\]\{\}\|\^\*\€\·\¬\…]/;#<string>
+my $Punct_urls = qr/[\:\/\~]/;#<string>
+
+##para splitter:
+##########INFORMAÇAO DEPENDENTE DA LINGUA###################
+#my $pron = "(me|te|se|le|les|la|lo|las|los|nos|os)";
+# Formas que não se separam do 's (e sim os nomes próprios) so ingles
+my $contr = "([Hh]e|[Hh]ere|[Hh]ow|[Ii]t|[Ss]he|[Tt]hat|[Tt]here|[Ww]hat|[Ww]hen|[Ww]here|[Ww]ho|[Ww]hy)";#<string>
+###########################################################
+my $w = "[A-ZÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÑÇÜa-záéíóúàèìòùâêîôûñçü]";#<string>
+
+sub tokens {
+	
+	my ($sentences) = @_;#<ref><list><string>
+
+	###puntuaçoes compostas
+	my $susp = "3SUSP012";#<string>
+	my $duplo1 = "2DOBR111";#<string>
+	my $duplo2 = "2DOBR222";#<string>
+	my $duplo3 = "2DOBR333";#<string>
+	my $duplo4 = "2DOBR444";#<string>
+
+	##pontos e virgulas entre numeros
+	my $dot_quant = "44DOTQUANT77";#<string>
+	my $comma_quant = "44COMMQUANT77";#<string>
+	my $quote_quant = "44QUOTQUANT77";#<string>
+
+	#(my @sentences) = split ('\n', $texto);
+
+	my @saida = ();#<list><string>
+	
+	foreach my $sentence (@{$sentences}) {
+	
+		chomp $sentence;
+		#substituir puntuaçoes 
+
+		$sentence =~ s/[ ]*$//;
+		$sentence =~ s/\.\.\./ $susp /g ;
+		$sentence =~ s/\<\</ $duplo1 /g ;
+		$sentence =~ s/\>\>/ $duplo2 /g ;
+		$sentence =~ s/\'\'/ $duplo3 /g ;
+		$sentence =~ s/\`\`/ $duplo4 /g ;
+
+		$sentence =~ s/([0-9]+)\.([0-9]+)/${1}$dot_quant$2 /g ;
+		$sentence =~ s/([0-9]+)\,([0-9]+)/${1}$comma_quant$2 /g ;
+		$sentence =~ s/([0-9]+)\'([0-9]+)/${1}$quote_quant$2 /g ;
+
+		#print STDERR "#$sentence#\n";
+		$sentence =~ s/($Punct)/ $1 /g ;
+		#print STDERR "2#$sentence#\n";
+		$sentence =~ s/($Punct_urls)(?:[\s\n]|$)/ $1 /g  ; 
+
+		##hypen - no fim de palavra ou no principio:
+		$sentence =~ s/(\w)- /$1 - /g  ;
+		$sentence =~ s/ -(\w)/ - $1/g  ;
+		$sentence =~ s/(\w)-$/$1 -/g  ;
+		$sentence =~ s/^-(\w)/- $1/g  ;
+
+
+		$sentence =~ s/\.$/ \. /g  ; ##ponto final
+
+		my @tokens = split (" ", $sentence);#<array><string> 
+
+		foreach my $token (@tokens) {
+
+			$token =~ s/^[\s]*//;
+			$token =~ s/[\s]*$//;
+			$token =~ s/$susp/\.\.\./;
+			$token =~ s/$duplo1/\<\</;
+			$token =~ s/$duplo2/\>\>/;
+			$token =~ s/$duplo3/\'\'/;
+			$token =~ s/$duplo4/\`\`/;
+			$token =~ s/$dot_quant/\./;
+			$token =~ s/$comma_quant/\,/;
+			$token =~ s/$quote_quant/\'/;
+
+			if($pipe){#<ignore-line>
+				print "$token ";#<ignore-line>
+			}else{#<ignore-line>
+				push (@saida, $token);
+			}#<ignore-line>
+		}
+	
+		if($pipe){#<ignore-line>
+			print "\n";#<ignore-line>
+		}else{#<ignore-line>
+			push (@saida, "");
+		}#<ignore-line>
+	}
+	
+	return \@saida;
+}
+
+#<ignore-block>
+if($pipe){
+	my @tokens=<STDIN>;
+	tokens(\@tokens);
+}
+#<ignore-block>
+
+###OUTRAS FUNÇOES
+
+sub punct {
+	my ($p) = @_ ;#<string>
+	my $result ="";#<string>
+
+	if ($p eq "\.") {
+		$result = "Fp"; 
+	}
+	elsif ($p eq "\,") {
+		$result = "Fc"; 
+	}
+	elsif ($p eq "\:") {
+		$result = "Fd"; 
+	}
+	elsif ($p eq "\;") {
+		$result = "Fx"; 
+	}
+	elsif ($p =~ /^(\-|\-\-)$/) {
+		$result = "Fg"; 
+	} 
+	elsif ($p =~ /^(\'|\"|\`\`|\'\')$/) {
+		$result = "Fe"; 
+	}
+	elsif ($p eq "\.\.\.") {
+		$result = "Fs"; 
+	}
+	elsif ($p =~ /^(\<\<|«)/) {
+		$result = "Fra"; 
+	}
+	elsif ($p =~ /^(\>\>|»)/) {
+		$result = "Frc"; 
+	}
+	elsif ($p eq "\%") {
+		$result = "Ft"; 
+	}
+	elsif ($p =~ /^(\/|\\)$/) {
+		$result = "Fh"; 
+	}
+	elsif ($p eq "\(") {
+		$result = "Fpa"; 
+	}
+	elsif ($p eq "\)") {
+		$result = "Fpt"; 
+	}
+	elsif ($p eq "\¿") {
+		$result = "Fia"; 
+	} 
+	elsif ($p eq "\?") {
+		$result = "Fit"; 
+	}
+	elsif ($p eq "\¡") {
+		$result = "Faa"; 
+	}
+	elsif ($p eq "\!") {
+		$result = "Fat"; 
+	}
+	elsif ($p eq "\[") {
+		$result = "Fca"; 
+	} 
+	elsif ($p eq "\]") {
+		$result = "Fct"; 
+	}
+	elsif ($p eq "\{") {
+		$result = "Fla"; 
+	} 
+	elsif ($p eq "\}") {
+		$result = "Flt"; 
+	}
+	return $result;
+}
+
+
+sub lowercase {
+  my ($x) = @_ ;#<string>
+  $x = lc ($x);
+  $x =~  tr/ÁÉÍÓÚÇÑ/áéíóúçñ/;
+
+  return $x;
+} 
+
+sub Trim {
+  my ($x) = @_ ;#<string>
+
+  $x =~ s/^[\s]*//;  
+  $x =~ s/[\s]$//;  
+
+  return $x;
+}