Hugging Face's logo

sleepyhead111
/
vllm_fairseq

ONNX
Safetensors
Model card Files
xet
Community
vllm_fairseq
File size: 4,109 Bytes
52db324
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
#!/usr/bin/env perl
#
# This file is part of moses.  Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.

use strict;
use warnings;

use autodie;
use FindBin qw($RealBin);
use Getopt::Long "GetOptions";

my ($SENNA,
    $SENNA_DIR,
    $SENNA_OPTIONS,
    $SPLIT_HYPHEN,
    $SPLIT_SLASH,
    $MARK_SPLIT,
    $BINARIZE,
    $UNPARSEABLE,
    $RAW_IN,
    $RAW_OUT);

$UNPARSEABLE = 0;

die("ERROR: syntax is: parse-en-senna.perl [-senna-options OPTIONS] [-split-hyphen] [-split-slash] [-mark-split] [-binarize] [-unparseable] [-raw-in PATH] [-raw-out PATH] -senna PATH -senna-dir PATH < in > out\n")
  unless &GetOptions
  ('senna=s' => \$SENNA,
   'senna-dir=s' => \$SENNA_DIR,
   'senna-options=s' => \$SENNA_OPTIONS,
   'split-hyphen' => \$SPLIT_HYPHEN,
   'split-slash' => \$SPLIT_SLASH,
   'mark-split' => \$MARK_SPLIT,
   'binarize' => \$BINARIZE,
   'unparseable' => \$UNPARSEABLE,
   'raw-in=s' => \$RAW_IN,
   'raw-out=s' => \$RAW_OUT
   )
  && defined($SENNA);

die("ERROR: file not found or not executable: '$SENNA'\n") unless -x $SENNA;
die("ERROR: could not find SENNA directory: '$SENNA_DIR'\n") unless -d $SENNA_DIR;

# Step 1: Read standard input and write two temporary files:
#
#     $tmpOriginal    Contains a copy of the input as-is
#
#     $tmpProcessed   Contains a copy of the input after pre-processing ready
#                     for input to SENNA

my $tmpOriginal = "/tmp/parse-en-senna.1.$$";
my $tmpProcessed = "/tmp/parse-en-senna.2.$$";

open(TMP_ORIGINAL, ">$tmpOriginal");

open(TMP_PROCESSED,
     "| $RealBin/../../tokenizer/deescape-special-chars.perl > $tmpProcessed;");

while(<STDIN>) {
  print TMP_ORIGINAL $_;

  # If the line is longer than 1023 bytes (including the newline) then replace
  # it with "SENTENCE_TOO_LONG\n".  This is because SENNA reads lines into a
  # 1024 character array and if a line is longer than 1023 characters then it
  # gets read in stages and treated as multiple input lines.
  my $num_bytes;
  {
    use bytes;
    $num_bytes = length($_);
  }
  if ($num_bytes > 1023) {
    print TMP_PROCESSED "SENTENCE_TOO_LONG\n";
    next;
  }

  # Replace "-LRB-", "-RRB-", etc. with "(", ")", etc.
  s/-LRB-/(/g;
  s/-RRB-/)/g;
  s/-LSB-/[/g;
  s/-RSB-/]/g;
  s/-LCB-/{/g;
  s/-RCB-/}/g;

  # Unsplit hyphens.
  s/ \@-\@ /-/g if $SPLIT_HYPHEN;
  # Unsplit slashes.
  s/ \@\/\@ /\//g if $SPLIT_SLASH;

  print TMP_PROCESSED $_;
}

close(TMP_ORIGINAL);
close(TMP_PROCESSED);

# Step 2: Parse $tmpProcessed then pass the raw output through a post-processing
#         pipeline.

my $pipeline = "";

# Stage 1: Parse input (unless given pre-parsed input via -raw-in option).
if (defined($RAW_IN)) {
  $pipeline .= "cat \"$RAW_IN\" |";
} else {
  $pipeline .= "cat $tmpProcessed |";
  my $path = $SENNA_DIR;
  # SENNA requires -path's argument to end with a slash.
  if ($path !~ /\/$/) {
    $path .= "/";
  }
  $pipeline .= " $SENNA -path $path -usrtokens";
  $pipeline .= " $SENNA_OPTIONS" if defined($SENNA_OPTIONS);
  $pipeline .= " |";
}

if (defined($RAW_OUT)) {
  $pipeline .= " tee \"$RAW_OUT\" |";
}

# Stage 2: Convert SENNA output to Moses XML (via Berkeley output format)
$pipeline .= " $RealBin/senna2brackets.py --berkeley-style |";
$pipeline .= " $RealBin/berkeleyparsed2mosesxml.perl |";

# Stage 3: Re-split hyphens / slashes.
if ($SPLIT_HYPHEN) {
  $pipeline .= " $RealBin/syntax-hyphen-splitting.perl";
  $pipeline .= " -binarize" if $BINARIZE;
  $pipeline .= " -mark-split" if $MARK_SPLIT;
  $pipeline .= " |";
}
if ($SPLIT_SLASH) {
  $pipeline .= " $RealBin/syntax-hyphen-splitting.perl -slash";
  $pipeline .= " -binarize" if $BINARIZE;
  $pipeline .= " -mark-split" if $MARK_SPLIT;
  $pipeline .= " |";
}

# Run the parsing + post-processing pipeline.
open(PARSE, $pipeline);
open(TMP_ORIGINAL, $tmpOriginal);
while (<PARSE>) {
  my $parsedLine = $_;
  my $originalLine = <TMP_ORIGINAL>;
  if ($UNPARSEABLE == 1 && length($parsedLine) == 1) {
    print $originalLine;
  } else {
    print $parsedLine;
  }
}
close(PARSE);

`rm $tmpOriginal`;
`rm $tmpProcessed`;