Hugging Face's logo

Spaces:
quickmt
/
quickmt-vs-libretranslate
Running

App Files Community
quickmt-vs-libretranslate / index.html
radinplaid's picture
radinplaid
Upload index.html
443a92a verified
raw
history blame contribute delete
133 kB
 <!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
<meta charset="utf-8">
<meta name="generator" content="quarto-1.8.25">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<title>quickmt vs.&nbsp;libretranslate</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
width: 0.8em;
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
vertical-align: middle;
}
/* CSS for syntax highlighting */
html { -webkit-text-size-adjust: 100%; }
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
{ counter-reset: source-line 0; }
pre.numberSource code > span
{ position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
{ content: counter(source-line);
position: relative; left: -1em; text-align: right; vertical-align: baseline;
border: none; display: inline-block;
-webkit-touch-callout: none; -webkit-user-select: none;
-khtml-user-select: none; -moz-user-select: none;
-ms-user-select: none; user-select: none;
padding: 0 4px; width: 4em;
}
pre.numberSource { margin-left: 3em; padding-left: 4px; }
div.sourceCode
{ }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
</style>
<script src="https://cdn.jsdelivr.net/npm/jquery@3.5.1/dist/jquery.min.js" integrity="sha384-ZvpUoO/+PpLXR1lu4jmpXWu80pZlYUAfxl5NsBMWOEPSjUn/6Z/hRTt8+pR6L4N2" crossorigin="anonymous"></script><script src="quickmt-vs-libretranslate_files/libs/clipboard/clipboard.min.js"></script>
<script src="quickmt-vs-libretranslate_files/libs/quarto-html/quarto.js" type="module"></script>
<script src="quickmt-vs-libretranslate_files/libs/quarto-html/tabsets/tabsets.js" type="module"></script>
<script src="quickmt-vs-libretranslate_files/libs/quarto-html/axe/axe-check.js" type="module"></script>
<script src="quickmt-vs-libretranslate_files/libs/quarto-html/popper.min.js"></script>
<script src="quickmt-vs-libretranslate_files/libs/quarto-html/tippy.umd.min.js"></script>
<script src="quickmt-vs-libretranslate_files/libs/quarto-html/anchor.min.js"></script>
<link href="quickmt-vs-libretranslate_files/libs/quarto-html/tippy.css" rel="stylesheet">
<link href="quickmt-vs-libretranslate_files/libs/quarto-html/quarto-syntax-highlighting-7b89279ff1a6dce999919e0e67d4d9ec.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="quickmt-vs-libretranslate_files/libs/bootstrap/bootstrap.min.js"></script>
<link href="quickmt-vs-libretranslate_files/libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="quickmt-vs-libretranslate_files/libs/bootstrap/bootstrap-607bf762899387bf4c6794576fa88f01.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">
<script src="https://cdn.jsdelivr.net/npm/requirejs@2.3.6/require.min.js" integrity="sha384-c9c+LnTbwQ3aujuU7ULEPVvgLs+Fn6fJUvIGTsuu1ZcCf11fiEubah0ttpca4ntM sha384-6V1/AdqZRWk1KAlWbKBlGhN7VG4iE/yAZcO6NZPMF8od0vukrvr0tg4qY6NSrItx" crossorigin="anonymous"></script>
<script type="application/javascript">define('jquery', [],function() {return window.jQuery;})</script>
<script type="text/javascript">
window.PlotlyConfig = {MathJaxConfig: 'local'};
if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}
</script>
<script type="module">import "https://cdn.plot.ly/plotly-3.2.0.min"</script>
</head>
<body class="fullcontent quarto-light">
<div id="quarto-content" class="page-columns page-rows-contents page-layout-article">
<main class="content" id="quarto-document-content">
<header id="title-block-header" class="quarto-title-block default">
<div class="quarto-title">
<h1 class="title">quickmt vs.&nbsp;libretranslate</h1>
</div>
<div class="quarto-title-meta">
</div>
</header>
<p><code>libretranslate</code> is a popular (<a href="https://github.com/LibreTranslate/LibreTranslate">13k stars</a> on Github!) library enabling users to self-host (or do offline) translation for many <a href="https://www.argosopentech.com/argospm/index/">different languages</a>. The neural machine translation component of the system is provided by <a href="https://github.com/argosopentech/argos-translate"><code>argos translate</code></a> (5k stars on Github). <code>libretranslate</code> and <code>argostranslate</code> are both quite easy to get up and running with and the <code>libretranslate</code> user interface is quite slick but neither of these libraries are <em>quick</em>.</p>
<p>In this short article we examine the quality and speed of <code>libretranslate</code> in comparison with <code>quickmt</code>.</p>
<p>In all of the experiments we are translating the <a href="https://huggingface.co/datasets/Muennighoff/flores200/tree/main"><code>flores-devtest</code></a> test set. This is a popular general-domain machine translation test set for evaluation between any of 200 different languages including 1012 segments (sentences or short paragraphs).</p>
<p>All of the <code>quickmt</code> models are available on <a href="https://huggingface.co/collections/quickmt/quickmt-models">Huggingface</a>, and the <code>libretranslate</code> models are available from <a href="https://www.argosopentech.com/argospm/index/"><code>argos translate</code></a>.</p>
<p><code>libretranslate</code> and <code>quickmt</code> were both configured to run on an Nvidia RTX 4070s with batch size 32, and <code>libretranslate</code> was launched withe the following command:</p>
<pre class="{bash}"><code>ARGOS_DEVICE_TYPE=cuda libretranslate --load-only pt,de,fa,cs,tr,es,pl,bn,vi,fr,it,id,hu,th,lv,ko,ar,en,zh,ro,hi,he,ja,da,el,ur,ru </code></pre>
<section id="speed-comparison" class="level2">
<h2 class="anchored" data-anchor-id="speed-comparison">Speed Comparison</h2>
<p><code>quickmt</code> is considerably faster than <code>libretranslate</code>:</p>
<div id="f635ea6a" class="cell" data-execution_count="1">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> pandas <span class="im">as</span> pd</span>
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> plotly <span class="im">import</span> express <span class="im">as</span> px</span>
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a>df_data <span class="op">=</span> {<span class="st">'index'</span>: {<span class="dv">0</span>: <span class="dv">0</span>, <span class="dv">1</span>: <span class="dv">1</span>, <span class="dv">2</span>: <span class="dv">2</span>, <span class="dv">3</span>: <span class="dv">3</span>, <span class="dv">4</span>: <span class="dv">4</span>, <span class="dv">5</span>: <span class="dv">5</span>, <span class="dv">6</span>: <span class="dv">6</span>, <span class="dv">7</span>: <span class="dv">7</span>, <span class="dv">8</span>: <span class="dv">8</span>, <span class="dv">9</span>: <span class="dv">9</span>, <span class="dv">10</span>: <span class="dv">10</span>, <span class="dv">11</span>: <span class="dv">11</span>, <span class="dv">12</span>: <span class="dv">12</span>, <span class="dv">13</span>: <span class="dv">13</span>, <span class="dv">14</span>: <span class="dv">14</span>, <span class="dv">15</span>: <span class="dv">15</span>, <span class="dv">16</span>: <span class="dv">16</span>, <span class="dv">17</span>: <span class="dv">17</span>, <span class="dv">18</span>: <span class="dv">18</span>, <span class="dv">19</span>: <span class="dv">19</span>, <span class="dv">20</span>: <span class="dv">20</span>, <span class="dv">21</span>: <span class="dv">21</span>, <span class="dv">22</span>: <span class="dv">22</span>, <span class="dv">23</span>: <span class="dv">23</span>, <span class="dv">24</span>: <span class="dv">24</span>, <span class="dv">25</span>: <span class="dv">25</span>, <span class="dv">26</span>: <span class="dv">26</span>, <span class="dv">27</span>: <span class="dv">27</span>, <span class="dv">28</span>: <span class="dv">28</span>, <span class="dv">29</span>: <span class="dv">29</span>, <span class="dv">30</span>: <span class="dv">30</span>, <span class="dv">31</span>: <span class="dv">31</span>, <span class="dv">32</span>: <span class="dv">32</span>, <span class="dv">33</span>: <span class="dv">33</span>, <span class="dv">34</span>: <span class="dv">34</span>, <span class="dv">35</span>: <span class="dv">35</span>, <span class="dv">36</span>: <span class="dv">36</span>, <span class="dv">37</span>: <span class="dv">37</span>, <span class="dv">38</span>: <span class="dv">38</span>, <span class="dv">39</span>: <span class="dv">39</span>, <span class="dv">40</span>: <span class="dv">40</span>, <span class="dv">41</span>: <span class="dv">41</span>, <span class="dv">42</span>: <span class="dv">42</span>, <span class="dv">43</span>: <span class="dv">43</span>, <span class="dv">44</span>: <span class="dv">44</span>, <span class="dv">45</span>: <span class="dv">45</span>, <span class="dv">46</span>: <span class="dv">46</span>, <span class="dv">47</span>: <span class="dv">47</span>, <span class="dv">48</span>: <span class="dv">48</span>, <span class="dv">49</span>: <span class="dv">49</span>, <span class="dv">50</span>: <span class="dv">50</span>, <span class="dv">51</span>: <span class="dv">51</span>, <span class="dv">52</span>: <span class="dv">0</span>, <span class="dv">53</span>: <span class="dv">1</span>, <span class="dv">54</span>: <span class="dv">2</span>, <span class="dv">55</span>: <span class="dv">3</span>, <span class="dv">56</span>: <span class="dv">4</span>, <span class="dv">57</span>: <span class="dv">5</span>, <span class="dv">58</span>: <span class="dv">6</span>, <span class="dv">59</span>: <span class="dv">7</span>, <span class="dv">60</span>: <span class="dv">8</span>, <span class="dv">61</span>: <span class="dv">9</span>, <span class="dv">62</span>: <span class="dv">10</span>, <span class="dv">63</span>: <span class="dv">11</span>, <span class="dv">64</span>: <span class="dv">12</span>, <span class="dv">65</span>: <span class="dv">13</span>, <span class="dv">66</span>: <span class="dv">14</span>, <span class="dv">67</span>: <span class="dv">15</span>, <span class="dv">68</span>: <span class="dv">16</span>, <span class="dv">69</span>: <span class="dv">17</span>, <span class="dv">70</span>: <span class="dv">18</span>, <span class="dv">71</span>: <span class="dv">19</span>, <span class="dv">72</span>: <span class="dv">20</span>, <span class="dv">73</span>: <span class="dv">21</span>, <span class="dv">74</span>: <span class="dv">22</span>, <span class="dv">75</span>: <span class="dv">23</span>, <span class="dv">76</span>: <span class="dv">24</span>, <span class="dv">77</span>: <span class="dv">25</span>, <span class="dv">78</span>: <span class="dv">26</span>, <span class="dv">79</span>: <span class="dv">27</span>, <span class="dv">80</span>: <span class="dv">28</span>, <span class="dv">81</span>: <span class="dv">29</span>, <span class="dv">82</span>: <span class="dv">30</span>, <span class="dv">83</span>: <span class="dv">31</span>, <span class="dv">84</span>: <span class="dv">32</span>, <span class="dv">85</span>: <span class="dv">33</span>, <span class="dv">86</span>: <span class="dv">34</span>, <span class="dv">87</span>: <span class="dv">35</span>, <span class="dv">88</span>: <span class="dv">36</span>, <span class="dv">89</span>: <span class="dv">37</span>, <span class="dv">90</span>: <span class="dv">38</span>, <span class="dv">91</span>: <span class="dv">39</span>, <span class="dv">92</span>: <span class="dv">40</span>, <span class="dv">93</span>: <span class="dv">41</span>, <span class="dv">94</span>: <span class="dv">42</span>, <span class="dv">95</span>: <span class="dv">43</span>, <span class="dv">96</span>: <span class="dv">44</span>, <span class="dv">97</span>: <span class="dv">45</span>, <span class="dv">98</span>: <span class="dv">46</span>, <span class="dv">99</span>: <span class="dv">47</span>, <span class="dv">100</span>: <span class="dv">48</span>, <span class="dv">101</span>: <span class="dv">49</span>, <span class="dv">102</span>: <span class="dv">50</span>}, <span class="st">'time'</span>: {<span class="dv">0</span>: <span class="fl">1.2328033447</span>, <span class="dv">1</span>: <span class="fl">1.7316634655</span>, <span class="dv">2</span>: <span class="fl">1.1428749561</span>, <span class="dv">3</span>: <span class="fl">1.2016947269</span>, <span class="dv">4</span>: <span class="fl">1.1951999664</span>, <span class="dv">5</span>: <span class="fl">1.2683308125</span>, <span class="dv">6</span>: <span class="fl">1.0825681686</span>, <span class="dv">7</span>: <span class="fl">1.2144613266</span>, <span class="dv">8</span>: <span class="fl">1.2252991199</span>, <span class="dv">9</span>: <span class="fl">1.2541656494</span>, <span class="dv">10</span>: <span class="fl">1.1193022728</span>, <span class="dv">11</span>: <span class="fl">1.0900781155</span>, <span class="dv">12</span>: <span class="fl">1.0908696651</span>, <span class="dv">13</span>: <span class="fl">1.1799731255</span>, <span class="dv">14</span>: <span class="fl">1.1092069149000001</span>, <span class="dv">15</span>: <span class="fl">1.3009929657</span>, <span class="dv">16</span>: <span class="fl">1.1924922466</span>, <span class="dv">17</span>: <span class="fl">1.321138382</span>, <span class="dv">18</span>: <span class="fl">1.404417038</span>, <span class="dv">19</span>: <span class="fl">1.2471661568</span>, <span class="dv">20</span>: <span class="fl">1.0757014751</span>, <span class="dv">21</span>: <span class="fl">1.1705803871</span>, <span class="dv">22</span>: <span class="fl">1.1219103336</span>, <span class="dv">23</span>: <span class="fl">1.2237184048</span>, <span class="dv">24</span>: <span class="fl">1.1639976501</span>, <span class="dv">25</span>: <span class="fl">1.1322209834999999</span>, <span class="dv">26</span>: <span class="fl">1.0975823402</span>, <span class="dv">27</span>: <span class="fl">1.0930819511</span>, <span class="dv">28</span>: <span class="fl">1.121234417</span>, <span class="dv">29</span>: <span class="fl">1.4847657681</span>, <span class="dv">30</span>: <span class="fl">1.1623337269</span>, <span class="dv">31</span>: <span class="fl">1.2030494213</span>, <span class="dv">32</span>: <span class="fl">1.2224872112</span>, <span class="dv">33</span>: <span class="fl">1.2217383385</span>, <span class="dv">34</span>: <span class="fl">1.0560030937</span>, <span class="dv">35</span>: <span class="fl">0.9569888115</span>, <span class="dv">36</span>: <span class="fl">1.1156563759</span>, <span class="dv">37</span>: <span class="fl">1.2865319252</span>, <span class="dv">38</span>: <span class="fl">1.1470780373</span>, <span class="dv">39</span>: <span class="fl">1.2559833527</span>, <span class="dv">40</span>: <span class="fl">1.2100195885</span>, <span class="dv">41</span>: <span class="fl">1.1664426327</span>, <span class="dv">42</span>: <span class="fl">1.2318534851</span>, <span class="dv">43</span>: <span class="fl">1.2992098331</span>, <span class="dv">44</span>: <span class="fl">1.4252364635</span>, <span class="dv">45</span>: <span class="fl">2.0381476879</span>, <span class="dv">46</span>: <span class="fl">1.3772408962</span>, <span class="dv">47</span>: <span class="fl">1.0434963703</span>, <span class="dv">48</span>: <span class="fl">1.0937628746</span>, <span class="dv">49</span>: <span class="fl">1.1264841557</span>, <span class="dv">50</span>: <span class="fl">1.1113624573</span>, <span class="dv">51</span>: <span class="fl">1.2100260258</span>, <span class="dv">52</span>: <span class="fl">52.0006783009</span>, <span class="dv">53</span>: <span class="fl">47.4294705391</span>, <span class="dv">54</span>: <span class="fl">48.9193472862</span>, <span class="dv">55</span>: <span class="fl">51.4381048679</span>, <span class="dv">56</span>: <span class="fl">51.9525535107</span>, <span class="dv">57</span>: <span class="fl">51.6938693523</span>, <span class="dv">58</span>: <span class="fl">50.0207080841</span>, <span class="dv">59</span>: <span class="fl">54.0675241947</span>, <span class="dv">60</span>: <span class="fl">50.6869974136</span>, <span class="dv">61</span>: <span class="fl">47.6485154629</span>, <span class="dv">62</span>: <span class="fl">50.6348552704</span>, <span class="dv">63</span>: <span class="fl">48.9413232803</span>, <span class="dv">64</span>: <span class="fl">49.865562439</span>, <span class="dv">65</span>: <span class="fl">53.2831482887</span>, <span class="dv">66</span>: <span class="fl">57.5475423336</span>, <span class="dv">67</span>: <span class="fl">52.2292399406</span>, <span class="dv">68</span>: <span class="fl">56.8881397247</span>, <span class="dv">69</span>: <span class="fl">51.0274362564</span>, <span class="dv">70</span>: <span class="fl">53.1056745052</span>, <span class="dv">71</span>: <span class="fl">50.9684550762</span>, <span class="dv">72</span>: <span class="fl">52.7217481136</span>, <span class="dv">73</span>: <span class="fl">48.5589823723</span>, <span class="dv">74</span>: <span class="fl">51.3466742039</span>, <span class="dv">75</span>: <span class="fl">49.7405352592</span>, <span class="dv">76</span>: <span class="fl">45.0998287201</span>, <span class="dv">77</span>: <span class="fl">49.0215342045</span>, <span class="dv">78</span>: <span class="fl">45.2211201191</span>, <span class="dv">79</span>: <span class="fl">53.4577124119</span>, <span class="dv">80</span>: <span class="fl">50.652441263200004</span>, <span class="dv">81</span>: <span class="fl">50.4762699604</span>, <span class="dv">82</span>: <span class="fl">51.7814288139</span>, <span class="dv">83</span>: <span class="fl">51.7295336723</span>, <span class="dv">84</span>: <span class="fl">52.9573953152</span>, <span class="dv">85</span>: <span class="fl">49.7250933647</span>, <span class="dv">86</span>: <span class="fl">46.9384317398</span>, <span class="dv">87</span>: <span class="fl">50.8940937519</span>, <span class="dv">88</span>: <span class="fl">56.0273029804</span>, <span class="dv">89</span>: <span class="fl">47.4176292419</span>, <span class="dv">90</span>: <span class="fl">53.34983325</span>, <span class="dv">91</span>: <span class="fl">51.4188220501</span>, <span class="dv">92</span>: <span class="fl">52.6518859863</span>, <span class="dv">93</span>: <span class="fl">52.7884061337</span>, <span class="dv">94</span>: <span class="fl">76.0510730743</span>, <span class="dv">95</span>: <span class="fl">51.4100003242</span>, <span class="dv">96</span>: <span class="fl">58.9121079445</span>, <span class="dv">97</span>: <span class="fl">49.9179136753</span>, <span class="dv">98</span>: <span class="fl">48.0125050545</span>, <span class="dv">99</span>: <span class="fl">47.6589736938</span>, <span class="dv">100</span>: <span class="fl">51.1337733269</span>, <span class="dv">101</span>: <span class="fl">55.2462155819</span>, <span class="dv">102</span>: <span class="fl">58.4858531952</span>}, <span class="st">'src_lang'</span>: {<span class="dv">0</span>: <span class="st">'ko'</span>, <span class="dv">1</span>: <span class="st">'en'</span>, <span class="dv">2</span>: <span class="st">'it'</span>, <span class="dv">3</span>: <span class="st">'en'</span>, <span class="dv">4</span>: <span class="st">'es'</span>, <span class="dv">5</span>: <span class="st">'en'</span>, <span class="dv">6</span>: <span class="st">'hu'</span>, <span class="dv">7</span>: <span class="st">'en'</span>, <span class="dv">8</span>: <span class="st">'zh'</span>, <span class="dv">9</span>: <span class="st">'en'</span>, <span class="dv">10</span>: <span class="st">'ar'</span>, <span class="dv">11</span>: <span class="st">'en'</span>, <span class="dv">12</span>: <span class="st">'hi'</span>, <span class="dv">13</span>: <span class="st">'en'</span>, <span class="dv">14</span>: <span class="st">'vi'</span>, <span class="dv">15</span>: <span class="st">'en'</span>, <span class="dv">16</span>: <span class="st">'ro'</span>, <span class="dv">17</span>: <span class="st">'en'</span>, <span class="dv">18</span>: <span class="st">'pl'</span>, <span class="dv">19</span>: <span class="st">'en'</span>, <span class="dv">20</span>: <span class="st">'pt'</span>, <span class="dv">21</span>: <span class="st">'en'</span>, <span class="dv">22</span>: <span class="st">'de'</span>, <span class="dv">23</span>: <span class="st">'en'</span>, <span class="dv">24</span>: <span class="st">'he'</span>, <span class="dv">25</span>: <span class="st">'en'</span>, <span class="dv">26</span>: <span class="st">'tr'</span>, <span class="dv">27</span>: <span class="st">'en'</span>, <span class="dv">28</span>: <span class="st">'ja'</span>, <span class="dv">29</span>: <span class="st">'en'</span>, <span class="dv">30</span>: <span class="st">'cs'</span>, <span class="dv">31</span>: <span class="st">'en'</span>, <span class="dv">32</span>: <span class="st">'lv'</span>, <span class="dv">33</span>: <span class="st">'en'</span>, <span class="dv">34</span>: <span class="st">'id'</span>, <span class="dv">35</span>: <span class="st">'en'</span>, <span class="dv">36</span>: <span class="st">'el'</span>, <span class="dv">37</span>: <span class="st">'en'</span>, <span class="dv">38</span>: <span class="st">'th'</span>, <span class="dv">39</span>: <span class="st">'en'</span>, <span class="dv">40</span>: <span class="st">'da'</span>, <span class="dv">41</span>: <span class="st">'en'</span>, <span class="dv">42</span>: <span class="st">'ru'</span>, <span class="dv">43</span>: <span class="st">'en'</span>, <span class="dv">44</span>: <span class="st">'fr'</span>, <span class="dv">45</span>: <span class="st">'en'</span>, <span class="dv">46</span>: <span class="st">'bn'</span>, <span class="dv">47</span>: <span class="st">'en'</span>, <span class="dv">48</span>: <span class="st">'fa'</span>, <span class="dv">49</span>: <span class="st">'en'</span>, <span class="dv">50</span>: <span class="st">'ur'</span>, <span class="dv">51</span>: <span class="st">'en'</span>, <span class="dv">52</span>: <span class="st">'ko'</span>, <span class="dv">53</span>: <span class="st">'en'</span>, <span class="dv">54</span>: <span class="st">'it'</span>, <span class="dv">55</span>: <span class="st">'en'</span>, <span class="dv">56</span>: <span class="st">'es'</span>, <span class="dv">57</span>: <span class="st">'en'</span>, <span class="dv">58</span>: <span class="st">'hu'</span>, <span class="dv">59</span>: <span class="st">'en'</span>, <span class="dv">60</span>: <span class="st">'zh'</span>, <span class="dv">61</span>: <span class="st">'en'</span>, <span class="dv">62</span>: <span class="st">'ar'</span>, <span class="dv">63</span>: <span class="st">'en'</span>, <span class="dv">64</span>: <span class="st">'hi'</span>, <span class="dv">65</span>: <span class="st">'en'</span>, <span class="dv">66</span>: <span class="st">'en'</span>, <span class="dv">67</span>: <span class="st">'ro'</span>, <span class="dv">68</span>: <span class="st">'en'</span>, <span class="dv">69</span>: <span class="st">'pl'</span>, <span class="dv">70</span>: <span class="st">'en'</span>, <span class="dv">71</span>: <span class="st">'pt'</span>, <span class="dv">72</span>: <span class="st">'en'</span>, <span class="dv">73</span>: <span class="st">'de'</span>, <span class="dv">74</span>: <span class="st">'en'</span>, <span class="dv">75</span>: <span class="st">'he'</span>, <span class="dv">76</span>: <span class="st">'en'</span>, <span class="dv">77</span>: <span class="st">'tr'</span>, <span class="dv">78</span>: <span class="st">'en'</span>, <span class="dv">79</span>: <span class="st">'ja'</span>, <span class="dv">80</span>: <span class="st">'en'</span>, <span class="dv">81</span>: <span class="st">'cs'</span>, <span class="dv">82</span>: <span class="st">'en'</span>, <span class="dv">83</span>: <span class="st">'lv'</span>, <span class="dv">84</span>: <span class="st">'en'</span>, <span class="dv">85</span>: <span class="st">'id'</span>, <span class="dv">86</span>: <span class="st">'en'</span>, <span class="dv">87</span>: <span class="st">'el'</span>, <span class="dv">88</span>: <span class="st">'en'</span>, <span class="dv">89</span>: <span class="st">'th'</span>, <span class="dv">90</span>: <span class="st">'en'</span>, <span class="dv">91</span>: <span class="st">'da'</span>, <span class="dv">92</span>: <span class="st">'en'</span>, <span class="dv">93</span>: <span class="st">'ru'</span>, <span class="dv">94</span>: <span class="st">'en'</span>, <span class="dv">95</span>: <span class="st">'fr'</span>, <span class="dv">96</span>: <span class="st">'en'</span>, <span class="dv">97</span>: <span class="st">'bn'</span>, <span class="dv">98</span>: <span class="st">'en'</span>, <span class="dv">99</span>: <span class="st">'fa'</span>, <span class="dv">100</span>: <span class="st">'en'</span>, <span class="dv">101</span>: <span class="st">'ur'</span>, <span class="dv">102</span>: <span class="st">'en'</span>}, <span class="st">'tgt_lang'</span>: {<span class="dv">0</span>: <span class="st">'en'</span>, <span class="dv">1</span>: <span class="st">'ko'</span>, <span class="dv">2</span>: <span class="st">'en'</span>, <span class="dv">3</span>: <span class="st">'it'</span>, <span class="dv">4</span>: <span class="st">'en'</span>, <span class="dv">5</span>: <span class="st">'es'</span>, <span class="dv">6</span>: <span class="st">'en'</span>, <span class="dv">7</span>: <span class="st">'hu'</span>, <span class="dv">8</span>: <span class="st">'en'</span>, <span class="dv">9</span>: <span class="st">'zh'</span>, <span class="dv">10</span>: <span class="st">'en'</span>, <span class="dv">11</span>: <span class="st">'ar'</span>, <span class="dv">12</span>: <span class="st">'en'</span>, <span class="dv">13</span>: <span class="st">'hi'</span>, <span class="dv">14</span>: <span class="st">'en'</span>, <span class="dv">15</span>: <span class="st">'vi'</span>, <span class="dv">16</span>: <span class="st">'en'</span>, <span class="dv">17</span>: <span class="st">'ro'</span>, <span class="dv">18</span>: <span class="st">'en'</span>, <span class="dv">19</span>: <span class="st">'pl'</span>, <span class="dv">20</span>: <span class="st">'en'</span>, <span class="dv">21</span>: <span class="st">'pt'</span>, <span class="dv">22</span>: <span class="st">'en'</span>, <span class="dv">23</span>: <span class="st">'de'</span>, <span class="dv">24</span>: <span class="st">'en'</span>, <span class="dv">25</span>: <span class="st">'he'</span>, <span class="dv">26</span>: <span class="st">'en'</span>, <span class="dv">27</span>: <span class="st">'tr'</span>, <span class="dv">28</span>: <span class="st">'en'</span>, <span class="dv">29</span>: <span class="st">'ja'</span>, <span class="dv">30</span>: <span class="st">'en'</span>, <span class="dv">31</span>: <span class="st">'cs'</span>, <span class="dv">32</span>: <span class="st">'en'</span>, <span class="dv">33</span>: <span class="st">'lv'</span>, <span class="dv">34</span>: <span class="st">'en'</span>, <span class="dv">35</span>: <span class="st">'id'</span>, <span class="dv">36</span>: <span class="st">'en'</span>, <span class="dv">37</span>: <span class="st">'el'</span>, <span class="dv">38</span>: <span class="st">'en'</span>, <span class="dv">39</span>: <span class="st">'th'</span>, <span class="dv">40</span>: <span class="st">'en'</span>, <span class="dv">41</span>: <span class="st">'da'</span>, <span class="dv">42</span>: <span class="st">'en'</span>, <span class="dv">43</span>: <span class="st">'ru'</span>, <span class="dv">44</span>: <span class="st">'en'</span>, <span class="dv">45</span>: <span class="st">'fr'</span>, <span class="dv">46</span>: <span class="st">'en'</span>, <span class="dv">47</span>: <span class="st">'bn'</span>, <span class="dv">48</span>: <span class="st">'en'</span>, <span class="dv">49</span>: <span class="st">'fa'</span>, <span class="dv">50</span>: <span class="st">'en'</span>, <span class="dv">51</span>: <span class="st">'ur'</span>, <span class="dv">52</span>: <span class="st">'en'</span>, <span class="dv">53</span>: <span class="st">'ko'</span>, <span class="dv">54</span>: <span class="st">'en'</span>, <span class="dv">55</span>: <span class="st">'it'</span>, <span class="dv">56</span>: <span class="st">'en'</span>, <span class="dv">57</span>: <span class="st">'es'</span>, <span class="dv">58</span>: <span class="st">'en'</span>, <span class="dv">59</span>: <span class="st">'hu'</span>, <span class="dv">60</span>: <span class="st">'en'</span>, <span class="dv">61</span>: <span class="st">'zh'</span>, <span class="dv">62</span>: <span class="st">'en'</span>, <span class="dv">63</span>: <span class="st">'ar'</span>, <span class="dv">64</span>: <span class="st">'en'</span>, <span class="dv">65</span>: <span class="st">'hi'</span>, <span class="dv">66</span>: <span class="st">'vi'</span>, <span class="dv">67</span>: <span class="st">'en'</span>, <span class="dv">68</span>: <span class="st">'ro'</span>, <span class="dv">69</span>: <span class="st">'en'</span>, <span class="dv">70</span>: <span class="st">'pl'</span>, <span class="dv">71</span>: <span class="st">'en'</span>, <span class="dv">72</span>: <span class="st">'pt'</span>, <span class="dv">73</span>: <span class="st">'en'</span>, <span class="dv">74</span>: <span class="st">'de'</span>, <span class="dv">75</span>: <span class="st">'en'</span>, <span class="dv">76</span>: <span class="st">'he'</span>, <span class="dv">77</span>: <span class="st">'en'</span>, <span class="dv">78</span>: <span class="st">'tr'</span>, <span class="dv">79</span>: <span class="st">'en'</span>, <span class="dv">80</span>: <span class="st">'ja'</span>, <span class="dv">81</span>: <span class="st">'en'</span>, <span class="dv">82</span>: <span class="st">'cs'</span>, <span class="dv">83</span>: <span class="st">'en'</span>, <span class="dv">84</span>: <span class="st">'lv'</span>, <span class="dv">85</span>: <span class="st">'en'</span>, <span class="dv">86</span>: <span class="st">'id'</span>, <span class="dv">87</span>: <span class="st">'en'</span>, <span class="dv">88</span>: <span class="st">'el'</span>, <span class="dv">89</span>: <span class="st">'en'</span>, <span class="dv">90</span>: <span class="st">'th'</span>, <span class="dv">91</span>: <span class="st">'en'</span>, <span class="dv">92</span>: <span class="st">'da'</span>, <span class="dv">93</span>: <span class="st">'en'</span>, <span class="dv">94</span>: <span class="st">'ru'</span>, <span class="dv">95</span>: <span class="st">'en'</span>, <span class="dv">96</span>: <span class="st">'fr'</span>, <span class="dv">97</span>: <span class="st">'en'</span>, <span class="dv">98</span>: <span class="st">'bn'</span>, <span class="dv">99</span>: <span class="st">'en'</span>, <span class="dv">100</span>: <span class="st">'fa'</span>, <span class="dv">101</span>: <span class="st">'en'</span>, <span class="dv">102</span>: <span class="st">'ur'</span>}, <span class="st">'bleu'</span>: {<span class="dv">0</span>: <span class="fl">27.08</span>, <span class="dv">1</span>: <span class="fl">14.93</span>, <span class="dv">2</span>: <span class="fl">32.09</span>, <span class="dv">3</span>: <span class="fl">30.52</span>, <span class="dv">4</span>: <span class="fl">28.63</span>, <span class="dv">5</span>: <span class="fl">26.64</span>, <span class="dv">6</span>: <span class="fl">35.0</span>, <span class="dv">7</span>: <span class="fl">28.71</span>, <span class="dv">8</span>: <span class="fl">28.74</span>, <span class="dv">9</span>: <span class="fl">2.35</span>, <span class="dv">10</span>: <span class="fl">42.88</span>, <span class="dv">11</span>: <span class="fl">29.6</span>, <span class="dv">12</span>: <span class="fl">39.84</span>, <span class="dv">13</span>: <span class="fl">35.94</span>, <span class="dv">14</span>: <span class="fl">37.57</span>, <span class="dv">15</span>: <span class="fl">43.69</span>, <span class="dv">16</span>: <span class="fl">44.93</span>, <span class="dv">17</span>: <span class="fl">42.29</span>, <span class="dv">18</span>: <span class="fl">27.46</span>, <span class="dv">19</span>: <span class="fl">21.76</span>, <span class="dv">20</span>: <span class="fl">48.68</span>, <span class="dv">21</span>: <span class="fl">50.51</span>, <span class="dv">22</span>: <span class="fl">44.16</span>, <span class="dv">23</span>: <span class="fl">40.15</span>, <span class="dv">24</span>: <span class="fl">45.01</span>, <span class="dv">25</span>: <span class="fl">34.32</span>, <span class="dv">26</span>: <span class="fl">39.46</span>, <span class="dv">27</span>: <span class="fl">32.75</span>, <span class="dv">28</span>: <span class="fl">27.85</span>, <span class="dv">29</span>: <span class="fl">3.59</span>, <span class="dv">30</span>: <span class="fl">39.61</span>, <span class="dv">31</span>: <span class="fl">33.73</span>, <span class="dv">32</span>: <span class="fl">35.3</span>, <span class="dv">33</span>: <span class="fl">31.47</span>, <span class="dv">34</span>: <span class="fl">44.51</span>, <span class="dv">35</span>: <span class="fl">48.68</span>, <span class="dv">36</span>: <span class="fl">35.45</span>, <span class="dv">37</span>: <span class="fl">28.86</span>, <span class="dv">38</span>: <span class="fl">29.32</span>, <span class="dv">39</span>: <span class="fl">9.56</span>, <span class="dv">40</span>: <span class="fl">49.02</span>, <span class="dv">41</span>: <span class="fl">46.61</span>, <span class="dv">42</span>: <span class="fl">34.69</span>, <span class="dv">43</span>: <span class="fl">32.28</span>, <span class="dv">44</span>: <span class="fl">44.35</span>, <span class="dv">45</span>: <span class="fl">50.05</span>, <span class="dv">46</span>: <span class="fl">32.9</span>, <span class="dv">47</span>: <span class="fl">19.26</span>, <span class="dv">48</span>: <span class="fl">37.55</span>, <span class="dv">49</span>: <span class="fl">26.17</span>, <span class="dv">50</span>: <span class="fl">31.48</span>, <span class="dv">51</span>: <span class="fl">20.81</span>, <span class="dv">52</span>: <span class="fl">14.13</span>, <span class="dv">53</span>: <span class="fl">6.11</span>, <span class="dv">54</span>: <span class="fl">28.58</span>, <span class="dv">55</span>: <span class="fl">25.9</span>, <span class="dv">56</span>: <span class="fl">25.63</span>, <span class="dv">57</span>: <span class="fl">23.43</span>, <span class="dv">58</span>: <span class="fl">29.71</span>, <span class="dv">59</span>: <span class="fl">24.3</span>, <span class="dv">60</span>: <span class="fl">20.51</span>, <span class="dv">61</span>: <span class="fl">1.1</span>, <span class="dv">62</span>: <span class="fl">29.36</span>, <span class="dv">63</span>: <span class="fl">17.24</span>, <span class="dv">64</span>: <span class="fl">26.72</span>, <span class="dv">65</span>: <span class="fl">28.76</span>, <span class="dv">66</span>: <span class="fl">33.8</span>, <span class="dv">67</span>: <span class="fl">39.71</span>, <span class="dv">68</span>: <span class="fl">33.06</span>, <span class="dv">69</span>: <span class="fl">26.18</span>, <span class="dv">70</span>: <span class="fl">18.82</span>, <span class="dv">71</span>: <span class="fl">46.33</span>, <span class="dv">72</span>: <span class="fl">45.83</span>, <span class="dv">73</span>: <span class="fl">36.3</span>, <span class="dv">74</span>: <span class="fl">31.75</span>, <span class="dv">75</span>: <span class="fl">32.19</span>, <span class="dv">76</span>: <span class="fl">24.37</span>, <span class="dv">77</span>: <span class="fl">23.9</span>, <span class="dv">78</span>: <span class="fl">18.75</span>, <span class="dv">79</span>: <span class="fl">13.55</span>, <span class="dv">80</span>: <span class="fl">1.85</span>, <span class="dv">81</span>: <span class="fl">35.05</span>, <span class="dv">82</span>: <span class="fl">28.51</span>, <span class="dv">83</span>: <span class="fl">31.1</span>, <span class="dv">84</span>: <span class="fl">29.67</span>, <span class="dv">85</span>: <span class="fl">32.65</span>, <span class="dv">86</span>: <span class="fl">37.75</span>, <span class="dv">87</span>: <span class="fl">31.4</span>, <span class="dv">88</span>: <span class="fl">26.34</span>, <span class="dv">89</span>: <span class="fl">15.33</span>, <span class="dv">90</span>: <span class="fl">1.05</span>, <span class="dv">91</span>: <span class="fl">44.64</span>, <span class="dv">92</span>: <span class="fl">42.43</span>, <span class="dv">93</span>: <span class="fl">36.95</span>, <span class="dv">94</span>: <span class="fl">32.07</span>, <span class="dv">95</span>: <span class="fl">42.0</span>, <span class="dv">96</span>: <span class="fl">47.23</span>, <span class="dv">97</span>: <span class="fl">15.84</span>, <span class="dv">98</span>: <span class="fl">7.92</span>, <span class="dv">99</span>: <span class="fl">26.25</span>, <span class="dv">100</span>: <span class="fl">21.79</span>, <span class="dv">101</span>: <span class="fl">13.29</span>, <span class="dv">102</span>: <span class="fl">13.12</span>}, <span class="st">'chrf2'</span>: {<span class="dv">0</span>: <span class="fl">56.24</span>, <span class="dv">1</span>: <span class="fl">36.89</span>, <span class="dv">2</span>: <span class="fl">61.47</span>, <span class="dv">3</span>: <span class="fl">59.73</span>, <span class="dv">4</span>: <span class="fl">58.61</span>, <span class="dv">5</span>: <span class="fl">55.13</span>, <span class="dv">6</span>: <span class="fl">62.41</span>, <span class="dv">7</span>: <span class="fl">59.4</span>, <span class="dv">8</span>: <span class="fl">57.9</span>, <span class="dv">9</span>: <span class="fl">34.5</span>, <span class="dv">10</span>: <span class="fl">67.01</span>, <span class="dv">11</span>: <span class="fl">61.65</span>, <span class="dv">12</span>: <span class="fl">65.0</span>, <span class="dv">13</span>: <span class="fl">59.92</span>, <span class="dv">14</span>: <span class="fl">62.84</span>, <span class="dv">15</span>: <span class="fl">60.76</span>, <span class="dv">16</span>: <span class="fl">69.3</span>, <span class="dv">17</span>: <span class="fl">66.07</span>, <span class="dv">18</span>: <span class="fl">57.18</span>, <span class="dv">19</span>: <span class="fl">52.1</span>, <span class="dv">20</span>: <span class="fl">71.48</span>, <span class="dv">21</span>: <span class="fl">71.75</span>, <span class="dv">22</span>: <span class="fl">68.83</span>, <span class="dv">23</span>: <span class="fl">66.25</span>, <span class="dv">24</span>: <span class="fl">68.39</span>, <span class="dv">25</span>: <span class="fl">62.37</span>, <span class="dv">26</span>: <span class="fl">65.02</span>, <span class="dv">27</span>: <span class="fl">63.78</span>, <span class="dv">28</span>: <span class="fl">56.98</span>, <span class="dv">29</span>: <span class="fl">42.02</span>, <span class="dv">30</span>: <span class="fl">65.63</span>, <span class="dv">31</span>: <span class="fl">60.29</span>, <span class="dv">32</span>: <span class="fl">62.91</span>, <span class="dv">33</span>: <span class="fl">60.5</span>, <span class="dv">34</span>: <span class="fl">68.77</span>, <span class="dv">35</span>: <span class="fl">71.94</span>, <span class="dv">36</span>: <span class="fl">61.89</span>, <span class="dv">37</span>: <span class="fl">54.93</span>, <span class="dv">38</span>: <span class="fl">58.4</span>, <span class="dv">39</span>: <span class="fl">54.79</span>, <span class="dv">40</span>: <span class="fl">71.78</span>, <span class="dv">41</span>: <span class="fl">70.07</span>, <span class="dv">42</span>: <span class="fl">62.31</span>, <span class="dv">43</span>: <span class="fl">59.11</span>, <span class="dv">44</span>: <span class="fl">68.15</span>, <span class="dv">45</span>: <span class="fl">71.31</span>, <span class="dv">46</span>: <span class="fl">59.7</span>, <span class="dv">47</span>: <span class="fl">53.32</span>, <span class="dv">48</span>: <span class="fl">63.34</span>, <span class="dv">49</span>: <span class="fl">54.07</span>, <span class="dv">50</span>: <span class="fl">58.35</span>, <span class="dv">51</span>: <span class="fl">48.55</span>, <span class="dv">52</span>: <span class="fl">42.15</span>, <span class="dv">53</span>: <span class="fl">23.95</span>, <span class="dv">54</span>: <span class="fl">58.81</span>, <span class="dv">55</span>: <span class="fl">56.37</span>, <span class="dv">56</span>: <span class="fl">56.91</span>, <span class="dv">57</span>: <span class="fl">52.71</span>, <span class="dv">58</span>: <span class="fl">58.63</span>, <span class="dv">59</span>: <span class="fl">55.59</span>, <span class="dv">60</span>: <span class="fl">51.7</span>, <span class="dv">61</span>: <span class="fl">29.57</span>, <span class="dv">62</span>: <span class="fl">56.44</span>, <span class="dv">63</span>: <span class="fl">44.69</span>, <span class="dv">64</span>: <span class="fl">55.25</span>, <span class="dv">65</span>: <span class="fl">53.65</span>, <span class="dv">66</span>: <span class="fl">52.98</span>, <span class="dv">67</span>: <span class="fl">66.1</span>, <span class="dv">68</span>: <span class="fl">60.73</span>, <span class="dv">69</span>: <span class="fl">55.67</span>, <span class="dv">70</span>: <span class="fl">49.69</span>, <span class="dv">71</span>: <span class="fl">69.84</span>, <span class="dv">72</span>: <span class="fl">69.27</span>, <span class="dv">73</span>: <span class="fl">63.56</span>, <span class="dv">74</span>: <span class="fl">60.49</span>, <span class="dv">75</span>: <span class="fl">56.77</span>, <span class="dv">76</span>: <span class="fl">53.62</span>, <span class="dv">77</span>: <span class="fl">52.91</span>, <span class="dv">78</span>: <span class="fl">52.15</span>, <span class="dv">79</span>: <span class="fl">42.83</span>, <span class="dv">80</span>: <span class="fl">29.2</span>, <span class="dv">81</span>: <span class="fl">62.47</span>, <span class="dv">82</span>: <span class="fl">56.74</span>, <span class="dv">83</span>: <span class="fl">59.21</span>, <span class="dv">84</span>: <span class="fl">59.08</span>, <span class="dv">85</span>: <span class="fl">60.71</span>, <span class="dv">86</span>: <span class="fl">65.21</span>, <span class="dv">87</span>: <span class="fl">58.94</span>, <span class="dv">88</span>: <span class="fl">53.27</span>, <span class="dv">89</span>: <span class="fl">45.03</span>, <span class="dv">90</span>: <span class="fl">44.97</span>, <span class="dv">91</span>: <span class="fl">68.93</span>, <span class="dv">92</span>: <span class="fl">67.42</span>, <span class="dv">93</span>: <span class="fl">63.11</span>, <span class="dv">94</span>: <span class="fl">59.01</span>, <span class="dv">95</span>: <span class="fl">66.74</span>, <span class="dv">96</span>: <span class="fl">69.45</span>, <span class="dv">97</span>: <span class="fl">42.95</span>, <span class="dv">98</span>: <span class="fl">37.76</span>, <span class="dv">99</span>: <span class="fl">53.3</span>, <span class="dv">100</span>: <span class="fl">49.47</span>, <span class="dv">101</span>: <span class="fl">38.88</span>, <span class="dv">102</span>: <span class="fl">38.33</span>}, <span class="st">'comet22'</span>: {<span class="dv">0</span>: <span class="fl">86.01</span>, <span class="dv">1</span>: <span class="fl">87.05</span>, <span class="dv">2</span>: <span class="fl">87.29</span>, <span class="dv">3</span>: <span class="fl">87.54</span>, <span class="dv">4</span>: <span class="fl">86.1</span>, <span class="dv">5</span>: <span class="fl">85.15</span>, <span class="dv">6</span>: <span class="fl">87.76</span>, <span class="dv">7</span>: <span class="fl">88.01</span>, <span class="dv">8</span>: <span class="fl">86.34</span>, <span class="dv">9</span>: <span class="fl">85.3</span>, <span class="dv">10</span>: <span class="fl">87.4</span>, <span class="dv">11</span>: <span class="fl">86.32</span>, <span class="dv">12</span>: <span class="fl">88.76</span>, <span class="dv">13</span>: <span class="fl">79.05</span>, <span class="dv">14</span>: <span class="fl">87.2</span>, <span class="dv">15</span>: <span class="fl">87.52</span>, <span class="dv">16</span>: <span class="fl">89.31</span>, <span class="dv">17</span>: <span class="fl">89.67</span>, <span class="dv">18</span>: <span class="fl">85.04</span>, <span class="dv">19</span>: <span class="fl">87.14</span>, <span class="dv">20</span>: <span class="fl">89.09</span>, <span class="dv">21</span>: <span class="fl">89.23</span>, <span class="dv">22</span>: <span class="fl">88.9</span>, <span class="dv">23</span>: <span class="fl">86.9</span>, <span class="dv">24</span>: <span class="fl">88.31</span>, <span class="dv">25</span>: <span class="fl">87.91</span>, <span class="dv">26</span>: <span class="fl">88.99</span>, <span class="dv">27</span>: <span class="fl">89.42</span>, <span class="dv">28</span>: <span class="fl">87.24</span>, <span class="dv">29</span>: <span class="fl">89.06</span>, <span class="dv">30</span>: <span class="fl">88.17</span>, <span class="dv">31</span>: <span class="fl">88.77</span>, <span class="dv">32</span>: <span class="fl">86.98</span>, <span class="dv">33</span>: <span class="fl">86.22</span>, <span class="dv">34</span>: <span class="fl">89.24</span>, <span class="dv">35</span>: <span class="fl">91.02</span>, <span class="dv">36</span>: <span class="fl">87.29</span>, <span class="dv">37</span>: <span class="fl">88.85</span>, <span class="dv">38</span>: <span class="fl">87.15</span>, <span class="dv">39</span>: <span class="fl">84.52</span>, <span class="dv">40</span>: <span class="fl">90.0</span>, <span class="dv">41</span>: <span class="fl">89.49</span>, <span class="dv">42</span>: <span class="fl">85.96</span>, <span class="dv">43</span>: <span class="fl">87.76</span>, <span class="dv">44</span>: <span class="fl">88.76</span>, <span class="dv">45</span>: <span class="fl">86.99</span>, <span class="dv">46</span>: <span class="fl">86.99</span>, <span class="dv">47</span>: <span class="fl">84.68</span>, <span class="dv">48</span>: <span class="fl">87.76</span>, <span class="dv">49</span>: <span class="fl">85.79</span>, <span class="dv">50</span>: <span class="fl">84.25</span>, <span class="dv">51</span>: <span class="fl">77.67</span>, <span class="dv">52</span>: <span class="fl">72.46</span>, <span class="dv">53</span>: <span class="fl">71.64</span>, <span class="dv">54</span>: <span class="fl">85.24</span>, <span class="dv">55</span>: <span class="fl">83.68</span>, <span class="dv">56</span>: <span class="fl">84.79</span>, <span class="dv">57</span>: <span class="fl">82.75</span>, <span class="dv">58</span>: <span class="fl">85.8</span>, <span class="dv">59</span>: <span class="fl">83.99</span>, <span class="dv">60</span>: <span class="fl">83.07</span>, <span class="dv">61</span>: <span class="fl">83.15</span>, <span class="dv">62</span>: <span class="fl">80.94</span>, <span class="dv">63</span>: <span class="fl">76.66</span>, <span class="dv">64</span>: <span class="fl">82.34</span>, <span class="dv">65</span>: <span class="fl">73.22</span>, <span class="dv">66</span>: <span class="fl">83.76</span>, <span class="dv">67</span>: <span class="fl">87.97</span>, <span class="dv">68</span>: <span class="fl">86.77</span>, <span class="dv">69</span>: <span class="fl">84.12</span>, <span class="dv">70</span>: <span class="fl">84.76</span>, <span class="dv">71</span>: <span class="fl">88.57</span>, <span class="dv">72</span>: <span class="fl">87.99</span>, <span class="dv">73</span>: <span class="fl">86.35</span>, <span class="dv">74</span>: <span class="fl">81.21</span>, <span class="dv">75</span>: <span class="fl">80.46</span>, <span class="dv">76</span>: <span class="fl">81.42</span>, <span class="dv">77</span>: <span class="fl">78.82</span>, <span class="dv">78</span>: <span class="fl">78.31</span>, <span class="dv">79</span>: <span class="fl">76.04</span>, <span class="dv">80</span>: <span class="fl">77.92</span>, <span class="dv">81</span>: <span class="fl">86.71</span>, <span class="dv">82</span>: <span class="fl">85.81</span>, <span class="dv">83</span>: <span class="fl">85.02</span>, <span class="dv">84</span>: <span class="fl">86.18</span>, <span class="dv">85</span>: <span class="fl">85.17</span>, <span class="dv">86</span>: <span class="fl">86.6</span>, <span class="dv">87</span>: <span class="fl">85.84</span>, <span class="dv">88</span>: <span class="fl">87.38</span>, <span class="dv">89</span>: <span class="fl">77.89</span>, <span class="dv">90</span>: <span class="fl">78.78</span>, <span class="dv">91</span>: <span class="fl">88.77</span>, <span class="dv">92</span>: <span class="fl">87.28</span>, <span class="dv">93</span>: <span class="fl">86.44</span>, <span class="dv">94</span>: <span class="fl">88.32</span>, <span class="dv">95</span>: <span class="fl">88.31</span>, <span class="dv">96</span>: <span class="fl">85.97</span>, <span class="dv">97</span>: <span class="fl">76.92</span>, <span class="dv">98</span>: <span class="fl">74.34</span>, <span class="dv">99</span>: <span class="fl">81.26</span>, <span class="dv">100</span>: <span class="fl">80.39</span>, <span class="dv">101</span>: <span class="fl">65.69</span>, <span class="dv">102</span>: <span class="fl">69.6</span>}, <span class="st">'system'</span>: {<span class="dv">0</span>: <span class="st">'quickmt'</span>, <span class="dv">1</span>: <span class="st">'quickmt'</span>, <span class="dv">2</span>: <span class="st">'quickmt'</span>, <span class="dv">3</span>: <span class="st">'quickmt'</span>, <span class="dv">4</span>: <span class="st">'quickmt'</span>, <span class="dv">5</span>: <span class="st">'quickmt'</span>, <span class="dv">6</span>: <span class="st">'quickmt'</span>, <span class="dv">7</span>: <span class="st">'quickmt'</span>, <span class="dv">8</span>: <span class="st">'quickmt'</span>, <span class="dv">9</span>: <span class="st">'quickmt'</span>, <span class="dv">10</span>: <span class="st">'quickmt'</span>, <span class="dv">11</span>: <span class="st">'quickmt'</span>, <span class="dv">12</span>: <span class="st">'quickmt'</span>, <span class="dv">13</span>: <span class="st">'quickmt'</span>, <span class="dv">14</span>: <span class="st">'quickmt'</span>, <span class="dv">15</span>: <span class="st">'quickmt'</span>, <span class="dv">16</span>: <span class="st">'quickmt'</span>, <span class="dv">17</span>: <span class="st">'quickmt'</span>, <span class="dv">18</span>: <span class="st">'quickmt'</span>, <span class="dv">19</span>: <span class="st">'quickmt'</span>, <span class="dv">20</span>: <span class="st">'quickmt'</span>, <span class="dv">21</span>: <span class="st">'quickmt'</span>, <span class="dv">22</span>: <span class="st">'quickmt'</span>, <span class="dv">23</span>: <span class="st">'quickmt'</span>, <span class="dv">24</span>: <span class="st">'quickmt'</span>, <span class="dv">25</span>: <span class="st">'quickmt'</span>, <span class="dv">26</span>: <span class="st">'quickmt'</span>, <span class="dv">27</span>: <span class="st">'quickmt'</span>, <span class="dv">28</span>: <span class="st">'quickmt'</span>, <span class="dv">29</span>: <span class="st">'quickmt'</span>, <span class="dv">30</span>: <span class="st">'quickmt'</span>, <span class="dv">31</span>: <span class="st">'quickmt'</span>, <span class="dv">32</span>: <span class="st">'quickmt'</span>, <span class="dv">33</span>: <span class="st">'quickmt'</span>, <span class="dv">34</span>: <span class="st">'quickmt'</span>, <span class="dv">35</span>: <span class="st">'quickmt'</span>, <span class="dv">36</span>: <span class="st">'quickmt'</span>, <span class="dv">37</span>: <span class="st">'quickmt'</span>, <span class="dv">38</span>: <span class="st">'quickmt'</span>, <span class="dv">39</span>: <span class="st">'quickmt'</span>, <span class="dv">40</span>: <span class="st">'quickmt'</span>, <span class="dv">41</span>: <span class="st">'quickmt'</span>, <span class="dv">42</span>: <span class="st">'quickmt'</span>, <span class="dv">43</span>: <span class="st">'quickmt'</span>, <span class="dv">44</span>: <span class="st">'quickmt'</span>, <span class="dv">45</span>: <span class="st">'quickmt'</span>, <span class="dv">46</span>: <span class="st">'quickmt'</span>, <span class="dv">47</span>: <span class="st">'quickmt'</span>, <span class="dv">48</span>: <span class="st">'quickmt'</span>, <span class="dv">49</span>: <span class="st">'quickmt'</span>, <span class="dv">50</span>: <span class="st">'quickmt'</span>, <span class="dv">51</span>: <span class="st">'quickmt'</span>, <span class="dv">52</span>: <span class="st">'libretranslate'</span>, <span class="dv">53</span>: <span class="st">'libretranslate'</span>, <span class="dv">54</span>: <span class="st">'libretranslate'</span>, <span class="dv">55</span>: <span class="st">'libretranslate'</span>, <span class="dv">56</span>: <span class="st">'libretranslate'</span>, <span class="dv">57</span>: <span class="st">'libretranslate'</span>, <span class="dv">58</span>: <span class="st">'libretranslate'</span>, <span class="dv">59</span>: <span class="st">'libretranslate'</span>, <span class="dv">60</span>: <span class="st">'libretranslate'</span>, <span class="dv">61</span>: <span class="st">'libretranslate'</span>, <span class="dv">62</span>: <span class="st">'libretranslate'</span>, <span class="dv">63</span>: <span class="st">'libretranslate'</span>, <span class="dv">64</span>: <span class="st">'libretranslate'</span>, <span class="dv">65</span>: <span class="st">'libretranslate'</span>, <span class="dv">66</span>: <span class="st">'libretranslate'</span>, <span class="dv">67</span>: <span class="st">'libretranslate'</span>, <span class="dv">68</span>: <span class="st">'libretranslate'</span>, <span class="dv">69</span>: <span class="st">'libretranslate'</span>, <span class="dv">70</span>: <span class="st">'libretranslate'</span>, <span class="dv">71</span>: <span class="st">'libretranslate'</span>, <span class="dv">72</span>: <span class="st">'libretranslate'</span>, <span class="dv">73</span>: <span class="st">'libretranslate'</span>, <span class="dv">74</span>: <span class="st">'libretranslate'</span>, <span class="dv">75</span>: <span class="st">'libretranslate'</span>, <span class="dv">76</span>: <span class="st">'libretranslate'</span>, <span class="dv">77</span>: <span class="st">'libretranslate'</span>, <span class="dv">78</span>: <span class="st">'libretranslate'</span>, <span class="dv">79</span>: <span class="st">'libretranslate'</span>, <span class="dv">80</span>: <span class="st">'libretranslate'</span>, <span class="dv">81</span>: <span class="st">'libretranslate'</span>, <span class="dv">82</span>: <span class="st">'libretranslate'</span>, <span class="dv">83</span>: <span class="st">'libretranslate'</span>, <span class="dv">84</span>: <span class="st">'libretranslate'</span>, <span class="dv">85</span>: <span class="st">'libretranslate'</span>, <span class="dv">86</span>: <span class="st">'libretranslate'</span>, <span class="dv">87</span>: <span class="st">'libretranslate'</span>, <span class="dv">88</span>: <span class="st">'libretranslate'</span>, <span class="dv">89</span>: <span class="st">'libretranslate'</span>, <span class="dv">90</span>: <span class="st">'libretranslate'</span>, <span class="dv">91</span>: <span class="st">'libretranslate'</span>, <span class="dv">92</span>: <span class="st">'libretranslate'</span>, <span class="dv">93</span>: <span class="st">'libretranslate'</span>, <span class="dv">94</span>: <span class="st">'libretranslate'</span>, <span class="dv">95</span>: <span class="st">'libretranslate'</span>, <span class="dv">96</span>: <span class="st">'libretranslate'</span>, <span class="dv">97</span>: <span class="st">'libretranslate'</span>, <span class="dv">98</span>: <span class="st">'libretranslate'</span>, <span class="dv">99</span>: <span class="st">'libretranslate'</span>, <span class="dv">100</span>: <span class="st">'libretranslate'</span>, <span class="dv">101</span>: <span class="st">'libretranslate'</span>, <span class="dv">102</span>: <span class="st">'libretranslate'</span>}}</span>
<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a>df <span class="op">=</span> pd.DataFrame(df_data)</span>
<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a>df[[<span class="st">"system"</span>, <span class="st">"time"</span>]].groupby(<span class="st">"system"</span>).mean()</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display" data-execution_count="1">
<div>
<table class="dataframe caption-top table table-sm table-striped small" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">time</th>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">system</th>
<th data-quarto-table-cell-role="th"></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<th data-quarto-table-cell-role="th">libretranslate</th>
<td>51.787339</td>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">quickmt</th>
<td>1.216358</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<p>It takes just over 1 second to translate 1000 sentences with <code>quickmt</code>, and more than <strong>50 seconds</strong> using <code>libretranslate</code>! The <code>libretranslate</code> models were warmed up (loaded) before starting the timer.</p>
<section id="why-is-there-such-a-large-difference-in-speed" class="level3">
<h3 class="anchored" data-anchor-id="why-is-there-such-a-large-difference-in-speed">Why is there such a large difference in speed?</h3>
<p>Both <code>libretranslate</code> and <code>quickmt</code> use <code>ctranslate2</code> for translation and <code>sentencepiece</code> for tokenization under the hood, but <code>libretranslate</code> uses <a href="https://github.com/stanfordnlp/stanza"><code>stanza</code></a> for sentence splitting while <code>quickmt</code> uses <a href="https://github.com/microsoft/BlingFire"><code>blingfire</code></a>. While <code>blingfire</code> is not a <a href="https://arxiv.org/abs/2010.09657">particularly <em>accurate</em></a> sentence segmenter, it is dramatically faster than <a href="https://github.com/nipunsadvilkar/pySBD?tab=readme-ov-file">other</a> <a href="https://github.com/segment-any-text/wtpsplit">methods</a>.</p>
<p>Another difference between <code>quickmt</code> and <code>libretranslate</code> is that our models were designed to be quick by having a small number of decoder layers. Most of the <code>libretranslate</code> models are re-packaged Opus-MT models which have 6 decoder layers.</p>
<p>How does <code>quickmt</code> compare to <code>libretranslate</code> in terms of translation quality?</p>
</section>
</section>
<section id="quality-comparison" class="level2">
<h2 class="anchored" data-anchor-id="quality-comparison">Quality Comparison</h2>
<p>We use <code>chrf</code> was calculated with <a href="https://github.com/mjpost/sacrebleu"><code>sacrebleu</code></a> and <code>comet</code> with the <a href="https://github.com/Unbabel/COMET"><code>comet</code></a> library and the default model (comet22).</p>
<p>First let’s take a look at translation from the languages supported by <code>quickmt</code> into English:</p>
<div id="84016fc4" class="cell" data-execution_count="2">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>px.bar(</span>
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a> df[df.src_lang <span class="op">!=</span> <span class="st">"en"</span>],</span>
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> x<span class="op">=</span><span class="st">"src_lang"</span>,</span>
<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a> y<span class="op">=</span><span class="st">"comet22"</span>,</span>
<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a> color<span class="op">=</span><span class="st">"system"</span>,</span>
<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a> barmode<span class="op">=</span><span class="st">"group"</span>,</span>
<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a> title<span class="op">=</span><span class="st">"XX-&gt;EN Machine Translation Quality - comet22 measure"</span>,</span>
<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a> range_y<span class="op">=</span>[<span class="dv">70</span>, <span class="dv">95</span>],</span>
<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a> height<span class="op">=</span><span class="dv">500</span></span>
<span id="cb3-10"><a href="#cb3-10" aria-hidden="true" tabindex="-1"></a>)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display">
<div> <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script> <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
<script charset="utf-8" src="https://cdn.plot.ly/plotly-3.2.0.min.js" integrity="sha256-iZ2u/oU2wf/vDbl/ChcX93WgbBRSBvUO6N413hDz7xM=" crossorigin="anonymous"></script> <div id="6abb4305-50ce-4929-95be-ca86945b0a22" class="plotly-graph-div" style="height:500px; width:100%;"></div> <script type="text/javascript"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById("6abb4305-50ce-4929-95be-ca86945b0a22")) { Plotly.newPlot( "6abb4305-50ce-4929-95be-ca86945b0a22", [{"alignmentgroup":"True","hovertemplate":"system=quickmt\u003cbr\u003esrc_lang=%{x}\u003cbr\u003ecomet22=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"quickmt","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"quickmt","offsetgroup":"quickmt","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","vi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"cT0K16OAVUDD9Shcj9JVQGZmZmZmhlVAcT0K16PwVUD2KFyPwpVVQJqZmZmZ2VVAcT0K16MwVkDNzMzMzMxVQKRwPQrXU1ZAw\u002fUoXI9CVUD2KFyPwkVWQJqZmZmZOVZApHA9CtcTVkCPwvUoXD9WQI\u002fC9Shcz1VAexSuR+EKVkAfhetRuL5VQI\u002fC9ShcT1ZAw\u002fUoXI\u002fSVUCamZmZmclVQAAAAAAAgFZAPQrXo3B9VUBxPQrXozBWQI\u002fC9Shcv1VAcT0K16PwVUAAAAAAABBVQA=="},"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"system=libretranslate\u003cbr\u003esrc_lang=%{x}\u003cbr\u003ecomet22=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"libretranslate","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"libretranslate","offsetgroup":"libretranslate","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"PQrXo3AdUkCPwvUoXE9VQMP1KFyPMlVAMzMzMzNzVUAUrkfhesRUQFyPwvUoPFRA9ihcj8KVVECuR+F6FP5VQEjhehSuB1VAFK5H4XokVkBmZmZmZpZVQD0K16NwHVRAFK5H4Xq0U0DD9ShcjwJTQD0K16NwrVVA4XoUrkdBVUB7FK5H4UpVQPYoXI\u002fCdVVAKVyPwvV4U0DhehSuRzFWQFyPwvUonFVApHA9CtcTVkB7FK5H4TpTQHE9CtejUFRAXI\u002fC9ShsUEA="},"yaxis":"y","type":"bar"}], {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"},"margin":{"b":0,"l":0,"r":0,"t":30}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"src_lang"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"comet22"},"range":[70,95]},"legend":{"title":{"text":"system"},"tracegroupgap":0},"title":{"text":"XX-\u003eEN Machine Translation Quality - comet22 measure"},"barmode":"group","height":500}, {"responsive": true} ).then(function(){
var gd = document.getElementById('6abb4305-50ce-4929-95be-ca86945b0a22');
var x = new MutationObserver(function (mutations, observer) {{
var display = window.getComputedStyle(gd).display;
if (!display || display === 'none') {{
console.log([gd, 'removed!']);
Plotly.purge(gd);
observer.disconnect();
}}
}});
// Listen for the removal of the full notebook cells
var notebookContainer = gd.closest('#notebook-container');
if (notebookContainer) {{
x.observe(notebookContainer, {childList: true});
}}
// Listen for the clearing of the current output cell
var outputEl = gd.closest('.output');
if (outputEl) {{
x.observe(outputEl, {childList: true});
}}
}) }; </script> </div>
</div>
</div>
<div id="51a7e312" class="cell" data-execution_count="3">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>px.bar(</span>
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a> df[df.src_lang <span class="op">!=</span> <span class="st">"en"</span>],</span>
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> x<span class="op">=</span><span class="st">"src_lang"</span>,</span>
<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a> y<span class="op">=</span><span class="st">"chrf2"</span>,</span>
<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a> color<span class="op">=</span><span class="st">"system"</span>,</span>
<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a> barmode<span class="op">=</span><span class="st">"group"</span>,</span>
<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a> title<span class="op">=</span><span class="st">"XX-&gt;EN Machine Translation Quality - chrf measure"</span>,</span>
<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a> height<span class="op">=</span><span class="dv">500</span></span>
<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display">
<div> <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script> <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
<script charset="utf-8" src="https://cdn.plot.ly/plotly-3.2.0.min.js" integrity="sha256-iZ2u/oU2wf/vDbl/ChcX93WgbBRSBvUO6N413hDz7xM=" crossorigin="anonymous"></script> <div id="4cf0e640-eb92-478d-8b1b-3a4bc58afd06" class="plotly-graph-div" style="height:500px; width:100%;"></div> <script type="text/javascript"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById("4cf0e640-eb92-478d-8b1b-3a4bc58afd06")) { Plotly.newPlot( "4cf0e640-eb92-478d-8b1b-3a4bc58afd06", [{"alignmentgroup":"True","hovertemplate":"system=quickmt\u003cbr\u003esrc_lang=%{x}\u003cbr\u003echrf2=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"quickmt","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"quickmt","offsetgroup":"quickmt","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","vi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"H4XrUbgeTEBcj8L1KLxOQK5H4XoUTk1AFK5H4Xo0T0AzMzMzM\u002fNMQHE9CtejwFBAAAAAAABAUEDsUbgehWtPQDMzMzMzU1FA16NwPQqXTEAfhetRuN5RQIXrUbgeNVFAKVyPwvUYUUDhehSuR0FQQD0K16NwfUxAuB6F61FoUEAUrkfhenRPQOF6FK5HMVFAUrgehevxTkAzMzMzMzNNQFK4HoXr8VFASOF6FK4nT0CamZmZmQlRQJqZmZmZ2U1A7FG4HoWrT0DNzMzMzCxNQA=="},"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"system=libretranslate\u003cbr\u003esrc_lang=%{x}\u003cbr\u003echrf2=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"libretranslate","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"libretranslate","offsetgroup":"libretranslate","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"MzMzMzMTRUBI4XoUrmdNQBSuR+F6dExAcT0K16NQTUCamZmZmdlJQLgehetROExAAAAAAACgS0BmZmZmZoZQQPYoXI\u002fC1UtA9ihcj8J1UUBI4XoUrsdPQMP1KFyPYkxAFK5H4Xp0SkAK16NwPWpFQFyPwvUoPE9AexSuR+GaTUB7FK5H4VpOQLgehetReE1ApHA9CteDRkDsUbgehTtRQK5H4XoUjk9Aj8L1KFyvUECamZmZmXlFQGZmZmZmpkpAcT0K16NwQ0A="},"yaxis":"y","type":"bar"}], {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"},"margin":{"b":0,"l":0,"r":0,"t":30}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"src_lang"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"chrf2"}},"legend":{"title":{"text":"system"},"tracegroupgap":0},"title":{"text":"XX-\u003eEN Machine Translation Quality - chrf measure"},"barmode":"group","height":500}, {"responsive": true} ).then(function(){
var gd = document.getElementById('4cf0e640-eb92-478d-8b1b-3a4bc58afd06');
var x = new MutationObserver(function (mutations, observer) {{
var display = window.getComputedStyle(gd).display;
if (!display || display === 'none') {{
console.log([gd, 'removed!']);
Plotly.purge(gd);
observer.disconnect();
}}
}});
// Listen for the removal of the full notebook cells
var notebookContainer = gd.closest('#notebook-container');
if (notebookContainer) {{
x.observe(notebookContainer, {childList: true});
}}
// Listen for the clearing of the current output cell
var outputEl = gd.closest('.output');
if (outputEl) {{
x.observe(outputEl, {childList: true});
}}
}) }; </script> </div>
</div>
</div>
<p>For both <code>chrf</code> and <code>comet22</code> and for all languages, with the notable exception of Russian (85.96 comet22 for <code>quickmt</code> vs.&nbsp;86.44 for <code>libretranslate</code>), <code>quickmt</code> is higher quality than <code>libretranslate</code>. The differences are <strong>huge</strong> for Korean (86.01 vs 72.46 comet22), Turkish (88.99 vs.&nbsp;78.82 comet22), japanese (87.24 vs.&nbsp;76.04 comet22), Thai (87.15 vs.&nbsp;77.79 comet22) and Bengali (86.99 vs.&nbsp;76.92 comet22).</p>
<p>On average, <code>quickmt</code> is 4.95 higher in comet22 and 7.14 higher in chrf2 than <code>libretranslate</code> for translation into English:</p>
<div id="479ccc51" class="cell" data-execution_count="4">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>df[df.tgt_lang<span class="op">==</span><span class="st">"en"</span>][[<span class="st">"system"</span>, <span class="st">"comet22"</span>]].groupby(<span class="st">"system"</span>).mean()</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display" data-execution_count="4">
<div>
<table class="dataframe caption-top table table-sm table-striped small" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">comet22</th>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">system</th>
<th data-quarto-table-cell-role="th"></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<th data-quarto-table-cell-role="th">libretranslate</th>
<td>82.599600</td>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">quickmt</th>
<td>87.549615</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div id="29a775e8" class="cell" data-execution_count="5">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>df[df.tgt_lang<span class="op">==</span><span class="st">"en"</span>][[<span class="st">"system"</span>, <span class="st">"chrf2"</span>]].groupby(<span class="st">"system"</span>).mean()</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display" data-execution_count="5">
<div>
<table class="dataframe caption-top table table-sm table-striped small" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">chrf2</th>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">system</th>
<th data-quarto-table-cell-role="th"></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<th data-quarto-table-cell-role="th">libretranslate</th>
<td>56.313600</td>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">quickmt</th>
<td>63.457308</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<p>Next we look at translation from English into the languages supported by <code>quickmt</code>:</p>
<div id="8a66d0df" class="cell" data-execution_count="6">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>px.bar(</span>
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> df[df.src_lang <span class="op">==</span> <span class="st">"en"</span>],</span>
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> x<span class="op">=</span><span class="st">"tgt_lang"</span>,</span>
<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> y<span class="op">=</span><span class="st">"comet22"</span>,</span>
<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a> color<span class="op">=</span><span class="st">"system"</span>,</span>
<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> barmode<span class="op">=</span><span class="st">"group"</span>,</span>
<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a> title<span class="op">=</span><span class="st">"EN-&gt;XX Machine Translation Quality - comet22 measure"</span>,</span>
<span id="cb7-8"><a href="#cb7-8" aria-hidden="true" tabindex="-1"></a> range_y<span class="op">=</span>[<span class="dv">70</span>, <span class="dv">95</span>],</span>
<span id="cb7-9"><a href="#cb7-9" aria-hidden="true" tabindex="-1"></a> height<span class="op">=</span><span class="dv">500</span></span>
<span id="cb7-10"><a href="#cb7-10" aria-hidden="true" tabindex="-1"></a>)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display">
<div> <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script> <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
<script charset="utf-8" src="https://cdn.plot.ly/plotly-3.2.0.min.js" integrity="sha256-iZ2u/oU2wf/vDbl/ChcX93WgbBRSBvUO6N413hDz7xM=" crossorigin="anonymous"></script> <div id="e006ea2f-e030-4a59-b958-e832545e8195" class="plotly-graph-div" style="height:500px; width:100%;"></div> <script type="text/javascript"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById("e006ea2f-e030-4a59-b958-e832545e8195")) { Plotly.newPlot( "e006ea2f-e030-4a59-b958-e832545e8195", [{"alignmentgroup":"True","hovertemplate":"system=quickmt\u003cbr\u003etgt_lang=%{x}\u003cbr\u003ecomet22=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"quickmt","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"quickmt","offsetgroup":"quickmt","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","vi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"MzMzMzPDVUDD9Shcj+JVQJqZmZmZSVVAcT0K16MAVkAzMzMzM1NVQBSuR+F6lFVAMzMzMzPDU0DhehSuR+FVQHsUrkfhalZAKVyPwvXIVUAfhetRuE5WQJqZmZmZuVVACtejcD36VUB7FK5H4VpWQKRwPQrXQ1ZA4XoUrkcxVkCuR+F6FI5VQOF6FK5HwVZAZmZmZmY2VkDhehSuRyFVQI\u002fC9ShcX1ZAcT0K16PwVUCPwvUoXL9VQOxRuB6FK1VAw\u002fUoXI9yVUB7FK5H4WpTQA=="},"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"system=libretranslate\u003cbr\u003etgt_lang=%{x}\u003cbr\u003ecomet22=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"libretranslate","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"libretranslate","offsetgroup":"libretranslate","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","vi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"KVyPwvXoUUDsUbgehetUQAAAAAAAsFRAj8L1KFz\u002fVECamZmZmclUQArXo3A9KlNArkfhehROUkBxPQrXo\u002fBUQOF6FK5HsVVAcT0K16MwVUCPwvUoXP9VQD0K16NwTVRAexSuR+FaVECkcD0K15NTQHsUrkfhelNApHA9CtdzVUDsUbgehYtVQGZmZmZmplVAuB6F61HYVUBSuB6F67FTQFK4HoXr0VVAFK5H4XoUVkCuR+F6FH5VQPYoXI\u002fClVJAKVyPwvUYVEBmZmZmZmZRQA=="},"yaxis":"y","type":"bar"}], {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"},"margin":{"b":0,"l":0,"r":0,"t":30}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"tgt_lang"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"comet22"},"range":[70,95]},"legend":{"title":{"text":"system"},"tracegroupgap":0},"title":{"text":"EN-\u003eXX Machine Translation Quality - comet22 measure"},"barmode":"group","height":500}, {"responsive": true} ).then(function(){
var gd = document.getElementById('e006ea2f-e030-4a59-b958-e832545e8195');
var x = new MutationObserver(function (mutations, observer) {{
var display = window.getComputedStyle(gd).display;
if (!display || display === 'none') {{
console.log([gd, 'removed!']);
Plotly.purge(gd);
observer.disconnect();
}}
}});
// Listen for the removal of the full notebook cells
var notebookContainer = gd.closest('#notebook-container');
if (notebookContainer) {{
x.observe(notebookContainer, {childList: true});
}}
// Listen for the clearing of the current output cell
var outputEl = gd.closest('.output');
if (outputEl) {{
x.observe(outputEl, {childList: true});
}}
}) }; </script> </div>
</div>
</div>
<div id="a0c73aa9" class="cell" data-execution_count="7">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>px.bar(</span>
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a> df[df.src_lang <span class="op">==</span> <span class="st">"en"</span>],</span>
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> x<span class="op">=</span><span class="st">"tgt_lang"</span>,</span>
<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a> y<span class="op">=</span><span class="st">"chrf2"</span>,</span>
<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a> color<span class="op">=</span><span class="st">"system"</span>,</span>
<span id="cb8-6"><a href="#cb8-6" aria-hidden="true" tabindex="-1"></a> barmode<span class="op">=</span><span class="st">"group"</span>,</span>
<span id="cb8-7"><a href="#cb8-7" aria-hidden="true" tabindex="-1"></a> title<span class="op">=</span><span class="st">"EN-&gt;XX Machine Translation Quality - chrf2 measure"</span>,</span>
<span id="cb8-8"><a href="#cb8-8" aria-hidden="true" tabindex="-1"></a> height<span class="op">=</span><span class="dv">500</span></span>
<span id="cb8-9"><a href="#cb8-9" aria-hidden="true" tabindex="-1"></a>)</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display">
<div> <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script> <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
<script charset="utf-8" src="https://cdn.plot.ly/plotly-3.2.0.min.js" integrity="sha256-iZ2u/oU2wf/vDbl/ChcX93WgbBRSBvUO6N413hDz7xM=" crossorigin="anonymous"></script> <div id="39c3dfcb-28d9-4388-b45b-6d033f0203af" class="plotly-graph-div" style="height:500px; width:100%;"></div> <script type="text/javascript"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById("39c3dfcb-28d9-4388-b45b-6d033f0203af")) { Plotly.newPlot( "39c3dfcb-28d9-4388-b45b-6d033f0203af", [{"alignmentgroup":"True","hovertemplate":"system=quickmt\u003cbr\u003etgt_lang=%{x}\u003cbr\u003echrf2=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"quickmt","marker":{"color":"#636efa","pattern":{"shape":""}},"name":"quickmt","offsetgroup":"quickmt","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","vi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"UrgehetxQkA9CtejcN1NQHE9CtejkEtAMzMzMzOzTUAAAAAAAEBBQDMzMzMz005A9ihcj8L1TUDhehSuR2FOQBSuR+F6hFBAzczMzMwMSkAAAAAAAPBRQAAAAAAAkFBAj8L1KFwvT0CkcD0K1+NPQMP1KFyPAkVAhetRuB4lTkAAAAAAAEBOQFyPwvUo\u002fFFA16NwPQp3S0CF61G4HmVLQBSuR+F6hFFArkfhehSOTUCkcD0K19NRQClcj8L1qEpAKVyPwvUIS0BmZmZmZkZIQA=="},"yaxis":"y","type":"bar"},{"alignmentgroup":"True","hovertemplate":"system=libretranslate\u003cbr\u003etgt_lang=%{x}\u003cbr\u003echrf2=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"libretranslate","marker":{"color":"#EF553B","pattern":{"shape":""}},"name":"libretranslate","offsetgroup":"libretranslate","orientation":"v","showlegend":true,"textposition":"auto","x":["ko","it","es","hu","zh","ar","hi","vi","ro","pl","pt","de","he","tr","ja","cs","lv","id","el","th","da","ru","fr","bn","fa","ur"],"xaxis":"x","y":{"dtype":"f8","bdata":"MzMzMzPzN0CPwvUoXC9MQHsUrkfhWkpA7FG4HoXLS0BSuB6F65E9QLgehetRWEZAMzMzMzPTSkA9CtejcH1KQD0K16NwXU5AuB6F61HYSEDhehSuR1FRQB+F61G4Pk5Aj8L1KFzPSkAzMzMzMxNKQDMzMzMzMz1AH4XrUbheTEAK16NwPYpNQD0K16NwTVBAw\u002fUoXI+iSkBcj8L1KHxGQHsUrkfh2lBA4XoUrkeBTUDNzMzMzFxRQOF6FK5H4UJAXI\u002fC9Si8SEAK16NwPSpDQA=="},"yaxis":"y","type":"bar"}], {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"},"margin":{"b":0,"l":0,"r":0,"t":30}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"tgt_lang"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"chrf2"}},"legend":{"title":{"text":"system"},"tracegroupgap":0},"title":{"text":"EN-\u003eXX Machine Translation Quality - chrf2 measure"},"barmode":"group","height":500}, {"responsive": true} ).then(function(){
var gd = document.getElementById('39c3dfcb-28d9-4388-b45b-6d033f0203af');
var x = new MutationObserver(function (mutations, observer) {{
var display = window.getComputedStyle(gd).display;
if (!display || display === 'none') {{
console.log([gd, 'removed!']);
Plotly.purge(gd);
observer.disconnect();
}}
}});
// Listen for the removal of the full notebook cells
var notebookContainer = gd.closest('#notebook-container');
if (notebookContainer) {{
x.observe(notebookContainer, {childList: true});
}}
// Listen for the clearing of the current output cell
var outputEl = gd.closest('.output');
if (outputEl) {{
x.observe(outputEl, {childList: true});
}}
}) }; </script> </div>
</div>
</div>
<p>The pattern here is similar - <code>quickmt</code> is higher-quality than <code>libretranslate</code> for all languages (except Russian!), sometimes dramatically but by an average of 4.97 comet22 and 6.38 chrf2.</p>
<div id="20b7bb4f" class="cell" data-execution_count="8">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>df[df.tgt_lang<span class="op">!=</span><span class="st">"en"</span>][[<span class="st">"system"</span>, <span class="st">"comet22"</span>]].groupby(<span class="st">"system"</span>).mean()</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display" data-execution_count="8">
<div>
<table class="dataframe caption-top table table-sm table-striped small" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">comet22</th>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">system</th>
<th data-quarto-table-cell-role="th"></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<th data-quarto-table-cell-role="th">libretranslate</th>
<td>81.841538</td>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">quickmt</th>
<td>86.808846</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div id="0c5249cc" class="cell" data-execution_count="9">
<details class="code-fold">
<summary>Code</summary>
<div class="code-copy-outer-scaffold"><div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>df[df.tgt_lang<span class="op">!=</span><span class="st">"en"</span>][[<span class="st">"system"</span>, <span class="st">"chrf2"</span>]].groupby(<span class="st">"system"</span>).mean()</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
</details>
<div class="cell-output cell-output-display" data-execution_count="9">
<div>
<table class="dataframe caption-top table table-sm table-striped small" data-border="1">
<thead>
<tr class="header">
<th data-quarto-table-cell-role="th"></th>
<th data-quarto-table-cell-role="th">chrf2</th>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">system</th>
<th data-quarto-table-cell-role="th"></th>
</tr>
</thead>
<tbody>
<tr class="odd">
<th data-quarto-table-cell-role="th">libretranslate</th>
<td>51.745000</td>
</tr>
<tr class="even">
<th data-quarto-table-cell-role="th">quickmt</th>
<td>58.123077</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</section>
<section id="conclusion" class="level2">
<h2 class="anchored" data-anchor-id="conclusion">Conclusion</h2>
<p><code>quickmt</code> is more than 40 times faster than <code>libretranslate</code> (on a RTX 4070s GPU) and quite a bit higher quality (for all languages except Russian).</p>
<p>You can give <code>quickmt</code> a try on <a href="https://huggingface.co/spaces/quickmt/QuickMT-Demo">Huggingface Spaces</a>, download the models from <a href="https://huggingface.co/collections/quickmt/quickmt-models">Huggingface</a> and make it your own by forking our <a href="https://github.com/quickmt/quickmt">Github repository</a>. Thanks for reading!</p>
</section>
</main>
<!-- /main column -->
<script id="quarto-html-after-body" type="application/javascript">
window.document.addEventListener("DOMContentLoaded", function (event) {
const icon = "";
const anchorJS = new window.AnchorJS();
anchorJS.options = {
placement: 'right',
icon: icon
};
anchorJS.add('.anchored');
const isCodeAnnotation = (el) => {
for (const clz of el.classList) {
if (clz.startsWith('code-annotation-')) {
return true;
}
}
return false;
}
const onCopySuccess = function(e) {
// button target
const button = e.trigger;
// don't keep focus
button.blur();
// flash "checked"
button.classList.add('code-copy-button-checked');
var currentTitle = button.getAttribute("title");
button.setAttribute("title", "Copied!");
let tooltip;
if (window.bootstrap) {
button.setAttribute("data-bs-toggle", "tooltip");
button.setAttribute("data-bs-placement", "left");
button.setAttribute("data-bs-title", "Copied!");
tooltip = new bootstrap.Tooltip(button,
{ trigger: "manual",
customClass: "code-copy-button-tooltip",
offset: [0, -8]});
tooltip.show();
}
setTimeout(function() {
if (tooltip) {
tooltip.hide();
button.removeAttribute("data-bs-title");
button.removeAttribute("data-bs-toggle");
button.removeAttribute("data-bs-placement");
}
button.setAttribute("title", currentTitle);
button.classList.remove('code-copy-button-checked');
}, 1000);
// clear code selection
e.clearSelection();
}
const getTextToCopy = function(trigger) {
const outerScaffold = trigger.parentElement.cloneNode(true);
const codeEl = outerScaffold.querySelector('code');
for (const childEl of codeEl.children) {
if (isCodeAnnotation(childEl)) {
childEl.remove();
}
}
return codeEl.innerText;
}
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
text: getTextToCopy
});
clipboard.on('success', onCopySuccess);
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
text: getTextToCopy,
container: window.document.getElementById('quarto-embedded-source-code-modal')
});
clipboardModal.on('success', onCopySuccess);
}
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
var mailtoRegex = new RegExp(/^mailto:/);
var filterRegex = new RegExp('/' + window.location.host + '/');
var isInternal = (href) => {
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
}
// Inspect non-navigation links and adorn them if external
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
for (var i=0; i<links.length; i++) {
const link = links[i];
if (!isInternal(link.href)) {
// undo the damage that might have been done by quarto-nav.js in the case of
// links that we want to consider external
if (link.dataset.originalHref !== undefined) {
link.href = link.dataset.originalHref;
}
}
}
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
const config = {
allowHTML: true,
maxWidth: 500,
delay: 100,
arrow: false,
appendTo: function(el) {
return el.parentElement;
},
interactive: true,
interactiveBorder: 10,
theme: 'quarto',
placement: 'bottom-start',
};
if (contentFn) {
config.content = contentFn;
}
if (onTriggerFn) {
config.onTrigger = onTriggerFn;
}
if (onUntriggerFn) {
config.onUntrigger = onUntriggerFn;
}
window.tippy(el, config);
}
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
for (var i=0; i<noterefs.length; i++) {
const ref = noterefs[i];
tippyHover(ref, function() {
// use id or data attribute instead here
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
try { href = new URL(href).hash; } catch {}
const id = href.replace(/^#\/?/, "");
const note = window.document.getElementById(id);
if (note) {
return note.innerHTML;
} else {
return "";
}
});
}
const xrefs = window.document.querySelectorAll('a.quarto-xref');
const processXRef = (id, note) => {
// Strip column container classes
const stripColumnClz = (el) => {
el.classList.remove("page-full", "page-columns");
if (el.children) {
for (const child of el.children) {
stripColumnClz(child);
}
}
}
stripColumnClz(note)
if (id === null || id.startsWith('sec-')) {
// Special case sections, only their first couple elements
const container = document.createElement("div");
if (note.children && note.children.length > 2) {
container.appendChild(note.children[0].cloneNode(true));
for (let i = 1; i < note.children.length; i++) {
const child = note.children[i];
if (child.tagName === "P" && child.innerText === "") {
continue;
} else {
container.appendChild(child.cloneNode(true));
break;
}
}
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(container);
}
return container.innerHTML
} else {
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(note);
}
return note.innerHTML;
}
} else {
// Remove any anchor links if they are present
const anchorLink = note.querySelector('a.anchorjs-link');
if (anchorLink) {
anchorLink.remove();
}
if (window.Quarto?.typesetMath) {
window.Quarto.typesetMath(note);
}
if (note.classList.contains("callout")) {
return note.outerHTML;
} else {
return note.innerHTML;
}
}
}
for (var i=0; i<xrefs.length; i++) {
const xref = xrefs[i];
tippyHover(xref, undefined, function(instance) {
instance.disable();
let url = xref.getAttribute('href');
let hash = undefined;
if (url.startsWith('#')) {
hash = url;
} else {
try { hash = new URL(url).hash; } catch {}
}
if (hash) {
const id = hash.replace(/^#\/?/, "");
const note = window.document.getElementById(id);
if (note !== null) {
try {
const html = processXRef(id, note.cloneNode(true));
instance.setContent(html);
} finally {
instance.enable();
instance.show();
}
} else {
// See if we can fetch this
fetch(url.split('#')[0])
.then(res => res.text())
.then(html => {
const parser = new DOMParser();
const htmlDoc = parser.parseFromString(html, "text/html");
const note = htmlDoc.getElementById(id);
if (note !== null) {
const html = processXRef(id, note);
instance.setContent(html);
}
}).finally(() => {
instance.enable();
instance.show();
});
}
} else {
// See if we can fetch a full url (with no hash to target)
// This is a special case and we should probably do some content thinning / targeting
fetch(url)
.then(res => res.text())
.then(html => {
const parser = new DOMParser();
const htmlDoc = parser.parseFromString(html, "text/html");
const note = htmlDoc.querySelector('main.content');
if (note !== null) {
// This should only happen for chapter cross references
// (since there is no id in the URL)
// remove the first header
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
note.children[0].remove();
}
const html = processXRef(null, note);
instance.setContent(html);
}
}).finally(() => {
instance.enable();
instance.show();
});
}
}, function(instance) {
});
}
let selectedAnnoteEl;
const selectorForAnnotation = ( cell, annotation) => {
let cellAttr = 'data-code-cell="' + cell + '"';
let lineAttr = 'data-code-annotation="' + annotation + '"';
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
return selector;
}
const selectCodeLines = (annoteEl) => {
const doc = window.document;
const targetCell = annoteEl.getAttribute("data-target-cell");
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
const lineIds = lines.map((line) => {
return targetCell + "-" + line;
})
let top = null;
let height = null;
let parent = null;
if (lineIds.length > 0) {
//compute the position of the single el (top and bottom and make a div)
const el = window.document.getElementById(lineIds[0]);
top = el.offsetTop;
height = el.offsetHeight;
parent = el.parentElement.parentElement;
if (lineIds.length > 1) {
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
height = bottom - top;
}
if (top !== null && height !== null && parent !== null) {
// cook up a div (if necessary) and position it
let div = window.document.getElementById("code-annotation-line-highlight");
if (div === null) {
div = window.document.createElement("div");
div.setAttribute("id", "code-annotation-line-highlight");
div.style.position = 'absolute';
parent.appendChild(div);
}
div.style.top = top - 2 + "px";
div.style.height = height + 4 + "px";
div.style.left = 0;
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
if (gutterDiv === null) {
gutterDiv = window.document.createElement("div");
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
gutterDiv.style.position = 'absolute';
const codeCell = window.document.getElementById(targetCell);
const gutter = codeCell.querySelector('.code-annotation-gutter');
gutter.appendChild(gutterDiv);
}
gutterDiv.style.top = top - 2 + "px";
gutterDiv.style.height = height + 4 + "px";
}
selectedAnnoteEl = annoteEl;
}
};
const unselectCodeLines = () => {
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
elementsIds.forEach((elId) => {
const div = window.document.getElementById(elId);
if (div) {
div.remove();
}
});
selectedAnnoteEl = undefined;
};
// Handle positioning of the toggle
window.addEventListener(
"resize",
throttle(() => {
elRect = undefined;
if (selectedAnnoteEl) {
selectCodeLines(selectedAnnoteEl);
}
}, 10)
);
function throttle(fn, ms) {
let throttle = false;
let timer;
return (...args) => {
if(!throttle) { // first call gets through
fn.apply(this, args);
throttle = true;
} else { // all the others get throttled
if(timer) clearTimeout(timer); // cancel #2
timer = setTimeout(() => {
fn.apply(this, args);
timer = throttle = false;
}, ms);
}
};
}
// Attach click handler to the DT
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
for (const annoteDlNode of annoteDls) {
annoteDlNode.addEventListener('click', (event) => {
const clickedEl = event.target;
if (clickedEl !== selectedAnnoteEl) {
unselectCodeLines();
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
if (activeEl) {
activeEl.classList.remove('code-annotation-active');
}
selectCodeLines(clickedEl);
clickedEl.classList.add('code-annotation-active');
} else {
// Unselect the line
unselectCodeLines();
clickedEl.classList.remove('code-annotation-active');
}
});
}
const findCites = (el) => {
const parentEl = el.parentElement;
if (parentEl) {
const cites = parentEl.dataset.cites;
if (cites) {
return {
el,
cites: cites.split(' ')
};
} else {
return findCites(el.parentElement)
}
} else {
return undefined;
}
};
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
for (var i=0; i<bibliorefs.length; i++) {
const ref = bibliorefs[i];
const citeInfo = findCites(ref);
if (citeInfo) {
tippyHover(citeInfo.el, function() {
var popup = window.document.createElement('div');
citeInfo.cites.forEach(function(cite) {
var citeDiv = window.document.createElement('div');
citeDiv.classList.add('hanging-indent');
citeDiv.classList.add('csl-entry');
var biblioDiv = window.document.getElementById('ref-' + cite);
if (biblioDiv) {
citeDiv.innerHTML = biblioDiv.innerHTML;
}
popup.appendChild(citeDiv);
});
return popup.innerHTML;
});
}
}
});
</script>
</div> <!-- /content -->
</body></html>