Spaces:

itramb
/

Template_test

Running

File size: 44,410 Bytes

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>AIRI institute</title>
    <link
      href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css"
      rel="stylesheet"
    />
    <meta name="description" content="AIRI is an autonomous non-profit organization bringing together researchers, scientists and data engineers engaged in breakthrough AI research." />
    <meta name="author" content="AIRI" />
    <link rel="stylesheet" href="styles/video.css" />
    <link rel="stylesheet" href="styles/main.css" />
    <link rel="stylesheet" href="styles/header.css" />
    <link rel="stylesheet" href="styles/columnsSection.css" />
    <link rel="stylesheet" href="styles/textSection.css" />
    <link rel="stylesheet" href="styles/textSectionBorder.css" />
    <link rel="stylesheet" href="styles/swiper.css" />
    <link rel="stylesheet" href="styles/charts.css" />
    <link rel="stylesheet" href="styles/numberSection.css" />
    <link rel="stylesheet" href="styles/table.css" />
    <link rel="stylesheet" href="styles/lightbox.css" />
    <link rel="stylesheet" href="styles/dropdown.css" />
    <link rel="stylesheet" href="styles/swiperOnButton.css" />
    <link rel="stylesheet" href="styles/codeSection.css" />
    <link rel="stylesheet" href="styles/sidebar.css" />
    <link rel="stylesheet" href="styles/verticalLine.css" />
    <link rel="stylesheet" href="styles/bibTex.css" />
    <link rel="stylesheet" href="styles/footer.css" />
    <link rel="stylesheet" href="styles/beforeAfter.css" />
    <link rel="preconnect" href="https://fonts.googleapis.com" />
    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
    <link
      href="https://fonts.googleapis.com/css2?family=Inter:ital,opsz,wght@0,14..32,100..900;1,14..32,100..900&family=Montserrat:ital,wght@0,100..900;1,100..900&family=PT+Sans&family=Roboto:wght@100..900&display=swap"
      rel="stylesheet"
    />
    <link
      rel="stylesheet"
      href="https://cdn.jsdelivr.net/npm/swiper@11/swiper-bundle.min.css"
    />
  </head>
  <body>
    <header>
      <section class="header__container" id="header">
        <span class="header-gradient"></span>
        <div class="header__participants-container">
          <img
            class="header__participants-logo"
            src="public/airi_logo_black.svg"
            alt="AIRI"
          />
          <img
            class="header__participants-logo"
            src="public/mtuci-logo.svg"
            alt="MTUCI"
          />
          <!-- <img
            class="header__participants-logo"
            src="public/hse-logo.svg"
            alt="HSE"
          />
          <img
            class="header__participants-logo"
            src="public/mirea-logo.svg"
            alt="MIREA"
          />
          <img
            class="header__participants-logo"
            src="public/MISIS-logo.svg"
            alt="MISIS"
          />
          <img
            class="header__participants-logo"
            src="public/mei-logo.svg"
            alt="MEI"
          />
          <img
            class="header__participants-logo"
            src="public/mirea-logo.svg"
            alt="MIREA"
          />
          <img
            class="header__participants-logo"
            src="public/MISIS-logo.svg"
            alt="MISIS"
          />
          <img
            class="header__participants-logo"
            src="public/mei-logo.svg"
            alt="MEI"
          /> -->
        </div>
        <h1 class="header-projectName">
          Fact-Checking the Output of Large Language Models via Token-Level
          Uncertainty Quantification.
        </h1>
        <div class="header__link-container">
          <div class="linkContainer__authors">
            <a href="">Ekaterina Fadeeva<sup>3,4</sup></a>
            <a href="">Aleksandr Rubashevskii<sup>1,3</sup></a>
            <a href="">Artem Shelmanov<sup>1</sup></a>
            <a href="">Sergey Petrakov<sup>3</sup></a>
            <a href="">Haonan Li<sup>1</sup></a>
            <a href="">Hamdy Mubarak<sup>7</sup></a>
            <a href="">Evgenii Tsymbalov<sup>8</sup></a>
            <a href="">Gleb Kuzmin<sup>2,5</sup></a>
            <a href="">Alexander Panchenko<sup>2,3</sup></a>
            <a href="">Timothy Baldwin<sup>1,6</sup></a>
            <a href="">Preslav Nakov<sup>1</sup></a>
            <a href="">Maxim Panov<sup>1</sup></a>
          </div>
          <div class="linkContainer__institutes">
            <a href=""><sup>1</sup>MBZUAI</a>
            <a href=""><sup>2</sup>AIRI</a>
            <a href=""
              ><sup>3</sup>Center for Artificial Intelligence Technology</a
            >
            <a href=""><sup>4</sup>HSE University</a>
            <a href=""><sup>5</sup>FRC CSC RAS</a>
            <a href=""><sup>6</sup>The University of Melbourne</a>
            <a href=""><sup>7</sup>QCRI</a>
            <a href=""><sup>8</sup>Independent Researcher</a>
          </div>
        </div>

        <div class="header__button-container">
          <button class="header__button">
            <img
              class="header__button-logo"
              src="public/paper-svgrepo-com.svg"
              alt=""
            />
            <p class="header__button-text">Paper</p>
          </button>
          <button class="header__button">
            <img
              class="header__button-logo"
              src="public/GithubOutlined.svg"
              alt=""
            />
            <p class="header__button-text">Github</p>
          </button>
          <button class="header__button">
            <img
              class="header__button-logo"
              src="public/SearchOutlined.svg"
              alt=""
            />
            <p class="header__button-text">Search</p>
          </button>
        </div>
      </section>
    </header>
    <div class="body-container">
      <aside class="sidebar-section">
        <div class="sidebar">
          <ul class="nav flex-column">
            <li class="nav-item" data-target="#header">
              <a class="nav-link" href="#header">Name</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#text-section">
              <a class="nav-link" href="#text-section">Text</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#video-section">
              <a class="nav-link" href="#video-section">Video</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#text-section-border">
              <a class="nav-link" href="#text-section-border">Text border</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#swiper-section">
              <a class="nav-link" href="#swiper-section">Swiper</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#lighbox-section">
              <a class="nav-link" href="#lighbox-section">Lightbox</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#before-after-section">
              <a class="nav-link" href="#before-after-section">Before-after</a>
              <span class="indicator"></span>
            </li>

            <li class="nav-item" data-target="#swiper-on-button-section">
              <a class="nav-link" href="#swiper-on-button-section"
                >Swiper diagramms</a
              >
              <span class="indicator"></span>
            </li>

            <li class="nav-item" data-target="#diagrams-section">
              <a class="nav-link" href="#diagrams-section">Diagramms</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item ms-3" data-target="#BarChart">
              <a class="nav-link small" href="#BarChart">Bar chart</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item ms-3" data-target="#LineChart">
              <a class="nav-link small" href="#LineChart">Line chart</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item ms-3" data-target="#PieChart">
              <a class="nav-link small" href="#PieChart">Pie chart</a>
              <span class="indicator"></span>
            </li>

            <li class="nav-item" data-target="#table-section">
              <a class="nav-link" href="#table-section">Table</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#numbers-section">
              <a class="nav-link" href="#numbers-section">Number of claims</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#columns-section">
              <a class="nav-link" href="#columns-section">Columns</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#dropdown-section">
              <a class="nav-link" href="#dropdown-section">Dropdown</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#code-section">
              <a class="nav-link" href="#code-section">Code</a>
              <span class="indicator"></span>
            </li>
            <li class="nav-item" data-target="#bibTeX-section">
              <a class="nav-link" href="#bibTeX-section">bibTeX</a>
              <span class="indicator"></span>
            </li>
          </ul>
        </div>
      </aside>
      <section class="text-section" id="text-section">
        <div class="text-section__container">
          <h2>Abstract</h2>
          <p>Large language models (LLMs) are notorious for hallucinating, i. e., producing erroneous claims in&nbsp;their output. Such hallucinations can be&nbsp;dangerous, as&nbsp;occasional factual inaccuracies in&nbsp;the generated text might be&nbsp;obscured by&nbsp;the rest of&nbsp;the output being generally factually correct, making it&nbsp;extremely hard for the users to&nbsp;spot them. Current services that leverage LLMs usually do&nbsp;not provide any means for detecting unreliable generations. Here, we&nbsp;aim to&nbsp;bridge this gap. In&nbsp;particular, we&nbsp;propose a&nbsp;novel <nobr>fact-checking</nobr> and hallucination detection pipeline based on&nbsp;<nobr>token-level</nobr> uncertainty quantification. Uncertainty scores leverage information encapsulated in&nbsp;the output of&nbsp;a&nbsp;neural network or&nbsp;its layers to&nbsp;detect unreliable predictions, and we&nbsp;show that they can be&nbsp;used to&nbsp;<nobr>fact-check</nobr> the atomic claims in&nbsp;the LLM output. Moreover, we&nbsp;present a&nbsp;novel <nobr>token-level</nobr> uncertainty quantification method that removes the impact of&nbsp;uncertainty about what claim to&nbsp;generate on&nbsp;the current step and what surface form to&nbsp;use. Our method Claim Conditioned Probability (CCP) measures only the uncertainty of&nbsp;a&nbsp;particular claim value expressed by&nbsp;the model. Experiments on&nbsp;the task of&nbsp;biography generation demonstrate strong improvements for CCP compared to&nbsp;the baselines for seven LLMs and four languages. Human evaluation reveals that the <nobr>fact-checking</nobr> pipeline based on&nbsp;uncertainty quantification is&nbsp;competitive with a&nbsp;<nobr>fact-checking</nobr> tool that leverages external knowledge.</p>
        </div>
      </section>
      <section class="video-section" id="video-section">
        <div class="video-container">
          <h2>Video</h2>
          <div class="video-wrapper">
            <iframe
              class="iframe"
              src="https://rutube.ru/play/embed/c6cc4d620b1d4338901770a44b3e82f4/"
              frameborder="0"
              allow="clipboard-write; autoplay"
              webkitAllowFullScreen
              mozallowfullscreen
              allowfullscreen
            ></iframe>
          </div>
        </div>
      </section>
      <section class="text-section-border" id="text-section-border">
        <div class="text-section-border__container">
          <h2>Introduction</h2>
          <p> Large language models (LLMs) have become a&nbsp;ubiquitous and versatile tool for addressing a&nbsp;variety of&nbsp;natural language processing (NLP) tasks. People use these models for tasks including information search Sun et&nbsp;al. (2023b), to&nbsp;ask medical questions Thirunavukarasu et&nbsp;al. (2023), or&nbsp;to&nbsp;generate new content Sun et&nbsp;al. (2023a). Recently, there has been a&nbsp;notable shift in&nbsp;user behavior, indicating an&nbsp;increasing reliance on&nbsp;and trust in&nbsp;LLMs as&nbsp;primary information sources, often surpassing traditional channels. However, a&nbsp;significant challenge with the spread of&nbsp;these models is&nbsp;their tendency to&nbsp;produce &laquo;hallucinations&raquo;, i.e., factually incorrect generations that contain misleading information Bang et&nbsp;al. (2023); Dale et&nbsp;al. (2023). This is&nbsp;a&nbsp;<nobr>side-effect</nobr> of&nbsp;the way modern LLMs are designed and trained Kalai and Vempala (2023). </p> <p> LLM hallucinations are a&nbsp;major concern because the deceptive content at&nbsp;the surface level can be&nbsp;highly coherent and persuasive. Common examples include the creation of&nbsp;fictitious biographies or&nbsp;the assertion of&nbsp;unfounded claims. The danger is&nbsp;that a&nbsp;few occasional false claims might be&nbsp;easily obscured by&nbsp;a&nbsp;large number of&nbsp;factual statements, making it&nbsp;extremely hard for people to&nbsp;spot them. As&nbsp;hallucinations in&nbsp;LLM outputs are hard to&nbsp;eliminate completely, users of&nbsp;such systems could be&nbsp;informed via highlighting some potential caveats in&nbsp;the text, and this is&nbsp;where our approach can help. </p> <p> <nobr>Fact-checking</nobr> is&nbsp;a&nbsp;research direction that addresses this problem. It&nbsp;is&nbsp;usually approached using complex systems that leverage external knowledge sources Guo et&nbsp;al. (2022); Nakov et&nbsp;al. (2021); Wadden et&nbsp;al. (2020). This introduces problems related to&nbsp;the incomplete nature of&nbsp;such sources and notable overhead in&nbsp;terms of&nbsp;storing the knowledge. We&nbsp;argue that information about whether a&nbsp;generation is&nbsp;a&nbsp;hallucination is&nbsp;encapsulated in&nbsp;the model output itself, and can be&nbsp;extracted using uncertainty quantification (UQ) Gal et&nbsp;al. (2016); Kotelevskii et&nbsp;al. (2022); Vazhentsev et&nbsp;al. (2022, 2023a). This avoids implementing complex and expensive <nobr>fact-checking</nobr> systems that require additional computational overhead and rely on&nbsp;external resources. </p> <p> Prior work has mainly focused on&nbsp;quantification of&nbsp;uncertainty for the whole generated text and been mostly limited to&nbsp;tasks such as&nbsp;machine translation Malinin and Gales (2020), question answering Kuhn et&nbsp;al. (2023), and text summarization van der Poel et&nbsp;al. (2022). However, the need for an&nbsp;uncertainty score for only a&nbsp;part of&nbsp;the generation substantially complicates the problem. We&nbsp;approach it&nbsp;by&nbsp;leveraging <nobr>token-level</nobr> uncertainty scores and aggregating them into <nobr>claim-level</nobr> scores. Moreover, we&nbsp;introduce a&nbsp;new <nobr>token-level</nobr> uncertainty score, namely <nobr>claim-conditioned</nobr> probability (CCP), which demonstrates confident improvements over several baselines for seven LLMs and four languages. </p> <p> To&nbsp;the best of&nbsp;our knowledge, there is&nbsp;no&nbsp;previous work that has investigated the quality of&nbsp;<nobr>claim-level</nobr> UQ&nbsp;techniques for LLM generation. Therefore, for this purpose, we&nbsp;construct a&nbsp;novel benchmark based on&nbsp;<nobr>fact-checking</nobr> of&nbsp;biographies of&nbsp;individuals generated using a&nbsp;range of&nbsp;LLMs. Note that different LLMs produce different outputs, which generally have higher variability than, e.g., outputs in&nbsp;such tasks as&nbsp;machine translation or&nbsp;question answering. Therefore, we&nbsp;compare the predictions and uncertainty scores to&nbsp;the results of&nbsp;an&nbsp;automatic external <nobr>fact-checking</nobr> system FactScore Min et&nbsp;al. (2023). Human evaluation verifies that our constructed benchmark based on&nbsp;FactScore can adequately evaluate the performance of&nbsp;the uncertainty scores. </p> <p>Our contributions are as&nbsp;follows:</p> <ul> <li> We&nbsp;propose a&nbsp;novel framework for <nobr>fact-checking</nobr> LLM generations using <nobr>token-level</nobr> uncertainty quantification. We&nbsp;provide a&nbsp;procedure for efficiently estimating the uncertainty of&nbsp;atomic claims generated by&nbsp;a&nbsp;<nobr>white-box</nobr> model and highlighting potentially deceptive fragments by&nbsp;mapping them back to&nbsp;the original response. </li> <li> We&nbsp;propose a&nbsp;novel method for <nobr>token-level</nobr> uncertainty quantification that outperforms baselines and can be&nbsp;used as&nbsp;a&nbsp;<nobr>plug-in</nobr> a&nbsp;<nobr>fact-checking</nobr> framework. </li> <li> We&nbsp;design a&nbsp;novel approach to&nbsp;evaluation of&nbsp;<nobr>token-level</nobr> UQ&nbsp;methods for <nobr>white-box</nobr> LLMs based on&nbsp;<nobr>fact-checking</nobr>, which can be&nbsp;applied to&nbsp;other <nobr>white-box</nobr> LLMs. </li> <li> We&nbsp;provide an&nbsp;empirical and ablation analysis of&nbsp;the method for <nobr>fact-checking</nobr> of&nbsp;LLM generations, and find that the uncertainty scores we&nbsp;produce can help to&nbsp;spot claims with factual errors for seven LLMs over four languages: English, Chinese, Arabic, and Russian. </li> <li> The method is&nbsp;implemented as&nbsp;a&nbsp;part of&nbsp;the <nobr>LM-Polygraph</nobr> library Fadeeva et&nbsp;al. (2023). All the code and data for experiments is&nbsp;publicly available1. </li>
          </ul>
        </div>
      </section>
      <section class="swiper-section" id="swiper-section">
        <div class="swiper-container">
          <h2>Swiper</h2>
          <div class="swiper mySwiper">
            <div class="swiper-wrapper">
            <div class="swiper-slide"><img src="public/flower1.jpg" alt="" /></div>
            <div class="swiper-slide"><img src="public/flower2.jpg" alt="" /></div>
            <div class="swiper-slide"><img src="public/flower3.jpg" alt="" /></div>
            <div class="swiper-slide"><img src="public/flower4.jpg" alt="" /></div>
            <div class="swiper-slide"><img src="public/flower5.jpg" alt="" /></div>
          </div>
          <div class="swiper-button-next"></div>
          <div class="swiper-button-prev"></div>
        </div>
      <div class="swiper-lightbox" id="swiper-lightbox">
        <span class="close-btn" id="swiper-lightbox__close">&times;</span>
        <img class="swiper-lightbox-content" id="swiper-lightbox-img" />
      </div>
      </section>
      <section class="lighbox-section" id="lighbox-section">
        <div class="lighbox-container">
          <h2>Lightbox</h2>
          
          <div class="lighbox-row">
            <img src="public/x5.png" alt="" class="thumbnail" id="openLightbox1" />
          
            <img src="public/x4.jpg" alt="" class="thumbnail" id="openLightbox2" />
        
            <img src="public/x6.png" alt="" class="thumbnail" id="openLightbox3" />
        
          </div>
          
          <div class="lightbox" id="lightbox1">
            <span class="close-btn" id="closeLightbox1">&times;</span>
            <img class="lightbox-content" id="lightbox-img1" />
          </div>
      
          <div class="lightbox" id="lightbox2">
            <span class="close-btn" id="closeLightbox2">&times;</span>
            <img class="lightbox-content" id="lightbox-img2" />
          </div>
      
          <div class="lightbox" id="lightbox3">
            <span class="close-btn" id="closeLightbox3">&times;</span>
            <img class="lightbox-content" id="lightbox-img3" />
          </div>
        </div>
      </section>

      <section class="before-after-section" id="before-after-section">
        <h2>Before-after</h2>
        <div class="compare-container">
          <div class="compare-images">
            <img
              src="public/flower1.jpg"
              alt="Before"
              class="compare-img before-img"
            />

            <img
              src="public/flower2.jpg"
              alt="After"
              class="compare-img after-img"
            />

            <div class="slider-handle"></div>
          </div>
        </div>
      </section>
      <section class="swiper-on-button-section" id="swiper-on-button-section">
        <div class="swiper-on-button__container">
          <h2>Swiper with buttons</h2>
          <div class="swiper-buttons">
            <button class="swiper-button active" data-slide-to="0">
              Slide 1
            </button>
            <button class="swiper-button" data-slide-to="1">Slide 2</button>
            <button class="swiper-button" data-slide-to="2">Slide 3</button>
          </div>

          <div class="swiper my-swiper-on-button">
            <div class="swiper-wrapper">
              <div class="swiper-slide">
                <canvas
                  class="myChart-slide swiper-slide"
                  id="BarChart-slider"
                ></canvas>
              </div>
              <div class="swiper-slide">
                <canvas
                  class="myChart-slide swiper-slide"
                  id="LineChart-slider"
                ></canvas>
              </div>
              <div class="swiper-slide">
                <canvas
                  class="myChart-slide swiper-slide"
                  id="PieChart-slider"
                ></canvas>
              </div>
            </div>
          </div>
        </div>
      </section>

      <section class="diagrams-section" id="diagrams-section">
        <div class="diagrams-section__container">
          <h2>Charts</h2>
          <canvas class="myChart" id="BarChart"></canvas>
          <p class="caption">
            The comparison of token-level uncertainty quantification methods in
            terms of ROC-AUC scores, measured for Chinese dataset. The results
            are split into bins when considering only facts from the first 2, 5,
            and all sentences.
          </p>
          <canvas class="myChart" id="LineChart"></canvas>
          <p class="caption">
            The comparison of token-level uncertainty quantification methods in
            terms of ROC-AUC scores, measured for Chinese dataset. The results
            are split into bins when considering only facts from the first 2, 5,
            and all sentences.
          </p>
          <canvas class="myChart" id="PieChart"></canvas>
          <p class="caption">
            The comparison of token-level uncertainty quantification methods in
            terms of ROC-AUC scores, measured for Chinese dataset. The results
            are split into bins when considering only facts from the first 2, 5,
            and all sentences.
          </p>
        </div>
      </section>

      <section class="table-section" id="table-section">
        <h2>
          ROC-AUC of claim-level UQ methods with manual annotation as the ground
          truth.
        </h2>
        <div class="table-section__container">
          <table>
            <thead>
              <th>Model</th>
              <th>Yi 6b, Chinese</th>
              <th>Jais 13b, Arabic</th>
              <th>GPT-4, Arabic</th>
              <th>Vikhr 7b, Russian</th>
            </thead>
            <tbody>
              <tr>
                <td>CCP (ours)</td>
                <td>0.64 ± 0.03</td>
                <td>0.66 ± 0.02</td>
                <td>0.56 ± 0.05</td>
                <td>0.68 ± 0.04</td>
              </tr>
              <tr>
                <td>Maximum Prob.</td>
                <td>0.52 ± 0.03</td>
                <td>0.59 ± 0.02</td>
                <td>0.55 ± 0.08</td>
                <td>0.63 ± 0.04</td>
              </tr>
              <tr>
                <td>Perplexity</td>
                <td>0.51 ± 0.04</td>
                <td>0.56 ± 0.02</td>
                <td>0.54 ± 0.08</td>
                <td>0.58 ± 0.04</td>
              </tr>
              <tr>
                <td>Token Entropy</td>
                <td>0.51 ± 0.04</td>
                <td>0.56 ± 0.02</td>
                <td>0.54 ± 0.08</td>
                <td>0.58 ± 0.04</td>
              </tr>
              <tr>
                <td>P(True)</td>
                <td>0.52 ± 0.03</td>
                <td>0.59 ± 0.02</td>
                <td>0.55 ± 0.08</td>
                <td>0.63 ± 0.04</td>
              </tr>
            </tbody>
          </table>
        </div>
      </section>
      <section class="numbers-section" id="numbers-section">
        <div class="numbers-section__container">
          <h2>Number of claims</h2>
          <div class="numbers__container">
            <div class="number__card">
              <h1>100</h1>
              <p>Vicuna 13b, English</p>
            </div>
            <div class="number__card">
              <h1>1,603</h1>
              <p>Yi 6b, Chinese</p>
            </div>
            <div class="number__card">
              <h1>200</h1>
              <p>GPT-4, Arabic</p>
            </div>
            <div class="number__card">
              <h1>146</h1>
              <p>Vikhr 7b, Russian</p>
            </div>
          </div>
        </div>
      </section>
      <section class="columns-section" id="columns-section">
        <div class="columns-section__container">
          <h2>Datasets and Statistics</h2>

          <div class="row g-6 justify-content-between">
            <div class="col-md-6 p-2">
              <div class="column">
                <p>For Arabic, using <nobr>GPT-4</nobr>, we&nbsp;generate 100 biographies of&nbsp;people randomly selected from the list of&nbsp;the most visited websites in&nbsp;Arabic Wikipedia. The used Arabic prompt is&nbsp;the translation of: &laquo;Tell me&nbsp;the biography of&nbsp;{person name}&raquo;. To&nbsp;extract claims, we&nbsp;prompt <nobr>GPT-4</nobr> in&nbsp;the following way: &laquo;Convert the following biography into Arabic atomic factual claims that can be&nbsp;verified, one claim per line. Biography is: {biography}&raquo;. Arabic biographies and claims are translated into English using Google Translate. It&nbsp;is&nbsp;worth mentioning that almost <nobr>one-third</nobr> of&nbsp;the names in&nbsp;the list of&nbsp;person names are foreign</p>
              </div>
            </div>
            <div class="col-md-6 p-2">
              <div class="column">
                <p>For Jais 13b experiments, we&nbsp;use the same prompts used for <nobr>GPT-4</nobr>. We&nbsp;notice that the biographies generated by&nbsp;Jais 13b are much shorter than the ones generated by&nbsp;<nobr>GPT-4</nobr> (almost <nobr>half-length</nobr>). Similarly, we&nbsp;use <nobr>GPT-4</nobr> to&nbsp;extract claims from the generated biographies. On&nbsp;average, biographies generated by&nbsp;Jais 13b have nine claims. Jais 13b generates empty biographies for seven names (out of&nbsp;100) with response messages like: &laquo;I&nbsp;am&nbsp;sorry! I&nbsp;cannot provide information about {name}&raquo;, or&nbsp;&laquo;What do&nbsp;you want to&nbsp;know exactly?&raquo;. Two random claims from each biography are verified manually (total = 186 claims).</p>
              </div>
            </div>
          </div>
          <div class="row g-6 justify-content-between">
            <div class="col-md-4 p-2">
              <div class="column">
                <p>Since FactScore only supports English, for Arabic, Chinese, and Russian, we&nbsp;generate biographies of&nbsp;<nobr>well-known</nobr> people and annotate them only manually. We&nbsp;also manually annotate English claims generated by&nbsp;Vicuna 13b. The statistics for annotated datasets are presented in&nbsp;Table 7.</p>
              </div>
            </div>
            <div class="col-md-4 p-2">
              <div class="column">
                <p>For Chinese, we&nbsp;first prompt ChatGPT to&nbsp;generate a&nbsp;list of&nbsp;famous people. Then use the same way as&nbsp;we&nbsp;have done Arabic, but change the prompt to&nbsp;Chinese, to&nbsp;biographies and claims. We&nbsp;use Yi&nbsp;6b to&nbsp;generate texts <nobr>GPT-4</nobr> to&nbsp;split them into atomic claims.</p>
              </div>
            </div>
            <div class="col-md-4 p-2">
              <div class="column">
                <p>For Russian, we&nbsp;conduct a&nbsp;similar approach, prompting to&nbsp;generate a&nbsp;list of&nbsp;100 famous people and checking result to&nbsp;obtain representative personalities from different areas such as&nbsp;science, sport, literature, art, activity, cinematography, heroes, etc.</p>
              </div>
            </div>
          </div>
        </div>
      </section>
      <section class="dropdown-section" id="dropdown-section">
        <div class="dropdown-section__container">
          <h2>Ablation Studies</h2>

          <div class="dropdown__row">
            <div class="dropdown" data-dropdown>
              <button class="dropdown__button">
                Aggregation of C⁢C⁢Pword for obtaining C⁢C⁢Pclaim.
              </button>
              <div class="dropdown__content">
                <div class="withline__container">
                  <span class="vertical-line blue-line"></span>
                  <div class="dropdown__item" data-value="opt1-1">
                    Besides the product of probabilities, we also tried the
                    normalized product, minimum, and average probability. All
                    these approaches perform slightly worse than the product
                    (see Table 9).
                  </div>
                </div>
              </div>
            </div>
            <div class="dropdown" data-dropdown>
              <button class="dropdown__button">NLI model</button>
              <div class="dropdown__content">
                <div class="withline__container">
                  <span class="vertical-line red-line"></span>
                  <div class="dropdown__item" data-value="opt2-1">
                    We investigate the influence of the specific NLI model on
                    the performance of CCP. Table 10 shows that CCP’s
                    effectiveness is not critically dependent on the complexity
                    of the NLI model employed. Notably, even a relatively small
                    model with 22M parameters maintains strong performance
                    without any degradation.
                  </div>
                </div>
              </div>
            </div>

            <div class="dropdown" data-dropdown>
              <button class="dropdown__button">NLI context</button>
              <div class="dropdown__content">
                <div class="withline__container">
                  <span class="vertical-line green-line"></span>
                  <div class="dropdown__item" data-value="opt3-1">
                    We analyze what context is sufficient for NLI in CCP (Table
                    11). In addition to the standard variant in CCP, where we
                    keep the claim that precedes the word in question, we
                    experiment with a single target word without context and the
                    whole sentence that precedes the target word. All variants
                    demonstrate lower performance. No context results in a drop
                    of 0.02 ROC-AUC, and longer contexts – of more than 0.07.
                  </div>
                </div>
              </div>
            </div>
          </div>
        </div>
      </section>
      <section class="code-section" id="code-section">
        <div class="code-section__container">
          <h2>Code</h2>
          <pre><code><div class="code__container" ><p class="code">items.forEach((item) => {
        item.addEventListener('click', function () {
        const value = this.textContent
        selectedValue.textContent = value
        dropdown.classList.remove('open')
    })
})</p></div></code></pre>
        </div>
      </section>

      <section class="bibTeX-section" id="bibTeX-section">
        <div class="bibTeX-section__container">
          <h2>BibTeX</h2>
          <pre><code><div class="bibTeX__container" ><p class="bibTeX">@misc{salnikov2025geopolitical,
    title={Geopolitical biases in LLMs: what are the "good" and the "bad" countries according to contemporary language models}, 
    author={Mikhail Salnikov and Dmitrii Korzh and Ivan Lazichny and Elvir Karimov and Artyom Iudin and Ivan Oseledets and Oleg Y. Rogov and Alexander Panchenko and Natalia Loukachevitch and Elena Tutubalina},
    year={2025},
    eprint={2506.06751},
    archivePrefix={arXiv},
    primaryClass={cs.CL},
    url={https://arxiv.org/abs/2506.06751}
}</p></div></code></pre>
          <button class="copy-button copy-button--active">
            <p class="active-content">Copy</p>
            <img src="public/ok.svg" class="inactive-content" alt="" />
          </button>
        </div>
      </section>
    </div>
    <footer id="footer">
      <section class="footer-section">
        <div class="footer-section__container">
          <div class="footer__information">
            <div class="airi-logo-section">
              <img
                class="participants-logo"
                src="public/airi_logo_white.svg"
                alt="MTUCI"
              />
              <p>This page was build with using AIRI Institute template</p>
            </div>

            <div class="footer__mails">

              <div class="mail__contanct">
                <a href="mailto:parther@airi.net" class="mail" type="email"
                  >parther@airi.net</a
                >
                <p class="mail__description">
                  For scientific cooperation and partnership
                </p>
              </div>
              <div class="mail__contanct">
                <a href="mailto:people@airi.net" class="mail" type="email"
                  >people@airi.net</a
                >
                <p class="mail__description">
                  For any questions connected with employees and employment
                </p>
              </div>
            </div>
            <div class="footer__social-media">
              <p class="gray">Join AIRI</p>
              <div class="social-media__row">
                <svg
                  class="media-logo"
                  width="40"
                  height="40"
                  viewBox="0 0 50 50"
                  fill="none"
                  xmlns="http://www.w3.org/2000/svg"
                >
                  <path
                    d="M25 6.10352e-05C11.193 6.10352e-05 0 11.1931 0 25.0001C0 38.807 11.193 50.0001 25 50.0001C38.807 50.0001 50 38.807 50 25.0001C50 11.1931 38.807 6.10352e-05 25 6.10352e-05ZM18.3059 36.6565H12.2754V18.5148H18.3059V36.6565V36.6565ZM15.2909 16.0393H15.2503C13.2275 16.0393 11.9165 14.6446 11.9165 12.9031C11.9165 11.1231 13.2665 9.76898 15.3304 9.76898C17.3943 9.76898 18.6633 11.1226 18.7029 12.9031C18.7029 14.6451 17.3949 16.0393 15.2909 16.0393ZM40.0547 36.6565H34.0237V26.9504C34.0237 24.5124 33.1512 22.8485 30.9686 22.8485C29.3022 22.8485 28.3106 23.9699 27.874 25.0548C27.7149 25.4427 27.6738 25.9826 27.6738 26.525V36.6565H21.6408C21.6408 36.6565 21.7219 20.2178 21.6408 18.5148H27.6728V21.0862C28.4743 19.8517 29.9055 18.0885 33.1091 18.0885C37.0782 18.0885 40.0536 20.6811 40.0536 26.2538V36.6565H40.0547Z"
                  ></path>
                </svg>

                <svg
                  class="media-logo"
                  xmlns="http://www.w3.org/2000/svg"
                  width="40"
                  height="40"
                  fill="none"
                >
                  <path
                    fill-rule="evenodd"
                    d="M20 40c11.046 0 20-8.954 20-20S31.046 0 20 0 0 8.954 0 20s8.954 20 20 20Zm9.754-30H26.38l-5.622 6.391L15.957 10H9l8.284 10.922L9.44 30h3.372l6.085-6.996L24.212 30H31l-8.65-11.513L29.754 10ZM27.07 28.007H25.2L12.966 11.92h2.009L27.07 28.007Z"
                    clip-rule="evenodd"
                  ></path>
                </svg>
                <svg
                  class="media-logo"
                  xmlns="http://www.w3.org/2000/svg"
                  width="40"
                  height="40"
                  fill="none"
                >
                  <path
                    fill-rule="evenodd"
                    d="M40 20c0 11.046-8.954 20-20 20S0 31.046 0 20 8.954 0 20 0s20 8.954 20 20Zm-20.068-6.124c-2.06.854-6.177 2.622-12.35 5.303-1.003.398-1.529.787-1.576 1.167-.081.642.726.895 1.825 1.24.15.047.304.095.463.147 1.082.35 2.536.76 3.292.776.686.015 1.452-.267 2.297-.846 5.769-3.881 8.746-5.843 8.933-5.885.132-.03.315-.068.439.042.124.11.111.318.098.373-.08.34-3.248 3.276-4.887 4.796-.512.473-.874.81-.948.886-.166.172-.335.335-.498.49-1.005.966-1.758 1.69.042 2.872.864.568 1.556 1.038 2.247 1.507.754.512 1.506 1.022 2.479 1.658.248.162.485.33.715.494.878.624 1.666 1.184 2.64 1.095.566-.052 1.15-.583 1.447-2.165.702-3.739 2.081-11.84 2.4-15.178a3.714 3.714 0 0 0-.036-.83c-.028-.165-.087-.4-.301-.572-.254-.206-.645-.249-.82-.246-.797.014-2.02.438-7.901 2.876Z"
                    clip-rule="evenodd"
                  ></path>
                </svg>
                <svg
                  class="media-logo"
                  xmlns="http://www.w3.org/2000/svg"
                  width="40"
                  height="40"
                  fill="none"
                >
                  <path
                    fill-rule="evenodd"
                    d="M20 40c11.046 0 20-8.954 20-20S31.046 0 20 0 0 8.954 0 20s8.954 20 20 20ZM7 13c.211 9.994 5.278 16 14.161 16h.504v-5.718c3.264.32 5.732 2.675 6.723 5.718H33c-1.267-4.549-4.596-7.063-6.675-8.024 2.079-1.185 5.002-4.068 5.7-7.976h-4.19c-.909 3.171-3.604 6.054-6.17 6.326V13h-4.19v11.083c-2.599-.64-5.88-3.748-6.025-11.083H7Z"
                    clip-rule="evenodd"
                  ></path>
                </svg>
                <svg
                  class="media-logo"
                  xmlns="http://www.w3.org/2000/svg"
                  width="40"
                  height="40"
                  fill="none"
                >
                  <path
                    fill-rule="evenodd"
                    d="M40 20c0 11.046-8.954 20-20 20S0 31.046 0 20 8.954 0 20 0s20 8.954 20 20ZM20.198 8.282c6.187 0 11.203 4.985 11.203 11.134a11.1 11.1 0 0 1-.302 2.577 1.678 1.678 0 0 0-1.266-.55c-.416 0-.843.136-1.27.406-.285.179-.598.496-.921.895-.3-.412-.718-.686-1.197-.761a1.788 1.788 0 0 0-.277-.022c-1.094 0-1.752.943-2 1.791-.124.287-.717 1.592-1.608 2.477-1.361 1.352-1.7 2.747-1.02 4.242a11.39 11.39 0 0 1-2.736-.006c.676-1.493.337-2.886-1.022-4.236-.892-.885-1.484-2.19-1.608-2.477-.248-.848-.906-1.79-2-1.79-.092 0-.185.006-.277.02-.48.076-.898.35-1.197.762-.323-.399-.637-.716-.92-.895-.428-.27-.856-.407-1.271-.407a1.68 1.68 0 0 0-1.223.504c-.19-.813-.29-1.66-.29-2.53 0-6.15 5.015-11.134 11.202-11.134ZM32.69 19.416c0 1.297-.2 2.548-.571 3.723l.07-.001c.496 0 .945.19 1.263.533.407.441.589.984.51 1.526-.038.259-.125.49-.255.705.274.22.476.526.574.895.076.29.154.891-.255 1.511.026.04.05.083.074.126.245.463.26.987.044 1.474-.328.739-1.145 1.321-2.731 1.946a21.05 21.05 0 0 1-1.897.64c-1.305.336-2.484.507-3.506.507-1.668 0-2.915-.452-3.715-1.344a12.648 12.648 0 0 1-4.24-.008c-.8.897-2.05 1.352-3.724 1.352-1.021 0-2.2-.171-3.505-.507-.008-.003-.91-.251-1.897-.64-1.586-.625-2.403-1.207-2.732-1.946a1.653 1.653 0 0 1 .118-1.6 1.861 1.861 0 0 1-.254-1.51c.098-.37.3-.676.573-.896a1.856 1.856 0 0 1-.254-.705c-.079-.542.102-1.085.51-1.526a1.693 1.693 0 0 1 1.39-.529 12.34 12.34 0 0 1-.573-3.726C7.707 12.559 13.299 7 20.198 7c6.9 0 12.492 5.559 12.492 12.416Zm-6.68 12.16c-.751 0-1.49-.047-2.085-.306l.004-.002a2.272 2.272 0 0 1-.872-.737c-.07-.101-.135-.2-.193-.298l-.009.002c-.64-1.117-.487-2.013.612-3.104 1.21-1.203 1.915-2.962 1.915-2.962s.24-.933.786-.933c.025 0 .05.002.077.006.6.094 1.04 1.62-.216 2.554-.835.621-.449 1.11.046 1.11.249 0 .525-.125.687-.418.483-.877 1.803-3.132 2.487-3.563.22-.139.42-.208.584-.208.344 0 .53.305.42.907-.08.441-.825 1.194-1.52 1.898-.707.713-1.363 1.376-1.223 1.61a.239.239 0 0 0 .219.123c.392 0 1.04-.67 1.04-.67s2.403-2.172 3.42-2.172c.132 0 .24.036.318.12.592.64-.279 1.154-1.649 1.963-.177.105-.362.214-.554.33-1.674 1.007-1.784 1.317-1.546 1.698.014.023.04.033.076.033.193 0 .682-.292 1.262-.638.845-.504 1.885-1.124 2.483-1.124.233 0 .4.094.46.326.188.709-.93 1.12-1.959 1.498-.876.322-1.686.62-1.564 1.057.038.137.138.24.27.24.281 0 .735-.249 1.217-.513.546-.298 1.127-.616 1.532-.616.189 0 .339.069.43.24.542 1.023-3.735 2.144-3.77 2.153a13.92 13.92 0 0 1-3.185.396Zm-8.517-1.353c.633-1.112.477-2.005-.618-3.092-1.21-1.203-1.916-2.962-1.916-2.962s-.24-.933-.785-.933a.49.49 0 0 0-.077.006c-.6.094-1.04 1.62.215 2.554.836.621.45 1.11-.045 1.11a.781.781 0 0 1-.688-.418c-.483-.877-1.803-3.132-2.487-3.563-.22-.139-.42-.208-.583-.208-.344 0-.53.305-.42.907.08.441.824 1.194 1.52 1.898.707.713 1.363 1.376 1.223 1.61a.239.239 0 0 1-.219.123c-.392 0-1.04-.67-1.04-.67s-2.403-2.172-3.42-2.172c-.132 0-.241.036-.318.12-.592.64.278 1.154 1.648 1.963.177.105.363.214.554.33 1.675 1.007 1.785 1.317 1.547 1.698-.014.023-.04.033-.076.033-.193 0-.682-.292-1.262-.637-.846-.505-1.885-1.125-2.483-1.125-.234 0-.4.094-.461.326-.188.709.931 1.12 1.96 1.498.876.322 1.686.62 1.564 1.057-.039.137-.138.24-.271.24-.28 0-.735-.249-1.217-.513-.545-.298-1.126-.616-1.531-.616-.189 0-.34.069-.43.24-.542 1.023 3.734 2.144 3.769 2.153.703.181 1.96.396 3.185.396.761 0 1.51-.048 2.108-.317l-.005-.001c.34-.173.632-.41.852-.727a5.21 5.21 0 0 0 .2-.31l.007.002Zm5.56-10.463c.84-.414 1.367-.673 1.367.092 0 .956-.458 2.511-1.687 3.454-.62.477-1.439.798-2.495.805h-.001c-1.056-.008-1.875-.328-2.496-.805-1.23-.943-1.687-2.498-1.687-3.454 0-.765.527-.506 1.366-.092.743.366 1.731.853 2.817.858 1.086-.005 2.074-.492 2.817-.858Zm-2.519 2.688c.063-.085.11-.168.152-.243-.043.076-.09.158-.152.243Zm7.16-4.602a1.036 1.036 0 1 0 0-2.072 1.036 1.036 0 0 0 0 2.072ZM13.798 16.81a1.036 1.036 0 1 1-2.071 0 1.036 1.036 0 0 1 2.071 0Zm10.242-.451c.174.061.304.249.428.427.166.241.322.465.56.34a1.604 1.604 0 0 0 .666-2.173 1.613 1.613 0 0 0-2.178-.664 1.604 1.604 0 0 0-.666 2.172c.11.207.356.11.614.006.203-.08.414-.165.576-.108Zm-8.015.427c.124-.178.254-.366.428-.427.161-.057.373.027.575.108.26.104.504.201.615-.006a1.604 1.604 0 0 0-.666-2.172 1.613 1.613 0 0 0-2.178.664 1.604 1.604 0 0 0 .666 2.172c.238.127.393-.098.56-.339Z"
                    clip-rule="evenodd"
                  ></path>
                </svg>
              </div>
            </div>
          </div>

          <p class="copyrighted">© 2025, AIRI</p>
        </div>
      </section>
    </footer>

    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/swiper@11/swiper-bundle.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
    <script src="scripts/sidebar.js"></script>
    <script src="scripts/beforeAfter.js"></script>
    <script src="scripts/swiper.js"></script>
    <script src="scripts/swiperCharts.js"></script>
    <script src="scripts/pieChart.js"></script>
    <script src="scripts/barChart.js"></script>
    <script src="scripts/lineChart.js"></script>
    <script src="scripts/pieChartSlider.js"></script>
    <script src="scripts/barChartSlider.js"></script>
    <script src="scripts/lineChartSlider.js"></script>
    <script src="scripts/lightbox.js"></script>
    <script src="scripts/dropdown.js"></script>
    <script src="scripts/codeCopy.js"></script>
  </body>
</html>