Upload 9 files

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/en.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+venv

OpenAIAPIModel.py

@@ -0,0 +1,50 @@
+import requests
+import time
+
+class GroqOpenAIAPIModel:
+    def __init__(self, api_key, url="https://api.groq.com/openai/v1/chat/completions", model="llama3-8b-8192"):
+        self.url = url
+        self.model = model
+        self.API_KEY = api_key
+
+        print(f"GroqOpenAIAPIModel initialized with model: {self.model}")
+
+    def generate(self, text: str, temperature=0.7, system="You are a helpful assistant.", top_p=1):
+        time.sleep(5)
+        headers = {
+            "Authorization": f"Bearer {self.API_KEY}",
+            "Content-Type": "application/json"
+        }
+
+        query = {
+            "model": self.model,
+            "temperature": temperature,
+            "top_p": top_p,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": text}
+            ],
+            "stream": False
+        }
+
+        try:
+            response = requests.post(self.url, headers=headers, json=query)
+            response.raise_for_status()
+
+            response_json = response.json()
+            if 'choices' not in response_json:
+                print("⚠️ 'choices' missing in response")
+                print("🔍 Input text:", text)
+                print("📦 Full response:", response_json)
+                return "[ERROR] Response missing 'choices'"
+
+            return response_json['choices'][0]['message']['content']
+
+        except requests.exceptions.RequestException as e:
+            print("❌ HTTP error during API call:", e)
+            print("🔍 Input text:", text)
+            return "[ERROR] API call failed"
+        except Exception as e:
+            print("❌ Unexpected error:", e)
+            print("🔍 Input text:", text)
+            return "[ERROR] Unexpected failure"

README.md

@@ -1,12 +1,12 @@
----
-title: RGB
-emoji: 💻
-colorFrom: red
-colorTo: pink
-sdk: gradio
-sdk_version: 5.38.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: RGB26Demo
+emoji: 📉
+colorFrom: gray
+colorTo: purple
+sdk: gradio
+sdk_version: 5.38.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,612 @@
+import time
+import gradio as gr
+import os
+import json
+import random
+import yaml
+import tqdm
+from utils import processdata, predict, supported_models
+from OpenAIAPIModel import GroqOpenAIAPIModel  # Adjusted import based on the provided context
+import utils
+import requests
+import pandas as pd
+
+def check_counterfactual(question, answer, url, apikey):
+    prompt = '''Please determine whether the model can identify factual errors in the document. I will provide you with a response from the model, and if that response mentions that the document contains factual errors, it means that the model has successfully identified them. Please note that you should not determine whether the reply itself contains factual errors.
+        Demonstrations:
+        Response:文档信息有误，2008年的奥运会在北京举办。
+        Yes, the model has identified the factual errors.
+
+        Response:2014年澳网女单冠军是大阪直美。
+        NO, the model fail to identify the factual errors.
+
+        Response: The director of the Silent Hill movie is Justin Kurzel.
+        NO, the model fail to identify the factual errors.
+
+        Response: Harry Potter is written by J. K. Rowling.
+        NO, the model fail to identify the factual errors.
+
+        Response:  There are factual errors in the provided documents. The correct answer is 2023.
+        Yes, the model has identified the factual errors.
+
+        Begin to generate:
+        Answer: {answer}
+            '''
+    text2 = prompt.format(answer=answer)
+    # return getdata(text2,url,apikey)
+    text2 = prompt.format(question=question,answer=answer)
+    return get_groq_response(text2, apikey)
+
+
+def check(question, answer, url, apikey):
+    prompt = '''I will give you a question and an answer generated through document retrieval. Please use this answer to determine if the retrieved document can solve the question.
+        Demonstrations:
+        Question: 2023年澳网女单冠军是谁
+        Answer:文档信息不足，因此我无法基于提供的文档回答该问题。
+        No, the question is not addressed by the documents.
+
+        Question: Who is the champion of Australian Open 2023 Women's Singles?
+        Answer: Serena Williams
+        Yes, the question is addressed by the documents.
+
+        Question: Where is ACL2023 held?
+        Answer: Location of ACL2023 has not been confirmed.
+        No, the question is not addressed by the documents.
+
+        Question:  2023年中国GDP是多少?
+        Answer: I can not answer this question。
+        No, the question is not addressed by the documents.
+
+        Begin to generate:
+        Question: {question}
+        Answer: {answer}
+    '''
+    text2 = prompt.format(question=question,answer=answer)
+    return get_groq_response(text2, apikey)
+
+def get_groq_response(prompt, api_key):
+    if api_key == "":
+        api_key = os.environ.get("GROQ_API_KEY")  # Safely loaded from HF Secrets
+
+    url = "https://api.groq.com/openai/v1/chat/completions"
+    headers = {
+        "Authorization": f"Bearer {api_key}", 
+        "Content-Type": "application/json"
+        }
+    data = {
+        "model": "llama-3.3-70b-versatile",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.7
+    }
+    for attempt in range(3):
+        try:
+            response = requests.post(url, json=data, headers=headers)
+            response.raise_for_status()  # Raise HTTP errors
+            json_response = response.json()
+            
+            if "choices" not in json_response:
+                print(f"Unexpected response format: {json_response}")
+                return "Error: Invalid API response format."
+                
+            return json_response["choices"][0]["message"]["content"]
+            
+        except requests.exceptions.RequestException as e:
+            print(f"Attempt {attempt + 1} failed: {e}")
+            time.sleep(2)  # Backoff before retry
+    
+    return "Error: Max retries reached."
+
+
+def run_reject_rate(
+    modelname='chatgpt',
+    dataset='en',
+    api_key='api_key',
+    url='https://api.openai.com/v1/completions',
+    temperature=0.7,
+    noise_rate=0.0,
+    correct_rate=0.0,
+    passage_num=5,
+    factchecking=False,
+    max_instances=2
+):
+    # Result path (in working dir)
+    resultpath = 'results/result-en' if 'en' in dataset else 'results/result-zh'
+
+    modelname = modelname.replace('/', '_')  # Replace '/' with '_' for file naming
+
+    evaluefile = f'{resultpath}/prediction_{dataset}_{modelname}_temp{temperature}_noise{1}_passage{passage_num}_correct{0}.json'
+
+    outputfile = f'{resultpath}/prediction_{dataset}_{modelname}_temp{temperature}_noise{1}_passage{passage_num}_correct{0}_chatgpt.json'
+
+    resultfile = f'{resultpath}/prediction_{dataset}_{modelname}_temp{temperature}_noise{1}_passage{passage_num}_correct{0}_chatgptresult.json'
+
+    results = []
+    useddata = {}
+    if os.path.exists(outputfile):
+        with open(outputfile) as f:
+            for line in f:
+                data = json.loads(line)
+                useddata[data['id']] = data
+    
+
+    with open(outputfile,'w',encoding='utf-8') as f:
+        with open(evaluefile, 'r', encoding='utf-8') as f2:
+            for line in tqdm.tqdm(f2):
+                data = json.loads(line)
+                if data['id'] in useddata and data['query'] == useddata[data['id']]['query'] and data['ans']  == useddata[data['id']]['ans'] :
+                    results.append(useddata[data['id']])
+                    f.write(json.dumps(useddata[data['id']],ensure_ascii=False)+'\n')
+                    continue
+                try:
+                    question = data['query']
+                    answer = data['prediction']
+                    
+                    evaluation = check(question, answer, url, api_key)
+                    data['evaluation'] = evaluation
+                    results.append(data)
+                    f.write(json.dumps(data,ensure_ascii=False)+'\n')
+                except Exception as e:
+                    print(f"Exception Generated: {e}")
+                    print(f"Questions :{question}, Answer :{answer}")
+                    continue
+    
+    rejecttt = 0
+    tt = 0
+    for i in results:
+        if "not addressed" in i['evaluation']:
+            rejecttt += 1
+        if 0 not in i['label'] and 1 in i['label']:
+            tt += 1
+    print(tt/len(results))
+    scores = {
+        'reject_rate': rejecttt/len(results),
+        'all_rate': (tt)/len(results),
+        'tt':tt,
+        'rejecttt':rejecttt,
+        'nums': len(results),
+    }
+    # json.dump(scores, open(resultfile, 'w', encoding='utf-8'), ensure_ascii=False, indent=4)
+
+    try:
+        utils.upload_file(outputfile, "")
+    except Exception as e:
+        print("Error saving outputfile {outputfile}:", e)
+
+    # Save results
+    try:
+        finalResults = {
+            'model': modelname,
+            'dataset': dataset,
+            'temperature': temperature,
+            'noise_rate': noise_rate,
+            'passage_num': passage_num,
+            'correct_rate': correct_rate,
+            'factchecking': factchecking,
+            'scores': scores,
+        } 
+        with open(resultfile, 'w') as f:
+            json.dump(finalResults, f, ensure_ascii=False, indent=4)
+        utils.upload_file(resultfile, "")
+    except Exception as e:
+        print("Error saving scores:", e)
+
+    return finalResults
+
+
+def run_information_integration(
+    modelname='chatgpt',
+    dataset='en_int',
+    api_key='api_key',
+    url='https://api.openai.com/v1/completions',
+    temperature=0.7,
+    noise_rate=0.0,
+    correct_rate=0.0,
+    passage_num=5,
+    factchecking=False,
+    max_instances=2
+):
+    return run_evaluation(
+        modelname=modelname,
+        dataset="en_int",
+        api_key=api_key,
+        url=url,
+        temperature=temperature,
+        noise_rate=noise_rate,
+        correct_rate=correct_rate,
+        passage_num=passage_num,
+        factchecking=factchecking,
+        max_instances=max_instances
+    )
+
+
+def run_counter_factual_checking(
+    modelname='chatgpt',
+    dataset='en_fact',
+    api_key='api_key',  # API key for the model
+    url='https://api.openai.com/v1/completions',
+    temperature=0.7,
+    noise_rate=0.0,
+    correct_rate=0.0,
+    passage_num=5,
+    factchecking=False,
+    max_instances=2
+):
+    resultpath = 'results/result-en' if 'en' in dataset else 'results/result-zh'
+    modelname = modelname.replace('/', '_')  # Replace '/' with '_' for file naming
+
+    evaluefile = f'{resultpath}/prediction_{dataset}_{modelname}_temp{temperature}_noise{noise_rate}_passage{passage_num}_correct{correct_rate}.json'
+    outputfile = f'{resultpath}/prediction_{dataset}_{modelname}_temp{temperature}_noise{noise_rate}_passage{passage_num}_correct{correct_rate}_chatgpt.json'
+    resultfile = f'{resultpath}/prediction_{dataset}_{modelname}_temp{temperature}_noise{noise_rate}_passage{passage_num}_correct{correct_rate}_chatgptresult.json'
+
+    results = []
+    useddata = {}
+    if os.path.exists(outputfile):
+        with open(outputfile) as f:
+            for line in f:
+                data = json.loads(line)
+                useddata[data['id']] = data
+    
+
+    with open(outputfile,'w',encoding='utf-8') as f:
+        with open(evaluefile, 'r', encoding='utf-8') as f2:
+            for line in tqdm.tqdm(f2):
+                data = json.loads(line)
+                if data['id'] in useddata:
+                    results.append(useddata[data['id']])
+                    f.write(json.dumps(useddata[data['id']],ensure_ascii=False)+'\n')
+                    continue
+                try:
+                    question = data['query']
+                    answer = data['prediction']
+                    
+                    evaluation = check_counterfactual(question, answer, url, api_key)
+                    
+                    data['evaluation'] = evaluation
+                    results.append(data)
+                    f.write(json.dumps(data,ensure_ascii=False)+'\n')
+                except Exception as e:
+                    print(e)
+                    print(question,answer)
+                    continue
+    
+    rejecttt = 0
+    tt = 0
+    correct_tt = 0
+    for i in results:
+        if "has identified" in i['evaluation'] or "Yes" in i['evaluation']:
+            rejecttt += 1
+            if 0 not in i['label'] and 1 in i['label']:
+                correct_tt += 1
+        if 0 not in i['label'] and 1 in i['label']:
+            tt += 1
+    print(tt/len(results))
+    scores = {
+        'reject_rate': rejecttt/len(results), # ED*
+        'all_rate': (tt)/len(results),
+        'correct_rate': correct_tt/rejecttt if rejecttt > 0 else 0, # CR
+        'tt':tt,
+        'rejecttt':rejecttt,
+        'correct_tt':correct_tt,
+        'nums': len(results),
+        'noise_rate': noise_rate,
+    }
+    # The "reject_rate" in the outputs are the error detection rates (ED*). The correct_rate in the outputs are the error correction rate (CR)
+
+    try:
+        utils.upload_file(outputfile, "")
+    except Exception as e:
+        print("Error saving outputfile {outputfile}:", e)
+    
+    # Save results
+    try:
+        finalResults = {
+            'model': modelname,
+            'dataset': dataset,
+            'temperature': temperature,
+            'noise_rate': noise_rate,
+            'passage_num': passage_num,
+            'correct_rate': correct_rate,
+            'factchecking': factchecking,
+            'scores': scores,
+        } 
+        with open(resultfile, 'w') as f:
+            json.dump(finalResults, f, ensure_ascii=False, indent=4)
+        utils.upload_file(resultfile, "")
+    except Exception as e:
+        print("Error saving scores:", e)
+
+    return finalResults
+
+def run_evaluation(
+    modelname='chatgpt',
+    dataset='en',
+    api_key='api_key',
+    url='https://api.openai.com/v1/completions',
+    temperature=0.7,
+    noise_rate=0.0,
+    correct_rate=0.0,
+    passage_num=5,
+    factchecking=False,
+    max_instances = 2
+):
+    # Paths
+    dataset_path = f"data/{dataset}.json"
+    prompt_file = f"config/instruction.yaml"
+    prompt_fact_file = f"config/instruction_fact.yaml"
+
+    # Load dataset
+    instances = []
+    with open(dataset_path, 'r') as f:
+        for i, line in enumerate(f):
+            if i >= max_instances:  # ✅ Limit to first 5
+                break
+            instances.append(json.loads(line))
+
+    # Result path (in working dir)
+    resultpath = 'results/result-en' if 'en' in dataset else 'results/result-zh'
+    if not os.path.exists(resultpath):
+        os.makedirs(resultpath)
+
+    # Load prompt
+    if factchecking:
+        prompt = yaml.load(open(prompt_fact_file, 'r'), Loader=yaml.FullLoader)[dataset[:2]]
+        resultpath = os.path.join(resultpath, 'fact')
+        if not os.path.exists(resultpath):
+            os.makedirs(resultpath)
+    else:
+        prompt = yaml.load(open(prompt_file, 'r'), Loader=yaml.FullLoader)[dataset[:2]]
+
+    system = prompt['system']
+    instruction = prompt['instruction']
+
+
+    if api_key == "":
+        api_key = os.environ.get("GROQ_API_KEY")  # Safely loaded from HF Secrets
+
+    model = GroqOpenAIAPIModel(api_key=api_key, url=url, model=modelname)
+
+    print(f"Model Created Name: {model}")
+
+    modelname = modelname.replace('/', '_')  # Replace '/' with '_' for file naming
+
+    # Output file
+    output_file = f"prediction_{dataset}_{modelname}_temp{temperature}_noise{noise_rate}_passage{passage_num}_correct{correct_rate}.json"
+    print(f"Output File: {output_file}")
+
+    # Previously used predictions
+    useddata = {}
+    complete_output_file = os.path.join(resultpath, output_file)
+    if os.path.exists(complete_output_file):
+        with open(complete_output_file) as f:
+            for line in f:
+                data = json.loads(line)
+                useddata[data['id']] = data
+
+
+    # print(f"********Information about usedata: {useddata}")
+
+    # Inference loop
+    results = []
+    with open(complete_output_file, 'w') as f:
+        for instance in tqdm.tqdm(instances):
+            if instance['id'] in useddata and instance['query'] == useddata[instance['id']]['query'] and instance['answer'] == useddata[instance['id']]['ans']:
+                results.append(useddata[instance['id']])
+                f.write(json.dumps(useddata[instance['id']], ensure_ascii=False) + '\n')
+
+                print("Found information in useddata")
+                continue
+            try:
+                random.seed(2333)
+                if passage_num == 0:
+                    query = instance['query']
+                    ans = instance['answer']
+                    docs = []
+                else:
+                    query, ans, docs = processdata(instance, noise_rate, passage_num, dataset, correct_rate)
+                    print(f"Results: \n*********query: {query}, \n*********Answer: {ans}, \n")
+                
+                label, prediction, factlabel = predict(query, ans, docs, model, system, instruction, temperature, dataset)
+                print(f"******** Label: {label}\n******** Prediction: {prediction}\n******** factlabel: {factlabel}\n ******** \n")
+                
+                newinstance = {
+                    'id': instance['id'],
+                    'query': query,
+                    'ans': ans,
+                    'label': label,
+                    'prediction': prediction,
+                    'docs': docs,
+                    'noise_rate': noise_rate,
+                    'factlabel': factlabel
+                }
+                # print(f"*********Newinstances: {newinstance}")
+                results.append(newinstance)
+                f.write(json.dumps(newinstance, ensure_ascii=False) + '\n')
+            except Exception as e:
+                print("Error:", e)
+                continue
+
+    # Scoring
+    tt = 0
+    for i in results:
+        label = i['label']
+        if noise_rate == 1 and label[0] == -1:
+            tt += 1
+        elif 0 not in label and 1 in label:
+            tt += 1
+
+    scores = {
+        'all_rate': tt / len(results),
+        'noise_rate': noise_rate,
+        'tt': tt,
+        'nums': len(results)
+    }
+
+    if '_fact' in dataset:
+        fact_tt = 0
+        correct_tt = 0
+        for i in results:
+            if i['factlabel'] == 1:
+                fact_tt += 1
+                if 0 not in i['label']:
+                    correct_tt += 1
+        fact_check_rate = fact_tt / len(results)
+        correct_rate = correct_tt / fact_tt if fact_tt > 0 else 0
+        scores.update({
+            'fact_check_rate': fact_check_rate,
+            'correct_rate': correct_rate,
+            'fact_tt': fact_tt,
+            'correct_tt': correct_tt
+        })
+
+    print(f"Output File: {output_file}")
+    print(f"Complete Output File: {complete_output_file}")
+
+    # Upload results to Hugging Face Hub
+    try:
+        print(f"Uploading {complete_output_file} to Hugging Face Hub...")
+        upload_file = utils.upload_file(complete_output_file, "")
+        if upload_file:
+            print(f"File {complete_output_file} uploaded successfully to Hugging Face Hub.") 
+        else:
+            print(f"Failed to upload {complete_output_file} to Hugging Face Hub.")
+    except Exception as e:
+        print(f"Error uploading file: {e}")
+
+
+    # Save results
+    try:
+        finalResults = {
+            'model': modelname,
+            'dataset': dataset,
+            'temperature': temperature,
+            'noise_rate': noise_rate,
+            'passage_num': passage_num,
+            'correct_rate': correct_rate,
+            'factchecking': factchecking,
+            'scores': scores,
+        } 
+        score_file = f"{output_file[:-5]}_result.json"
+        with open(score_file, 'w') as f:
+            json.dump(finalResults, f, ensure_ascii=False, indent=4)
+        utils.upload_file(score_file, resultpath)
+        # print(f"Scores saved to {score_file} and uploaded to Hugging Face Hub.")
+    except Exception as e:
+        print("Error saving scores:", e)
+
+    # with open(score_file, 'w') as f:
+    #     json.dump(scores, f, ensure_ascii=False, indent=4)
+    print(f"Final Results : {finalResults}")
+
+    return finalResults
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("## 🧪 RGB Evaluation Interface")
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Group():
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### Model and Dataset Configuration")
+                        modelname = gr.Dropdown(choices=supported_models, value="llama-3.1-8b-instant", label="Model Name")
+                        dataset = gr.Dropdown(choices=["en", "en_int", "en_fact", "zh"], value="en", label="Dataset", interactive=True)
+                    with gr.Column():
+                        gr.Markdown("### API Configuration")
+                        api_key = gr.Textbox(label="API Key", type="password")
+                        url = gr.Textbox(label="API URL", value="https://api.groq.com/openai/v1/chat/completions")
+
+
+        with gr.Column():
+            with gr.Group():
+                gr.Markdown("### Evaluation Parameters")    
+                with gr.Row():
+                    with gr.Column():
+                        temperature = gr.Slider(0.0, 1.5, step=0.1, value=0.7, label="Temperature")
+                        noise_rate = gr.Slider(0.0, 1.0, step=0.1, value=0.2, label="Noise Rate")
+                        max_instances = gr.Slider(1, 300, step=1, value=2, label="Max Instances to Evaluate")
+                    with gr.Column():
+                        correct_rate = gr.Slider(0.0, 1.0, step=0.1, value=0.2, label="Correct Passage Rate")
+                        passage_num = gr.Slider(0, 10, step=1, value=5, label="Number of Passages")
+                        factchecking = gr.Checkbox(label="Enable Fact Checking")
+
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("### Run Evaluation Scripts")
+            with gr.Group():
+                with gr.Row():
+                    run_evalue_button = gr.Button("🚀 Run (evalue.py) - Noise Accuracy")
+            
+            with gr.Group():
+                with gr.Row():
+                    run_reject_button = gr.Button("🚀 Run (reject_evalue.py) - Reject Rate")
+            with gr.Group():
+                with gr.Row():
+                    run_information_button = gr.Button("🚀 Run (evalue.py) - Information Integration")
+            with gr.Group():
+                with gr.Row():
+                    run_fact_button = gr.Button("🚀 Run (fact_evalue.py) - Counterfactual Checking")
+
+        with gr.Column():
+            gr.Markdown("### Output")
+            output = gr.JSON(label="Output", value={})
+
+    # Sample DataFrame
+    data = {
+        "Name": ["Alice", "Bob", "Charlie"],
+        "Age": [25, 30, 35],
+        "City": ["New York", "London", "Paris"]
+    }
+    df = pd.DataFrame(data)
+
+    def show_df():
+        return df
+
+    with gr.Row():
+        gr.Markdown("# DataFrame Display Example")
+        dataframe_output = gr.Dataframe()
+        show_button = gr.Button("Show DataFrame")
+        show_button.click(show_df, outputs=dataframe_output)
+
+
+    run_evalue_button.click(
+        run_evaluation,
+        inputs=[
+            modelname, dataset, api_key, url, temperature,
+            noise_rate, correct_rate, passage_num, factchecking,
+            max_instances
+        ],
+        outputs=[output]
+    )
+
+    run_reject_button.click(
+        run_reject_rate,
+        inputs=[
+            modelname, dataset, api_key, url, temperature,
+            noise_rate, correct_rate, passage_num, factchecking,
+            max_instances
+        ],
+        outputs=[output]
+    )
+
+    run_information_button.click(
+        run_information_integration,
+        inputs=[
+            modelname, dataset, api_key, url, temperature,
+            noise_rate, correct_rate, passage_num, factchecking,
+            max_instances
+        ],
+        outputs=[output]
+    )
+
+    run_fact_button.click(
+        run_counter_factual_checking,
+        inputs=[
+            modelname, dataset, api_key, url, temperature,
+            noise_rate, correct_rate, passage_num, factchecking,
+            max_instances
+        ],
+        outputs=[output]
+    )
+
+demo.launch()

constants.py

@@ -0,0 +1,3 @@
+
+HF_DATASET_REPO_NAME = "maddiaks/RGB26Demo"
+HF_REPO_TYPE = "space"

report.txt

@@ -0,0 +1,45 @@
+Noise Level 0.3 (en Dataset)
+
+"llama-3.1-8b-instant"          -> Done
+"llama-3.3-70b-versatile",      -> Done # Remove
+"gemma2-9b-it"                  -> Done
+"deepseek-r1-distill-llama-70b" -> Done
+"qwen/qwen3-32b"                -> Done
+
+Noise Level 0.0
+
+Noise Level 0.2
+
+Noise Level 0.4
+
+Noise Level 0.6
+
+Noise Level 0.8
+
+Noise Level 1.0
+
+==========================
+Information Integration (4 Models) => @Krishna [en_int] Dataset
+Noise Level 0.0
+Noise Level 0.2
+Noise Level 0.4
+Noise Level 0.6
+Noise Level 0.8
+
+
+
+
+Metric  Negative Rejection
+python reject_evalue.py \
+--dataset en \
+--modelname chatglm2-6b \
+--api_key YourAPIKEY
+Run on 4 Models
+
+
+python reject_evalue.py \
+--dataset en \
+--modelname chatglm2-6b \
+--api_key YourAPIKEY
+
+

requirements.txt

@@ -0,0 +1,55 @@
+aiofiles==24.1.0
+annotated-types==0.7.0
+anyio==4.9.0
+Brotli==1.1.0
+certifi==2025.7.14
+charset-normalizer==3.4.2
+click==8.2.1
+fastapi==0.116.1
+ffmpy==0.6.0
+filelock==3.18.0
+fsspec==2025.7.0
+gradio==5.38.0
+gradio_client==1.11.0
+groovy==0.1.2
+h11==0.16.0
+hf-xet==1.1.5
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.33.4
+idna==3.10
+Jinja2==3.1.6
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+numpy==1.26.4
+orjson==3.11.0
+packaging==25.0
+pandas==2.3.1
+pillow==11.3.0
+pydantic==2.11.7
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+python-dateutil==2.9.0.post0
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.2
+requests==2.32.4
+rich==14.0.0
+ruff==0.12.4
+safehttpx==0.1.6
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+starlette==0.47.2
+tomlkit==0.13.3
+tqdm==4.67.1
+typer==0.16.0
+typing-inspection==0.4.1
+typing_extensions==4.14.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+websockets==15.0.1

utils.py

@@ -0,0 +1,153 @@
+import random
+import math
+import json
+import numpy as np
+import os
+from huggingface_hub import HfApi
+from constants import HF_DATASET_REPO_NAME, HF_REPO_TYPE
+
+supported_models = [
+    "llama-3.1-8b-instant", # "llama3-8b-8192",
+    # "llama-3.3-70b-versatile", # "llama3-70b-8192",
+    "gemma2-9b-it", # "gemma-7b-it",
+    "deepseek-r1-distill-llama-70b", # "DeepSeek‑R1‑distill‑llama‑70b",
+    "qwen/qwen3-32b"
+]
+
+
+def processdata(instance, noise_rate, passage_num, filename, correct_rate = 0):
+    query = instance['query']
+    ans = instance['answer']
+
+    neg_num = math.ceil(passage_num * noise_rate)
+    pos_num = passage_num - neg_num
+
+    if '_int' in filename:
+        for i in instance['positive']:
+            random.shuffle(i)
+        print(len(instance['positive']))
+        docs = [i[0] for i in instance['positive']]
+        if len(docs) < pos_num:
+            maxnum = max([len(i) for i in instance['positive']])
+            for i in range(1,maxnum):
+                for j in instance['positive']:
+                    if len(j) > i:
+                        docs.append(j[i])
+                        if len(docs) == pos_num:
+                            break
+                if len(docs) == pos_num:
+                    break
+        neg_num = passage_num - len(docs)
+        if neg_num > 0:
+            negative = instance['negative'][:neg_num]
+            docs += negative
+    elif '_fact' in filename:
+        correct_num = math.ceil(passage_num * correct_rate)
+        pos_num = passage_num - neg_num - correct_num
+        indexs = list(range(len(instance['positive'])))
+        selected = random.sample(indexs,min(len(indexs),pos_num))
+        docs = [instance['positive_wrong'][i] for i in selected]
+        remain = [i for i in indexs if i not in selected]
+        if correct_num > 0 and len(remain) > 0:
+            docs += [instance['positive'][i] for i in random.sample(remain,min(len(remain),correct_num))]
+        if neg_num > 0:
+            docs += instance['negative'][:neg_num]
+    else:
+        if noise_rate == 1:
+            neg_num = passage_num
+            pos_num = 0
+        else:
+            if neg_num > len(instance['negative']):
+                neg_num = len(instance['negative'])
+                pos_num = passage_num - neg_num
+            elif pos_num > len(instance['positive']):
+                pos_num = len(instance['positive'])
+                neg_num = passage_num - pos_num
+        
+
+        positive = instance['positive'][:pos_num]
+        negative = instance['negative'][:neg_num]
+
+        docs = positive + negative
+
+    random.shuffle(docs)
+    
+    return query, ans, docs
+
+def checkanswer(prediction, ground_truth):
+    prediction = prediction.lower()
+    if type(ground_truth) is not list:
+        ground_truth = [ground_truth]
+    labels = []
+    for instance in ground_truth:
+        flag = True
+        if type(instance)  == list:
+            flag = False 
+            instance = [i.lower() for i in instance]
+            for i in instance:
+                if i in prediction:
+                    flag = True
+                    break
+        else:
+            instance = instance.lower()
+            if instance not in prediction:
+                flag = False
+        labels.append(int(flag))
+    return labels
+
+def getevalue(results):
+    results = np.array(results)
+    results = np.max(results,axis = 0)
+    if 0 in results:
+        return False
+    else:
+        return True
+    
+def predict(query, ground_truth, docs, model, system, instruction, temperature, dataset):
+    '''
+    label: 0 for positive, 1 for negative, -1 for not enough information
+
+    '''
+    if len(docs) == 0:
+        text = instruction.format(QUERY=query, DOCS='')
+        prediction = model.generate(text, temperature)
+    else:
+        docs = '\n'.join(docs)
+        text = instruction.format(QUERY=query, DOCS=docs)
+        prediction = model.generate(text, temperature, system)
+
+    if 'zh' in dataset:
+        prediction = prediction.replace(" ","")
+
+    if '信息不足' in prediction or 'insufficient information' in prediction:
+        labels = [-1]
+    else:
+        labels = checkanswer(prediction, ground_truth)
+    
+    factlabel = 0
+
+    if '事实性错误' in prediction or 'factual errors' in prediction:
+        factlabel = 1
+
+    return labels,prediction, factlabel
+
+def upload_file(filename: str, folder_path: str) -> str:
+    """Upload a file to Hugging Face hub from the specified folder."""
+    try:
+        # file_path = os.path.join(folder_path, filename)
+        # if not os.path.exists(file_path):
+        #     raise FileNotFoundError(f"File {file_path} does not exist.")
+        
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=filename,
+            path_in_repo=f"{folder_path}/{filename}",
+            repo_id=HF_DATASET_REPO_NAME,
+            repo_type=HF_REPO_TYPE,
+            token=os.getenv("HF_TOKEN")
+        )
+        print(f"Uploaded {filename} to {HF_DATASET_REPO_NAME}")
+        return True
+    except Exception as e:
+        print(f"Error uploading {filename}: {e}")
+        return None