fix: gguf dequantize

model/mistral_7b_ielts_evaluator.py

@@ -1,32 +1,66 @@
 from typing import *
 import torch
-import numpy as np
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import spaces
 
 model_name = "chillies/mistral-7b-ielts-evaluator-q4"
-model = AutoModelForSequenceClassification.from_pretrained(model_name)
-tokenizer = AutoTokenizer.from_pretrained(model_name)
+file_name = "GUFF-ielts-fighter-unsloth.Q4_K_M.gguf"
+model = AutoModelForCausalLM.from_pretrained(model_name, gguf_file=file_name)
+tokenizer = AutoTokenizer.from_pretrained(model_name, gguf_file=file_name)
+pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
 
+system = """In this task, you are required to evaluate an IELTS Writing Task 2 essay. Consider the following four criteria and provide a detailed assessment for each, along with a suggested band score:
 
-@spaces.GPU()
+## Task Achievement:
+- Evaluate how well the candidate has addressed the given task.
+- Assess the clarity and coherence of the response in presenting ideas.
+- Identify if the candidate has fully covered all parts of the task and supported arguments appropriately.
+- Suggested Band Score (Task Achievement): [Insert Score]
+
+## Coherence and Cohesion:
+- Assess the overall organization and structure of the essay.
+- Evaluate the use of linking devices to connect ideas and paragraphs.
+- Identify if there is a logical flow of information.
+- Suggested Band Score (Coherence and Cohesion): [Insert Score]
+
+## Lexical Resource (Vocabulary):
+- Examine the range and accuracy of vocabulary used in the essay.
+- Point out specific mistakes in vocabulary, such as inaccuracies or overuse of certain words and Suggest modified versions or alternatives for the identified mistakes. [list of mistakes and rectify]
+- Assess the appropriateness of vocabulary for the given context.
+- Suggested Band Score (Lexical Resource): [Insert Score]
+
+## Grammatical Range and Accuracy:
+- Evaluate the variety and complexity of sentence structures.
+- Point out specific grammatical errors, such as incorrect verb forms or sentence construction and Suggest modified versions or corrections for the identified mistakes. [list of mistakes and rectify]
+- Examine the use of punctuation and sentence formation.
+- Suggested Band Score (Grammatical Range and Accuracy): [Insert Score]
+
+## Overall Band Score:
+
+- Provide an overall band score for the essay, considering the holistic performance across all criteria.
+- Consider the synergy of the essay in meeting the task requirements cohesively.
+- Suggested Overall Band Score: [Insert Score]
+
+## Feedback and Additional Comments:
+- Provide constructive feedback highlighting specific strengths and areas for improvement.
+- Suggest strategies for enhancement in weaker areas."""
+
+
+@spaces.GPU(duration=120)
 @torch.no_grad()
 def grade_mistral_7b_ielts_evaluator(question: str, answer: str) -> Tuple[float, str]:
-    text = f"{question} {answer}"
+    text = f"""{system}
 
-    inputs = tokenizer(
-        text,
-        return_tensors="pt",
-        padding=True,
-        truncation=True,
-    )
+## Prompt:
+{question}
 
-    outputs = model(**inputs)
-    score = outputs.logits.argmax(dim=-1).item()
-    print(score)
+## Essay:
+{answer}
 
-    overall_score = float(score)
+## Evaluation:
+"""
 
-    comment = ""
+    outputs = pipe(text)
+    overall_score = float(0.0)
 
-    return overall_score, comment
+    return overall_score, outputs

requirements.txt

@@ -1,6 +1,9 @@
 torch
-transformers
+# transformers (wait for 4.44.0)
+git+https://github.com/huggingface/transformers
 accelerate
 spaces
 gradio
 numpy
+gguf
+sentencepiece