Spaces:
Sleeping
Sleeping
File size: 4,368 Bytes
d8e1c3b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | from sklearn import svm
import re
import nltk
import pandas as pd
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report
import pickle
""""
*** USAGE ***
Proprocessing and model loading should be done at the initialization of the app,
/model.pkl/ can be found at ./app/ml/ alongside the dataset, can be done by
/init_model/
from utils import init_model, inference
model = init_model("BooksDataSet.csv", "model.pkl")
Then, for inference we can simply call /inference/ which returns a string (genre)
print(inference(model, "spaceship armada")) # returns "Science Fiction"
"""
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)
LE = LabelEncoder()
def setup_vectorizer(xtrain, xval):
"vectorizer used for converting words to vector"
xtrain_tfidf = tfidf_vectorizer.fit_transform(xtrain.values.astype('U'))
xval_tfidf = tfidf_vectorizer.transform(xval.values.astype('U'))
return xtrain_tfidf, xval_tfidf
def setup_nltk():
"""
Initialize nltk before preprocessing, needs to be called only once for python
dependencies
"""
nltk.download('stopwords')
nltk.download('wordnet')
def lematizing(sentence):
lemma=WordNetLemmatizer()
stemSentence = ""
for word in sentence.split():
stem = lemma.lemmatize(word)
stemSentence += stem
stemSentence += " "
stemSentence = stemSentence.strip()
return stemSentence
def remove_stopwords(text):
"""
function to remove stopwords
"""
stop_words = set(stopwords.words('english'))
no_stopword_text = [w for w in text.split() if not w in stop_words]
return ' '.join(no_stopword_text)
def stemming(sentence):
stemmer = PorterStemmer()
stemSentence = ""
for word in sentence.split():
stem = stemmer.stem(word)
stemSentence += stem
stemSentence += " "
stemSentence = stemSentence.strip()
return stemSentence
def clean(text):
"""
remove backslash-apostrophe, whitespace and convert to lowercase
"""
text = re.sub("\'", "", text)
text = re.sub("[^a-zA-Z]"," ",text)
text = ' '.join(text.split())
text = text.lower()
return text
def train_model(xtrain, xval, ytrain, yval):
"""
Train the model on the dataset
"""
svc = svm.SVC(kernel='linear').fit(xtrain, ytrain)
return svc
def validate_model(model, xval, yval):
"""
Validate the model for accuracy
"""
svpred=model.predict(xval)
# for accuracy reports
print(classification_report(yval,svpred))
return svpred
def save_model(model):
pickle.dump(model, open("model.pkl", 'wb'))
def preprocess_data(filename: str):
"""
Data preprocessing for variability and cleanliness
"""
books=pd.read_csv(filename)
books=pd.DataFrame(books,columns=['book_id','book_name','genre','summary'])
books.loc[:,'summary']=books.loc[:,'summary'].apply(lambda x: clean(x))
books['summary'] = books['summary'].apply(lambda x: remove_stopwords(x))
books['summary'] = books['summary'].apply(lambda x: lematizing(x))
books['summary'] = books['summary'].apply(lambda x: stemming(x))
y=LE.fit_transform(books['genre'])
xtrain, xval, ytrain, yval = train_test_split(books['summary'], y, test_size=0.15, random_state=557)
xtrain, xval = setup_vectorizer(xtrain, xval)
# print(books['summary'])
return [xtrain, xval, ytrain, yval]
def load_model(filename: str):
"load a model from a filename"
model = pickle.load(open("model.pkl", 'rb'))
return model
def inference(model, q):
"enter a string to return the genre of the summary"
q = clean(q)
q = remove_stopwords(q)
q = lematizing(q)
q = stemming(q)
q_vec = tfidf_vectorizer.transform([q])
q_pred = model.predict(q_vec)
return LE.inverse_transform(q_pred)[0]
def init_model(dataset: str, model: str):
"""
Initializes the model should be called at the start of the application
"""
setup_nltk()
xtrain, xval, ytrain, yval = preprocess_data(dataset)
model = load_model(model)
return model
|