Delete ResumeCode.txt

ResumeCode.txt

@@ -1,214 +0,0 @@
-!pip install opendatasets
-
-#!pip install wandb
-
-!pip install transformers[torch]
-
-!pip install evaluate
-
-import pandas as pd
-import numpy as np
-import os
-import random
-from datasets import Dataset
-import opendatasets as od
-import matplotlib.pyplot as plt
-
-import torch
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-    TrainingArguments,
-    Trainer,
-    DataCollatorWithPadding,
-    pipeline
-)
-import evaluate
-
-plt.style.use('seaborn-v0_8')
-
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import LabelEncoder
-from sklearn.naive_bayes import MultinomialNB
-from sklearn import metrics
-from sklearn.metrics import accuracy_score
-from pandas.plotting import scatter_matrix
-from sklearn import metrics
-from sklearn.feature_extraction.text import TfidfVectorizer
-from matplotlib.gridspec import GridSpec
-import nltk
-nltk.download('stopwords')
-nltk.download('punkt')
-from nltk.corpus import stopwords
-import string
-from wordcloud import WordCloud
-
-DIRECTORY = '/content/UpdatedResumeDataSet.csv'
-MODEL_NAME = 'distilbert-base-uncased'
-BATCH_SIZE = 32
-LR = 2e-5
-EPOCHS = 10
-DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-# read the dataset
-df = pd.read_csv('UpdatedResumeDataSet.csv')
-print(df.shape)
-df.head(10)      # first 10 rows
-
-# Display the distinct categories of resume
-df['Category'].unique()
-
-# Display the distinct categories of resume and the number of records belonging to each category
-df['Category'].value_counts()
-
-import seaborn as sns
-
-sns.countplot(y = df['Category'], data = df['Resume'])
-
-# Convert all characters to lowercase
-def convert_lower(text):
-    return text.lower()
-
-df['Resume'] = df['Resume'].apply(convert_lower)
-
-import re
-def cleanResume(resumeText):
-  resumeText = re.sub(r'http\S+', '', resumeText,flags = re.MULTILINE) # remove URLs
-  resumeText = re.sub('RT|cc', '', resumeText)  # remove RT and cc
-  resumeText = re.sub('#\S+', '', resumeText)  # remove hashtags
-  resumeText = re.sub('@\S+', '', resumeText)  # remove mentions
-  resumeText = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), '', resumeText)  # remove punctuations
-  resumeText = re.sub('â\S+', '', resumeText)  # remove •
-  resumeText = re.sub('€+', '', resumeText)  # remove €
-  resumeText = re.sub('\s+', ' ', resumeText)  # remove extra whitespace
-
-  return resumeText
-
- # apply the function defined above and save the
-df['cleaned_resume'] = df['Resume'].apply(cleanResume)
-
-# stop words
-stopword_list = nltk.corpus.stopwords.words('english')
-print(stopword_list)
-
-# removing the stopwords
-from nltk.tokenize import word_tokenize
-
-def remove_stopwords(text, is_lower_case=False):
-    # splitting strings into tokens (list of words)
-    tokens = word_tokenize(text)
-    tokens = [token.strip() for token in tokens]
-    filtered_tokens = [token for token in tokens if token not in stopword_list]
-    filtered_text = ' '.join(filtered_tokens)
-    return filtered_text
-
-# apply function on cleaned resume to remove stopwords
-df['text'] = df['cleaned_resume'].apply(remove_stopwords)
-df['label'] = df['Category']
-
-# reorder dataframe columns
-df = df[['text', 'label']]
-
-# view shape
-df.shape
-
-# view number of classes
-n_classes = df['label'].nunique()
-print(f"Number of Resume classes: {n_classes}")
-
-# view some statistics about are texts
-lengths = df['text'].apply(lambda x: len(x))
-print(
-    f'Max text length: {lengths.max()}\nMin text length: {lengths.min()}\nAvg text length: {lengths.mean():.2f}'
-)
-
-# create mappings
-id2label = {idx: label for idx, label in enumerate(df['label'].unique())}
-label2id = {label: idx for idx, label in id2label.items()}
-
-# label encode our labels
-df['label'] = df['label'].map(label2id)
-
-# create and split dataset
-dataset = Dataset.from_pandas(df).train_test_split(train_size=0.8)
-print(dataset)
-
-# initialize tokenizer
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-
-# Tokenize and encode the dataset
-def tokenize(batch):
-    tokenized_batch = tokenizer(batch['text'], padding=True, truncation=True)
-    return tokenized_batch
-
-dataset_enc = dataset.map(tokenize, batched=True)
-
-print(dataset_enc)
-
-accuracy = evaluate.load('accuracy')
-
-def compute_metrics(eval_pred):
-    predictions, labels = eval_pred
-    predictions = np.argmax(predictions, axis=1)
-    return accuracy.compute(predictions=predictions, references=labels)
-
-# define model
-model = AutoModelForSequenceClassification.from_pretrained(
-    MODEL_NAME,
-    num_labels=n_classes,
-    id2label=id2label,
-    label2id=label2id
-)
-
-model.to(DEVICE)
-
-# define collator function
-collator_fn = DataCollatorWithPadding(tokenizer, return_tensors='pt')
-
-pip install accelerate -U
-
-import accelerate
-import transformers
-
-transformers.__version__, accelerate.__version__
-
-from transformers import TrainingArguments
-
-training_args = TrainingArguments(
-    output_dir = "Resume_training",
-    learning_rate=LR,
-    per_device_train_batch_size= BATCH_SIZE,
-    per_device_eval_batch_size = BATCH_SIZE,
-    num_train_epochs = EPOCHS,
-    weight_decay = 0.01,
-    evaluation_strategy = "epoch",
-    save_strategy = "epoch",
-    load_best_model_at_end = True,
-    push_to_hub = False,
-    report_to="none"
-)
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=dataset_enc["train"],
-    eval_dataset=dataset_enc["test"],
-    tokenizer=tokenizer,
-    data_collator=collator_fn,
-    compute_metrics=compute_metrics
-)
-
-trainer.train()
-
-trainer.save_model('ResumeClassification_distilBERT')
-
-trainer.evaluate()
-
-def predict(sample, validate=True):
-  classifier = pipeline("text-classification", model=model, tokenizer=tokenizer)
-  pred = classifier(sample)[0]['label']
-  return pred
-
-sample1 = "I have working expereince in Java and javascript"
-
-predict(sample1)