ZinebSN commited on
Commit
c576e80
·
1 Parent(s): f451bc7

Delete iris-batch-inference-pipeline.py

Browse files
Files changed (1) hide show
  1. iris-batch-inference-pipeline.py +0 -108
iris-batch-inference-pipeline.py DELETED
@@ -1,108 +0,0 @@
1
- import os
2
- import modal
3
-
4
- LOCAL=True
5
-
6
- if LOCAL == False:
7
- stub = modal.Stub()
8
- hopsworks_image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"])
9
- @stub.function(image=hopsworks_image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("jim-hopsworks-ai"))
10
- def f():
11
- g()
12
-
13
- def g():
14
- import pandas as pd
15
- import hopsworks
16
- import joblib
17
- import datetime
18
- from PIL import Image
19
- from datetime import datetime
20
- import dataframe_image as dfi
21
- from sklearn.metrics import confusion_matrix
22
- from matplotlib import pyplot
23
- import seaborn as sns
24
- import requests
25
-
26
- project = hopsworks.login()
27
- fs = project.get_feature_store()
28
-
29
- mr = project.get_model_registry()
30
- model = mr.get_model("iris_modal", version=1)
31
- model_dir = model.download()
32
- model = joblib.load(model_dir + "/iris_model.pkl")
33
-
34
- feature_view = fs.get_feature_view(name="iris_modal", version=1)
35
- batch_data = feature_view.get_batch_data()
36
-
37
- y_pred = model.predict(batch_data)
38
- # print(y_pred)
39
- flower = y_pred[y_pred.size-1]
40
- flower_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + flower + ".png"
41
- print("Flower predicted: " + flower)
42
- img = Image.open(requests.get(flower_url, stream=True).raw)
43
- img.save("./latest_iris.png")
44
- dataset_api = project.get_dataset_api()
45
- dataset_api.upload("./latest_iris.png", "Resources/images", overwrite=True)
46
-
47
- iris_fg = fs.get_feature_group(name="iris_modal", version=1)
48
- df = iris_fg.read()
49
- # print(df["variety"])
50
- label = df.iloc[-1]["variety"]
51
- label_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + label + ".png"
52
- print("Flower actual: " + label)
53
- img = Image.open(requests.get(label_url, stream=True).raw)
54
- img.save("./actual_iris.png")
55
- dataset_api.upload("./actual_iris.png", "Resources/images", overwrite=True)
56
-
57
- monitor_fg = fs.get_or_create_feature_group(name="iris_predictions",
58
- version=1,
59
- primary_key=["datetime"],
60
- description="Iris flower Prediction/Outcome Monitoring"
61
- )
62
-
63
- now = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
64
- data = {
65
- 'prediction': [flower],
66
- 'label': [label],
67
- 'datetime': [now],
68
- }
69
- monitor_df = pd.DataFrame(data)
70
- monitor_fg.insert(monitor_df, write_options={"wait_for_job" : False})
71
-
72
- history_df = monitor_fg.read()
73
- # Add our prediction to the history, as the history_df won't have it -
74
- # the insertion was done asynchronously, so it will take ~1 min to land on App
75
- history_df = pd.concat([history_df, monitor_df])
76
-
77
-
78
- df_recent = history_df.tail(5)
79
- dfi.export(df_recent, './df_recent.png', table_conversion = 'matplotlib')
80
- dataset_api.upload("./df_recent.png", "Resources/images", overwrite=True)
81
-
82
- predictions = history_df[['prediction']]
83
- labels = history_df[['label']]
84
-
85
- # Only create the confusion matrix when our iris_predictions feature group has examples of all 3 iris flowers
86
- print("Number of different flower predictions to date: " + str(predictions.value_counts().count()))
87
- if predictions.value_counts().count() == 3:
88
- results = confusion_matrix(labels, predictions)
89
-
90
- df_cm = pd.DataFrame(results, ['True Setosa', 'True Versicolor', 'True Virginica'],
91
- ['Pred Setosa', 'Pred Versicolor', 'Pred Virginica'])
92
-
93
- cm = sns.heatmap(df_cm, annot=True)
94
- fig = cm.get_figure()
95
- fig.savefig("./confusion_matrix.png")
96
- dataset_api.upload("./confusion_matrix.png", "Resources/images", overwrite=True)
97
- else:
98
- print("You need 3 different flower predictions to create the confusion matrix.")
99
- print("Run the batch inference pipeline more times until you get 3 different iris flower predictions")
100
-
101
-
102
- if __name__ == "__main__":
103
- if LOCAL == True :
104
- g()
105
- else:
106
- with stub.run():
107
- f()
108
-