Changed back to set, with jsondumps for serializing

handler.py

@@ -15,6 +15,7 @@ from decord import cpu
 
 import timeit
 import easyocr
+import json
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -161,6 +162,11 @@ class EndpointHandler:
         # self.logger.info("Returning batch_emb list")
         return batch_emb
 
+    def set_default(self, obj):
+        if isinstance(obj, set):
+            return list(obj)
+        raise TypeError
+
     def process_video(self, video_url, video_metadata):
         try:
             self.logger.info("Downloading video as bytes.")
@@ -190,10 +196,12 @@ class EndpointHandler:
             self.logger.info("Extracting text from frames.")
             text_extraction_start_time = timeit.default_timer()
             frame_texts = [self.reader.readtext(frame, detail=0) for frame in frames]
-            all_texts_list = []
+            texts_set = set()
             for text_list in frame_texts:
-                [all_texts_list.append(text) for text in text_list]
-            video_metadata["extracted_text"] = all_texts_list
+                [texts_set.add(text) for text in text_list]
+            video_metadata["extracted_text"] = json.dumps(
+                texts_set, default=self.set_default
+            )
             text_extraction_end_time = timeit.default_timer()
             self.logger.info(
                 f"Text extraction took {text_extraction_end_time - text_extraction_start_time} seconds"