added training sources

get_data.py

@@ -0,0 +1,56 @@
+'''
+File: get_data.py
+Project: PatternsRecognizer
+Author: Milko Videv (milko.videv@thalesgroup.com)
+-----
+Last Modified: Friday, 1st March 2024 6:04:04 pm
+Modified By: Milko Videv (milko.videv@thalesgroup.com>)
+-----
+Copyright 2017 - 2024, Thales DIS, MCS SSH
+-----
+HISTORY:
+Date      	By	Comments
+----------	---	---------------------------------------------------------
+'''
+
+from random import random
+from fastdownload import download_url
+from fastai.vision.all import *
+from time import sleep
+from fastbook import search_images_ddg
+from fastcore.all import *
+
+def search_images_fastbook(term, max_images=30):
+    print(f"Searching for {max_images} {term}")
+    return search_images_ddg(term, max_images=max_images)
+
+def get_images(target_path, count, searches):
+    path = Path(target_path)    
+
+    for o in searches:
+        dest = (path/o)
+        dest.mkdir(exist_ok=True, parents=True)
+        download_images(dest, urls=search_images_fastbook(f'{o} patterns images', max_images=count))
+        sleep(10)
+        resize_images(path/o, max_size=400, dest=path/o)
+
+    print(f"Checking for bad images ...")
+    failed = verify_images(get_image_files(path))
+    failed.map(Path.unlink)
+    print(f"Removed {len(failed)} bad images")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3 or not sys.argv[2].isdigit():
+        print("Use: python get_data.py <target path> <images count> <one or more pattern kinds>\n")
+        print("Example: python get_data.py patterns 1 bulgarian indian japanese")
+    else:        
+        target_path = sys.argv[1]
+        count = sys.argv[2]
+        searches = sys.argv[3:] if len(sys.argv) > 3 else "bulgarian"
+
+        if len(searches) == 0:
+            print(f"Nothing to search for. Exitting.")
+            sys.exit()
+
+        #print(target_path, int(count), searches)
+        get_images(target_path, int(count), searches)

train.py

@@ -0,0 +1,72 @@
+'''
+File: train.py
+Project: PatternsRecognizer
+Author: Milko Videv (milko.videv@thalesgroup.com)
+-----
+Last Modified: Tuesday, 5th March 2024 10:02:08 am
+Modified By: Milko Videv (milko.videv@thalesgroup.com>)
+-----
+Copyright 2017 - 2024, Thales DIS, MCS SSH
+-----
+HISTORY:
+Date      	By	Comments
+----------	---	---------------------------------------------------------
+'''
+
+from fastai.vision.all import *
+from fastcore.all import *
+from tools import *
+
+def train(src, count):
+    path = Path(src)
+    dls = DataBlock(
+        blocks=(ImageBlock, CategoryBlock), # block for image and category
+        get_items=get_image_files, # recursively get files in path
+        splitter=RandomSplitter(valid_pct=0.3, seed=42), # 30% for check
+        get_y=parent_label, # use the dir as label
+        item_tfms=[Resize(192, method='squish')] # resize them a uniformly
+    ).dataloaders(path, bs=32, verbose=True)    # batch size 32 
+
+    # using resnet Neural Network library 
+    # Note: on 1st run will download it from https://download.pytorch.org/models)
+    # so set up proxy:     
+    # set HTTP_PROXY=http://proxy-us-austin.gemalto.com:8080
+    # set HTTPS_PROXY=http://proxy-us-austin.gemalto.com:8080
+
+    sav_path = Path('./models')  
+    sav_path.mkdir(exist_ok=True, parents=True)
+
+    learn = vision_learner(dls, resnet18, metrics=error_rate)
+
+    if os.path.exists("./models/current.pth"):
+        print(f"Loading current.pth to continue learning ...")
+        learn.load("current")
+
+    stopwatch = Stopwatch()
+    stopwatch.start()
+    print(f"Training ...")
+    learn.fine_tune(count)      
+    stopwatch.stop()
+    print(f"Training took ", stopwatch.elapsed_time(), "seconds")  
+
+    learn.save('current') # todo: load it if existing during next run to continue training 
+
+    learn.path = sav_path
+    learn.export('trained.pkl')
+
+if __name__ == "__main__":
+    if debugger_is_active():
+        train(3)
+        sys.exit()
+    else:
+        if len(sys.argv) < 3 or not sys.argv[2].isdigit():
+            print("Use: python train.py <images path> <epochs count>")   
+            print("Example: python train.py patterns 3")
+            sys.exit()
+        elif int(sys.argv[2]) > 5:
+            if not get_yes_no("Epoch count is too big, might take time. Continue?"):
+                sys.exit()
+                
+        src = sys.argv[1]    
+        count = sys.argv[2]    
+        train(src, int(count))