Upload tuned models

README.md

@@ -62,7 +62,7 @@ See the `inference_examples` directory. Models in the Safetensors format were us
 
 ### Training Data
 
-All of the data can be found [here](link to Dataset card once it becomes public). Refer to the Dataset card to find the relevant training data, as different kinds exist.
+All of the data can be found [here](https://huggingface.co/datasets/flowlabcu/PoreSimNet-Data). Refer to the Dataset card to find the relevant training data, as different kinds exist.
 
 ### Training Procedure
 

inference_examples/inference_cnn_safetensors.py

@@ -0,0 +1,102 @@
+import os, sys, torch, torchvision
+from safetensors.torch import load_file 
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__name__), '..', '..')))          
+from data_modules import image_transformations   
+from model_classes import EfficientNets, ResNets, ConvNeXt           
+from PIL import Image
+import pandas as pd
+import numpy as np
+import piexif
+from tqdm import tqdm
+
+def run_inference_cnn_base(model_name: str, state_dict: str, images_dir: str, save_file: str):
+    if model_name == 'ENet-B0':
+        model = EfficientNets.ENetB0()
+    elif model_name == 'ENet-B3':
+        model = EfficientNets.ENetB3()
+    elif model_name == 'ENet-B7':
+        model = EfficientNets.ENetB7()
+    elif model_name == 'ResNet-18':
+        model = ResNets.ResNet18()
+    elif model_name == 'ResNet-50':
+        model = ResNets.ResNet50()
+    elif model_name == 'ResNet-152':
+        model = ResNets.ResNet152()
+    elif model_name == 'ConvNeXt-Tiny':
+        model = ConvNeXt.ConvNeXtTiny()
+    else:
+        print('Model not found')
+        print('Models available:')
+        print('ENet-B0: \t\t EfficientNet-B0')
+        print('ENet-B3: \t\t EfficientNet-B3')
+        print('ENet-B7: \t\t EfficientNet-B7')
+        print('ResNet-18: \t\t ResNet-18')
+        print('ResNet-50: \t\t ResNet-50')
+        print('ResNet-152: \t\t ResNet-152')
+        print('ConvNeXt-Tiny: \t\t ConvNeXt Tiny')
+        
+    state_dict = load_file(
+    state_dict,
+    device="cpu" # keep on CPU while loading
+    )
+    
+    print(f'Running inference on model {model_name}')
+    
+    model.load_state_dict(state_dict, strict=True)
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    
+    model.to(device).eval() # Send model to GPU or CPU and put in evaluation mode
+    
+    val_tf = image_transformations.transform_val()    # Load in validation transforms for consistent evaluations
+
+    results = []
+    for fn in tqdm(sorted(os.listdir(images_dir)), desc="Processing images", unit=" images"):
+        fullpath = os.path.join(images_dir, fn)
+        
+        # Extract permeability from metadata in each image using EXIF (UserComment field)
+    
+        pil = Image.open(fullpath)
+        label_k = None
+        exif_bytes = pil.info.get('exif')
+        if exif_bytes:
+            exif_dict = piexif.load(exif_bytes)
+            user_comment = exif_dict['Exif'].get(piexif.ExifIFD.UserComment)
+            if user_comment:
+                label_k = float(user_comment.decode('utf-8'))
+
+        # Load the image as a tensor, convert to grayscale
+        img_tensor = torchvision.io.read_image(
+        fullpath, mode=torchvision.io.image.ImageReadMode.GRAY
+        ).float()
+
+        # Pass image through the validation image transformation pipeline
+        x = val_tf(img_tensor)           # now shape [1,512,512], dtype=bfloat16
+        x = x.to(torch.float32)
+        x = x.unsqueeze(0).to(device)    # Send image to the GPU or CPU
+
+        # --- inference ---
+        with torch.no_grad():
+            logk = model(x)              # network trained on out log(k)
+            pred_k = float(logk.exp().cpu())
+
+        # Append the results to the results dictionary
+        results.append({
+            'filename':  os.path.basename(fn),
+            'permeability': label_k,
+            'prediction': pred_k,
+        })
+        
+    # Save predictions to CSV
+    pd.options.display.float_format = '{:.6e}'.format
+    df = pd.DataFrame(results)
+    df.to_csv(save_file, index=False, float_format='%.6e')
+    print(f'Predictions saved to {save_file}')
+        
+       
+
+images_dir = '/path/to/images/'
+
+run_inference_cnn_base(model_name='ENet-B0',
+                  state_dict='/path/to/model/enet_b0_base.safetensors', 
+                  images_dir=images_dir, 
+                  save_file='enet_b0_preds.csv')

inference_examples/inference_mlp_full_safetensors.py

@@ -0,0 +1,66 @@
+import os
+import sys
+import torch
+import pandas as pd
+import numpy as np
+from safetensors.torch import load_file
+from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+
+# ── Adjust these paths as needed ──
+STATE_PATH = '/path/to/model/mlp_full_base.safetensors'
+DATA_PATH  = '/path/to/image/data.csv'
+save_file = 'predictions.csv' # Name of file to store predictions
+
+# ── Setup device ──
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+# ── Load state dict ──
+state_dict = load_file(STATE_PATH, device='cpu')
+
+# ── Load & prep data ──
+data = pd.read_csv(DATA_PATH).dropna(axis=1)
+
+# Identify feature columns by name
+start_idx = data.columns.get_loc('img_mean_intensity')
+feature_cols = data.columns[start_idx:]
+
+input_data = data[feature_cols].values.astype('float32')
+scaler = StandardScaler()
+input_data = scaler.fit_transform(input_data)
+
+labels = data['k'].values.astype('float32').reshape(-1,1)
+
+# Pull file name
+if 'id' in data.columns:
+    filenames = data['id'].values
+
+
+# ── Load model ──
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+from model_classes import MLP
+
+model = MLP.MLP(input_size=input_data.shape[1])
+model.load_state_dict(state_dict, strict=True)
+model.to(device).eval()
+
+# ── Run inference ──
+results = []
+with torch.no_grad():
+    for i in tqdm(range(len(input_data)), desc='Processing images', unit=' image(s)'):
+        sample = torch.tensor(input_data[i:i+1], dtype=torch.float32, device=device)
+        
+        permeability = float(labels[i])
+        
+        output = model(sample).cpu().numpy()[0][0]
+        pred = float(np.exp(output))
+        results.append({
+            'filename': os.path.basename(filenames[i]) + '_crop.jpeg', # To match the CNN predictions to be joined to a master CSV later
+            'permeability': permeability,
+            'prediction': pred
+        })
+
+# Save to CSV to save_file specified at beginning of script
+pd.options.display.float_format = '{:.6e}'.format
+results_df = pd.DataFrame(results)
+results_df.to_csv(save_file, index=False, float_format='%.6e')

inference_examples/inference_mlp_full_safetensors_tuned.py

@@ -0,0 +1,83 @@
+import os
+import sys
+import json
+import torch
+import pandas as pd
+import numpy as np
+from safetensors.torch import load_file
+from sklearn.preprocessing import StandardScaler
+
+# ── Adjust these paths as needed ──
+STATE_PATH = '/path/to/model/mlp_full_base.safetensors'
+DATA_PATH = '/path/to/image/data.csv'
+save_file = 'predictions.csv' # Name of file to store predictions
+
+# ── Setup device ──
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+# ── Load state dict ──
+state_dict = load_file(STATE_PATH, device='cpu')
+
+# ── Load & prep data ──
+data = pd.read_csv(DATA_PATH).dropna(axis=1)
+
+# Get feature columns
+start_idx = data.columns.get_loc('img_mean_intensity')
+feature_cols = data.columns[start_idx:]
+
+input_data = data[feature_cols].values.astype('float32')
+scaler = StandardScaler()
+input_data = scaler.fit_transform(input_data)
+
+labels = data['k'].values.astype('float32').reshape(-1,1)
+
+# Pull file name
+if 'id' in data.columns:
+    filenames = data['id'].values
+
+
+# ── Load model ──
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+from model_classes import MLP
+
+# Path to hyperparameter JSON file
+hyperparams_path = '/path/to/hyperparams/json/best_params_mlp.json'
+with open(hyperparams_path, 'r') as f:
+        hyperparams = json.load(f)
+        
+hyperparameters = hyperparams['hyperparameters']
+
+
+model = MLP.MLP(
+    input_size=input_data.shape[1],
+    hidden_size=hyperparameters['hidden_size'],
+    num_layers=hyperparameters['num_layers'],
+    learning_rate=hyperparameters['lr'],
+    weight_decay=hyperparameters['weight_decay'],
+    scheduler_factor=hyperparameters['scheduler_factor'],
+    scheduler_patience=hyperparameters['scheduler_patience'],
+    scheduler_threshold=hyperparameters['scheduler_threshold']
+    )
+model.load_state_dict(state_dict, strict=True)
+model.to(device).eval()
+
+# ── Run inference ──
+results = []
+with torch.no_grad():
+    for i in range(len(input_data)):
+        sample = torch.tensor(input_data[i:i+1], dtype=torch.float32, device=device)
+        
+        permeability = float(labels[i])
+        
+        output = model(sample).cpu().numpy()[0][0]
+        pred = float(np.exp(output))
+        results.append({
+            'filename': os.path.basename(filenames[i]),
+            'permeability': permeability,
+            'prediction': pred
+        })
+
+# Save to CSV to save_file specified at beginning of script
+pd.options.display.float_format = '{:.6e}'.format
+results_df = pd.DataFrame(results)
+results_df.to_csv(save_file, index=False, float_format='%.6e')

tuned/convnext_tiny/convnext_tiny_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83de8cc13c31a797d29080f84fcb17a7fe4f7db2386717db6d2146b011842906
+size 111350066

tuned/convnext_tiny/convnext_tiny_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a884ed3259e0d80987a894abf7ab067b484ccb3c75297fc8dc4825d9347c7885
+size 111283644

tuned/enet_b0/enet_b0_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:032094ed9087d94734c38424466bb15c45d526523d0586ec8b4444e839fe5845
+size 16021996

tuned/enet_b0/enet_b0_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf7df5f0673e36f4a7842356596c02396627694ed47fd91e653f32928f83d667
+size 16237652

tuned/enet_b3/enet_b3_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e36fc9e5f958e859956679d6f2ca4acdefbe50e67fd4d305e6cbdbc735ae6c0
+size 42736143

tuned/enet_b3/enet_b3_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fae288a552eea077976723bd3d0e7131461a38075239e87ab302f8ffa71caeb
+size 43200316

tuned/enet_b7/enet_b7_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41b9232aee65d493b0d1dc52a547c16b4c841beff75efbeac33d03b0901267fe
+size 254785573

tuned/enet_b7/enet_b7_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edbc94032df5b45429298ab8d56e6e4d90c924b674e3431b53e1bcf5eeaa3c3a
+size 256524508

tuned/mlp_dend/mlp_dend_3d_pretrain_300.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6914733a5758100deb9a38ac70f6cb074f7cc624df84e8cb01bba684bc8c7d
+size 60616

tuned/mlp_dend/mlp_dend_3d_pretrain_300.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e8549e0f7ea3772995f5d6810e12811a8132ea853bbdf95c7b1fae81441b777
+size 62872

tuned/mlp_dend/mlp_dend_3d_pretrain_300.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf3f07a9337834db1e825b219059be90687da2fbf4a9a8835a59269425e326f4
+size 60348

tuned/mlp_dend/mlp_dend_3d_pretrain_300_tscript.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4130f3c9006482c9797c25cf18c6ab89ca3eab1e7411560b6c1e93753e03a56
+size 102680

tuned/mlp_dend/mlp_dend_3d_scratch_300.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a7d2f6ebd06f4f756e1fabd589101ad4a7b04c5a6cc25ba55f7e050161af1ea
+size 76559

tuned/mlp_dend/mlp_dend_3d_scratch_300.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0d0d8721f7dd2ae0b7182ac70ddb38f13a2c893226f2bc2a57853c4fe375122
+size 80101

tuned/mlp_dend/mlp_dend_3d_scratch_300.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f240254877695ccd8d30b82eec32fab8950619ee2b6aa3b949c164a8334e66c8
+size 76124

tuned/mlp_dend/mlp_dend_3d_scratch_300_tscript.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12ede0587491fb3b707c71ec7e7552d5f7c3e4fc987a2987db94264d1b064241
+size 125596

tuned/mlp_dend/mlp_dend_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2f4f2d5f2ab1d8d83b2bb42fee21ebbd7aa00154192196b18822f9a26afa4fe
+size 60616

tuned/mlp_dend/mlp_dend_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d35307e6634d1f4ba31f54d8f7eb2a1149e8a7bfda16e5c5161f2c172e08989c
+size 60348

tuned/mlp_full/mlp_full_3d_pretrain_300.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:066150f017d26bcfa0f7be7a065bb1497164eeb63ed838621cc01d571cc79bd2
+size 1551626

tuned/mlp_full/mlp_full_3d_pretrain_300.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cdf469292917c2fcf3daae8e9bf3a4a91f90c769544be4db3b08b3597931371
+size 1551364

tuned/mlp_full/mlp_full_3d_scratch_300.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7675d5a23c67d9d5d24376163e6ca3396951c22a4e00dfd20cb9ac7c81877ca5
+size 203539

tuned/mlp_full/mlp_full_3d_scratch_300.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1102c981efb7d0eee802b9807691ccd0a8f8880f7a208508bbeef6765ae537a4
+size 203124

tuned/mlp_full/mlp_full_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b639130ca4374e73cb4029e35d255421a32a418ca5ffb58b2e3751358529d4f
+size 1551626

tuned/mlp_full/mlp_full_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d981362613e9c13e3b3ce680866b4e5c678e11b5ca2af899f550cc1ba525ac8
+size 1551364

tuned/mlp_o1/mlp_o1_3d_pretrain_300.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c47df13ed2ef20026846992f6cfe3ca8a67cb6aff5b4f0c77342b2a5dd75e14
+size 59784

tuned/mlp_o1/mlp_o1_3d_pretrain_300.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae3963683e3093c09e6c5db6021bf2526ab417c5b20849421a3ced0ba326f036
+size 59516

tuned/mlp_o1/mlp_o1_3d_scratch_300.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b220b45de6498e811702254f8ef11d0fa47b65c3db15cceb85444cc1ef8f41b
+size 74511

tuned/mlp_o1/mlp_o1_3d_scratch_300.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d91b408d5224bf7990bd3ae322b8ce6ac20b9f2a449b5b43e17d7a11cb88470
+size 74076

tuned/mlp_o1/mlp_o1_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5150de6b5df6bdaddd29f0301d55d7e77ca602a490ac660898818b5ddc485218
+size 59784

tuned/mlp_o1/mlp_o1_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63202059f526646ba730ff8def123e20be9d5110b9db1d2bbe097dfda8ee9373
+size 59516

tuned/resnet_152/resnet_152_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16504a487ee384f7569c3777dc1e8668b5b40bae8c9980b3c0325faf22f6f2e8
+size 232317981

tuned/resnet_152/resnet_152_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d4a371e4cd2496f70df81db182fe016717e28c5d99e9899e9af54d456814e03
+size 233245860

tuned/resnet_18/resnet_18_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c23b278bbcf87af8a025317ae58aaa9780b35a17b6e50a83798634911f7bed0d
+size 44663281

tuned/resnet_18/resnet_18_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f12d4f747bfdb8cff9d3de2ba6c589409740c2bc1f1b8bd04c4915e8b931429
+size 44723124

tuned/resnet_50/resnet_50_tuned.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:471d9bf03e6235366d1a2c35d560f2d97c836c4795c9ed57c4c5acadd14fbc6d
+size 93923245

tuned/resnet_50/resnet_50_tuned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d3b37455cc184e98e860a7cc891e22acfe846a6aec2660e38f6d2cd6734cb1c
+size 94248660