updates to quantity module; adding qrf_model_bundle trained in environment matching container

CHRIS.py

@@ -1,5 +1,4 @@
 from flask import Flask, render_template
-#from functions import weight_func
 
 app = Flask(__name__)
 app.debug = False

qrf_functions.py

@@ -9,7 +9,7 @@ import mordred.descriptors
 import rdkit
 from rdkit import Chem
 
-def QRF_Ceramic(density, polytg, quantiles=[0.03,0.5,0.97], T=37):
+def QRF_Ceramic(density, polytg, quantiles=[0.03,0.5,0.97], T=37, worstcase='hi'):
     with open(f'qrf_model_bundle_{int(T)}.pkl','rb') as f:
         reg, imp, scaler_X, sub_desc_list = pickle.load(f)
     df_X = pd.read_excel('qrf_x.xlsx')
@@ -17,8 +17,12 @@ def QRF_Ceramic(density, polytg, quantiles=[0.03,0.5,0.97], T=37):
     X_all = imp.transform(df_X)
     X_all_scale = scaler_X.transform(X_all)
     ## use "worst-case" solute values
-    tmpq = np.array([0.95]*len(sub_desc_list))
-    tmpq[df_X.corrwith(df_y['LogD'])>0] = 0.05 # positive correlations (increase in variable increases D) use low values of variable, negative correlations use high values of variable
+    if worstcase == 'hi':
+        tmpq = np.array([0.95]*len(sub_desc_list))
+        tmpq[df_X.corrwith(df_y['LogD'])>0] = 0.05 # positive correlations (increase in variable increases D) use low values of variable, negative correlations use high values of variable
+    elif worstcase == 'lo':
+        tmpq = np.array([0.05]*len(sub_desc_list))
+        tmpq[df_X.corrwith(df_y['LogD'])>0] = 0.95 # inverse of above
     tmpv = [np.nanquantile(X_all_scale[:,i], q) for i,q in enumerate(tmpq)] # "worst-case" values of scaled descriptors
     tmps = [polytg if n == 'Polymer_Tg' else (density if n == 'Polymer_Density' else 0) for i,n in enumerate(sub_desc_list)]
     tmps = scaler_X.transform([tmps])[0] # scaled values of polymer descriptors

qrf_model_bundle_37.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5c2b2899461c3597863a6271f0bb37658f053daf1d0cba509b98b445fb45698
+size 15583438

qrf_train.py

@@ -5,7 +5,7 @@ import sklearn
 import sklearn.impute
 from quantile_forest import RandomForestQuantileRegressor
 
-T_target = 50
+T_target = 37
 T_cut = 5
 qhiv, qlov = 0.97, 0.03
 state = 12345

quantity_functions.py

@@ -15,10 +15,8 @@ import mordred.descriptors
 
 from functions import PowerLaw, Piringer
 
-### TODO arbitrary T
-
 ## sampling parameters
-N_sample = int(1e6)
+N_sample = int(1e5)
 #rng = np.random.Generator(np.random.PCG64(seed=12345))
 
 ## list of solvents to include
@@ -32,11 +30,6 @@ use_new = True
 T_cut_new = 0.5 
 
 #### read data files
-# CHRIS bounds
-#with open('quantity_module/data/bounds.pkl', 'rb') as fp:
-#    params_dict_ub, params_dict_lb, params_dict_ub_band, params_dict_lb_band, params_dict_q95, params_dict_q50, params_dict_q05 = pickle.load(fp)
-#with open('quantity_module/data/bounds-50.pkl', 'rb') as fp:
-#    params_dict_ub_50, params_dict_lb_50, params_dict_ub_band_50, params_dict_lb_band_50, params_dict_q95_50, params_dict_q50_50, params_dict_q05_50 = pickle.load(fp)
 # CHRIS parameter distributions
 if not use_new:
     param_dists = {}
@@ -50,10 +43,6 @@ else:
 # other stuff
 df_visc = pd.read_excel('quantity_module/data/solvent-viscosity.xlsx')
 df_desc = pd.read_excel(f'quantity_module/data/data-descriptors-mordred-numconfs51.xlsx', usecols=['Solute_InChIKey', 'Vabc','VMcGowan'])
-#dfp = pd.read_excel('quantity_module/data/db-properties-polymer.xlsx') ## XXX?
-#df2 = pd.read_excel('quantity_module/data/db-D.xlsx', sheet_name=1) ## XXX?
-#df2['CHRIS Class'] = df2['CHRIS Class'].fillna('none')
-#df2['New Class'] = df2['New Class'].fillna('none')
 if not use_new:
     ## clean data
     df_final_37 = pd.read_excel('quantity_module/data/db-D-interp-37-clean.xlsx')
@@ -86,7 +75,7 @@ for solv in df_visc['Solvent_Name']:
     mws.append(mw)
 df_visc['MW'] = mws
 ## selected solvent MWs
-SolventMWs = {solv:df_visc.loc[df_visc['Solvent_Name']==solv,'MW'].iloc[0] for solv in solvents}
+Solvent_MWs = {solv:df_visc.loc[df_visc['Solvent_Name']==solv,'MW'].iloc[0] for solv in solvents}
 # linear relation to estimate Vabc when it fails for a molecule
 Vabc = df_desc['Vabc']
 Vmcg = df_desc['VMcGowan']
@@ -241,7 +230,7 @@ def get_D_dists(w,T,Polymer_Tg,Solvent_Name,Solvent_MW,Solute_MW,CHRIS_category,
             else:
                 Ball = params[1]
                 A_list = params[2:]
-                D_list = np.exp([PowerLaw(Solute_MW, Ai, Ball) for Ai in A_list])
+                D_list = np.array([PowerLaw(Solute_MW, Ai, Ball) for Ai in A_list])
         else:
             D_list = input_Ds
     else:
@@ -260,7 +249,7 @@ def get_D_dists(w,T,Polymer_Tg,Solvent_Name,Solvent_MW,Solute_MW,CHRIS_category,
                 else:
                     Ball = params[1]
                     A_list = params[2:]
-                    D_list += list(np.exp([PowerLaw(Solute_MW, Ai, Ball) for Ai in A_list]))
+                    D_list += [PowerLaw(Solute_MW, Ai, Ball) for Ai in A_list]
         else:
             D_list = input_Ds
     D_dist_noswell = rng.choice(D_list, N)

quantity_module/quantity.py

@@ -3,7 +3,7 @@ import numpy as np
 import pandas as pd
 from flask import render_template, request
 from functions import SigFigs, HtmlNumber, Piringer, WilkeChang, CdfPlot
-from functions import Piecewise, PowerLaw
+#from functions import Piecewise, PowerLaw
 from qrf_functions import QRF_Apply, QRF_Ceramic
 from . import blueprint
 from polymers import Polymers, Polymers3
@@ -36,7 +36,10 @@ def exp_post():
 
     Polymer_Tg = float(request.form['Polymer_Tg'])  ## NOTE Tg is provided in C
     T = float(request.form['T']) 
+    Polymer_Tg += 273.15
+    T += 273.15
     rng = np.random.Generator(np.random.PCG64(seed=12345))
+    CHRIS_category = None
 
     if T<Polymer_Tg:
         return render_template('quantity_temperatureError.html')
@@ -89,43 +92,44 @@ def exp_post():
     M_expt = float(request.form['amount']) # amount
     units = request.form['units'] 
     mass = float(request.form['mass'])
-    PolymerDensity = float(request.form['density'])
-    PolymerVolume = mass / PolymerDensity # vol
-    SurfaceArea = float(request.form['area']) # area
-    SolventVolume = float(request.form['solventvol']) 
-    SolventName = request.form['solventname'] 
+    Polymer_Density = float(request.form['density'])
+    Polymer_Volume = mass / Polymer_Density # vol
+    Surface_Area = float(request.form['area']) # area
+    Solvent_Volume = float(request.form['solventvol']) 
+    Solvent_Name = request.form['solventname'] 
     Swelling_percent = float(request.form['swelling']) 
     Swelling_wtfrac = Swelling_percent/100
-    ExtractionTime = float(request.form['time']) 
+    Extraction_Time = float(request.form['time']) 
     K_expt = float(request.form['K']) 
-    SolventMW = SolventMWs[SolventName]
-    SoluteMW = MW
+    Solvent_MW = Solvent_MWs[Solvent_Name]
+    Solute_MW = MW
 
     polymer = request.form['polymer']
     pIndex = np.argmax(polymers == polymer)
 
-    use_qrf = False
-    if polymer == 'Other polymer':
+    # QRF is only implemented for 37 and 50 C
+    if polymer == 'Other polymer' and round(T) in [310,323]:
         use_qrf = True
+    else:
+        use_qrf = False
 
     if use_qrf:
         method = 'qrf'
-        quantiles = list(np.linspace(0,1,101))
+        ## XXX fix this so the density of quantiles matches density of distribution??
+        #quantiles = list(np.linspace(0,1,101))
+        quantiles = list(np.linspace(0.05,0.95,181))
         if is_ceramic:
-            diff,domain_extrap = QRF_Ceramic(PolymerDensity, Polymer_Tg, quantiles=quantiles, T=T)
+            diff,domain_extrap = QRF_Ceramic(Polymer_Density, Polymer_Tg, quantiles=quantiles, T=T-273.15, worstcase='lo')
         else:
-            diff,domain_extrap = QRF_Apply(PolymerDensity, Polymer_Tg, smiles, quantiles=quantiles, T=T)
-        #diff = diff[2] # upper bound
+            diff,domain_extrap = QRF_Apply(Polymer_Density, Polymer_Tg, smiles, quantiles=quantiles, T=T-273.15)
         if domain_extrap:
             # outside training domain, default to Wilke-Chang
-            #diff = Piecewise(MW, params[None])
-            D_dist_noswell, D_dist_swell = get_D_dists(Swelling_wtfrac, T+273.15, Polymer_Tg+273.15, SolventName, SolventMW, SoluteMW, 'G2', rng, return_DCs=False, N=N_sample)
-            M0_pred = get_M_dist(D_dist_swell, M_expt, PolymerVolume, SurfaceArea, SolventVolume, ExtractionTime*3600, K_expt=K_expt)
+            D_dist_noswell, D_dist_swell = get_D_dists(Swelling_wtfrac, T, Polymer_Tg, Solvent_Name, Solvent_MW, Solute_MW, 'G2', rng, return_DCs=False, N=N_sample)
+            M0_pred = get_M_dist(D_dist_swell, M_expt, Polymer_Volume, Surface_Area, Solvent_Volume, Extraction_Time*3600, K_expt=K_expt)
             method = 'qrf/wc'
         else:
-            ## TODO implement total quantity prediction with QRF
-            D_dist_noswell, D_dist_swell = get_D_dists(Swelling_wtfrac, T+273.15, Polymer_Tg+273.15, SolventName, SolventMW, SoluteMW, 'G2', rng, return_DCs=False, N=N_sample, input_Ds=diff)
-            M0_pred = get_M_dist(D_dist_swell, M_expt, PolymerVolume, SurfaceArea, SolventVolume, ExtractionTime*3600, K_expt=K_expt)
+            D_dist_noswell, D_dist_swell = get_D_dists(Swelling_wtfrac, T, Polymer_Tg, Solvent_Name, Solvent_MW, Solute_MW, None, rng, return_DCs=False, N=N_sample, input_Ds=diff)
+            M0_pred = get_M_dist(D_dist_swell, M_expt, Polymer_Volume, Surface_Area, Solvent_Volume, Extraction_Time*3600, K_expt=K_expt)
     else:
         ## use categories
         CHRIS_category = categories[pIndex]
@@ -134,19 +138,21 @@ def exp_post():
             ## worst-case for a generic polymer --> G2
             CHRIS_flag = 'wc'
             CHRIS_category = 'G2'
-        D_dist_noswell, D_dist_swell = get_D_dists(Swelling_wtfrac, T+273.15, Polymer_Tg+273.15, SolventName, SolventMW, SoluteMW, CHRIS_category, rng, return_DCs=False, N=N_sample)
-        M0_pred = get_M_dist(D_dist_swell, M_expt, PolymerVolume, SurfaceArea, SolventVolume, ExtractionTime*3600, K_expt=K_expt)
-        if 0:
-            print('Swelling_wtfrac, T+273.15, Polymer_Tg+273.15, SolventName, SolventMW, SoluteMW, CHRIS_category')
-            print(Swelling_wtfrac, T+273.15, Polymer_Tg+273.15, SolventName, SolventMW, SoluteMW, CHRIS_category)
-            print(np.nanquantile(D_dist_swell, [0.05,0.5,0.95]))
-            print('M_expt, PolymerVolume, SurfaceArea, SolventVolume, ExtractionTime*3600, K_expt')
-            print(M_expt, PolymerVolume, SurfaceArea, SolventVolume, ExtractionTime*3600, K_expt)
-            print(np.nanquantile(M0_pred, [0.05,0.5,0.95]))
+        D_dist_noswell, D_dist_swell = get_D_dists(Swelling_wtfrac, T, Polymer_Tg, Solvent_Name, Solvent_MW, Solute_MW, CHRIS_category, rng, return_DCs=False, N=N_sample)
+        M0_pred = get_M_dist(D_dist_swell, M_expt, Polymer_Volume, Surface_Area, Solvent_Volume, Extraction_Time*3600, K_expt=K_expt)
         if CHRIS_flag is None:
             method = 'category'
         else:
             method = 'wc'
+    if 1:
+        print('Swelling_wtfrac, T, Polymer_Tg, Solvent_Name, Solvent_MW, Solute_MW, CHRIS_category')
+        print(Swelling_wtfrac, T, Polymer_Tg, Solvent_Name, Solvent_MW, Solute_MW, CHRIS_category)
+        print(D_dist_noswell)
+        print(np.nanquantile(D_dist_noswell, [0.05,0.5,0.95]))
+        print(np.nanquantile(D_dist_swell, [0.05,0.5,0.95]))
+        print('M_expt, Polymer_Volume, Surface_Area, Solvent_Volume, Extraction_Time*3600, K_expt')
+        print(M_expt, Polymer_Volume, Surface_Area, Solvent_Volume, Extraction_Time*3600, K_expt)
+        print(np.nanquantile(M0_pred, [0.05,0.5,0.95]))
 
     # Generate the rate plot using matplotlib
     #pngImageB64String = CdfPlot(M0_pred[~np.isnan(M0_pred)], units=units)
@@ -165,15 +171,15 @@ def exp_post():
     table = df_table.style.set_properties(subset=[f'\( M_0 \) ({units})', r'\( M_0 \) (% median)'], **{'text-align': 'right'}).set_table_attributes('border="1"').hide(axis='index').to_html(index=False, escape=False, justify='center')
     #print(table)
 
-    tau = np.nanquantile(D_dist_swell,0.5) * (ExtractionTime*3600) / (PolymerVolume/SurfaceArea)**2
+    tau = np.nanquantile(D_dist_swell,0.5) * (Extraction_Time*3600) / (Polymer_Volume/Surface_Area)**2
 
     M0_out = SigFigs(np.nanquantile(M0_pred,0.5),6)
     tau_out = SigFigs(tau,6)
 
     return render_template('quantity_report.html', show_properties=show_properties, polymers=polymers, pIndex=pIndex, 
-                           area=SurfaceArea, vol=PolymerVolume, units=units, M=M_expt, M0=M0_out, time=ExtractionTime, 
-                           solventvol=SolventVolume, solventname=SolventName, swelling=Swelling_percent, K=K_expt, T=T, tau=tau_out,
+                           area=Surface_Area, vol=Polymer_Volume, units=units, M=M_expt, M0=M0_out, time=Extraction_Time, 
+                           solventvol=Solvent_Volume, solventname=Solvent_Name, swelling=Swelling_percent, K=K_expt, T=T, tau=tau_out,
                            chemName=chemName, MW=MW, LogP=LogP, rho=rho, mp=mp, iupac=iupac, cas=cas, smiles=smiles, molImage=molImage, table=table,
-                           LogP_origin=LogP_origin, rho_origin=rho_origin, mp_origin=mp_origin, ceramic=is_ceramic, methods=[method,Polymer_Tg,PolymerDensity],
-                           mass=mass, density=PolymerDensity)
+                           LogP_origin=LogP_origin, rho_origin=rho_origin, mp_origin=mp_origin, ceramic=is_ceramic, methods=[method,Polymer_Tg,Polymer_Density],
+                           mass=mass, density=Polymer_Density)