File size: 4,830 Bytes
6918c6b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
from scipy.interpolate import interp1d
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def load_spectra_from_QM9(data_dir):
df = pd.read_csv(data_dir)
filtered_data = df.dropna().values
print(filtered_data[0])
def normalize_data(data):
min_val = np.min(data, axis=0, keepdims=True)
max_val = np.max(data, axis=0, keepdims=True)
return (data - min_val) / (max_val - min_val)
if filtered_data.shape[1] > 3500:
filtered_data = filtered_data[:, 1:3501]
filtered_data = np.array([normalize_data(row) for row in filtered_data])
return filtered_data
def gpu_normalize_safe(data_tensor):
nan_mask = torch.isnan(data_tensor)
inf_mask = torch.isinf(data_tensor)
if torch.any(nan_mask) or torch.any(inf_mask):
data_tensor = torch.nan_to_num(data_tensor, nan=0.0, posinf=1.0, neginf=0.0)
min_vals = torch.min(data_tensor, dim=1, keepdim=True)[0]
max_vals = torch.max(data_tensor, dim=1, keepdim=True)[0]
range_vals = max_vals - min_vals
range_vals[range_vals == 0] = 1.0
normalized = (data_tensor - min_vals) / range_vals
return normalized
def load_spectra_from_QMe14S(data_dir):
df = pd.read_csv(data_dir)
spectral_columns = [col for col in df.columns if col.startswith('wavenumber_')]
if len(spectral_columns) == 0:
raise ValueError("did not find spectral data columns starting with 'wavenumber_'")
spectral_data = df[spectral_columns].copy()
spectral_data = spectral_data.apply(pd.to_numeric, errors='coerce')
spectral_data = spectral_data.dropna()
if len(spectral_data) == 0:
raise ValueError("all rows contain NaN values after conversion")
# Ensure writable contiguous memory before converting to torch tensor.
filtered_data = spectral_data.to_numpy(dtype=np.float32, copy=True)
filtered_data = np.ascontiguousarray(filtered_data)
data_tensor = torch.from_numpy(filtered_data).to(device)
print(f"data has transferred to device: {data_tensor.device}")
if data_tensor.shape[1] > 3500:
data_tensor = data_tensor[:, :3500]
data_tensor = gpu_normalize_safe(data_tensor)
filtered_data = data_tensor.cpu().numpy()
dedimed_spectrum_list = []
for i, spectrum in enumerate(filtered_data):
original_wave_min = 500
original_wave_max = 4000
original_wavenumbers = np.linspace(original_wave_min, original_wave_max, len(spectrum))
target_wave_min = 0
target_wave_max = 3500
target_wavenumbers = np.linspace(target_wave_min, target_wave_max, 3500)
interp_func = interp1d(
original_wavenumbers,
spectrum,
kind='linear',
bounds_error=False,
fill_value=0.0,
assume_sorted=True
)
interpolated_spectrum = interp_func(target_wavenumbers)
mask_low = target_wavenumbers < 500
interpolated_spectrum[mask_low] = 1e-10
interpolated_spectrum = np.nan_to_num(interpolated_spectrum, nan=1e-10)
dedimed_spectrum_list.append(interpolated_spectrum)
dedimed_spectrum_list = np.array(dedimed_spectrum_list)
return dedimed_spectrum_list
def load_real_data(data_path, labels_path=None, wavenumbers_path=None, normalize=True):
try:
spectra = np.load(data_path)
if np.isnan(spectra).any():
spectra = np.nan_to_num(spectra, nan=0.0)
if normalize:
for i in range(spectra.shape[0]):
spectrum = spectra[i]
min_val = np.min(spectrum)
max_val = np.max(spectrum)
if max_val > min_val:
spectra[i] = (spectrum - min_val) / (max_val - min_val)
labels = None
if labels_path:
try:
labels = np.load(labels_path)
if labels.shape[0] != spectra.shape[0]:
print(f"warning：label_num({labels.shape[0]})doesn't match spectra_num({spectra.shape[0]})!")
except Exception as e:
print(f"fail to load labels: {e}")
wavenumbers = None
if wavenumbers_path is not None:
try:
wavenumbers = np.load(wavenumbers_path)
if wavenumbers.shape[0] != spectra.shape[1]:
print(f"warning：wavenumber_length({wavenumbers.shape[0]})doesn't match spectra_length({spectra.shape[1]})!")
except Exception as e:
print(f"fail to load wavenumbers: {e}")
return spectra, labels, wavenumbers
except Exception as e:
print(f"fail to load spectra data: {e}")
return None, None, None |