import time from pathlib import Path import cv2 import torch import torch.nn as nn import torch.backends.cudnn as cudnn from numpy import random from torchvision import models, transforms import matplotlib.pyplot as plt from PIL import Image import numpy as np from numpy import random import torchvision import sys sys.path.append('yolov7-main') sys.path.append('./') # to run '$ python *.py' files in subdirectories from models.experimental import attempt_load from utils.datasets import LoadStreams, LoadImages from utils.general import check_img_size, check_imshow, non_max_suppression, apply_classifier, \ scale_coords, xyxy2xywh, set_logging from utils.plots import plot_one_box from utils.torch_utils import select_device, load_classifier, time_synchronized, TracedModel # from smooth_grad import generate_vanilla_grad from plaus_functs import generate_vanilla_grad transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=0., std=1.) ]) thisPath = "" def generate_feature_maps(img, con_layer): this_img = np.array(img) image = Image.fromarray(this_img, 'RGB') plt.imshow(image) # model = models.resnet18(weights=torchvision.models.ResNet18_Weights.IMAGENET1K_V1) model = models.resnet18(weights=torchvision.models.ResNet18_Weights.DEFAULT) # we will save the conv layer weights in this list model_weights =[] #we will save the 49 conv layers in this list conv_layers = [] # get all the model children as list model_children = list(model.children()) #counter to keep count of the conv layers counter = 0 #append all the conv layers and their respective wights to the list for i in range(len(model_children)): if type(model_children[i]) == nn.Conv2d: counter+=1 model_weights.append(model_children[i].weight) conv_layers.append(model_children[i]) elif type(model_children[i]) == nn.Sequential: for j in range(len(model_children[i])): for child in model_children[i][j].children(): if type(child) == nn.Conv2d: counter+=1 model_weights.append(child.weight) conv_layers.append(child) if torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') model = model.to(device) image = transform(image) image = image.unsqueeze(0) image = image.to(device) outputs = [] names = [] for layer in conv_layers[0:]: image = layer(image) outputs.append(image) names.append(str(layer)) processed = [] for feature_map in outputs: feature_map = feature_map.squeeze(0) gray_scale = torch.sum(feature_map,0) gray_scale = gray_scale / feature_map.shape[0] processed.append(gray_scale.data.cpu().numpy()) # Plot and save feature maps for each layer for i, (fm, name) in enumerate(zip(processed, names)): fig = plt.figure(figsize=(10, 10)) a = fig.add_subplot(1, 1, 1) # You should adjust the layout as needed imgplot = plt.imshow(fm, cmap='viridis') # Adjust the colormap if needed a.axis("off") filename = f'layer{i}.jpg' plt.savefig("outputs\\runs\\detect\\exp\\layers\\" + filename, bbox_inches='tight') plt.close(fig) # Close the figure after saving this_dir = "outputs\\runs\\detect\\exp\\layers\\layer" + str(int(int(con_layer) - 1)) + '.jpg' print("Convolutional layers Generated") return this_dir def detect(opt, is_stream, outputNum=1, norm=False, save_img=False): source, weights, view_img, save_txt, imgsz, trace = opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size, not opt.no_trace save_img = not opt.nosave and not source.endswith('.txt') # save inference images webcam = source.isnumeric() or source.endswith('.txt') or source.lower().startswith( ('rtsp://', 'rtmp://', 'http://', 'https://')) # Directories save_dir = Path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir # Initialize set_logging() device = select_device(opt.device) half = device.type != 'cpu' # half precision only supported on CUDA half = False # Load model model = attempt_load(weights, map_location=device) # load FP32 model stride = int(model.stride.max()) # model stride imgsz = check_img_size(imgsz, s=stride) # check img_size if trace: model = TracedModel(model, device, opt.img_size) if half: model.half() # to FP16 # Second-stage classifier classify = False if classify: modelc = load_classifier(name='resnet101', n=2) # initialize modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']).to(device).eval() # Set Dataloader vid_path, vid_writer = None, None if webcam: view_img = check_imshow() cudnn.benchmark = True # set True to speed up constant image size inference dataset = LoadStreams(source, img_size=imgsz, stride=stride) else: dataset = LoadImages(source, img_size=imgsz, stride=stride) # Get names and colors names = model.module.names if hasattr(model, 'module') else model.names colors = [[random.randint(0, 255) for _ in range(3)] for _ in names] # Run inference if device.type != 'cpu': model(torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(next(model.parameters()))) # run once old_img_w = old_img_h = imgsz old_img_b = 1 t0 = time.time() for path, img, im0s, vid_cap in dataset: img = torch.from_numpy(img).to(device) img = img.half() if half else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 if img.ndimension() == 3: img = img.unsqueeze(0) # Warmup if device.type != 'cpu' and (old_img_b != img.shape[0] or old_img_h != img.shape[2] or old_img_w != img.shape[3]): old_img_b = img.shape[0] old_img_h = img.shape[2] old_img_w = img.shape[3] for i in range(3): model(img, augment=opt.augment)[0] # Inference t1 = time_synchronized() with torch.no_grad(): # Calculating gradients would cause a GPU memory leak pred = model(img, augment=opt.augment)[0] t2 = time_synchronized() # Apply NMS pred = non_max_suppression(pred.cpu(), opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) t3 = time_synchronized() # Apply Classifier if classify: pred = apply_classifier(pred, modelc, img, im0s) # Process detections labels = {} allDetcs = [] for i, det in enumerate(pred): # detections per image if webcam: # batch_size >= 1 p, s, im0, frame = path[i], '%g: ' % i, im0s[i].copy(), dataset.count else: p, s, im0, frame = path, '', im0s, getattr(dataset, 'frame', 0) p = Path(p) # to Path save_path = str(save_dir / p.name) # img.jpg txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # img.txt gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh if len(det): # Rescale boxes from img_size to im0 size det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() # Print results for c in det[:, -1].unique(): n = (det[:, -1] == c).sum() # detections per class s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string if dataset.mode == 'image': model.train() smooth_gradient1 = generate_vanilla_grad(model=model, input_tensor=img, out_num=1, targets=None, norm=norm, device=device) torchvision.utils.save_image(smooth_gradient1,fp="outputs\\runs\\detect\\exp\\smoothGrad0.jpg") smooth_gradient2 = generate_vanilla_grad(model=model, input_tensor=img, out_num=2, targets=None, norm=norm, device=device) torchvision.utils.save_image(smooth_gradient2,fp="outputs\\runs\\detect\\exp\\smoothGrad1.jpg") smooth_gradient3 = generate_vanilla_grad(model=model, input_tensor=img, out_num=3, targets=None, norm=norm, device=device) torchvision.utils.save_image(smooth_gradient3,fp="outputs\\runs\\detect\\exp\\smoothGrad2.jpg") model.eval() # Write results for *xyxy, conf, cls in reversed(det): if save_txt: # Write to file xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh line = (cls, *xywh, conf) if opt.save_conf else (cls, *xywh) # label format with open(txt_path + '.txt', 'a') as f: f.write(('%g ' * len(line)).rstrip() % line + '\n') if save_img or view_img: # Add bbox to image label = f'{names[int(cls)]} {conf:.2f}' allDetcs.append(label) if (names[int(cls)] not in labels or labels[names[int(cls)]] < conf.item()) and conf is not None: plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=1) # Print time (inference + NMS) print(f'{s}Done. ({(1E3 * (t2 - t1)):.1f}ms) Inference, ({(1E3 * (t3 - t2)):.1f}ms) NMS') # Stream results if view_img: cv2.imshow(str(p), im0) cv2.waitKey(1) # 1 millisecond # Save results (image with detections) if save_img: if dataset.mode == 'image': cv2.imwrite(save_path, im0) print(f" The image with the result is saved in: {save_path}") else: # 'video' or 'stream' if vid_path != save_path: # new video vid_path = save_path if isinstance(vid_writer, cv2.VideoWriter): vid_writer.release() # release previous video writer if vid_cap: # video fps = vid_cap.get(cv2.CAP_PROP_FPS) w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) else: # stream fps, w, h = 30, im0.shape[1], im0.shape[0] save_path += '.mp4' vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'h264'), fps, (w, h)) vid_writer.write(im0) if save_txt or save_img: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' print(f"Results saved to {save_dir}{s}") if dataset.mode == 'image': formatted_time = f"{time.time() - t0:.2f}" print(f'Done. ({formatted_time}s)') print(allDetcs) return [str(save_path), "outputs\\runs\\detect\\exp\\smoothGrad" + str(int(int(outputNum) -1)) + ".jpg", allDetcs, formatted_time] else: return str(save_path)