Upload json_data/parts_creation.py with huggingface_hub

json_data/parts_creation.py

@@ -0,0 +1,264 @@
+from sklearn.cluster import KMeans
+import numpy as np
+import cv2
+from matplotlib import pyplot as plt
+from bs4 import BeautifulSoup
+import json
+
+def is_ring_contour(contour):
+    """
+    简单的检查轮廓是否可能是环状的。
+    这个函数可以根据需要进一步完善。
+    """
+    # 计算轮廓的面积
+    area = cv2.contourArea(contour)
+    # 计算轮廓的周长
+    perimeter = cv2.arcLength(contour, True)
+    # 使用简单的准则判断轮廓是否是环状的
+    if perimeter == 0:
+        return False
+    else:
+        compactness = 4 * np.pi * area / (perimeter ** 2)
+        return compactness<1  # 这个阈值可以调整
+
+
+def get_the_parts(img_dir):
+
+    img = cv2.imread(img_dir)
+    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)  # 转换为灰度图像
+    _, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)  # 可能需要调整阈值
+    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) #整体的外轮廓
+    mask = np.zeros_like(img)
+    for contour in contours: 
+        cv2.drawContours(mask, [contour], -1, (255, 255, 255), thickness=cv2.FILLED)
+        result = cv2.bitwise_and(img, mask)
+    # 将BGR图像转换为RGB图像
+    result_rgb = cv2.cvtColor(result, cv2.COLOR_BGR2RGB)
+    # 步骤2: 准备数据
+    pixels = result_rgb.reshape((-1, 3))
+    pixels = np.float32(pixels)
+
+    # 步骤3: 应用K-means聚类
+    k = 12  # 聚类的数量
+    kmeans = KMeans(n_clusters=k, random_state=0).fit(pixels)
+    # 获取每个聚类的颜色中心
+    colors = kmeans.cluster_centers_
+
+    # 步骤4: 显示聚类颜色结果
+    # 创建一个空白图片，用于显示聚类的颜色
+    colors_img = np.zeros((50, 50*k, 3), dtype=np.uint8)
+    for i, color in enumerate(colors):
+        colors_img[:, i*50:(i+1)*50] = color
+    cnt_list=np.zeros(len(colors))
+    all_C=[]
+    
+    for i, color in enumerate(colors):
+    # 将RGB颜色转换为HSV颜色
+        color_hsv = cv2.cvtColor(np.uint8([[color]]), cv2.COLOR_RGB2HSV)[0][0]
+        lower_bound = color_hsv - np.array([15, 20, 20])
+        upper_bound = color_hsv + np.array([15, 20, 20])
+        mask = cv2.inRange(hsv, lower_bound, upper_bound)
+
+
+        # 轮廓检测
+        contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+        all_C.append(contours)
+    #     print(contours)
+        # 检查是否有环状轮廓
+        for contour in contours:
+            if is_ring_contour(contour):
+                # 环状轮廓绘制到原图上
+#                 cv2.drawContours(img, [contour], -1, (0, 255, 0), 2)
+                cnt_list[i]+=1
+#                 print(f"环状轮廓在颜色 {i} 中找到。")
+
+
+
+    cnt_list_copy = cnt_list.copy()
+
+    # 将第一个元素和所有0值替换为np.max(cnt_list) + 1，确保这些值不会是最小值
+    cnt_list_copy[0] = np.max(cnt_list) + 1
+    cnt_list_copy[cnt_list_copy == 0] = np.max(cnt_list) + 1
+    min_value_non_zero_non_first = np.min(cnt_list_copy)
+
+# 找到所有等于这个最小值的索引
+    min_indices_non_zero_non_first = np.where(np.abs(cnt_list_copy - min_value_non_zero_non_first) <= 3)[0]
+#     print(cnt_list)
+#     print(min_indices_non_zero_non_first)
+    mask_p = np.zeros_like(img)
+    output_list=[]
+    for this_c in min_indices_non_zero_non_first:
+#     this_c = np.argmin(cnt_list_copy)
+        
+#         print(this_c)
+        
+        for c in all_C[this_c]:
+            cv2.drawContours(mask_p, [c], -1, (255, 255, 255), thickness=cv2.FILLED)
+            result_p = cv2.bitwise_and(img, mask_p)
+        # 将BGR图像转换为RGB图像
+        result_p_rgb = cv2.cvtColor(result_p, cv2.COLOR_BGR2RGB)
+#         print(colors[this_c])
+        output_list.append(colors[this_c])
+    return output_list
+
+
+# 读取HTML文件
+
+def make_new_json(fn):
+    with open('lego/'+fn+'/'+fn+'.html', 'r', encoding='utf-8') as file:
+        html_content = file.read()
+
+    # 使用BeautifulSoup解析HTML
+    soup = BeautifulSoup(html_content, 'lxml')
+    rows = soup.find_all(class_='row')
+#     print(soup)
+
+    # 将提取的数据转换为JSON
+    data = []
+    snellius_dir='/gpfs/home4/hhuang/MiniGPT/MiniGPT-4/lego/'+fn+'/images/'
+    instruction_id = 0  # 初始化instruction_id
+    for row in rows:
+    #     row_data = {'instruction_id': instruction_id}
+        row_data = {
+            'instruction_id': instruction_id,
+            
+            'text':'None',
+            'VLM': {
+                'img_path':'/gpfs/home4/hhuang/MiniGPT/MiniGPT-4/lego/',
+                'id_sign':'None',
+                'step_num':'None',
+               'step_class':'None',
+                'other_sign':'None',
+                'bound_color':[],
+                'task_label': 'None',
+                'query': 'None',
+                'MiniGPTv2_output': 'None'
+            }
+        }
+        
+        
+        # 提取class="img"的元素
+        img = row.find(class_='image').find('img')  # 假设class="img"内有<img>标签
+        row_data['img'] = img['src'].replace("./LEGO 60274 Elite Police Lighthouse Capture_files/","https://legoaudioinstructions.com/wp-content/themes/mtt-wordpress-theme/assets/manual/manual-images/60274/").replace("#", "%23").replace(" ", "%20")if img and img.has_attr('src') else 'No image found'
+#         print(row_data['img'])
+
+        img_name=img['src'].split('/')[-1].replace("#", "%23").replace(" ", "%20").replace("./LEGO 60274 Elite Police Lighthouse Capture_files/https://legoaudioinstructions.com/wp-content/themes/mtt-wordpress-theme/assets/manual/manual-images/60274/", "%20")
+        print(img_name)
+        detect_list=img_name.split('-')[1:]
+
+        row_data['VLM']['id_sign']=detect_list[0].split('.png')[0]
+        if len(detect_list)==1:
+            row_data['VLM']['id_sign']='None'
+        else:
+            more_list=detect_list[1].split('_')
+            row_data['VLM']['step_num']=more_list[0]
+            row_data['VLM']['step_class']=more_list[1]
+            row_data['VLM']['other_sign']=detect_list[-1].split('%23')[-1][:-4]
+        
+        
+        
+        
+
+#         print('lego/'+fn+'/images/'+img['src'].split('/')[-1].replace("#", "%23").replace(" ", "%20").replace("./LEGO 60274 Elite Police Lighthouse Capture_files/https://legoaudioinstructions.com/wp-content/themes/mtt-wordpress-theme/assets/manual/manual-images/60274/", "%20")
+# )       
+        if row_data['img'] != 'No image found':
+            row_data['VLM']['img_path']= snellius_dir+img['src'].split('/')[-1].replace("#", "%23").replace(" ", "%20").replace("./LEGO 60274 Elite Police Lighthouse Capture_files/https://legoaudioinstructions.com/wp-content/themes/mtt-wordpress-theme/assets/manual/manual-images/60274/", "%20")
+            img_dir='lego/'+fn+'/images/'+img['src'].split('/')[-1].replace("#", "%23").replace(" ", "%20").replace("./LEGO 60274 Elite Police Lighthouse Capture_files/https://legoaudioinstructions.com/wp-content/themes/mtt-wordpress-theme/assets/manual/manual-images/60274/", "%20")
+            
+            
+            if row_data['VLM']['step_class']=="step":
+                row_data['VLM']['bound_color']=get_the_parts(img_dir)
+                print(row_data['VLM']['bound_color'])
+        
+
+        text = row.find(class_='text')
+    #     print(text.find_all(class_='txtbox'))
+        if text:
+            # 处理所有class="txtbox"的子元素
+            
+            
+            
+            
+            
+            
+            txtboxes = text.find_all(class_='txtbox')
+            row_data['text'] = [txtbox.text.strip() for txtbox in txtboxes]
+            row_data['entities'] = []
+            query_texts = []
+            for txtbox in txtboxes:
+                txtbox_data = [{child['class'][0] if child.has_attr('class') else 'this_line': child.text.strip()} for child in txtbox.find_all()]
+                row_data['entities'].append(txtbox_data)
+                query_texts.extend([child.text.strip() for child in txtbox.find_all() if child.has_attr('class') and child['class'][0] != 'verb'])
+
+            # 检查是否需要更改VLM的task_label
+
+
+            # 检查是否需要更改VLM的task_label
+            for line in row_data['entities']:
+                if len(line) > 1:
+                    if 'verb' in line[1]:
+                        if line[1]['verb'] == 'Collect':
+                            row_data['VLM']['task_label'] = '[detection-collect]'
+                            row_data['VLM']['query'] = 'Collect '+' '.join(query_texts)
+                            break
+                        elif line[1]['verb'] == 'Find':
+                            row_data['VLM']['task_label'] = '[detection]'
+    #                         print(line[0]['this_line'])
+                            row_data['VLM']['query'] = ''.join(line[0]['this_line'])
+                            break
+        else:
+            row_data['text'] = 'No text found'
+
+        # 添加到数据列表中
+        data.append(row_data)
+
+        # 为下一个row元素增加instruction_id
+        instruction_id += 1
+    whole_json={
+            "manual_id":fn,
+            "manual_type":"lego",
+            "instructions":data
+        }
+
+
+    json_data = json.dumps(whole_json, indent=4, ensure_ascii=False)
+
+    # 输出或保存JSON数据
+    # print(json_data)
+    # 或者写入文件
+    # json_name
+    with open('new_json_color/'+fn+'_new_color.json', 'w', encoding='utf-8') as json_file:
+        json_file.write(json_data)
+
+
+
+import wandb
+import random
+
+# start a new wandb run to track this script
+wandb.init(
+    # set the wandb project where this run will be logged
+    project="my-awesome-project",
+    
+    # track hyperparameters and run metadata
+    config={
+    "learning_rate": 0.02,
+    "architecture": "CNN",
+    "dataset": "CIFAR-100",
+    "epochs": 10,
+    }
+)
+
+# simulate training
+epochs = 10
+offset = random.random() / 5
+for epoch in range(2, epochs):
+    acc = 1 - 2 ** -epoch - random.random() / epoch - offset
+    loss = 2 ** -epoch + random.random() / epoch + offset
+    
+    # log metrics to wandb
+    wandb.log({"acc": acc, "loss": loss})
+    
+# [optional] finish the wandb run, necessary in notebooks
+wandb.finish()