Remove unnecessary files (batch 111)

research/securebert2/opensource_data/data_sentence_pairs.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d00b5141b51b7a0c546374f29ffe7fa57c5f448b516642399985a1e82ce225b3
-size 879057

research/securebert2/opensource_data/data_sentence_pairs_test.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3190943fc8528a62d8bdbbdc122e3d706b4eb97c5fa6235f03c6caaa916706a3
-size 197063

research/securebert2/opensource_data/data_vuln_dataset.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:294ddc5c3732ca5ccf087bfe9bc49e08fff84102c9fee79845ad8bbd2b81e193
-size 338245

research/securebert2/opensource_data/data_vuln_dataset_test.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5297d540407236def70adac3866f60ed1f4c980bad854cafb7473e07d13a82bc
-size 88363

research/securebert2/requirements.txt

@@ -1,211 +0,0 @@
-absl-py==2.3.0
-accelerate==1.9.0
-aiohappyeyeballs==2.6.1
-aiohttp==3.12.9
-aiosignal==1.3.2
-annotated-types==0.7.0
-anyio==4.9.0
-argon2-cffi==25.1.0
-argon2-cffi-bindings==21.2.0
-arrow==1.3.0
-asttokens==3.0.0
-async-lru==2.0.5
-async-timeout==5.0.1
-attrs==25.3.0
-babel==2.17.0
-backcall==0.2.0
-backoff==2.2.1
-beautifulsoup4==4.13.4
-bleach==6.2.0
-boto3==1.38.29
-botocore==1.38.29
-cachetools==5.5.2
-certifi==2025.4.26
-cffi==1.17.1
-charset-normalizer==3.4.2
-click==8.2.1
-comm==0.2.2
-contourpy==1.3.2
-cycler==0.12.1
-datasets==3.6.0
-debugpy==1.8.14
-decorator==5.2.1
-defusedxml==0.7.1
-dill==0.3.8
-docker==7.1.0
-docopt==0.6.2
-exceptiongroup==1.3.0
-executing==2.2.0
-fastapi==0.115.12
-fastjsonschema==2.21.1
-filelock==3.18.0
-fonttools==4.58.1
-fqdn==1.5.1
-frozenlist==1.6.2
-fsspec==2025.3.0
-google-auth==2.40.3
-google-auth-oauthlib==1.2.2
-grpcio==1.72.1
-h11==0.16.0
-hf-xet==1.1.3
-httpcore==1.0.9
-httptools==0.6.4
-httpx==0.28.1
-huggingface-hub==0.32.4
-idna==3.10
-inquirerpy==0.3.4
-ipykernel==6.26.0
-ipython==8.12.3
-ipywidgets==8.1.1
-isoduration==20.11.0
-jedi==0.19.2
-Jinja2==3.1.6
-jmespath==1.0.1
-joblib==1.5.1
-json5==0.12.0
-jsonpointer==3.0.0
-jsonschema==4.24.0
-jsonschema-specifications==2025.4.1
-jupyter-events==0.12.0
-jupyter-lsp==2.2.5
-jupyter_client==8.6.3
-jupyter_core==5.8.1
-jupyter_server==2.16.0
-jupyter_server_terminals==0.5.3
-jupyterlab==4.2.0
-jupyterlab_pygments==0.3.0
-jupyterlab_server==2.27.3
-jupyterlab_widgets==3.0.15
-kiwisolver==1.4.8
-lightning==2.5.1.post0
-lightning-cloud==0.5.70
-lightning-utilities==0.14.3
-lightning_sdk==0.2.18
-litdata==0.2.45
-litserve==0.2.11
-Markdown==3.8
-markdown-it-py==3.0.0
-MarkupSafe==3.0.2
-matplotlib==3.8.2
-matplotlib-inline==0.1.7
-mdurl==0.1.2
-mistune==3.1.3
-mpmath==1.3.0
-multidict==6.4.4
-multiprocess==0.70.16
-nbclient==0.10.2
-nbconvert==7.16.6
-nbformat==5.10.4
-nest-asyncio==1.6.0
-networkx==3.4.2
-notebook_shim==0.2.4
-numpy==1.26.4
-nvidia-cublas-cu12==12.8.3.14
-nvidia-cuda-cupti-cu12==12.8.57
-nvidia-cuda-nvrtc-cu12==12.8.61
-nvidia-cuda-runtime-cu12==12.8.57
-nvidia-cudnn-cu12==9.7.1.26
-nvidia-cufft-cu12==11.3.3.41
-nvidia-cufile-cu12==1.13.0.11
-nvidia-curand-cu12==10.3.9.55
-nvidia-cusolver-cu12==11.7.2.55
-nvidia-cusparse-cu12==12.5.7.53
-nvidia-cusparselt-cu12==0.6.3
-nvidia-nccl-cu12==2.26.2
-nvidia-nvjitlink-cu12==12.8.61
-nvidia-nvtx-cu12==12.8.55
-oauthlib==3.2.2
-overrides==7.7.0
-packaging==24.2
-pandas==2.1.4
-pandocfilters==1.5.1
-parso==0.8.4
-pexpect==4.9.0
-pfzy==0.3.4
-pickleshare==0.7.5
-pillow==11.2.1
-pipreqs==0.5.0
-platformdirs==4.3.8
-prometheus_client==0.22.1
-prompt_toolkit==3.0.51
-propcache==0.3.1
-protobuf==4.23.4
-psutil==7.0.0
-ptyprocess==0.7.0
-pure_eval==0.2.3
-pyarrow==20.0.0
-pyasn1==0.6.1
-pyasn1_modules==0.4.2
-pycparser==2.22
-pydantic==2.11.5
-pydantic_core==2.33.2
-Pygments==2.19.1
-PyJWT==2.10.1
-pyparsing==3.2.3
-python-dateutil==2.9.0.post0
-python-dotenv==1.1.0
-python-json-logger==3.3.0
-python-multipart==0.0.20
-pytorch-lightning==2.5.1.post0
-pytz==2025.2
-PyYAML==6.0.2
-pyzmq==26.4.0
-referencing==0.36.2
-regex==2024.11.6
-requests==2.32.3
-requests-oauthlib==2.0.0
-rfc3339-validator==0.1.4
-rfc3986-validator==0.1.1
-rich==14.0.0
-rpds-py==0.25.1
-rsa==4.9.1
-s3transfer==0.13.0
-safetensors==0.5.3
-scikit-learn==1.3.2
-scipy==1.11.4
-Send2Trash==1.8.3
-sentence-transformers==5.0.0
-seqeval==1.2.2
-simple-term-menu==1.6.6
-six==1.17.0
-sniffio==1.3.1
-soupsieve==2.7
-stack-data==0.6.3
-starlette==0.46.2
-sympy==1.14.0
-tensorboard==2.15.1
-tensorboard-data-server==0.7.2
-terminado==0.18.1
-threadpoolctl==3.6.0
-tifffile==2025.5.10
-tinycss2==1.4.0
-tokenizers==0.21.1
-tomli==2.2.1
-torch==2.7.0+cu128
-torchmetrics==1.3.1
-torchvision==0.22.0+cu128
-tornado==6.5.1
-tqdm==4.67.1
-traitlets==5.14.3
-transformers==4.52.4
-triton==3.3.0
-types-python-dateutil==2.9.0.20250516
-typing-inspection==0.4.1
-typing_extensions==4.14.0
-tzdata==2025.2
-uri-template==1.3.0
-urllib3==2.4.0
-uvicorn==0.34.3
-uvloop==0.21.0
-watchfiles==1.0.5
-wcwidth==0.2.13
-webcolors==24.11.1
-webencodings==0.5.1
-websocket-client==1.8.0
-websockets==15.0.1
-Werkzeug==3.1.3
-wget==3.2
-widgetsnbextension==4.0.14
-xxhash==3.5.0
-yarg==0.1.9
-yarl==1.20.0

research/securebert2/vuln_classification/CodeVuln_eval.py

@@ -1,119 +0,0 @@
-# Copyright 2025 Cisco Systems, Inc. and its affiliates
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import argparse
-import torch
-from torch.utils.data import DataLoader
-from transformers import (
-    AutoTokenizer,
-    AutoModelForSequenceClassification,
-)
-from sklearn.metrics import (
-    accuracy_score,
-    precision_recall_fscore_support,
-    confusion_matrix,
-)
-
-from dataset import Eval_SentimentVulnerabilityDataset  # your dataset class
-from tqdm import tqdm
-
-
-# ---------- Collate ----------
-def cls_collate_fn(batch):
-    texts, labels = zip(*batch)
-    return list(texts), torch.tensor(labels, dtype=torch.long)
-
-
-# ---------- Evaluation ----------
-def evaluate(
-    ckpt_path: str,
-    batch_size: int = 32,
-    max_len: int = 1024,
-    device: torch.device = torch.device("cuda:0"),
-):
-    # 1. Dataset and dataloader
-    test_ds = Eval_SentimentVulnerabilityDataset()  # adjust if needed
-    test_dl = DataLoader(
-        test_ds,
-        batch_size=batch_size,
-        shuffle=False,
-        collate_fn=cls_collate_fn,
-    )
-
-    # 2. Tokenizer & model
-    tokenizer = AutoTokenizer.from_pretrained("answerdotai/ModernBERT-base")
-    model = AutoModelForSequenceClassification.from_pretrained(
-        "answerdotai/ModernBERT-base",
-        num_labels=2,
-        attn_implementation="sdpa",
-    )
-
-    # Load model directly
-
-    tokenizer = AutoTokenizer.from_pretrained("SynamicTechnologies/CYBERT")
-    model = AutoModelForSequenceClassification.from_pretrained("SynamicTechnologies/CYBERT")
-
-    
-    # 3. Load checkpoint
-    # ckpt = torch.load(ckpt_path, map_location="cpu")
-    # model.load_state_dict(ckpt["model_state_dict"], strict=True)
-    model.to(device)
-    model.eval()
-
-    # tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base")
-    # model = AutoModelForSequenceClassification.from_pretrained("mahdin70/codebert-devign-code-vulnerability-detector")
-
-    # Load model directly
-
-    model.eval() 
-    model.to(device)
-
-    # 4. Inference loop
-    all_preds, all_labels = [], []
-    with torch.no_grad():
-        for texts, labels in tqdm(test_dl):
-            enc = tokenizer(
-                texts,
-                padding="max_length",
-                truncation=True,
-                max_length=max_len,
-                return_tensors="pt",
-            ).to(device)
-
-            labels = labels.to(device)
-            logits = model(**enc).logits
-            preds = logits.argmax(dim=-1)
-
-            all_preds.append(preds.cpu())
-            all_labels.append(labels.cpu())
-
-    # 5. Metrics
-    y_pred = torch.cat(all_preds).numpy()
-    y_true = torch.cat(all_labels).numpy()
-
-    acc = accuracy_score(y_true, y_pred)
-    prec, rec, f1, _ = precision_recall_fscore_support(
-        y_true, y_pred, average="binary"
-    )
-    cm = confusion_matrix(y_true, y_pred)
-
-    print("======= Evaluation (single GPU) =======")
-    print(f"Accuracy : {acc:.4f}")
-    print(f"Precision: {prec:.4f}")
-    print(f"Recall   : {rec:.4f}")
-    print(f"F1-score : {f1:.4f}")
-    print("Confusion matrix (rows: true, cols: pred):")
-    print(cm)
-
-
-# ---------- CLI ----------
-if __name__ == "__main__":
-    # evaluate(
-    #     ckpt_path="sentiment_classif/checkpoint_epoch_6.pth", max_len = 1024
-    # )
-    # If running codebert-devign. If you get error RuntimeError: The expanded size of the tensor (1024) must match the existing size (514) at non-singleton dimension 1.  Target sizes: [32, 1024].  Tensor sizes: [1, 514]
-    # Then chnage max_len from 1024 to 512
-    evaluate(
-        ckpt_path="sentiment_classif/checkpoint_epoch_6.pth", max_len = 512
-    )

research/securebert2/vuln_classification/CodeVuln_train.py

@@ -1,124 +0,0 @@
-# Copyright 2025 Cisco Systems, Inc. and its affiliates
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from dataset import ModernBertDataset, ContrastiveLearningDataset, SentimentVulnerabilityDataset
-import torch
-import tqdm
-from transformers import AutoModelForMaskedLM, AutoTokenizer, get_scheduler, DataCollatorForLanguageModeling
-from torch.nn.parallel import DistributedDataParallel as DDP
-from torch.utils.data.distributed import DistributedSampler
-import random
-import os
-from lightning.fabric import Fabric
-from lightning.fabric.strategies import DDPStrategy
-from transformers import AutoModelForSequenceClassification
-torch.set_float32_matmul_precision('high')
-
-fabric = Fabric(accelerator="gpu", devices=4, strategy=DDPStrategy(find_unused_parameters=True))
-
-def cls_collate_fn(batch):
-    texts, labels = zip(*batch)
-    labels = torch.tensor(labels, dtype=torch.long)
-    return list(texts), labels
-
-def train_model(checkpoint_path):
-    fabric.launch()
-    # torch.distributed.init_process_group(backend="nccl", rank=rank, world_size=world_size)
-    # torch.cuda.set_device(rank)
-    # if fabric.is_global_zero:  # Only the main process writes to the log file
-    log_file = open("cont_training_log.txt", "w")
-    # Paths to your datasets
-    df = SentimentVulnerabilityDataset()
-    rng = random.Random()
-    # Hyperparameters
-    max_seq_length = 1024
-    batch_size_per_gpu = 8  # Adjust based on memory usage
-    print(f"Using per-GPU batch size: {batch_size_per_gpu}")
-    # print(f"Total effective batch size: {total_batch_size}")
-    num_epochs = 10
-    learning_rate = 1e-5
-    weight_decay = 0.01
-    mlm_prob = 0.15
-    # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    # device = torch.device(f"cuda:{rank}")
-    # Model init
-    tokenizer = AutoTokenizer.from_pretrained(
-    "answerdotai/ModernBERT-base",
-    )
-    model = AutoModelForSequenceClassification.from_pretrained(
-        "answerdotai/ModernBERT-base",       # or SecureBERT checkpoint if you have one
-        num_labels=2,                        # binary (vulnerable / non-vulnerable)
-        attn_implementation="sdpa"
-    )
-    # model = DDP(model, device_ids=[rank], output_device=rank)
-    # Use DistributedSampler for the DataLoader
-    # sampler = DistributedSampler(df, num_replicas=world_size, rank=rank, shuffle=True)
-    sampler = DistributedSampler(df, num_replicas=fabric.world_size, rank=fabric.global_rank, shuffle=True)
-    dataloader = torch.utils.data.DataLoader(
-        df, batch_size=batch_size_per_gpu, sampler=sampler, collate_fn = cls_collate_fn
-    )
-    num_training_steps = len(dataloader) * num_epochs
-    dataloader = fabric.setup_dataloaders(dataloader)
-    # dataloader = torch.utils.data.DataLoader(df, batch_size=batch_size, shuffle=True)
-    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
-    model, optimizer = fabric.setup(model, optimizer)
-    # 2 to 5 thousands warmup steeps if checkpoint resume
-    print("Has no warmup steps")
-    lr_scheduler = get_scheduler("linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps)
-    mask_id = tokenizer.mask_token_id
-    vocab_size = tokenizer.vocab_size
-    model.train()
-    # Load base model
-    checkpoint = torch.load(checkpoint_path, map_location=fabric.device)
-    model.load_state_dict(checkpoint["model_state_dict"], strict = False)
-    for epoch in range(num_epochs):
-        epoch_loss = 0
-        sampler.set_epoch(epoch)
-        skipped = 0
-        for batch in tqdm.tqdm(dataloader, disable=not fabric.is_global_zero):
-            optimizer.zero_grad()
-            text, labels = batch
-            try:
-                enc = tokenizer(
-                    text,
-                    padding="max_length",
-                    truncation=True,
-                    max_length=max_seq_length,
-                    return_tensors="pt", 
-                    add_special_tokens=True
-                )
-            except:
-                skipped += 1
-                continue
-            labels = labels.to(fabric.device)
-            outputs = model(**enc, labels=labels)
-            loss = outputs.loss
-            fabric.backward(loss)
-            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Gradient clipping
-            optimizer.step()
-            lr_scheduler.step()
-
-            epoch_loss += loss.item()
-        avg_epoch_loss = epoch_loss / len(dataloader)
-        if fabric.is_global_zero:
-            log_message = f"Epoch Number: {epoch + 1} | Average Epoch Loss: {avg_epoch_loss} | Skipped: {skipped}\n"
-            print(log_message.strip())  # Print to console
-            log_file.write(log_message)  # Write to log file
-
-            # save_path = f"model_epoch_{epoch + 1}.pth"
-            checkpoint_path = f"checkpoint_epoch_{epoch + 1}.pth"
-            torch.save({
-                "epoch": epoch + 1,
-                "model_state_dict": model.state_dict(),
-                "optimizer_state_dict": optimizer.state_dict(),
-                "lr_scheduler_state_dict": lr_scheduler.state_dict(),
-                "loss": avg_epoch_loss,
-            }, checkpoint_path)
-            # torch.save(model.state_dict(), save_path)  # Save the model weights
-            print(f"Model saved to {checkpoint_path}")
-    log_file.close()
-    # torch.distributed.destroy_process_group()
-
-if __name__ == "__main__":
-    train_model(checkpoint_path = "final_base_modernsecurebert_pths/checkpoint_epoch_20.pth")

research/securebert2/vuln_classification/init.txt

@@ -1 +0,0 @@
-