Upload 4 files

converter.py

@@ -0,0 +1,57 @@
+import os
+import argparse
+import torch
+from safetensors.torch import save_file
+
+def extract_state_dict(checkpoint):
+    """
+    Extracts the tensor dictionary from common .pth formats.
+    """
+    if isinstance(checkpoint, dict):
+        for key in ["state_dict", "model", "model_state_dict", "module"]:
+            if key in checkpoint and isinstance(checkpoint[key], dict):
+                return checkpoint[key]
+    return checkpoint
+
+def convert_pth_to_safetensors(input_path, output_path=None):
+    print(f"🔍 Loading checkpoint from: {input_path}")
+
+    try:
+        checkpoint = torch.load(input_path, map_location="cpu", weights_only=True)
+    except Exception as e:
+        print(f"❌ Error loading .pth file: {e}")
+        return
+
+    state_dict = extract_state_dict(checkpoint)
+
+    if not isinstance(state_dict, dict):
+        print("❌ Invalid checkpoint: not a dictionary.")
+        return
+
+    tensor_dict = {k: v for k, v in state_dict.items() if isinstance(v, torch.Tensor)}
+
+    if not tensor_dict:
+        print("❌ No tensor values found to convert.")
+        return
+
+    # Optionally add "model." prefix to HuggingFace-compatible keys
+    if not all(k.startswith("model.") for k in tensor_dict):
+        tensor_dict = {f"model.{k}": v for k, v in tensor_dict.items()}
+
+    if output_path is None:
+        output_path = os.path.splitext(input_path)[0] + ".safetensors"
+
+    try:
+        print(f"💾 Saving to: {output_path}")
+        save_file(tensor_dict, output_path)
+        print("✅ Conversion to .safetensors successful!")
+    except Exception as e:
+        print(f"❌ Saving failed: {e}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Convert .pth to .safetensors")
+    parser.add_argument("input", help="Path to input .pth file")
+    parser.add_argument("--output", help="Path to output .safetensors file (optional)")
+
+    args = parser.parse_args()
+    convert_pth_to_safetensors(args.input, args.output)

enhanced_transformer_model_500M_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecf29d34e90a5bc9d26cd19401753a24b8dda6d9afc726681363023c26b6acb2
+size 112823067

enhanced_transformer_model_500M_final.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f80dcb5311ccae55f66030c0b3285c37bae862d3cd48657a8dc33f04e6e3435c
+size 112807876

textgen.py

@@ -0,0 +1,35 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+def load_model(model_path):
+    # Load the model
+    model = AutoModelForCausalLM.from_pretrained(model_path)
+    # Load the tokenizer (assuming it's saved alongside the model)
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+    return model, tokenizer
+
+def generate_text(model, tokenizer, prompt, max_length=100, temperature=0.7):
+    # Encode the prompt
+    input_ids = tokenizer.encode(prompt, return_tensors="pt")
+
+    # Generate text
+    with torch.no_grad():
+        output = model.generate(
+            input_ids,
+            max_length=max_length,
+            temperature=temperature,
+            num_return_sequences=1,
+            pad_token_id=tokenizer.eos_token_id
+        )
+
+    # Decode and return the generated text
+    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
+    return generated_text
+
+if __name__ == "__main__":
+    model_path = "enhanced_transformer_model_500M_final.pth"
+    model, tokenizer = load_model(model_path)
+
+    prompt = "Once upon a time"
+    generated_text = generate_text(model, tokenizer, prompt)
+    print(generated_text)