Update README.md

README.md

@@ -11,17 +11,40 @@ This model is finetuned on mutiple datasets related to ABC notation (mostly Iris
 
 
 
-## CLI demo
+## CLI demo for 4-bit quantize 
 ```python
-from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig, BitsAndBytesConfig
 import torch
 import torchaudio
 import re
-from string import Template
-prompt_template = Template("Human: ${inst} </s> Assistant: ")
 
-tokenizer = AutoTokenizer.from_pretrained("Seeker38/gemma-2-9b-it-abc-notation", trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("Seeker38/gemma-2-9b-it-abc-notation", torch_dtype=torch.float16, device_map="cuda", resume_download=True).eval()
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4"
+)
+
+# Alpaca prompt template
+alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+### Instruction:
+{}
+### Input:
+{}
+### Response:
+{}"""
+
+
+tokenizer = AutoTokenizer.from_pretrained("Seeker38/gemma-2-9b-it-abc-notation")
+
+# model 4-bit quant
+model = AutoModelForCausalLM.from_pretrained(
+    "Seeker38/gemma-2-9b-it-abc-notation", 
+    quantization_config=quantization_config,
+    device_map="auto",
+    resume_download=True
+).eval()
+
 
 generation_config = GenerationConfig(
     temperature=0.2,
@@ -34,30 +57,62 @@ generation_config = GenerationConfig(
     max_new_tokens=1536
 )
 
-instruction = """Develop a musical piece using the given chord progression.
-'Dm', 'C', 'Dm', 'Dm', 'C', 'Dm', 'C', 'Dm'
-"""
-
-prompt = prompt_template.safe_substitute({"inst": instruction})
-inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
-response = model.generate(
-        input_ids=inputs["input_ids"].to(model.device),
-        attention_mask=inputs['attention_mask'].to(model.device),
-        eos_token_id=tokenizer.eos_token_id,
-        generation_config=generation_config,
-        )
-response = tokenizer.decode(response[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
-print(response)
+instruction = """Create a musical composition using the given motif and adhering to the specified musical form represented by alphabet characters.
+X:1
+L:1/8
+Q:3/8=90
+M:6/8
+K:A
+['e cAA ABc dBB Tf2 e fdd', 'e fga']"""
+# input_context = "'A', 'D', 'E7', 'A', 'E/G#', 'A', 'Bm', 'A7/C#', 'D', 'E7', 'A', 'A', 'D', 'A', 'A', 'D', 'A', 'A', 'D', 'A', 'D', 'A/D#', 'E', 'A', 'D', 'A', 'A', 'D', 'A', 'E7'"
+input_context = ""
+
+prompt = alpaca_prompt.format(
+    instruction,    # instruction
+    input_context,  # input
+    "",            # output - leave this blank for generation!
+)
+
+# Tokenize input
+inputs = tokenizer([prompt], return_tensors="pt").to("cuda")
+
+# Generate response with specified parameters
+with torch.no_grad():
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=2048,
+        temperature=0.2,
+        top_p=0.9,
+        top_k=40,
+        use_cache=True,
+        do_sample=True,
+        repetition_penalty=1.1,
+        pad_token_id=tokenizer.eos_token_id
+    )
+
+result = tokenizer.batch_decode(outputs, skip_special_tokens=True)
+print("Generated Response:")
+print(result[0])
 
 # to render abc notation, you need to install symusic
 # pip install symusic
-from symusic import Score, Synthesizer, BuiltInSF3, dump_wav
+import re
+from symusic import Score, Synthesizer
+
+abc_notation = re.search(r'### Response:\s*(.*)', result[0], re.DOTALL).group(1).strip()
 
-abc_pattern = r'(X:\d+\n(?:[^\n]*\n)+)'
-abc_notation = re.findall(abc_pattern, response+'\n')[0]
 s = Score.from_abc(abc_notation)
 audio = Synthesizer().render(s, stereo=True)
 torchaudio.save('cm_music_piece.wav', torch.FloatTensor(audio), 44100)
+
+from IPython.display import Audio, display
+
+from pydub import AudioSegment
+wav_link = "cm_music_piece.wav"
+mp3_file = AudioSegment.from_wav(wav_link).export("cm_music_piece.mp3", format="mp3")
+
+display(Audio(wav_link))
+display(Audio('cm_music_piece.mp3'))
 ```