| import argparse | |
| import sys | |
| sys.path.append('../') | |
| from time import perf_counter | |
| from lyra_llama import lyraLlama | |
| def get_args(): | |
| parser = argparse.ArgumentParser(description="Faster ChatGLM6B Demo") | |
| parser.add_argument('--model-path', type=str, required=True, | |
| help='Model Path, include config.ini and tokenizer files') | |
| parser.add_argument('--tokenizer-path', type=str, default=None) | |
| parser.add_argument( | |
| '--data-type', type=str, metavar='TYPE', default='fp16', | |
| choices=[None, 'fp32', 'fp16', 'bf16', 'int8'], | |
| help='The data type to inference. If None, the data type follows the ' | |
| 'checkpoint data type.') | |
| parser.add_argument( | |
| '--memopt-mode', type=int, default=0, choices=[0, 1], | |
| help='Use MEMOPT mode to increase speed and reduce VRAM usage.' | |
| ' 0: FP16 mode' | |
| ' 1: Use MEMOPT mode') | |
| parser.add_argument( | |
| '--quant-type', type=str, metavar='TYPE', default='int8', | |
| choices=['int4', 'int8'], | |
| help='The data type of quantization. Only used in MEMOPT.') | |
| parser.add_argument( | |
| '--kvqparams-fpath', type=str, required=False, default="", | |
| help='File path of kv quantized params.') | |
| parser.add_argument("--prompt", type=str, required=False) | |
| parser.add_argument("--max-output-length", type=int, default=512) | |
| parser.add_argument("--warmups", type=int, default=10) | |
| parser.add_argument("--avgnums", type=int, default=10) | |
| args = parser.parse_args() | |
| print('\n=================== Arguments ===================') | |
| for k, v in vars(args).items(): | |
| print(f' - {k.ljust(25, ".")}: {v}') | |
| print('=================================================') | |
| return args | |
| def main(): | |
| args = get_args() | |
| model = lyraLlama(args.model_path, args.tokenizer_path, args.data_type, args.memopt_mode, args.quant_type, args.kvqparams_fpath) | |
| # args.prompt = '''<context>/*\n * Implement the \"Falling Rocks\" game in the text console. \n * A small dwarf stays at the bottom of the screen and can \n * move left and right (by the arrows keys). A number of rocks \n * of different sizes and forms constantly fall down and you \n * need to avoid a crash.\n * Rocks are the symbols ^, @, *, &, +, %, $, #, !, ., ;, - distributed \n * with appropriate density. The dwarf is (O). \n * Ensure a constant game speed by Thread.Sleep(150).\n * Implement collision detection and scoring system.\n*/\n\nusing System;\nusing System.Threading;\nusing System.Collections.Generic;\nusing System.Threading.Tasks;\n\nclass FallingRocks\n{\n struct Position\n {\n public int X, Y;\n public string symbol;\n public ConsoleColor color;\n\n public Position(int x, int y, string symbol, ConsoleColor color)\n {\n this.X = x;\n this.Y = y;\n this.symbol = symbol;\n this.color = color;\n }\n }\n\n static void Main()\n {\n Thread oThread = new Thread(new ThreadStart(Mainn));\n Thread aThread = new Thread(new ThreadStart(Clr));\n \n aThread.Start();\n oThread.Start();\n oThread.Join();\n aThread.Join();\n }\n\n static void Clr()\n {\n while (true)\n {\n Thread.Sleep(10);\n Console.Clear();\n }\n }\n static void Mainn()\n {\n //Random generator for rocks color, position and symbol\n Random randomGenerator = new Random();\n \n //Sleep time for the game loop\n double sleepTime = 150;\n //Console settings\n Console.CursorVisible = false;\n Console.BufferHeight = Console.WindowHeight;\n \n //number of rocks in the Array rocks\n int rocksCount = 0;\n\n //array with the symbols of the rocks\n string[] symbols = new string[] { \"^\", \"@\", \"*\", \"&\", \"+\", \"%\", \"$\", \"#\", \"!\", \".\", \";\" };\n \n //array with colors for the rocks\n ConsoleColor[] colors = new ConsoleColor[] {ConsoleColor.Yellow, ConsoleColor.White, ConsoleColor.Gray};\n \n //array with rocks\n Position[] rocks = new Position[200];\n \n //position for the dwarf\n Position dwarf = new Position(10, Console.WindowHeight - 1,\"(0)\",ConsoleColor.Red);\n \n //bool variable to say when the game loop to be over\n bool gameLoop = true;\n\n //variable keeping the score\n ulong score = 0;\n\n //the game loop\n while (gameLoop)\n {\n //score is growing as the cycle runs\n score++;\n\n //setting the Y component for all the rocks in the array to grow with 2\n for (int i = 0; i <= rocks.Length - 1; i++)\n {\n rocks[i].Y = rocks[i].Y + 2;\n }\n\n //generating rocks\n for (int x = 0; x <= randomGenerator.Next(2, 4); x++)\n {\n rocks[rocksCount] = new Position(randomGenerator.Next(x * 15, x * 15 + 20), 0\n , symbols[randomGenerator.Next(0, symbols.Length - 1)]\n , colors[randomGenerator.Next(0, colors.Length - 1)]);\n if (rocksCount >= 199) rocksCount = 0;\n rocksCount++;\n }\n\n //printing the rocks and other stuff\n foreach (var item in rocks)\n {\n foreach (var rock in rocks)\n {\n //checking for colision\n if ((rock.X >= dwarf.X) && (rock.X <= (dwarf.X + 2)) && (rock.Y == dwarf.Y))\n {\n gameLoop = false;\n break;\n }\n } \n\n //printing the rocks\n if (item.Y < Console.WindowHeight)\n { \n Console.SetCursorPosition(item.X, item.Y);\n Console.ForegroundColor = item.color;\n Console.Write(item.symbol);\n }\n\n //checking for key pressed\n if (Console.KeyAvailable)\n {\n ConsoleKeyInfo pressedKey = Console.ReadKey();\n if (pressedKey.Key == ConsoleKey.RightArrow)\n {\n if(dwarf.X < Console.WindowWidth - 20)\n {\n //removing the old positions of the dwarf and increasing his X value\n Console.SetCursorPosition(dwarf.X, dwarf.Y);\n Console.Write(\" \");\n dwarf.X++;\n }\n }\n if (pressedKey.Key == ConsoleKey.LeftArrow) \n {\n if(dwarf.X >= 1)\n {\n //removing the old positions of the dwarf and decreasing his X value\n Console.SetCursorPosition(dwarf.X, dwarf.Y);\n Console.Write(\" \");\n dwarf.X--;\n }\n }\n }\n }\n \n //printing the dwarf\n Console.SetCursorPosition(dwarf.X, dwarf.Y);\n Console.ForegroundColor = dwarf.color;\n Console.Write(dwarf.symbol); \n \n //sleeping the loop for sometime\n //Thread.Sleep((int)sleepTime);\n\n //reducing the sleep time of the loop\n sleepTime -= 0.5;\n\n \n //removing the rocks \n //foreach (var item in rocks)\n //{\n // if (item.Y < Console.WindowHeight)\n // {\n // Console.SetCursorPosition(item.X, item.Y);\n // Console.Write(\" \");\n // }\n //} \n }\n //Printing the score after the game is over\n Console.Clear();\n Console.WriteLine(\"Game over! Your score is: \" + score);\n\n }\n}\n</context>\n\n这个\"Falling Rocks\"游戏是如何工作的呢？可以详细解释一下代码的运作机制吗？ \n\n\n\n''' | |
| prompt_template = "Human: {}\n\nAssistant:" # xverse | |
| # prompt_template = "<human>:{}\n<bot>:" # llama-ziya 13b | |
| prompt = prompt_template.format(args.prompt) | |
| test_batch_size = [1, 8, 16, 32, 64] # 8, 16, 32, 64 | |
| print("test_batch_size: ", test_batch_size) | |
| for i, bs in enumerate(test_batch_size): | |
| prompts = [prompt, ] * bs | |
| # warmup gpu | |
| for _ in range(args.warmups): | |
| output_texts = model.generate( | |
| prompts, output_length=args.max_output_length, | |
| top_k=30, top_p=0.85, temperature=1.0, repetition_penalty=1.0, do_sample=False) | |
| start = perf_counter() | |
| for _ in range(args.avgnums): | |
| output_texts = model.generate( | |
| prompts, output_length=args.max_output_length, | |
| top_k=30, top_p=0.85, temperature=1.0, repetition_penalty=1.0, do_sample=False) | |
| end = perf_counter() | |
| cost = (end - start) / args.avgnums | |
| input_output_texts = [prompt + ' ' + gtext for prompt, | |
| gtext in zip(prompts, output_texts)] | |
| tokens = 0 | |
| input_tokens = len(model.tokenizer.encode(prompt)) | |
| words = 0 | |
| for text in input_output_texts: | |
| tokens += len(model.tokenizer.encode(text)) | |
| words += len(text) | |
| avg_output_tokens = tokens / len(input_output_texts) - input_tokens | |
| print( | |
| f"\nFaster-Dtype: {args.data_type}, Batch Size: {bs}, All tokens: {tokens}. Input tokens: {input_tokens}. Output tokens: {avg_output_tokens} Cost: {cost} seconds. Speed: {tokens/cost} tokens/s." | |
| ) | |
| print( | |
| f"Faster-Dtype: {args.data_type}, Batch Size: {bs}, All generated words: {words}. Cost: {cost} seconds. Speed: {words/cost} words/s." | |
| ) | |
| if i == 0: | |
| for k in range(bs): | |
| print( | |
| f"The {k} Sample, \n\t\tInputs: {prompts[k]}. \n\t\tOutputs: {output_texts[k].lstrip()}") | |
| if k > 2: | |
| break | |
| if __name__ == "__main__": | |
| main() | |