\u60a8\u53ef\u4ee5\u901a\u8fc7\u4f7f\u7528 Hugging Face \u7684\u00a0 \u4ee5\u4e0b\u6570\u636e\u5df2\u91cd\u65b0\u8bc4\u4f30\u5e76\u8ba1\u7b97\u4e3a\u6bcf\u6b21\u6d4b\u8bd5\u7684\u5e73\u5747\u503c\u3002<\/p>\n\n\n\nLlama-3.1-8B-Fusion-9010<\/code>\u662f\u4e00\u4e2a\u6df7\u5408\u6a21\u578b\uff0c\u5b83\u7ed3\u5408\u4e86\u4e24\u4e2a\u5f3a\u5927\u7684\u57fa\u4e8e Llama \u7684\u6a21\u578b\u7684\u4f18\u52bf\uff1aarcee-ai\/Llama-3.1-SuperNova-Lite<\/a> \u548c mlabonne\/Meta-Llama-3.1-8B-Instruct-abliterated<\/a>\u3002\u6743\u91cd\u4ee5 9\uff1a1 \u7684\u6bd4\u4f8b\u6df7\u5408\uff0c\u5176\u4e2d 90% \u7684\u6743\u91cd\u6765\u81ea SuperNova-Lite\uff0c10% \u6765\u81ea\u88ab\u5220\u51cf\u7684 Meta-Llama-3.1-8B-Instruct \u6a21\u578b\u3002\u867d\u7136\u662f\u7b80\u5355\u7684\u6df7\u5408\uff0c\u4f46\u6a21\u578b\u662f\u53ef\u7528\u7684\uff0c\u5e76\u4e14\u6ca1\u6709\u51fa\u73b0\u4e71\u7801<\/strong>\u3002 \u8fd9\u662f\u4e00\u4e2a\u5b9e\u9a8c\u3002\u6211\u5206\u522b\u6d4b\u8bd5 9\uff1a1<\/a>\u30018\uff1a2<\/a>\u30017\uff1a3<\/a>\u30016\uff1a4<\/a> \u548c 5\uff1a5<\/a> \u7684\u6bd4\u7387\uff0c\u4ee5\u67e5\u770b\u5b83\u4eec\u5bf9\u6a21\u578b\u7684\u5f71\u54cd\u6709\u591a\u5927\u3002 \u6240\u6709\u6a21\u578b\u8bc4\u4f30\u62a5\u544a\u5c06\u5728\u968f\u540e\u63d0\u4f9b\u3002<\/p>\n\n\n\n<\/a>\u578b\u53f7\u8be6\u7ec6\u4fe1\u606f<\/h2>\n\n\n\n
\n
\n
<\/a>\u4e3b\u8981\u7279\u70b9<\/h2>\n\n\n\n
\n
<\/a>\u7528\u6cd5<\/h2>\n\n\n\n
transformers<\/code>\u00a0\u5e93\u52a0\u8f7d\u5b83\u6765\u5728\u60a8\u7684\u5e94\u7528\u7a0b\u5e8f\u4e2d\u4f7f\u7528\u8fd9\u4e2a\u6df7\u5408\u6a21\u578b\uff1a<\/p>\n\n\n\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer\nimport time\n\nmixed_model_name = \"huihui-ai\/Llama-3.1-8B-Fusion-9010\"\n\n# Check if CUDA is available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load model and tokenizer\nmixed_model = AutoModelForCausalLM.from_pretrained(mixed_model_name, device_map=device, torch_dtype=torch.bfloat16)\ntokenizer = AutoTokenizer.from_pretrained(mixed_model_name)\n\n# Ensure the tokenizer has pad_token_id set\ntokenizer.pad_token_id = tokenizer.eos_token_id\n\n# Input loop\nprint(\"Start inputting text for inference (type 'exit' to quit)\")\nwhile True:\n prompt = input(\"Enter your prompt: \")\n if prompt.lower() == \"exit\":\n print(\"Exiting inference loop.\")\n break\n\n # Inference phase: Generate text using the modified model\n chat = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": prompt}\n ]\n\n # Prepare input data\n input_ids = tokenizer.apply_chat_template(\n chat, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\"\n ).to(device)\n\n # Use TextStreamer for streaming output\n streamer = TextStreamer(tokenizer, skip_special_tokens=True)\n\n # Record the start time\n start_time = time.time()\n\n # Generate text and stream output character by character\n outputs = mixed_model.generate(\n input_ids,\n max_new_tokens=8192,\n do_sample=True,\n temperature=0.6,\n top_p=0.9,\n streamer=streamer # Enable streaming output\n )\n\n # Record the end time\n end_time = time.time()\n\n # Calculate the number of generated tokens\n generated_tokens = outputs[0][input_ids.shape[-1]:].shape[0]\n\n # Calculate the total time taken\n total_time = end_time - start_time\n\n # Calculate tokens generated per second\n tokens_per_second = generated_tokens \/ total_time\n\n print(f\"\\nGenerated {generated_tokens} tokens in total, took {total_time:.2f} seconds, generating {tokens_per_second:.2f} tokens per second.\")\n<\/pre><\/div>\n\n\n\n<\/a>\u8bc4\u4f30<\/h2>\n\n\n\n
Benchmark<\/th> SuperNova-Lite<\/th> Meta-Llama-3.1-8B-Instruct-abliterated<\/th> Llama-3.1-8B-Fusion-9010<\/th> Llama-3.1-8B-Fusion-8020<\/th> Llama-3.1-8B-Fusion-7030<\/th> Llama-3.1-8B-Fusion-6040<\/th> Llama-3.1-8B-Fusion-5050<\/th><\/tr><\/thead> IF_Eval<\/td> 82.09<\/td> 76.29<\/td> 82.44<\/td> 82.93<\/td> 83.10<\/strong><\/td> 82.94<\/td> 82.03<\/td><\/tr> MMLU Pro<\/td> 35.87<\/strong><\/td> 33.1<\/td> 35.65<\/td> 35.32<\/td> 34.91<\/td> 34.5<\/td> 33.96<\/td><\/tr> TruthfulQA<\/td> 64.35<\/strong><\/td> 53.25<\/td> 62.67<\/td> 61.04<\/td> 59.09<\/td> 57.8<\/td> 56.75<\/td><\/tr> BBH<\/td> 49.48<\/strong><\/td> 44.87<\/td> 48.86<\/td> 48.47<\/td> 48.30<\/td> 48.19<\/td> 47.93<\/td><\/tr> GPQA<\/td> 31.98<\/td> 29.50<\/td> 32.25<\/td> 32.38<\/td> 32.61<\/strong><\/td> 31.14<\/td> 30.6<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n