{"id":2458,"date":"2024-03-11T14:26:13","date_gmt":"2024-03-11T06:26:13","guid":{"rendered":"https:\/\/www.aqwu.net\/wp\/?p=2458"},"modified":"2024-04-28T20:02:43","modified_gmt":"2024-04-28T12:02:43","slug":"%e4%ba%86%e8%a7%a3-llama-2-%e6%a8%a1%e5%9e%8b%e7%bb%93%e6%9e%845","status":"publish","type":"post","link":"https:\/\/www.aqwu.net\/wp\/?p=2458","title":{"rendered":"\u4e86\u89e3 LLaMA-2 \u6a21\u578b\u7ed3\u6784(5)"},"content":{"rendered":"\n<h2 class=\"wp-block-heading\"><strong>9. \u8f6c\u6362 tokenizer.model \u5e76\u4fdd\u5b58<\/strong><\/h2>\n\n\n\n<p>\u524d\u9762\u7684\u7ae0\u8282\u8f6c\u6362\u6a21\u578b\u7684\u6240\u6709\u6743\u91cd\u540e\uff0c\u8fd8\u9700\u8981\u8f6c\u6362 tokenizer.model \u4e3a\u81ea\u5df1\u9700\u8981\u7684\u683c\u5f0f\u3002<\/p>\n\n\n\n<p>\u628a meta-llama\/Llama-2-7b-chat-hf\/ \u76ee\u5f55\u4e0b\u7684 tokenizer.model \u62f7\u8d1d\u5230 newsrc \u76ee\u5f55\u4e0b\u3002<\/p>\n\n\n\n<p>\u53c2\u7167 https:\/\/github.com\/karpathy\/llama2.c \u9879\u76ee\u4e0b\u7684 tokenizer.py \u6587\u4ef6\uff0c\u547d\u540d\u4e3a test09.py\uff0c\u6587\u4ef6\u4fdd\u5b58\u5230 newsrc \u76ee\u5f55\u4e0b\uff1a<\/p>\n\n\n\n<div class=\"wp-block-urvanov-syntax-highlighter-code-block\"><pre class=\"lang:python decode:true \">import os\nimport struct\nimport argparse\nfrom typing import List\n\nfrom sentencepiece import SentencePieceProcessor\n\nclass Tokenizer:\n    def __init__(self, tokenizer_model=None):\n        model_path = tokenizer_model\n        assert os.path.isfile(model_path), model_path\n        self.sp_model = SentencePieceProcessor(model_file=model_path)\n        self.model_path = model_path\n\n        # BOS \/ EOS token IDs\n        self.n_words: int = self.sp_model.vocab_size()\n        self.bos_id: int = self.sp_model.bos_id()\n        self.eos_id: int = self.sp_model.eos_id()\n        self.pad_id: int = self.sp_model.pad_id()\n        #print(f\"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}\")\n        assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()\n\n    def encode(self, s: str, bos: bool, eos: bool) -&gt; List[int]:\n        assert type(s) is str\n        t = self.sp_model.encode(s)\n        if bos:\n            t = [self.bos_id] + t\n        if eos:\n            t = t + [self.eos_id]\n        return t\n\n    def decode(self, t: List[int]) -&gt; str:\n        return self.sp_model.decode(t)\n\n    def export(self):\n\n        # get all the tokens (postprocessed) and their scores as floats\n        tokens, scores = [], []\n        for i in range(self.n_words):\n\n            # decode the token and light postprocessing\n            t = self.sp_model.id_to_piece(i)\n            s = self.sp_model.get_score(i)\n            if i == self.bos_id:\n                t = '\\n&lt;s&gt;\\n'\n            elif i == self.eos_id:\n                t = '\\n&lt;\/s&gt;\\n'\n            t = t.replace('\u2581', ' ') # sentencepiece uses this character as whitespace\n            b = t.encode('utf-8') # bytes of this token, utf-8 encoded\n\n            tokens.append(b)\n            scores.append(s)\n\n        # record the max token length\n        max_token_length = max(len(t) for t in tokens)\n\n        # write to a binary file\n        # the tokenizer.bin file is the same as .model file, but .bin\n        tokenizer_bin = self.model_path.replace('.model', '.bin')\n        with open(tokenizer_bin, 'wb') as f:\n            f.write(struct.pack(\"I\", max_token_length))\n            for bytes, score in zip(tokens, scores):\n                f.write(struct.pack(\"fI\", score, len(bytes)))\n                f.write(bytes)\n\nt = Tokenizer(\"newsrc\/tokenizer.model\")\nt.export()\n\n<\/pre><\/div>\n\n\n\n<p>\u8fd0\u884c test09.py, \u67e5\u770bnewsrc \u7684\u6587\u4ef6\u76ee\u5f55<\/p>\n\n\n\n<div class=\"wp-block-urvanov-syntax-highlighter-code-block\"><pre class=\"lang:python decode:true \">ls -l newsrc\/tokenizer.*\n-rwxrwxrwx 1 tony tony 433869 Mar 11 14:24 newsrc\/tokenizer.bin\n-rwxrwxrwx 1 tony tony 499723 Mar 10 23:51 newsrc\/tokenizer.model<\/pre><\/div>\n\n\n\n<h2 class=\"wp-block-heading\"><strong>10. \u67e5\u770b tokenizer.bin<\/strong><\/h2>\n\n\n\n<p>\u4e0b\u9762\u7ed9\u51fa\u7684\u4f8b\u5b50\u5f00\u59cb\u7684\u90fd\u662fC\/C++\u4ee3\u7801\uff0c\u8fd9\u6837\u66f4\u597d\u7406\u89e3\u6587\u4ef6\u91cc\u9762\u7684\u5185\u5bb9<\/p>\n\n\n\n<p>\u53c2\u7167 https:\/\/github.com\/karpathy\/llama2.c \u9879\u76ee\u4e0b\u7684 run.c \u6587\u4ef6\uff0c\u547d\u540d\u4e3a test01.c\uff0c\u6587\u4ef6\u4fdd\u5b58\u5230 newsrc \u76ee\u5f55\u4e0b\uff1a<\/p>\n\n\n\n<div class=\"wp-block-urvanov-syntax-highlighter-code-block\"><pre class=\"lang:c decode:true \" >#include &lt;stdio.h&gt;\n#include &lt;stdlib.h&gt;\n\/\/ The Byte Pair Encoding (BPE) Tokenizer that translates strings &lt;-&gt; tokens\n\ntypedef struct {\n    \/\/ \u4e00\u4e2a\u5b57\u7b26\u6307\u9488\uff0c\u6307\u5411\u4e0eID\u5173\u8054\u7684\u5b57\u7b26\u4e32\u3002\n    char *str;\n    \/\/ \u4e00\u4e2a\u6574\u6570\uff0c\u8868\u793a\u4e0e\u5b57\u7b26\u4e32\u5173\u8054\u7684ID\u3002\n    int id;\n} TokenIndex;\n\ntypedef struct {\n    \/\/ \u4e00\u4e2a\u6307\u9488\u6570\u7ec4\uff0c\u5b58\u50a8\u8bcd\u6c47\u8868\u4e2d\u6bcf\u4e2a\u5355\u8bcd\u7684\u5b57\u7b26\u4e32\u8868\u793a\u3002\n    \/\/ \u4f8b\u5982\uff0c\u5982\u679c vocab[0] \u662f \"apple\"\uff0c\u90a3\u4e48 vocab[0][0] \u5c31\u662f\u5b57\u7b26 'a'\u3002\t\n    char** vocab;\n    \/\/ \u4e00\u4e2a\u6d6e\u70b9\u6570\u6570\u7ec4\uff0c\u53ef\u80fd\u5b58\u50a8\u4e0e\u8bcd\u6c47\u8868\u4e2d\u7684\u6bcf\u4e2a\u5355\u8bcd\u76f8\u5173\u8054\u7684\u5206\u6570\u6216\u6743\u91cd\u3002\n    float* vocab_scores;\n    \/\/ \u7ed3\u6784\u4f53\uff0c\u5b58\u50a8\u4e00\u4e2a\u5b57\u7b26\u4e32\u548c\u4e0e\u5176\u76f8\u5173\u8054\u7684\u6574\u6570ID\n    TokenIndex *sorted_vocab;\n    \/\/ \u4e00\u4e2a\u6574\u6570\uff0c\u8868\u793a\u8bcd\u6c47\u8868\u7684\u5927\u5c0f\uff0c\u5373vocab\u548cvocab_scores\u6570\u7ec4\u4e2d\u7684\u5143\u7d20\u6570\u91cf\u3002\n    int vocab_size;\n    \/\/ \u4e00\u4e2a\u65e0\u7b26\u53f7\u6574\u6570\uff0c\u8868\u793a\u8bcd\u6c47\u8868\u4e2d\u7684\u6700\u5927token\u957f\u5ea6\u3002\n    unsigned int max_token_length;\n    \/\/ \u4e00\u4e2a\u65e0\u7b26\u53f7\u5b57\u7b26\u6570\u7ec4\uff0c\u5b58\u50a8\u6240\u6709\u5355\u5b57\u8282\u5b57\u7b26\u4e32\n    \/\/ \u8fd9\u4e2a\u6570\u7ec4\u7684\u5927\u5c0f\u88ab\u8bbe\u5b9a\u4e3a512\uff0c\u53ef\u80fd\u662f\u4e3a\u4e86\u5b58\u50a8ASCII\u5b57\u7b26\u8868\u4e2d\u7684\u6240\u6709\u53ef\u80fd\u7684\u5355\u5b57\u8282\u5b57\u7b26\u4e32\u3002\n    unsigned char byte_pieces[512]; \/\/ stores all single-byte strings\n} Tokenizer;\n\nvoid build_tokenizer(Tokenizer* t, char* tokenizer_path, int vocab_size) {\n    \/\/ i should have written the vocab_size into the tokenizer file... sigh\n    \/\/ \u8bbe\u7f6e\u4e86Tokenizer\u7ed3\u6784\u4f53\u4e2dvocab_size\u7684\u503c\u3002\n    t-&gt;vocab_size = vocab_size;\n    \/\/ malloc space to hold the scores and the strings\n    \/\/ \u4e3avocab\uff0cvocab_scores\u6570\u7ec4\u5206\u914d\u4e86\u5185\u5b58\uff0c\u800csorted_vocab\u88ab\u521d\u59cb\u5316\u4e3aNULL\uff0c\u8868\u793a\u5b83\u5c06\u5728\u540e\u7eed\u88ab\u201c\u61d2\u60f0\u5730\u201d\u521d\u59cb\u5316\u3002\n    t-&gt;vocab = (char**)malloc(vocab_size * sizeof(char*));\n    t-&gt;vocab_scores = (float*)malloc(vocab_size * sizeof(float));\n    t-&gt;sorted_vocab = NULL; \/\/ initialized lazily\n    \/\/ \u521d\u59cb\u5316byte_pieces\u6570\u7ec4\uff0c\u8be5\u6570\u7ec4\u5b58\u50a8\u6240\u6709\u5355\u5b57\u8282\u5b57\u7b26\u4e32\n    for (int i = 0; i &lt; 256; i++) {\n        t-&gt;byte_pieces[i * 2] = (unsigned char)i;\n        t-&gt;byte_pieces[i * 2 + 1] = '\\0';\n    }\n    \/\/ read in the file\n    FILE *file = fopen(tokenizer_path, \"rb\");\n    if (!file) { fprintf(stderr, \"couldn't load %s\\n\", tokenizer_path); exit(EXIT_FAILURE); }\n    \/\/ \u4ece\u6587\u4ef6\u4e2d\u8bfb\u53d6max_token_length\u7684\u503c\n    if (fread(&amp;t-&gt;max_token_length, sizeof(int), 1, file) != 1) { fprintf(stderr, \"failed read\\n\"); exit(EXIT_FAILURE); }\n    int len;\n    \/\/ \u8fd9\u4e2a\u5faa\u73af\u5757\u8bfb\u53d6\u6bcf\u4e2avocab\u5b57\u7b26\u4e32\u53ca\u5176\u5bf9\u5e94\u7684vocab_scores\u503c\u3002\n    for (int i = 0; i &lt; vocab_size; i++) {\n        if (fread(t-&gt;vocab_scores + i, sizeof(float), 1, file) != 1) { fprintf(stderr, \"failed read\\n\"); exit(EXIT_FAILURE);}\n        if (fread(&amp;len, sizeof(int), 1, file) != 1) { fprintf(stderr, \"failed read\\n\"); exit(EXIT_FAILURE); }\n        t-&gt;vocab[i] = (char *)malloc(len + 1);\n        if (fread(t-&gt;vocab[i], len, 1, file) != 1) { fprintf(stderr, \"failed read\\n\"); exit(EXIT_FAILURE); }\n        t-&gt;vocab[i][len] = '\\0'; \/\/ add the string terminating token\n    }\n    fclose(file);\n}\n\nvoid free_tokenizer(Tokenizer* t) {\n    for (int i = 0; i &lt; t-&gt;vocab_size; i++) \n    { \n    \tfree(t-&gt;vocab[i]); \n    }\n    if(t-&gt;vocab) free(t-&gt;vocab);\n    if(t-&gt;vocab_scores) free(t-&gt;vocab_scores);\n    if(t-&gt;sorted_vocab) free(t-&gt;sorted_vocab);\n}\n\nvoid print_tokenizer(Tokenizer* t) {\n\t\tprintf(\"vocab = %d\\n\", t-&gt;vocab_size);\n\t\tprintf(\"max_token_length = %d\\n\", t-&gt;max_token_length);\n    for (int i = 0; i &lt; t-&gt;vocab_size; i++) \n    { \n    \tprintf(\"%5d, %12.6lf, (%s)\\n\", i, t-&gt;vocab_scores[i], t-&gt;vocab[i]);\n    }\n}\n\nint main(int argc, char *argv[]) {\n\n    char *tokenizer_path = \"tokenizer.bin\";\n    \n    \/\/ build the Tokenizer via the tokenizer .bin file\n    Tokenizer tokenizer;\n    int vocab_size = 32000; \/\/ \u4ece\u6a21\u578b\u6587\u4ef6\u7684 config.json \u83b7\u53d6\n    build_tokenizer(&amp;tokenizer, tokenizer_path, vocab_size);\n    \n    print_tokenizer(&amp;tokenizer);\n    free_tokenizer(&amp;tokenizer);\n    \n    return 0;\n}<\/pre><\/div>\n\n\n\n<p>\u7f16\u8bd1 test01.c<\/p>\n\n\n\n<div class=\"wp-block-urvanov-syntax-highlighter-code-block\"><pre class=\"lang:sh decode:true \" >make test01\ncc     test01.c   -o test01<\/pre><\/div>\n\n\n\n<p>\u8fd0\u884c test01<\/p>\n\n\n\n<div class=\"wp-block-urvanov-syntax-highlighter-code-block\"><pre class=\"lang:python decode:true \" >.\/test01 &gt; 1.txt<\/pre><\/div>\n\n\n\n<p>\u7531\u4e8e\u8f93\u51fa\u7684\u5185\u5bb9\u5f88\u591a\uff0c\u6240\u4ee5\u6211\u4eec\u628a\u8f93\u51fa\u91cd\u5b9a\u5411\u5230 1.txt \u6587\u4ef6\u4e2d\uff0c\u4e0b\u9762\u662f 1.txt \u6587\u4ef6\u7684\u5f00\u5934\u548c\u7ed3\u5c3e\u90e8\u5206\u5185\u5bb9<\/p>\n\n\n\n<div class=\"wp-block-urvanov-syntax-highlighter-code-block\"><pre class=\"lang:sh decode:true \" >vocab = 32000\nmax_token_length = 27\n    0,     0.000000, (&lt;unk&gt;)\n    1,     0.000000, (\n&lt;s&gt;\n)\n    2,     0.000000, (\n&lt;\/s&gt;\n)\n    3,     0.000000, (&lt;0x00&gt;)\n    4,     0.000000, (&lt;0x01&gt;)\n    5,     0.000000, (&lt;0x02&gt;)\n    6,     0.000000, (&lt;0x03&gt;)\n    7,     0.000000, (&lt;0x04&gt;)\n    8,     0.000000, (&lt;0x05&gt;)\n    9,     0.000000, (&lt;0x06&gt;)\n   10,     0.000000, (&lt;0x07&gt;)\n   11,     0.000000, (&lt;0x08&gt;)\n   12,     0.000000, (&lt;0x09&gt;)\n...\n  259, -1000000000.000000, (  )\n  260,    -1.000000, ( t)\n  261,    -2.000000, (er)\n  262,    -3.000000, (in)\n  263,    -4.000000, ( a)\n  264,    -5.000000, (en)\n  265,    -6.000000, (on)\n  266,    -7.000000, ( th)\n  267,    -8.000000, (es)\n  268, -1000000000.000000, (    )\n  269,   -10.000000, ( s)\n  270,   -11.000000, ( d)\n  271,   -12.000000, (at)\n...\n31985, -31726.000000, (\u602a)\n31986, -31727.000000, (\u8054)\n31987, -31728.000000, (\uc5ed)\n31988, -31729.000000, (\u6cf0)\n31989, -31730.000000, (\ubc31)\n31990, -31731.000000, (\u1f40)\n31991, -31732.000000, (\u3052)\n31992, -31733.000000, (\u3079)\n31993, -31734.000000, (\u8fb9)\n31994, -31735.000000, (\u8fd8)\n31995, -31736.000000, (\u9ec3)\n31996, -31737.000000, (\uc655)\n31997, -31738.000000, (\u6536)\n31998, -31739.000000, (\u5f18)\n31999, -31740.000000, (\u7ed9)\n<\/pre><\/div>\n\n\n\n<p>\u53ef\u4ee5\u770b\u5230\uff0ctoken \u7684\u6700\u5927\u957f\u5ea6\u4e3a27<\/p>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>9. \u8f6c\u6362 tokenizer.model \u5e76\u4fdd\u5b58 \u524d\u9762\u7684\u7ae0\u8282\u8f6c\u6362\u6a21\u578b\u7684\u6240\u6709\u6743\u91cd\u540e\uff0c\u8fd8\u9700\u8981\u8f6c\u6362 tokeniz [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"set","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[313,289,443,442,312],"tags":[242,314],"class_list":["post-2458","post","type-post","status-publish","format-standard","hentry","category-chatgpt","category-gpt","category-llm","category-llms","category-openai","tag-chatgpt","tag-openai-api"],"views":1386,"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=\/wp\/v2\/posts\/2458","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2458"}],"version-history":[{"count":14,"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=\/wp\/v2\/posts\/2458\/revisions"}],"predecessor-version":[{"id":2481,"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=\/wp\/v2\/posts\/2458\/revisions\/2481"}],"wp:attachment":[{"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2458"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2458"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.aqwu.net\/wp\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2458"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}