Update chat-template
Browse files- special_tokens_map.json +7 -1
- tokenizer_config.json +1 -0
special_tokens_map.json
CHANGED
@@ -12,5 +12,11 @@
|
|
12 |
">>SUFFIX<<",
|
13 |
">>MIDDLE<<"
|
14 |
],
|
15 |
-
"eos_token":
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
}
|
|
|
12 |
">>SUFFIX<<",
|
13 |
">>MIDDLE<<"
|
14 |
],
|
15 |
+
"eos_token": {
|
16 |
+
"content": "<|endoftext|>",
|
17 |
+
"lstrip": false,
|
18 |
+
"normalized": false,
|
19 |
+
"rstrip": false,
|
20 |
+
"single_word": false
|
21 |
+
}
|
22 |
}
|
tokenizer_config.json
CHANGED
@@ -111,6 +111,7 @@
|
|
111 |
">>SUFFIX<<",
|
112 |
">>MIDDLE<<"
|
113 |
],
|
|
|
114 |
"clean_up_tokenization_spaces": true,
|
115 |
"eos_token": "<|endoftext|>",
|
116 |
"model_input_names": [
|
|
|
111 |
">>SUFFIX<<",
|
112 |
">>MIDDLE<<"
|
113 |
],
|
114 |
+
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{% if not loop.first %}{% endif %}{% if message['role'] == 'system' %}{{ message['content'] + ' ' }}{% elif message['role'] == 'user' %}{{ 'USER: ' + message['content'] + ' ' }}{% elif message['role'] == 'assistant' %}{{ 'ASSISTANT: ' + message['content'] + '</s>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}",
|
115 |
"clean_up_tokenization_spaces": true,
|
116 |
"eos_token": "<|endoftext|>",
|
117 |
"model_input_names": [
|