tjtanaa commited on
Commit
58316e9
·
1 Parent(s): 8cb6b5b

Update chat-template

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +7 -1
  2. tokenizer_config.json +1 -0
special_tokens_map.json CHANGED
@@ -12,5 +12,11 @@
12
  ">>SUFFIX<<",
13
  ">>MIDDLE<<"
14
  ],
15
- "eos_token": "<|endoftext|>"
 
 
 
 
 
 
16
  }
 
12
  ">>SUFFIX<<",
13
  ">>MIDDLE<<"
14
  ],
15
+ "eos_token": {
16
+ "content": "<|endoftext|>",
17
+ "lstrip": false,
18
+ "normalized": false,
19
+ "rstrip": false,
20
+ "single_word": false
21
+ }
22
  }
tokenizer_config.json CHANGED
@@ -111,6 +111,7 @@
111
  ">>SUFFIX<<",
112
  ">>MIDDLE<<"
113
  ],
 
114
  "clean_up_tokenization_spaces": true,
115
  "eos_token": "<|endoftext|>",
116
  "model_input_names": [
 
111
  ">>SUFFIX<<",
112
  ">>MIDDLE<<"
113
  ],
114
+ "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{% if not loop.first %}{% endif %}{% if message['role'] == 'system' %}{{ message['content'] + ' ' }}{% elif message['role'] == 'user' %}{{ 'USER: ' + message['content'] + ' ' }}{% elif message['role'] == 'assistant' %}{{ 'ASSISTANT: ' + message['content'] + '</s>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}",
115
  "clean_up_tokenization_spaces": true,
116
  "eos_token": "<|endoftext|>",
117
  "model_input_names": [