zf tddlZddlZddlmZddlmZddlmZm Z m Z m Z ddl m Z ddlZddlZddlmZmZdZdZeejd d Zd Zd d gdd gdd gdd ggZ dZejeddZejeZde_e dd5Z!ej"e!Z#dddn #1swxYwYe#dZ#e#dZ$e#dZ%e#dZ&e#dZ'eddd d!fd"e(d#e(d$ed%e)d&e)d'ed(e)d)e e(e(ffd*Z*d+e(d,e)d)e(fd-Z+ej,d.d/05Z-ej.eej/5ej05ej1d12Z2ej1d3d45Z3dddn #1swxYwYej05ej4d6d7ed7e8Z5ej4d9d:d;dd?d>d8Z7ej4d@d7dAd7d 8Z8ej4dBd?d;d>d!8Z9dddn #1swxYwYdddn #1swxYwYej/5ej:dCZ;dddn #1swxYwYej.dDej/5ej05ej.dEZ<ej.dF2Z=dddn #1swxYwYej.dGej05ej.dHZ<ej.dI2Z>dddn #1swxYwYdddn #1swxYwYej?ee2e3ge=e>ge*Je;@e*e2e3e5e6e7e8e9ge=e>gKdddn #1swxYwYeAdLkre-BdSdS)MN)Path)Thread)ListTupleIteratorOptional)Queue)AutoModelForCausalLM AutoTokenizeriiMAX_INPUT_TOKEN_LENGTH4096aG

Llama-2 7B Chat with Uncertainty Probes

This Space demonstrates the Llama-2-7b-chat model with a semantic uncertainty probe.

This demo is based on our paper: "Semantic Entropy Probes: Robust and Cheap Hallucination Detection in LLMs" by Jannik Kossen*, Jiatong Han*, Muhammed Razzak*, Lisa Schut, Shreshth Malik and Yarin Gal.

The highlighted text shows the model's uncertainty in real-time:

The demo compares the model's uncertainty with two different probes:

Please see our paper for more details. NOTE: This demo is a work in progress.

zWhat is the capital of France?zWho landed on the moon?zWho is Yarin Gal?z1Explain the theory of relativity in simple terms.Tzmeta-llama/Llama-2-7b-chat-hfautoF) device_map load_in_8bitz ./model/20240625-131035_demo.pklrbt_bmodelsep_layer_ranget_amodelap_layer_rangeu;

Running on CPU 🥶 This demo does not work on CPU.

g333333?g?2g333333?message system_promptmax_new_tokens temperaturetop_ptop_krepetition_penaltyreturnc\g}|r|d|d|d|dt|d}|jdtkr1|ddt df}t jdtd|tj }t||d ||||d d tj | } tj 5tjdi| } dddn #1swxYwY| jd |jddf} t| d } t#| | j} d }d }t'dt)| D]}tjd| |D}|t.d t.dd}t2|ddd ddzdz }|t6d t6dd}dt8|ddd dz dzdz }| jd |jd|zf}t|}t#||||t;||}t;||}|d|z }|d|z }||fS)Nsystem)rolecontentuserpt)return_tensorsz6Trimmed input from conversation as it was longer than z tokens.T) input_idsr do_samplerrrroutput_hidden_statesreturn_dict_in_generateattention_maskr)skip_special_tokensrcLg|]!}|ddddf"S)rN)cpu).0generated_tokens o/Users/shreshth/Library/CloudStorage/OneDrive-Nexus365/Projects/semantic_entropy/semantic-entropy-probes/app.py zgenerate..us3'h'h'h?1aaa(@(D(D(F(F'h'h'h )append tokenizerapply_chat_templateshaper grWarningtomodeldevicedicttorch ones_likeno_gradgenerate sequencesdecodeprint hidden_statesrangelenstacknumpyse_layer_rangereshapese_probe predict_probaacc_layer_range acc_probehighlight_text)rrrrrrr conversationr)generation_kwargsoutputsgenerated_tokensgenerated_texthiddense_highlighted_textacc_highlighted_textitoken_embeddingsse_concat_layers se_probe_predacc_concat_layersacc_probe_pred output_id output_wordse_new_highlighted_textacc_new_highlighted_texts r3rGrGEsLJX-HHIIIG<<===--l4-PPIq222aaa"8!8!9!99:  lLblllmmm U\**I%-! $y11    66.55#455666666666666666(IOA,>,?,?)?@%%&6D%QQN .  "F1c&kk " "??!;'h'h^def^g'h'h'hiiooqq+N1,=nQ>O,OPXXY[\\ ../?/G/G2/N/NOOPQRSTUXYY\]] -_Q-?PQ@R-RS[[\^__i556G6O6OPQSU6V6VWWXYZ[\]]abbeff%a);A)=&=> &&y11  im^DDD"0m"L"L#1+~#N#N <#:<<< >$< > >>  4 44s5DDDtextuncertainty_scorec|dkr/ddtdd|z ztdd|z zfz}n.dtdd|zzdtdd|zzfz}d||S)Nrz #%02X%02X%02Xr(z:{})intformat)rirj html_colors r3rVrVs1$  q,,- . . q,,- . .(  % q,,- . .  q,,- . .(  H N ND  r5z Llama-2 7B Chat with Dual Probeszfooter {visibility: hidden})titlecssMessage)labelz System promptr7)rslineszMax new tokensr()rsminimummaximumstepvalue Temperatureg{Gz?g@g?zTop-p (nucleus sampling)g?g?zTop-kizRepetition penaltyGeneratez

z#

Semantic Uncertainty Probe

zSemantic Uncertainty Probez
z

Accuracy Probe

zAccuracy Probe)examplesinputsrYfn)r|rY__main__)Cospicklepklpathlibr threadingrtypingrrrrqueuer gradior>rD transformersr r MAX_MAX_NEW_TOKENSDEFAULT_MAX_NEW_TOKENSrmgetenvr DESCRIPTIONEXAMPLESmodel_idfrom_pretrainedrAr;use_default_system_promptopenfload probe_datarRrPrUrTstrfloatrGrVBlocksdemoHTMLRowColumnTextboxrrSliderrrrrrButton generate_btnrp se_output acc_outputExamplesclick__name__launchr9r5r3rs 222222222222 <<<<<<<<YRY'?HHII $&r*#"8"=  R.H 0 0f[` a a aE- -h77I*/I' 0$ 7 7!1SXa[[ !!!!!!!!!!!!!!!BJ*%H 12N:&I !12O1 #A5A5 A5A5A5 A5  A5  A5A5 38_A5A5A5A5J3"RY7=Z[[[,_c BGK  w w RY[[ G G bjy111G&BJ_AFFFM G G G G G G G G G G G G G G GRY[[ w w&RY-=qRdkluKLLLN#")-sY\dhiiiKBI$>VY`dlopppEBIGQ1TVWWWE!*1Es\_fjru!v!v!v   w w w w w w w w w w w w w w w w w w w w w w w w w w w w w w -- ry,, --------------- BGJ  9 9 RY[[ D DBGABBE&BCCCI D D D D D D D D D D D D D D D 2333 RY[[ 9 9BG566E '7888J 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9BK'J'   UESefJ'Q,,,,,,,,,,,,,,,^ zKKMMMMMs!B;;B?B?, N H3$F  H3 F H3F H3&A*H H3H H3#H $H3' N3H7 7N:H7 ;N I* N*I. .N1I. 2"NM$"K MK MK "M<"L* M*L. .M1L. 2M5 NM NM