add examples, math, and auth

app.py

@@ -80,7 +80,7 @@ class Chatbot():
                 d.metadata['page'] = ''
 
         output = ' '.join([
-            f'SOURCE {i}\n' + d.page_content + '\n\nSource: ' + d.metadata['source'] + '\nPage: ' + str(d.metadata['page']) + '\n\n\n'
+            f'SOURCE {i}\n' + d.page_content.replace('$', '') + '\n\nSource: ' + d.metadata['source'] + '\nPage: ' + str(d.metadata['page']) + '\n\n\n' + '-'*100
             for i, d in enumerate(documents)
         ])
 
@@ -207,6 +207,14 @@ class Chatbot():
                 submit.click(self.chat, inputs=[message, state, module, model], outputs=[chatbot, state])
                 message.submit(self.chat, inputs=[message, state, module, model], outputs=[chatbot, state])
                 
+                gr.Examples(
+                    examples=[
+                    'Answer the following question, explain your reasoning:\n'
+                    'Answer the following question, explain your reasoning, use latex format:\n'
+                    'Answer the following multiple choice question, explain your reasoning:\n'
+                    ],
+                    inputs=message
+                )
             
             with gr.Tab("Search"):
 
@@ -244,7 +252,7 @@ class Chatbot():
                 message.submit(self.chat_vanilla, inputs=[message, vanilla_state, model], outputs=[vanilla_chatbot, vanilla_state])
 
 
-        block.launch(debug=True, share=False)
+        block.launch(debug=True, share=False, auth=("bread", os.environ.get('PASSWORD')))
         
 
 if __name__ == '__main__':

chroma/chroma-collections.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df46d39377ae5665c6f52890a51c1b39d55ce5cca43006a62d775f558d01a3d7
+oid sha256:a1c64656c27f073dc82f7d7e20cd31059ed38e6ac344408db5c052e56983ee59
 size 557

chroma/chroma-embeddings.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed9ecd5766659fab4de819f1149e7022fa7bac9700fed908c8bb8cd7fbee2f18
-size 5751374
+oid sha256:1655cb7989cce3e21c033127f334c095e5cca13458d9318ed1544bfed657f005
+size 7266587

chroma/index/{id_to_uuid_48820301-4b52-46b2-8746-e343bf602b95.pkl → id_to_uuid_33066827-7eb2-42ca-8a41-4459ce4b0011.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d94ee3ba8d5ddcb17a8ca31845f96988cae28412304707b61389d72bc38d1a1c
-size 14648
+oid sha256:09d54e466950cbc8b35a45821e139193bebccca0124a74635335341ad6f2dbf9
+size 18544

chroma/index/{index_48820301-4b52-46b2-8746-e343bf602b95.bin → index_33066827-7eb2-42ca-8a41-4459ce4b0011.bin}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cd4d746421153bac9a1ee713618ae655718472fb597caf0a54998fd08908c25
-size 2882380
+oid sha256:2f58846f6d4fa248d0775e83d21968429f064925379b0e32aa8696719df67c46
+size 3631264

chroma/index/{index_metadata_48820301-4b52-46b2-8746-e343bf602b95.pkl → index_metadata_33066827-7eb2-42ca-8a41-4459ce4b0011.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7384c6ad2206ea30195640a1aa2b8721341f26aae78fa7fc9f584705dbceb88c
+oid sha256:55a26232ed945d77cb53a3db5983204243c0987f20fc0ca9b7a98b294b3956d1
 size 74

chroma/index/{uuid_to_id_48820301-4b52-46b2-8746-e343bf602b95.pkl → uuid_to_id_33066827-7eb2-42ca-8a41-4459ce4b0011.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c28815d0892801f4674238321a1bfe266dd5f4f802bfd85faea9cc82c3dad51f
-size 17165
+oid sha256:dd5b8c41ef8f11902aed5a95e89fd585402cf1f072216ae9794c9b950f939e7d
+size 21687

ingest.py

@@ -3,7 +3,7 @@ import argparse
 import os
 
 from langchain.document_loaders import PyPDFLoader
-from langchain.text_splitter import CharacterTextSplitter
+from langchain.text_splitter import CharacterTextSplitter, LatexTextSplitter
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores import Chroma
 from langchain.schema import Document
@@ -16,6 +16,7 @@ parser.add_argument("folder", help="The folder to be ingested", type=str)
 parser.add_argument("--chunk_size", help="Chunk size", type=int, default=1500)
 parser.add_argument('--chunk_overlap', help='Chunk overlap', type=int, default=400)
 parser.add_argument('--separator', help='Separator', type=str, default='\n')
+parser.add_argument('--use_tex_splitter', help='Use tex splitter', type=bool, default=False)
 
 args = parser.parse_args()
 
@@ -23,6 +24,7 @@ FOLDER = args.folder
 CHUNK_SIZE = args.chunk_size
 CHUNK_OVERLAP = args.chunk_overlap
 SEPARATOR = args.separator
+USE_TEX_SPLITTER = args.use_tex_splitter
 
 
 class Ingest():
@@ -33,7 +35,14 @@ class Ingest():
             chunk_size,
             separator,
             chunk_overlap,
+            use_tex_splitter,
     ):
+        self.vectorstore = Chroma(persist_directory='./chroma', embedding_function=OpenAIEmbeddings())
+        print(f"Count of {self.vectorstore._collection.count()} in vectostore")
+        print(f"Deleting previous items from {folder}")
+        self.vectorstore._collection.delete(where={'module' : folder})
+        print(f"New count, {self.vectorstore._collection.count()}")
+
         self.folder = folder
         self.chunk_size = chunk_size
 
@@ -46,6 +55,15 @@ class Ingest():
             length_function = len,
         )
 
+        if use_tex_splitter:
+            self.splitter = LatexTextSplitter(
+                chunk_size = chunk_size,
+                chunk_overlap  = chunk_overlap,
+            )
+
+    def _load_tex(self, path):
+        with open(path, "r") as f:
+            return f.read()
 
     def ingest(self):
         # find all .pdf files in the data folder
@@ -65,6 +83,12 @@ class Ingest():
             with open(os.path.join('./data', os.path.join(self.folder, t)), "r") as f:
                 documents.append(Document(page_content=f.read(), metadata={"source": os.path.basename(t).split(".")[0] + ' transcript'}))
 
+        # tex
+        texfiles = [f for f in os.listdir(os.path.join('./data', self.folder)) if f.endswith(".tex")]
+        for t in texfiles:
+            documents.append(Document(page_content=self._load_tex(os.path.join('./data', os.path.join(self.folder, t))), metadata={"source": os.path.basename(t).split(".")[0] + ' transcript'}))
+
+
         for i in documents:
             i.metadata['module'] = self.folder
 
@@ -75,7 +99,7 @@ class Ingest():
         embeddings = OpenAIEmbeddings()
         # create store
         print("Embedding chunks...")
-        Chroma.from_documents(chunks, embeddings, persist_directory='./chroma')
+        self.vectorstore.add_texts(texts=[d.page_content for d in chunks], metadatas=[d.metadata for d in chunks])
 
 if __name__ == "__main__":
     ingest = Ingest(
@@ -83,5 +107,6 @@ if __name__ == "__main__":
         chunk_size = CHUNK_SIZE,
         separator = SEPARATOR,
         chunk_overlap = CHUNK_OVERLAP,
+        use_tex_splitter = USE_TEX_SPLITTER,
     )
     ingest.ingest()