added sound

.gitea/workflows/deploy_via_docker_compose.yml

@@ -6,13 +6,13 @@ jobs:
   Explore-Gitea-Actions:
     #runs-on: ubuntu-latest
     steps:
-      - run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event."
+      #- run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event."
-      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
+      #- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
 
       - name: List files in the repository
         run: |
           ls ~
 
-      - run: echo "🍏 This job's status is ${{ job.status }}."
+      #- run: echo "🍏 This job's status is ${{ job.status }}."
 
 

README.md

@@ -54,7 +54,6 @@ sudo docker rmi $(sudo docker images -f "dangling=true" -q)
 Give just the translation of the given input to German and nothing else.
 
 
-Give the JSON of a graph linking Germanys 9 biggest cities
 
 
 

backend/Dockerfile

@@ -3,15 +3,19 @@ FROM python:3.12
 RUN apt-get update
 RUN apt-get install -y firefox-esr
 RUN apt-get install -y ffmpeg
-RUN apt-get install -y espeak-ng
+RUN apt-get install -y espeak
 
 #RUN curl https://ollama.ai/install.sh | sh
 #RUN ollama run llama2
+WORKDIR /code
+
+COPY requirements.txt /code/requirements.txt
+#RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
+
 
-COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
 COPY . .
 
-ENTRYPOINT ["python3", "app.py"]
+ENTRYPOINT ["python3", "/code/app.py"]
 #ENTRYPOINT ["fastapi", "run", "main.py", "--port", "8000"]
 

backend/app.py

@@ -6,33 +6,32 @@ OpenAPI access via http://localhost:5000/openapi/ on local docker-compose deploy
 #import warnings
 #warnings.filterwarnings("ignore")
 
-#std lib modules:
+#------std lib modules:-------
 import os, sys, json, time
+import os.path
 from typing import Any, Tuple, List, Dict, Any, Callable, Optional
 from datetime import datetime, date
 from collections import namedtuple
 import hashlib, traceback, logging
 from functools import wraps
+import base64
 
+#-------ext libs--------------
 #llm
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
-from langchain_community.llms import Ollama
 
 import tiktoken
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.chains import RetrievalQA
+from langchain.callbacks.base import BaseCallbackHandler, BaseCallbackManager
+from langchain.prompts import PromptTemplate
 
-
+from langchain_community.llms import Ollama
 from langchain_community.vectorstores.elasticsearch import ElasticsearchStore
 from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
 from langchain_community.embeddings import OllamaEmbeddings
 
-
-from langchain.callbacks.base import BaseCallbackHandler, BaseCallbackManager
-from langchain.prompts import PromptTemplate
-
-#ext libs
 from elasticsearch import NotFoundError, Elasticsearch # for normal read/write without vectors
 from elasticsearch_dsl import Search, A, Document, Date, Integer, Keyword, Float, Long, Text, connections
 from elasticsearch.exceptions import ConnectionError
@@ -44,31 +43,24 @@ import jwt as pyjwt
 
 
 #flask, openapi
-from flask import Flask, send_from_directory, Response, request, jsonify
+from flask import Flask, send_from_directory, send_file, Response, request, jsonify
-import sys, os
 from flask_cors import CORS, cross_origin
 from werkzeug.utils import secure_filename
 from flask_openapi3 import Info, Tag, OpenAPI, Server, FileStorage
 from flask_socketio import SocketIO, join_room, leave_room, rooms, send
 
-
-import base64
-import os
 from cryptography.fernet import Fernet
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
+import pyttsx3
+
 #----------home grown--------------
 #from scraper import WebScraper
 from funcs import group_by
 from elastictools import get_by_id, update_by_id, delete_by_id
 from models import QueryLog, Chatbot, User
 
-import pyttsx3
-
-engine = pyttsx3.init()
-
-
 
 #LLM_PAYLOAD = int(os.getenv("LLM_PAYLOAD"))
 #CHUNK_SIZE = int(os.getenv("CHUNK_SIZE"))
@@ -260,11 +252,13 @@ def handle_message(message):
     #try:
     room = message["room"]
     question = message["question"]
+    system_prompt = message["system_prompt"]
     bot_id = message["bot_id"]
+
     #except:
     #    return
 
-    for chunk in ask_bot(question, bot_id):
+    for chunk in ask_bot(system_prompt + " " + question, bot_id):
         socket.emit('backend token', {'data': chunk, "done": False}, to=room)
     socket.emit('backend token', {'done': True}, to=room)
 
@@ -347,40 +341,56 @@ class GetSpeechRequest(BaseModel):
 @app.post('/text2speech', summary="", tags=[], security=security)
 @uses_jwt()
 def text2speech(form: GetSpeechRequest, decoded_jwt, user):
+    engine = pyttsx3.init()
 
-    #def get_voice(s):
+    def get_voice(s):
-    #    for v in engine.getProperty("voices"):
+        for v in engine.getProperty("voices"):
-    #        if s == v.id:
+            if s == v.id:
-    #            return v
+                return v
-
-    #def set_voice(v):
-    #    engine.setProperty("voice", v.id)
-
-    #def set_volume(n):
-    #    engine.setProperty('volume', engine.getProperty('volume') + n)
-
-    #def set_rate(n):
-    #    engine.setProperty('rate', engine.getProperty('rate') + n)
 
+    def set_voice(v):
+        engine.setProperty("voice", v.id)
 
+    def set_volume(n):
+        engine.setProperty('volume', engine.getProperty('volume') + n)
 
+    def set_rate(n):
+        engine.setProperty('rate', engine.getProperty('rate') + n)
 
     #voices = engine.getProperty('voices')
     #engine.setProperty('voice', voices[1].id)
-    #set_voice(get_voice("english"))
+    set_voice(get_voice("english"))
-    #set_volume(-5.0)
+    set_volume(-5.0)
-    #set_rate(-40)
+    set_rate(-40)
 
-    # Speak the response
+    #espeak -v mb-en1 -s 120 "Hello world"
-    #engine.say(response)
+    #sudo apt-get install mbrola mbrola-en1
-    #ngine.say("Hello World!")
-    #engine.say("Neuroscience!")
 
+    unix_timestamp = datetime.now().timestamp()
+    file_name = f'speech_{unix_timestamp}.mp3'
+    file_path = f'./public/{file_name}'
 
-    file_name = 'speech.mp3'
+    engine.save_to_file(form.text, file_path)
-    engine.save_to_file(form.text, file_name)
     engine.runAndWait()
-    return send_file(file_name) #, mimetype = 'zip', attachment_filename= 'Audiofiles.zip', as_attachment = True)
+
+    timeout = 10
+    t = 0
+    step = 0.1
+    while not os.path.isfile(file_path):
+        time.sleep(step)
+        t += step
+        if t > timeout:
+            raise Exception("Timeout(%s s) for creating speech.mp3!" % timeout)
+
+    time.sleep(step)
+
+
+    #return send_file(file_path, mimetype='audio/mpeg') #, attachment_filename= 'Audiofiles.zip', as_attachment = True)
+    return jsonify({
+        "status": "success",
+        "file": "/" + file_name
+    })
+
 
 
 

backend/public/index.html

@@ -144,6 +144,11 @@
           <li class="nav-item">
             <a class="nav-link" data-bs-toggle="tab" href="#create_bot_tab">Create bot</a>
           </li>
+
+          <li class="nav-item">
+            <a class="nav-link" data-bs-toggle="tab" href="#tweak_bot_tab">Tweak bot</a>
+          </li>
+
         </ul>
 
         <!-- Tab panes -->
@@ -237,7 +242,7 @@
                       <div class="col"></div>
                       <div class="col"></div>
                       <div class="col">
-                        <button id="create_bot_btn" disabled "type="button" class="btn btn-primary text-white">Create bot</button>
+                        <button id="create_bot_btn" disabled type="button" class="btn btn-primary text-white">Create bot</button>
                      </div>
                 </div>
 
@@ -264,6 +269,17 @@
 
             </div>
 
+
+            <div class="tab-pane container fade" id="tweak_bot_tab">
+
+                <div style="height: 10px !important;"></div>
+
+                <i>Tweaking a new bot requires an account and login via settings!</i>
+                <br>
+                <br>
+
+
+            </div>
         </div>
 
 
@@ -335,6 +351,20 @@
         }
     }
 
+    async function text2speech(jwt, txt){
+        const formData = new FormData();
+        formData.append("text", txt);
+        const response = await fetch("/text2speech", {
+            method: "POST",
+            headers: {
+                'accept': '*/*',
+                'Authorization': 'Bearer ' + jwt
+            },
+            body: formData
+        });
+        return response.json();
+    }
+
     async function create_bot(jwt, name, visibility, description, llm, sys_prompt){
         const formData = new FormData();
         formData.append("name", name);
@@ -394,25 +424,23 @@
                 else{
                     done = true;
                     socket.off('backend token');
+                    dom_ele.dispatchEvent(new CustomEvent(evt_name, { detail: "" }));
                 }
             });
-            socket.emit('client message', {
+            socket.emit('client message', {question, system_prompt, bot_id, room});
-                question: system_prompt + " " + question,
-                bot_id: bot_id,
-                room: room
-            });
             while(!done){
                 yield f();
             }
+            return;
         }
         catch(e){
             console.error(e);
+            return;
         }
         finally{
-            //socket.off('backend token');
-            //dom_ele.removeEventListener(evt_name, evt_listener);
             socket.emit('end');
             socket.close();
+            return;
         }
     }
 
@@ -421,7 +449,9 @@
         document.documentElement.style.setProperty("--bs-primary-rgb", "45, 124, 172");
 
         //chat
-        let tA = document.getElementById("user_input");
+        let user_input = document.getElementById("user_input");
+        let system_prompt = document.getElementById("system_prompt");
+
         let log = document.getElementById("log");
         let submit_btn = document.getElementById("submit_btn");
         let scroll_div = document.getElementById("scroll_div");
@@ -612,20 +642,20 @@
         };
 
         submit_btn.onclick = async evt =>{
-            let s = tA.value;
+            let input_string = user_input.value;
-            //if(s.trim() !== '' && room){
+
-            if(s.trim() !== ''){
+            if(input_string.trim() !== ''){
                 answer_count += 1;
 
-                tA.value = "";
+                user_input.value = "";
-                log_msg('User', s);
+                log_msg('User', input_string);
 
-                let tA2 = document.getElementById("system_prompt");
+                //let tA2 = document.getElementById("system_prompt");
                 let acc_text = "";
 
                 log.innerHTML += `<tr><td><b>${get_bot_name()}</b>:</td><td id="${answer_count}"></td></tr>`;
 
-                for await (let token of ask_question(bot_select.value, s, tA2.value)){
+                for await (let token of ask_question(bot_select.value, input_string, system_prompt.value)){
                     console.log(token);
 
                     acc_text += "" + token;
@@ -635,13 +665,38 @@
 
                 }
 
+                /*
+                function play() {
+                  var audio = new Audio('https://interactive-examples.mdn.mozilla.net/media/cc0-audio/t-rex-roar.mp3');
+                  audio.play();
+                }
+
+                */
+
+
                 let final_answer = acc_text;
                 console.log(final_answer);
 
+
+                let extra_s = "";
+                let jwt = localStorage.getItem("jwt");
+                if(jwt){
+                    let{file} = await text2speech(jwt, final_answer);
+
+                    //autoplay controls
+                    extra_s = `
+                        <audio controls>
+                            <source src="${file}" type="audio/mpeg">
+                        </audio>`;
+
+                    console.log(file);
+                }
+
+
                 switch(view_select.value){
 
                     case "md":
-                        document.getElementById(answer_count).innerHTML = marked.parse(final_answer);
+                        document.getElementById(answer_count).innerHTML = marked.parse(final_answer) + extra_s;
                         break;
 
                     case "dot":

backend/requirements.txt

@@ -28,11 +28,9 @@ cryptography
 
 neo4j
 
-
 pyttsx3
 
 
 
 
 
-