From 6017d0f537baa1b28ece1b8aca7d8be1d7e82cf9 Mon Sep 17 00:00:00 2001 From: Tobias Weise Date: Fri, 2 Aug 2024 23:51:28 +0200 Subject: [PATCH] added sound --- .../workflows/deploy_via_docker_compose.yml | 6 +- README.md | 1 - backend/Dockerfile | 12 ++- backend/app.py | 94 ++++++++++--------- backend/public/index.html | 89 ++++++++++++++---- backend/requirements.txt | 2 - 6 files changed, 135 insertions(+), 69 deletions(-) diff --git a/.gitea/workflows/deploy_via_docker_compose.yml b/.gitea/workflows/deploy_via_docker_compose.yml index dfa8058..7d8a29a 100644 --- a/.gitea/workflows/deploy_via_docker_compose.yml +++ b/.gitea/workflows/deploy_via_docker_compose.yml @@ -6,13 +6,13 @@ jobs: Explore-Gitea-Actions: #runs-on: ubuntu-latest steps: - - run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!" + #- run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event." + #- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!" - name: List files in the repository run: | ls ~ - - run: echo "🍏 This job's status is ${{ job.status }}." + #- run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/README.md b/README.md index 3ca968a..d6a3543 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,6 @@ sudo docker rmi $(sudo docker images -f "dangling=true" -q) Give just the translation of the given input to German and nothing else. -Give the JSON of a graph linking Germanys 9 biggest cities diff --git a/backend/Dockerfile b/backend/Dockerfile index bd7d40c..2657393 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -3,15 +3,19 @@ FROM python:3.12 RUN apt-get update RUN apt-get install -y firefox-esr RUN apt-get install -y ffmpeg -RUN apt-get install -y espeak-ng +RUN apt-get install -y espeak #RUN curl https://ollama.ai/install.sh | sh #RUN ollama run llama2 +WORKDIR /code + +COPY requirements.txt /code/requirements.txt +#RUN pip3 install --no-cache-dir --upgrade -r requirements.txt +RUN pip3 install --no-cache-dir -r requirements.txt + -COPY requirements.txt requirements.txt -RUN pip3 install -r requirements.txt COPY . . -ENTRYPOINT ["python3", "app.py"] +ENTRYPOINT ["python3", "/code/app.py"] #ENTRYPOINT ["fastapi", "run", "main.py", "--port", "8000"] diff --git a/backend/app.py b/backend/app.py index 1bfc9b8..5e7031e 100644 --- a/backend/app.py +++ b/backend/app.py @@ -6,33 +6,32 @@ OpenAPI access via http://localhost:5000/openapi/ on local docker-compose deploy #import warnings #warnings.filterwarnings("ignore") -#std lib modules: +#------std lib modules:------- import os, sys, json, time +import os.path from typing import Any, Tuple, List, Dict, Any, Callable, Optional from datetime import datetime, date from collections import namedtuple import hashlib, traceback, logging from functools import wraps +import base64 +#-------ext libs-------------- #llm from langchain.callbacks.manager import CallbackManager from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler -from langchain_community.llms import Ollama import tiktoken from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.chains import RetrievalQA +from langchain.callbacks.base import BaseCallbackHandler, BaseCallbackManager +from langchain.prompts import PromptTemplate - +from langchain_community.llms import Ollama from langchain_community.vectorstores.elasticsearch import ElasticsearchStore from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader from langchain_community.embeddings import OllamaEmbeddings - -from langchain.callbacks.base import BaseCallbackHandler, BaseCallbackManager -from langchain.prompts import PromptTemplate - -#ext libs from elasticsearch import NotFoundError, Elasticsearch # for normal read/write without vectors from elasticsearch_dsl import Search, A, Document, Date, Integer, Keyword, Float, Long, Text, connections from elasticsearch.exceptions import ConnectionError @@ -44,31 +43,24 @@ import jwt as pyjwt #flask, openapi -from flask import Flask, send_from_directory, Response, request, jsonify -import sys, os +from flask import Flask, send_from_directory, send_file, Response, request, jsonify from flask_cors import CORS, cross_origin from werkzeug.utils import secure_filename from flask_openapi3 import Info, Tag, OpenAPI, Server, FileStorage from flask_socketio import SocketIO, join_room, leave_room, rooms, send - -import base64 -import os from cryptography.fernet import Fernet from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC +import pyttsx3 + #----------home grown-------------- #from scraper import WebScraper from funcs import group_by from elastictools import get_by_id, update_by_id, delete_by_id from models import QueryLog, Chatbot, User -import pyttsx3 - -engine = pyttsx3.init() - - #LLM_PAYLOAD = int(os.getenv("LLM_PAYLOAD")) #CHUNK_SIZE = int(os.getenv("CHUNK_SIZE")) @@ -260,11 +252,13 @@ def handle_message(message): #try: room = message["room"] question = message["question"] + system_prompt = message["system_prompt"] bot_id = message["bot_id"] + #except: # return - for chunk in ask_bot(question, bot_id): + for chunk in ask_bot(system_prompt + " " + question, bot_id): socket.emit('backend token', {'data': chunk, "done": False}, to=room) socket.emit('backend token', {'done': True}, to=room) @@ -347,40 +341,56 @@ class GetSpeechRequest(BaseModel): @app.post('/text2speech', summary="", tags=[], security=security) @uses_jwt() def text2speech(form: GetSpeechRequest, decoded_jwt, user): + engine = pyttsx3.init() - #def get_voice(s): - # for v in engine.getProperty("voices"): - # if s == v.id: - # return v - - #def set_voice(v): - # engine.setProperty("voice", v.id) - - #def set_volume(n): - # engine.setProperty('volume', engine.getProperty('volume') + n) - - #def set_rate(n): - # engine.setProperty('rate', engine.getProperty('rate') + n) + def get_voice(s): + for v in engine.getProperty("voices"): + if s == v.id: + return v + def set_voice(v): + engine.setProperty("voice", v.id) + def set_volume(n): + engine.setProperty('volume', engine.getProperty('volume') + n) + def set_rate(n): + engine.setProperty('rate', engine.getProperty('rate') + n) #voices = engine.getProperty('voices') #engine.setProperty('voice', voices[1].id) - #set_voice(get_voice("english")) - #set_volume(-5.0) - #set_rate(-40) + set_voice(get_voice("english")) + set_volume(-5.0) + set_rate(-40) - # Speak the response - #engine.say(response) - #ngine.say("Hello World!") - #engine.say("Neuroscience!") + #espeak -v mb-en1 -s 120 "Hello world" + #sudo apt-get install mbrola mbrola-en1 + unix_timestamp = datetime.now().timestamp() + file_name = f'speech_{unix_timestamp}.mp3' + file_path = f'./public/{file_name}' - file_name = 'speech.mp3' - engine.save_to_file(form.text, file_name) + engine.save_to_file(form.text, file_path) engine.runAndWait() - return send_file(file_name) #, mimetype = 'zip', attachment_filename= 'Audiofiles.zip', as_attachment = True) + + timeout = 10 + t = 0 + step = 0.1 + while not os.path.isfile(file_path): + time.sleep(step) + t += step + if t > timeout: + raise Exception("Timeout(%s s) for creating speech.mp3!" % timeout) + + time.sleep(step) + + + #return send_file(file_path, mimetype='audio/mpeg') #, attachment_filename= 'Audiofiles.zip', as_attachment = True) + return jsonify({ + "status": "success", + "file": "/" + file_name + }) + diff --git a/backend/public/index.html b/backend/public/index.html index 2cfa6fb..f2d1a6d 100644 --- a/backend/public/index.html +++ b/backend/public/index.html @@ -144,6 +144,11 @@
  • Create bot
    • + +
  • + Tweak bot +
    • + @@ -237,7 +242,7 @@
    - +
    @@ -264,6 +269,17 @@ + +
    + +
    + + Tweaking a new bot requires an account and login via settings! +
    +
    + + +
    @@ -335,6 +351,20 @@ } } + async function text2speech(jwt, txt){ + const formData = new FormData(); + formData.append("text", txt); + const response = await fetch("/text2speech", { + method: "POST", + headers: { + 'accept': '*/*', + 'Authorization': 'Bearer ' + jwt + }, + body: formData + }); + return response.json(); + } + async function create_bot(jwt, name, visibility, description, llm, sys_prompt){ const formData = new FormData(); formData.append("name", name); @@ -394,25 +424,23 @@ else{ done = true; socket.off('backend token'); + dom_ele.dispatchEvent(new CustomEvent(evt_name, { detail: "" })); } }); - socket.emit('client message', { - question: system_prompt + " " + question, - bot_id: bot_id, - room: room - }); + socket.emit('client message', {question, system_prompt, bot_id, room}); while(!done){ yield f(); } + return; } catch(e){ console.error(e); + return; } finally{ - //socket.off('backend token'); - //dom_ele.removeEventListener(evt_name, evt_listener); socket.emit('end'); socket.close(); + return; } } @@ -421,7 +449,9 @@ document.documentElement.style.setProperty("--bs-primary-rgb", "45, 124, 172"); //chat - let tA = document.getElementById("user_input"); + let user_input = document.getElementById("user_input"); + let system_prompt = document.getElementById("system_prompt"); + let log = document.getElementById("log"); let submit_btn = document.getElementById("submit_btn"); let scroll_div = document.getElementById("scroll_div"); @@ -612,20 +642,20 @@ }; submit_btn.onclick = async evt =>{ - let s = tA.value; - //if(s.trim() !== '' && room){ - if(s.trim() !== ''){ + let input_string = user_input.value; + + if(input_string.trim() !== ''){ answer_count += 1; - tA.value = ""; - log_msg('User', s); + user_input.value = ""; + log_msg('User', input_string); - let tA2 = document.getElementById("system_prompt"); + //let tA2 = document.getElementById("system_prompt"); let acc_text = ""; log.innerHTML += `${get_bot_name()}:`; - for await (let token of ask_question(bot_select.value, s, tA2.value)){ + for await (let token of ask_question(bot_select.value, input_string, system_prompt.value)){ console.log(token); acc_text += "" + token; @@ -635,13 +665,38 @@ } + /* + function play() { + var audio = new Audio('https://interactive-examples.mdn.mozilla.net/media/cc0-audio/t-rex-roar.mp3'); + audio.play(); + } + + */ + + let final_answer = acc_text; console.log(final_answer); + + let extra_s = ""; + let jwt = localStorage.getItem("jwt"); + if(jwt){ + let{file} = await text2speech(jwt, final_answer); + + //autoplay controls + extra_s = ` + `; + + console.log(file); + } + + switch(view_select.value){ case "md": - document.getElementById(answer_count).innerHTML = marked.parse(final_answer); + document.getElementById(answer_count).innerHTML = marked.parse(final_answer) + extra_s; break; case "dot": diff --git a/backend/requirements.txt b/backend/requirements.txt index cbeb33c..2af0653 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -28,11 +28,9 @@ cryptography neo4j - pyttsx3 -