async gen, ui improvements

.gitignore

@@ -1,3 +1,4 @@
 backend/__pycache__/
-
+ollama
+deployment/ollama
 

README.md

@@ -1,30 +1,38 @@
 # Ollama bot
 
-
 After deploy:
 
 ## WebUI for Ollama:
-
 * http://localhost:8888
 * use to install models like llama2, llama3 (https://ollama.com/library)
 
 ## Frontend
-
 * simple FE: http://localhost:5000/
 
+### Stack
+* Gitea actions + Google lighthouse
+* Gitea actions + Playright
+* Nuxt.js + Bootstrap 5
+
 
 ## Backend:
-
 * http://localhost:5000/openapi/swagger
 * http://localhost/backend/openapi/swagger
 
+### Stack
+* FastAPI
+* RabbitMQ/Kafka?
+* OpenSearch
+
 
 ### Push image
 
 ```bash
-sudo docker tag llm-python-backend nucberlin:5123/llm-python-backend
+docker login registry.tobiasweise.dev
+docker-compose push
 
-sudo docker push nucberlin:5123/llm-python-backend
+#sudo docker tag llm-python-backend nucberlin:5123/llm-python-backend
+#sudo docker push nucberlin:5123/llm-python-backend
 ```
 
 ----

backend/Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.12
-
+RUN apt-get update && apt-get install -y firefox-esr
 #RUN curl https://ollama.ai/install.sh | sh
 #RUN ollama run llama2
 
@@ -8,4 +8,5 @@ RUN pip3 install -r requirements.txt
 COPY . .
 
 ENTRYPOINT ["python3", "app.py"]
+#ENTRYPOINT ["fastapi", "run", "main.py", "--port", "8000"]
 

backend/app.py

@@ -214,7 +214,7 @@ for env_key, conf_key in env_to_conf.items():
         app.config[conf_key] = x
 
 
-
+#TODO add history
 
 def ask_bot(question, bot_id):
     bot = Chatbot.get(id=bot_id)
@@ -247,30 +247,18 @@ def sockcon(data):
     socket.emit('backend response', {'msg': f'Connected to room {room} !', "room": room}) # looks like iOS needs an answer
 
 
+#TODO: pydantic message type validation
+
+
 @socket.on('client message')
 def handle_message(message):
 
-    #room = message["room"]
+    #try:
-    #stream_key = "chatbot_stream"
-
-    #llm = Ollama(
-    #    model="llama3",
-    #    base_url="http://ollama:11434"
-    #)
-
-    #system_prompt = ""
-
-    #query = system_prompt + " " + message["data"]
-    #print(message["data"])
-
-    #for chunks in llm.stream(query):
-    #    socket.emit('backend token', {'data': chunks, "done": False}, to=room)
-
-    #socket.emit('backend token', {'done': True}, to=room)
-
     room = message["room"]
     question = message["question"]
     bot_id = message["bot_id"]
+    #except:
+    #    return
 
     for chunk in ask_bot(question, bot_id):
         socket.emit('backend token', {'data': chunk, "done": False}, to=room)
@@ -585,15 +573,13 @@ def get_schema():
 
 @app.route("/") #Index Verzeichnis
 def index():
-    return send_from_directory('.', "index.html")
+    return send_from_directory('./public', "index.html")
 
-#@app.route("/info") #spezielle Nutzer definierte Route
-#def info():
-#    return sys.version+" "+os.getcwd()
 
 @app.route('/<path:path>') #generische Route (auch Unterordner)
 def catchAll(path):
-    return send_from_directory('.', path)
+    #return send_from_directory('.', path)
+    return send_from_directory('./public', path)
 
 
 

backend/main.py

@@ -0,0 +1,126 @@
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from jinja2 import Environment, FileSystemLoader
+from pydantic import BaseModel
+
+from neo4j import GraphDatabase
+
+import os, sys
+from multiprocessing import Pool
+from bs4 import BeautifulSoup
+import requests
+
+
+from webbot import * #Bot, innerHTML
+from xing import *
+
+
+env = Environment(loader=FileSystemLoader('templates'))
+app = FastAPI()
+
+class JobSearch(BaseModel):
+    location: str
+    language: str
+
+
+
+def xing_job_search(location: str, radius: int) -> list:
+    with Bot() as bot:
+        vars_ = {
+            "page": 1,
+            "filter.industry%5B%5D": 90000,
+            "filter.type%5B%5D": "FULL_TIME",
+            "filter.level%5B%5D": 2,
+            "location": location,
+            "radius": radius
+        }
+        start_url = "https://www.xing.com/jobs/search?" + "&".join([k + "=" + str(v) for k, v in vars_.items()])
+
+
+        def kill_cookie_questions():
+            bot.click_id("consent-accept-button")
+
+
+        def next_page():
+            nav = bot.get_elements_by_tag_name("nav")[1]
+            next_site_link = get_elements_by_tag_name(nav, "a")[-1]
+            bot.click(next_site_link)
+
+        def get_nr_pages():
+            nav = bot.get_elements_by_tag_name("nav")[1]
+            return int(get_elements_by_tag_name(nav, "a")[-2].text)
+
+        def get_items():
+            rs = []
+            for article in bot.get_elements_by_tag_name("article"):
+                rs.append( get_children(article)[0].get_attribute("href") )
+            return rs
+
+        return collect_pagination_items(bot, start_url, next_page, get_nr_pages, get_items, kill_cookie_questions)
+
+
+
+
+
+"""
+pwd = "neo4j2"
+proto = "bolt"
+host = "192.168.99.101"
+
+driver = GraphDatabase.driver("%s://%s:7687" % (proto, host), auth=("neo4j", pwd), encrypted=False)
+
+def add_friend(tx, name, friend_name):
+    tx.run("MERGE (a:Person {name: $name}) "
+           "MERGE (a)-[:KNOWS]->(friend:Person {name: $friend_name})",
+           name=name, friend_name=friend_name)
+
+def print_friends(tx, name):
+    for record in tx.run("MATCH (a:Person)-[:KNOWS]->(friend) WHERE a.name = $name "
+                         "RETURN friend.name ORDER BY friend.name", name=name):
+        print(record["friend.name"])
+
+with driver.session() as session:
+    session.write_transaction(add_friend, "Arthur", "Guinevere")
+    session.write_transaction(add_friend, "Arthur", "Lancelot")
+    session.write_transaction(add_friend, "Arthur", "Merlin")
+    session.read_transaction(print_friends, "Arthur")
+
+driver.close()
+"""
+
+
+@app.post("/search")
+def job_search(js: JobSearch):
+
+
+    #https://berlinstartupjobs.com/?s=python&page=3
+
+    location = "Berlin"
+    radius = 50
+
+
+    with Bot() as bot:
+        vars_ = {
+            "page": 1,
+            "filter.industry%5B%5D": 90000,
+            "filter.type%5B%5D": "FULL_TIME",
+            "filter.level%5B%5D": 2,
+            "location": location,
+            "radius": radius
+        }
+        start_url = "https://www.xing.com/jobs/search?" + "&".join([k + "=" + str(v) for k, v in vars_.items()])
+
+        bot.set_url(start_url)
+        return bot.get_page_content()
+
+
+
+
+@app.get("/")
+async def root():
+    template = env.get_template('index.twig')
+    html = template.render()
+    return HTMLResponse(html)
+
+
+

backend/public/favicon.svg

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="91mm"
+   height="91mm"
+   viewBox="0 0 91 91"
+   version="1.1"
+   id="svg5"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   sodipodi:docname="logo.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview7"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:document-units="mm"
+     showgrid="false"
+     inkscape:zoom="0.75098906"
+     inkscape:cx="721.71491"
+     inkscape:cy="134.48931"
+     inkscape:window-width="1896"
+     inkscape:window-height="1022"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1"
+     inkscape:lockguides="false"
+     width="100mm" />
+  <defs
+     id="defs2">
+    <inkscape:perspective
+       sodipodi:type="inkscape:persp3d"
+       inkscape:vp_x="-281.05291 : 147.87572 : 1"
+       inkscape:vp_y="0 : 2002.8478 : 0"
+       inkscape:vp_z="329.34254 : 147.87572 : 1"
+       inkscape:persp3d-origin="24.144846 : 48.734748 : 1"
+       id="perspective14132" />
+    <rect
+       x="81.08667"
+       y="730.20618"
+       width="93.026169"
+       height="105.77888"
+       id="rect4964" />
+  </defs>
+  <g
+     inkscape:label="Ebene 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <ellipse
+       style="fill:#007ec1;fill-opacity:1;fill-rule:evenodd;stroke-width:0.264583"
+       id="path31"
+       cx="45.785828"
+       cy="46.287594"
+       rx="44.359997"
+       ry="44.235622" />
+    <text
+       xml:space="preserve"
+       transform="scale(0.26458333)"
+       id="text4962"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;white-space:pre;shape-inside:url(#rect4964);fill:#000000;fill-opacity:1;stroke:none" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:63.3606px;line-height:1.25;font-family:sans-serif;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1.58402"
+       x="22.167429"
+       y="68.303719"
+       id="text9216"
+       transform="scale(1.001691,0.99831186)"><tspan
+         sodipodi:role="line"
+         id="tspan9214"
+         style="fill:#ffffff;stroke-width:1.58402"
+         x="22.167429"
+         y="68.303719">C</tspan></text>
+  </g>
+</svg>

backend/public/index.html

@@ -3,6 +3,9 @@
 <head>
     <title>Ollama Chatbot</title>
     <meta charset="utf-8">
+    <link rel="icon" href="favicon.svg">
+
+
     <script src="viz.js"></script>
     <script src="viz_widget.js"></script>
     <script src="tabs.js"></script>
@@ -19,12 +22,64 @@
 </head>
 <body>
 
-    <div class="container-fluid p-3 bg-primary text-white text-center">
+    <!-- The Login Modal -->
-      <h1>Ollama Chatbot</h1>
+    <div class="modal fade" id="myModal">
-      <p>Create and talk to chatbots!</p> 
+      <div class="modal-dialog modal-dialog-centered">
+        <div class="modal-content">
+
+          <!-- Login Modal Header -->
+          <div class="modal-header">
+            <h4 class="modal-title">Login to account</h4>
+            <button type="button" class="btn-close" data-bs-dismiss="modal"></button>
+          </div>
+
+          <form>
+
+              <!-- Login Modal body -->
+              <div class="modal-body">
+
+                <div class="mb-3 mt-3">
+                    <label for="email" class="form-label">Email:</label>
+                    <input type="email" class="form-control" id="email" placeholder="Enter email" name="email">
+                </div>
+
+                <div class="mb-3">
+                    <label for="pwd" class="form-label">Password:</label>
+                    <input type="password" class="form-control" id="pass" placeholder="Enter password" name="pswd">
+                </div>
+
+                <div class="form-check">
+                  <input type="checkbox" class="form-check-input" onclick="show_password()">
+                  <label class="form-check-label">Show Password</label>
+                </div> 
+
+                <script>
+                function show_password() {
+                    let ele = document.getElementById("pass");
+                    if(ele.type === "password"){
+                        ele.type = "text";
+                    } else {
+                        ele.type = "password";
+                    }
+                }
+                </script>
+             </div>
+             <!-- Login Modal footer -->
+             <div class="modal-footer">
+                <button id="submit_login_btn" type="button" class="btn btn-primary">Submit</button>
+             </div>
+          </form>
+
+        </div>
+      </div>
     </div>
 
 
+    <div class="container-fluid p-3 bg-primary text-white text-center">
+      <h1>Creative Bots</h1>
+      <p>Create and talk to chatbots!</p> 
+    </div>
+
     <div class="container"> 
 
          <!-- Offcanvas Sidebar -->
@@ -35,8 +90,18 @@
           </div>
           <div class="offcanvas-body">
 
+            <!-- Button to Open the Modal -->
+            <button id="login_btn" type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#myModal">Login</button>
+
+            <!--
             <button id="login_btn" type="button" class="btn btn-primary text-white">Login</button>
+            -->
+
+
             <button id="logout_btn" type="button" class="btn btn-danger text-white">Logout</button>
+
+
+
             <br>
 
             <label for="system_prompt">System prompt:</label>
@@ -44,10 +109,14 @@
 
             <br>
 
+
+            <!--
             <label for="bots">Choose a bot:</label>
             <select name="bots" id="bot_select" class="form-select"></select> 
 
             <br>
+            -->
+
 
             <label for="views">Choose a view:</label>
             <select name="views" id="view_select" class="form-select">
@@ -82,8 +151,37 @@
 
                 <br>
                 <div class="input-group">
-                    <span class="input-group-text">@bot</span>
+                    <span class="input-group-text">@
+                        <select name="bots" id="bot_select" class="form-select"></select> 
+
+                    <!--
+                        <input size="8" class="form-control" list="bot_select" name="bots" placeholder="bot">
+                        <datalist id="bot_select">
+
+                          <option value="Superman">
+
+
+                        </datalist> 
+                    -->
+
+                    </span>
+
+
+
+                    <!--
                     <input id="user_input" type="text" class="form-control" placeholder="What is...">
+                    -->
+
+                    <input class="form-control" list="questions" name="question" id="user_input" placeholder="What is...">
+                    <datalist id="questions">
+                      <option value="Write all the ministries of Germany and their suborganizations in dot lang and return the source code!">
+                      <option value="What is a whale?">
+                      <option value="Is a monad a burito?">
+                      <option value="Give the JSON of a graph linking Germanys 9 biggest cities">
+                    </datalist> 
+
+
+
                     <button id="submit_btn" class="btn btn-success" type="submit">Send</button>
                 </div>
 
@@ -103,6 +201,9 @@
                 <br>
                 <br>
 
+                <form>
+
+
                 <label for="bot_name" class="form-label">Name:</label>
                 <input type="bot_name" class="form-control" id="bot_name" placeholder="MyNewBot">
 
@@ -163,6 +264,8 @@
                 </div>
                 -->
 
+                </form>
+
             </div>
 
         </div>
@@ -198,7 +301,6 @@
     </footer>
 
     <script>
-
     //idea: generate proxy opject via openapi.json   api(url).login_now()
 
     async function login(email, pwd){
@@ -237,8 +339,6 @@
         }
     }
 
-
-
     async function create_bot(jwt, name, visibility, description, llm, sys_prompt){
         const formData = new FormData();
         formData.append("name", name);
@@ -258,101 +358,76 @@
         return response.json();
     }
 
-
+    async function* ask_question(bot_id, question, system_prompt=""){
-    function ask_question(s){
+        let socket;
-        const socket = io();
-
         let room = null;
-        socket.on('backend response', data =>{
+        //let evt_listener = null;
-            console.log(data);
+        let dom_ele = document.head;
-            if(data.room) room = data.room;
+        const evt_name = "tokenstream";
-        });
+        try{
-
+            socket = io();
-
+            socket.on('backend response', data =>{
-
+                console.log(data);
-        let answer_count = 0;
+                if(data.room){
-        let acc_text = "";
+                    room = data.room;
-        let first_token = true;
+                    socket.off('backend response');
-
-        socket.on('backend token', obj =>{
-            console.log(obj);
-            if(first_token){
-                let id = answer_count;
-                log.innerHTML += "<tr><td><b>Bot</b>:</td><td id='" + id + "'></td></tr>";
-            }
-
-            if(!obj.done){
-                acc_text += "" + obj.data;
-                first_token = false;
-                document.getElementById(answer_count).innerHTML += obj.data;
-                scroll_down();
-            }
-            else{
-                //log_msg("Bot", acc_text);
-
-                let final_answer = document.getElementById(answer_count).textContent;
-
-                //alert(final_answer);
-
-                final_answer = final_answer.replace("```", "").replace("```", "");
-
-                switch(view_select.value){
-
-                    case "md":
-                        //document.getElementById(answer_count).innerHTML += obj.data;
-                        document.getElementById(answer_count).innerHTML = marked.parse(final_answer);
-                        break;
-                    case "dot":
-                        //let layout = "fdp";
-                        let layout = "dot";
-                        document.getElementById(answer_count).innerHTML = `<dot-graph layout="${layout}" style="width:100%; height:100%;">${final_answer}</dot-graph>`;
-                        break;
-                    default:
-                        //document.getElementById(answer_count).innerHTML += obj.data;
-                        break;
                 }
+            });
 
-                acc_text = "";
+            let done = false;
-                first_token = true;
+            //let last_timestamp = null;
-                answer_count += 1;
+
-                scroll_down();
+            function f(){
+                return new Promise((resolve,reject)=>{
+                    let evt_listener = evt => {
+                        //if(evt.timeStamp !== last_timestamp){
+                        //last_timestamp = evt.timeStamp;
+                        dom_ele.removeEventListener(evt_name, evt_listener);
+                        resolve(evt.detail);
+                        //}
+                        //last_timestamp = evt.timeStamp;
+                    };
+                    dom_ele.addEventListener(evt_name, evt_listener);
+                });
             }
-        });
 
-
+            socket.on('backend token', obj =>{
-        //send the request
+                if(!obj.done){
-        let tA2 = document.getElementById("system_prompt");
+                    dom_ele.dispatchEvent(new CustomEvent(evt_name, { detail: obj.data }));
-        socket.emit('client message', {
-            question: tA2.value + " " + s,
-            bot_id: bot_select.value,
-            room: room
-        });
-
-
-
-        return {
-            next(){
-                return {
-                    done: false,
-                    value: new Promise((resolve,reject)=>{
-                        resolve(1);
-                    })
                 }
-            },
+                else{
-            [Symbol.iterator]() {
+                    done = true;
-                return this;
+                    socket.off('backend token');
+                }
+            });
+            socket.emit('client message', {
+                question: system_prompt + " " + question,
+                bot_id: bot_id,
+                room: room
+            });
+            while(!done){
+                yield f();
             }
         }
-
+        catch(e){
+            console.error(e);
+        }
+        finally{
+            //socket.off('backend token');
+            //dom_ele.removeEventListener(evt_name, evt_listener);
+            socket.emit('end');
+            socket.close();
+        }
     }
 
+
     window.onload = async ()=>{
         document.documentElement.style.setProperty("--bs-primary-rgb", "45, 124, 172");
 
         //chat
         let tA = document.getElementById("user_input");
         let log = document.getElementById("log");
-        let btn = document.getElementById("submit_btn");
+        let submit_btn = document.getElementById("submit_btn");
         let scroll_div = document.getElementById("scroll_div");
 
         //settings
@@ -360,6 +435,7 @@
         let view_select = document.getElementById("view_select");
         let login_btn = document.getElementById("login_btn");
         let logout_btn = document.getElementById("logout_btn");
+        let submit_login_btn = document.getElementById("submit_login_btn");
 
         //create bot form
         let create_bot_btn = document.getElementById("create_bot_btn");
@@ -368,9 +444,9 @@
         let bot_description = document.getElementById("bot_description");
         let bot_llm_select = document.getElementById("bot_llm_select");
         let bot_system_prompt = document.getElementById("bot_system_prompt");
-
         let alert_spawn = document.getElementById("alert_spawn");
 
+        let answer_count = 0;
 
         function log_msg(nick, msg){
             console.log(nick + ": " + msg);
@@ -388,37 +464,38 @@
         }
 
         function set_bot_list(ls){
-            bot_select.innerHTML = ls.map(bot => `<option value="${bot.id}">${bot.name}</option>`).join("");
+            if(ls.length === 0){
+                console.error("No bots found!");
+            }
+            else{
+                bot_select.innerHTML = ls.map(bot => `<option value="${bot.id}">${bot.name}</option>`).join("");
+            }
         }
 
-
         function clean_bot_create_form(){
             bot_name.value = "";
             bot_description.value = "";
             bot_system_prompt.value = "";
         }
 
-
         function alert_bot_creation(success){
+            let msg, s;
             if(success){
-                alert_spawn.innerHTML = `
+                s = "success";
-                <div class="alert alert-success alert-dismissible fade show">
+                msg = "<strong>Success!</strong> Bot created!";
-                    <button type="button" class="btn-close" data-bs-dismiss="alert"></button>
-                    <strong>Success!</strong> Bot created!
-                </div>
-                `;
             }
             else{
-                alert_spawn.innerHTML = `
+                s = "danger";
-                <div class="alert alert-danger alert-dismissible fade show">
+                msg = "<strong>Couldn't create bot!</strong> Something killed that bot!";
-                    <button type="button" class="btn-close" data-bs-dismiss="alert"></button>
-                    <strong>Couldn't create bot!</strong> Something killed that bot!
-                </div>
-                `;
             }
+            alert_spawn.innerHTML = `
+            <div class="alert alert-${s} alert-dismissible fade show">
+                <button type="button" class="btn-close" data-bs-dismiss="alert"></button>
+                ${msg}
+            </div>
+            `;
         }
 
-
         function set_ui_loggedin(b){
             if(b){
                 //enable create bot button
@@ -438,32 +515,19 @@
         let jwt = localStorage.getItem("jwt");
         if(jwt === null){
             let ls = await get_bots();
-            if(ls.length === 0){
+            set_bot_list(ls);
-                console.error("No bots found!");
-            }
-            else{
-                set_bot_list(ls);
-            }
             set_ui_loggedin(false);
         }
         else{
             let ls = await get_bots(jwt);
-            if(ls.length === 0){
+            set_bot_list(ls);
-                console.error("No bots found!");
-            }
-            else{
-                set_bot_list(ls);
-            }
             set_ui_loggedin(true);
         }
 
+        //init chat
+        log_msg(get_bot_name(), "Ask a question!");
 
-        //for await (let x of new_async_gen()){
+        //-----init buttons------------
-        //    alert(x);
-        //}
-
-
-        //init buttons
         create_bot_btn.onclick = async ()=>{
             let jwt = localStorage.getItem("jwt");
             if(jwt){
@@ -473,10 +537,25 @@
                 let llm = bot_llm_select.value;
                 let sys_prompt = bot_system_prompt.value;
 
+                if(!name){
+                    bot_name.focus();
+                    return;
+                }
+
+                if(!sys_prompt){
+                    bot_system_prompt.focus();
+                    return;
+                }
+
                 try{
                     await create_bot(jwt, name, visibility, description, llm, sys_prompt);
                     alert_bot_creation(true);
                     clean_bot_create_form();
+
+                    //update bot list
+                    let ls = await get_bots(jwt);
+                    set_bot_list(ls);
+
                 }
                 catch(err){
                     console.error(err);
@@ -486,10 +565,26 @@
             }
         };
 
+        submit_login_btn.onclick = async ()=>{
+            //let nick = prompt("Please enter your email");
+            //let pwd = prompt("Please enter your password");
+
+            let nick_ele = document.getElementById("email");
+            let pwd_ele = document.getElementById("pass");
+
+            if(!nick_ele.value){
+                nick_ele.focus();
+                return;
+            }
+
+            if(!pwd_ele.value){
+                pwd_ele.focus();
+                return;
+            }
+
+            let nick = nick_ele.value;
+            let pwd = pwd_ele.value;
 
-        login_btn.onclick = async ()=>{
-            let nick = prompt("Please enter your email");
-            let pwd = prompt("Please enter your password");
 
             try{
                 let{jwt} = await login(nick, pwd);
@@ -497,17 +592,14 @@
                 if(!jwt) throw Error("No JWT!");
 
                 localStorage.setItem("jwt", jwt);
-
                 set_ui_loggedin(true);
 
                 let ls = await get_bots(jwt);
-                if(ls.length === 0){
+                set_bot_list(ls);
-                    console.error("No bots found!");
-                }
-                else{
-                    set_bot_list(ls);
-                }
 
+                let myModalEl = document.querySelector('#myModal');
+                let myModal = bootstrap.Modal.getOrCreateInstance(myModalEl);
+                myModal.hide();
             }
             catch(e){
                 console.error("Login failed!");
@@ -520,53 +612,39 @@
             set_ui_loggedin(false);
 
             let ls = await get_bots();
-            if(ls.length === 0){
+            set_bot_list(ls);
-                console.error("No bots found!");
-            }
-            else{
-                set_bot_list(ls);
-            }
         };
 
+        submit_btn.onclick = async evt =>{
+            let s = tA.value;
+            //if(s.trim() !== '' && room){
+            if(s.trim() !== ''){
+                answer_count += 1;
 
-        //init chat
+                tA.value = "";
-        log_msg(get_bot_name(), "Ask a question!");
+                log_msg('User', s);
 
-        const socket = io();
+                let tA2 = document.getElementById("system_prompt");
+                let acc_text = "";
 
-        let room = null;
+                log.innerHTML += `<tr><td><b>${get_bot_name()}</b>:</td><td id="${answer_count}"></td></tr>`;
-        socket.on('backend response', data =>{
-            console.log(data);
-            if(data.room) room = data.room;
-        });
 
-        let answer_count = 0;
+                for await (let token of ask_question(bot_select.value, s, tA2.value)){
-        let acc_text = "";
+                    console.log(token);
-        let first_token = true;
 
-        socket.on('backend token', obj =>{
+                    acc_text += "" + token;
-            console.log(obj);
+                    //document.getElementById(answer_count).innerHTML += obj.data;
-            if(first_token){
+                    document.getElementById(answer_count).innerHTML = marked.parse(acc_text);
-                let id = answer_count;
+                    scroll_down();
-                log.innerHTML += `<tr><td><b>${get_bot_name()}</b>:</td><td id="${id}"></td></tr>`;
-            }
 
-            if(!obj.done){
+                }
-                acc_text += "" + obj.data;
-                first_token = false;
-                document.getElementById(answer_count).innerHTML += obj.data;
-                scroll_down();
-            }
-            else{
 
                 let final_answer = acc_text;
                 console.log(final_answer);
 
-
                 switch(view_select.value){
 
                     case "md":
-                        //document.getElementById(answer_count).innerHTML += obj.data;
                         document.getElementById(answer_count).innerHTML = marked.parse(final_answer);
                         break;
 
@@ -578,31 +656,14 @@
                         break;
 
                     default:
-                        //document.getElementById(answer_count).innerHTML += obj.data;
+                        document.getElementById(answer_count).innerHTML = `<pre>${final_answer}</pre>`;
                         break;
                 }
 
-                acc_text = "";
+                //answer_count += 1;
-                first_token = true;
-                answer_count += 1;
                 scroll_down();
-            }
-        });
 
-        btn.onclick = evt=>{
-            let s = tA.value;
-            if(s.trim() != '' && room){
-                tA.value = "";
-                log_msg('User', s);
-
-                let tA2 = document.getElementById("system_prompt");
-                socket.emit('client message', {
-                    question: tA2.value + " " + s,
-                    bot_id: bot_select.value,
-                    room: room
-                });
             }
-            scroll_down();
         };
 
     };

backend/requirements.txt

@@ -1,4 +1,9 @@
 
+webdriver_manager
+requests
+selenium
+bs4
+
 elasticsearch
 elasticsearch-dsl
 langchain
@@ -6,6 +11,9 @@ langchain-community
 tiktoken
 
 pydantic
+
+fastapi
+
 Werkzeug
 flask
 Flask-Cors
@@ -18,4 +26,5 @@ python-logging-loki
 pyjwt
 cryptography
 
+neo4j
 

backend/w3.css

@@ -1,361 +0,0 @@
-/* W3.CSS 2.99 Mar 2017 by Jan Egil and Borge Refsnes */
-html{box-sizing:border-box}*,*:before,*:after{box-sizing:inherit}
-/* Extract from normalize.css by Nicolas Gallagher and Jonathan Neal git.io/normalize */
-html{-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}
-article,aside,details,figcaption,figure,footer,header,main,menu,nav,section,summary{display:block}
-audio,canvas,progress,video{display:inline-block}progress{vertical-align:baseline}
-audio:not([controls]){display:none;height:0}[hidden],template{display:none}
-a{background-color:transparent;-webkit-text-decoration-skip:objects}
-a:active,a:hover{outline-width:0}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}
-dfn{font-style:italic}mark{background:#ff0;color:#000}
-small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}
-sub{bottom:-0.25em}sup{top:-0.5em}figure{margin:1em 40px}
-img{border-style:none}svg:not(:root){overflow:hidden}
-code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}
-hr{box-sizing:content-box;height:0;overflow:visible}
-button,input,select,textarea{font:inherit;margin:0}optgroup{font-weight:bold}
-button,input{overflow:visible}button,select{text-transform:none}
-button,html [type=button],[type=reset],[type=submit]{-webkit-appearance:button}
-button::-moz-focus-inner, [type=button]::-moz-focus-inner, [type=reset]::-moz-focus-inner, [type=submit]::-moz-focus-inner{border-style:none;padding:0}
-button:-moz-focusring, [type=button]:-moz-focusring, [type=reset]:-moz-focusring, [type=submit]:-moz-focusring{outline:1px dotted ButtonText}
-fieldset{border:1px solid #c0c0c0;margin:0 2px;padding:.35em .625em .75em}
-legend{color:inherit;display:table;max-width:100%;padding:0;white-space:normal}textarea{overflow:auto}
-[type=checkbox],[type=radio]{padding:0}
-[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}
-[type=search]{-webkit-appearance:textfield;outline-offset:-2px}
-[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}
-::-webkit-input-placeholder{color:inherit;opacity:0.54}
-::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}
-/* End extract */
-html,body{font-family:Verdana,sans-serif;font-size:15px;line-height:1.5}html{overflow-x:hidden}
-h1,h2,h3,h4,h5,h6,.w3-slim,.w3-wide{font-family:"Segoe UI",Arial,sans-serif}
-h1{font-size:36px}h2{font-size:30px}h3{font-size:24px}h4{font-size:20px}h5{font-size:18px}h6{font-size:16px}
-.w3-serif{font-family:"Times New Roman",Times,serif}
-h1,h2,h3,h4,h5,h6{font-weight:400;margin:10px 0}.w3-wide{letter-spacing:4px}
-h1 a,h2 a,h3 a,h4 a,h5 a,h6 a{font-weight:inherit}
-hr{border:0;border-top:1px solid #eee;margin:20px 0}
-img{margin-bottom:-5px}a{color:inherit}
-.w3-image{max-width:100%;height:auto}
-.w3-table,.w3-table-all{border-collapse:collapse;border-spacing:0;width:100%;display:table}
-.w3-table-all{border:1px solid #ccc}
-.w3-bordered tr,.w3-table-all tr{border-bottom:1px solid #ddd}
-.w3-striped tbody tr:nth-child(even){background-color:#f1f1f1}
-.w3-table-all tr:nth-child(odd){background-color:#fff}
-.w3-table-all tr:nth-child(even){background-color:#f1f1f1}
-.w3-hoverable tbody tr:hover,.w3-ul.w3-hoverable li:hover{background-color:#ccc}
-.w3-centered tr th,.w3-centered tr td{text-align:center}
-.w3-table td,.w3-table th,.w3-table-all td,.w3-table-all th{padding:8px 8px;display:table-cell;text-align:left;vertical-align:top}
-.w3-table th:first-child,.w3-table td:first-child,.w3-table-all th:first-child,.w3-table-all td:first-child{padding-left:16px}
-.w3-btn,.w3-btn-block,.w3-button{border:none;display:inline-block;outline:0;padding:6px 16px;vertical-align:middle;overflow:hidden;text-decoration:none!important;color:#fff;background-color:#000;text-align:center;cursor:pointer;white-space:nowrap}
-.w3-btn:hover,.w3-btn-block:hover,.w3-btn-floating:hover,.w3-btn-floating-large:hover{box-shadow:0 8px 16px 0 rgba(0,0,0,0.2),0 6px 20px 0 rgba(0,0,0,0.19)}
-.w3-button{color:#000;background-color:#f1f1f1;padding:8px 16px}.w3-button:hover{color:#000!important;background-color:#ccc!important}
-.w3-btn,.w3-btn-floating,.w3-btn-floating-large,.w3-closenav,.w3-opennav,.w3-btn-block,.w3-button{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}   
-.w3-btn-floating,.w3-btn-floating-large{display:inline-block;text-align:center;color:#fff;background-color:#000;position:relative;overflow:hidden;z-index:1;padding:0;border-radius:50%;cursor:pointer;font-size:24px}
-.w3-btn-floating{width:40px;height:40px;line-height:40px}.w3-btn-floating-large{width:56px;height:56px;line-height:56px}
-.w3-disabled,.w3-btn:disabled,.w3-button:disabled,.w3-btn-floating:disabled,.w3-btn-floating-large:disabled{cursor:not-allowed;opacity:0.3}.w3-disabled *,:disabled *{pointer-events:none}
-.w3-btn.w3-disabled:hover,.w3-btn-block.w3-disabled:hover,.w3-btn:disabled:hover,.w3-btn-floating.w3-disabled:hover,.w3-btn-floating:disabled:hover,
-.w3-btn-floating-large.w3-disabled:hover,.w3-btn-floating-large:disabled:hover{box-shadow:none}
-.w3-btn-group .w3-btn{float:left}.w3-btn-block{width:100%}
-.w3-btn-bar .w3-btn{box-shadow:none;background-color:inherit;color:inherit;float:left}.w3-btn-bar .w3-btn:hover{background-color:#ccc}
-.w3-badge,.w3-tag,.w3-sign{background-color:#000;color:#fff;display:inline-block;padding-left:8px;padding-right:8px;text-align:center}
-.w3-badge{border-radius:50%}
-ul.w3-ul{list-style-type:none;padding:0;margin:0}ul.w3-ul li{padding:6px 2px 6px 16px;border-bottom:1px solid #ddd}ul.w3-ul li:last-child{border-bottom:none}
-.w3-tooltip,.w3-display-container{position:relative}.w3-tooltip .w3-text{display:none}.w3-tooltip:hover .w3-text{display:inline-block}
-.w3-navbar{list-style-type:none;margin:0;padding:0;overflow:hidden}
-.w3-navbar li{float:left}.w3-navbar li a,.w3-navitem,.w3-navbar li .w3-btn,.w3-navbar li .w3-input{display:block;padding:8px 16px}.w3-navbar li .w3-btn,.w3-navbar li .w3-input{border:none;outline:none;width:100%}
-.w3-navbar li a:hover{color:#000;background-color:#ccc}
-.w3-navbar .w3-dropdown-hover,.w3-navbar .w3-dropdown-click{position:static}
-.w3-navbar .w3-dropdown-hover:hover,.w3-navbar .w3-dropdown-hover:first-child,.w3-navbar .w3-dropdown-click:hover{background-color:#ccc;color:#000}
-.w3-navbar a,.w3-topnav a,.w3-sidenav a,.w3-dropdown-content a,.w3-accordion-content a,.w3-dropnav a,.w3-navblock a{text-decoration:none!important}
-.w3-navbar .w3-opennav.w3-right{float:right!important}.w3-topnav{padding:8px 8px}
-.w3-navblock .w3-dropdown-hover:hover,.w3-navblock .w3-dropdown-hover:first-child,.w3-navblock .w3-dropdown-click:hover{background-color:#ccc;color:#000}
-.w3-navblock .w3-dropdown-hover,.w3-navblock .w3-dropdown-click{width:100%}.w3-navblock .w3-dropdown-hover .w3-dropdown-content,.w3-navblock .w3-dropdown-click .w3-dropdown-content{min-width:100%}
-.w3-topnav a{padding:0 8px;border-bottom:3px solid transparent;-webkit-transition:border-bottom .25s;transition:border-bottom .25s}
-.w3-topnav a:hover{border-bottom:3px solid #fff}.w3-topnav .w3-dropdown-hover a{border-bottom:0}
-.w3-opennav,.w3-closenav{color:inherit}.w3-opennav:hover,.w3-closenav:hover{cursor:pointer;opacity:0.8}
-.w3-btn,.w3-btn-floating,.w3-dropnav a,.w3-btn-floating-large,.w3-btn-block, .w3-navbar a,.w3-navblock a,.w3-sidenav a,.w3-pagination li a,.w3-hoverable tbody tr,.w3-hoverable li,
-.w3-accordion-content a,.w3-dropdown-content a,.w3-dropdown-click:hover,.w3-dropdown-hover:hover,.w3-opennav,.w3-closenav,.w3-closebtn,*[class*="w3-hover-"]
-{-webkit-transition:background-color .25s,color .15s,box-shadow .25s,opacity 0.25s,filter 0.25s,border 0.15s;transition:background-color .25s,color .15s,box-shadow .15s,opacity .25s,filter .25s,border .15s}
-.w3-ripple:active{opacity:0.5}.w3-ripple{-webkit-transition:opacity 0s;transition:opacity 0s}
-.w3-sidenav,.w3-sidebar{height:100%;width:200px;background-color:#fff;position:fixed!important;z-index:1;overflow:auto}
-.w3-sidenav a,.w3-navblock a{padding:4px 2px 4px 16px}.w3-sidenav a:hover,.w3-navblock a:hover{background-color:#ccc;color:#000}.w3-sidenav a,.w3-dropnav a,.w3-navblock a{display:block}
-.w3-sidenav .w3-dropdown-hover:hover,.w3-sidenav .w3-dropdown-hover:first-child,.w3-sidenav .w3-dropdown-click:hover,.w3-dropnav a:hover{background-color:#ccc;color:#000}
-.w3-sidenav .w3-dropdown-hover,.w3-sidenav .w3-dropdown-click,.w3-bar-block .w3-dropdown-hover,.w3-bar-block .w3-dropdown-click{width:100%}
-.w3-sidenav .w3-dropdown-hover .w3-dropdown-content,.w3-sidenav .w3-dropdown-click .w3-dropdown-content,.w3-bar-block .w3-dropdown-hover .w3-dropdown-content,.w3-bar-block .w3-dropdown-click .w3-dropdown-content{min-width:100%}
-.w3-bar-block .w3-dropdown-hover .w3-button,.w3-bar-block .w3-dropdown-click .w3-button{width:100%;text-align:left;background-color:inherit;color:inherit;padding:6px 2px 6px 16px}
-.w3-main,#main{transition:margin-left .4s}
-.w3-modal{z-index:3;display:none;padding-top:100px;position:fixed;left:0;top:0;width:100%;height:100%;overflow:auto;background-color:rgb(0,0,0);background-color:rgba(0,0,0,0.4)}
-.w3-modal-content{margin:auto;background-color:#fff;position:relative;padding:0;outline:0;width:600px}.w3-closebtn{text-decoration:none;float:right;font-size:24px;font-weight:bold;color:inherit}
-.w3-closebtn:hover,.w3-closebtn:focus{color:#000;text-decoration:none;cursor:pointer}
-.w3-pagination{display:inline-block;padding:0;margin:0}.w3-pagination li{display:inline}
-.w3-pagination li a{text-decoration:none;color:#000;float:left;padding:8px 16px}
-.w3-pagination li a:hover{background-color:#ccc}
-.w3-input-group,.w3-group{margin-top:24px;margin-bottom:24px}
-.w3-input{padding:8px;display:block;border:none;border-bottom:1px solid #808080;width:100%}
-.w3-label{color:#009688}.w3-input:not(:valid)~.w3-validate{color:#f44336}
-.w3-select{padding:9px 0;width:100%;color:#000;border:1px solid transparent;border-bottom:1px solid #009688}
-.w3-select select:focus{color:#000;border:1px solid #009688}.w3-select option[disabled]{color:#009688}
-.w3-dropdown-click,.w3-dropdown-hover{position:relative;display:inline-block;cursor:pointer}
-.w3-dropdown-hover:hover .w3-dropdown-content{display:block;z-index:1}
-.w3-dropdown-hover:first-child,.w3-dropdown-click:hover{background-color:#ccc;color:#000}
-.w3-dropdown-hover:hover > .w3-button:first-child,.w3-dropdown-click:hover > .w3-button:first-child{background-color:#ccc;color:#000}
-.w3-dropdown-content{cursor:auto;color:#000;background-color:#fff;display:none;position:absolute;min-width:160px;margin:0;padding:0}
-.w3-dropdown-content a{padding:6px 16px;display:block}
-.w3-dropdown-content a:hover{background-color:#ccc}
-.w3-accordion{width:100%;cursor:pointer}
-.w3-accordion-content{cursor:auto;display:none;position:relative;width:100%;margin:0;padding:0}
-.w3-accordion-content a{padding:6px 16px;display:block}.w3-accordion-content a:hover{background-color:#ccc}
-.w3-progress-container{width:100%;height:1.5em;position:relative;background-color:#f1f1f1}
-.w3-progressbar{background-color:#757575;height:100%;position:absolute;line-height:inherit}
-input[type=checkbox].w3-check,input[type=radio].w3-radio{width:24px;height:24px;position:relative;top:6px}
-input[type=checkbox].w3-check:checked+.w3-validate,input[type=radio].w3-radio:checked+.w3-validate{color:#009688} 
-input[type=checkbox].w3-check:disabled+.w3-validate,input[type=radio].w3-radio:disabled+.w3-validate{color:#aaa}
-.w3-bar{width:100%;overflow:hidden}.w3-center .w3-bar{display:inline-block;width:auto}
-.w3-bar .w3-bar-item{padding:8px 16px;float:left;background-color:inherit;color:inherit;width:auto;border:none;outline:none;display:block}
-.w3-bar .w3-dropdown-hover,.w3-bar .w3-dropdown-click{position:static;float:left}
-.w3-bar .w3-button{background-color:inherit;color:inherit;white-space:normal}
-.w3-bar-block .w3-bar-item{width:100%;display:block;padding:8px 16px;text-align:left;background-color:inherit;color:inherit;border:none;outline:none;white-space:normal}
-.w3-bar-block.w3-center .w3-bar-item{text-align:center}
-.w3-block{display:block;width:100%}
-.w3-responsive{overflow-x:auto}
-.w3-container:after,.w3-container:before,.w3-panel:after,.w3-panel:before,.w3-row:after,.w3-row:before,.w3-row-padding:after,.w3-row-padding:before,.w3-cell-row:before,.w3-cell-row:after,
-.w3-topnav:after,.w3-topnav:before,.w3-clear:after,.w3-clear:before,.w3-btn-group:before,.w3-btn-group:after,.w3-btn-bar:before,.w3-btn-bar:after,.w3-bar:before,.w3-bar:after
-{content:"";display:table;clear:both}
-.w3-col,.w3-half,.w3-third,.w3-twothird,.w3-threequarter,.w3-quarter{float:left;width:100%}
-.w3-col.s1{width:8.33333%}
-.w3-col.s2{width:16.66666%}
-.w3-col.s3{width:24.99999%}
-.w3-col.s4{width:33.33333%}
-.w3-col.s5{width:41.66666%}
-.w3-col.s6{width:49.99999%}
-.w3-col.s7{width:58.33333%}
-.w3-col.s8{width:66.66666%}
-.w3-col.s9{width:74.99999%}
-.w3-col.s10{width:83.33333%}
-.w3-col.s11{width:91.66666%}
-.w3-col.s12,.w3-half,.w3-third,.w3-twothird,.w3-threequarter,.w3-quarter{width:99.99999%}
-@media (min-width:601px){
-.w3-col.m1{width:8.33333%}
-.w3-col.m2{width:16.66666%}
-.w3-col.m3,.w3-quarter{width:24.99999%}
-.w3-col.m4,.w3-third{width:33.33333%}
-.w3-col.m5{width:41.66666%}
-.w3-col.m6,.w3-half{width:49.99999%}
-.w3-col.m7{width:58.33333%}
-.w3-col.m8,.w3-twothird{width:66.66666%}
-.w3-col.m9,.w3-threequarter{width:74.99999%}
-.w3-col.m10{width:83.33333%}
-.w3-col.m11{width:91.66666%}
-.w3-col.m12{width:99.99999%}}
-@media (min-width:993px){
-.w3-col.l1{width:8.33333%}
-.w3-col.l2{width:16.66666%}
-.w3-col.l3,.w3-quarter{width:24.99999%}
-.w3-col.l4,.w3-third{width:33.33333%}
-.w3-col.l5{width:41.66666%}
-.w3-col.l6,.w3-half{width:49.99999%}
-.w3-col.l7{width:58.33333%}
-.w3-col.l8,.w3-twothird{width:66.66666%}
-.w3-col.l9,.w3-threequarter{width:74.99999%}
-.w3-col.l10{width:83.33333%}
-.w3-col.l11{width:91.66666%}
-.w3-col.l12{width:99.99999%}}
-.w3-content{max-width:980px;margin:auto}
-.w3-rest{overflow:hidden}
-.w3-layout-container,.w3-cell-row{display:table;width:100%}.w3-layout-row{display:table-row}.w3-layout-cell,.w3-layout-col,.w3-cell{display:table-cell}
-.w3-layout-top,.w3-cell-top{vertical-align:top}.w3-layout-middle,.w3-cell-middle{vertical-align:middle}.w3-layout-bottom,.w3-cell-bottom{vertical-align:bottom}
-.w3-hide{display:none!important}.w3-show-block,.w3-show{display:block!important}.w3-show-inline-block{display:inline-block!important}
-@media (max-width:600px){.w3-modal-content{margin:0 10px;width:auto!important}.w3-modal{padding-top:30px}
-.w3-topnav a{display:block}.w3-navbar li:not(.w3-opennav){float:none;width:100%!important}.w3-navbar li.w3-right{float:none!important}
-.w3-topnav .w3-dropdown-hover .w3-dropdown-content,.w3-navbar .w3-dropdown-click .w3-dropdown-content,.w3-navbar .w3-dropdown-hover .w3-dropdown-content,.w3-dropdown-hover.w3-mobile .w3-dropdown-content,.w3-dropdown-click.w3-mobile .w3-dropdown-content{position:relative}	
-.w3-topnav,.w3-navbar{text-align:center}.w3-hide-small{display:none!important}.w3-layout-col,.w3-mobile{display:block;width:100%!important}.w3-bar-item.w3-mobile,.w3-dropdown-hover.w3-mobile,.w3-dropdown-click.w3-mobile{text-align:center}
-.w3-dropdown-hover.w3-mobile,.w3-dropdown-hover.w3-mobile .w3-btn,.w3-dropdown-hover.w3-mobile .w3-button,.w3-dropdown-click.w3-mobile,.w3-dropdown-click.w3-mobile .w3-btn,.w3-dropdown-click.w3-mobile .w3-button{width:100%}}
-@media (max-width:768px){.w3-modal-content{width:500px}.w3-modal{padding-top:50px}}
-@media (min-width:993px){.w3-modal-content{width:900px}.w3-hide-large{display:none!important}.w3-sidenav.w3-collapse,.w3-sidebar.w3-collapse{display:block!important}}
-@media (max-width:992px) and (min-width:601px){.w3-hide-medium{display:none!important}}
-@media (max-width:992px){.w3-sidenav.w3-collapse,.w3-sidebar.w3-collapse{display:none}.w3-main{margin-left:0!important;margin-right:0!important}}
-.w3-top,.w3-bottom{position:fixed;width:100%;z-index:1}.w3-top{top:0}.w3-bottom{bottom:0}
-.w3-overlay{position:fixed;display:none;width:100%;height:100%;top:0;left:0;right:0;bottom:0;background-color:rgba(0,0,0,0.5);z-index:2}
-.w3-left{float:left!important}.w3-right{float:right!important}
-.w3-tiny{font-size:10px!important}.w3-small{font-size:12px!important}
-.w3-medium{font-size:15px!important}.w3-large{font-size:18px!important}
-.w3-xlarge{font-size:24px!important}.w3-xxlarge{font-size:36px!important}
-.w3-xxxlarge{font-size:48px!important}.w3-jumbo{font-size:64px!important}
-.w3-vertical{word-break:break-all;line-height:1;text-align:center;width:0.6em}
-.w3-left-align{text-align:left!important}.w3-right-align{text-align:right!important}
-.w3-justify{text-align:justify!important}.w3-center{text-align:center!important}
-.w3-display-topleft{position:absolute;left:0;top:0}.w3-display-topright{position:absolute;right:0;top:0}
-.w3-display-bottomleft{position:absolute;left:0;bottom:0}.w3-display-bottomright{position:absolute;right:0;bottom:0}
-.w3-display-middle{position:absolute;top:50%;left:50%;transform:translate(-50%,-50%);-ms-transform:translate(-50%,-50%)}
-.w3-display-left{position:absolute;top:50%;left:0%;transform:translate(0%,-50%);-ms-transform:translate(-0%,-50%)}
-.w3-display-right{position:absolute;top:50%;right:0%;transform:translate(0%,-50%);-ms-transform:translate(0%,-50%)}
-.w3-display-topmiddle{position:absolute;left:50%;top:0;transform:translate(-50%,0%);-ms-transform:translate(-50%,0%)}
-.w3-display-bottommiddle{position:absolute;left:50%;bottom:0;transform:translate(-50%,0%);-ms-transform:translate(-50%,0%)}
-.w3-display-container:hover .w3-display-hover{display:block}.w3-display-container:hover span.w3-display-hover{display:inline-block}.w3-display-hover{display:none}
-.w3-display-position{position:absolute}
-.w3-circle{border-radius:50%!important}
-.w3-round-small{border-radius:2px!important}.w3-round,.w3-round-medium{border-radius:4px!important}
-.w3-round-large{border-radius:8px!important}.w3-round-xlarge{border-radius:16px!important}
-.w3-round-xxlarge{border-radius:32px!important}.w3-round-jumbo{border-radius:64px!important}
-.w3-border-0{border:0!important}.w3-border{border:1px solid #ccc!important}
-.w3-border-top{border-top:1px solid #ccc!important}.w3-border-bottom{border-bottom:1px solid #ccc!important}
-.w3-border-left{border-left:1px solid #ccc!important}.w3-border-right{border-right:1px solid #ccc!important}
-.w3-margin{margin:16px!important}.w3-margin-0{margin:0!important}
-.w3-margin-top{margin-top:16px!important}.w3-margin-bottom{margin-bottom:16px!important}
-.w3-margin-left{margin-left:16px!important}.w3-margin-right{margin-right:16px!important}
-.w3-section{margin-top:16px!important;margin-bottom:16px!important}
-.w3-padding-tiny{padding:2px 4px!important}.w3-padding-small{padding:4px 8px!important}
-.w3-padding-medium,.w3-padding,.w3-form{padding:8px 16px!important}
-.w3-padding-large{padding:12px 24px!important}.w3-padding-xlarge{padding:16px 32px!important}
-.w3-padding-xxlarge{padding:24px 48px!important}.w3-padding-jumbo{padding:32px 64px!important}
-.w3-padding-4{padding-top:4px!important;padding-bottom:4px!important}
-.w3-padding-8{padding-top:8px!important;padding-bottom:8px!important}
-.w3-padding-12{padding-top:12px!important;padding-bottom:12px!important}
-.w3-padding-16{padding-top:16px!important;padding-bottom:16px!important}
-.w3-padding-24{padding-top:24px!important;padding-bottom:24px!important}
-.w3-padding-32{padding-top:32px!important;padding-bottom:32px!important}
-.w3-padding-48{padding-top:48px!important;padding-bottom:48px!important}
-.w3-padding-64{padding-top:64px!important;padding-bottom:64px!important}
-.w3-padding-128{padding-top:128px!important;padding-bottom:128px!important}
-.w3-padding-0{padding:0!important}
-.w3-padding-top{padding-top:8px!important}.w3-padding-bottom{padding-bottom:8px!important}
-.w3-padding-left{padding-left:16px!important}.w3-padding-right{padding-right:16px!important}
-.w3-topbar{border-top:6px solid #ccc!important}.w3-bottombar{border-bottom:6px solid #ccc!important}
-.w3-leftbar{border-left:6px solid #ccc!important}.w3-rightbar{border-right:6px solid #ccc!important}
-.w3-row-padding,.w3-row-padding>.w3-half,.w3-row-padding>.w3-third,.w3-row-padding>.w3-twothird,.w3-row-padding>.w3-threequarter,.w3-row-padding>.w3-quarter,.w3-row-padding>.w3-col{padding:0 8px}
-.w3-spin{animation:w3-spin 2s infinite linear;-webkit-animation:w3-spin 2s infinite linear}
-@-webkit-keyframes w3-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}
-@keyframes w3-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}
-.w3-container{padding:0.01em 16px}
-.w3-panel{padding:0.01em 16px;margin-top:16px!important;margin-bottom:16px!important}
-.w3-example{background-color:#f1f1f1;padding:0.01em 16px}
-.w3-code,.w3-codespan{font-family:Consolas,"courier new";font-size:16px}
-.w3-code{line-height:1.4;width:auto;background-color:#fff;padding:8px 12px;border-left:4px solid #4CAF50;word-wrap:break-word}
-.w3-codespan{color:crimson;background-color:#f1f1f1;padding-left:4px;padding-right:4px;font-size:110%}
-.w3-example,.w3-code{margin:20px 0}.w3-card{border:1px solid #ccc}
-.w3-card-2,.w3-example{box-shadow:0 2px 4px 0 rgba(0,0,0,0.16),0 2px 10px 0 rgba(0,0,0,0.12)!important}
-.w3-card-4,.w3-hover-shadow:hover{box-shadow:0 4px 8px 0 rgba(0,0,0,0.2),0 6px 20px 0 rgba(0,0,0,0.19)!important}
-.w3-card-8{box-shadow:0 8px 16px 0 rgba(0,0,0,0.2),0 6px 20px 0 rgba(0,0,0,0.19)!important}
-.w3-card-12{box-shadow:0 12px 16px 0 rgba(0,0,0,0.24),0 17px 50px 0 rgba(0,0,0,0.19)!important}
-.w3-card-16{box-shadow:0 16px 24px 0 rgba(0,0,0,0.22),0 25px 55px 0 rgba(0,0,0,0.21)!important}
-.w3-card-24{box-shadow:0 24px 24px 0 rgba(0,0,0,0.2),0 40px 77px 0 rgba(0,0,0,0.22)!important}
-.w3-animate-fading{-webkit-animation:fading 10s infinite;animation:fading 10s infinite}
-@-webkit-keyframes fading{0%{opacity:0}50%{opacity:1}100%{opacity:0}}
-@keyframes fading{0%{opacity:0}50%{opacity:1}100%{opacity:0}}
-.w3-animate-opacity{-webkit-animation:opac 0.8s;animation:opac 0.8s}
-@-webkit-keyframes opac{from{opacity:0} to{opacity:1}}
-@keyframes opac{from{opacity:0} to{opacity:1}}
-.w3-animate-top{position:relative;-webkit-animation:animatetop 0.4s;animation:animatetop 0.4s}
-@-webkit-keyframes animatetop{from{top:-300px;opacity:0} to{top:0;opacity:1}}
-@keyframes animatetop{from{top:-300px;opacity:0} to{top:0;opacity:1}}
-.w3-animate-left{position:relative;-webkit-animation:animateleft 0.4s;animation:animateleft 0.4s}
-@-webkit-keyframes animateleft{from{left:-300px;opacity:0} to{left:0;opacity:1}}
-@keyframes animateleft{from{left:-300px;opacity:0} to{left:0;opacity:1}}
-.w3-animate-right{position:relative;-webkit-animation:animateright 0.4s;animation:animateright 0.4s}
-@-webkit-keyframes animateright{from{right:-300px;opacity:0} to{right:0;opacity:1}}
-@keyframes animateright{from{right:-300px;opacity:0} to{right:0;opacity:1}}
-.w3-animate-bottom{position:relative;-webkit-animation:animatebottom 0.4s;animation:animatebottom 0.4s}
-@-webkit-keyframes animatebottom{from{bottom:-300px;opacity:0} to{bottom:0px;opacity:1}}
-@keyframes animatebottom{from{bottom:-300px;opacity:0} to{bottom:0;opacity:1}}
-.w3-animate-zoom {-webkit-animation:animatezoom 0.6s;animation:animatezoom 0.6s}
-@-webkit-keyframes animatezoom{from{-webkit-transform:scale(0)} to{-webkit-transform:scale(1)}}
-@keyframes animatezoom{from{transform:scale(0)} to{transform:scale(1)}}
-.w3-animate-input{-webkit-transition:width 0.4s ease-in-out;transition:width 0.4s ease-in-out}.w3-animate-input:focus{width:100%!important}
-.w3-opacity,.w3-hover-opacity:hover{opacity:0.60;-webkit-backface-visibility:hidden}
-.w3-opacity-off,.w3-hover-opacity-off:hover{opacity:1;-webkit-backface-visibility:hidden}
-.w3-opacity-max{opacity:0.25;-webkit-backface-visibility:hidden}
-.w3-opacity-min{opacity:0.75;-webkit-backface-visibility:hidden}
-.w3-greyscale-max,.w3-grayscale-max,.w3-hover-greyscale:hover,.w3-hover-grayscale:hover{-webkit-filter:grayscale(100%);filter:grayscale(100%)}
-.w3-greyscale,.w3-grayscale{-webkit-filter:grayscale(75%);filter:grayscale(75%)}
-.w3-greyscale-min,.w3-grayscale-min{-webkit-filter:grayscale(50%);filter:grayscale(50%)}
-.w3-sepia{-webkit-filter:sepia(75%);filter:sepia(75%)}
-.w3-sepia-max,.w3-hover-sepia:hover{-webkit-filter:sepia(100%);filter:sepia(100%)}
-.w3-sepia-min{-webkit-filter:sepia(50%);filter:sepia(50%)}
-.w3-text-shadow{text-shadow:1px 1px 0 #444}.w3-text-shadow-white{text-shadow:1px 1px 0 #ddd}
-.w3-transparent{background-color:transparent!important}
-.w3-hover-none:hover{box-shadow:none!important;background-color:transparent!important}
-/* Colors */
-.w3-amber,.w3-hover-amber:hover{color:#000!important;background-color:#ffc107!important}
-.w3-aqua,.w3-hover-aqua:hover{color:#000!important;background-color:#00ffff!important}
-.w3-blue,.w3-hover-blue:hover{color:#fff!important;background-color:#2196F3!important}
-.w3-light-blue,.w3-hover-light-blue:hover{color:#000!important;background-color:#87CEEB!important}
-.w3-brown,.w3-hover-brown:hover{color:#fff!important;background-color:#795548!important}
-.w3-cyan,.w3-hover-cyan:hover{color:#000!important;background-color:#00bcd4!important}
-.w3-blue-grey,.w3-hover-blue-grey:hover,.w3-blue-gray,.w3-hover-blue-gray:hover{color:#fff!important;background-color:#607d8b!important}
-.w3-green,.w3-hover-green:hover{color:#fff!important;background-color:#4CAF50!important}
-.w3-light-green,.w3-hover-light-green:hover{color:#000!important;background-color:#8bc34a!important}
-.w3-indigo,.w3-hover-indigo:hover{color:#fff!important;background-color:#3f51b5!important}
-.w3-khaki,.w3-hover-khaki:hover{color:#000!important;background-color:#f0e68c!important}
-.w3-lime,.w3-hover-lime:hover{color:#000!important;background-color:#cddc39!important}
-.w3-orange,.w3-hover-orange:hover{color:#000!important;background-color:#ff9800!important}
-.w3-deep-orange,.w3-hover-deep-orange:hover{color:#fff!important;background-color:#ff5722!important}
-.w3-pink,.w3-hover-pink:hover{color:#fff!important;background-color:#e91e63!important}
-.w3-purple,.w3-hover-purple:hover{color:#fff!important;background-color:#9c27b0!important}
-.w3-deep-purple,.w3-hover-deep-purple:hover{color:#fff!important;background-color:#673ab7!important}
-.w3-red,.w3-hover-red:hover{color:#fff!important;background-color:#f44336!important}
-.w3-sand,.w3-hover-sand:hover{color:#000!important;background-color:#fdf5e6!important}
-.w3-teal,.w3-hover-teal:hover{color:#fff!important;background-color:#009688!important}
-.w3-yellow,.w3-hover-yellow:hover{color:#000!important;background-color:#ffeb3b!important}
-.w3-white,.w3-hover-white:hover{color:#000!important;background-color:#fff!important}
-.w3-black,.w3-hover-black:hover{color:#fff!important;background-color:#000!important}
-.w3-grey,.w3-hover-grey:hover,.w3-gray,.w3-hover-gray:hover{color:#000!important;background-color:#9e9e9e!important}
-.w3-light-grey,.w3-hover-light-grey:hover,.w3-light-gray,.w3-hover-light-gray:hover{color:#000!important;background-color:#f1f1f1!important}
-.w3-dark-grey,.w3-hover-dark-grey:hover,.w3-dark-gray,.w3-hover-dark-gray:hover{color:#fff!important;background-color:#616161!important}
-.w3-pale-red,.w3-hover-pale-red:hover{color:#000!important;background-color:#ffdddd!important}
-.w3-pale-green,.w3-hover-pale-green:hover{color:#000!important;background-color:#ddffdd!important}
-.w3-pale-yellow,.w3-hover-pale-yellow:hover{color:#000!important;background-color:#ffffcc!important}
-.w3-pale-blue,.w3-hover-pale-blue:hover{color:#000!important;background-color:#ddffff!important}
-.w3-text-amber,.w3-hover-text-amber:hover{color:#ffc107!important}
-.w3-text-aqua,.w3-hover-text-aqua:hover{color:#00ffff!important}
-.w3-text-blue,.w3-hover-text-blue:hover{color:#2196F3!important}
-.w3-text-light-blue,.w3-hover-text-light-blue:hover{color:#87CEEB!important}
-.w3-text-brown,.w3-hover-text-brown:hover{color:#795548!important}
-.w3-text-cyan,.w3-hover-text-cyan:hover{color:#00bcd4!important}
-.w3-text-blue-grey,.w3-hover-text-blue-grey:hover,.w3-text-blue-gray,.w3-hover-text-blue-gray:hover{color:#607d8b!important}
-.w3-text-green,.w3-hover-text-green:hover{color:#4CAF50!important}
-.w3-text-light-green,.w3-hover-text-light-green:hover{color:#8bc34a!important}
-.w3-text-indigo,.w3-hover-text-indigo:hover{color:#3f51b5!important}
-.w3-text-khaki,.w3-hover-text-khaki:hover{color:#b4aa50!important}
-.w3-text-lime,.w3-hover-text-lime:hover{color:#cddc39!important}
-.w3-text-orange,.w3-hover-text-orange:hover{color:#ff9800!important}
-.w3-text-deep-orange,.w3-hover-text-deep-orange:hover{color:#ff5722!important}
-.w3-text-pink,.w3-hover-text-pink:hover{color:#e91e63!important}
-.w3-text-purple,.w3-hover-text-purple:hover{color:#9c27b0!important}
-.w3-text-deep-purple,.w3-hover-text-deep-purple:hover{color:#673ab7!important}
-.w3-text-red,.w3-hover-text-red:hover{color:#f44336!important}
-.w3-text-sand,.w3-hover-text-sand:hover{color:#fdf5e6!important}
-.w3-text-teal,.w3-hover-text-teal:hover{color:#009688!important}
-.w3-text-yellow,.w3-hover-text-yellow:hover{color:#d2be0e!important}
-.w3-text-white,.w3-hover-text-white:hover{color:#fff!important}
-.w3-text-black,.w3-hover-text-black:hover{color:#000!important}
-.w3-text-grey,.w3-hover-text-grey:hover,.w3-text-gray,.w3-hover-text-gray:hover{color:#757575!important}
-.w3-text-light-grey,.w3-hover-text-light-grey:hover,.w3-text-light-gray,.w3-hover-text-light-gray:hover{color:#f1f1f1!important}
-.w3-text-dark-grey,.w3-hover-text-dark-grey:hover,.w3-text-dark-gray,.w3-hover-text-dark-gray:hover{color:#3a3a3a!important}
-.w3-border-amber,.w3-hover-border-amber:hover{border-color:#ffc107!important}
-.w3-border-aqua,.w3-hover-border-aqua:hover{border-color:#00ffff!important}
-.w3-border-blue,.w3-hover-border-blue:hover{border-color:#2196F3!important}
-.w3-border-light-blue,.w3-hover-border-light-blue:hover{border-color:#87CEEB!important}
-.w3-border-brown,.w3-hover-border-brown:hover{border-color:#795548!important}
-.w3-border-cyan,.w3-hover-border-cyan:hover{border-color:#00bcd4!important}
-.w3-border-blue-grey,.w3-hover-border-blue-grey:hover,.w3-border-blue-gray,.w3-hover-border-blue-gray:hover{border-color:#607d8b!important}
-.w3-border-green,.w3-hover-border-green:hover{border-color:#4CAF50!important}
-.w3-border-light-green,.w3-hover-border-light-green:hover{border-color:#8bc34a!important}
-.w3-border-indigo,.w3-hover-border-indigo:hover{border-color:#3f51b5!important}
-.w3-border-khaki,.w3-hover-border-khaki:hover{border-color:#f0e68c!important}
-.w3-border-lime,.w3-hover-border-lime:hover{border-color:#cddc39!important}
-.w3-border-orange,.w3-hover-border-orange:hover{border-color:#ff9800!important}
-.w3-border-deep-orange,.w3-hover-border-deep-orange:hover{border-color:#ff5722!important}
-.w3-border-pink,.w3-hover-border-pink:hover{border-color:#e91e63!important}
-.w3-border-purple,.w3-hover-border-purple:hover{border-color:#9c27b0!important}
-.w3-border-deep-purple,.w3-hover-border-deep-purple:hover{border-color:#673ab7!important}
-.w3-border-red,.w3-hover-border-red:hover{border-color:#f44336!important}
-.w3-border-sand,.w3-hover-border-sand:hover{border-color:#fdf5e6!important}
-.w3-border-teal,.w3-hover-border-teal:hover{border-color:#009688!important}
-.w3-border-yellow,.w3-hover-border-yellow:hover{border-color:#ffeb3b!important}
-.w3-border-white,.w3-hover-border-white:hover{border-color:#fff!important}
-.w3-border-black,.w3-hover-border-black:hover{border-color:#000!important}
-.w3-border-grey,.w3-hover-border-grey:hover,.w3-border-gray,.w3-hover-border-gray:hover{border-color:#9e9e9e!important}
-.w3-border-light-grey,.w3-hover-border-light-grey:hover,.w3-border-light-gray,.w3-hover-border-light-gray:hover{border-color:#f1f1f1!important}
-.w3-border-dark-grey,.w3-hover-border-dark-grey:hover,.w3-border-dark-gray,.w3-hover-border-dark-gray:hover{border-color:#616161!important}
-.w3-border-pale-red,.w3-hover-border-pale-red:hover{border-color:#ffe7e7!important}.w3-border-pale-green,.w3-hover-border-pale-green:hover{border-color:#e7ffe7!important}
-.w3-border-pale-yellow,.w3-hover-border-pale-yellow:hover{border-color:#ffffcc!important}.w3-border-pale-blue,.w3-hover-border-pale-blue:hover{border-color:#e7ffff!important}

backend/webbot.py

@@ -0,0 +1,191 @@
+import os
+
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+#from selenium.webdriver.chrome.options import Options
+
+
+from webdriver_manager.firefox import GeckoDriverManager
+from selenium.webdriver.firefox.service import Service as FirefoxService
+
+
+from tempfile import mkdtemp
+from time import sleep
+from bs4 import BeautifulSoup
+
+#element = driver.find_element_by_xpath("//div[@class='blockUI blockOverlay']")
+#wait.until(EC.invisibility_of_element_located((By.XPATH, "//div[@class='blockUI blockOverlay']")))
+#ele = WebDriverWait(browser, 10).until(
+#    EC.presence_of_element_located((By.ID, "myDynamicElement"))
+#)
+#print( browser.title )
+#ele = browser.find_element_by_css_selector(".myclass")
+#ele.get_attribute("href")
+#ele.send_keys("test")
+#ele.send_keys(Keys.RETURN)
+#ele.click()
+
+def innerHTML(element):
+    """
+    Returns the inner HTML of an element as a UTF-8 encoded bytestring
+    """
+    return element.encode_contents()
+
+def get_elements_by_tag_name(ele, tag_name):
+    return ele.find_elements(By.TAG_NAME, tag_name)
+
+def get_children(ele):
+    return ele.find_elements(By.XPATH, "./child::*")
+
+
+class Bot:
+
+    def __init__(self, display=False):
+        self.__display = display
+        self.__current_url = None
+
+    def __enter__(self):
+        if not self.__display:
+            os.environ['MOZ_HEADLESS'] = '1'
+
+        #firefox_executable_path = '/usr/local/bin/geckodriver'
+        #firefox_service = webdriver.firefox.service.Service()
+        #options = webdriver.FirefoxOptions()
+        #driver = webdriver.Firefox(service=firefox_service, options=firefox_options)
+
+        service = FirefoxService(executable_path=GeckoDriverManager().install())
+
+        self.__browser = webdriver.Firefox(service=service)
+        self.__browser.implicitly_wait(5000)
+        return self
+
+    def __exit__(self, *args):
+        #driver.quit()
+        self.__browser.close()
+
+    def click(self, ele):
+        if self.__current_url is None:
+            raise Exception("No URL set! No DOM to affect!")
+        self.__browser.execute_script("arguments[0].click()", ele)
+
+    def click_id(self, id):
+        self.click(self.__browser.find_element("id", id))
+
+    def set_url(self, url):
+        self.__browser.get(url)
+        self.__current_url = url
+
+    #DOM methods
+    def get_elements_by_class_name(self, cls_name):
+        return self.__browser.find_elements(By.CLASS_NAME, cls_name)
+
+    def get_elements_by_tag_name(self, tag_name):
+        return self.__browser.find_elements(By.TAG_NAME, tag_name)
+
+    def get_elements_by_xpath(self, path):
+        return self.__browser.find_elements(By.XPATH, path)
+
+
+    def get_page_content(self):
+        #WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, 'body')))
+        return self.__browser.execute_script("return document.documentElement.innerHTML")
+
+
+    def get_page_soup(self):
+        return BeautifulSoup(self.get_page_content(), "html.parser")
+
+
+    def sleep(self, t):
+        sleep(t)
+
+    def scroll(self, d=250):
+        #if self.__current_url is None:
+        #    raise Exception("No URL set! No DOM to affect!")
+        self.__browser.execute_script(f"window.scrollBy(0,{d})")
+
+
+
+
+
+
+def collect_pagination_items(bot, start_url, next_page, get_nr_pages, get_items, kill_cookie_questions=lambda: None):
+    """
+    Collect all the content of a pagination
+    """
+    bot.set_url(start_url)
+    kill_cookie_questions()
+    bot.sleep(4)
+    nr_pages = get_nr_pages()
+    bot.sleep(2)
+    results = []
+    for page_nr in range(nr_pages):
+        #print("Page %s..." % (page_nr + 1))
+        for item in get_items():
+            results.append(item)
+        bot.sleep(0.5)
+        next_page()
+        bot.sleep(2)
+
+    return results
+
+
+
+
+
+"""
+#ele.text
+#ele = browser.find_element_by_id("")
+ele = browser.find_element_by_name("s")
+#ele = browser.find_element_by_css_selector(".myclass")
+#ele.get_attribute("href")
+
+print(ele)
+
+ele.send_keys("test")
+ele.send_keys(Keys.RETURN)
+
+
+
+
+def getKeiserHeadlines():
+    url = "http://maxkeiser.com/"
+    soup = BeautifulSoup(readUrl(url), "html.parser")
+    for h1 in soup.findAll("h1"):
+        if "post-title" in h1["class"]:
+            if h1.a.string != None:
+                print(h1.a.string)
+
+    soup = BeautifulSoup(readUrl(url), "html.parser")
+    for ele in soup.findAll("title"):
+        return ele.string
+
+    soup = BeautifulSoup(getSiteContent(url))
+    ls = []
+    for i in soup.findAll(tagName):
+        if i.get("class") == className:
+            if link:
+                if i.a.string != None:
+                    ls.append(i.a.string)
+            else:
+                ls.append(i.string)
+    return ls
+
+
+def getWeltHeadLines():
+    soup = BeautifulSoup(getSiteContent("http://welt.de"))
+    ls = []
+    for i in soup.findAll("h4"):
+        if i.get("class") == "headline":
+            if i.a.string != None:
+                ls.append(i.a.string)
+    return ls
+
+"""
+
+
+
+

deployment/.env

@@ -1,2 +1,2 @@
 APP_PREFIX=llm
-
+OPENSEARCH_INITIAL_ADMIN_PASSWORD=my_Pwd12345

deployment/docker-compose.yml

@@ -35,21 +35,30 @@ services:
   elasticsearch:
     container_name: ${APP_PREFIX}_elasticsearch
     image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
+    #image: opensearchproject/opensearch
     restart: always
     mem_limit: 4024m
     ports:
       - "9200:9200"
     environment:
       - discovery.type=single-node
-      - xpack.security.enabled=false
       - logger.level=ERROR
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping
+
+      - xpack.security.enabled=false
+
+      #- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}    # Sets the demo admin user password when using demo configuration, required for OpenSearch 2.12 and later
+      #- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
       #- ES_JAVA_OPTS="-Xms2g -Xmx2g"
     volumes:
       - esdata:/usr/share/elasticsearch/data
     ulimits:
       memlock:
-        soft: -1
+        soft: -1 # Set memlock to unlimited (no soft or hard limit)
         hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
+        hard: 65536
     networks:
       - llm_network