diff --git a/app/backend/app.py b/app/backend/app.py index 2cb2299e..676ddac9 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -3,10 +3,8 @@ import os import time from typing import AsyncGenerator, cast -import base64 import csv import io -from azure.identity.aio import DefaultAzureCredential from azure.monitor.opentelemetry import configure_azure_monitor from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware @@ -20,20 +18,13 @@ send_from_directory, send_file ) +from init_app import initApp from core.modelhelper import num_tokens_from_message -from core.llmhelper import createAzureChatGPT, getAzureChatGPT from core.types.Chunk import Chunk -from core.datahelper import Repository, Base, Requestinfo +from core.datahelper import Requestinfo from core.authentification import AuthentificationHelper, AuthError -from core.confighelper import ConfigHelper -from core.types.AppConfig import AppConfig, OpenaiInfo -from core.textsplit import splitPDF, splitText -from core.types.Config import BackendConfig - -from approaches.summarize import Summarize -from approaches.simplechat import SimpleChatApproach -from approaches.brainstorm import Brainstorm - +from core.types.AppConfig import AppConfig +from core.types.SupportedModels import SupportedModels bp = Blueprint("routes", __name__, static_folder='static') APPCONFIG_KEY = "APPCONFIG" @@ -45,9 +36,7 @@ async def handleAuthError(error: AuthError): @bp.route("/") async def index(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() return await bp.send_static_file("index.html") @bp.route("/favicon.ico") @@ -56,43 +45,27 @@ async def favicon(): @bp.route("/assets/") async def assets(path): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() return await send_from_directory("static/assets", path) @bp.route("/sum", methods=["POST"]) async def sum(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() + department = get_department(request=request) + files = await request.files forms = await request.form request_json = json.loads(forms.get("body")) file = files.get("file", None) - - # detailed summarization summarizes lower chunks detaillevel = request_json["detaillevel"] - switcher = { - "short": 2100, - "medium": 1500, - "long": 700, - } - splitsize = switcher.get(detaillevel, 700) - - #TODO Wenn Cleanup tokenlimit sprengt, was machen? In mehreren Schritten übersetzen. - if(file is not None): - splits = splitPDF(file, splitsize, 0) - else: - text = request_json["text"] - splits = splitText(text, splitsize, 0) - - department = get_department(request=request) - try: + impl = cfg["sum_approaches"] + text = request_json["text"] if file is None else None + splits = impl.split(detaillevel=detaillevel, file=file, text=text) + r = await impl.run(splits = splits, department=department, language=request_json["language"] or "Deutsch") return jsonify(r) except Exception as e: @@ -101,9 +74,7 @@ async def sum(): @bp.route("/brainstorm", methods=["POST"]) async def brainstorm(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() if not request.is_json: return jsonify({"error": "request must be json"}), 415 @@ -119,22 +90,9 @@ async def brainstorm(): msg = "Momentan liegt eine starke Auslastung vor. Bitte in einigen Sekunden erneut versuchen." if "Rate limit" in str(e) else str(e) return jsonify({"error": msg}), 500 - -async def format_as_ndjson(r: AsyncGenerator[Chunk, None]) -> AsyncGenerator[str, None]: - try: - async for event in r: - yield json.dumps(event, ensure_ascii=False) + "\n" - except Exception as e: - logging.exception("Exception while generating response stream: %s", e) - msg = "Momentan liegt eine starke Auslastung vor. Bitte in einigen Sekunden erneut versuchen." if "Rate limit" in str(e) else str(e) - yield json.dumps(Chunk(type="E", message=msg)) - - @bp.route("/chat_stream", methods=["POST"]) async def chat_stream(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() department = get_department(request=request) if not request.is_json: @@ -156,19 +114,39 @@ async def chat_stream(): except Exception as e: logging.exception("Exception in /chat") return jsonify({"error": str(e)}), 500 + +@bp.route("/chat", methods=["POST"]) +async def chat(): + cfg = get_config_and_authentificate() + department = get_department(request=request) + + if not request.is_json: + return jsonify({"error": "request must be json"}), 415 + request_json = await request.get_json() + try: + impl = cfg["chat_approaches"] + temperature=request_json['temperature'] or 0.7 + max_tokens=request_json['max_tokens'] or 4096 + system_message = request_json['system_message'] or None + history = request_json["history"] + response = impl.run_without_streaming(history= history, + temperature=temperature, + max_tokens=max_tokens, + system_message=system_message, + department= department) + return jsonify({"content": response}) + except Exception as e: + logging.exception("Exception in /chat") + return jsonify({"error": str(e)}), 500 @bp.route("/config", methods=["GET"]) async def getConfig(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() return jsonify(cfg["configuration_features"]) @bp.route("/statistics", methods=["GET"]) async def getStatistics(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() repo = cfg["repository"] sum_by_department = repo.sumByDepartment() avg_by_department = repo.avgByDepartment() @@ -179,9 +157,7 @@ async def getStatistics(): @bp.route("/counttokens", methods=["POST"]) async def counttokens(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() if not request.is_json: return jsonify({"error": "request must be json"}), 415 @@ -195,9 +171,7 @@ async def counttokens(): @bp.route("/statistics/export", methods=["GET"]) async def getStatisticsCSV(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) - if cfg["configuration_features"]["backend"]["enable_auth"]: - ensure_authentification(request=request) + cfg = get_config_and_authentificate() repo = cfg["repository"] memfile = io.StringIO() @@ -217,52 +191,24 @@ async def getStatisticsCSV(): def health_check(): return "OK" +def get_config(): + return cast(AppConfig, current_app.config[APPCONFIG_KEY]) -@bp.route("/checkauth", methods=["GET"]) -async def checkauth(): - auth_client, claims = ensure_authentification(request=request) - return "OK" - -@bp.route("/token", methods=["GET"]) -async def readToken(): - principalID = request.headers.get('X-MS-CLIENT-PRINCIPAL-ID') - principalName = request.headers.get('X-MS-CLIENT-PRINCIPAL-NAME') - idProviderId = request.headers.get('X-MS-CLIENT-PRINCIPAL-IDP') - ssoidtoken = request.headers.get("X-Ms-Token-Lhmsso-Access-Token") - clientPrincipal = request.headers.get('X-MS-CLIENT-PRINCIPAL') - clientPrincipal= base64.b64decode(clientPrincipal) - - auth_client, claims = ensure_authentification(request=request) - - id_claims = auth_client.decode(ssoidtoken) - - result = "\n" - myDict = sorted(dict(request.headers)) - for key in myDict: - result += f"{key} = {dict(request.headers)[key]}\n" - - response = await make_response(f"Hi {auth_client.getName(claims=claims)} aus {auth_client.getDepartment(claims=id_claims)}"+ - f"\n\nHier sind deine Client Principals von Azure:"+ - f"\nThis is your X-MS-CLIENT-PRINCIPAL-ID: {principalID}"+ - f"\nThis is your X-MS-CLIENT-PRINCIPAL-NAME: {principalName}"+ - f"\nThis is your X-MS-CLIENT-PRINCIPAL-IDP: {idProviderId}"+ - f"\nThis is your X-MS-CLIENT-PRINCIPAL: {clientPrincipal}"+ - f"\n\nJetzt gehts um deine Rollen:"+ - f"\n\nRollen: {auth_client.getRoles(claims)}"+ - f"\n\n\n Und alle Daten: {result}", 200) - - response.mimetype = "text/plain" - return response +def get_config_and_authentificate(): + cfg = get_config() + if cfg["configuration_features"]["backend"]["enable_auth"]: + ensure_authentification(request=request) + return cfg def ensure_authentification(request: request): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) + cfg = get_config() ssoaccesstoken = request.headers.get("X-Ms-Token-Lhmsso-Access-Token") auth_client : AuthentificationHelper = cfg["authentification_client"] claims = auth_client.authentificate(ssoaccesstoken) return auth_client,claims def get_department(request: request): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) + cfg = get_config() if cfg["configuration_features"]["backend"]["enable_auth"]: ssoidtoken = request.headers.get('X-Ms-Token-Lhmsso-Id-Token') auth_client : AuthentificationHelper = cfg["authentification_client"] @@ -270,126 +216,32 @@ def get_department(request: request): return auth_client.getDepartment(claims=id_claims) else: return None + +async def format_as_ndjson(r: AsyncGenerator[Chunk, None]) -> AsyncGenerator[str, None]: + try: + async for event in r: + yield json.dumps(event, ensure_ascii=False) + "\n" + except Exception as e: + logging.exception("Exception while generating response stream: %s", e) + msg = "Momentan liegt eine starke Auslastung vor. Bitte in einigen Sekunden erneut versuchen." if "Rate limit" in str(e) else str(e) + yield json.dumps(Chunk(type="E", message=msg)) + @bp.before_request async def ensure_openai_token(): - cfg = cast(AppConfig, current_app.config[APPCONFIG_KEY]) + cfg = get_config() openai_token = cfg["model_info"]["openai_token"] if openai_token.expires_on < time.time() + 60: openai_token = await cfg["azure_credential"].get_token("https://cognitiveservices.azure.com/.default") - # new token, + # updates tokens, the approaches should get the newest version of the token via reference cfg["model_info"]["openai_token"] = openai_token cfg["model_info"]["openai_api_key"] = openai_token.token - (chat_approaches, brainstorm_approaches, sum_approaches) = initApproaches(model_info=cfg["model_info"], cfg=cfg["backend_config"], repoHelper=cfg["repository"]) - cfg["chat_approaches"] = chat_approaches - cfg["brainstorm_approaches"] = brainstorm_approaches - cfg["sum_approaches"] = sum_approaches - - @bp.before_app_serving async def setup_clients(): - - # Replace these with your own values, either in environment variables or directly here - AZURE_OPENAI_SERVICE = os.environ["AZURE_OPENAI_SERVICE"] - AZURE_OPENAI_CHATGPT_DEPLOYMENT = os.environ["AZURE_OPENAI_CHATGPT_DEPLOYMENT"] - AZURE_OPENAI_CHATGPT_MODEL = os.environ["AZURE_OPENAI_CHATGPT_MODEL"] - SSO_ISSUER = os.environ["SSO_ISSUER"] - CONFIG_NAME = os.environ["CONFIG_NAME"] - DB_HOST = os.environ["DB_HOST"] - DB_NAME = os.environ["DB_NAME"] - DB_USER = os.environ["DB_USER"] - DB_PASSWORD = os.environ["DB_PASSWORD"] - - # Use the current user identity to authenticate with Azure OpenAI, Cognitive Search and Blob Storage (no secrets needed, - # just use 'az login' locally, and managed identity when deployed on Azure). If you need to use keys, use separate AzureKeyCredential instances with the - # keys for each service - # If you encounter a blocking error during a DefaultAzureCredential resolution, you can exclude the problematic credential by using a parameter (ex. exclude_shared_token_cache_credential=True) - azure_credential = DefaultAzureCredential(exclude_shared_token_cache_credential = True) - - # Used by the OpenAI SDK - openai_api_base = f"https://{AZURE_OPENAI_SERVICE}.openai.azure.com" - openai_api_version = "2023-05-15" - openai_api_type = "azure_ad" - openai_token = await azure_credential.get_token( - "https://cognitiveservices.azure.com/.default" - ) - openai_api_key = openai_token.token - # Set up authentication helper - auth_helper = AuthentificationHelper( - issuer=SSO_ISSUER, - role="lhm-ab-mucgpt-user" - ) - - repoHelper = Repository( - username=DB_USER, - host=DB_HOST, - database=DB_NAME, - password=DB_PASSWORD - ) - - - config_helper = ConfigHelper(base_path=os.getcwd()+"/ressources/", env=CONFIG_NAME, base_config_name="base") - cfg = config_helper.loadData() - model_info = OpenaiInfo( - model=AZURE_OPENAI_CHATGPT_MODEL, - openai_token = openai_token, - openai_api_key = openai_api_key, - openai_api_base = openai_api_base, - openai_api_version = openai_api_version, - openai_api_type = openai_api_type - ) - - (chat_approaches, brainstorm_approaches, sum_approaches) = initApproaches(model_info=model_info, cfg=cfg["backend"], repoHelper=repoHelper) - - current_app.config[APPCONFIG_KEY] = AppConfig( - model_info= model_info, - azure_credential=azure_credential, - authentification_client=auth_helper, - configuration_features=cfg, - chat_approaches= chat_approaches, - brainstorm_approaches= brainstorm_approaches, - sum_approaches= sum_approaches, - repository=repoHelper, - backend_config=cfg["backend"] - ) - if cfg["backend"]["enable_database"]: - repoHelper.setup_schema(base=Base) - -def initApproaches(model_info: OpenaiInfo, cfg: BackendConfig, repoHelper: Repository): - brainstormllm = getAzureChatGPT( - chatgpt_model= model_info["model"], - max_tokens = 4000, - n = 1, - openai_api_key = model_info["openai_api_key"], - openai_api_base = model_info["openai_api_base"], - openai_api_version = model_info["openai_api_version"], - openai_api_type = model_info["openai_api_type"], - temperature=0.9) - sumllm = getAzureChatGPT( - chatgpt_model= model_info["model"], - max_tokens = 1000, - n = 1, - openai_api_key = model_info["openai_api_key"], - openai_api_base = model_info["openai_api_base"], - openai_api_version = model_info["openai_api_version"], - openai_api_type = model_info["openai_api_type"], - temperature=0.7) - chatlllm = createAzureChatGPT( - chatgpt_model= model_info["model"], - n = 1, - openai_api_key = model_info["openai_api_key"], - openai_api_base = model_info["openai_api_base"], - openai_api_version = model_info["openai_api_version"], - openai_api_type = model_info["openai_api_type"]) - chat_approaches = SimpleChatApproach(createLLM=chatlllm, config=cfg["chat"], repo=repoHelper, chatgpt_model=model_info["model"]) - brainstorm_approaches = Brainstorm(llm=brainstormllm, config=cfg["brainstorm"], repo=repoHelper) - sum_approaches = Summarize(llm=sumllm, config=cfg["sum"], repo=repoHelper) - return (chat_approaches, brainstorm_approaches, sum_approaches) - - + current_app.config[APPCONFIG_KEY] = await initApp() def create_app(): if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): diff --git a/app/backend/approaches/brainstorm.py b/app/backend/approaches/brainstorm.py index 98ddef43..fe0ab07f 100644 --- a/app/backend/approaches/brainstorm.py +++ b/app/backend/approaches/brainstorm.py @@ -1,13 +1,15 @@ from typing import Any, Optional from langchain.chains import LLMChain - from langchain.prompts import PromptTemplate from langchain.chains import SequentialChain from langchain_community.callbacks import get_openai_callback +from langchain_core.runnables.base import RunnableSerializable + +from core.types.LlmConfigs import LlmConfigs from core.datahelper import Repository from core.types.Config import ApproachConfig +from core.types.AzureChatGPTConfig import AzureChatGPTConfig from core.datahelper import Requestinfo -from langchain_core.language_models.chat_models import BaseChatModel class Brainstorm(): """ @@ -53,9 +55,10 @@ class Brainstorm(): Text: {brainstorm}""" - def __init__(self, llm: BaseChatModel, config: ApproachConfig, repo: Repository): + def __init__(self, llm: RunnableSerializable, config: ApproachConfig, model_info: AzureChatGPTConfig, repo: Repository): self.llm = llm self.config = config + self.model_info = model_info self.repo = repo def getBrainstormPrompt(self) -> PromptTemplate: @@ -66,8 +69,12 @@ def getTranslationPrompt(self) -> PromptTemplate: async def run(self, topic: str, language: str, department: Optional[str]) -> Any: - brainstormChain = LLMChain(llm=self.llm, prompt=self.getBrainstormPrompt(), output_key="brainstorm") - translationChain = LLMChain(llm=self.llm, prompt=self.getTranslationPrompt(), output_key="translation") + config: LlmConfigs = { + "llm_api_key": self.model_info["openai_api_key"] + } + llm = self.llm.with_config(configurable=config) + brainstormChain = LLMChain(llm=llm, prompt=self.getBrainstormPrompt(), output_key="brainstorm") + translationChain = LLMChain(llm=llm, prompt=self.getTranslationPrompt(), output_key="translation") overall_chain = SequentialChain( chains=[brainstormChain, translationChain], input_variables=["language", "topic"], diff --git a/app/backend/approaches/simplechat.py b/app/backend/approaches/chat.py similarity index 62% rename from app/backend/approaches/simplechat.py rename to app/backend/approaches/chat.py index 84f94f3e..ced9d855 100644 --- a/app/backend/approaches/simplechat.py +++ b/app/backend/approaches/chat.py @@ -1,65 +1,50 @@ -from typing import Any, AsyncGenerator, Callable, Optional, Sequence - -from core.datahelper import Requestinfo -from core.modelhelper import num_tokens_from_message, num_tokens_from_messages -from core.datahelper import Repository -from core.types.Config import ApproachConfig -from core.types.Chunk import Chunk, ChunkInfo +from typing import Any, AsyncGenerator, Optional, Sequence +import asyncio from langchain.prompts import ( ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, SystemMessagePromptTemplate ) - -from langchain_core.language_models.chat_models import BaseChatModel from langchain.chains import LLMChain from langchain.memory import ConversationBufferMemory from langchain.callbacks.streaming_aiter import AsyncIteratorCallbackHandler +from langchain_core.messages import AIMessage +from langchain_community.callbacks import get_openai_callback +from langchain_core.runnables.base import RunnableSerializable -import asyncio +from core.datahelper import Requestinfo +from core.modelhelper import num_tokens_from_message, num_tokens_from_messages +from core.datahelper import Repository +from core.types.Config import ApproachConfig +from core.types.Chunk import Chunk, ChunkInfo +from core.types.LlmConfigs import LlmConfigs +from core.types.AzureChatGPTConfig import AzureChatGPTConfig -class SimpleChatApproach(): +class ChatApproach(): - def __init__(self, createLLM: Callable[[int, float, AsyncIteratorCallbackHandler], BaseChatModel], config: ApproachConfig, repo: Repository, chatgpt_model: str): - self.createLLM = createLLM + def __init__(self, llm: RunnableSerializable, config: ApproachConfig, model_info: AzureChatGPTConfig, repo: Repository, chatgpt_model: str): + self.llm = llm self.config = config + self.model_info = model_info self.repo = repo self.chatgpt_model = chatgpt_model - async def run_until_final_call(self, history: "Sequence[dict[str, str]]", llm: BaseChatModel, system_message: Optional[str]) -> Any: - user_q = history[-1]["user"] - messages = [ - # The `variable_name` here is what must align with memory - MessagesPlaceholder(variable_name="chat_history"), - HumanMessagePromptTemplate.from_template("{question}") - ] - if(system_message and system_message.strip() !=""): - messages.insert(0, - SystemMessagePromptTemplate.from_template( - system_message - )) - prompt = ChatPromptTemplate( - messages=messages - ) - # Notice that we `return_messages=True` to fit into the MessagesPlaceholder - # Notice that `"chat_history"` aligns with the MessagesPlaceholder name. - memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) - ## initialize memory with our own chat model. - self.initializeMemory(history[:-1],memory=memory) - conversation = LLMChain( - llm=llm, - prompt=prompt, - memory=memory - ) - + async def run_until_final_call(self, history: "Sequence[dict[str, str]]", llm: RunnableSerializable, system_message: Optional[str]) -> Any: + user_q, conversation = self.init_conversation(history, llm, system_message) chat_coroutine = conversation.acall({"question": user_q}) return (chat_coroutine) - async def run_with_streaming(self, history: 'list[dict[str, str]]',max_tokens: int, temperature: float, system_message: Optional[str], department: Optional[str]) -> AsyncGenerator[dict, None]: handler = AsyncIteratorCallbackHandler() - llm = self.createLLM(max_tokens, temperature, handler) + config: LlmConfigs = { + "llm_max_tokens": max_tokens, + "llm_api_key": self.model_info["openai_api_key"], + "llm_temperature": temperature, + "llm_streaming": True, + "llm_callbacks": [handler], + } + llm = self.llm.with_config(configurable=config) chat_coroutine = await self.run_until_final_call(history, llm=llm, system_message=system_message) task = asyncio.create_task(chat_coroutine) result = "" @@ -92,9 +77,59 @@ async def run_with_streaming(self, history: 'list[dict[str, str]]',max_tokens: i info = ChunkInfo(requesttokens=num_tokens_from_message(history[-1]["user"],self.chatgpt_model), streamedtokens=num_tokens_from_message(result,self.chatgpt_model)) yield Chunk(type="I", message=info, order=position) + + def run_without_streaming(self, history: "Sequence[dict[str, str]]", max_tokens: int, temperature: float, system_message: Optional[str], department: Optional[str]) -> str: + config: LlmConfigs = { + "llm_max_tokens": max_tokens, + "llm_api_key": self.model_info["openai_api_key"], + "llm_temperature": temperature, + "llm_streaming": False, + } + llm = self.llm.with_config(configurable=config) + user_q, conversation = self.init_conversation(history, llm, system_message) + with get_openai_callback() as cb: + ai_message: AIMessage = conversation.invoke({"question": user_q}) + total_tokens = cb.total_tokens + + info = ChunkInfo(requesttokens=cb.prompt_tokens, streamedtokens=cb.completion_tokens) + if self.config["log_tokens"]: + self.repo.addInfo(Requestinfo( + tokencount = total_tokens, + department = department, + messagecount= 1, + method = "Brainstorm")) + return ai_message["chat_history"][-1].content + + def init_conversation(self, history, llm, system_message): + user_q = history[-1]["user"] + messages = [ + # The `variable_name` here is what must align with memory + MessagesPlaceholder(variable_name="chat_history"), + HumanMessagePromptTemplate.from_template("{question}") + ] + if(system_message and system_message.strip() !=""): + messages.insert(0, + SystemMessagePromptTemplate.from_template( + system_message + )) + prompt = ChatPromptTemplate( + messages=messages + ) + # Notice that we `return_messages=True` to fit into the MessagesPlaceholder + # Notice that `"chat_history"` aligns with the MessagesPlaceholder name. + memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) + ## initialize memory with our own chat model. + self.init_mem(history[:-1],memory=memory) + conversation = LLMChain( + llm=llm, + prompt=prompt, + memory=memory + ) + + return user_q,conversation - def initializeMemory(self, messages:"Sequence[dict[str, str]]", memory: ConversationBufferMemory) : + def init_mem(self, messages:"Sequence[dict[str, str]]", memory: ConversationBufferMemory) : for conversation in messages: if("user" in conversation and conversation["user"]): userMsg = conversation["user"] diff --git a/app/backend/approaches/summarize.py b/app/backend/approaches/summarize.py index 4e053ec3..00f1df56 100644 --- a/app/backend/approaches/summarize.py +++ b/app/backend/approaches/summarize.py @@ -1,15 +1,19 @@ from concurrent.futures import ThreadPoolExecutor from typing import Any, List, Optional - +import json +import re from langchain.prompts import PromptTemplate from langchain.chains import SequentialChain, LLMChain -from langchain_core.language_models.chat_models import BaseChatModel from langchain_community.callbacks import get_openai_callback +from langchain_core.runnables.base import RunnableSerializable + from core.datahelper import Repository from core.types.Config import ApproachConfig from core.datahelper import Requestinfo -import json -import re +from core.textsplit import splitPDF, splitText +from core.types.AzureChatGPTConfig import AzureChatGPTConfig +from core.types.LlmConfigs import LlmConfigs + class Summarize(): """Chain of Density prompting: https://arxiv.org/abs/2309.04269""" @@ -79,10 +83,16 @@ class Summarize(): - def __init__(self, llm: BaseChatModel, config: ApproachConfig, repo: Repository): + def __init__(self, llm: RunnableSerializable, config: ApproachConfig, model_info: AzureChatGPTConfig, repo: Repository, short_split = 2100, medium_split = 1500, long_split = 700): self.llm = llm self.config = config + self.model_info = model_info self.repo = repo + self.switcher = { + "short": short_split, + "medium": medium_split, + "long": long_split, + } def getSummarizationPrompt(self) -> PromptTemplate: return PromptTemplate(input_variables=["text"], template=self.user_sum_prompt) @@ -95,9 +105,13 @@ def getTranslationCleanupPrompt(self) -> PromptTemplate: def setup(self) -> SequentialChain: + config: LlmConfigs = { + "llm_api_key": self.model_info["openai_api_key"] + } + llm = self.llm.with_config(configurable=config) # setup model - summarizationChain = LLMChain(llm=self.llm, prompt=self.getSummarizationPrompt(), output_key="sum") - translationChain = LLMChain(llm=self.llm, prompt=self.getTranslationCleanupPrompt(), output_key="translation") + summarizationChain = LLMChain(llm=llm, prompt=self.getSummarizationPrompt(), output_key="sum") + translationChain = LLMChain(llm=llm, prompt=self.getTranslationCleanupPrompt(), output_key="translation") return (summarizationChain, translationChain) @@ -202,3 +216,13 @@ async def run(self, splits: List[str], language: str, department: Optional[str] return {"answer": final_summarys} + def split(self, detaillevel: str, file = None, text = None): + splitsize = self.switcher.get(detaillevel, 700) + + #TODO Wenn Cleanup tokenlimit sprengt, was machen? In mehreren Schritten übersetzen. + if(file is not None): + splits = splitPDF(file, splitsize, 0) + else: + splits = splitText(text, splitsize, 0) + return splits + diff --git a/app/backend/core/llmhelper.py b/app/backend/core/llmhelper.py index 650b6447..25259b2e 100644 --- a/app/backend/core/llmhelper.py +++ b/app/backend/core/llmhelper.py @@ -1,46 +1,55 @@ - - -from typing import Callable from langchain_openai import AzureChatOpenAI -from langchain_core.language_models.chat_models import BaseChatModel -from langchain.callbacks.streaming_aiter import AsyncIteratorCallbackHandler - -def getAzureChatGPT(chatgpt_model: str, - max_tokens: int, - n: int, - openai_api_key: str, - openai_api_base: str, - openai_api_version: str, - openai_api_type: str, - temperature: float) -> BaseChatModel: - return AzureChatOpenAI( - model=chatgpt_model, - max_tokens=max_tokens, - n=n, - deployment_name= "chat", - openai_api_key=openai_api_key, - azure_endpoint=openai_api_base, - openai_api_version=openai_api_version, - openai_api_type=openai_api_type, - temperature=temperature - ) +from langchain_core.runnables import ConfigurableField +from langchain_community.llms.fake import FakeListLLM +from langchain_core.runnables.base import RunnableSerializable +from core.types.SupportedModels import SupportedModels -def createAzureChatGPT(chatgpt_model: str, - n: int, - openai_api_key: str, - openai_api_base: str, - openai_api_version: str, - openai_api_type: str,) -> Callable[[AsyncIteratorCallbackHandler], BaseChatModel]: - return lambda max_tokens, temperature, callback : AzureChatOpenAI( - model=chatgpt_model, +def getModel(chatgpt_model: str, + max_tokens: int, + n: int, + api_key: str, + api_base: str, + api_version: str, + api_type: str, + temperature: float, + streaming: bool) -> RunnableSerializable: + llm = AzureChatOpenAI( + model=chatgpt_model, max_tokens=max_tokens, n=n, deployment_name= "chat", - openai_api_key=openai_api_key, - azure_endpoint=openai_api_base, - openai_api_version=openai_api_version, - openai_api_type=openai_api_type, + openai_api_key=api_key, + azure_endpoint=api_base, + openai_api_version=api_version, + openai_api_type=api_type, temperature=temperature, - streaming=True, - callbacks=[callback] - ) \ No newline at end of file + streaming=streaming, + ).configurable_fields( + temperature=ConfigurableField( + id="llm_temperature", + name="LLM Temperature", + description="The temperature of the LLM", + ), + max_tokens= ConfigurableField( + id="llm_max_tokens", + name="LLM max Tokens", + description="The token Limit of the LLM", + ), + openai_api_key = ConfigurableField( + id="llm_api_key", + name="The api key", + description="The api key"), + streaming = ConfigurableField( + id="llm_streaming", + name="Streaming", + description="Should the LLM Stream"), + callbacks = ConfigurableField( + id="llm_callbacks", + name="Callbacks", + description="Callbacks for the llm") + + ).configurable_alternatives( + ConfigurableField(id="llm"), + default_key=SupportedModels.AZURE_CHATGPT.value, + fake= FakeListLLM(responses=["Hi diggi"])) + return llm diff --git a/app/backend/core/types/AppConfig.py b/app/backend/core/types/AppConfig.py index 5d4b9925..1d7c6239 100644 --- a/app/backend/core/types/AppConfig.py +++ b/app/backend/core/types/AppConfig.py @@ -1,26 +1,18 @@ from typing import Any, Mapping, TypedDict from azure.identity.aio import DefaultAzureCredential -from azure.core.credentials import AccessToken +from core.types.AzureChatGPTConfig import AzureChatGPTConfig from approaches.summarize import Summarize -from approaches.simplechat import SimpleChatApproach +from approaches.chat import ChatApproach from approaches.brainstorm import Brainstorm from core.authentification import AuthentificationHelper from core.types.Config import Config from core.datahelper import Repository from core.types.Config import BackendConfig -class OpenaiInfo(TypedDict): - model: str - openai_token: AccessToken - openai_api_key: str - openai_api_base: str - openai_api_version: str - openai_api_type: str - class AppConfig(TypedDict): - model_info: OpenaiInfo + model_info: AzureChatGPTConfig azure_credential: DefaultAzureCredential - chat_approaches: SimpleChatApproach + chat_approaches: ChatApproach sum_approaches: Summarize brainstorm_approaches: Brainstorm authentification_client: AuthentificationHelper diff --git a/app/backend/core/types/AzureChatGPTConfig.py b/app/backend/core/types/AzureChatGPTConfig.py new file mode 100644 index 00000000..f7a898ef --- /dev/null +++ b/app/backend/core/types/AzureChatGPTConfig.py @@ -0,0 +1,11 @@ +from azure.core.credentials import AccessToken +from typing import TypedDict + + +class AzureChatGPTConfig(TypedDict): + model: str + openai_token: AccessToken + openai_api_key: str + openai_api_base: str + openai_api_version: str + openai_api_type: str \ No newline at end of file diff --git a/app/backend/core/types/Config.py b/app/backend/core/types/Config.py index 6d6ba1b7..16408770 100644 --- a/app/backend/core/types/Config.py +++ b/app/backend/core/types/Config.py @@ -15,6 +15,7 @@ class LabelsConfig(TypedDict): class FrontendConfig(TypedDict): labels: LabelsConfig + alternative_logo: bool class Config(TypedDict): version: str diff --git a/app/backend/core/types/LlmConfigs.py b/app/backend/core/types/LlmConfigs.py new file mode 100644 index 00000000..de4c91e1 --- /dev/null +++ b/app/backend/core/types/LlmConfigs.py @@ -0,0 +1,10 @@ +from typing_extensions import List, NotRequired, TypedDict + + +class LlmConfigs(TypedDict, total=False): + llm: NotRequired[str] # one of the SupportedModels + llm_max_tokens: NotRequired[int] + llm_temperature: NotRequired[float] + llm_api_key: NotRequired[str] + llm_streaming: NotRequired[bool] + llm_callbacks: NotRequired[List] \ No newline at end of file diff --git a/app/backend/core/types/SupportedModels.py b/app/backend/core/types/SupportedModels.py new file mode 100644 index 00000000..932dc4e9 --- /dev/null +++ b/app/backend/core/types/SupportedModels.py @@ -0,0 +1,7 @@ +from enum import Enum, unique + + +@unique +class SupportedModels(Enum): + AZURE_CHATGPT = "AZURE_CHATGPT" + FAKE = "FAKE" \ No newline at end of file diff --git a/app/backend/core/types/__init__.py b/app/backend/core/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/backend/init_app.py b/app/backend/init_app.py new file mode 100644 index 00000000..b61965c5 --- /dev/null +++ b/app/backend/init_app.py @@ -0,0 +1,118 @@ +import os +from core.types.SupportedModels import SupportedModels +from core.types.AzureChatGPTConfig import AzureChatGPTConfig +from core.types.Config import BackendConfig +from core.llmhelper import getModel +from core.datahelper import Repository, Base +from core.authentification import AuthentificationHelper +from core.confighelper import ConfigHelper +from core.types.AppConfig import AppConfig +from approaches.summarize import Summarize +from approaches.chat import ChatApproach +from approaches.brainstorm import Brainstorm +from azure.identity.aio import DefaultAzureCredential + + +def initApproaches(model_info: AzureChatGPTConfig, cfg: BackendConfig, repoHelper: Repository): + brainstormllm = getModel( + chatgpt_model= model_info["model"], + max_tokens = 4000, + n = 1, + api_key = model_info["openai_api_key"], + api_base = model_info["openai_api_base"], + api_version = model_info["openai_api_version"], + api_type = model_info["openai_api_type"], + streaming=False, + temperature=0.9) + sumllm = getModel( + chatgpt_model= model_info["model"], + max_tokens = 1000, + n = 1, + api_key = model_info["openai_api_key"], + api_base = model_info["openai_api_base"], + api_version = model_info["openai_api_version"], + api_type = model_info["openai_api_type"], + streaming=False, + temperature=0.7) + chatlllm = getModel( + chatgpt_model= model_info["model"], + max_tokens=4000, + n = 1, + api_key = model_info["openai_api_key"], + api_base = model_info["openai_api_base"], + api_version = model_info["openai_api_version"], + api_type = model_info["openai_api_type"], + streaming=True, + temperature=0.7) + chat_approaches = ChatApproach(llm=chatlllm, config=cfg["chat"], model_info=model_info, repo=repoHelper, chatgpt_model=model_info["model"]) + brainstorm_approaches = Brainstorm(llm=brainstormllm, model_info=model_info, config=cfg["brainstorm"], repo=repoHelper) + sum_approaches = Summarize(llm=sumllm, config=cfg["sum"],model_info=model_info, repo=repoHelper) + return (chat_approaches, brainstorm_approaches, sum_approaches) + +async def initApp() -> AppConfig: + # Replace these with your own values, either in environment variables or directly here + AZURE_OPENAI_SERVICE = os.environ["AZURE_OPENAI_SERVICE"] + AZURE_OPENAI_CHATGPT_DEPLOYMENT = os.environ["AZURE_OPENAI_CHATGPT_DEPLOYMENT"] + AZURE_OPENAI_CHATGPT_MODEL = os.environ["AZURE_OPENAI_CHATGPT_MODEL"] + SSO_ISSUER = os.environ["SSO_ISSUER"] + CONFIG_NAME = os.environ["CONFIG_NAME"] + DB_HOST = os.environ["DB_HOST"] + DB_NAME = os.environ["DB_NAME"] + DB_USER = os.environ["DB_USER"] + DB_PASSWORD = os.environ["DB_PASSWORD"] + + # Use the current user identity to authenticate with Azure OpenAI, Cognitive Search and Blob Storage (no secrets needed, + # just use 'az login' locally, and managed identity when deployed on Azure). If you need to use keys, use separate AzureKeyCredential instances with the + # keys for each service + # If you encounter a blocking error during a DefaultAzureCredential resolution, you can exclude the problematic credential by using a parameter (ex. exclude_shared_token_cache_credential=True) + azure_credential = DefaultAzureCredential(exclude_shared_token_cache_credential = True) + + # Used by the OpenAI SDK + openai_api_base = f"https://{AZURE_OPENAI_SERVICE}.openai.azure.com" + openai_api_version = "2023-05-15" + openai_api_type = "azure_ad" + openai_token = await azure_credential.get_token( + "https://cognitiveservices.azure.com/.default" + ) + openai_api_key = openai_token.token + # Set up authentication helper + auth_helper = AuthentificationHelper( + issuer=SSO_ISSUER, + role="lhm-ab-mucgpt-user" + ) + + repoHelper = Repository( + username=DB_USER, + host=DB_HOST, + database=DB_NAME, + password=DB_PASSWORD + ) + + + config_helper = ConfigHelper(base_path=os.getcwd()+"/ressources/", env=CONFIG_NAME, base_config_name="base") + cfg = config_helper.loadData() + model_info = AzureChatGPTConfig( + model=AZURE_OPENAI_CHATGPT_MODEL, + openai_token = openai_token, + openai_api_key = openai_api_key, + openai_api_base = openai_api_base, + openai_api_version = openai_api_version, + openai_api_type = openai_api_type + ) + + (chat_approaches, brainstorm_approaches, sum_approaches) = initApproaches(model_info=model_info, cfg=cfg["backend"], repoHelper=repoHelper) + + if cfg["backend"]["enable_database"]: + repoHelper.setup_schema(base=Base) + + return AppConfig( + model_info=model_info, + azure_credential=azure_credential, + authentification_client=auth_helper, + configuration_features=cfg, + chat_approaches= chat_approaches, + brainstorm_approaches= brainstorm_approaches, + sum_approaches= sum_approaches, + repository=repoHelper, + backend_config=cfg["backend"] + ) \ No newline at end of file diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index b3d4fe9b..ecc53549 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -15,3 +15,4 @@ requests sqlalchemy==2.0.19 psycopg2==2.9.9 pypdf2==3.0.1 +tenacity==8.2.3 \ No newline at end of file diff --git a/app/backend/ressources/demo.json b/app/backend/ressources/demo.json index 5eb01545..78cc1081 100644 --- a/app/backend/ressources/demo.json +++ b/app/backend/ressources/demo.json @@ -2,7 +2,8 @@ "frontend": { "labels": { "env_name": "PILOT" - } + }, + "alternative_logo": false }, "backend": { "enable_auth": true, diff --git a/app/backend/ressources/dev.json b/app/backend/ressources/dev.json index 7c22981f..d9b75028 100644 --- a/app/backend/ressources/dev.json +++ b/app/backend/ressources/dev.json @@ -1,8 +1,9 @@ { "frontend": { "labels": { - "env_name": "PILOT-C" - } + "env_name": "MUC tschibidi-C" + }, + "alternative_logo": true }, "backend": { "enable_auth": true, diff --git a/app/backend/ressources/local.json b/app/backend/ressources/local.json index eb22a178..44df80e0 100644 --- a/app/backend/ressources/local.json +++ b/app/backend/ressources/local.json @@ -1,8 +1,9 @@ { "frontend": { "labels": { - "env_name": "PILOT-L" - } + "env_name": "MUC tschibidi-C" + }, + "alternative_logo": true }, "backend": { "enable_auth": false, diff --git a/app/backend/ressources/nosec.json b/app/backend/ressources/nosec.json index 1faca003..b6152c26 100644 --- a/app/backend/ressources/nosec.json +++ b/app/backend/ressources/nosec.json @@ -2,7 +2,8 @@ "frontend": { "labels": { "env_name": "NOSEC" - } + }, + "alternative_logo": false }, "backend": { "enable_auth": false, diff --git a/app/backend/ressources/prod.json b/app/backend/ressources/prod.json index 4ab1a92f..5861cf2f 100644 --- a/app/backend/ressources/prod.json +++ b/app/backend/ressources/prod.json @@ -2,7 +2,8 @@ "frontend": { "labels": { "env_name": "PROD" - } + }, + "alternative_logo": false }, "backend": { "enable_auth": true, diff --git a/app/frontend/src/api/models.ts b/app/frontend/src/api/models.ts index 88394cc3..d439bf90 100644 --- a/app/frontend/src/api/models.ts +++ b/app/frontend/src/api/models.ts @@ -42,20 +42,14 @@ export interface ApplicationConfig { } export interface Backend { - features: BackendFeatures; -} - -export interface BackendFeatures { enable_auth: boolean; } export interface Frontend { - features: FrontendFeatures; + alternative_logo: boolean; labels: Labels; } -export interface FrontendFeatures {} - export interface Labels { env_name: string; } diff --git a/app/frontend/src/assets/mugg_tschibidi.png b/app/frontend/src/assets/mugg_tschibidi.png new file mode 100644 index 00000000..1b6f04d1 Binary files /dev/null and b/app/frontend/src/assets/mugg_tschibidi.png differ diff --git a/app/frontend/src/pages/layout/Layout.tsx b/app/frontend/src/pages/layout/Layout.tsx index 1ddab493..8465e2e7 100644 --- a/app/frontend/src/pages/layout/Layout.tsx +++ b/app/frontend/src/pages/layout/Layout.tsx @@ -2,6 +2,7 @@ import { Outlet, NavLink, Link, Navigate, useNavigate } from "react-router-dom"; import styles from "./Layout.module.css"; import { useContext, useEffect, useState } from "react"; import logo from "../../assets/mucgpt_logo.png"; +import alternative_logo from "../../assets/mugg_tschibidi.png"; import logo_black from "../../assets/mucgpt_black.png"; import { SelectionEvents, OptionOnSelectData } from "@fluentui/react-combobox"; import { DEFAULTLANG, LanguageContext } from "../../components/LanguageSelector/LanguageContextProvider"; @@ -31,15 +32,13 @@ export const Layout = () => { const { t, i18n } = useTranslation(); const [config, setConfig] = useState({ backend: { - features: { - enable_auth: true - } + enable_auth: true }, frontend: { - features: {}, labels: { - env_name: "PILOT-C" - } + "env_name": "MUC tschibidi-C" + }, + alternative_logo: true, }, version: "DEV 1.0.0" }); @@ -93,7 +92,7 @@ export const Layout = () => {