From a46f9ee2cc06dbcdc366d9ca17a9be320bcc3ceb Mon Sep 17 00:00:00 2001 From: herdogan Date: Mon, 11 Dec 2023 14:04:42 +0300 Subject: [PATCH] upstream updated and changes are made according to reviews of pull request --- fern/docs/pages/manual/llms.mdx | 86 ++++++++++++++++++- .../embedding/embedding_component.py | 23 +++++ private_gpt/components/llm/llm_component.py | 23 +++++ private_gpt/settings/settings.py | 12 ++- settings-bedrock.yaml | 19 ++++ settings.yaml | 8 +- 6 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 settings-bedrock.yaml diff --git a/fern/docs/pages/manual/llms.mdx b/fern/docs/pages/manual/llms.mdx index 8b56f758d5..6e5c1d3728 100644 --- a/fern/docs/pages/manual/llms.mdx +++ b/fern/docs/pages/manual/llms.mdx @@ -82,4 +82,88 @@ or `PGPT_PROFILES=sagemaker poetry run python -m private_gpt` When the server is started it will print a log *Application startup complete*. -Navigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API. \ No newline at end of file +Navigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API. + + + +### Use AWS Bedrock Models + +Another alternative if you don't have the necessary GPU to run local models, is using AWS Bedrock models. The default chosen models in Bedrock are Anthropic Claude v2 (LLM) and Titan Embeddings G1 - Text (embedding). You can navigate to `settings-bedrock.yaml` and change the `llm_modelid` and `embedding_modelid` parameters. You can find the available models and their IDs in the link https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html + +In order to use the Bedrock models, you need to do the following configurations. + +#### Install AWS CLI + +* Go to `https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html` and follow the instructions + +#### Configure AWS CLI + +* Open a powershell and run the following command: `aws configure` + +* You'll be prompted to enter the following information: + + + * AWS Access Key ID\ + AWS Secret Access Key\ + Default region name\ + Default output format (optional)\ + These credentials should have the necessary permissions to interact with AWS services. + +#### Set the Environment Variable + +To add Bedrock as an active profile of PrivateGPT, you should add `'bedrock'` as value to an environment variable named **PGPT_PROFILES**: + +First, check if the variable exists with one of the commands below based on your OS. If the variable exists, it will print the value. If not, it will do nothing: + +```bash +# macOS (terminal) +echo $PGPT_PROFILES + +# Windows (cmd) +echo %PGPT_PROFILES% + +# Windows (Powershell) +$env:PGPT_PROFILES + +# Linux (Bash) +echo $PGPT_PROFILES +``` + +If variable does not exist, set the value to `'bedrock'` with one the following commands: + +```bash +# macOS (terminal) +export PGPT_PROFILES="bedrock" + +# Windows (cmd) +set PGPT_PROFILES=bedrock + +# Windows (Powershell) +$env:PGPT_PROFILES="bedrock" + +# Linux (Bash) +export PGPT_PROFILES="bedrock" +``` + +If variable does exist, add the value `'bedrock'` to env var with one the following commands: + +```bash +# macOS (terminal) +export PGPT_PROFILES="${PGPT_PROFILES}:bedrock" + +# Windows (cmd) : Windows Command Prompt doesn't have a straightforward way to directly append to an environment variable without extra tooling or scripting. + +# Windows (Powershell) +$env:PGPT_PROFILES += ";bedrock" + +# Linux (Bash) +export PGPT_PROFILES="${PGPT_PROFILES}:bedrock" +``` + +#### Run the program + +Then, you can run the program with the command: + +```bash +poetry run python -m private_gpt +``` \ No newline at end of file diff --git a/private_gpt/components/embedding/embedding_component.py b/private_gpt/components/embedding/embedding_component.py index e60c7af631..3af83ac922 100644 --- a/private_gpt/components/embedding/embedding_component.py +++ b/private_gpt/components/embedding/embedding_component.py @@ -44,3 +44,26 @@ def __init__(self, settings: Settings) -> None: # Not a random number, is the dimensionality used by # the default embedding model self.embedding_model = MockEmbedding(384) + + case "bedrock": + + from llama_index.embeddings import BedrockEmbedding + + self.embedding_model = BedrockEmbedding( + model_name=settings.bedrock.embedding_modelid, + ) + + from boto3 import Session + + # Access credentials using boto3 + session = Session() + credentials = session.get_credentials() + + # Access key ID and secret access key + access_key = credentials.access_key + secret_key = credentials.secret_key + + self.embedding_model.set_credentials(aws_region=settings.bedrock.region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) \ No newline at end of file diff --git a/private_gpt/components/llm/llm_component.py b/private_gpt/components/llm/llm_component.py index 88485f4e93..05d9c0d11b 100644 --- a/private_gpt/components/llm/llm_component.py +++ b/private_gpt/components/llm/llm_component.py @@ -56,3 +56,26 @@ def __init__(self, settings: Settings) -> None: ) case "mock": self.llm = MockLLM() + + case "bedrock": + from llama_index.llms import Bedrock + + self.llm = Bedrock( + model=settings.bedrock.llm_modelid, + temperature=settings.bedrock.llm_temperature + ) + + from boto3 import Session + + # Access credentials using boto3 + session = Session() + credentials = session.get_credentials() + + # Access key ID and secret access key + access_key = credentials.access_key + secret_key = credentials.secret_key + + self.llm.set_credentials(aws_region=settings.bedrock.region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 8b03f6111d..79f11a61e8 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -81,7 +81,7 @@ class DataSettings(BaseModel): class LLMSettings(BaseModel): - mode: Literal["local", "openai", "sagemaker", "mock"] + mode: Literal["local", "openai", "sagemaker", "mock", "bedrock"] max_new_tokens: int = Field( 256, description="The maximum number of token that the LLM is authorized to generate in one completion.", @@ -111,7 +111,7 @@ class LocalSettings(BaseModel): class EmbeddingSettings(BaseModel): - mode: Literal["local", "openai", "sagemaker", "mock"] + mode: Literal["local", "openai", "sagemaker", "mock", "bedrock"] ingest_mode: Literal["simple", "batch", "parallel"] = Field( "simple", description=( @@ -151,6 +151,13 @@ class OpenAISettings(BaseModel): ) +class BedrockSettings(BaseModel): + llm_modelid: str + llm_temperature: float + embedding_modelid: str + region: str + + class UISettings(BaseModel): enabled: bool path: str @@ -226,6 +233,7 @@ class Settings(BaseModel): local: LocalSettings sagemaker: SagemakerSettings openai: OpenAISettings + bedrock: BedrockSettings vectorstore: VectorstoreSettings qdrant: QdrantSettings | None = None diff --git a/settings-bedrock.yaml b/settings-bedrock.yaml new file mode 100644 index 0000000000..33635f361b --- /dev/null +++ b/settings-bedrock.yaml @@ -0,0 +1,19 @@ +server: + env_name: ${APP_ENV:prod} + port: ${PORT:8001} + +ui: + enabled: true + path: / + +llm: + mode: bedrock + +embedding: + mode: bedrock + +bedrock: + llm_modelid: anthropic.claude-v2 + llm_temperature: 0.5 + embedding_modelid: amazon.titan-embed-text-v1 + region: us-east-1 \ No newline at end of file diff --git a/settings.yaml b/settings.yaml index af51a7f7ee..2049596f7e 100644 --- a/settings.yaml +++ b/settings.yaml @@ -56,4 +56,10 @@ sagemaker: openai: api_key: ${OPENAI_API_KEY:} - model: gpt-3.5-turbo \ No newline at end of file + model: gpt-3.5-turbo + +bedrock: + llm_modelid: anthropic.claude-v2 + llm_temperature: 0.5 + embedding_modelid: amazon.titan-embed-text-v1 + region: us-east-1 \ No newline at end of file