upstream updated and changes are made according to reviews of pull re…

…quest

fern/docs/pages/manual/llms.mdx

@@ -82,4 +82,88 @@ or
 `PGPT_PROFILES=sagemaker poetry run python -m private_gpt`
 
 When the server is started it will print a log *Application startup complete*.
-Navigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API.
+Navigate to http://localhost:8001 to use the Gradio UI or to http://localhost:8001/docs (API section) to try the API.
+
+
+
+### Use AWS Bedrock Models
+
+Another alternative if you don't have the necessary GPU to run local models, is using AWS Bedrock models. The default chosen models in Bedrock are Anthropic Claude v2 (LLM) and Titan Embeddings G1 - Text (embedding). You can navigate to `settings-bedrock.yaml` and change the `llm_modelid` and `embedding_modelid` parameters. You can find the available models and their IDs in the link https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html
+
+In order to use the Bedrock models, you need to do the following configurations.
+
+#### Install AWS CLI
+
+* Go to `https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html` and follow the instructions
+
+#### Configure AWS CLI
+
+* Open a powershell and run the following command: `aws configure`
+
+* You'll be prompted to enter the following information:
+
+
+  * AWS Access Key ID\
+  AWS Secret Access Key\
+  Default region name\
+  Default output format (optional)\
+  These credentials should have the necessary permissions to interact with AWS services.
+
+#### Set the Environment Variable
+
+To add Bedrock as an active profile of PrivateGPT, you should add `'bedrock'` as value to an environment variable named **PGPT_PROFILES**:
+
+First, check if the variable exists with one of the commands below based on your OS. If the variable exists, it will print the value. If not, it will do nothing:
+
+```bash
+# macOS (terminal)
+echo $PGPT_PROFILES
+
+# Windows (cmd)
+echo %PGPT_PROFILES%
+
+# Windows (Powershell)
+$env:PGPT_PROFILES
+
+# Linux (Bash)
+echo $PGPT_PROFILES
+```
+
+If variable does not exist, set the value to `'bedrock'` with one the following commands:
+
+```bash
+# macOS (terminal)
+export PGPT_PROFILES="bedrock"
+
+# Windows (cmd)
+set PGPT_PROFILES=bedrock
+
+# Windows (Powershell)
+$env:PGPT_PROFILES="bedrock"
+
+# Linux (Bash)
+export PGPT_PROFILES="bedrock"
+```
+
+If variable does exist, add the value `'bedrock'` to env var with one the following commands:
+
+```bash
+# macOS (terminal)
+export PGPT_PROFILES="${PGPT_PROFILES}:bedrock"
+
+# Windows (cmd) : Windows Command Prompt doesn't have a straightforward way to directly append to an environment variable without extra tooling or scripting.
+
+# Windows (Powershell)
+$env:PGPT_PROFILES += ";bedrock"
+
+# Linux (Bash)
+export PGPT_PROFILES="${PGPT_PROFILES}:bedrock"
+```
+
+#### Run the program
+
+Then, you can run the program with the command:
+
+```bash
+poetry run python -m private_gpt
+```

private_gpt/components/embedding/embedding_component.py

@@ -44,3 +44,26 @@ def __init__(self, settings: Settings) -> None:
                 # Not a random number, is the dimensionality used by
                 # the default embedding model
                 self.embedding_model = MockEmbedding(384)
+
+            case "bedrock":
+
+                from llama_index.embeddings import BedrockEmbedding
+
+                self.embedding_model = BedrockEmbedding(
+                    model_name=settings.bedrock.embedding_modelid,
+                )
+
+                from boto3 import Session
+
+                # Access credentials using boto3
+                session = Session()
+                credentials = session.get_credentials()
+
+                # Access key ID and secret access key
+                access_key = credentials.access_key
+                secret_key = credentials.secret_key
+
+                self.embedding_model.set_credentials(aws_region=settings.bedrock.region,
+                                                     aws_access_key_id=access_key,
+                                                     aws_secret_access_key=secret_key
+                                                     )

private_gpt/components/llm/llm_component.py

@@ -56,3 +56,26 @@ def __init__(self, settings: Settings) -> None:
                 )
             case "mock":
                 self.llm = MockLLM()
+
+            case "bedrock":
+                from llama_index.llms import Bedrock
+
+                self.llm = Bedrock(
+                    model=settings.bedrock.llm_modelid,
+                    temperature=settings.bedrock.llm_temperature
+                )
+
+                from boto3 import Session
+
+                # Access credentials using boto3
+                session = Session()
+                credentials = session.get_credentials()
+
+                # Access key ID and secret access key
+                access_key = credentials.access_key
+                secret_key = credentials.secret_key
+
+                self.llm.set_credentials(aws_region=settings.bedrock.region,
+                                                     aws_access_key_id=access_key,
+                                                     aws_secret_access_key=secret_key
+                                                     )

private_gpt/settings/settings.py

@@ -81,7 +81,7 @@ class DataSettings(BaseModel):
 
 
 class LLMSettings(BaseModel):
-    mode: Literal["local", "openai", "sagemaker", "mock"]
+    mode: Literal["local", "openai", "sagemaker", "mock", "bedrock"]
     max_new_tokens: int = Field(
         256,
         description="The maximum number of token that the LLM is authorized to generate in one completion.",
@@ -111,7 +111,7 @@ class LocalSettings(BaseModel):
 
 
 class EmbeddingSettings(BaseModel):
-    mode: Literal["local", "openai", "sagemaker", "mock"]
+    mode: Literal["local", "openai", "sagemaker", "mock", "bedrock"]
     ingest_mode: Literal["simple", "batch", "parallel"] = Field(
         "simple",
         description=(
@@ -151,6 +151,13 @@ class OpenAISettings(BaseModel):
     )
 
 
+class BedrockSettings(BaseModel):
+    llm_modelid: str
+    llm_temperature: float
+    embedding_modelid: str
+    region: str
+
+
 class UISettings(BaseModel):
     enabled: bool
     path: str
@@ -226,6 +233,7 @@ class Settings(BaseModel):
     local: LocalSettings
     sagemaker: SagemakerSettings
     openai: OpenAISettings
+    bedrock: BedrockSettings
     vectorstore: VectorstoreSettings
     qdrant: QdrantSettings | None = None
 

settings-bedrock.yaml

@@ -0,0 +1,19 @@
+server:
+  env_name: ${APP_ENV:prod}
+  port: ${PORT:8001}
+
+ui:
+  enabled: true
+  path: /
+
+llm:
+  mode: bedrock
+
+embedding:
+  mode: bedrock
+
+bedrock:
+  llm_modelid: anthropic.claude-v2
+  llm_temperature: 0.5
+  embedding_modelid: amazon.titan-embed-text-v1
+  region: us-east-1

settings.yaml

@@ -56,4 +56,10 @@ sagemaker:
 
 openai:
   api_key: ${OPENAI_API_KEY:}
-  model: gpt-3.5-turbo
+  model: gpt-3.5-turbo
+
+bedrock:
+  llm_modelid: anthropic.claude-v2
+  llm_temperature: 0.5
+  embedding_modelid: amazon.titan-embed-text-v1
+  region: us-east-1