diff --git a/README.md b/README.md index b933a603b..68725281a 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ through programming." The agents within ChatDev **collaborate** by participating in specialized functional seminars, including tasks such as designing, coding, testing, and documenting. - The primary objective of ChatDev is to offer an **easy-to-use**, **highly customizable** and **extendable** framework, - which is based on large language models (LLMs) and serves as an ideal scenario for studying collective intelligence. + which is based on large language models (this fork uses OpenHermes 2.5 Mistral 7B) and serves as an ideal scenario for studying collective intelligence.

diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.py index c824a1528..22b3a1cad 100644 --- a/camel/agents/chat_agent.py +++ b/camel/agents/chat_agent.py @@ -29,6 +29,7 @@ openai_api_key_required, ) from chatdev.utils import log_visualize +import re try: from openai.types.chat import ChatCompletion @@ -109,6 +110,11 @@ def __init__( else: self.memory = None + def parse_number_bulets(self, text): + pattern = r'\d+\.\s+([^\n]+)' + matches = re.findall(pattern, text) + return matches + def reset(self) -> List[MessageType]: r"""Resets the :obj:`ChatAgent` to its initial state and returns the stored messages. @@ -201,7 +207,7 @@ def use_memory(self,input_message) -> List[MessageType]: return target_memory - @retry(wait=wait_exponential(min=5, max=60), stop=stop_after_attempt(5)) + @retry(wait=wait_exponential(min=1, max=1), stop=stop_after_attempt(1)) @openai_api_key_required def step( self, @@ -238,31 +244,36 @@ def step( if num_tokens < self.model_token_limit: response = self.model_backend.run(messages=openai_messages) if openai_new_api: - if not isinstance(response, ChatCompletion): - raise RuntimeError("OpenAI returned unexpected struct") + #if not isinstance(response, ChatCompletion): + # raise RuntimeError("OpenAI returned unexpected struct") output_messages = [ ChatMessage(role_name=self.role_name, role_type=self.role_type, meta_dict=dict(), **dict(choice.message)) - for choice in response.choices + #for choice in response.choices + for choice in self.parse_number_bulets(response) + ] info = self.get_info( - response.id, - response.usage, - [str(choice.finish_reason) for choice in response.choices], + #response.id, + #response.usage, + #[str(choice.finish_reason) for choice in self.parse_number_bulets(response)], + [str(choice) for choice in self.parse_number_bulets(response)], num_tokens, ) else: - if not isinstance(response, dict): - raise RuntimeError("OpenAI returned unexpected struct") + #if not isinstance(response, dict): + # raise RuntimeError("OpenAI returned unexpected struct") output_messages = [ ChatMessage(role_name=self.role_name, role_type=self.role_type, meta_dict=dict(), **dict(choice["message"])) - for choice in response["choices"] + #for choice in response["choices"] + for choice in self.parse_number_bulets(response) ] info = self.get_info( - response["id"], - response["usage"], - [str(choice["finish_reason"]) for choice in response["choices"]], + #response["id"], + #response["usage"], + #[str(choice["finish_reason"]) for choice in self.parse_number_bulets(response)], + [str(choice) for choice in self.parse_number_bulets(response)], num_tokens, ) diff --git a/camel/model_backend.py b/camel/model_backend.py index 394171a32..aeb2a302e 100644 --- a/camel/model_backend.py +++ b/camel/model_backend.py @@ -14,28 +14,38 @@ from abc import ABC, abstractmethod from typing import Any, Dict -import openai +import os import tiktoken from camel.typing import ModelType from chatdev.statistics import prompt_cost from chatdev.utils import log_visualize -try: - from openai.types.chat import ChatCompletion - - openai_new_api = True # new openai api version -except ImportError: - openai_new_api = False # old openai api version - -import os - -OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] -if 'BASE_URL' in os.environ: - BASE_URL = os.environ['BASE_URL'] +USE_OPENAI = False +if USE_OPENAI == True: + import openai + try: + from openai.types.chat import ChatCompletion + + openai_new_api = True # new openai api version + except ImportError: + openai_new_api = False # old openai api version + + OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] + if 'BASE_URL' in os.environ: + BASE_URL = os.environ['BASE_URL'] + else: + BASE_URL = None else: - BASE_URL = None - + import re + from urllib.parse import urlencode + import subprocess + import json + import jsonstreams + from io import StringIO + from contextlib import redirect_stdout + BASE_URL = "http://localhost:11434/api/generate" + mistral_new_api = True # new mistral api version class ModelBackend(ABC): r"""Base class for different model backends. @@ -145,6 +155,107 @@ def run(self, *args, **kwargs): return response +class MistralAIModel(ModelBackend): + r"""Mistral API in a unified ModelBackend interface.""" + + def __init__(self, model_type: ModelType, model_config_dict: Dict) -> None: + super().__init__() + self.model_type = model_type + self.model_config_dict = model_config_dict + + def generate_stream_json_response(self, prompt): + data = json.dumps({"model": "openhermes", "prompt": prompt}) + process = subprocess.Popen(["curl", "-X", "POST", "-d", data, "http://localhost:11434/api/generate"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + full_response = "" + with jsonstreams.Stream(jsonstreams.Type.array, filename='./response_log.txt') as output: + while True: + line, _ = process.communicate() + if not line: + break + try: + record = line.decode("utf-8").split("\n") + for i in range(len(record)-1): + data = json.loads(record[i].replace('\0', '')) + if "response" in data: + full_response += data["response"] + with output.subobject() as output_e: + output_e.write('response', data["response"]) + else: + return full_response.replace('\0', '') + if len(record)==1: + data = json.loads(record[0].replace('\0', '')) + if "error" in data: + full_response += data["error"] + with output.subobject() as output_e: + output_e.write('error', data["error"]) + return full_response.replace('\0', '') + except Exception as error: + # handle the exception + print("An exception occurred:", error) + return full_response.replace('\0', '') + + def run(self, *args, **kwargs): + string = "\n".join([message["content"] for message in kwargs["messages"]]) + #fake model to enable tiktoken to work with mistral + #encoding = tiktoken.encoding_for_model(self.model_type.value) + encoding = tiktoken.encoding_for_model(ModelType.GPT_3_5_TURBO.value) #fake model to enable tiktoken to work with mistral + num_prompt_tokens = len(encoding.encode(string)) + gap_between_send_receive = 15 * len(kwargs["messages"]) + num_prompt_tokens += gap_between_send_receive + + if mistral_new_api: + # Experimental, add base_url + num_max_token_map = { + "Mistral-7B": 8192, + } + num_max_token = num_max_token_map["Mistral-7B"] #hard coded model to enable tiktoken to work with mistral + num_max_completion_tokens = num_max_token - num_prompt_tokens + self.model_config_dict['max_tokens'] = num_max_completion_tokens + + #response = client.chat.completions.create(*args, **kwargs, model=self.model_type.value, + # **self.model_config_dict) + print("DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG") + print("args:", args) + print("message: ", kwargs["messages"]) + print("self.model_config_dict:", self.model_config_dict['max_tokens']) + print("prompt: ", string) + response = self.generate_stream_json_response("<|im_start|>system" + '\n' + string + "<|im_end|>") + print("--> ", response) + print("DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG") + log_visualize( + "**[Mistral_Usage_Info Receive]**\ncost: ${:.6f}\n".format(len(response.split()))) + return response + else: + num_max_token_map = { + "Mistral-7B": 8192, + } + num_max_token = num_max_token_map[self.model_type.value] + num_max_completion_tokens = num_max_token - num_prompt_tokens + self.model_config_dict['max_tokens'] = num_max_completion_tokens + print("DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG") + print("args:", args) + print("message: ", kwargs["messages"]) + print("self.model_config_dict:", self.model_config_dict['max_tokens']) + print("prompt: ", string) + response = self.generate_stream_json_response("<|im_start|>system" + '\n' + string + '\n' + "And always answer with a number of choices" +"<|im_end|>") + print("--> ", response) + print("DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG") + log_visualize( + "**[Mistral_Usage_Info Receive]**\ncost: ${:.6f}\n".format(len(response.split()))) + + cost = prompt_cost( + self.model_type.value, + num_prompt_tokens=len(response.split()),#response["usage"]["prompt_tokens"], + num_completion_tokens=len(response.split()) #response["usage"]["completion_tokens"] + ) + + log_visualize( + "**[Mistral_Usage_Info Receive]**\n\ncost: ${:.6f}\n".format( + response["usage"]["total_tokens"], cost)) + + return response + + class StubModel(ModelBackend): r"""A dummy model used for unit tests.""" @@ -173,7 +284,7 @@ class ModelFactory: @staticmethod def create(model_type: ModelType, model_config_dict: Dict) -> ModelBackend: - default_model_type = ModelType.GPT_3_5_TURBO + default_model_type = ModelType.MISTRAL_7B if model_type in { ModelType.GPT_3_5_TURBO, @@ -182,9 +293,10 @@ def create(model_type: ModelType, model_config_dict: Dict) -> ModelBackend: ModelType.GPT_4_32k, ModelType.GPT_4_TURBO, ModelType.GPT_4_TURBO_V, + ModelType.MISTRAL_7B, None }: - model_class = OpenAIModel + model_class = MistralAIModel elif model_type == ModelType.STUB: model_class = StubModel else: diff --git a/camel/typing.py b/camel/typing.py index e922c5b52..700d1aba7 100644 --- a/camel/typing.py +++ b/camel/typing.py @@ -50,12 +50,13 @@ class ModelType(Enum): GPT_4_32k = "gpt-4-32k" GPT_4_TURBO = "gpt-4-1106-preview" GPT_4_TURBO_V = "gpt-4-1106-vision-preview" + MISTRAL_7B = "Mistral-7B" STUB = "stub" @property def value_for_tiktoken(self): - return self.value if self.name != "STUB" else "gpt-3.5-turbo-16k-0613" + return self.value if self.name != "STUB" else "Mistral-7B" class PhaseType(Enum): diff --git a/camel/utils.py b/camel/utils.py index a2713af34..700d1aba7 100644 --- a/camel/utils.py +++ b/camel/utils.py @@ -11,219 +11,76 @@ # See the License for the specific language governing permissions and # limitations under the License. # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. =========== -import os -import re -import zipfile -from functools import wraps -from typing import Any, Callable, List, Optional, Set, TypeVar - -import requests -import tiktoken - -from camel.messages import OpenAIMessage -from camel.typing import ModelType, TaskType - -F = TypeVar('F', bound=Callable[..., Any]) - -import time - - -def count_tokens_openai_chat_models( - messages: List[OpenAIMessage], - encoding: Any, -) -> int: - r"""Counts the number of tokens required to generate an OpenAI chat based - on a given list of messages. - - Args: - messages (List[OpenAIMessage]): The list of messages. - encoding (Any): The encoding method to use. - - Returns: - int: The number of tokens required. - """ - num_tokens = 0 - for message in messages: - # message follows {role/name}\n{content}\n - num_tokens += 4 - for key, value in message.items(): - num_tokens += len(encoding.encode(value)) - if key == "name": # if there's a name, the role is omitted - num_tokens += -1 # role is always 1 token - num_tokens += 2 # every reply is primed with assistant - return num_tokens - - -def num_tokens_from_messages( - messages: List[OpenAIMessage], - model: ModelType, -) -> int: - r"""Returns the number of tokens used by a list of messages. - - Args: - messages (List[OpenAIMessage]): The list of messages to count the - number of tokens for. - model (ModelType): The OpenAI model used to encode the messages. - - Returns: - int: The total number of tokens used by the messages. - - Raises: - NotImplementedError: If the specified `model` is not implemented. - - References: - - https://github.com/openai/openai-python/blob/main/chatml.md - - https://platform.openai.com/docs/models/gpt-4 - - https://platform.openai.com/docs/models/gpt-3-5 - """ - try: - value_for_tiktoken = model.value_for_tiktoken - encoding = tiktoken.encoding_for_model(value_for_tiktoken) - except KeyError: - encoding = tiktoken.get_encoding("cl100k_base") - - if model in { - ModelType.GPT_3_5_TURBO, - ModelType.GPT_3_5_TURBO_NEW, - ModelType.GPT_4, - ModelType.GPT_4_32k, - ModelType.GPT_4_TURBO, - ModelType.GPT_4_TURBO_V, - ModelType.STUB - }: - return count_tokens_openai_chat_models(messages, encoding) - else: - raise NotImplementedError( - f"`num_tokens_from_messages`` is not presently implemented " - f"for model {model}. " - f"See https://github.com/openai/openai-python/blob/main/chatml.md " - f"for information on how messages are converted to tokens. " - f"See https://platform.openai.com/docs/models/gpt-4" - f"or https://platform.openai.com/docs/models/gpt-3-5" - f"for information about openai chat models.") - - -def get_model_token_limit(model: ModelType) -> int: - r"""Returns the maximum token limit for a given model. - - Args: - model (ModelType): The type of the model. - - Returns: - int: The maximum token limit for the given model. - """ - if model == ModelType.GPT_3_5_TURBO: - return 16384 - elif model == ModelType.GPT_3_5_TURBO_NEW: - return 16384 - elif model == ModelType.GPT_4: - return 8192 - elif model == ModelType.GPT_4_32k: - return 32768 - elif model == ModelType.GPT_4_TURBO: - return 128000 - elif model == ModelType.STUB: - return 4096 - else: - raise ValueError("Unknown model type") - - -def openai_api_key_required(func: F) -> F: - r"""Decorator that checks if the OpenAI API key is available in the - environment variables. - - Args: - func (callable): The function to be wrapped. - - Returns: - callable: The decorated function. - - Raises: - ValueError: If the OpenAI API key is not found in the environment - variables. - """ - - @wraps(func) - def wrapper(self, *args, **kwargs): - from camel.agents.chat_agent import ChatAgent - if not isinstance(self, ChatAgent): - raise ValueError("Expected ChatAgent") - if self.model == ModelType.STUB: - return func(self, *args, **kwargs) - elif 'OPENAI_API_KEY' in os.environ: - return func(self, *args, **kwargs) - else: - raise ValueError('OpenAI API key not found.') - - return wrapper - - -def print_text_animated(text, delay: float = 0.005, end: str = ""): - r"""Prints the given text with an animated effect. - - Args: - text (str): The text to print. - delay (float, optional): The delay between each character printed. - (default: :obj:`0.02`) - end (str, optional): The end character to print after the text. - (default: :obj:`""`) - """ - for char in text: - print(char, end=end, flush=True) - time.sleep(delay) - print('\n') - - -def get_prompt_template_key_words(template: str) -> Set[str]: - r"""Given a string template containing curly braces {}, return a set of - the words inside the braces. - - Args: - template (str): A string containing curly braces. - - Returns: - List[str]: A list of the words inside the curly braces. - - Example: - >>> get_prompt_template_key_words('Hi, {name}! How are you {status}?') - {'name', 'status'} - """ - return set(re.findall(r'{([^}]*)}', template)) - - -def get_first_int(string: str) -> Optional[int]: - r"""Returns the first integer number found in the given string. - - If no integer number is found, returns None. - - Args: - string (str): The input string. - - Returns: - int or None: The first integer number found in the string, or None if - no integer number is found. - """ - match = re.search(r'\d+', string) - if match: - return int(match.group()) - else: - return None - - -def download_tasks(task: TaskType, folder_path: str) -> None: - # Define the path to save the zip file - zip_file_path = os.path.join(folder_path, "tasks.zip") - - # Download the zip file from the Google Drive link - response = requests.get("https://huggingface.co/datasets/camel-ai/" - f"metadata/resolve/main/{task.value}_tasks.zip") - - # Save the zip file - with open(zip_file_path, "wb") as f: - f.write(response.content) - - with zipfile.ZipFile(zip_file_path, "r") as zip_ref: - zip_ref.extractall(folder_path) - - # Delete the zip file - os.remove(zip_file_path) +from enum import Enum + + +class TaskType(Enum): + AI_SOCIETY = "ai_society" + CODE = "code" + MISALIGNMENT = "misalignment" + TRANSLATION = "translation" + EVALUATION = "evaluation" + SOLUTION_EXTRACTION = "solution_extraction" + CHATDEV = "chat_dev" + DEFAULT = "default" + + +class RoleType(Enum): + ASSISTANT = "assistant" + USER = "user" + CRITIC = "critic" + EMBODIMENT = "embodiment" + DEFAULT = "default" + CHATDEV = "AgentTech" + CHATDEV_COUNSELOR = "counselor" + CHATDEV_CEO = "chief executive officer (CEO)" + CHATDEV_CHRO = "chief human resource officer (CHRO)" + CHATDEV_CPO = "chief product officer (CPO)" + CHATDEV_CTO = "chief technology officer (CTO)" + CHATDEV_PROGRAMMER = "programmer" + CHATDEV_REVIEWER = "code reviewer" + CHATDEV_TESTER = "software test engineer" + CHATDEV_CCO = "chief creative officer (CCO)" + + +class ModelType(Enum): + GPT_3_5_TURBO = "gpt-3.5-turbo-16k-0613" + GPT_3_5_TURBO_NEW = "gpt-3.5-turbo-16k" + GPT_4 = "gpt-4" + GPT_4_32k = "gpt-4-32k" + GPT_4_TURBO = "gpt-4-1106-preview" + GPT_4_TURBO_V = "gpt-4-1106-vision-preview" + MISTRAL_7B = "Mistral-7B" + + STUB = "stub" + + @property + def value_for_tiktoken(self): + return self.value if self.name != "STUB" else "Mistral-7B" + + +class PhaseType(Enum): + REFLECTION = "reflection" + RECRUITING_CHRO = "recruiting CHRO" + RECRUITING_CPO = "recruiting CPO" + RECRUITING_CTO = "recruiting CTO" + DEMAND_ANALYSIS = "demand analysis" + CHOOSING_LANGUAGE = "choosing language" + RECRUITING_PROGRAMMER = "recruiting programmer" + RECRUITING_REVIEWER = "recruiting reviewer" + RECRUITING_TESTER = "recruiting software test engineer" + RECRUITING_CCO = "recruiting chief creative officer" + CODING = "coding" + CODING_COMPLETION = "coding completion" + CODING_AUTOMODE = "coding auto mode" + REVIEWING_COMMENT = "review comment" + REVIEWING_MODIFICATION = "code modification after reviewing" + ERROR_SUMMARY = "error summary" + MODIFICATION = "code modification" + ART_ELEMENT_ABSTRACTION = "art element abstraction" + ART_ELEMENT_INTEGRATION = "art element integration" + CREATING_ENVIRONMENT_DOCUMENT = "environment document" + CREATING_USER_MANUAL = "user manual" + + +__all__ = ["TaskType", "RoleType", "ModelType", "PhaseType"] diff --git a/camel/web_spider.py b/camel/web_spider.py index 31e1f70d2..2aec0b971 100644 --- a/camel/web_spider.py +++ b/camel/web_spider.py @@ -1,23 +1,67 @@ import requests from bs4 import BeautifulSoup -import openai -from openai import OpenAI import wikipediaapi import os import time -self_api_key = os.environ.get('OPENAI_API_KEY') -BASE_URL = os.environ.get('BASE_URL') +USE_OPENAI = False +if USE_OPENAI == True: + import openai + from openai import OpenAI + self_api_key = os.environ.get('OPENAI_API_KEY') + BASE_URL = os.environ.get('BASE_URL') -if BASE_URL: - client = openai.OpenAI( - api_key=self_api_key, - base_url=BASE_URL, - ) + if BASE_URL: + client = openai.OpenAI( + api_key=self_api_key, + base_url=BASE_URL, + ) + else: + client = openai.OpenAI( + api_key=self_api_key + ) else: - client = openai.OpenAI( - api_key=self_api_key - ) + import re + from urllib.parse import urlencode + import subprocess + import json + import jsonstreams + from io import StringIO + from contextlib import redirect_stdout + BASE_URL = "http://localhost:11434/api/generate" + mistral_new_api = True # new mistral api version + + def generate_stream_json_response(prompt): + data = json.dumps({"model": "openhermes", "prompt": prompt}) + process = subprocess.Popen(["curl", "-X", "POST", "-d", data, "http://localhost:11434/api/generate"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + full_response = "" + with jsonstreams.Stream(jsonstreams.Type.array, filename='./response_log.txt') as output: + while True: + line, _ = process.communicate() + if not line: + break + try: + record = line.decode("utf-8").split("\n") + for i in range(len(record)-1): + data = json.loads(record[i].replace('\0', '')) + if "response" in data: + full_response += data["response"] + with output.subobject() as output_e: + output_e.write('response', data["response"]) + else: + return full_response.replace('\0', '') + if len(record)==1: + data = json.loads(record[0].replace('\0', '')) + if "error" in data: + full_response += data["error"] + with output.subobject() as output_e: + output_e.write('error', data["error"]) + return full_response.replace('\0', '') + except Exception as error: + # handle the exception + print("An exception occurred:", error) + return full_response.replace('\0', '') + def get_baidu_baike_content(keyword): # design api by the baidubaike @@ -56,34 +100,36 @@ def modal_trans(task_dsp): try: task_in ="'" + task_dsp + \ "'Just give me the most important keyword about this sentence without explaining it and your answer should be only one keyword." - messages = [{"role": "user", "content": task_in}] - response = client.chat.completions.create(messages=messages, - model="gpt-3.5-turbo-16k", - temperature=0.2, - top_p=1.0, - n=1, - stream=False, - frequency_penalty=0.0, - presence_penalty=0.0, - logit_bias={}) + #messages = [{"role": "user", "content": task_in}] + #response = client.chat.completions.create(messages=messages, + #model="gpt-3.5-turbo-16k", + #temperature=0.2, + #top_p=1.0, + #n=1, + #stream=False, + #frequency_penalty=0.0, + #presence_penalty=0.0, + #logit_bias={}) + response = generate_stream_json_response("<|im_start|>user" + '\n' + task_in + "<|im_end|>") response_text = response.choices[0].message.content spider_content = get_wiki_content(response_text) # time.sleep(1) task_in = "'" + spider_content + \ "',Summarize this paragraph and return the key information." messages = [{"role": "user", "content": task_in}] - response = client.chat.completions.create(messages=messages, - model="gpt-3.5-turbo-16k", - temperature=0.2, - top_p=1.0, - n=1, - stream=False, - frequency_penalty=0.0, - presence_penalty=0.0, - logit_bias={}) + #response = client.chat.completions.create(messages=messages, + #model="gpt-3.5-turbo-16k", + #temperature=0.2, + #top_p=1.0, + #n=1, + #stream=False, + #frequency_penalty=0.0, + #presence_penalty=0.0, + #logit_bias={}) + response = generate_stream_json_response("<|im_start|>user" + '\n' + task_in + "<|im_end|>") result = response.choices[0].message.content print("web spider content:", result) except: result = '' print("the content is none") - return result \ No newline at end of file + return result diff --git a/ecl/embedding.py b/ecl/embedding.py index f8e90bc49..5f9e6f102 100644 --- a/ecl/embedding.py +++ b/ecl/embedding.py @@ -1,11 +1,24 @@ import os -import openai -from openai import OpenAI -OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] -if 'BASE_URL' in os.environ: - BASE_URL = os.environ['BASE_URL'] + +USE_OPENAI = False +if USE_OPENAI == True: + import openai + from openai import OpenAI + OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] + if 'BASE_URL' in os.environ: + BASE_URL = os.environ['BASE_URL'] + else: + BASE_URL = None else: - BASE_URL = None + import re + from urllib.parse import urlencode + import subprocess + import json + import jsonstreams + from io import StringIO + from contextlib import redirect_stdout + BASE_URL = "http://localhost:11434/api/generate" + mistral_new_api = True # new mistral api version import sys import time from tenacity import ( @@ -27,7 +40,8 @@ def __init__(self, **params): self.prompt_tokens = 0 self.total_tokens = 0 - @retry(wait=wait_random_exponential(min=2, max=5), stop=stop_after_attempt(10)) + #@retry(wait=wait_random_exponential(min=2, max=5), stop=stop_after_attempt(10)) + @retry(wait=wait_random_exponential(min=1, max=1), stop=stop_after_attempt(1)) def get_text_embedding(self,text: str): if BASE_URL: client = openai.OpenAI( @@ -82,3 +96,84 @@ def get_code_embedding(self,code: str): return embedding +class MistralAIEmbedding: + def __init__(self, **params): + self.code_prompt_tokens = 0 + self.text_prompt_tokens = 0 + self.code_total_tokens = 0 + self.text_total_tokens = 0 + + self.prompt_tokens = 0 + self.total_tokens = 0 + + def generate_stream_json_response(self, prompt): + data = json.dumps({"model": "openhermes", "prompt": prompt}) + process = subprocess.Popen(["curl", "-X", "POST", "-d", data, "http://localhost:11434/api/generate"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + full_response = "" + with jsonstreams.Stream(jsonstreams.Type.array, filename='./response_log.txt') as output: + while True: + line, _ = process.communicate() + if not line: + break + try: + record = line.decode("utf-8").split("\n") + for i in range(len(record)-1): + data = json.loads(record[i].replace('\0', '')) + if "response" in data: + full_response += data["response"] + with output.subobject() as output_e: + output_e.write('response', data["response"]) + else: + return full_response.replace('\0', '') + if len(record)==1: + data = json.loads(record[0].replace('\0', '')) + if "error" in data: + full_response += data["error"] + with output.subobject() as output_e: + output_e.write('error', data["error"]) + return full_response.replace('\0', '') + except Exception as error: + # handle the exception + print("An exception occurred:", error) + return full_response.replace('\0', '') + + #@retry(wait=wait_random_exponential(min=2, max=5), stop=stop_after_attempt(10)) + @retry(wait=wait_random_exponential(min=1, max=1), stop=stop_after_attempt(1)) + def get_text_embedding(self,text: str): + if len(text)>8191: + text = text[:8190] + response = self.generate_stream_json_response("<|im_start|>user" + '\n' + text + "<|im_end|>") + embedding = response['data'][0]['embedding'] + log_and_print_online( + "Get text embedding from {}:\n**[Mistral_Usage_Info Receive]**\nprompt_tokens: {}\ntotal_tokens: {}\n".format( + response["model"],response["usage"]["prompt_tokens"],response["usage"]["total_tokens"])) + self.text_prompt_tokens += response["usage"]["prompt_tokens"] + self.text_total_tokens += response["usage"]["total_tokens"] + self.prompt_tokens += response["usage"]["prompt_tokens"] + self.total_tokens += response["usage"]["total_tokens"] + + return embedding + + #@retry(wait=wait_random_exponential(min=10, max=60), stop=stop_after_attempt(10)) + @retry(wait=wait_random_exponential(min=1, max=1), stop=stop_after_attempt(1)) + def get_code_embedding(self,code: str): + if len(code) == 0: + code = "#" + elif len(code) >8191: + code = code[0:8190] + response = self.generate_stream_json_response("<|im_start|>user" + '\n' + code + "<|im_end|>") + embedding = response['data'][0]['embedding'] + log_and_print_online( + "Get code embedding from {}:\n**[Mistral_Usage_Info Receive]**\nprompt_tokens: {}\ntotal_tokens: {}\n".format( + response["model"],response["usage"]["prompt_tokens"],response["usage"]["total_tokens"])) + + self.code_prompt_tokens += response["usage"]["prompt_tokens"] + self.code_total_tokens += response["usage"]["total_tokens"] + self.prompt_tokens += response["usage"]["prompt_tokens"] + self.total_tokens += response["usage"]["total_tokens"] + + return embedding + + + + diff --git a/ecl/experience.py b/ecl/experience.py index 6e363d6f6..67f1ec30a 100644 --- a/ecl/experience.py +++ b/ecl/experience.py @@ -5,7 +5,7 @@ import openai import numpy as np from codes import Codes -from utils import get_easyDict_from_filepath,OpenAIModel,log_and_print_online +from utils import get_easyDict_from_filepath,OpenAIModel,MistralAIModel,log_and_print_online from embedding import OpenAIEmbedding sys.path.append(os.path.join(os.getcwd(),"ecl")) class Shortcut: @@ -29,7 +29,9 @@ def __init__(self, graph: Graph, directory: str): self.upperLimit = cfg.experience.upper_limit self.experiences = [] - self.model = OpenAIModel(model_type="gpt-3.5-turbo-16k") + #self.model = OpenAIModel(model_type="gpt-3.5-turbo-16k") + #self.embedding_method = OpenAIEmbedding() + self.model = MistralAIModel(model_type="Mistral-7B") self.embedding_method = OpenAIEmbedding() for edge in self.graph.edges: diff --git a/ecl/utils.py b/ecl/utils.py index cb11cda1e..17b88789c 100644 --- a/ecl/utils.py +++ b/ecl/utils.py @@ -16,11 +16,24 @@ stop_after_attempt, wait_exponential ) -OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] -if 'BASE_URL' in os.environ: - BASE_URL = os.environ['BASE_URL'] + +USE_OPENAI = False +if USE_OPENAI == True: + OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] + if 'BASE_URL' in os.environ: + BASE_URL = os.environ['BASE_URL'] + else: + BASE_URL = None else: - BASE_URL = None + import re + from urllib.parse import urlencode + import subprocess + import json + import jsonstreams + from io import StringIO + from contextlib import redirect_stdout + BASE_URL = "http://localhost:11434/api/generate" + mistral_new_api = True # new mistral api version def getFilesFromType(sourceDir, filetype): files = [] @@ -51,8 +64,10 @@ def get_easyDict_from_filepath(path: str): def calc_max_token(messages, model): + #fake model to enable tiktoken to work with mistral + model = "gpt-3.5-turbo-16k" string = "\n".join([message["content"] for message in messages]) - encoding = tiktoken.encoding_for_model(model) + encoding = tiktoken.encoding_for_model(model) #fake model to enable tiktoken to work with mistral num_prompt_tokens = len(encoding.encode(string)) gap_between_send_receive = 50 num_prompt_tokens += gap_between_send_receive @@ -65,6 +80,7 @@ def calc_max_token(messages, model): "gpt-4": 8192, "gpt-4-0613": 8192, "gpt-4-32k": 32768, + "Mistral-7B": 8192, } num_max_token = num_max_token_map[model] num_max_completion_tokens = num_max_token - num_prompt_tokens @@ -166,6 +182,104 @@ def run(self, messages) : raise RuntimeError("Unexpected return from OpenAI API") return response + +class MistralAIModel(ModelBackend): + r"""Mistral API in a unified ModelBackend interface.""" + + def __init__(self, model_type, model_config_dict: Dict=None) -> None: + super().__init__() + self.model_type = model_type + self.model_config_dict = model_config_dict + if self.model_config_dict == None: + self.model_config_dict = {"temperature": 0.2, + "top_p": 1.0, + "n": 1, + "stream": False, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "logit_bias": {}, + } + self.prompt_tokens = 0 + self.completion_tokens = 0 + self.total_tokens = 0 + + def generate_stream_json_response(self, prompt): + data = json.dumps({"model": "openhermes", "prompt": prompt}) + process = subprocess.Popen(["curl", "-X", "POST", "-d", data, "http://localhost:11434/api/generate"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + full_response = "" + with jsonstreams.Stream(jsonstreams.Type.array, filename='./response_log.txt') as output: + while True: + line, _ = process.communicate() + if not line: + break + try: + record = line.decode("utf-8").split("\n") + for i in range(len(record)-1): + data = json.loads(record[i].replace('\0', '')) + if "response" in data: + full_response += data["response"] + with output.subobject() as output_e: + output_e.write('response', data["response"]) + else: + return full_response.replace('\0', '') + if len(record)==1: + data = json.loads(record[0].replace('\0', '')) + if "error" in data: + full_response += data["error"] + with output.subobject() as output_e: + output_e.write('error', data["error"]) + return full_response.replace('\0', '') + except Exception as error: + # handle the exception + print("An exception occurred:", error) + return full_response.replace('\0', '') + + #@retry(wait=wait_exponential(min=5, max=60), stop=stop_after_attempt(5)) + @retry(wait=wait_exponential(min=1, max=1), stop=stop_after_attempt(1)) + def run(self, messages) : + if BASE_URL: + client = openai.OpenAI( + api_key=OPENAI_API_KEY, + base_url=BASE_URL, + ) + else: + client = openai.OpenAI( + api_key=OPENAI_API_KEY + ) + current_retry = 0 + max_retry = 1 + + string = "\n".join([message["content"] for message in messages]) + encoding = tiktoken.encoding_for_model(self.model_type) + num_prompt_tokens = len(encoding.encode(string)) + gap_between_send_receive = 15 * len(messages) + num_prompt_tokens += gap_between_send_receive + + num_max_token_map = { + "Mistral-7B": 8192, + } + #response = client.chat.completions.create(messages = messages, + #model = "gpt-3.5-turbo-16k", + #temperature = 0.2, + #top_p = 1.0, + #n = 1, + #stream = False, + #frequency_penalty = 0.0, + #presence_penalty = 0.0, + #logit_bias = {}, + #).model_dump() + response = self.generate_stream_json_response("<|im_start|>user" + '\n' + messages + "<|im_end|>") + #response_text = response['choices'][0]['message']['content'] + response_text = response + num_max_token = num_max_token_map[self.model_type] + num_max_completion_tokens = num_max_token - num_prompt_tokens + self.model_config_dict['max_tokens'] = num_max_completion_tokens + log_and_print_online("InstructionStar generation:\n**[Mistral_Usage_Info Receive]**\nprompt_tokens: {}\n".format(len(response.split()))) + self.prompt_tokens += len(response.split()) #response["usage"]["prompt_tokens"] + self.completion_tokens += len(response.split())#response["usage"]["completion_tokens"] + self.total_tokens +=len(response.split())# response["usage"]["total_tokens"] + + return response def now(): return time.strftime("%Y%m%d%H%M%S", time.localtime()) diff --git a/requirements.txt b/requirements.txt index 0e9f28136..e0537e99d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ beautifulsoup4==4.12.2 faiss-cpu==1.7.4 pyyaml==6.0 easydict==1.10 +jsonstreams==0.6.0 diff --git a/run.py b/run.py index 0cea00f01..ee48fd973 100644 --- a/run.py +++ b/run.py @@ -35,6 +35,7 @@ "Please update as specified in requirement.txt. \n " "The old API interface is deprecated and will no longer be supported.") +mistral_new_api = True # new mistral api version def get_config(company): """ @@ -78,8 +79,8 @@ def get_config(company): help="Prompt of software") parser.add_argument('--name', type=str, default="Gomoku", help="Name of software, your software will be generated in WareHouse/name_org_timestamp") -parser.add_argument('--model', type=str, default="GPT_3_5_TURBO", - help="GPT Model, choose from {'GPT_3_5_TURBO','GPT_4','GPT_4_32K', 'GPT_4_TURBO'}") +parser.add_argument('--model', type=str, default="MISTRAL-7B", + help="GPT Model, choose from {'GPT_3_5_TURBO','GPT_4','GPT_4_32K','GPT_4_TURBO','MISTRAL-7B'}") parser.add_argument('--path', type=str, default="", help="Your file directory, ChatDev will build upon your software in the Incremental mode") args = parser.parse_args() @@ -94,10 +95,14 @@ def get_config(company): 'GPT_4': ModelType.GPT_4, 'GPT_4_32K': ModelType.GPT_4_32k, 'GPT_4_TURBO': ModelType.GPT_4_TURBO, - 'GPT_4_TURBO_V': ModelType.GPT_4_TURBO_V + 'GPT_4_TURBO_V': ModelType.GPT_4_TURBO_V, + 'MISTRAL-7B': ModelType.MISTRAL_7B } -if openai_new_api: - args2type['GPT_3_5_TURBO'] = ModelType.GPT_3_5_TURBO_NEW +#if openai_new_api: +# args2type['GPT_3_5_TURBO'] = ModelType.GPT_3_5_TURBO_NEW + +if mistral_new_api: + args2type['MISTRAL-7B'] = ModelType.MISTRAL_7B chat_chain = ChatChain(config_path=config_path, config_phase_path=config_phase_path,