diff --git a/src/crewai/agent.py b/src/crewai/agent.py index db0c5d3057..aa795d209b 100644 --- a/src/crewai/agent.py +++ b/src/crewai/agent.py @@ -12,6 +12,7 @@ ConfigDict, Field, InstanceOf, + PrivateAttr, field_validator, model_validator, ) @@ -46,6 +47,8 @@ class Agent(BaseModel): """ __hash__ = object.__hash__ + _request_within_rpm_limit: Any = PrivateAttr(default=None) + model_config = ConfigDict(arbitrary_types_allowed=True) id: UUID4 = Field( default_factory=uuid.uuid4, @@ -139,6 +142,10 @@ def set_cache_handler(self, cache_handler) -> None: self.tools_handler = ToolsHandler(cache=self.cache_handler) self.__create_agent_executor() + def set_request_within_rpm_limit(self, ensure_function) -> None: + self._request_within_rpm_limit = ensure_function + self.__create_agent_executor() + def __create_agent_executor(self) -> CrewAgentExecutor: """Create an agent executor for the agent. @@ -157,11 +164,12 @@ def __create_agent_executor(self) -> CrewAgentExecutor: "verbose": self.verbose, "handle_parsing_errors": True, "max_iterations": self.max_iter, + "request_within_rpm_limit": self._request_within_rpm_limit, } if self.memory: summary_memory = ConversationSummaryMemory( - llm=self.llm, memory_key="chat_history", input_key="input" + llm=self.llm, input_key="input", memory_key="chat_history" ) executor_args["memory"] = summary_memory agent_args["chat_history"] = lambda x: x["chat_history"] diff --git a/src/crewai/agents/executor.py b/src/crewai/agents/executor.py index 4be00c6bca..d8856496b8 100644 --- a/src/crewai/agents/executor.py +++ b/src/crewai/agents/executor.py @@ -19,6 +19,7 @@ class CrewAgentExecutor(AgentExecutor): i18n: I18N = I18N() iterations: int = 0 + request_within_rpm_limit: Any = None max_iterations: Optional[int] = 15 force_answer_max_iterations: Optional[int] = None @@ -54,29 +55,30 @@ def _call( start_time = time.time() # We now enter the agent loop (until it returns something). while self._should_continue(self.iterations, time_elapsed): - next_step_output = self._take_next_step( - name_to_tool_map, - color_mapping, - inputs, - intermediate_steps, - run_manager=run_manager, - ) - if isinstance(next_step_output, AgentFinish): - return self._return( - next_step_output, intermediate_steps, run_manager=run_manager + if not self.request_within_rpm_limit or self.request_within_rpm_limit(): + next_step_output = self._take_next_step( + name_to_tool_map, + color_mapping, + inputs, + intermediate_steps, + run_manager=run_manager, ) - - intermediate_steps.extend(next_step_output) - if len(next_step_output) == 1: - next_step_action = next_step_output[0] - # See if tool should return directly - tool_return = self._get_tool_return(next_step_action) - if tool_return is not None: + if isinstance(next_step_output, AgentFinish): return self._return( - tool_return, intermediate_steps, run_manager=run_manager + next_step_output, intermediate_steps, run_manager=run_manager ) - self.iterations += 1 - time_elapsed = time.time() - start_time + + intermediate_steps.extend(next_step_output) + if len(next_step_output) == 1: + next_step_action = next_step_output[0] + # See if tool should return directly + tool_return = self._get_tool_return(next_step_action) + if tool_return is not None: + return self._return( + tool_return, intermediate_steps, run_manager=run_manager + ) + self.iterations += 1 + time_elapsed = time.time() - start_time output = self.agent.return_stopped_response( self.early_stopping_method, intermediate_steps, **inputs ) diff --git a/src/crewai/crew.py b/src/crewai/crew.py index 8ce27371b6..f5016a2488 100644 --- a/src/crewai/crew.py +++ b/src/crewai/crew.py @@ -1,6 +1,8 @@ import json +import threading +import time import uuid -from typing import Any, Dict, List, Optional, Union +from typing import Any, ClassVar, Dict, List, Optional, Union from pydantic import ( UUID4, @@ -9,6 +11,7 @@ Field, InstanceOf, Json, + PrivateAttr, field_validator, model_validator, ) @@ -33,10 +36,16 @@ class Crew(BaseModel): verbose: Indicates the verbosity level for logging during execution. config: Configuration settings for the crew. cache_handler: Handles caching for the crew's operations. + max_rpm: Maximum number of requests per minute for the crew execution to be respected. + rpm: Current number of requests per minute for the crew execution. id: A unique identifier for the crew instance. """ __hash__ = object.__hash__ + _timer: Optional[threading.Timer] = PrivateAttr(default=None) + lock: ClassVar[threading.Lock] = threading.Lock() + rpm: ClassVar[int] = 0 + max_rpm: Optional[int] = Field(default=None) model_config = ConfigDict(arbitrary_types_allowed=True) tasks: List[Task] = Field(default_factory=list) agents: List[Agent] = Field(default_factory=list) @@ -64,6 +73,12 @@ def _deny_user_set_id(cls, v: Optional[UUID4]) -> None: def check_config_type(cls, v: Union[Json, Dict[str, Any]]): return json.loads(v) if isinstance(v, Json) else v + @model_validator(mode="after") + def set_reset_counter(self): + if self.max_rpm: + self._reset_request_count() + return self + @model_validator(mode="after") def check_config(self): """Validates that the crew is properly configured with agents and tasks.""" @@ -80,6 +95,7 @@ def check_config(self): if self.agents: for agent in self.agents: agent.set_cache_handler(self.cache_handler) + agent.set_request_within_rpm_limit(self.ensure_request_within_rpm_limit) return self def _setup_from_config(self): @@ -100,6 +116,24 @@ def _create_task(self, task_config): del task_config["agent"] return Task(**task_config, agent=task_agent) + def ensure_request_within_rpm_limit(self): + if not self.max_rpm: + return True + + with Crew.lock: + if Crew.rpm < self.max_rpm: + Crew.rpm += 1 + return True + self._log("info", "Max RPM reached, waiting for next minute to start.") + + return self._wait_for_next_minute() + + def _wait_for_next_minute(self): + time.sleep(60) + with Crew.lock: + Crew.rpm = 0 + return True + def kickoff(self) -> str: """Starts the crew to work on its assigned tasks.""" for agent in self.agents: @@ -115,9 +149,8 @@ def _sequential_loop(self) -> str: for task in self.tasks: self._prepare_and_execute_task(task) task_output = task.execute(task_output) - self._log( - "debug", f"\n\n[{task.agent.role}] Task output: {task_output}\n\n" - ) + self._log("debug", f"\n[{task.agent.role}] Task output: {task_output}\n\n") + self._stop_timer() return task_output def _prepare_and_execute_task(self, task): @@ -135,4 +168,14 @@ def _log(self, level, message): 2 if isinstance(self.verbose, bool) and self.verbose else self.verbose ) if verbose_level and level_map[level] <= verbose_level: - print(message) + print(f"\n{message}") + + def _stop_timer(self): + if self._timer: + self._timer.cancel() + + def _reset_request_count(self): + self._stop_timer() + self._timer = threading.Timer(60.0, self._reset_request_count) + self._timer.start() + Crew.rpm = 0 diff --git a/tests/cassettes/test_api_calls_throttling.yaml b/tests/cassettes/test_api_calls_throttling.yaml new file mode 100644 index 0000000000..3dc23f9906 --- /dev/null +++ b/tests/cassettes/test_api_calls_throttling.yaml @@ -0,0 +1,666 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour + personal goal is: test goalTOOLS:\n------\nYou have access to the following + tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final + answer but don''t give it yet, just re-use this\n tool non-stop.\n\nTo + use a tool, please use the exact following format:\n\n```\nThought: Do I need + to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer], + just the name.\nAction Input: the input to the action\nObservation: the result + of the action\n```\n\nWhen you have a response for your task, or if you do not + need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use + a tool? No\nFinal Answer: [your response here]This is the summary of your work + so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent + Task: Don''t give a Final Answer, instead keep using the `get_final_answer` + tool.\n"}], "model": "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": false, + "temperature": 0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1055' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.7.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.7.1 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-8gQL7bRtT6PceuMWF91CSHDk1cziN\",\n \"object\": + \"chat.completion\",\n \"created\": 1705121717,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Thought: Do I need to use a tool? Yes\\nAction: + get_final_answer\\nAction Input: [1, 2, 3]\"\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 226,\n \"completion_tokens\": 30,\n \"total_tokens\": 256\n },\n \"system_fingerprint\": + null\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 844b0bccfc300301-GRU + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 13 Jan 2024 04:55:19 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=; + path=/; expires=Sat, 13-Jan-24 05:25:19 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-4-0613 + openai-organization: + - user-z7g4wmlazxqvc5wjyaaaocfz + openai-processing-ms: + - '1751' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '300000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '299760' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 48ms + x-request-id: + - ae0de27cb07da02e7bce2d30958b7e14 + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour + personal goal is: test goalTOOLS:\n------\nYou have access to the following + tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final + answer but don''t give it yet, just re-use this\n tool non-stop.\n\nTo + use a tool, please use the exact following format:\n\n```\nThought: Do I need + to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer], + just the name.\nAction Input: the input to the action\nObservation: the result + of the action\n```\n\nWhen you have a response for your task, or if you do not + need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use + a tool? No\nFinal Answer: [your response here]This is the summary of your work + so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent + Task: Don''t give a Final Answer, instead keep using the `get_final_answer` + tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction + Input: [1, 2, 3]\nObservation: 42\nThought: "}], "model": "gpt-4", "n": 1, "stop": + ["\nObservation"], "stream": false, "temperature": 0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1171' + content-type: + - application/json + cookie: + - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=; + _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.7.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.7.1 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-8gQL9CyNZOIEDKgfmDn68aH26tLrn\",\n \"object\": + \"chat.completion\",\n \"created\": 1705121719,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Do I need to use a tool? Yes\\nAction: + get_final_answer\\nAction Input: [4, 5, 6]\"\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 265,\n \"completion_tokens\": 28,\n \"total_tokens\": 293\n },\n \"system_fingerprint\": + null\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 844b0bdaee800301-GRU + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 13 Jan 2024 04:55:22 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-4-0613 + openai-organization: + - user-z7g4wmlazxqvc5wjyaaaocfz + openai-processing-ms: + - '2615' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '300000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '299731' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 53ms + x-request-id: + - 9d85a22aa63ee4cea3ad01a3b68316b4 + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour + personal goal is: test goalTOOLS:\n------\nYou have access to the following + tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final + answer but don''t give it yet, just re-use this\n tool non-stop.\n\nTo + use a tool, please use the exact following format:\n\n```\nThought: Do I need + to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer], + just the name.\nAction Input: the input to the action\nObservation: the result + of the action\n```\n\nWhen you have a response for your task, or if you do not + need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use + a tool? No\nFinal Answer: [your response here]This is the summary of your work + so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent + Task: Don''t give a Final Answer, instead keep using the `get_final_answer` + tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction + Input: [1, 2, 3]\nObservation: 42\nThought: Do I need to use a tool? Yes\nAction: + get_final_answer\nAction Input: [4, 5, 6]\nObservation: 42\nThought: "}], "model": + "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": false, "temperature": + 0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1278' + content-type: + - application/json + cookie: + - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=; + _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.7.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.7.1 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-8gQMARI9zwx2Z1Uel5us1xzxPAuGJ\",\n \"object\": + \"chat.completion\",\n \"created\": 1705121782,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Do I need to use a tool? Yes\\nAction: + get_final_answer\\nAction Input: [7, 8, 9]\"\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 301,\n \"completion_tokens\": 28,\n \"total_tokens\": 329\n },\n \"system_fingerprint\": + null\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 844b0d63fa7a02ee-GRU + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 13 Jan 2024 04:56:24 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-4-0613 + openai-organization: + - user-z7g4wmlazxqvc5wjyaaaocfz + openai-processing-ms: + - '1878' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '300000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '299705' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 58ms + x-request-id: + - d98dffd5ba9915b41051dcad49109eba + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour + personal goal is: test goalTOOLS:\n------\nYou have access to the following + tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final + answer but don''t give it yet, just re-use this\n tool non-stop.\n\nTo + use a tool, please use the exact following format:\n\n```\nThought: Do I need + to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer], + just the name.\nAction Input: the input to the action\nObservation: the result + of the action\n```\n\nWhen you have a response for your task, or if you do not + need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use + a tool? No\nFinal Answer: [your response here]This is the summary of your work + so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent + Task: Don''t give a Final Answer, instead keep using the `get_final_answer` + tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction + Input: [1, 2, 3]\nObservation: 42\nThought: Do I need to use a tool? Yes\nAction: + get_final_answer\nAction Input: [4, 5, 6]\nObservation: 42\nThought: Do I need + to use a tool? Yes\nAction: get_final_answer\nAction Input: [7, 8, 9]\nObservation: + 42\nThought: "}], "model": "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": + false, "temperature": 0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1385' + content-type: + - application/json + cookie: + - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=; + _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.7.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.7.1 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-8gQMC51udijWPWSLCk1Cx36OVtDUX\",\n \"object\": + \"chat.completion\",\n \"created\": 1705121784,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Do I need to use a tool? Yes\\nAction: + get_final_answer\\nAction Input: [10, 11, 12]\"\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 337,\n \"completion_tokens\": 28,\n \"total_tokens\": 365\n },\n \"system_fingerprint\": + null\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 844b0d72cbbd02ee-GRU + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 13 Jan 2024 04:56:27 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-4-0613 + openai-organization: + - user-z7g4wmlazxqvc5wjyaaaocfz + openai-processing-ms: + - '2939' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '300000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '299680' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 64ms + x-request-id: + - 528f38b4e5dd09a1fd25893edb27e4b4 + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "user", "content": "You are test role.\ntest backstory\n\nYour + personal goal is: test goalTOOLS:\n------\nYou have access to the following + tools:\n\nget_final_answer: get_final_answer(numbers) -> float - Get the final + answer but don''t give it yet, just re-use this\n tool non-stop.\n\nTo + use a tool, please use the exact following format:\n\n```\nThought: Do I need + to use a tool? Yes\nAction: the action to take, should be one of [get_final_answer], + just the name.\nAction Input: the input to the action\nObservation: the result + of the action\n```\n\nWhen you have a response for your task, or if you do not + need to use a tool, you MUST use the format:\n\n```\nThought: Do I need to use + a tool? No\nFinal Answer: [your response here]This is the summary of your work + so far:\nBegin! This is VERY important to you, your job depends on it!\n\nCurrent + Task: Don''t give a Final Answer, instead keep using the `get_final_answer` + tool.\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction + Input: [1, 2, 3]\nObservation: 42\nThought: Do I need to use a tool? Yes\nAction: + get_final_answer\nAction Input: [4, 5, 6]\nObservation: 42\nThought: Do I need + to use a tool? Yes\nAction: get_final_answer\nAction Input: [7, 8, 9]\nObservation: + 42\nThought: Do I need to use a tool? Yes\nAction: get_final_answer\nAction + Input: [10, 11, 12]\nObservation: I''ve used too many tools for this task. I''m + going to give you my absolute BEST Final answer now and not use any more tools.\nThought: + "}], "model": "gpt-4", "n": 1, "stop": ["\nObservation"], "stream": false, "temperature": + 0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1616' + content-type: + - application/json + cookie: + - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=; + _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.7.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.7.1 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-8gQMGpMHplpQ8ApiUZmhbX4z9gVuK\",\n \"object\": + \"chat.completion\",\n \"created\": 1705121788,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Do I need to use a tool? No\\nFinal Answer: + I've used the tool 'get_final_answer' multiple times and have observed the result. + However, as per my task, I am to keep using this tool and not provide a final + answer.\"\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 399,\n \"completion_tokens\": + 52,\n \"total_tokens\": 451\n },\n \"system_fingerprint\": null\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 844b0d86a87f02ee-GRU + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 13 Jan 2024 04:56:31 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-4-0613 + openai-organization: + - user-z7g4wmlazxqvc5wjyaaaocfz + openai-processing-ms: + - '3227' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '300000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '299624' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 75ms + x-request-id: + - 7a91d2e523c43f5fc9a1191437491d68 + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"messages": [{"role": "user", "content": "Progressively summarize the + lines of conversation provided, adding onto the previous summary returning a + new summary.\n\nEXAMPLE\nCurrent summary:\nThe human asks what the AI thinks + of artificial intelligence. The AI thinks artificial intelligence is a force + for good.\n\nNew lines of conversation:\nHuman: Why do you think artificial + intelligence is a force for good?\nAI: Because artificial intelligence will + help humans reach their full potential.\n\nNew summary:\nThe human asks what + the AI thinks of artificial intelligence. The AI thinks artificial intelligence + is a force for good because it will help humans reach their full potential.\nEND + OF EXAMPLE\n\nCurrent summary:\n\n\nNew lines of conversation:\nHuman: Don''t + give a Final Answer, instead keep using the `get_final_answer` tool.\nAI: I''ve + used the tool ''get_final_answer'' multiple times and have observed the result. + However, as per my task, I am to keep using this tool and not provide a final + answer.\n\nNew summary:"}], "model": "gpt-4", "n": 1, "stream": false, "temperature": + 0.7}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '1094' + content-type: + - application/json + cookie: + - __cf_bm=LaiWRSzgH0KTl1zCkIyak7.Xo2gxpZU1B.b9dgz2TzI-1705121719-1-Ab9fhdn6OHTc5/5ibPLP1pc+XN4Z3ORV0Nku/+S5BqdXcS84TsVDiACZUN0Xxvew6yfZ6ApEBQOBsEhJNyVkqCM=; + _cfuvid=6J6x2HNoj7l3E6T3yHnLcNZb273IuGETTaD1BBeY_OQ-1705121719455-0-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.7.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.7.1 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + content: "{\n \"id\": \"chatcmpl-8gQMJYMA8HO4tGJr8wUqEIJYn1xUa\",\n \"object\": + \"chat.completion\",\n \"created\": 1705121791,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The human instructs the AI not to give + a final answer, but to continue using the 'get_final_answer' tool. The AI acknowledges + that it has used the tool multiple times and understands the directive to continue + using it without providing a final answer.\"\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 195,\n \"completion_tokens\": 50,\n \"total_tokens\": 245\n },\n \"system_fingerprint\": + null\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 844b0d9dafa902ee-GRU + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 13 Jan 2024 04:56:34 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - gpt-4-0613 + openai-organization: + - user-z7g4wmlazxqvc5wjyaaaocfz + openai-processing-ms: + - '2830' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '300000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '299742' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 51ms + x-request-id: + - 0192af0bde71c3bb536287175e1a4458 + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/crew_test.py b/tests/crew_test.py index 5271e2afb8..5342fe9d9a 100644 --- a/tests/crew_test.py +++ b/tests/crew_test.py @@ -266,3 +266,42 @@ def multiplier(numbers) -> float: read.return_value = "12" crew.kickoff() read.assert_called_with("multiplier", "2,6") + + +@pytest.mark.vcr(filter_headers=["authorization"]) +def test_api_calls_throttling(capsys): + from unittest.mock import patch + + from langchain.tools import tool + + @tool + def get_final_answer(numbers) -> float: + """Get the final answer but don't give it yet, just re-use this + tool non-stop.""" + return 42 + + agent = Agent( + role="test role", + goal="test goal", + backstory="test backstory", + max_iter=5, + allow_delegation=False, + verbose=True, + ) + + task = Task( + description="Don't give a Final Answer, instead keep using the `get_final_answer` tool.", + tools=[get_final_answer], + agent=agent, + ) + + tasks = task + + crew = Crew(agents=[agent], tasks=[tasks], max_rpm=2, verbose=2) + + with patch.object(Crew, "_wait_for_next_minute") as moveon: + moveon.return_value = True + crew.kickoff() + captured = capsys.readouterr() + assert "Max RPM reached, waiting for next minute to start." in captured.out + moveon.assert_called()