Add back AgentOps as Optional Dependency (#543)

* implements agentops with a langchain handler, agent tracking and tool call recording * track tool usage * end session after completion * track tool usage time * better tool and llm tracking * code cleanup * make agentops optional * optional dependency usage * remove telemetry code * optional agentops * agentops version bump * remove org key * true dependency * add crew org key to agentops * cleanup * Update pyproject.toml * Revert "true dependency" This reverts commit e52e8e9. * Revert "cleanup" This reverts commit 7f5635f. * optional parent key * agentops 0.1.5 * Revert "Revert "cleanup"" This reverts commit cea33d9. * Revert "Revert "true dependency"" This reverts commit 4d1b460 * cleanup * Forcing version 0.1.5 * Update pyproject.toml * agentops update * noop * add crew tag * black formatting * use langchain callback handler to support all LLMs * agentops version bump * track task evaluator * merge upstream * Fix typo in instruction en.json (#676) * Enable search in docs (#663) * Clarify text in docstring (#662) * Update agent.py (#655) Changed default model value from gpt-4 to gpt-4o. Reasoning. gpt-4 costs 30$ per million tokens while gpt-4o costs 5$. This is more cost friendly for default option. * Update README.md (#652) Rework example so that if you use a custom LLM it doesn't throw code errors by uncommenting. * Update BrowserbaseLoadTool.md (#647) * Update crew.py (#644) Fixed Type on line 53 * fixes #665 (#666) * Added timestamp to logger (#646) * Added timestamp to logger Updated the logger.py file to include timestamps when logging output. For example: [2024-05-20 15:32:48][DEBUG]: == Working Agent: Researcher [2024-05-20 15:32:48][INFO]: == Starting Task: Research the topic [2024-05-20 15:33:22][DEBUG]: == [Researcher] Task output: * Update tool_usage.py * Revert "Update tool_usage.py" This reverts commit 95d18d5. incorrect bramch for this commit * support skip auto end session * conditional protect agentops use * fix crew logger bug * fix crew logger bug * Update crew.py * Update tool_usage.py --------- Co-authored-by: João Moura <[email protected]> Co-authored-by: Howard Gil <[email protected]> Co-authored-by: Olivier Roberdet <[email protected]> Co-authored-by: Paul Sanders <[email protected]> Co-authored-by: Anudeep Kolluri <[email protected]> Co-authored-by: Mike Heavers <[email protected]> Co-authored-by: Mish Ushakov <[email protected]> Co-authored-by: theCyberTech - Rip&Tear <[email protected]> Co-authored-by: Saif Mahmud <[email protected]>
crewAIInc · Jul 3, 2024 · f479041 · f479041
1 parent d72b00a
commit f479041
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 12 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -26,10 +26,12 @@ click = "^8.1.7"
 python-dotenv = "^1.0.0"
 appdirs = "^1.4.4"
 jsonref = "^1.1.0"
+agentops = { version = "^0.1.9", optional = true }
 embedchain = "^0.1.113"
 
 [tool.poetry.extras]
 tools = ["crewai-tools"]
+agentops = ["agentops"]
 
 [tool.poetry.group.dev.dependencies]
 isort = "^5.13.2"
@@ -60,4 +62,4 @@ exclude = ["cli/templates/main.py", "cli/templates/crew.py"]
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"
diff --git a/src/crewai/agent.py b/src/crewai/agent.py
@@ -19,7 +19,20 @@
 from crewai.agents.agent_builder.base_agent import BaseAgent
 from crewai.utilities.training_handler import CrewTrainingHandler
 
+agentops = None
+try:
+    import agentops
+    from agentops import track_agent
+except ImportError:
 
+    def track_agent():
+        def noop(f):
+            return f
+
+        return noop
+
+
+@track_agent()
 class Agent(BaseAgent):
     """Represents an agent in a system.
 
@@ -48,6 +61,8 @@ class Agent(BaseAgent):
         default=None,
         description="Maximum execution time for an agent to execute a task",
     )
+    agent_ops_agent_name: str = None
+    agent_ops_agent_id: str = None
     cache_handler: InstanceOf[CacheHandler] = Field(
         default=None, description="An instance of the CacheHandler class."
     )
@@ -84,6 +99,7 @@ class Agent(BaseAgent):
     def __init__(__pydantic_self__, **data):
         config = data.pop("config", {})
         super().__init__(**config, **data)
+        __pydantic_self__.agent_ops_agent_name = __pydantic_self__.role
 
     @model_validator(mode="after")
     def set_agent_executor(self) -> "Agent":
@@ -101,6 +117,12 @@ def set_agent_executor(self) -> "Agent":
             ):
                 self.llm.callbacks.append(token_handler)
 
+            if agentops and not any(
+                isinstance(handler, agentops.LangchainCallbackHandler) for handler in self.llm.callbacks
+            ):
+                agentops.stop_instrumenting()
+                self.llm.callbacks.append(agentops.LangchainCallbackHandler())
+
         if not self.agent_executor:
             if not self.cache_handler:
                 self.cache_handler = CacheHandler()

diff --git a/src/crewai/crew.py b/src/crewai/crew.py
@@ -31,6 +31,11 @@
 from crewai.utilities.evaluators.task_evaluator import TaskEvaluator
 from crewai.utilities.training_handler import CrewTrainingHandler
 
+try:
+    import agentops
+except ImportError:
+    agentops = None
+
 
 class Crew(BaseModel):
     """
@@ -536,6 +541,10 @@ def _format_output(
     def _finish_execution(self, output) -> None:
         if self.max_rpm:
             self._rpm_controller.stop_rpm_counter()
+        if agentops:
+            agentops.end_session(
+                end_state="Success", end_state_reason="Finished Execution", is_auto_end=True
+            )
         self._telemetry.end_crew(self, output)
 
     def __repr__(self):

diff --git a/src/crewai/tools/tool_usage.py b/src/crewai/tools/tool_usage.py
@@ -11,6 +11,12 @@
 from crewai.tools.tool_calling import InstructorToolCalling, ToolCalling
 from crewai.utilities import I18N, Converter, ConverterError, Printer
 
+agentops = None
+try:
+    import agentops
+except ImportError:
+    pass
+
 OPENAI_BIGGER_MODELS = ["gpt-4"]
 
 
@@ -91,15 +97,16 @@ def use(
             self.task.increment_tools_errors()
             self._printer.print(content=f"\n\n{error}\n", color="red")
             return error
-        return f"{self._use(tool_string=tool_string, tool=tool, calling=calling)}"  # type: ignore # BUG?: "_use" of "ToolUsage" does not return a value (it only ever returns None)
+        return f"{self._use(tool_string=tool_string, tool=tool, calling=calling)}" # type: ignore # BUG?: "_use" of "ToolUsage" does not return a value (it only ever returns None)
 
     def _use(
         self,
         tool_string: str,
         tool: BaseTool,
         calling: Union[ToolCalling, InstructorToolCalling],
-    ) -> str:  # TODO: Fix this return type --> finecwg : I updated return type to str
-        if self._check_tool_repeated_usage(calling=calling):  # type: ignore # _check_tool_repeated_usage of "ToolUsage" does not return a value (it only ever returns None)
+    ) -> str:  # TODO: Fix this return type
+        tool_event = agentops.ToolEvent(name=calling.tool_name) if agentops else None        
+        if self._check_tool_repeated_usage(calling=calling): # type: ignore # _check_tool_repeated_usage of "ToolUsage" does not return a value (it only ever returns None)
             try:
                 result = self._i18n.errors("task_repeated_usage").format(
                     tool_names=self.tools_names
@@ -110,13 +117,13 @@ def _use(
                     tool_name=tool.name,
                     attempts=self._run_attempts,
                 )
-                result = self._format_result(result=result)  # type: ignore #  "_format_result" of "ToolUsage" does not return a value (it only ever returns None)
+                result = self._format_result(result=result) # type: ignore #  "_format_result" of "ToolUsage" does not return a value (it only ever returns None)
                 return result  # type: ignore # Fix the reutrn type of this function
 
             except Exception:
                 self.task.increment_tools_errors()
 
-        result = None  # type: ignore # Incompatible types in assignment (expression has type "None", variable has type "str")
+        result = None # type: ignore # Incompatible types in assignment (expression has type "None", variable has type "str")
 
         if self.tools_handler.cache:
             result = self.tools_handler.cache.read(  # type: ignore # Incompatible types in assignment (expression has type "str | None", variable has type "str")
@@ -133,7 +140,7 @@ def _use(
 
                 if calling.arguments:
                     try:
-                        acceptable_args = tool.args_schema.schema()["properties"].keys()  # type: ignore # Item "None" of "type[BaseModel] | None" has no attribute "schema"
+                        acceptable_args = tool.args_schema.schema()["properties"].keys() # type: ignore # Item "None" of "type[BaseModel] | None" has no attribute "schema"
                         arguments = {
                             k: v
                             for k, v in calling.arguments.items()
@@ -145,7 +152,7 @@ def _use(
                             arguments = calling.arguments
                             result = tool._run(**arguments)
                         else:
-                            arguments = calling.arguments.values()  # type: ignore # Incompatible types in assignment (expression has type "dict_values[str, Any]", variable has type "dict[str, Any]")
+                            arguments = calling.arguments.values() # type: ignore # Incompatible types in assignment (expression has type "dict_values[str, Any]", variable has type "dict[str, Any]")
                             result = tool._run(*arguments)
                 else:
                     result = tool._run()
@@ -164,6 +171,10 @@ def _use(
                     return error  # type: ignore # No return value expected
 
                 self.task.increment_tools_errors()
+                if agentops:
+                    agentops.record(
+                        agentops.ErrorEvent(exception=e, trigger_event=tool_event)
+                    )
                 return self.use(calling=calling, tool_string=tool_string)  # type: ignore # No return value expected
 
             if self.tools_handler:
@@ -184,18 +195,20 @@ def _use(
                 )
 
         self._printer.print(content=f"\n\n{result}\n", color="purple")
+        if agentops:
+            agentops.record(tool_event)
         self._telemetry.tool_usage(
             llm=self.function_calling_llm,
             tool_name=tool.name,
             attempts=self._run_attempts,
-        )
-        result = self._format_result(result=result)  # type: ignore # "_format_result" of "ToolUsage" does not return a value (it only ever returns None)
+        )        
+        result = self._format_result(result=result) # type: ignore # "_format_result" of "ToolUsage" does not return a value (it only ever returns None)
         return result  # type: ignore # No return value expected
 
     def _format_result(self, result: Any) -> None:
         self.task.used_tools += 1
         if self._should_remember_format():  # type: ignore # "_should_remember_format" of "ToolUsage" does not return a value (it only ever returns None)
-            result = self._remember_format(result=result)  # type: ignore # "_remember_format" of "ToolUsage" does not return a value (it only ever returns None)
+            result = self._remember_format(result=result) # type: ignore # "_remember_format" of "ToolUsage" does not return a value (it only ever returns None)
         return result
 
     def _should_remember_format(self) -> None:

diff --git a/src/crewai/utilities/evaluators/task_evaluator.py b/src/crewai/utilities/evaluators/task_evaluator.py
@@ -5,6 +5,17 @@
 
 from crewai.utilities import Converter
 from crewai.utilities.pydantic_schema_parser import PydanticSchemaParser
+agentops = None
+try:
+    import agentops
+    from agentops import track_agent
+except ImportError:
+
+    def track_agent(name):
+        def noop(f):
+            return f
+
+        return noop
 
 
 class Entity(BaseModel):
@@ -38,6 +49,7 @@ class TrainingTaskEvaluation(BaseModel):
     )
 
 
+@track_agent(name="Task Evaluator")
 class TaskEvaluator:
     def __init__(self, original_agent):
         self.llm = original_agent.llm

diff --git a/src/crewai/utilities/logger.py b/src/crewai/utilities/logger.py
@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from crewai.utilities.printer import Printer
-
+from datetime import datetime
 
 class Logger:
     _printer = Printer()