Skip to content

Crawl4AITool

autogen.tools.experimental.Crawl4AITool #

Crawl4AITool(llm_config=None, extraction_model=None, llm_strategy_kwargs=None)

Bases: Tool

Crawl a website and extract information using the crawl4ai library.

Initialize the Crawl4AITool.

PARAMETER DESCRIPTION
llm_config

The config dictionary for the LLM model. If None, the tool will run without LLM.

TYPE: Optional[dict[str, Any]] DEFAULT: None

extraction_model

The Pydantic model to use for extraction. If None, the tool will use the default schema.

TYPE: Optional[type[BaseModel]] DEFAULT: None

llm_strategy_kwargs

The keyword arguments to pass to the LLM extraction strategy.

TYPE: Optional[dict[str, Any]] DEFAULT: None

Source code in autogen/tools/experimental/crawl4ai/crawl4ai.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def __init__(
    self,
    llm_config: Optional[dict[str, Any]] = None,
    extraction_model: Optional[type[BaseModel]] = None,
    llm_strategy_kwargs: Optional[dict[str, Any]] = None,
) -> None:
    """
    Initialize the Crawl4AITool.

    Args:
        llm_config: The config dictionary for the LLM model. If None, the tool will run without LLM.
        extraction_model: The Pydantic model to use for extraction. If None, the tool will use the default schema.
        llm_strategy_kwargs: The keyword arguments to pass to the LLM extraction strategy.
    """
    Crawl4AITool._validate_llm_strategy_kwargs(llm_strategy_kwargs, llm_config_provided=(llm_config is not None))

    async def crawl4ai_helper(  # type: ignore[no-any-unimported]
        url: str,
        browser_cfg: Optional["BrowserConfig"] = None,
        crawl_config: Optional["CrawlerRunConfig"] = None,
    ) -> Any:
        async with AsyncWebCrawler(config=browser_cfg) as crawler:
            result = await crawler.arun(
                url=url,
                config=crawl_config,
            )

        if crawl_config is None:
            response = result.markdown
        else:
            response = result.extracted_content if result.success else result.error_message

        return response

    async def crawl4ai_without_llm(
        url: Annotated[str, "The url to crawl and extract information from."],
    ) -> Any:
        return await crawl4ai_helper(url=url)

    async def crawl4ai_with_llm(
        url: Annotated[str, "The url to crawl and extract information from."],
        instruction: Annotated[str, "The instruction to provide on how and what to extract."],
        llm_config: Annotated[Any, Depends(on(llm_config))],
        llm_strategy_kwargs: Annotated[Optional[dict[str, Any]], Depends(on(llm_strategy_kwargs))],
        extraction_model: Annotated[Optional[type[BaseModel]], Depends(on(extraction_model))],
    ) -> Any:
        browser_cfg = BrowserConfig(headless=True)
        crawl_config = Crawl4AITool._get_crawl_config(
            llm_config=llm_config,
            instruction=instruction,
            extraction_model=extraction_model,
            llm_strategy_kwargs=llm_strategy_kwargs,
        )

        return await crawl4ai_helper(url=url, browser_cfg=browser_cfg, crawl_config=crawl_config)

    super().__init__(
        name="crawl4ai",
        description="Crawl a website and extract information.",
        func_or_tool=crawl4ai_without_llm if llm_config is None else crawl4ai_with_llm,
    )

name property #

name

description property #

description

func property #

func

tool_schema property #

tool_schema

Get the schema for the tool.

This is the preferred way of handling function calls with OpeaAI and compatible frameworks.

function_schema property #

function_schema

Get the schema for the function.

This is the old way of handling function calls with OpenAI and compatible frameworks. It is provided for backward compatibility.

realtime_tool_schema property #

realtime_tool_schema

Get the schema for the tool.

This is the preferred way of handling function calls with OpeaAI and compatible frameworks.

register_for_llm #

register_for_llm(agent)

Registers the tool for use with a ConversableAgent's language model (LLM).

This method registers the tool so that it can be invoked by the agent during interactions with the language model.

PARAMETER DESCRIPTION
agent

The agent to which the tool will be registered.

TYPE: ConversableAgent

Source code in autogen/tools/tool.py 
72
73
74
75
76
77
78
79
80
81
def register_for_llm(self, agent: "ConversableAgent") -> None:
    """Registers the tool for use with a ConversableAgent's language model (LLM).

    This method registers the tool so that it can be invoked by the agent during
    interactions with the language model.

    Args:
        agent (ConversableAgent): The agent to which the tool will be registered.
    """
    agent.register_for_llm()(self)

register_for_execution #

register_for_execution(agent)

Registers the tool for direct execution by a ConversableAgent.

This method registers the tool so that it can be executed by the agent, typically outside of the context of an LLM interaction.

PARAMETER DESCRIPTION
agent

The agent to which the tool will be registered.

TYPE: ConversableAgent

Source code in autogen/tools/tool.py 
83
84
85
86
87
88
89
90
91
92
def register_for_execution(self, agent: "ConversableAgent") -> None:
    """Registers the tool for direct execution by a ConversableAgent.

    This method registers the tool so that it can be executed by the agent,
    typically outside of the context of an LLM interaction.

    Args:
        agent (ConversableAgent): The agent to which the tool will be registered.
    """
    agent.register_for_execution()(self)

register_tool #

register_tool(agent)

Register a tool to be both proposed and executed by an agent.

Equivalent to calling both register_for_llm and register_for_execution with the same agent.

Note: This will not make the agent recommend and execute the call in the one step. If the agent recommends the tool, it will need to be the next agent to speak in order to execute the tool.

PARAMETER DESCRIPTION
agent

The agent to which the tool will be registered.

TYPE: ConversableAgent

Source code in autogen/tools/tool.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
def register_tool(self, agent: "ConversableAgent") -> None:
    """Register a tool to be both proposed and executed by an agent.

    Equivalent to calling both `register_for_llm` and `register_for_execution` with the same agent.

    Note: This will not make the agent recommend and execute the call in the one step. If the agent
    recommends the tool, it will need to be the next agent to speak in order to execute the tool.

    Args:
        agent (ConversableAgent): The agent to which the tool will be registered.
    """
    self.register_for_llm(agent)
    self.register_for_execution(agent)