How to give CrewAI and LangChain agents clean web scraping without Cloudflare blocks

import os
import requests
from typing import Type
from pydantic import BaseModel, Field
from crewai.tools import BaseTool

class MessoraScrapeInput(BaseModel):
    url: str = Field(..., description="The full HTTP/HTTPS URL of the page to scrape.")
    only_main_content: bool = Field(default=True, description="When True, removes navigation and footer boilerplate.")

class MessoraScrapeTool(BaseTool):
    name: str = "web_content_scraper"
    description: str = (
        "Extracts clean markdown from any web page, handling JavaScript rendering "
        "and anti-bot mechanisms automatically."
    )
    args_schema: Type[BaseModel] = MessoraScrapeInput

    def _run(self, url: str, only_main_content: bool = True) -> str:
        api_key = os.environ.get("MESSORA_API_KEY")
        if not api_key:
            return "Error: MESSORA_API_KEY environment variable is not set."

        try:
            resp = requests.post(
                "https://api.messora.dev/v1/extract",
                headers={"Authorization": f"Bearer {api_key}"},
                json={"url": url, "only_main_content": only_main_content},
                timeout=30,
            )
            resp.raise_for_status()
            data = resp.json()
            return data.get("markdown", "No content returned.")
        except requests.exceptions.RequestException as err:
            return f"Scraping failed for {url}: {err}"

from crewai import Agent, Task, Crew

researcher = Agent(
    role="Senior Market Analyst",
    goal="Extract and synthesize competitive data from target URLs",
    backstory="You research companies and pricing pages accurately using live web data.",
    tools=[MessoraScrapeTool()],
    verbose=True,
)

research_task = Task(
    description="Analyze the pricing page at https://example.com/pricing and summarize the tiers.",
    expected_output="A bulleted summary of available plans and monthly prices.",
    agent=researcher,
)

crew = Crew(agents=[researcher], tasks=[research_task])
result = crew.kickoff()
print(result)