from openai import OpenAI
from dotenv import load_dotenv
from tavily import TavilyClient
from firecrawl import FirecrawlApp
# Upload the api keys
load_dotenv()
class WebTools:
def __init__(self) -> None:
self.client = OpenAI()
self.tavily_client = TavilyClient()
self.firecrawl = FirecrawlApp()
def search(self, query: str):
try:
response = self.firecrawl.search(query)
return response
except Exception as e:
return f"Error performing search: {str(e)}"
def crawl(self, url: str, maxDepth: int, limit: int):
try:
crawl_page = self.firecrawl.crawl_url(
url,
params={
"limit": limit,
"maxDepth": maxDepth,
"scrapeOptions": {"formats": ["markdown", "html"]},
},
poll_interval=30,
)
return crawl_page
except Exception as e:
return f"Error crawling pages: {str(e)}"
def extract_info(
self, url: list[str], enableWebSearch: bool, prompt: str, showSources: bool
):
try:
info_extracted = self.firecrawl.extract(
url,
{
"prompt": prompt,
"enableWebSearch": enableWebSearch,
"showSources": showSources,
"scrapeOptions": {
"formats": ["markdown"],
"blockAds": True,
},
},
)
return info_extracted
except Exception as e:
return f"Error extracting information from page {url}: {str(e)}"
def scrape_urls(self, url: list[str]):
try:
urls_scraped = self.firecrawl.scrape_url(
url,
params={
"formats": ["markdown", "html"],
"actions": [
{"type": "screenshot"},
],
},
)
return urls_scraped
except Exception as e:
return f"Error scrapping ulr {url}: {str(e)}"