feat: add new examples from pocketflow-academy

cookbook/pocketflow-tool-crawler/README.md

@@ -0,0 +1,71 @@
+# Web Crawler with Content Analysis
+
+A web crawler tool built with PocketFlow that crawls websites and analyzes content using LLM.
+
+## Features
+
+- Crawls websites while respecting domain boundaries
+- Extracts text content and links from pages
+- Analyzes content using GPT-4 to generate:
+  - Page summaries
+  - Main topics/keywords
+  - Content type classification
+- Processes pages in batches for efficiency
+- Generates a comprehensive analysis report
+
+## Installation
+
+1. Clone the repository
+2. Install dependencies:
+   ```bash
+   pip install -r requirements.txt
+   ```
+3. Set your OpenAI API key:
+   ```bash
+   export OPENAI_API_KEY='your-api-key'
+   ```
+
+## Usage
+
+Run the crawler:
+```bash
+python main.py
+```
+
+You will be prompted to:
+1. Enter the website URL to crawl
+2. Specify maximum number of pages to crawl (default: 10)
+
+The tool will then:
+1. Crawl the specified website
+2. Extract and analyze content using GPT-4
+3. Generate a report with findings
+
+## Project Structure
+
+```
+pocketflow-tool-crawler/
+├── tools/
+│   ├── crawler.py     # Web crawling functionality
+│   └── parser.py      # Content analysis using LLM
+├── utils/
+│   └── call_llm.py    # LLM API wrapper
+├── nodes.py           # PocketFlow nodes
+├── flow.py           # Flow configuration
+├── main.py           # Main script
+└── requirements.txt   # Dependencies
+```
+
+## Limitations
+
+- Only crawls within the same domain
+- Text content only (no images/media)
+- Rate limited by OpenAI API
+- Basic error handling
+
+## Dependencies
+
+- pocketflow: Flow-based processing
+- requests: HTTP requests
+- beautifulsoup4: HTML parsing
+- openai: GPT-4 API access

cookbook/pocketflow-tool-crawler/flow.py

@@ -0,0 +1,19 @@
+from pocketflow import Flow
+from nodes import CrawlWebsiteNode, AnalyzeContentBatchNode, GenerateReportNode
+
+def create_flow() -> Flow:
+    """Create and configure the crawling flow
+    
+    Returns:
+        Flow: Configured flow ready to run
+    """
+    # Create nodes
+    crawl = CrawlWebsiteNode()
+    analyze = AnalyzeContentBatchNode()
+    report = GenerateReportNode()
+    
+    # Connect nodes
+    crawl >> analyze >> report
+    
+    # Create flow starting with crawl
+    return Flow(start=crawl)

cookbook/pocketflow-tool-crawler/main.py

@@ -0,0 +1,26 @@
+import os
+from flow import create_flow
+
+def main():
+    """Run the web crawler flow"""
+    
+    # Get website URL from user
+    url = input("Enter website URL to crawl (e.g., https://example.com): ")
+    if not url:
+        print("Error: URL is required")
+        return
+        
+    # Initialize shared data
+    shared = {
+        "base_url": url,
+        "max_pages": 1
+    }
+    
+    # Create and run flow
+    flow = create_flow()
+    flow.run(shared)
+    
+    # Results are in shared["report"]
+    
+if __name__ == "__main__":
+    main()

cookbook/pocketflow-tool-crawler/nodes.py

@@ -0,0 +1,75 @@
+from pocketflow import Node, BatchNode
+from tools.crawler import WebCrawler
+from tools.parser import analyze_site
+from typing import List, Dict
+
+class CrawlWebsiteNode(Node):
+    """Node to crawl a website and extract content"""
+    
+    def prep(self, shared):
+        return shared.get("base_url"), shared.get("max_pages", 10)
+        
+    def exec(self, inputs):
+        base_url, max_pages = inputs
+        if not base_url:
+            return []
+            
+        crawler = WebCrawler(base_url, max_pages)
+        return crawler.crawl()
+        
+    def post(self, shared, prep_res, exec_res):
+        shared["crawl_results"] = exec_res
+        return "default"
+
+class AnalyzeContentBatchNode(BatchNode):
+    """Node to analyze crawled content in batches"""
+    
+    def prep(self, shared):
+        results = shared.get("crawl_results", [])
+        # Process in batches of 5 pages
+        batch_size = 5
+        return [results[i:i+batch_size] for i in range(0, len(results), batch_size)]
+        
+    def exec(self, batch):
+        return analyze_site(batch)
+        
+    def post(self, shared, prep_res, exec_res_list):
+        # Flatten results from all batches
+        all_results = []
+        for batch_results in exec_res_list:
+            all_results.extend(batch_results)
+            
+        shared["analyzed_results"] = all_results
+        return "default"
+
+class GenerateReportNode(Node):
+    """Node to generate a summary report of the analysis"""
+    
+    def prep(self, shared):
+        return shared.get("analyzed_results", [])
+        
+    def exec(self, results):
+        if not results:
+            return "No results to report"
+            
+        report = []
+        report.append(f"Analysis Report\n")
+        report.append(f"Total pages analyzed: {len(results)}\n")
+        
+        for page in results:
+            report.append(f"\nPage: {page['url']}")
+            report.append(f"Title: {page['title']}")
+            
+            analysis = page.get("analysis", {})
+            report.append(f"Summary: {analysis.get('summary', 'N/A')}")
+            report.append(f"Topics: {', '.join(analysis.get('topics', []))}")
+            report.append(f"Content Type: {analysis.get('content_type', 'unknown')}")
+            report.append("-" * 80)
+            
+        return "\n".join(report)
+        
+    def post(self, shared, prep_res, exec_res):
+        shared["report"] = exec_res
+        print("\nReport generated:")
+        print(exec_res)
+        return "default"

cookbook/pocketflow-tool-crawler/requirements.txt

@@ -0,0 +1,4 @@
+pocketflow>=0.1.0
+requests>=2.31.0
+beautifulsoup4>=4.12.0
+openai>=1.0.0  # for content analysis

cookbook/pocketflow-tool-crawler/tools/crawler.py

@@ -0,0 +1,74 @@
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from typing import Dict, List, Set
+
+class WebCrawler:
+    """Simple web crawler that extracts content and follows links"""
+    
+    def __init__(self, base_url: str, max_pages: int = 10):
+        self.base_url = base_url
+        self.max_pages = max_pages
+        self.visited: Set[str] = set()
+        
+    def is_valid_url(self, url: str) -> bool:
+        """Check if URL belongs to the same domain"""
+        base_domain = urlparse(self.base_url).netloc
+        url_domain = urlparse(url).netloc
+        return base_domain == url_domain
+        
+    def extract_page_content(self, url: str) -> Dict:
+        """Extract content from a single page"""
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.text, "html.parser")
+            
+            # Extract main content
+            content = {
+                "url": url,
+                "title": soup.title.string if soup.title else "",
+                "text": soup.get_text(separator="\n", strip=True),
+                "links": []
+            }
+            
+            # Extract links
+            for link in soup.find_all("a"):
+                href = link.get("href")
+                if href:
+                    absolute_url = urljoin(url, href)
+                    if self.is_valid_url(absolute_url):
+                        content["links"].append(absolute_url)
+            
+            return content
+            
+        except Exception as e:
+            print(f"Error crawling {url}: {str(e)}")
+            return None
+    
+    def crawl(self) -> List[Dict]:
+        """Crawl website starting from base_url"""
+        to_visit = [self.base_url]
+        results = []
+        
+        while to_visit and len(self.visited) < self.max_pages:
+            url = to_visit.pop(0)
+            
+            if url in self.visited:
+                continue
+                
+            print(f"Crawling: {url}")
+            content = self.extract_page_content(url)
+            
+            if content:
+                self.visited.add(url)
+                results.append(content)
+                
+                # Add new URLs to visit
+                new_urls = [url for url in content["links"] 
+                          if url not in self.visited 
+                          and url not in to_visit]
+                to_visit.extend(new_urls)
+        
+        return results

cookbook/pocketflow-tool-crawler/tools/parser.py

@@ -0,0 +1,77 @@
+from typing import Dict, List
+from utils.call_llm import call_llm
+
+def analyze_content(content: Dict) -> Dict:
+    """Analyze webpage content using LLM
+    
+    Args:
+        content (Dict): Webpage content with url, title and text
+        
+    Returns:
+        Dict: Analysis results including summary and topics
+    """
+    prompt = f"""
+Analyze this webpage content:
+
+Title: {content['title']}
+URL: {content['url']}
+Content: {content['text'][:2000]}  # Limit content length
+
+Please provide:
+1. A brief summary (2-3 sentences)
+2. Main topics/keywords (up to 5)
+3. Content type (article, product page, etc)
+
+Output in YAML format:
+```yaml
+summary: >
+    brief summary here
+topics:
+    - topic 1
+    - topic 2
+content_type: type here
+```
+"""
+    
+    try:
+        response = call_llm(prompt)
+        # Extract YAML between code fences
+        yaml_str = response.split("```yaml")[1].split("```")[0].strip()
+        
+        import yaml
+        analysis = yaml.safe_load(yaml_str)
+        
+        # Validate required fields
+        assert "summary" in analysis
+        assert "topics" in analysis
+        assert "content_type" in analysis
+        assert isinstance(analysis["topics"], list)
+        
+        return analysis
+        
+    except Exception as e:
+        print(f"Error analyzing content: {str(e)}")
+        return {
+            "summary": "Error analyzing content",
+            "topics": [],
+            "content_type": "unknown"
+        }
+
+def analyze_site(crawl_results: List[Dict]) -> List[Dict]:
+    """Analyze all crawled pages
+    
+    Args:
+        crawl_results (List[Dict]): List of crawled page contents
+        
+    Returns:
+        List[Dict]: Original content with added analysis
+    """
+    analyzed_results = []
+    
+    for content in crawl_results:
+        if content and content.get("text"):
+            analysis = analyze_content(content)
+            content["analysis"] = analysis
+            analyzed_results.append(content)
+            
+    return analyzed_results

cookbook/pocketflow-tool-crawler/utils/call_llm.py

@@ -0,0 +1,30 @@
+from openai import OpenAI
+import os
+
+# Initialize OpenAI client
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+def call_llm(prompt: str) -> str:
+    """Call OpenAI API to analyze text
+    
+    Args:
+        prompt (str): Input prompt for the model
+        
+    Returns:
+        str: Model response
+    """
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4",
+            messages=[{"role": "user", "content": prompt}]
+        )
+        return response.choices[0].message.content
+        
+    except Exception as e:
+        print(f"Error calling LLM API: {str(e)}")
+        return ""
+
+if __name__ == "__main__":
+    # Test LLM call
+    response = call_llm("What is web crawling?")
+    print("Response:", response)