feat: add new examples from pocketflow-academy
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
# PocketFlow BatchNode Example
|
||||
|
||||
This example demonstrates the BatchNode concept in PocketFlow by implementing a CSV processor that handles large files by processing them in chunks.
|
||||
|
||||
## What this Example Demonstrates
|
||||
|
||||
- How to use BatchNode to process large inputs in chunks
|
||||
- The three key methods of BatchNode:
|
||||
1. `prep`: Splits input into chunks
|
||||
2. `exec`: Processes each chunk independently
|
||||
3. `post`: Combines results from all chunks
|
||||
|
||||
## Project Structure
|
||||
```
|
||||
pocketflow-batch-node/
|
||||
├── README.md
|
||||
├── requirements.txt
|
||||
├── data/
|
||||
│ └── sales.csv # Sample large CSV file
|
||||
├── main.py # Entry point
|
||||
├── flow.py # Flow definition
|
||||
└── nodes.py # BatchNode implementation
|
||||
```
|
||||
|
||||
## How it Works
|
||||
|
||||
The example processes a large CSV file containing sales data:
|
||||
|
||||
1. **Chunking (prep)**: The CSV file is read and split into chunks of N rows
|
||||
2. **Processing (exec)**: Each chunk is processed to calculate:
|
||||
- Total sales
|
||||
- Average sale value
|
||||
- Number of transactions
|
||||
3. **Combining (post)**: Results from all chunks are aggregated into final statistics
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
## Sample Output
|
||||
|
||||
```
|
||||
Processing sales.csv in chunks...
|
||||
|
||||
Final Statistics:
|
||||
- Total Sales: $1,234,567.89
|
||||
- Average Sale: $123.45
|
||||
- Total Transactions: 10,000
|
||||
```
|
||||
|
||||
## Key Concepts Illustrated
|
||||
|
||||
1. **Chunk-based Processing**: Shows how BatchNode handles large inputs by breaking them into manageable pieces
|
||||
2. **Independent Processing**: Demonstrates how each chunk is processed separately
|
||||
3. **Result Aggregation**: Shows how individual results are combined into a final output
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
from pocketflow import Flow, Node
|
||||
from nodes import CSVProcessor
|
||||
|
||||
class ShowStats(Node):
|
||||
"""Node to display the final statistics."""
|
||||
|
||||
def prep(self, shared):
|
||||
"""Get statistics from shared store."""
|
||||
return shared["statistics"]
|
||||
|
||||
def post(self, shared, prep_res, exec_res):
|
||||
"""Display the statistics."""
|
||||
stats = prep_res
|
||||
print("\nFinal Statistics:")
|
||||
print(f"- Total Sales: ${stats['total_sales']:,.2f}")
|
||||
print(f"- Average Sale: ${stats['average_sale']:,.2f}")
|
||||
print(f"- Total Transactions: {stats['total_transactions']:,}\n")
|
||||
return "end"
|
||||
|
||||
def create_flow():
|
||||
"""Create and return the processing flow."""
|
||||
# Create nodes
|
||||
processor = CSVProcessor(chunk_size=1000)
|
||||
show_stats = ShowStats()
|
||||
|
||||
# Connect nodes
|
||||
processor - "show_stats" >> show_stats
|
||||
|
||||
# Create and return flow
|
||||
return Flow(start=processor)
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
from flow import create_flow
|
||||
|
||||
def main():
|
||||
"""Run the batch processing example."""
|
||||
# Create data directory if it doesn't exist
|
||||
os.makedirs("data", exist_ok=True)
|
||||
|
||||
# Create sample CSV if it doesn't exist
|
||||
if not os.path.exists("data/sales.csv"):
|
||||
print("Creating sample sales.csv...")
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
# Generate sample data
|
||||
np.random.seed(42)
|
||||
n_rows = 10000
|
||||
df = pd.DataFrame({
|
||||
"date": pd.date_range("2024-01-01", periods=n_rows),
|
||||
"amount": np.random.normal(100, 30, n_rows).round(2),
|
||||
"product": np.random.choice(["A", "B", "C"], n_rows)
|
||||
})
|
||||
df.to_csv("data/sales.csv", index=False)
|
||||
|
||||
# Initialize shared store
|
||||
shared = {
|
||||
"input_file": "data/sales.csv"
|
||||
}
|
||||
|
||||
# Create and run flow
|
||||
print(f"Processing sales.csv in chunks...")
|
||||
flow = create_flow()
|
||||
flow.run(shared)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
import pandas as pd
|
||||
from pocketflow import BatchNode
|
||||
|
||||
class CSVProcessor(BatchNode):
|
||||
"""BatchNode that processes a large CSV file in chunks."""
|
||||
|
||||
def __init__(self, chunk_size=1000):
|
||||
"""Initialize with chunk size."""
|
||||
super().__init__()
|
||||
self.chunk_size = chunk_size
|
||||
|
||||
def prep(self, shared):
|
||||
"""Split CSV file into chunks.
|
||||
|
||||
Returns an iterator of DataFrames, each containing chunk_size rows.
|
||||
"""
|
||||
# Read CSV in chunks
|
||||
chunks = pd.read_csv(
|
||||
shared["input_file"],
|
||||
chunksize=self.chunk_size
|
||||
)
|
||||
return chunks
|
||||
|
||||
def exec(self, chunk):
|
||||
"""Process a single chunk of the CSV.
|
||||
|
||||
Args:
|
||||
chunk: pandas DataFrame containing chunk_size rows
|
||||
|
||||
Returns:
|
||||
dict: Statistics for this chunk
|
||||
"""
|
||||
return {
|
||||
"total_sales": chunk["amount"].sum(),
|
||||
"num_transactions": len(chunk),
|
||||
"total_amount": chunk["amount"].sum()
|
||||
}
|
||||
|
||||
def post(self, shared, prep_res, exec_res_list):
|
||||
"""Combine results from all chunks.
|
||||
|
||||
Args:
|
||||
prep_res: Original chunks iterator
|
||||
exec_res_list: List of results from each chunk
|
||||
|
||||
Returns:
|
||||
str: Action to take next
|
||||
"""
|
||||
# Combine statistics from all chunks
|
||||
total_sales = sum(res["total_sales"] for res in exec_res_list)
|
||||
total_transactions = sum(res["num_transactions"] for res in exec_res_list)
|
||||
total_amount = sum(res["total_amount"] for res in exec_res_list)
|
||||
|
||||
# Calculate final statistics
|
||||
shared["statistics"] = {
|
||||
"total_sales": total_sales,
|
||||
"average_sale": total_amount / total_transactions,
|
||||
"total_transactions": total_transactions
|
||||
}
|
||||
|
||||
return "show_stats"
|
||||
@@ -0,0 +1,2 @@
|
||||
pocketflow
|
||||
pandas>=2.0.0
|
||||
Reference in New Issue
Block a user