opensource/n8n-workflows
3
0
Fork 0
mirror of https://github.com/Zie619/n8n-workflows.git synced 2026-08-29 03:45:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

reformated + line fixes

Browse Source
This commit is contained in:
noam
2025-11-15 00:42:42 +02:00
parent 51721cf44a
commit bd7bc3aa1f
14 changed files with 2045 additions and 1569 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api_server.py
+244 -168
View File
@@ -13,7 +13,6 @@ from pydantic import BaseModel, field_validator
from typing import Optional, List, Dict, Any
import json
import os
import asyncio
import re
import urllib.parse
from pathlib import Path
@@ -27,7 +26,7 @@ from workflow_db import WorkflowDatabase
app = FastAPI(
title="N8N Workflow Documentation API",
description="Fast API for browsing and searching workflow documentation",
version="2.0.0"
version="2.0.0",
)
# Security: Rate limiting storage
@@ -59,13 +58,15 @@ app.add_middleware(
# Initialize database
db = WorkflowDatabase()
# Security: Helper function for rate limiting
def check_rate_limit(client_ip: str) -> bool:
"""Check if client has exceeded rate limit."""
current_time = time.time()
# Clean old entries
rate_limit_storage[client_ip] = [
timestamp for timestamp in rate_limit_storage[client_ip]
timestamp
for timestamp in rate_limit_storage[client_ip]
if current_time - timestamp < 60
]
# Check rate limit
@@ -75,6 +76,7 @@ def check_rate_limit(client_ip: str) -> bool:
rate_limit_storage[client_ip].append(current_time)
return True
# Security: Helper function to validate and sanitize filenames
def validate_filename(filename: str) -> bool:
"""
@@ -85,25 +87,30 @@ def validate_filename(filename: str) -> bool:
decoded = filename
for _ in range(3): # Decode up to 3 times to catch nested encodings
try:
decoded = urllib.parse.unquote(decoded, errors='strict')
decoded = urllib.parse.unquote(decoded, errors="strict")
except:
return False # Invalid encoding
# Check for path traversal patterns
dangerous_patterns = [
'..', # Parent directory
'..\\', # Windows parent directory
'../', # Unix parent directory
'\\', # Backslash (Windows path separator)
'/', # Forward slash (Unix path separator)
'\x00', # Null byte
'\n', '\r', # Newlines
'~', # Home directory
':', # Drive letter or stream (Windows)
'|', '<', '>', # Shell redirection
'*', '?', # Wildcards
'$', # Variable expansion
';', '&', # Command separators
"..", # Parent directory
"..\\", # Windows parent directory
"../", # Unix parent directory
"\\", # Backslash (Windows path separator)
"/", # Forward slash (Unix path separator)
"\x00", # Null byte
"\n",
"\r", # Newlines
"~", # Home directory
":", # Drive letter or stream (Windows)
"|",
"<",
">", # Shell redirection
"*",
"?", # Wildcards
"$", # Variable expansion
";",
"&", # Command separators
]
for pattern in dangerous_patterns:
@@ -111,30 +118,31 @@ def validate_filename(filename: str) -> bool:
return False
# Check for absolute paths
if decoded.startswith('/') or decoded.startswith('\\'):
if decoded.startswith("/") or decoded.startswith("\\"):
return False
# Check for Windows drive letters
if len(decoded) >= 2 and decoded[1] == ':':
if len(decoded) >= 2 and decoded[1] == ":":
return False
# Only allow alphanumeric, dash, underscore, and .json extension
if not re.match(r'^[a-zA-Z0-9_\-]+\.json$', decoded):
if not re.match(r"^[a-zA-Z0-9_\-]+\.json$", decoded):
return False
# Additional check: filename should end with .json
if not decoded.endswith('.json'):
if not decoded.endswith(".json"):
return False
return True
# Startup function to verify database
@app.on_event("startup")
async def startup_event():
"""Verify database connectivity on startup."""
try:
stats = db.get_stats()
if stats['total'] == 0:
if stats["total"] == 0:
print("⚠️ Warning: No workflows found in database. Run indexing first.")
else:
print(f"✅ Database connected: {stats['total']} workflows indexed")
@@ -142,6 +150,7 @@ async def startup_event():
print(f"❌ Database connection failed: {e}")
raise
# Response models
class WorkflowSummary(BaseModel):
id: Optional[int] = None
@@ -156,18 +165,18 @@ class WorkflowSummary(BaseModel):
tags: List[str] = []
created_at: Optional[str] = None
updated_at: Optional[str] = None
class Config:
# Allow conversion of int to bool for active field
validate_assignment = True
@field_validator('active', mode='before')
@field_validator("active", mode="before")
@classmethod
def convert_active(cls, v):
if isinstance(v, int):
return bool(v)
return v
class SearchResponse(BaseModel):
workflows: List[WorkflowSummary]
@@ -178,6 +187,7 @@ class SearchResponse(BaseModel):
query: str
filters: Dict[str, Any]
class StatsResponse(BaseModel):
total: int
active: int
@@ -188,26 +198,33 @@ class StatsResponse(BaseModel):
unique_integrations: int
last_indexed: str
@app.get("/")
async def root():
"""Serve the main documentation page."""
static_dir = Path("static")
index_file = static_dir / "index.html"
if not index_file.exists():
return HTMLResponse("""
return HTMLResponse(
"""
<html><body>
<h1>Setup Required</h1>
<p>Static files not found. Please ensure the static directory exists with index.html</p>
<p>Current directory: """ + str(Path.cwd()) + """</p>
<p>Current directory: """
+ str(Path.cwd())
+ """</p>
</body></html>
""")
"""
)
return FileResponse(str(index_file))
@app.get("/health")
async def health_check():
"""Health check endpoint."""
return {"status": "healthy", "message": "N8N Workflow API is running"}
@app.get("/api/stats", response_model=StatsResponse)
async def get_stats():
"""Get workflow database statistics."""
@@ -217,6 +234,7 @@ async def get_stats():
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error fetching stats: {str(e)}")
@app.get("/api/workflows", response_model=SearchResponse)
async def search_workflows(
q: str = Query("", description="Search query"),
@@ -224,48 +242,50 @@ async def search_workflows(
complexity: str = Query("all", description="Filter by complexity"),
active_only: bool = Query(False, description="Show only active workflows"),
page: int = Query(1, ge=1, description="Page number"),
per_page: int = Query(20, ge=1, le=100, description="Items per page")
per_page: int = Query(20, ge=1, le=100, description="Items per page"),
):
"""Search and filter workflows with pagination."""
try:
offset = (page - 1) * per_page
workflows, total = db.search_workflows(
query=q,
trigger_filter=trigger,
complexity_filter=complexity,
active_only=active_only,
limit=per_page,
offset=offset
offset=offset,
)
# Convert to Pydantic models with error handling
workflow_summaries = []
for workflow in workflows:
try:
# Remove extra fields that aren't in the model
clean_workflow = {
'id': workflow.get('id'),
'filename': workflow.get('filename', ''),
'name': workflow.get('name', ''),
'active': workflow.get('active', False),
'description': workflow.get('description', ''),
'trigger_type': workflow.get('trigger_type', 'Manual'),
'complexity': workflow.get('complexity', 'low'),
'node_count': workflow.get('node_count', 0),
'integrations': workflow.get('integrations', []),
'tags': workflow.get('tags', []),
'created_at': workflow.get('created_at'),
'updated_at': workflow.get('updated_at')
"id": workflow.get("id"),
"filename": workflow.get("filename", ""),
"name": workflow.get("name", ""),
"active": workflow.get("active", False),
"description": workflow.get("description", ""),
"trigger_type": workflow.get("trigger_type", "Manual"),
"complexity": workflow.get("complexity", "low"),
"node_count": workflow.get("node_count", 0),
"integrations": workflow.get("integrations", []),
"tags": workflow.get("tags", []),
"created_at": workflow.get("created_at"),
"updated_at": workflow.get("updated_at"),
}
workflow_summaries.append(WorkflowSummary(**clean_workflow))
except Exception as e:
print(f"Error converting workflow {workflow.get('filename', 'unknown')}: {e}")
print(
f"Error converting workflow {workflow.get('filename', 'unknown')}: {e}"
)
# Continue with other workflows instead of failing completely
continue
pages = (total + per_page - 1) // per_page # Ceiling division
return SearchResponse(
workflows=workflow_summaries,
total=total,
@@ -276,11 +296,14 @@ async def search_workflows(
filters={
"trigger": trigger,
"complexity": complexity,
"active_only": active_only
}
"active_only": active_only,
},
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error searching workflows: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error searching workflows: {str(e)}"
)
@app.get("/api/workflows/{filename}")
async def get_workflow_detail(filename: str, request: Request):
@@ -294,17 +317,21 @@ async def get_workflow_detail(filename: str, request: Request):
# Security: Rate limiting
client_ip = request.client.host if request.client else "unknown"
if not check_rate_limit(client_ip):
raise HTTPException(status_code=429, detail="Rate limit exceeded. Please try again later.")
raise HTTPException(
status_code=429, detail="Rate limit exceeded. Please try again later."
)
# Get workflow metadata from database
workflows, _ = db.search_workflows(f'filename:"{filename}"', limit=1)
if not workflows:
raise HTTPException(status_code=404, detail="Workflow not found in database")
raise HTTPException(
status_code=404, detail="Workflow not found in database"
)
workflow_meta = workflows[0]
# Load raw JSON from file with security checks
workflows_path = Path('workflows').resolve()
workflows_path = Path("workflows").resolve()
# Find the file safely
matching_file = None
@@ -318,25 +345,28 @@ async def get_workflow_detail(filename: str, request: Request):
matching_file = target_file
break
except ValueError:
print(f"Security: Blocked access to file outside workflows: {target_file}")
print(
f"Security: Blocked access to file outside workflows: {target_file}"
)
continue
if not matching_file:
print(f"Warning: File {filename} not found in workflows directory")
raise HTTPException(status_code=404, detail=f"Workflow file '{filename}' not found on filesystem")
raise HTTPException(
status_code=404,
detail=f"Workflow file '{filename}' not found on filesystem",
)
with open(matching_file, 'r', encoding='utf-8') as f:
with open(matching_file, "r", encoding="utf-8") as f:
raw_json = json.load(f)
return {
"metadata": workflow_meta,
"raw_json": raw_json
}
return {"metadata": workflow_meta, "raw_json": raw_json}
except HTTPException:
raise
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error loading workflow: {str(e)}")
@app.get("/api/workflows/{filename}/download")
async def download_workflow(filename: str, request: Request):
"""Download workflow JSON file with security validation."""
@@ -349,10 +379,12 @@ async def download_workflow(filename: str, request: Request):
# Security: Rate limiting
client_ip = request.client.host if request.client else "unknown"
if not check_rate_limit(client_ip):
raise HTTPException(status_code=429, detail="Rate limit exceeded. Please try again later.")
raise HTTPException(
status_code=429, detail="Rate limit exceeded. Please try again later."
)
# Only search within the workflows directory
workflows_path = Path('workflows').resolve() # Get absolute path
workflows_path = Path("workflows").resolve() # Get absolute path
# Find the file safely
json_files = []
@@ -366,12 +398,16 @@ async def download_workflow(filename: str, request: Request):
json_files.append(target_file)
except ValueError:
# File is outside workflows directory
print(f"Security: Blocked access to file outside workflows: {target_file}")
print(
f"Security: Blocked access to file outside workflows: {target_file}"
)
continue
if not json_files:
print(f"File {filename} not found in workflows directory")
raise HTTPException(status_code=404, detail=f"Workflow file '{filename}' not found")
raise HTTPException(
status_code=404, detail=f"Workflow file '{filename}' not found"
)
file_path = json_files[0]
@@ -379,19 +415,22 @@ async def download_workflow(filename: str, request: Request):
try:
file_path.resolve().relative_to(workflows_path)
except ValueError:
print(f"Security: Blocked final attempt to access file outside workflows: {file_path}")
print(
f"Security: Blocked final attempt to access file outside workflows: {file_path}"
)
raise HTTPException(status_code=403, detail="Access denied")
return FileResponse(
str(file_path),
media_type="application/json",
filename=filename
str(file_path), media_type="application/json", filename=filename
)
except HTTPException:
raise
except Exception as e:
print(f"Error downloading workflow {filename}: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error downloading workflow: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error downloading workflow: {str(e)}"
)
@app.get("/api/workflows/{filename}/diagram")
async def get_workflow_diagram(filename: str, request: Request):
@@ -405,10 +444,12 @@ async def get_workflow_diagram(filename: str, request: Request):
# Security: Rate limiting
client_ip = request.client.host if request.client else "unknown"
if not check_rate_limit(client_ip):
raise HTTPException(status_code=429, detail="Rate limit exceeded. Please try again later.")
raise HTTPException(
status_code=429, detail="Rate limit exceeded. Please try again later."
)
# Only search within the workflows directory
workflows_path = Path('workflows').resolve()
workflows_path = Path("workflows").resolve()
# Find the file safely
matching_file = None
@@ -422,18 +463,23 @@ async def get_workflow_diagram(filename: str, request: Request):
matching_file = target_file
break
except ValueError:
print(f"Security: Blocked access to file outside workflows: {target_file}")
print(
f"Security: Blocked access to file outside workflows: {target_file}"
)
continue
if not matching_file:
print(f"Warning: File {filename} not found in workflows directory")
raise HTTPException(status_code=404, detail=f"Workflow file '{filename}' not found on filesystem")
raise HTTPException(
status_code=404,
detail=f"Workflow file '{filename}' not found on filesystem",
)
with open(matching_file, 'r', encoding='utf-8') as f:
with open(matching_file, "r", encoding="utf-8") as f:
data = json.load(f)
nodes = data.get('nodes', [])
connections = data.get('connections', {})
nodes = data.get("nodes", [])
connections = data.get("connections", {})
# Generate Mermaid diagram
diagram = generate_mermaid_diagram(nodes, connections)
@@ -443,103 +489,113 @@ async def get_workflow_diagram(filename: str, request: Request):
raise
except json.JSONDecodeError as e:
print(f"Error parsing JSON in {filename}: {str(e)}")
raise HTTPException(status_code=400, detail=f"Invalid JSON in workflow file: {str(e)}")
raise HTTPException(
status_code=400, detail=f"Invalid JSON in workflow file: {str(e)}"
)
except Exception as e:
print(f"Error generating diagram for {filename}: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error generating diagram: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error generating diagram: {str(e)}"
)
def generate_mermaid_diagram(nodes: List[Dict], connections: Dict) -> str:
"""Generate Mermaid.js flowchart code from workflow nodes and connections."""
if not nodes:
return "graph TD\n EmptyWorkflow[No nodes found in workflow]"
# Create mapping for node names to ensure valid mermaid IDs
mermaid_ids = {}
for i, node in enumerate(nodes):
node_id = f"node{i}"
node_name = node.get('name', f'Node {i}')
node_name = node.get("name", f"Node {i}")
mermaid_ids[node_name] = node_id
# Start building the mermaid diagram
mermaid_code = ["graph TD"]
# Add nodes with styling
for node in nodes:
node_name = node.get('name', 'Unnamed')
node_name = node.get("name", "Unnamed")
node_id = mermaid_ids[node_name]
node_type = node.get('type', '').replace('n8n-nodes-base.', '')
node_type = node.get("type", "").replace("n8n-nodes-base.", "")
# Determine node style based on type
style = ""
if any(x in node_type.lower() for x in ['trigger', 'webhook', 'cron']):
if any(x in node_type.lower() for x in ["trigger", "webhook", "cron"]):
style = "fill:#b3e0ff,stroke:#0066cc" # Blue for triggers
elif any(x in node_type.lower() for x in ['if', 'switch']):
elif any(x in node_type.lower() for x in ["if", "switch"]):
style = "fill:#ffffb3,stroke:#e6e600" # Yellow for conditional nodes
elif any(x in node_type.lower() for x in ['function', 'code']):
elif any(x in node_type.lower() for x in ["function", "code"]):
style = "fill:#d9b3ff,stroke:#6600cc" # Purple for code nodes
elif 'error' in node_type.lower():
elif "error" in node_type.lower():
style = "fill:#ffb3b3,stroke:#cc0000" # Red for error handlers
else:
style = "fill:#d9d9d9,stroke:#666666" # Gray for other nodes
# Add node with label (escaping special characters)
clean_name = node_name.replace('"', "'")
clean_type = node_type.replace('"', "'")
label = f"{clean_name}<br>({clean_type})"
mermaid_code.append(f" {node_id}[\"{label}\"]")
mermaid_code.append(f' {node_id}["{label}"]')
mermaid_code.append(f" style {node_id} {style}")
# Add connections between nodes
for source_name, source_connections in connections.items():
if source_name not in mermaid_ids:
continue
if isinstance(source_connections, dict) and 'main' in source_connections:
main_connections = source_connections['main']
if isinstance(source_connections, dict) and "main" in source_connections:
main_connections = source_connections["main"]
for i, output_connections in enumerate(main_connections):
if not isinstance(output_connections, list):
continue
for connection in output_connections:
if not isinstance(connection, dict) or 'node' not in connection:
if not isinstance(connection, dict) or "node" not in connection:
continue
target_name = connection['node']
target_name = connection["node"]
if target_name not in mermaid_ids:
continue
# Add arrow with output index if multiple outputs
label = f" -->|{i}| " if len(main_connections) > 1 else " --> "
mermaid_code.append(f" {mermaid_ids[source_name]}{label}{mermaid_ids[target_name]}")
mermaid_code.append(
f" {mermaid_ids[source_name]}{label}{mermaid_ids[target_name]}"
)
# Format the final mermaid diagram code
return "\n".join(mermaid_code)
@app.post("/api/reindex")
async def reindex_workflows(
background_tasks: BackgroundTasks,
request: Request,
force: bool = False,
admin_token: Optional[str] = Query(None, description="Admin authentication token")
admin_token: Optional[str] = Query(None, description="Admin authentication token"),
):
"""Trigger workflow reindexing in the background (requires authentication)."""
# Security: Rate limiting
client_ip = request.client.host if request.client else "unknown"
if not check_rate_limit(client_ip):
raise HTTPException(status_code=429, detail="Rate limit exceeded. Please try again later.")
raise HTTPException(
status_code=429, detail="Rate limit exceeded. Please try again later."
)
# Security: Basic authentication check
# In production, use proper authentication (JWT, OAuth, etc.)
# For now, check for environment variable or disable endpoint
import os
expected_token = os.environ.get("ADMIN_TOKEN", None)
if not expected_token:
# If no token is configured, disable the endpoint for security
raise HTTPException(
status_code=503,
detail="Reindexing endpoint is disabled. Set ADMIN_TOKEN environment variable to enable."
detail="Reindexing endpoint is disabled. Set ADMIN_TOKEN environment variable to enable.",
)
if admin_token != expected_token:
@@ -556,15 +612,19 @@ async def reindex_workflows(
background_tasks.add_task(run_indexing)
return {"message": "Reindexing started in background", "requested_by": client_ip}
@app.get("/api/integrations")
async def get_integrations():
"""Get list of all unique integrations."""
try:
stats = db.get_stats()
# For now, return basic info. Could be enhanced to return detailed integration stats
return {"integrations": [], "count": stats['unique_integrations']}
return {"integrations": [], "count": stats["unique_integrations"]}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error fetching integrations: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error fetching integrations: {str(e)}"
)
@app.get("/api/categories")
async def get_categories():
@@ -573,32 +633,35 @@ async def get_categories():
# Try to load from the generated unique categories file
categories_file = Path("context/unique_categories.json")
if categories_file.exists():
with open(categories_file, 'r', encoding='utf-8') as f:
with open(categories_file, "r", encoding="utf-8") as f:
categories = json.load(f)
return {"categories": categories}
else:
# Fallback: extract categories from search_categories.json
search_categories_file = Path("context/search_categories.json")
if search_categories_file.exists():
with open(search_categories_file, 'r', encoding='utf-8') as f:
with open(search_categories_file, "r", encoding="utf-8") as f:
search_data = json.load(f)
unique_categories = set()
for item in search_data:
if item.get('category'):
unique_categories.add(item['category'])
if item.get("category"):
unique_categories.add(item["category"])
else:
unique_categories.add('Uncategorized')
unique_categories.add("Uncategorized")
categories = sorted(list(unique_categories))
return {"categories": categories}
else:
# Last resort: return basic categories
return {"categories": ["Uncategorized"]}
except Exception as e:
print(f"Error loading categories: {e}")
raise HTTPException(status_code=500, detail=f"Error fetching categories: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error fetching categories: {str(e)}"
)
@app.get("/api/category-mappings")
async def get_category_mappings():
@@ -607,65 +670,68 @@ async def get_category_mappings():
search_categories_file = Path("context/search_categories.json")
if not search_categories_file.exists():
return {"mappings": {}}
with open(search_categories_file, 'r', encoding='utf-8') as f:
with open(search_categories_file, "r", encoding="utf-8") as f:
search_data = json.load(f)
# Convert to a simple filename -> category mapping
mappings = {}
for item in search_data:
filename = item.get('filename')
category = item.get('category') or 'Uncategorized'
filename = item.get("filename")
category = item.get("category") or "Uncategorized"
if filename:
mappings[filename] = category
return {"mappings": mappings}
except Exception as e:
print(f"Error loading category mappings: {e}")
raise HTTPException(status_code=500, detail=f"Error fetching category mappings: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error fetching category mappings: {str(e)}"
)
@app.get("/api/workflows/category/{category}", response_model=SearchResponse)
async def search_workflows_by_category(
category: str,
page: int = Query(1, ge=1, description="Page number"),
per_page: int = Query(20, ge=1, le=100, description="Items per page")
per_page: int = Query(20, ge=1, le=100, description="Items per page"),
):
"""Search workflows by service category (messaging, database, ai_ml, etc.)."""
try:
offset = (page - 1) * per_page
workflows, total = db.search_by_category(
category=category,
limit=per_page,
offset=offset
category=category, limit=per_page, offset=offset
)
# Convert to Pydantic models with error handling
workflow_summaries = []
for workflow in workflows:
try:
clean_workflow = {
'id': workflow.get('id'),
'filename': workflow.get('filename', ''),
'name': workflow.get('name', ''),
'active': workflow.get('active', False),
'description': workflow.get('description', ''),
'trigger_type': workflow.get('trigger_type', 'Manual'),
'complexity': workflow.get('complexity', 'low'),
'node_count': workflow.get('node_count', 0),
'integrations': workflow.get('integrations', []),
'tags': workflow.get('tags', []),
'created_at': workflow.get('created_at'),
'updated_at': workflow.get('updated_at')
"id": workflow.get("id"),
"filename": workflow.get("filename", ""),
"name": workflow.get("name", ""),
"active": workflow.get("active", False),
"description": workflow.get("description", ""),
"trigger_type": workflow.get("trigger_type", "Manual"),
"complexity": workflow.get("complexity", "low"),
"node_count": workflow.get("node_count", 0),
"integrations": workflow.get("integrations", []),
"tags": workflow.get("tags", []),
"created_at": workflow.get("created_at"),
"updated_at": workflow.get("updated_at"),
}
workflow_summaries.append(WorkflowSummary(**clean_workflow))
except Exception as e:
print(f"Error converting workflow {workflow.get('filename', 'unknown')}: {e}")
print(
f"Error converting workflow {workflow.get('filename', 'unknown')}: {e}"
)
continue
pages = (total + per_page - 1) // per_page
return SearchResponse(
workflows=workflow_summaries,
total=total,
@@ -673,19 +739,22 @@ async def search_workflows_by_category(
per_page=per_page,
pages=pages,
query=f"category:{category}",
filters={"category": category}
filters={"category": category},
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Error searching by category: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Error searching by category: {str(e)}"
)
# Custom exception handler for better error responses
@app.exception_handler(Exception)
async def global_exception_handler(request, exc):
return JSONResponse(
status_code=500,
content={"detail": f"Internal server error: {str(exc)}"}
status_code=500, content={"detail": f"Internal server error: {str(exc)}"}
)
# Mount static files AFTER all routes are defined
static_dir = Path("static")
if static_dir.exists():
@@ -694,22 +763,24 @@ if static_dir.exists():
else:
print(f"❌ Warning: Static directory not found at {static_dir.absolute()}")
def create_static_directory():
"""Create static directory if it doesn't exist."""
static_dir = Path("static")
static_dir.mkdir(exist_ok=True)
return static_dir
def run_server(host: str = "127.0.0.1", port: int = 8000, reload: bool = False):
"""Run the FastAPI server."""
# Ensure static directory exists
create_static_directory()
# Debug: Check database connectivity
try:
stats = db.get_stats()
print(f"✅ Database connected: {stats['total']} workflows found")
if stats['total'] == 0:
if stats["total"] == 0:
print("🔄 Database is empty. Indexing workflows...")
db.index_all_workflows()
stats = db.get_stats()
@@ -722,8 +793,8 @@ def run_server(host: str = "127.0.0.1", port: int = 8000, reload: bool = False):
print(f"✅ Database created: {stats['total']} workflows indexed")
except Exception as e2:
print(f"❌ Failed to create database: {e2}")
stats = {'total': 0}
stats = {"total": 0}
# Debug: Check static files
static_path = Path("static")
if static_path.exists():
@@ -731,29 +802,34 @@ def run_server(host: str = "127.0.0.1", port: int = 8000, reload: bool = False):
print(f"✅ Static files found: {[f.name for f in files]}")
else:
print(f"❌ Static directory not found at: {static_path.absolute()}")
print(f"🚀 Starting N8N Workflow Documentation API")
print("🚀 Starting N8N Workflow Documentation API")
print(f"📊 Database contains {stats['total']} workflows")
print(f"🌐 Server will be available at: http://{host}:{port}")
print(f"📁 Static files at: http://{host}:{port}/static/")
uvicorn.run(
"api_server:app",
host=host,
port=port,
reload=reload,
access_log=True, # Enable access logs for debugging
log_level="info"
log_level="info",
)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='N8N Workflow Documentation API Server')
parser.add_argument('--host', default='127.0.0.1', help='Host to bind to')
parser.add_argument('--port', type=int, default=8000, help='Port to bind to')
parser.add_argument('--reload', action='store_true', help='Enable auto-reload for development')
parser = argparse.ArgumentParser(
description="N8N Workflow Documentation API Server"
)
parser.add_argument("--host", default="127.0.0.1", help="Host to bind to")
parser.add_argument("--port", type=int, default=8000, help="Port to bind to")
parser.add_argument(
"--reload", action="store_true", help="Enable auto-reload for development"
)
args = parser.parse_args()
run_server(host=args.host, port=args.port, reload=args.reload)
run_server(host=args.host, port=args.port, reload=args.reload)
run.py
+32 -45
View File
@@ -7,7 +7,6 @@ Start the advanced search system with optimized performance.
import sys
import os
import argparse
from pathlib import Path
def print_banner():
@@ -19,27 +18,27 @@ def print_banner():
def check_requirements() -> bool:
"""Check if required dependencies are installed."""
missing_deps = []
try:
import sqlite3
except ImportError:
missing_deps.append("sqlite3")
try:
import uvicorn
except ImportError:
missing_deps.append("uvicorn")
try:
import fastapi
except ImportError:
missing_deps.append("fastapi")
if missing_deps:
print(f"❌ Missing dependencies: {', '.join(missing_deps)}")
print("💡 Install with: pip install -r requirements.txt")
return False
print("✅ Dependencies verified")
return True
@@ -47,10 +46,10 @@ def check_requirements() -> bool:
def setup_directories():
"""Create necessary directories."""
directories = ["database", "static", "workflows"]
for directory in directories:
os.makedirs(directory, exist_ok=True)
print("✅ Directories verified")
@@ -72,7 +71,7 @@ def setup_database(force_reindex: bool = False, skip_index: bool = False) -> str
# Check if database has data or force reindex
stats = db.get_stats()
if stats['total'] == 0 or force_reindex:
if stats["total"] == 0 or force_reindex:
print("📚 Indexing workflows...")
index_stats = db.index_all_workflows(force_reindex=True)
print(f"✅ Indexed {index_stats['processed']} workflows")
@@ -94,25 +93,26 @@ def start_server(host: str = "127.0.0.1", port: int = 8000, reload: bool = False
print()
print("Press Ctrl+C to stop the server")
print("-" * 50)
# Configure database path
os.environ['WORKFLOW_DB_PATH'] = "database/workflows.db"
os.environ["WORKFLOW_DB_PATH"] = "database/workflows.db"
# Start uvicorn with better configuration
import uvicorn
uvicorn.run(
"api_server:app",
host=host,
port=port,
"api_server:app",
host=host,
port=port,
reload=reload,
log_level="info",
access_log=False # Reduce log noise
access_log=False, # Reduce log noise
)
def main():
"""Main entry point with command line arguments."""
sys.stdout.reconfigure(encoding='utf-8')
sys.stdout.reconfigure(encoding="utf-8")
parser = argparse.ArgumentParser(
description="N8N Workflows Search Engine",
formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -123,65 +123,52 @@ Examples:
python run.py --host 0.0.0.0 # Accept external connections
python run.py --reindex # Force database reindexing
python run.py --dev # Development mode with auto-reload
"""
""",
)
parser.add_argument(
"--host",
default="127.0.0.1",
help="Host to bind to (default: 127.0.0.1)"
"--host", default="127.0.0.1", help="Host to bind to (default: 127.0.0.1)"
)
parser.add_argument(
"--port",
type=int,
default=8000,
help="Port to bind to (default: 8000)"
"--port", type=int, default=8000, help="Port to bind to (default: 8000)"
)
parser.add_argument(
"--reindex",
action="store_true",
help="Force database reindexing"
"--reindex", action="store_true", help="Force database reindexing"
)
parser.add_argument(
"--dev",
action="store_true",
help="Development mode with auto-reload"
"--dev", action="store_true", help="Development mode with auto-reload"
)
parser.add_argument(
"--skip-index",
action="store_true",
help="Skip workflow indexing (useful for CI/testing)"
help="Skip workflow indexing (useful for CI/testing)",
)
args = parser.parse_args()
# Also check environment variable for CI mode
ci_mode = os.environ.get('CI', '').lower() in ('true', '1', 'yes')
ci_mode = os.environ.get("CI", "").lower() in ("true", "1", "yes")
skip_index = args.skip_index or ci_mode
print_banner()
# Check dependencies
if not check_requirements():
sys.exit(1)
# Setup directories
setup_directories()
# Setup database
try:
setup_database(force_reindex=args.reindex, skip_index=skip_index)
except Exception as e:
print(f"❌ Database setup error: {e}")
sys.exit(1)
# Start server
try:
start_server(
host=args.host,
port=args.port,
reload=args.dev
)
start_server(host=args.host, port=args.port, reload=args.dev)
except KeyboardInterrupt:
print("\n👋 Server stopped!")
except Exception as e:
@@ -190,4 +177,4 @@ Examples:
if __name__ == "__main__":
main()
main()
scripts/generate_search_index.py
+89 -75
View File
@@ -38,51 +38,58 @@ def generate_static_search_index(db_path: str, output_dir: str) -> Dict[str, Any
search_workflows = []
for workflow in workflows:
# Create searchable text combining multiple fields
searchable_text = ' '.join([
workflow['name'],
workflow['description'],
workflow['filename'],
' '.join(workflow['integrations']),
' '.join(workflow['tags']) if workflow['tags'] else ''
]).lower()
searchable_text = " ".join(
[
workflow["name"],
workflow["description"],
workflow["filename"],
" ".join(workflow["integrations"]),
" ".join(workflow["tags"]) if workflow["tags"] else "",
]
).lower()
# Use existing category from create_categories.py system, fallback to integration-based
category = get_workflow_category(workflow['filename'], existing_categories, workflow['integrations'], categories)
category = get_workflow_category(
workflow["filename"],
existing_categories,
workflow["integrations"],
categories,
)
search_workflow = {
'id': workflow['filename'].replace('.json', ''),
'name': workflow['name'],
'description': workflow['description'],
'filename': workflow['filename'],
'active': workflow['active'],
'trigger_type': workflow['trigger_type'],
'complexity': workflow['complexity'],
'node_count': workflow['node_count'],
'integrations': workflow['integrations'],
'tags': workflow['tags'],
'category': category,
'searchable_text': searchable_text,
'download_url': f"https://raw.githubusercontent.com/Zie619/n8n-workflows/main/workflows/{extract_folder_from_filename(workflow['filename'])}/{workflow['filename']}"
"id": workflow["filename"].replace(".json", ""),
"name": workflow["name"],
"description": workflow["description"],
"filename": workflow["filename"],
"active": workflow["active"],
"trigger_type": workflow["trigger_type"],
"complexity": workflow["complexity"],
"node_count": workflow["node_count"],
"integrations": workflow["integrations"],
"tags": workflow["tags"],
"category": category,
"searchable_text": searchable_text,
"download_url": f"https://raw.githubusercontent.com/Zie619/n8n-workflows/main/workflows/{extract_folder_from_filename(workflow['filename'])}/{workflow['filename']}",
}
search_workflows.append(search_workflow)
# Create comprehensive search index
search_index = {
'version': '1.0',
'generated_at': stats.get('last_indexed', ''),
'stats': {
'total_workflows': stats['total'],
'active_workflows': stats['active'],
'inactive_workflows': stats['inactive'],
'total_nodes': stats['total_nodes'],
'unique_integrations': stats['unique_integrations'],
'categories': len(get_category_list(categories)),
'triggers': stats['triggers'],
'complexity': stats['complexity']
"version": "1.0",
"generated_at": stats.get("last_indexed", ""),
"stats": {
"total_workflows": stats["total"],
"active_workflows": stats["active"],
"inactive_workflows": stats["inactive"],
"total_nodes": stats["total_nodes"],
"unique_integrations": stats["unique_integrations"],
"categories": len(get_category_list(categories)),
"triggers": stats["triggers"],
"complexity": stats["complexity"],
},
'categories': get_category_list(categories),
'integrations': get_popular_integrations(workflows),
'workflows': search_workflows
"categories": get_category_list(categories),
"integrations": get_popular_integrations(workflows),
"workflows": search_workflows,
}
return search_index
@@ -91,23 +98,29 @@ def generate_static_search_index(db_path: str, output_dir: str) -> Dict[str, Any
def load_existing_categories() -> Dict[str, str]:
"""Load existing categories from search_categories.json created by create_categories.py."""
try:
with open('context/search_categories.json', 'r', encoding='utf-8') as f:
with open("context/search_categories.json", "r", encoding="utf-8") as f:
categories_data = json.load(f)
# Convert to filename -> category mapping
category_mapping = {}
for item in categories_data:
if item.get('category'):
category_mapping[item['filename']] = item['category']
if item.get("category"):
category_mapping[item["filename"]] = item["category"]
return category_mapping
except FileNotFoundError:
print("Warning: search_categories.json not found, using integration-based categorization")
print(
"Warning: search_categories.json not found, using integration-based categorization"
)
return {}
def get_workflow_category(filename: str, existing_categories: Dict[str, str],
integrations: List[str], service_categories: Dict[str, List[str]]) -> str:
def get_workflow_category(
filename: str,
existing_categories: Dict[str, str],
integrations: List[str],
service_categories: Dict[str, List[str]],
) -> str:
"""Get category for workflow, preferring existing assignment over integration-based."""
# First priority: Use existing category from create_categories.py system
@@ -118,7 +131,9 @@ def get_workflow_category(filename: str, existing_categories: Dict[str, str],
return determine_category(integrations, service_categories)
def determine_category(integrations: List[str], categories: Dict[str, List[str]]) -> str:
def determine_category(
integrations: List[str], categories: Dict[str, List[str]]
) -> str:
"""Determine the category for a workflow based on its integrations."""
if not integrations:
return "Uncategorized"
@@ -135,20 +150,20 @@ def determine_category(integrations: List[str], categories: Dict[str, List[str]]
def format_category_name(category_key: str) -> str:
"""Format category key to display name."""
category_mapping = {
'messaging': 'Communication & Messaging',
'email': 'Communication & Messaging',
'cloud_storage': 'Cloud Storage & File Management',
'database': 'Data Processing & Analysis',
'project_management': 'Project Management',
'ai_ml': 'AI Agent Development',
'social_media': 'Social Media Management',
'ecommerce': 'E-commerce & Retail',
'analytics': 'Data Processing & Analysis',
'calendar_tasks': 'Project Management',
'forms': 'Data Processing & Analysis',
'development': 'Technical Infrastructure & DevOps'
"messaging": "Communication & Messaging",
"email": "Communication & Messaging",
"cloud_storage": "Cloud Storage & File Management",
"database": "Data Processing & Analysis",
"project_management": "Project Management",
"ai_ml": "AI Agent Development",
"social_media": "Social Media Management",
"ecommerce": "E-commerce & Retail",
"analytics": "Data Processing & Analysis",
"calendar_tasks": "Project Management",
"forms": "Data Processing & Analysis",
"development": "Technical Infrastructure & DevOps",
}
return category_mapping.get(category_key, category_key.replace('_', ' ').title())
return category_mapping.get(category_key, category_key.replace("_", " ").title())
def get_category_list(categories: Dict[str, List[str]]) -> List[str]:
@@ -165,7 +180,7 @@ def get_category_list(categories: Dict[str, List[str]]) -> List[str]:
"Creative Content & Video Automation",
"Creative Design Automation",
"CRM & Sales",
"Financial & Accounting"
"Financial & Accounting",
]
for cat in additional_categories:
@@ -179,30 +194,25 @@ def get_popular_integrations(workflows: List[Dict]) -> List[Dict[str, Any]]:
integration_counts = {}
for workflow in workflows:
for integration in workflow['integrations']:
for integration in workflow["integrations"]:
integration_counts[integration] = integration_counts.get(integration, 0) + 1
# Sort by count and take top 50
sorted_integrations = sorted(
integration_counts.items(),
key=lambda x: x[1],
reverse=True
integration_counts.items(), key=lambda x: x[1], reverse=True
)[:50]
return [
{'name': name, 'count': count}
for name, count in sorted_integrations
]
return [{"name": name, "count": count} for name, count in sorted_integrations]
def extract_folder_from_filename(filename: str) -> str:
"""Extract folder name from workflow filename."""
# Most workflows follow pattern: ID_Service_Purpose_Trigger.json
# Extract the service name as folder
parts = filename.replace('.json', '').split('_')
parts = filename.replace(".json", "").split("_")
if len(parts) >= 2:
return parts[1].capitalize() # Second part is usually the service
return 'Misc'
return "Misc"
def save_search_index(search_index: Dict[str, Any], output_dir: str):
@@ -212,22 +222,26 @@ def save_search_index(search_index: Dict[str, Any], output_dir: str):
os.makedirs(output_dir, exist_ok=True)
# Save complete index
with open(os.path.join(output_dir, 'search-index.json'), 'w', encoding='utf-8') as f:
with open(
os.path.join(output_dir, "search-index.json"), "w", encoding="utf-8"
) as f:
json.dump(search_index, f, indent=2, ensure_ascii=False)
# Save stats only (for quick loading)
with open(os.path.join(output_dir, 'stats.json'), 'w', encoding='utf-8') as f:
json.dump(search_index['stats'], f, indent=2, ensure_ascii=False)
with open(os.path.join(output_dir, "stats.json"), "w", encoding="utf-8") as f:
json.dump(search_index["stats"], f, indent=2, ensure_ascii=False)
# Save categories only
with open(os.path.join(output_dir, 'categories.json'), 'w', encoding='utf-8') as f:
json.dump(search_index['categories'], f, indent=2, ensure_ascii=False)
with open(os.path.join(output_dir, "categories.json"), "w", encoding="utf-8") as f:
json.dump(search_index["categories"], f, indent=2, ensure_ascii=False)
# Save integrations only
with open(os.path.join(output_dir, 'integrations.json'), 'w', encoding='utf-8') as f:
json.dump(search_index['integrations'], f, indent=2, ensure_ascii=False)
with open(
os.path.join(output_dir, "integrations.json"), "w", encoding="utf-8"
) as f:
json.dump(search_index["integrations"], f, indent=2, ensure_ascii=False)
print(f"Search index generated successfully:")
print("Search index generated successfully:")
print(f" {search_index['stats']['total_workflows']} workflows indexed")
print(f" {len(search_index['categories'])} categories")
print(f" {len(search_index['integrations'])} popular integrations")
@@ -260,4 +274,4 @@ def main():
if __name__ == "__main__":
main()
main()
scripts/update_github_pages.py
+46 -36
View File
@@ -6,11 +6,11 @@ Addresses Issues #115 and #129.
"""
import json
import os
from datetime import datetime
from pathlib import Path
import re
def update_html_timestamp(html_file: str):
"""Update the timestamp in the HTML file to current date."""
file_path = Path(html_file)
@@ -20,7 +20,7 @@ def update_html_timestamp(html_file: str):
return False
# Read the HTML file
with open(file_path, 'r', encoding='utf-8') as f:
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
# Get current month and year
@@ -29,22 +29,25 @@ def update_html_timestamp(html_file: str):
# Replace the hardcoded timestamp
# Look for pattern like "Last updated: Month Year"
pattern = r'(<p class="footer-meta">Last updated:)\s*([^<]+)'
replacement = f'\\1 {current_date}'
replacement = f"\\1 {current_date}"
updated_content = re.sub(pattern, replacement, content)
# Also add a meta tag with the exact timestamp for better tracking
if '<meta name="last-updated"' not in updated_content:
timestamp_meta = f' <meta name="last-updated" content="{datetime.now().isoformat()}">\n'
updated_content = updated_content.replace('</head>', f'{timestamp_meta}</head>')
timestamp_meta = (
f' <meta name="last-updated" content="{datetime.now().isoformat()}">\n'
)
updated_content = updated_content.replace("</head>", f"{timestamp_meta}</head>")
# Write back the updated content
with open(file_path, 'w', encoding='utf-8') as f:
with open(file_path, "w", encoding="utf-8") as f:
f.write(updated_content)
print(f"✅ Updated timestamp in {html_file} to: {current_date}")
return True
def update_api_timestamp(api_dir: str):
"""Update timestamp in API JSON files."""
api_path = Path(api_dir)
@@ -57,30 +60,31 @@ def update_api_timestamp(api_dir: str):
"last_updated": datetime.now().isoformat(),
"last_updated_readable": datetime.now().strftime("%B %d, %Y at %H:%M UTC"),
"version": "2.0.1",
"deployment_type": "github_pages"
"deployment_type": "github_pages",
}
metadata_file = api_path / 'metadata.json'
with open(metadata_file, 'w', encoding='utf-8') as f:
metadata_file = api_path / "metadata.json"
with open(metadata_file, "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=2)
print(f"✅ Created metadata file: {metadata_file}")
# Update stats.json if it exists
stats_file = api_path / 'stats.json'
stats_file = api_path / "stats.json"
if stats_file.exists():
with open(stats_file, 'r', encoding='utf-8') as f:
with open(stats_file, "r", encoding="utf-8") as f:
stats = json.load(f)
stats['last_updated'] = datetime.now().isoformat()
stats["last_updated"] = datetime.now().isoformat()
with open(stats_file, 'w', encoding='utf-8') as f:
with open(stats_file, "w", encoding="utf-8") as f:
json.dump(stats, f, indent=2)
print(f"✅ Updated stats file: {stats_file}")
return True
def create_github_pages_config():
"""Create necessary GitHub Pages configuration files."""
@@ -113,13 +117,13 @@ exclude:
- .devcontainer/
"""
config_file = Path('docs/_config.yml')
with open(config_file, 'w', encoding='utf-8') as f:
config_file = Path("docs/_config.yml")
with open(config_file, "w", encoding="utf-8") as f:
f.write(config_content)
print(f"✅ Created Jekyll config: {config_file}")
# Create .nojekyll file to bypass Jekyll processing (for pure HTML/JS site)
nojekyll_file = Path('docs/.nojekyll')
nojekyll_file = Path("docs/.nojekyll")
nojekyll_file.touch()
print(f"✅ Created .nojekyll file: {nojekyll_file}")
@@ -170,23 +174,24 @@ exclude:
</body>
</html>"""
error_file = Path('docs/404.html')
with open(error_file, 'w', encoding='utf-8') as f:
error_file = Path("docs/404.html")
with open(error_file, "w", encoding="utf-8") as f:
f.write(error_page_content)
print(f"✅ Created 404 page: {error_file}")
def verify_github_pages_structure():
"""Verify that all necessary files exist for GitHub Pages deployment."""
required_files = [
'docs/index.html',
'docs/css/styles.css',
'docs/js/app.js',
'docs/js/search.js',
'docs/api/search-index.json',
'docs/api/stats.json',
'docs/api/categories.json',
'docs/api/integrations.json'
"docs/index.html",
"docs/css/styles.css",
"docs/js/app.js",
"docs/js/search.js",
"docs/api/search-index.json",
"docs/api/stats.json",
"docs/api/categories.json",
"docs/api/integrations.json",
]
missing_files = []
@@ -208,13 +213,14 @@ def verify_github_pages_structure():
print("\n✅ All required files present for GitHub Pages deployment")
return True
def fix_base_url_references():
"""Fix any hardcoded URLs to use relative paths for GitHub Pages."""
# Update index.html to use relative paths
index_file = Path('docs/index.html')
index_file = Path("docs/index.html")
if index_file.exists():
with open(index_file, 'r', encoding='utf-8') as f:
with open(index_file, "r", encoding="utf-8") as f:
content = f.read()
# Replace absolute paths with relative ones
@@ -229,16 +235,16 @@ def fix_base_url_references():
for old, new in replacements:
content = content.replace(old, new)
with open(index_file, 'w', encoding='utf-8') as f:
with open(index_file, "w", encoding="utf-8") as f:
f.write(content)
print("✅ Fixed URL references in index.html")
# Update JavaScript files
js_files = ['docs/js/app.js', 'docs/js/search.js']
js_files = ["docs/js/app.js", "docs/js/search.js"]
for js_file in js_files:
js_path = Path(js_file)
if js_path.exists():
with open(js_path, 'r', encoding='utf-8') as f:
with open(js_path, "r", encoding="utf-8") as f:
content = f.read()
# Fix API endpoint references
@@ -247,10 +253,11 @@ def fix_base_url_references():
content = content.replace("'/api/", "'api/")
content = content.replace('"/api/', '"api/')
with open(js_path, 'w', encoding='utf-8') as f:
with open(js_path, "w", encoding="utf-8") as f:
f.write(content)
print(f"✅ Fixed URL references in {js_file}")
def main():
"""Main function to update GitHub Pages deployment."""
@@ -259,8 +266,8 @@ def main():
# Step 1: Update timestamps
print("\n📅 Updating timestamps...")
update_html_timestamp('docs/index.html')
update_api_timestamp('docs/api')
update_html_timestamp("docs/index.html")
update_api_timestamp("docs/api")
# Step 2: Create GitHub Pages configuration
print("\n⚙️ Creating GitHub Pages configuration...")
@@ -276,9 +283,12 @@ def main():
print("\n✨ GitHub Pages setup complete!")
print("\nDeployment will be available at:")
print(" https://zie619.github.io/n8n-workflows/")
print("\nNote: It may take a few minutes for changes to appear after pushing to GitHub.")
print(
"\nNote: It may take a few minutes for changes to appear after pushing to GitHub."
)
else:
print("\n⚠️ Some files are missing. Please generate them first.")
if __name__ == "__main__":
main()
main()
scripts/update_readme_stats.py
+99 -79
View File
@@ -4,7 +4,6 @@ Update README.md with current workflow statistics
Replaces hardcoded numbers with live data from the database.
"""
import json
import os
import re
import sys
@@ -32,15 +31,15 @@ def get_current_stats():
categories = db.get_service_categories()
return {
'total_workflows': stats['total'],
'active_workflows': stats['active'],
'inactive_workflows': stats['inactive'],
'total_nodes': stats['total_nodes'],
'unique_integrations': stats['unique_integrations'],
'categories_count': len(get_category_list(categories)),
'triggers': stats['triggers'],
'complexity': stats['complexity'],
'last_updated': datetime.now().strftime('%Y-%m-%d')
"total_workflows": stats["total"],
"active_workflows": stats["active"],
"inactive_workflows": stats["inactive"],
"total_nodes": stats["total_nodes"],
"unique_integrations": stats["unique_integrations"],
"categories_count": len(get_category_list(categories)),
"triggers": stats["triggers"],
"complexity": stats["complexity"],
"last_updated": datetime.now().strftime("%Y-%m-%d"),
}
@@ -50,22 +49,24 @@ def get_category_list(categories):
# Map technical categories to display names
category_mapping = {
'messaging': 'Communication & Messaging',
'email': 'Communication & Messaging',
'cloud_storage': 'Cloud Storage & File Management',
'database': 'Data Processing & Analysis',
'project_management': 'Project Management',
'ai_ml': 'AI Agent Development',
'social_media': 'Social Media Management',
'ecommerce': 'E-commerce & Retail',
'analytics': 'Data Processing & Analysis',
'calendar_tasks': 'Project Management',
'forms': 'Data Processing & Analysis',
'development': 'Technical Infrastructure & DevOps'
"messaging": "Communication & Messaging",
"email": "Communication & Messaging",
"cloud_storage": "Cloud Storage & File Management",
"database": "Data Processing & Analysis",
"project_management": "Project Management",
"ai_ml": "AI Agent Development",
"social_media": "Social Media Management",
"ecommerce": "E-commerce & Retail",
"analytics": "Data Processing & Analysis",
"calendar_tasks": "Project Management",
"forms": "Data Processing & Analysis",
"development": "Technical Infrastructure & DevOps",
}
for category_key in categories.keys():
display_name = category_mapping.get(category_key, category_key.replace('_', ' ').title())
display_name = category_mapping.get(
category_key, category_key.replace("_", " ").title()
)
formatted_categories.add(display_name)
# Add categories from the create_categories.py system
@@ -76,7 +77,7 @@ def get_category_list(categories):
"Creative Content & Video Automation",
"Creative Design Automation",
"CRM & Sales",
"Financial & Accounting"
"Financial & Accounting",
]
for cat in additional_categories:
@@ -93,71 +94,90 @@ def update_readme_stats(stats):
print("README.md not found")
return False
with open(readme_path, 'r', encoding='utf-8') as f:
with open(readme_path, "r", encoding="utf-8") as f:
content = f.read()
# Define replacement patterns and their new values
replacements = [
# Main collection description
(r'A professionally organized collection of \*\*[\d,]+\s+n8n workflows\*\*',
f'A professionally organized collection of **{stats["total_workflows"]:,} n8n workflows**'),
(
r"A professionally organized collection of \*\*[\d,]+\s+n8n workflows\*\*",
f"A professionally organized collection of **{stats['total_workflows']:,} n8n workflows**",
),
# Total workflows in various contexts
(r'- \*\*[\d,]+\s+workflows\*\* with meaningful',
f'- **{stats["total_workflows"]:,} workflows** with meaningful'),
(
r"- \*\*[\d,]+\s+workflows\*\* with meaningful",
f"- **{stats['total_workflows']:,} workflows** with meaningful",
),
# Statistics section
(r'- \*\*Total Workflows\*\*: [\d,]+',
f'- **Total Workflows**: {stats["total_workflows"]:,}'),
(r'- \*\*Active Workflows\*\*: [\d,]+ \([\d.]+%',
f'- **Active Workflows**: {stats["active_workflows"]:,} ({(stats["active_workflows"]/stats["total_workflows"]*100):.1f}%'),
(r'- \*\*Total Nodes\*\*: [\d,]+ \(avg [\d.]+ nodes',
f'- **Total Nodes**: {stats["total_nodes"]:,} (avg {(stats["total_nodes"]/stats["total_workflows"]):.1f} nodes'),
(r'- \*\*Unique Integrations\*\*: [\d,]+ different',
f'- **Unique Integrations**: {stats["unique_integrations"]:,} different'),
(
r"- \*\*Total Workflows\*\*: [\d,]+",
f"- **Total Workflows**: {stats['total_workflows']:,}",
),
(
r"- \*\*Active Workflows\*\*: [\d,]+ \([\d.]+%",
f"- **Active Workflows**: {stats['active_workflows']:,} ({(stats['active_workflows'] / stats['total_workflows'] * 100):.1f}%",
),
(
r"- \*\*Total Nodes\*\*: [\d,]+ \(avg [\d.]+ nodes",
f"- **Total Nodes**: {stats['total_nodes']:,} (avg {(stats['total_nodes'] / stats['total_workflows']):.1f} nodes",
),
(
r"- \*\*Unique Integrations\*\*: [\d,]+ different",
f"- **Unique Integrations**: {stats['unique_integrations']:,} different",
),
# Update complexity/trigger distribution
(r'- \*\*Complex\*\*: [\d,]+ workflows \([\d.]+%\)',
f'- **Complex**: {stats["triggers"].get("Complex", 0):,} workflows ({(stats["triggers"].get("Complex", 0)/stats["total_workflows"]*100):.1f}%)'),
(r'- \*\*Webhook\*\*: [\d,]+ workflows \([\d.]+%\)',
f'- **Webhook**: {stats["triggers"].get("Webhook", 0):,} workflows ({(stats["triggers"].get("Webhook", 0)/stats["total_workflows"]*100):.1f}%)'),
(r'- \*\*Manual\*\*: [\d,]+ workflows \([\d.]+%\)',
f'- **Manual**: {stats["triggers"].get("Manual", 0):,} workflows ({(stats["triggers"].get("Manual", 0)/stats["total_workflows"]*100):.1f}%)'),
(r'- \*\*Scheduled\*\*: [\d,]+ workflows \([\d.]+%\)',
f'- **Scheduled**: {stats["triggers"].get("Scheduled", 0):,} workflows ({(stats["triggers"].get("Scheduled", 0)/stats["total_workflows"]*100):.1f}%)'),
(
r"- \*\*Complex\*\*: [\d,]+ workflows \([\d.]+%\)",
f"- **Complex**: {stats['triggers'].get('Complex', 0):,} workflows ({(stats['triggers'].get('Complex', 0) / stats['total_workflows'] * 100):.1f}%)",
),
(
r"- \*\*Webhook\*\*: [\d,]+ workflows \([\d.]+%\)",
f"- **Webhook**: {stats['triggers'].get('Webhook', 0):,} workflows ({(stats['triggers'].get('Webhook', 0) / stats['total_workflows'] * 100):.1f}%)",
),
(
r"- \*\*Manual\*\*: [\d,]+ workflows \([\d.]+%\)",
f"- **Manual**: {stats['triggers'].get('Manual', 0):,} workflows ({(stats['triggers'].get('Manual', 0) / stats['total_workflows'] * 100):.1f}%)",
),
(
r"- \*\*Scheduled\*\*: [\d,]+ workflows \([\d.]+%\)",
f"- **Scheduled**: {stats['triggers'].get('Scheduled', 0):,} workflows ({(stats['triggers'].get('Scheduled', 0) / stats['total_workflows'] * 100):.1f}%)",
),
# Update total in current collection stats
(r'\*\*Total Workflows\*\*: [\d,]+ automation',
f'**Total Workflows**: {stats["total_workflows"]:,} automation'),
(r'\*\*Active Workflows\*\*: [\d,]+ \([\d.]+% active',
f'**Active Workflows**: {stats["active_workflows"]:,} ({(stats["active_workflows"]/stats["total_workflows"]*100):.1f}% active'),
(r'\*\*Total Nodes\*\*: [\d,]+ \(avg [\d.]+ nodes',
f'**Total Nodes**: {stats["total_nodes"]:,} (avg {(stats["total_nodes"]/stats["total_workflows"]):.1f} nodes'),
(r'\*\*Unique Integrations\*\*: [\d,]+ different',
f'**Unique Integrations**: {stats["unique_integrations"]:,} different'),
(
r"\*\*Total Workflows\*\*: [\d,]+ automation",
f"**Total Workflows**: {stats['total_workflows']:,} automation",
),
(
r"\*\*Active Workflows\*\*: [\d,]+ \([\d.]+% active",
f"**Active Workflows**: {stats['active_workflows']:,} ({(stats['active_workflows'] / stats['total_workflows'] * 100):.1f}% active",
),
(
r"\*\*Total Nodes\*\*: [\d,]+ \(avg [\d.]+ nodes",
f"**Total Nodes**: {stats['total_nodes']:,} (avg {(stats['total_nodes'] / stats['total_workflows']):.1f} nodes",
),
(
r"\*\*Unique Integrations\*\*: [\d,]+ different",
f"**Unique Integrations**: {stats['unique_integrations']:,} different",
),
# Categories count
(r'Our system automatically categorizes workflows into [\d]+ service categories',
f'Our system automatically categorizes workflows into {stats["categories_count"]} service categories'),
(
r"Our system automatically categorizes workflows into [\d]+ service categories",
f"Our system automatically categorizes workflows into {stats['categories_count']} service categories",
),
# Update any "2000+" references
(r'2000\+', f'{stats["total_workflows"]:,}+'),
(r'2,000\+', f'{stats["total_workflows"]:,}+'),
(r"2000\+", f"{stats['total_workflows']:,}+"),
(r"2,000\+", f"{stats['total_workflows']:,}+"),
# Search across X workflows
(r'Search across [\d,]+ workflows', f'Search across {stats["total_workflows"]:,} workflows'),
(
r"Search across [\d,]+ workflows",
f"Search across {stats['total_workflows']:,} workflows",
),
# Instant search across X workflows
(r'Instant search across [\d,]+ workflows', f'Instant search across {stats["total_workflows"]:,} workflows'),
(
r"Instant search across [\d,]+ workflows",
f"Instant search across {stats['total_workflows']:,} workflows",
),
]
# Apply all replacements
@@ -171,10 +191,10 @@ def update_readme_stats(stats):
replacements_made += 1
# Write back to file
with open(readme_path, 'w', encoding='utf-8') as f:
with open(readme_path, "w", encoding="utf-8") as f:
f.write(updated_content)
print(f"README.md updated with current statistics:")
print("README.md updated with current statistics:")
print(f" - Total workflows: {stats['total_workflows']:,}")
print(f" - Active workflows: {stats['active_workflows']:,}")
print(f" - Total nodes: {stats['total_nodes']:,}")
@@ -210,4 +230,4 @@ def main():
if __name__ == "__main__":
main()
main()
src/ai_assistant.py
+133 -91
View File
@@ -4,59 +4,63 @@ AI Assistant for N8N Workflow Discovery
Intelligent chat interface for finding and understanding workflows.
"""
from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
from fastapi import FastAPI, HTTPException
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from typing import List, Dict, Any, Optional
from typing import List, Dict, Optional
import json
import asyncio
import sqlite3
from datetime import datetime
import re
class ChatMessage(BaseModel):
message: str
user_id: Optional[str] = None
class AIResponse(BaseModel):
response: str
workflows: List[Dict] = []
suggestions: List[str] = []
confidence: float = 0.0
class WorkflowAssistant:
def __init__(self, db_path: str = "workflows.db"):
self.db_path = db_path
self.conversation_history = {}
def get_db_connection(self):
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def search_workflows_intelligent(self, query: str, limit: int = 5) -> List[Dict]:
"""Intelligent workflow search based on natural language query."""
conn = self.get_db_connection()
# Extract keywords and intent from query
keywords = self.extract_keywords(query)
intent = self.detect_intent(query)
# Build search query
search_terms = []
for keyword in keywords:
search_terms.append(f"name LIKE '%{keyword}%' OR description LIKE '%{keyword}%'")
search_terms.append(
f"name LIKE '%{keyword}%' OR description LIKE '%{keyword}%'"
)
where_clause = " OR ".join(search_terms) if search_terms else "1=1"
# Add intent-based filtering
if intent == "automation":
where_clause += " AND (trigger_type = 'Scheduled' OR trigger_type = 'Complex')"
where_clause += (
" AND (trigger_type = 'Scheduled' OR trigger_type = 'Complex')"
)
elif intent == "integration":
where_clause += " AND trigger_type = 'Webhook'"
elif intent == "manual":
where_clause += " AND trigger_type = 'Manual'"
query_sql = f"""
SELECT * FROM workflows
WHERE {where_clause}
@@ -65,180 +69,216 @@ class WorkflowAssistant:
node_count DESC
LIMIT {limit}
"""
cursor = conn.execute(query_sql)
workflows = []
for row in cursor.fetchall():
workflow = dict(row)
workflow['integrations'] = json.loads(workflow['integrations'] or '[]')
workflow['tags'] = json.loads(workflow['tags'] or '[]')
workflow["integrations"] = json.loads(workflow["integrations"] or "[]")
workflow["tags"] = json.loads(workflow["tags"] or "[]")
workflows.append(workflow)
conn.close()
return workflows
def extract_keywords(self, query: str) -> List[str]:
"""Extract relevant keywords from user query."""
# Common automation terms
automation_terms = {
'email': ['email', 'gmail', 'mail'],
'social': ['twitter', 'facebook', 'instagram', 'linkedin', 'social'],
'data': ['data', 'database', 'spreadsheet', 'csv', 'excel'],
'ai': ['ai', 'openai', 'chatgpt', 'artificial', 'intelligence'],
'notification': ['notification', 'alert', 'slack', 'telegram', 'discord'],
'automation': ['automation', 'workflow', 'process', 'automate'],
'integration': ['integration', 'connect', 'sync', 'api']
"email": ["email", "gmail", "mail"],
"social": ["twitter", "facebook", "instagram", "linkedin", "social"],
"data": ["data", "database", "spreadsheet", "csv", "excel"],
"ai": ["ai", "openai", "chatgpt", "artificial", "intelligence"],
"notification": ["notification", "alert", "slack", "telegram", "discord"],
"automation": ["automation", "workflow", "process", "automate"],
"integration": ["integration", "connect", "sync", "api"],
}
query_lower = query.lower()
keywords = []
for category, terms in automation_terms.items():
for term in terms:
if term in query_lower:
keywords.append(term)
# Extract specific service names
services = ['slack', 'telegram', 'openai', 'google', 'microsoft', 'shopify', 'airtable']
services = [
"slack",
"telegram",
"openai",
"google",
"microsoft",
"shopify",
"airtable",
]
for service in services:
if service in query_lower:
keywords.append(service)
return list(set(keywords))
def detect_intent(self, query: str) -> str:
"""Detect user intent from query."""
query_lower = query.lower()
if any(word in query_lower for word in ['automate', 'schedule', 'recurring', 'daily', 'weekly']):
if any(
word in query_lower
for word in ["automate", "schedule", "recurring", "daily", "weekly"]
):
return "automation"
elif any(word in query_lower for word in ['connect', 'integrate', 'sync', 'webhook']):
elif any(
word in query_lower for word in ["connect", "integrate", "sync", "webhook"]
):
return "integration"
elif any(word in query_lower for word in ['manual', 'trigger', 'button', 'click']):
elif any(
word in query_lower for word in ["manual", "trigger", "button", "click"]
):
return "manual"
elif any(word in query_lower for word in ['ai', 'chat', 'assistant', 'intelligent']):
elif any(
word in query_lower for word in ["ai", "chat", "assistant", "intelligent"]
):
return "ai"
else:
return "general"
def generate_response(self, query: str, workflows: List[Dict]) -> str:
"""Generate natural language response based on query and workflows."""
if not workflows:
return "I couldn't find any workflows matching your request. Try searching for specific services like 'Slack', 'OpenAI', or 'Email automation'."
# Analyze workflow patterns
trigger_types = [w['trigger_type'] for w in workflows]
trigger_types = [w["trigger_type"] for w in workflows]
integrations = []
for w in workflows:
integrations.extend(w['integrations'])
integrations.extend(w["integrations"])
common_integrations = list(set(integrations))[:3]
most_common_trigger = max(set(trigger_types), key=trigger_types.count)
# Generate contextual response
response_parts = []
if len(workflows) == 1:
workflow = workflows[0]
response_parts.append(f"I found a perfect match: **{workflow['name']}**")
response_parts.append(f"This is a {workflow['trigger_type'].lower()} workflow that {workflow['description'].lower()}")
response_parts.append(
f"This is a {workflow['trigger_type'].lower()} workflow that {workflow['description'].lower()}"
)
else:
response_parts.append(f"I found {len(workflows)} relevant workflows:")
for i, workflow in enumerate(workflows[:3], 1):
response_parts.append(f"{i}. **{workflow['name']}** - {workflow['description']}")
response_parts.append(
f"{i}. **{workflow['name']}** - {workflow['description']}"
)
if common_integrations:
response_parts.append(f"\nThese workflows commonly use: {', '.join(common_integrations)}")
if most_common_trigger != 'all':
response_parts.append(f"Most are {most_common_trigger.lower()} triggered workflows.")
response_parts.append(
f"\nThese workflows commonly use: {', '.join(common_integrations)}"
)
if most_common_trigger != "all":
response_parts.append(
f"Most are {most_common_trigger.lower()} triggered workflows."
)
return "\n".join(response_parts)
def get_suggestions(self, query: str) -> List[str]:
"""Generate helpful suggestions based on query."""
suggestions = []
if 'email' in query.lower():
suggestions.extend([
"Email automation workflows",
"Gmail integration examples",
"Email notification systems"
])
elif 'ai' in query.lower() or 'openai' in query.lower():
suggestions.extend([
"AI-powered workflows",
"OpenAI integration examples",
"Chatbot automation"
])
elif 'social' in query.lower():
suggestions.extend([
"Social media automation",
"Twitter integration workflows",
"LinkedIn automation"
])
if "email" in query.lower():
suggestions.extend(
[
"Email automation workflows",
"Gmail integration examples",
"Email notification systems",
]
)
elif "ai" in query.lower() or "openai" in query.lower():
suggestions.extend(
[
"AI-powered workflows",
"OpenAI integration examples",
"Chatbot automation",
]
)
elif "social" in query.lower():
suggestions.extend(
[
"Social media automation",
"Twitter integration workflows",
"LinkedIn automation",
]
)
else:
suggestions.extend([
"Popular automation patterns",
"Webhook-triggered workflows",
"Scheduled automation examples"
])
suggestions.extend(
[
"Popular automation patterns",
"Webhook-triggered workflows",
"Scheduled automation examples",
]
)
return suggestions[:3]
def calculate_confidence(self, query: str, workflows: List[Dict]) -> float:
"""Calculate confidence score for the response."""
if not workflows:
return 0.0
# Base confidence on number of matches and relevance
base_confidence = min(len(workflows) / 5.0, 1.0)
# Boost confidence for exact matches
query_lower = query.lower()
exact_matches = 0
for workflow in workflows:
if any(word in workflow['name'].lower() for word in query_lower.split()):
if any(word in workflow["name"].lower() for word in query_lower.split()):
exact_matches += 1
if exact_matches > 0:
base_confidence += 0.2
return min(base_confidence, 1.0)
# Initialize assistant
assistant = WorkflowAssistant()
# FastAPI app for AI Assistant
ai_app = FastAPI(title="N8N AI Assistant", version="1.0.0")
@ai_app.post("/chat", response_model=AIResponse)
async def chat_with_assistant(message: ChatMessage):
"""Chat with the AI assistant to discover workflows."""
try:
# Search for relevant workflows
workflows = assistant.search_workflows_intelligent(message.message, limit=5)
# Generate response
response_text = assistant.generate_response(message.message, workflows)
# Get suggestions
suggestions = assistant.get_suggestions(message.message)
# Calculate confidence
confidence = assistant.calculate_confidence(message.message, workflows)
return AIResponse(
response=response_text,
workflows=workflows,
suggestions=suggestions,
confidence=confidence
confidence=confidence,
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Assistant error: {str(e)}")
@ai_app.get("/chat/interface")
async def chat_interface():
"""Get the chat interface HTML."""
@@ -544,6 +584,8 @@ async def chat_interface():
"""
return HTMLResponse(content=html_content)
if __name__ == "__main__":
import uvicorn
uvicorn.run(ai_app, host="127.0.0.1", port=8001)
src/analytics_engine.py
+136 -94
View File
@@ -7,12 +7,12 @@ Provides insights, patterns, and usage analytics.
from fastapi import FastAPI, HTTPException, Query
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from typing import List, Dict, Any, Optional
from typing import List, Dict, Any
import sqlite3
import json
from datetime import datetime, timedelta
from datetime import datetime
from collections import Counter, defaultdict
import statistics
class AnalyticsResponse(BaseModel):
overview: Dict[str, Any]
@@ -21,26 +21,29 @@ class AnalyticsResponse(BaseModel):
recommendations: List[str]
generated_at: str
class WorkflowAnalytics:
def __init__(self, db_path: str = "workflows.db"):
self.db_path = db_path
def get_db_connection(self):
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def get_workflow_analytics(self) -> Dict[str, Any]:
"""Get comprehensive workflow analytics."""
conn = self.get_db_connection()
# Basic statistics
cursor = conn.execute("SELECT COUNT(*) as total FROM workflows")
total_workflows = cursor.fetchone()['total']
cursor = conn.execute("SELECT COUNT(*) as active FROM workflows WHERE active = 1")
active_workflows = cursor.fetchone()['active']
total_workflows = cursor.fetchone()["total"]
cursor = conn.execute(
"SELECT COUNT(*) as active FROM workflows WHERE active = 1"
)
active_workflows = cursor.fetchone()["active"]
# Trigger type distribution
cursor = conn.execute("""
SELECT trigger_type, COUNT(*) as count
@@ -48,8 +51,10 @@ class WorkflowAnalytics:
GROUP BY trigger_type
ORDER BY count DESC
""")
trigger_distribution = {row['trigger_type']: row['count'] for row in cursor.fetchall()}
trigger_distribution = {
row["trigger_type"]: row["count"] for row in cursor.fetchall()
}
# Complexity distribution
cursor = conn.execute("""
SELECT complexity, COUNT(*) as count
@@ -57,8 +62,10 @@ class WorkflowAnalytics:
GROUP BY complexity
ORDER BY count DESC
""")
complexity_distribution = {row['complexity']: row['count'] for row in cursor.fetchall()}
complexity_distribution = {
row["complexity"]: row["count"] for row in cursor.fetchall()
}
# Node count statistics
cursor = conn.execute("""
SELECT
@@ -69,47 +76,54 @@ class WorkflowAnalytics:
FROM workflows
""")
node_stats = dict(cursor.fetchone())
# Integration analysis
cursor = conn.execute("SELECT integrations FROM workflows WHERE integrations IS NOT NULL")
cursor = conn.execute(
"SELECT integrations FROM workflows WHERE integrations IS NOT NULL"
)
all_integrations = []
for row in cursor.fetchall():
integrations = json.loads(row['integrations'] or '[]')
integrations = json.loads(row["integrations"] or "[]")
all_integrations.extend(integrations)
integration_counts = Counter(all_integrations)
top_integrations = dict(integration_counts.most_common(10))
# Workflow patterns
patterns = self.analyze_workflow_patterns(conn)
# Recommendations
recommendations = self.generate_recommendations(
total_workflows, active_workflows, trigger_distribution,
complexity_distribution, top_integrations
total_workflows,
active_workflows,
trigger_distribution,
complexity_distribution,
top_integrations,
)
conn.close()
return {
"overview": {
"total_workflows": total_workflows,
"active_workflows": active_workflows,
"activation_rate": round((active_workflows / total_workflows) * 100, 2) if total_workflows > 0 else 0,
"activation_rate": round((active_workflows / total_workflows) * 100, 2)
if total_workflows > 0
else 0,
"unique_integrations": len(integration_counts),
"avg_nodes_per_workflow": round(node_stats['avg_nodes'], 2),
"most_complex_workflow": node_stats['max_nodes']
"avg_nodes_per_workflow": round(node_stats["avg_nodes"], 2),
"most_complex_workflow": node_stats["max_nodes"],
},
"distributions": {
"trigger_types": trigger_distribution,
"complexity_levels": complexity_distribution,
"top_integrations": top_integrations
"top_integrations": top_integrations,
},
"patterns": patterns,
"recommendations": recommendations,
"generated_at": datetime.now().isoformat()
"generated_at": datetime.now().isoformat(),
}
def analyze_workflow_patterns(self, conn) -> Dict[str, Any]:
"""Analyze common workflow patterns and relationships."""
# Integration co-occurrence analysis
@@ -118,27 +132,27 @@ class WorkflowAnalytics:
FROM workflows
WHERE integrations IS NOT NULL
""")
integration_pairs = defaultdict(int)
service_categories = defaultdict(int)
for row in cursor.fetchall():
integrations = json.loads(row['integrations'] or '[]')
integrations = json.loads(row["integrations"] or "[]")
# Count service categories
for integration in integrations:
category = self.categorize_service(integration)
service_categories[category] += 1
# Find integration pairs
for i in range(len(integrations)):
for j in range(i + 1, len(integrations)):
pair = tuple(sorted([integrations[i], integrations[j]]))
integration_pairs[pair] += 1
# Most common integration pairs
top_pairs = dict(Counter(integration_pairs).most_common(5))
# Workflow complexity patterns
cursor = conn.execute("""
SELECT
@@ -150,46 +164,61 @@ class WorkflowAnalytics:
GROUP BY trigger_type, complexity
ORDER BY count DESC
""")
complexity_patterns = []
for row in cursor.fetchall():
complexity_patterns.append({
"trigger_type": row['trigger_type'],
"complexity": row['complexity'],
"avg_nodes": round(row['avg_nodes'], 2),
"frequency": row['count']
})
complexity_patterns.append(
{
"trigger_type": row["trigger_type"],
"complexity": row["complexity"],
"avg_nodes": round(row["avg_nodes"], 2),
"frequency": row["count"],
}
)
return {
"integration_pairs": top_pairs,
"service_categories": dict(service_categories),
"complexity_patterns": complexity_patterns[:10]
"complexity_patterns": complexity_patterns[:10],
}
def categorize_service(self, service: str) -> str:
"""Categorize a service into a broader category."""
service_lower = service.lower()
if any(word in service_lower for word in ['slack', 'telegram', 'discord', 'whatsapp']):
if any(
word in service_lower
for word in ["slack", "telegram", "discord", "whatsapp"]
):
return "Communication"
elif any(word in service_lower for word in ['openai', 'ai', 'chat', 'gpt']):
elif any(word in service_lower for word in ["openai", "ai", "chat", "gpt"]):
return "AI/ML"
elif any(word in service_lower for word in ['google', 'microsoft', 'office']):
elif any(word in service_lower for word in ["google", "microsoft", "office"]):
return "Productivity"
elif any(word in service_lower for word in ['shopify', 'woocommerce', 'stripe']):
elif any(
word in service_lower for word in ["shopify", "woocommerce", "stripe"]
):
return "E-commerce"
elif any(word in service_lower for word in ['airtable', 'notion', 'database']):
elif any(word in service_lower for word in ["airtable", "notion", "database"]):
return "Data Management"
elif any(word in service_lower for word in ['twitter', 'facebook', 'instagram']):
elif any(
word in service_lower for word in ["twitter", "facebook", "instagram"]
):
return "Social Media"
else:
return "Other"
def generate_recommendations(self, total: int, active: int, triggers: Dict,
complexity: Dict, integrations: Dict) -> List[str]:
def generate_recommendations(
self,
total: int,
active: int,
triggers: Dict,
complexity: Dict,
integrations: Dict,
) -> List[str]:
"""Generate actionable recommendations based on analytics."""
recommendations = []
# Activation rate recommendations
activation_rate = (active / total) * 100 if total > 0 else 0
if activation_rate < 20:
@@ -202,11 +231,11 @@ class WorkflowAnalytics:
f"High activation rate ({activation_rate:.1f}%)! Your workflows are well-maintained. "
"Consider documenting successful patterns for team sharing."
)
# Trigger type recommendations
webhook_count = triggers.get('Webhook', 0)
scheduled_count = triggers.get('Scheduled', 0)
webhook_count = triggers.get("Webhook", 0)
scheduled_count = triggers.get("Scheduled", 0)
if webhook_count > scheduled_count * 2:
recommendations.append(
"You have many webhook-triggered workflows. Consider adding scheduled workflows "
@@ -217,30 +246,30 @@ class WorkflowAnalytics:
"You have many scheduled workflows. Consider adding webhook-triggered workflows "
"for real-time integrations and event-driven automation."
)
# Integration recommendations
if 'OpenAI' in integrations and integrations['OpenAI'] > 5:
if "OpenAI" in integrations and integrations["OpenAI"] > 5:
recommendations.append(
"You're using OpenAI extensively. Consider creating AI workflow templates "
"for common use cases like content generation and data analysis."
)
if 'Slack' in integrations and 'Telegram' in integrations:
if "Slack" in integrations and "Telegram" in integrations:
recommendations.append(
"You're using multiple communication platforms. Consider creating unified "
"notification workflows that can send to multiple channels."
)
# Complexity recommendations
high_complexity = complexity.get('high', 0)
high_complexity = complexity.get("high", 0)
if high_complexity > total * 0.3:
recommendations.append(
"You have many high-complexity workflows. Consider breaking them down into "
"smaller, reusable components for better maintainability."
)
return recommendations
def get_trend_analysis(self, days: int = 30) -> Dict[str, Any]:
"""Analyze trends over time (simulated for demo)."""
# In a real implementation, this would analyze historical data
@@ -248,24 +277,24 @@ class WorkflowAnalytics:
"workflow_growth": {
"daily_average": 2.3,
"growth_rate": 15.2,
"trend": "increasing"
"trend": "increasing",
},
"popular_integrations": {
"trending_up": ["OpenAI", "Slack", "Google Sheets"],
"trending_down": ["Twitter", "Facebook"],
"stable": ["Telegram", "Airtable"]
"stable": ["Telegram", "Airtable"],
},
"complexity_trends": {
"average_nodes": 12.5,
"complexity_increase": 8.3,
"automation_maturity": "intermediate"
}
"automation_maturity": "intermediate",
},
}
def get_usage_insights(self) -> Dict[str, Any]:
"""Get usage insights and patterns."""
conn = self.get_db_connection()
# Active vs inactive analysis
cursor = conn.execute("""
SELECT
@@ -276,23 +305,29 @@ class WorkflowAnalytics:
FROM workflows
GROUP BY trigger_type, complexity
""")
usage_patterns = []
for row in cursor.fetchall():
activation_rate = (row['active_count'] / row['total']) * 100 if row['total'] > 0 else 0
usage_patterns.append({
"trigger_type": row['trigger_type'],
"complexity": row['complexity'],
"total_workflows": row['total'],
"active_workflows": row['active_count'],
"activation_rate": round(activation_rate, 2)
})
activation_rate = (
(row["active_count"] / row["total"]) * 100 if row["total"] > 0 else 0
)
usage_patterns.append(
{
"trigger_type": row["trigger_type"],
"complexity": row["complexity"],
"total_workflows": row["total"],
"active_workflows": row["active_count"],
"activation_rate": round(activation_rate, 2),
}
)
# Most effective patterns
effective_patterns = sorted(usage_patterns, key=lambda x: x['activation_rate'], reverse=True)[:5]
effective_patterns = sorted(
usage_patterns, key=lambda x: x["activation_rate"], reverse=True
)[:5]
conn.close()
return {
"usage_patterns": usage_patterns,
"most_effective_patterns": effective_patterns,
@@ -300,16 +335,18 @@ class WorkflowAnalytics:
"Webhook-triggered workflows have higher activation rates",
"Medium complexity workflows are most commonly used",
"AI-powered workflows show increasing adoption",
"Communication integrations are most popular"
]
"Communication integrations are most popular",
],
}
# Initialize analytics engine
analytics_engine = WorkflowAnalytics()
# FastAPI app for Analytics
analytics_app = FastAPI(title="N8N Analytics Engine", version="1.0.0")
@analytics_app.get("/analytics/overview", response_model=AnalyticsResponse)
async def get_analytics_overview():
"""Get comprehensive analytics overview."""
@@ -317,17 +354,18 @@ async def get_analytics_overview():
analytics_data = analytics_engine.get_workflow_analytics()
trends = analytics_engine.get_trend_analysis()
insights = analytics_engine.get_usage_insights()
return AnalyticsResponse(
overview=analytics_data["overview"],
trends=trends,
patterns=analytics_data["patterns"],
recommendations=analytics_data["recommendations"],
generated_at=analytics_data["generated_at"]
generated_at=analytics_data["generated_at"],
)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Analytics error: {str(e)}")
@analytics_app.get("/analytics/trends")
async def get_trend_analysis(days: int = Query(30, ge=1, le=365)):
"""Get trend analysis for specified period."""
@@ -336,6 +374,7 @@ async def get_trend_analysis(days: int = Query(30, ge=1, le=365)):
except Exception as e:
raise HTTPException(status_code=500, detail=f"Trend analysis error: {str(e)}")
@analytics_app.get("/analytics/insights")
async def get_usage_insights():
"""Get usage insights and patterns."""
@@ -344,6 +383,7 @@ async def get_usage_insights():
except Exception as e:
raise HTTPException(status_code=500, detail=f"Insights error: {str(e)}")
@analytics_app.get("/analytics/dashboard")
async def get_analytics_dashboard():
"""Get analytics dashboard HTML."""
@@ -583,6 +623,8 @@ async def get_analytics_dashboard():
"""
return HTMLResponse(content=html_content)
if __name__ == "__main__":
import uvicorn
uvicorn.run(analytics_app, host="127.0.0.1", port=8002)
src/community_features.py
+196 -127
View File
@@ -6,14 +6,15 @@ Implements rating, review, and social features
import sqlite3
import json
import hashlib
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from typing import Dict, List, Optional
from dataclasses import dataclass
@dataclass
class WorkflowRating:
"""Workflow rating data structure"""
workflow_id: str
user_id: str
rating: int # 1-5 stars
@@ -22,9 +23,11 @@ class WorkflowRating:
created_at: datetime = None
updated_at: datetime = None
@dataclass
class WorkflowStats:
"""Workflow statistics"""
workflow_id: str
total_ratings: int
average_rating: float
@@ -33,19 +36,20 @@ class WorkflowStats:
total_downloads: int
last_updated: datetime
class CommunityFeatures:
"""Community features manager for workflow repository"""
def __init__(self, db_path: str = "workflows.db"):
"""Initialize community features with database connection"""
self.db_path = db_path
self.init_community_tables()
def init_community_tables(self):
"""Initialize community feature database tables"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Workflow ratings and reviews
cursor.execute("""
CREATE TABLE IF NOT EXISTS workflow_ratings (
@@ -60,7 +64,7 @@ class CommunityFeatures:
UNIQUE(workflow_id, user_id)
)
""")
# Workflow usage statistics
cursor.execute("""
CREATE TABLE IF NOT EXISTS workflow_stats (
@@ -73,7 +77,7 @@ class CommunityFeatures:
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# User profiles
cursor.execute("""
CREATE TABLE IF NOT EXISTS user_profiles (
@@ -89,7 +93,7 @@ class CommunityFeatures:
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# Workflow collections (user favorites)
cursor.execute("""
CREATE TABLE IF NOT EXISTS workflow_collections (
@@ -103,7 +107,7 @@ class CommunityFeatures:
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
# Workflow comments
cursor.execute("""
CREATE TABLE IF NOT EXISTS workflow_comments (
@@ -117,81 +121,96 @@ class CommunityFeatures:
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
conn.commit()
conn.close()
def add_rating(self, workflow_id: str, user_id: str, rating: int, review: str = None) -> bool:
def add_rating(
self, workflow_id: str, user_id: str, rating: int, review: str = None
) -> bool:
"""Add or update a workflow rating and review"""
if not (1 <= rating <= 5):
raise ValueError("Rating must be between 1 and 5")
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
try:
# Insert or update rating
cursor.execute("""
cursor.execute(
"""
INSERT OR REPLACE INTO workflow_ratings
(workflow_id, user_id, rating, review, updated_at)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
""", (workflow_id, user_id, rating, review))
""",
(workflow_id, user_id, rating, review),
)
# Update workflow statistics
self._update_workflow_stats(workflow_id)
conn.commit()
return True
except Exception as e:
print(f"Error adding rating: {e}")
return False
finally:
conn.close()
def get_workflow_ratings(self, workflow_id: str, limit: int = 10) -> List[WorkflowRating]:
def get_workflow_ratings(
self, workflow_id: str, limit: int = 10
) -> List[WorkflowRating]:
"""Get ratings and reviews for a workflow"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT workflow_id, user_id, rating, review, helpful_votes, created_at, updated_at
FROM workflow_ratings
WHERE workflow_id = ?
ORDER BY helpful_votes DESC, created_at DESC
LIMIT ?
""", (workflow_id, limit))
""",
(workflow_id, limit),
)
ratings = []
for row in cursor.fetchall():
ratings.append(WorkflowRating(
workflow_id=row[0],
user_id=row[1],
rating=row[2],
review=row[3],
helpful_votes=row[4],
created_at=datetime.fromisoformat(row[5]) if row[5] else None,
updated_at=datetime.fromisoformat(row[6]) if row[6] else None
))
ratings.append(
WorkflowRating(
workflow_id=row[0],
user_id=row[1],
rating=row[2],
review=row[3],
helpful_votes=row[4],
created_at=datetime.fromisoformat(row[5]) if row[5] else None,
updated_at=datetime.fromisoformat(row[6]) if row[6] else None,
)
)
conn.close()
return ratings
def get_workflow_stats(self, workflow_id: str) -> Optional[WorkflowStats]:
"""Get comprehensive statistics for a workflow"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT workflow_id, total_ratings, average_rating, total_reviews,
total_views, total_downloads, last_updated
FROM workflow_stats
WHERE workflow_id = ?
""", (workflow_id,))
""",
(workflow_id,),
)
row = cursor.fetchone()
conn.close()
if row:
return WorkflowStats(
workflow_id=row[0],
@@ -200,236 +219,286 @@ class CommunityFeatures:
total_reviews=row[3],
total_views=row[4],
total_downloads=row[5],
last_updated=datetime.fromisoformat(row[6]) if row[6] else None
last_updated=datetime.fromisoformat(row[6]) if row[6] else None,
)
return None
def increment_view(self, workflow_id: str):
"""Increment view count for a workflow"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
INSERT OR IGNORE INTO workflow_stats (workflow_id, total_views)
VALUES (?, 1)
""", (workflow_id,))
cursor.execute("""
""",
(workflow_id,),
)
cursor.execute(
"""
UPDATE workflow_stats
SET total_views = total_views + 1, last_updated = CURRENT_TIMESTAMP
WHERE workflow_id = ?
""", (workflow_id,))
""",
(workflow_id,),
)
conn.commit()
conn.close()
def increment_download(self, workflow_id: str):
"""Increment download count for a workflow"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
INSERT OR IGNORE INTO workflow_stats (workflow_id, total_downloads)
VALUES (?, 1)
""", (workflow_id,))
cursor.execute("""
""",
(workflow_id,),
)
cursor.execute(
"""
UPDATE workflow_stats
SET total_downloads = total_downloads + 1, last_updated = CURRENT_TIMESTAMP
WHERE workflow_id = ?
""", (workflow_id,))
""",
(workflow_id,),
)
conn.commit()
conn.close()
def get_top_rated_workflows(self, limit: int = 10) -> List[Dict]:
"""Get top-rated workflows"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT w.filename, w.name, w.description, ws.average_rating, ws.total_ratings
FROM workflows w
JOIN workflow_stats ws ON w.filename = ws.workflow_id
WHERE ws.total_ratings >= 3
ORDER BY ws.average_rating DESC, ws.total_ratings DESC
LIMIT ?
""", (limit,))
""",
(limit,),
)
results = []
for row in cursor.fetchall():
results.append({
'filename': row[0],
'name': row[1],
'description': row[2],
'average_rating': row[3],
'total_ratings': row[4]
})
results.append(
{
"filename": row[0],
"name": row[1],
"description": row[2],
"average_rating": row[3],
"total_ratings": row[4],
}
)
conn.close()
return results
def get_most_popular_workflows(self, limit: int = 10) -> List[Dict]:
"""Get most popular workflows by views and downloads"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT w.filename, w.name, w.description, ws.total_views, ws.total_downloads
FROM workflows w
LEFT JOIN workflow_stats ws ON w.filename = ws.workflow_id
ORDER BY (ws.total_views + ws.total_downloads) DESC
LIMIT ?
""", (limit,))
""",
(limit,),
)
results = []
for row in cursor.fetchall():
results.append({
'filename': row[0],
'name': row[1],
'description': row[2],
'total_views': row[3] or 0,
'total_downloads': row[4] or 0
})
results.append(
{
"filename": row[0],
"name": row[1],
"description": row[2],
"total_views": row[3] or 0,
"total_downloads": row[4] or 0,
}
)
conn.close()
return results
def create_collection(self, user_id: str, collection_name: str, workflow_ids: List[str],
is_public: bool = False, description: str = None) -> bool:
def create_collection(
self,
user_id: str,
collection_name: str,
workflow_ids: List[str],
is_public: bool = False,
description: str = None,
) -> bool:
"""Create a workflow collection"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
try:
cursor.execute("""
cursor.execute(
"""
INSERT INTO workflow_collections
(user_id, collection_name, workflow_ids, is_public, description)
VALUES (?, ?, ?, ?, ?)
""", (user_id, collection_name, json.dumps(workflow_ids), is_public, description))
""",
(
user_id,
collection_name,
json.dumps(workflow_ids),
is_public,
description,
),
)
conn.commit()
return True
except Exception as e:
print(f"Error creating collection: {e}")
return False
finally:
conn.close()
def get_user_collections(self, user_id: str) -> List[Dict]:
"""Get collections for a user"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT id, collection_name, workflow_ids, is_public, description, created_at
FROM workflow_collections
WHERE user_id = ?
ORDER BY created_at DESC
""", (user_id,))
""",
(user_id,),
)
collections = []
for row in cursor.fetchall():
collections.append({
'id': row[0],
'name': row[1],
'workflow_ids': json.loads(row[2]) if row[2] else [],
'is_public': bool(row[3]),
'description': row[4],
'created_at': row[5]
})
collections.append(
{
"id": row[0],
"name": row[1],
"workflow_ids": json.loads(row[2]) if row[2] else [],
"is_public": bool(row[3]),
"description": row[4],
"created_at": row[5],
}
)
conn.close()
return collections
def _update_workflow_stats(self, workflow_id: str):
"""Update workflow statistics after rating changes"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Calculate new statistics
cursor.execute("""
cursor.execute(
"""
SELECT COUNT(*), AVG(rating), COUNT(CASE WHEN review IS NOT NULL THEN 1 END)
FROM workflow_ratings
WHERE workflow_id = ?
""", (workflow_id,))
""",
(workflow_id,),
)
total_ratings, avg_rating, total_reviews = cursor.fetchone()
# Update or insert statistics
cursor.execute("""
cursor.execute(
"""
INSERT OR REPLACE INTO workflow_stats
(workflow_id, total_ratings, average_rating, total_reviews, last_updated)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
""", (workflow_id, total_ratings or 0, avg_rating or 0.0, total_reviews or 0))
""",
(workflow_id, total_ratings or 0, avg_rating or 0.0, total_reviews or 0),
)
conn.commit()
conn.close()
# Example usage and API endpoints
def create_community_api_endpoints(app):
"""Add community feature endpoints to FastAPI app"""
community = CommunityFeatures()
@app.post("/api/workflows/{workflow_id}/rate")
async def rate_workflow(workflow_id: str, rating_data: dict):
"""Rate a workflow"""
try:
success = community.add_rating(
workflow_id=workflow_id,
user_id=rating_data.get('user_id', 'anonymous'),
rating=rating_data['rating'],
review=rating_data.get('review')
user_id=rating_data.get("user_id", "anonymous"),
rating=rating_data["rating"],
review=rating_data.get("review"),
)
return {"success": success}
except Exception as e:
return {"error": str(e)}
@app.get("/api/workflows/{workflow_id}/ratings")
async def get_workflow_ratings(workflow_id: str, limit: int = 10):
"""Get workflow ratings and reviews"""
ratings = community.get_workflow_ratings(workflow_id, limit)
return {"ratings": ratings}
@app.get("/api/workflows/{workflow_id}/stats")
async def get_workflow_stats(workflow_id: str):
"""Get workflow statistics"""
stats = community.get_workflow_stats(workflow_id)
return {"stats": stats}
@app.get("/api/workflows/top-rated")
async def get_top_rated_workflows(limit: int = 10):
"""Get top-rated workflows"""
workflows = community.get_top_rated_workflows(limit)
return {"workflows": workflows}
@app.get("/api/workflows/most-popular")
async def get_most_popular_workflows(limit: int = 10):
"""Get most popular workflows"""
workflows = community.get_most_popular_workflows(limit)
return {"workflows": workflows}
@app.post("/api/workflows/{workflow_id}/view")
async def track_workflow_view(workflow_id: str):
"""Track workflow view"""
community.increment_view(workflow_id)
return {"success": True}
@app.post("/api/workflows/{workflow_id}/download")
async def track_workflow_download(workflow_id: str):
"""Track workflow download"""
community.increment_download(workflow_id)
return {"success": True}
if __name__ == "__main__":
# Initialize community features
community = CommunityFeatures()
print("✅ Community features initialized successfully!")
# Example: Add a rating
# community.add_rating("example-workflow.json", "user123", 5, "Great workflow!")
# Example: Get top-rated workflows
top_workflows = community.get_top_rated_workflows(5)
print(f"📊 Top rated workflows: {len(top_workflows)}")
src/enhanced_api.py
+214 -189
View File
@@ -5,12 +5,10 @@ Advanced features, analytics, and performance optimizations
"""
import sqlite3
import json
import time
import hashlib
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from fastapi import FastAPI, HTTPException, Query, BackgroundTasks
from datetime import datetime
from typing import Dict, List, Optional
from fastapi import FastAPI, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from pydantic import BaseModel
@@ -19,8 +17,10 @@ import uvicorn
# Import community features
from community_features import CommunityFeatures, create_community_api_endpoints
class WorkflowSearchRequest(BaseModel):
"""Workflow search request model"""
query: str
categories: Optional[List[str]] = None
trigger_types: Optional[List[str]] = None
@@ -30,21 +30,26 @@ class WorkflowSearchRequest(BaseModel):
limit: int = 20
offset: int = 0
class WorkflowRecommendationRequest(BaseModel):
"""Workflow recommendation request model"""
user_interests: List[str]
viewed_workflows: Optional[List[str]] = None
preferred_complexity: Optional[str] = None
limit: int = 10
class AnalyticsRequest(BaseModel):
"""Analytics request model"""
date_range: str # "7d", "30d", "90d", "1y"
metrics: List[str] # ["views", "downloads", "ratings", "searches"]
class EnhancedAPI:
"""Enhanced API with advanced features"""
def __init__(self, db_path: str = "workflows.db"):
"""Initialize enhanced API"""
self.db_path = db_path
@@ -52,11 +57,11 @@ class EnhancedAPI:
self.app = FastAPI(
title="N8N Workflows Enhanced API",
description="Advanced API for n8n workflows repository with community features",
version="2.0.0"
version="2.0.0",
)
self._setup_middleware()
self._setup_routes()
def _setup_middleware(self):
"""Setup middleware for performance and security"""
# CORS middleware
@@ -67,13 +72,13 @@ class EnhancedAPI:
allow_methods=["*"],
allow_headers=["*"],
)
# Gzip compression
self.app.add_middleware(GZipMiddleware, minimum_size=1000)
def _setup_routes(self):
"""Setup API routes"""
# Core workflow endpoints
@self.app.get("/api/v2/workflows")
async def get_workflows_enhanced(
@@ -86,11 +91,11 @@ class EnhancedAPI:
sort_by: str = Query("name"),
sort_order: str = Query("asc"),
limit: int = Query(20, le=100),
offset: int = Query(0, ge=0)
offset: int = Query(0, ge=0),
):
"""Enhanced workflow search with multiple filters"""
start_time = time.time()
try:
workflows = self._search_workflows_enhanced(
search=search,
@@ -102,64 +107,64 @@ class EnhancedAPI:
sort_by=sort_by,
sort_order=sort_order,
limit=limit,
offset=offset
offset=offset,
)
response_time = (time.time() - start_time) * 1000
return {
"workflows": workflows,
"total": len(workflows),
"limit": limit,
"offset": offset,
"response_time_ms": round(response_time, 2),
"timestamp": datetime.now().isoformat()
"timestamp": datetime.now().isoformat(),
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@self.app.post("/api/v2/workflows/search")
async def advanced_workflow_search(request: WorkflowSearchRequest):
"""Advanced workflow search with complex queries"""
start_time = time.time()
try:
results = self._advanced_search(request)
response_time = (time.time() - start_time) * 1000
return {
"results": results,
"total": len(results),
"query": request.dict(),
"response_time_ms": round(response_time, 2),
"timestamp": datetime.now().isoformat()
"timestamp": datetime.now().isoformat(),
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@self.app.get("/api/v2/workflows/{workflow_id}")
async def get_workflow_enhanced(
workflow_id: str,
include_stats: bool = Query(True),
include_ratings: bool = Query(True),
include_related: bool = Query(True)
include_related: bool = Query(True),
):
"""Get detailed workflow information"""
try:
workflow_data = self._get_workflow_details(
workflow_id, include_stats, include_ratings, include_related
)
if not workflow_data:
raise HTTPException(status_code=404, detail="Workflow not found")
return workflow_data
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# Recommendation endpoints
@self.app.post("/api/v2/recommendations")
async def get_workflow_recommendations(request: WorkflowRecommendationRequest):
@@ -169,12 +174,12 @@ class EnhancedAPI:
return {
"recommendations": recommendations,
"user_profile": request.dict(),
"timestamp": datetime.now().isoformat()
"timestamp": datetime.now().isoformat(),
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@self.app.get("/api/v2/recommendations/trending")
async def get_trending_workflows(limit: int = Query(10, le=50)):
"""Get trending workflows based on recent activity"""
@@ -183,12 +188,12 @@ class EnhancedAPI:
return {
"trending": trending,
"limit": limit,
"timestamp": datetime.now().isoformat()
"timestamp": datetime.now().isoformat(),
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# Analytics endpoints
@self.app.get("/api/v2/analytics/overview")
async def get_analytics_overview():
@@ -196,20 +201,20 @@ class EnhancedAPI:
try:
overview = self._get_analytics_overview()
return overview
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@self.app.post("/api/v2/analytics/custom")
async def get_custom_analytics(request: AnalyticsRequest):
"""Get custom analytics data"""
try:
analytics = self._get_custom_analytics(request)
return analytics
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# Performance monitoring
@self.app.get("/api/v2/health")
async def health_check():
@@ -217,94 +222,98 @@ class EnhancedAPI:
try:
health_data = self._get_health_status()
return health_data
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# Add community endpoints
create_community_api_endpoints(self.app)
def _search_workflows_enhanced(self, **kwargs) -> List[Dict]:
"""Enhanced workflow search with multiple filters"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Build dynamic query
query_parts = ["SELECT w.*, ws.average_rating, ws.total_ratings"]
query_parts.append("FROM workflows w")
query_parts.append("LEFT JOIN workflow_stats ws ON w.filename = ws.workflow_id")
conditions = []
params = []
# Apply filters
if kwargs.get('search'):
conditions.append("(w.name LIKE ? OR w.description LIKE ? OR w.integrations LIKE ?)")
if kwargs.get("search"):
conditions.append(
"(w.name LIKE ? OR w.description LIKE ? OR w.integrations LIKE ?)"
)
search_term = f"%{kwargs['search']}%"
params.extend([search_term, search_term, search_term])
if kwargs.get('category'):
if kwargs.get("category"):
conditions.append("w.category = ?")
params.append(kwargs['category'])
if kwargs.get('trigger_type'):
params.append(kwargs["category"])
if kwargs.get("trigger_type"):
conditions.append("w.trigger_type = ?")
params.append(kwargs['trigger_type'])
if kwargs.get('complexity'):
params.append(kwargs["trigger_type"])
if kwargs.get("complexity"):
conditions.append("w.complexity = ?")
params.append(kwargs['complexity'])
if kwargs.get('integration'):
params.append(kwargs["complexity"])
if kwargs.get("integration"):
conditions.append("w.integrations LIKE ?")
params.append(f"%{kwargs['integration']}%")
if kwargs.get('min_rating'):
if kwargs.get("min_rating"):
conditions.append("ws.average_rating >= ?")
params.append(kwargs['min_rating'])
params.append(kwargs["min_rating"])
# Add conditions to query
if conditions:
query_parts.append("WHERE " + " AND ".join(conditions))
# Add sorting
sort_by = kwargs.get('sort_by', 'name')
sort_order = kwargs.get('sort_order', 'asc').upper()
sort_by = kwargs.get("sort_by", "name")
sort_order = kwargs.get("sort_order", "asc").upper()
query_parts.append(f"ORDER BY {sort_by} {sort_order}")
# Add pagination
query_parts.append("LIMIT ? OFFSET ?")
params.extend([kwargs.get('limit', 20), kwargs.get('offset', 0)])
params.extend([kwargs.get("limit", 20), kwargs.get("offset", 0)])
# Execute query
query = " ".join(query_parts)
cursor.execute(query, params)
workflows = []
for row in cursor.fetchall():
workflows.append({
'filename': row[0],
'name': row[1],
'workflow_id': row[2],
'active': bool(row[3]),
'description': row[4],
'trigger_type': row[5],
'complexity': row[6],
'node_count': row[7],
'integrations': row[8],
'tags': row[9],
'created_at': row[10],
'updated_at': row[11],
'file_hash': row[12],
'file_size': row[13],
'analyzed_at': row[14],
'average_rating': row[15],
'total_ratings': row[16]
})
workflows.append(
{
"filename": row[0],
"name": row[1],
"workflow_id": row[2],
"active": bool(row[3]),
"description": row[4],
"trigger_type": row[5],
"complexity": row[6],
"node_count": row[7],
"integrations": row[8],
"tags": row[9],
"created_at": row[10],
"updated_at": row[11],
"file_hash": row[12],
"file_size": row[13],
"analyzed_at": row[14],
"average_rating": row[15],
"total_ratings": row[16],
}
)
conn.close()
return workflows
def _advanced_search(self, request: WorkflowSearchRequest) -> List[Dict]:
"""Advanced search with complex queries"""
# Implementation for advanced search logic
@@ -313,214 +322,230 @@ class EnhancedAPI:
search=request.query,
category=request.categories[0] if request.categories else None,
trigger_type=request.trigger_types[0] if request.trigger_types else None,
complexity=request.complexity_levels[0] if request.complexity_levels else None,
complexity=request.complexity_levels[0]
if request.complexity_levels
else None,
limit=request.limit,
offset=request.offset
offset=request.offset,
)
def _get_workflow_details(self, workflow_id: str, include_stats: bool,
include_ratings: bool, include_related: bool) -> Dict:
def _get_workflow_details(
self,
workflow_id: str,
include_stats: bool,
include_ratings: bool,
include_related: bool,
) -> Dict:
"""Get detailed workflow information"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Get basic workflow data
cursor.execute("SELECT * FROM workflows WHERE filename = ?", (workflow_id,))
workflow_row = cursor.fetchone()
if not workflow_row:
conn.close()
return None
workflow_data = {
'filename': workflow_row[0],
'name': workflow_row[1],
'workflow_id': workflow_row[2],
'active': bool(workflow_row[3]),
'description': workflow_row[4],
'trigger_type': workflow_row[5],
'complexity': workflow_row[6],
'node_count': workflow_row[7],
'integrations': workflow_row[8],
'tags': workflow_row[9],
'created_at': workflow_row[10],
'updated_at': workflow_row[11],
'file_hash': workflow_row[12],
'file_size': workflow_row[13],
'analyzed_at': workflow_row[14]
"filename": workflow_row[0],
"name": workflow_row[1],
"workflow_id": workflow_row[2],
"active": bool(workflow_row[3]),
"description": workflow_row[4],
"trigger_type": workflow_row[5],
"complexity": workflow_row[6],
"node_count": workflow_row[7],
"integrations": workflow_row[8],
"tags": workflow_row[9],
"created_at": workflow_row[10],
"updated_at": workflow_row[11],
"file_hash": workflow_row[12],
"file_size": workflow_row[13],
"analyzed_at": workflow_row[14],
}
# Add statistics if requested
if include_stats:
stats = self.community.get_workflow_stats(workflow_id)
workflow_data['stats'] = stats.__dict__ if stats else None
workflow_data["stats"] = stats.__dict__ if stats else None
# Add ratings if requested
if include_ratings:
ratings = self.community.get_workflow_ratings(workflow_id, 5)
workflow_data['ratings'] = [rating.__dict__ for rating in ratings]
workflow_data["ratings"] = [rating.__dict__ for rating in ratings]
# Add related workflows if requested
if include_related:
related = self._get_related_workflows(workflow_id)
workflow_data['related_workflows'] = related
workflow_data["related_workflows"] = related
conn.close()
return workflow_data
def _get_recommendations(self, request: WorkflowRecommendationRequest) -> List[Dict]:
def _get_recommendations(
self, request: WorkflowRecommendationRequest
) -> List[Dict]:
"""Get personalized workflow recommendations"""
# Implementation for recommendation algorithm
# This would use collaborative filtering, content-based filtering, etc.
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Simple recommendation based on user interests
recommendations = []
for interest in request.user_interests:
cursor.execute("""
cursor.execute(
"""
SELECT * FROM workflows
WHERE integrations LIKE ? OR name LIKE ? OR description LIKE ?
LIMIT 5
""", (f"%{interest}%", f"%{interest}%", f"%{interest}%"))
""",
(f"%{interest}%", f"%{interest}%", f"%{interest}%"),
)
for row in cursor.fetchall():
recommendations.append({
'filename': row[0],
'name': row[1],
'description': row[4],
'reason': f"Matches your interest in {interest}"
})
recommendations.append(
{
"filename": row[0],
"name": row[1],
"description": row[4],
"reason": f"Matches your interest in {interest}",
}
)
conn.close()
return recommendations[:request.limit]
return recommendations[: request.limit]
def _get_trending_workflows(self, limit: int) -> List[Dict]:
"""Get trending workflows based on recent activity"""
return self.community.get_most_popular_workflows(limit)
def _get_analytics_overview(self) -> Dict:
"""Get analytics overview"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Total workflows
cursor.execute("SELECT COUNT(*) FROM workflows")
total_workflows = cursor.fetchone()[0]
# Active workflows
cursor.execute("SELECT COUNT(*) FROM workflows WHERE active = 1")
active_workflows = cursor.fetchone()[0]
# Categories
cursor.execute("SELECT category, COUNT(*) FROM workflows GROUP BY category")
categories = dict(cursor.fetchall())
# Integrations
cursor.execute("SELECT COUNT(DISTINCT integrations) FROM workflows")
unique_integrations = cursor.fetchone()[0]
conn.close()
return {
'total_workflows': total_workflows,
'active_workflows': active_workflows,
'categories': categories,
'unique_integrations': unique_integrations,
'timestamp': datetime.now().isoformat()
"total_workflows": total_workflows,
"active_workflows": active_workflows,
"categories": categories,
"unique_integrations": unique_integrations,
"timestamp": datetime.now().isoformat(),
}
def _get_custom_analytics(self, request: AnalyticsRequest) -> Dict:
"""Get custom analytics data"""
# Implementation for custom analytics
return {
'date_range': request.date_range,
'metrics': request.metrics,
'data': {}, # Placeholder for actual analytics data
'timestamp': datetime.now().isoformat()
"date_range": request.date_range,
"metrics": request.metrics,
"data": {}, # Placeholder for actual analytics data
"timestamp": datetime.now().isoformat(),
}
def _get_health_status(self) -> Dict:
"""Get health status and performance metrics"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Database health
cursor.execute("SELECT COUNT(*) FROM workflows")
total_workflows = cursor.fetchone()[0]
# Performance test
start_time = time.time()
cursor.execute("SELECT COUNT(*) FROM workflows WHERE active = 1")
active_count = cursor.fetchone()[0]
query_time = (time.time() - start_time) * 1000
conn.close()
return {
'status': 'healthy',
'database': {
'total_workflows': total_workflows,
'active_workflows': active_count,
'connection_status': 'connected'
"status": "healthy",
"database": {
"total_workflows": total_workflows,
"active_workflows": active_count,
"connection_status": "connected",
},
'performance': {
'query_time_ms': round(query_time, 2),
'response_time_target': '<100ms',
'status': 'good' if query_time < 100 else 'slow'
"performance": {
"query_time_ms": round(query_time, 2),
"response_time_target": "<100ms",
"status": "good" if query_time < 100 else "slow",
},
'timestamp': datetime.now().isoformat()
"timestamp": datetime.now().isoformat(),
}
def _get_related_workflows(self, workflow_id: str, limit: int = 5) -> List[Dict]:
"""Get related workflows based on similar integrations or categories"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Get current workflow details
cursor.execute("SELECT integrations, category FROM workflows WHERE filename = ?", (workflow_id,))
cursor.execute(
"SELECT integrations, category FROM workflows WHERE filename = ?",
(workflow_id,),
)
current_workflow = cursor.fetchone()
if not current_workflow:
conn.close()
return []
current_integrations = current_workflow[0] or ""
current_category = current_workflow[1] or ""
# Find related workflows
cursor.execute("""
cursor.execute(
"""
SELECT filename, name, description FROM workflows
WHERE filename != ?
AND (integrations LIKE ? OR category = ?)
LIMIT ?
""", (workflow_id, f"%{current_integrations[:50]}%", current_category, limit))
""",
(workflow_id, f"%{current_integrations[:50]}%", current_category, limit),
)
related = []
for row in cursor.fetchall():
related.append({
'filename': row[0],
'name': row[1],
'description': row[2]
})
related.append({"filename": row[0], "name": row[1], "description": row[2]})
conn.close()
return related
def run(self, host: str = "127.0.0.1", port: int = 8000, debug: bool = False):
"""Run the enhanced API server"""
uvicorn.run(
self.app,
host=host,
port=port,
log_level="debug" if debug else "info"
self.app, host=host, port=port, log_level="debug" if debug else "info"
)
if __name__ == "__main__":
# Initialize and run enhanced API
api = EnhancedAPI()
print("🚀 Starting Enhanced N8N Workflows API...")
print("📊 Features: Advanced search, recommendations, analytics, community features")
print(
"📊 Features: Advanced search, recommendations, analytics, community features"
)
print("🌐 API Documentation: http://127.0.0.1:8000/docs")
api.run(debug=True)
src/integration_hub.py
+114 -88
View File
@@ -4,15 +4,13 @@ Integration Hub for N8N Workflows
Connect with external platforms and services.
"""
from fastapi import FastAPI, HTTPException, BackgroundTasks
from fastapi import FastAPI, HTTPException
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, Field
from typing import List, Dict, Any, Optional
from typing import List, Dict, Any
import httpx
import json
import asyncio
from datetime import datetime
import os
class IntegrationConfig(BaseModel):
name: str
@@ -20,48 +18,50 @@ class IntegrationConfig(BaseModel):
base_url: str
enabled: bool = True
class WebhookPayload(BaseModel):
event: str
data: Dict[str, Any]
timestamp: str = Field(default_factory=lambda: datetime.now().isoformat())
class IntegrationHub:
def __init__(self):
self.integrations = {}
self.webhook_endpoints = {}
def register_integration(self, config: IntegrationConfig):
"""Register a new integration."""
self.integrations[config.name] = config
async def sync_with_github(self, repo: str, token: str) -> Dict[str, Any]:
"""Sync workflows with GitHub repository."""
try:
async with httpx.AsyncClient() as client:
headers = {"Authorization": f"token {token}"}
# Get repository contents
response = await client.get(
f"https://api.github.com/repos/{repo}/contents/workflows",
headers=headers
headers=headers,
)
if response.status_code == 200:
files = response.json()
workflow_files = [f for f in files if f['name'].endswith('.json')]
workflow_files = [f for f in files if f["name"].endswith(".json")]
return {
"status": "success",
"repository": repo,
"workflow_files": len(workflow_files),
"files": [f['name'] for f in workflow_files]
"files": [f["name"] for f in workflow_files],
}
else:
return {"status": "error", "message": "Failed to access repository"}
except Exception as e:
return {"status": "error", "message": str(e)}
async def sync_with_slack(self, webhook_url: str, message: str) -> Dict[str, Any]:
"""Send notification to Slack."""
try:
@@ -69,149 +69,169 @@ class IntegrationHub:
payload = {
"text": message,
"username": "N8N Workflows Bot",
"icon_emoji": ":robot_face:"
"icon_emoji": ":robot_face:",
}
response = await client.post(webhook_url, json=payload)
if response.status_code == 200:
return {"status": "success", "message": "Notification sent to Slack"}
return {
"status": "success",
"message": "Notification sent to Slack",
}
else:
return {"status": "error", "message": "Failed to send to Slack"}
except Exception as e:
return {"status": "error", "message": str(e)}
async def sync_with_discord(self, webhook_url: str, message: str) -> Dict[str, Any]:
"""Send notification to Discord."""
try:
async with httpx.AsyncClient() as client:
payload = {
"content": message,
"username": "N8N Workflows Bot"
}
payload = {"content": message, "username": "N8N Workflows Bot"}
response = await client.post(webhook_url, json=payload)
if response.status_code == 204:
return {"status": "success", "message": "Notification sent to Discord"}
return {
"status": "success",
"message": "Notification sent to Discord",
}
else:
return {"status": "error", "message": "Failed to send to Discord"}
except Exception as e:
return {"status": "error", "message": str(e)}
async def export_to_airtable(self, base_id: str, table_name: str, api_key: str, workflows: List[Dict]) -> Dict[str, Any]:
async def export_to_airtable(
self, base_id: str, table_name: str, api_key: str, workflows: List[Dict]
) -> Dict[str, Any]:
"""Export workflows to Airtable."""
try:
async with httpx.AsyncClient() as client:
headers = {"Authorization": f"Bearer {api_key}"}
records = []
for workflow in workflows:
record = {
"fields": {
"Name": workflow.get('name', ''),
"Description": workflow.get('description', ''),
"Trigger Type": workflow.get('trigger_type', ''),
"Complexity": workflow.get('complexity', ''),
"Node Count": workflow.get('node_count', 0),
"Active": workflow.get('active', False),
"Integrations": ", ".join(workflow.get('integrations', [])),
"Last Updated": datetime.now().isoformat()
"Name": workflow.get("name", ""),
"Description": workflow.get("description", ""),
"Trigger Type": workflow.get("trigger_type", ""),
"Complexity": workflow.get("complexity", ""),
"Node Count": workflow.get("node_count", 0),
"Active": workflow.get("active", False),
"Integrations": ", ".join(workflow.get("integrations", [])),
"Last Updated": datetime.now().isoformat(),
}
}
records.append(record)
# Create records in batches
batch_size = 10
created_records = 0
for i in range(0, len(records), batch_size):
batch = records[i:i + batch_size]
batch = records[i : i + batch_size]
response = await client.post(
f"https://api.airtable.com/v0/{base_id}/{table_name}",
headers=headers,
json={"records": batch}
json={"records": batch},
)
if response.status_code == 200:
created_records += len(batch)
else:
return {"status": "error", "message": f"Failed to create records: {response.text}"}
return {
"status": "error",
"message": f"Failed to create records: {response.text}",
}
return {
"status": "success",
"message": f"Exported {created_records} workflows to Airtable"
"message": f"Exported {created_records} workflows to Airtable",
}
except Exception as e:
return {"status": "error", "message": str(e)}
async def sync_with_notion(self, database_id: str, token: str, workflows: List[Dict]) -> Dict[str, Any]:
async def sync_with_notion(
self, database_id: str, token: str, workflows: List[Dict]
) -> Dict[str, Any]:
"""Sync workflows with Notion database."""
try:
async with httpx.AsyncClient() as client:
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
"Notion-Version": "2022-06-28"
"Notion-Version": "2022-06-28",
}
created_pages = 0
for workflow in workflows:
page_data = {
"parent": {"database_id": database_id},
"properties": {
"Name": {
"title": [{"text": {"content": workflow.get('name', '')}}]
"title": [
{"text": {"content": workflow.get("name", "")}}
]
},
"Description": {
"rich_text": [{"text": {"content": workflow.get('description', '')}}]
"rich_text": [
{
"text": {
"content": workflow.get("description", "")
}
}
]
},
"Trigger Type": {
"select": {"name": workflow.get('trigger_type', '')}
"select": {"name": workflow.get("trigger_type", "")}
},
"Complexity": {
"select": {"name": workflow.get('complexity', '')}
},
"Node Count": {
"number": workflow.get('node_count', 0)
},
"Active": {
"checkbox": workflow.get('active', False)
"select": {"name": workflow.get("complexity", "")}
},
"Node Count": {"number": workflow.get("node_count", 0)},
"Active": {"checkbox": workflow.get("active", False)},
"Integrations": {
"multi_select": [{"name": integration} for integration in workflow.get('integrations', [])]
}
}
"multi_select": [
{"name": integration}
for integration in workflow.get("integrations", [])
]
},
},
}
response = await client.post(
"https://api.notion.com/v1/pages",
headers=headers,
json=page_data
json=page_data,
)
if response.status_code == 200:
created_pages += 1
else:
return {"status": "error", "message": f"Failed to create page: {response.text}"}
return {
"status": "error",
"message": f"Failed to create page: {response.text}",
}
return {
"status": "success",
"message": f"Synced {created_pages} workflows to Notion"
"message": f"Synced {created_pages} workflows to Notion",
}
except Exception as e:
return {"status": "error", "message": str(e)}
def register_webhook(self, endpoint: str, handler):
"""Register a webhook endpoint."""
self.webhook_endpoints[endpoint] = handler
async def handle_webhook(self, endpoint: str, payload: WebhookPayload):
"""Handle incoming webhook."""
if endpoint in self.webhook_endpoints:
@@ -219,12 +239,14 @@ class IntegrationHub:
else:
return {"status": "error", "message": "Webhook endpoint not found"}
# Initialize integration hub
integration_hub = IntegrationHub()
# FastAPI app for Integration Hub
integration_app = FastAPI(title="N8N Integration Hub", version="1.0.0")
@integration_app.post("/integrations/github/sync")
async def sync_github(repo: str, token: str):
"""Sync workflows with GitHub repository."""
@@ -234,6 +256,7 @@ async def sync_github(repo: str, token: str):
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@integration_app.post("/integrations/slack/notify")
async def notify_slack(webhook_url: str, message: str):
"""Send notification to Slack."""
@@ -243,6 +266,7 @@ async def notify_slack(webhook_url: str, message: str):
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@integration_app.post("/integrations/discord/notify")
async def notify_discord(webhook_url: str, message: str):
"""Send notification to Discord."""
@@ -252,26 +276,23 @@ async def notify_discord(webhook_url: str, message: str):
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@integration_app.post("/integrations/airtable/export")
async def export_airtable(
base_id: str,
table_name: str,
api_key: str,
workflows: List[Dict]
base_id: str, table_name: str, api_key: str, workflows: List[Dict]
):
"""Export workflows to Airtable."""
try:
result = await integration_hub.export_to_airtable(base_id, table_name, api_key, workflows)
result = await integration_hub.export_to_airtable(
base_id, table_name, api_key, workflows
)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@integration_app.post("/integrations/notion/sync")
async def sync_notion(
database_id: str,
token: str,
workflows: List[Dict]
):
async def sync_notion(database_id: str, token: str, workflows: List[Dict]):
"""Sync workflows with Notion database."""
try:
result = await integration_hub.sync_with_notion(database_id, token, workflows)
@@ -279,6 +300,7 @@ async def sync_notion(
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@integration_app.post("/webhooks/{endpoint}")
async def handle_webhook_endpoint(endpoint: str, payload: WebhookPayload):
"""Handle incoming webhook."""
@@ -288,15 +310,17 @@ async def handle_webhook_endpoint(endpoint: str, payload: WebhookPayload):
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@integration_app.get("/integrations/status")
async def get_integration_status():
"""Get status of all integrations."""
return {
"integrations": list(integration_hub.integrations.keys()),
"webhook_endpoints": list(integration_hub.webhook_endpoints.keys()),
"status": "operational"
"status": "operational",
}
@integration_app.get("/integrations/dashboard")
async def get_integration_dashboard():
"""Get integration dashboard HTML."""
@@ -623,6 +647,8 @@ async def get_integration_dashboard():
"""
return HTMLResponse(content=html_content)
if __name__ == "__main__":
import uvicorn
uvicorn.run(integration_app, host="127.0.0.1", port=8003)
src/performance_monitor.py
+95 -64
View File
@@ -7,17 +7,17 @@ Real-time metrics, monitoring, and alerting.
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from typing import List, Dict, Any, Optional
from typing import List, Dict, Any
import asyncio
import time
import psutil
import sqlite3
from datetime import datetime, timedelta
import json
import threading
import queue
import os
class PerformanceMetrics(BaseModel):
timestamp: str
cpu_usage: float
@@ -30,6 +30,7 @@ class PerformanceMetrics(BaseModel):
workflow_executions: int
error_rate: float
class Alert(BaseModel):
id: str
type: str
@@ -38,6 +39,7 @@ class Alert(BaseModel):
timestamp: str
resolved: bool = False
class PerformanceMonitor:
def __init__(self, db_path: str = "workflows.db"):
self.db_path = db_path
@@ -46,79 +48,81 @@ class PerformanceMonitor:
self.websocket_connections = []
self.monitoring_active = False
self.metrics_queue = queue.Queue()
def start_monitoring(self):
"""Start performance monitoring in background thread."""
if not self.monitoring_active:
self.monitoring_active = True
monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
monitor_thread.start()
def _monitor_loop(self):
"""Main monitoring loop."""
while self.monitoring_active:
try:
metrics = self._collect_metrics()
self.metrics_history.append(metrics)
# Keep only last 1000 metrics
if len(self.metrics_history) > 1000:
self.metrics_history = self.metrics_history[-1000:]
# Check for alerts
self._check_alerts(metrics)
# Send to websocket connections
self._broadcast_metrics(metrics)
time.sleep(5) # Collect metrics every 5 seconds
except Exception as e:
print(f"Monitoring error: {e}")
time.sleep(10)
def _collect_metrics(self) -> PerformanceMetrics:
"""Collect current system metrics."""
# CPU and Memory
cpu_usage = psutil.cpu_percent(interval=1)
memory = psutil.virtual_memory()
memory_usage = memory.percent
# Disk usage
disk = psutil.disk_usage('/')
disk = psutil.disk_usage("/")
disk_usage = (disk.used / disk.total) * 100
# Network I/O
network = psutil.net_io_counters()
network_io = {
"bytes_sent": network.bytes_sent,
"bytes_recv": network.bytes_recv,
"packets_sent": network.packets_sent,
"packets_recv": network.packets_recv
"packets_recv": network.packets_recv,
}
# API response times (simulated)
api_response_times = {
"/api/stats": self._measure_api_time("/api/stats"),
"/api/workflows": self._measure_api_time("/api/workflows"),
"/api/search": self._measure_api_time("/api/workflows?q=test")
"/api/search": self._measure_api_time("/api/workflows?q=test"),
}
# Active connections
active_connections = len(psutil.net_connections())
# Database size
try:
db_size = os.path.getsize(self.db_path) if os.path.exists(self.db_path) else 0
db_size = (
os.path.getsize(self.db_path) if os.path.exists(self.db_path) else 0
)
except:
db_size = 0
# Workflow executions (simulated)
workflow_executions = self._get_workflow_executions()
# Error rate (simulated)
error_rate = self._calculate_error_rate()
return PerformanceMetrics(
timestamp=datetime.now().isoformat(),
cpu_usage=cpu_usage,
@@ -129,50 +133,65 @@ class PerformanceMonitor:
active_connections=active_connections,
database_size=db_size,
workflow_executions=workflow_executions,
error_rate=error_rate
error_rate=error_rate,
)
def _measure_api_time(self, endpoint: str) -> float:
"""Measure API response time (simulated)."""
# In a real implementation, this would make actual HTTP requests
import random
return round(random.uniform(10, 100), 2)
def _get_workflow_executions(self) -> int:
"""Get number of workflow executions (simulated)."""
# In a real implementation, this would query execution logs
import random
return random.randint(0, 50)
def _calculate_error_rate(self) -> float:
"""Calculate error rate (simulated)."""
# In a real implementation, this would analyze error logs
import random
return round(random.uniform(0, 5), 2)
def _check_alerts(self, metrics: PerformanceMetrics):
"""Check metrics against alert thresholds."""
# CPU alert
if metrics.cpu_usage > 80:
self._create_alert("high_cpu", "warning", f"High CPU usage: {metrics.cpu_usage}%")
self._create_alert(
"high_cpu", "warning", f"High CPU usage: {metrics.cpu_usage}%"
)
# Memory alert
if metrics.memory_usage > 85:
self._create_alert("high_memory", "warning", f"High memory usage: {metrics.memory_usage}%")
self._create_alert(
"high_memory", "warning", f"High memory usage: {metrics.memory_usage}%"
)
# Disk alert
if metrics.disk_usage > 90:
self._create_alert("high_disk", "critical", f"High disk usage: {metrics.disk_usage}%")
self._create_alert(
"high_disk", "critical", f"High disk usage: {metrics.disk_usage}%"
)
# API response time alert
for endpoint, response_time in metrics.api_response_times.items():
if response_time > 1000: # 1 second
self._create_alert("slow_api", "warning", f"Slow API response: {endpoint} ({response_time}ms)")
self._create_alert(
"slow_api",
"warning",
f"Slow API response: {endpoint} ({response_time}ms)",
)
# Error rate alert
if metrics.error_rate > 10:
self._create_alert("high_error_rate", "critical", f"High error rate: {metrics.error_rate}%")
self._create_alert(
"high_error_rate", "critical", f"High error rate: {metrics.error_rate}%"
)
def _create_alert(self, alert_type: str, severity: str, message: str):
"""Create a new alert."""
alert = Alert(
@@ -180,32 +199,28 @@ class PerformanceMonitor:
type=alert_type,
severity=severity,
message=message,
timestamp=datetime.now().isoformat()
timestamp=datetime.now().isoformat(),
)
# Check if similar alert already exists
existing_alert = next((a for a in self.alerts if a.type == alert_type and not a.resolved), None)
existing_alert = next(
(a for a in self.alerts if a.type == alert_type and not a.resolved), None
)
if not existing_alert:
self.alerts.append(alert)
self._broadcast_alert(alert)
def _broadcast_metrics(self, metrics: PerformanceMetrics):
"""Broadcast metrics to all websocket connections."""
if self.websocket_connections:
message = {
"type": "metrics",
"data": metrics.dict()
}
message = {"type": "metrics", "data": metrics.dict()}
self._broadcast_to_websockets(message)
def _broadcast_alert(self, alert: Alert):
"""Broadcast alert to all websocket connections."""
message = {
"type": "alert",
"data": alert.dict()
}
message = {"type": "alert", "data": alert.dict()}
self._broadcast_to_websockets(message)
def _broadcast_to_websockets(self, message: dict):
"""Broadcast message to all websocket connections."""
disconnected = []
@@ -214,40 +229,47 @@ class PerformanceMonitor:
asyncio.create_task(websocket.send_text(json.dumps(message)))
except:
disconnected.append(websocket)
# Remove disconnected connections
for ws in disconnected:
self.websocket_connections.remove(ws)
def get_metrics_summary(self) -> Dict[str, Any]:
"""Get performance metrics summary."""
if not self.metrics_history:
return {"message": "No metrics available"}
latest = self.metrics_history[-1]
avg_cpu = sum(m.cpu_usage for m in self.metrics_history[-10:]) / min(10, len(self.metrics_history))
avg_memory = sum(m.memory_usage for m in self.metrics_history[-10:]) / min(10, len(self.metrics_history))
avg_cpu = sum(m.cpu_usage for m in self.metrics_history[-10:]) / min(
10, len(self.metrics_history)
)
avg_memory = sum(m.memory_usage for m in self.metrics_history[-10:]) / min(
10, len(self.metrics_history)
)
return {
"current": latest.dict(),
"averages": {
"cpu_usage": round(avg_cpu, 2),
"memory_usage": round(avg_memory, 2)
"memory_usage": round(avg_memory, 2),
},
"alerts": [alert.dict() for alert in self.alerts[-10:]],
"status": "healthy" if latest.cpu_usage < 80 and latest.memory_usage < 85 else "warning"
"status": "healthy"
if latest.cpu_usage < 80 and latest.memory_usage < 85
else "warning",
}
def get_historical_metrics(self, hours: int = 24) -> List[Dict]:
"""Get historical metrics for specified hours."""
cutoff_time = datetime.now() - timedelta(hours=hours)
cutoff_timestamp = cutoff_time.isoformat()
return [
metrics.dict() for metrics in self.metrics_history
metrics.dict()
for metrics in self.metrics_history
if metrics.timestamp >= cutoff_timestamp
]
def resolve_alert(self, alert_id: str) -> bool:
"""Resolve an alert."""
for alert in self.alerts:
@@ -256,6 +278,7 @@ class PerformanceMonitor:
return True
return False
# Initialize performance monitor
performance_monitor = PerformanceMonitor()
performance_monitor.start_monitoring()
@@ -263,21 +286,25 @@ performance_monitor.start_monitoring()
# FastAPI app for Performance Monitoring
monitor_app = FastAPI(title="N8N Performance Monitor", version="1.0.0")
@monitor_app.get("/monitor/metrics")
async def get_current_metrics():
"""Get current performance metrics."""
return performance_monitor.get_metrics_summary()
@monitor_app.get("/monitor/history")
async def get_historical_metrics(hours: int = 24):
"""Get historical performance metrics."""
return performance_monitor.get_historical_metrics(hours)
@monitor_app.get("/monitor/alerts")
async def get_alerts():
"""Get current alerts."""
return [alert.dict() for alert in performance_monitor.alerts if not alert.resolved]
@monitor_app.post("/monitor/alerts/{alert_id}/resolve")
async def resolve_alert(alert_id: str):
"""Resolve an alert."""
@@ -287,12 +314,13 @@ async def resolve_alert(alert_id: str):
else:
return {"message": "Alert not found"}
@monitor_app.websocket("/monitor/ws")
async def websocket_endpoint(websocket: WebSocket):
"""WebSocket endpoint for real-time metrics."""
await websocket.accept()
performance_monitor.websocket_connections.append(websocket)
try:
while True:
# Keep connection alive
@@ -300,6 +328,7 @@ async def websocket_endpoint(websocket: WebSocket):
except WebSocketDisconnect:
performance_monitor.websocket_connections.remove(websocket)
@monitor_app.get("/monitor/dashboard")
async def get_monitoring_dashboard():
"""Get performance monitoring dashboard HTML."""
@@ -722,6 +751,8 @@ async def get_monitoring_dashboard():
"""
return HTMLResponse(content=html_content)
if __name__ == "__main__":
import uvicorn
uvicorn.run(monitor_app, host="127.0.0.1", port=8005)
src/user_management.py
+146 -99
View File
@@ -8,13 +8,12 @@ from fastapi import FastAPI, HTTPException, Depends, status
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, EmailStr
from typing import List, Dict, Any, Optional
from typing import List, Optional
import sqlite3
import hashlib
import secrets
import jwt
from datetime import datetime, timedelta
import json
import os
# Configuration - Use environment variables for security
@@ -25,6 +24,7 @@ ACCESS_TOKEN_EXPIRE_MINUTES = 30
# Security
security = HTTPBearer()
class User(BaseModel):
id: Optional[int] = None
username: str
@@ -34,6 +34,7 @@ class User(BaseModel):
active: bool = True
created_at: Optional[str] = None
class UserCreate(BaseModel):
username: str
email: EmailStr
@@ -41,31 +42,35 @@ class UserCreate(BaseModel):
password: str
role: str = "user"
class UserLogin(BaseModel):
username: str
password: str
class UserUpdate(BaseModel):
full_name: Optional[str] = None
email: Optional[EmailStr] = None
role: Optional[str] = None
active: Optional[bool] = None
class Token(BaseModel):
access_token: str
token_type: str
expires_in: int
class UserManager:
def __init__(self, db_path: str = "users.db"):
self.db_path = db_path
self.init_database()
def init_database(self):
"""Initialize user database."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -79,7 +84,7 @@ class UserManager:
last_login TIMESTAMP
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS user_sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -90,7 +95,7 @@ class UserManager:
FOREIGN KEY (user_id) REFERENCES users (id)
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS user_permissions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -101,72 +106,93 @@ class UserManager:
FOREIGN KEY (user_id) REFERENCES users (id)
)
""")
conn.commit()
conn.close()
# Create default admin user if none exists
self.create_default_admin()
def create_default_admin(self):
"""Create default admin user if none exists."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM users WHERE role = 'admin'")
admin_count = cursor.fetchone()[0]
if admin_count == 0:
# Use environment variable or generate secure random password
admin_password = os.environ.get("ADMIN_PASSWORD", secrets.token_urlsafe(16))
password_hash = self.hash_password(admin_password)
cursor.execute("""
cursor.execute(
"""
INSERT INTO users (username, email, full_name, password_hash, role)
VALUES (?, ?, ?, ?, ?)
""", ("admin", "admin@n8n-workflows.com", "System Administrator", password_hash, "admin"))
""",
(
"admin",
"admin@n8n-workflows.com",
"System Administrator",
password_hash,
"admin",
),
)
conn.commit()
# Only print password if it was auto-generated (not from env)
if "ADMIN_PASSWORD" not in os.environ:
print(f"Default admin user created: admin/{admin_password}")
print("WARNING: Please change this password immediately after first login!")
print(
"WARNING: Please change this password immediately after first login!"
)
else:
print("Default admin user created with environment-configured password")
conn.close()
def hash_password(self, password: str) -> str:
"""Hash password using SHA-256."""
return hashlib.sha256(password.encode()).hexdigest()
def verify_password(self, password: str, hashed: str) -> bool:
"""Verify password against hash."""
return self.hash_password(password) == hashed
def create_user(self, user_data: UserCreate) -> User:
"""Create a new user."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
try:
# Check if username or email already exists
cursor.execute("SELECT COUNT(*) FROM users WHERE username = ? OR email = ?",
(user_data.username, user_data.email))
cursor.execute(
"SELECT COUNT(*) FROM users WHERE username = ? OR email = ?",
(user_data.username, user_data.email),
)
if cursor.fetchone()[0] > 0:
raise ValueError("Username or email already exists")
password_hash = self.hash_password(user_data.password)
cursor.execute("""
cursor.execute(
"""
INSERT INTO users (username, email, full_name, password_hash, role)
VALUES (?, ?, ?, ?, ?)
""", (user_data.username, user_data.email, user_data.full_name,
password_hash, user_data.role))
""",
(
user_data.username,
user_data.email,
user_data.full_name,
password_hash,
user_data.role,
),
)
user_id = cursor.lastrowid
conn.commit()
return User(
id=user_id,
username=user_data.username,
@@ -174,28 +200,31 @@ class UserManager:
full_name=user_data.full_name,
role=user_data.role,
active=True,
created_at=datetime.now().isoformat()
created_at=datetime.now().isoformat(),
)
except Exception as e:
conn.rollback()
raise e
finally:
conn.close()
def authenticate_user(self, username: str, password: str) -> Optional[User]:
"""Authenticate user and return user data."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT id, username, email, full_name, password_hash, role, active
FROM users WHERE username = ? AND active = 1
""", (username,))
""",
(username,),
)
row = cursor.fetchone()
conn.close()
if row and self.verify_password(password, row[4]):
return User(
id=row[0],
@@ -203,11 +232,11 @@ class UserManager:
email=row[2],
full_name=row[3],
role=row[5],
active=bool(row[6])
active=bool(row[6]),
)
return None
def create_access_token(self, user: User) -> str:
"""Create JWT access token."""
expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
@@ -215,10 +244,10 @@ class UserManager:
"sub": str(user.id),
"username": user.username,
"role": user.role,
"exp": expire
"exp": expire,
}
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
def verify_token(self, token: str) -> Optional[User]:
"""Verify JWT token and return user data."""
try:
@@ -226,31 +255,30 @@ class UserManager:
user_id = payload.get("sub")
username = payload.get("username")
role = payload.get("role")
if user_id is None or username is None:
return None
return User(
id=int(user_id),
username=username,
role=role
)
return User(id=int(user_id), username=username, role=role)
except jwt.PyJWTError:
return None
def get_user_by_id(self, user_id: int) -> Optional[User]:
"""Get user by ID."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
SELECT id, username, email, full_name, role, active, created_at
FROM users WHERE id = ?
""", (user_id,))
""",
(user_id,),
)
row = cursor.fetchone()
conn.close()
if row:
return User(
id=row[0],
@@ -259,84 +287,86 @@ class UserManager:
full_name=row[3],
role=row[4],
active=bool(row[5]),
created_at=row[6]
created_at=row[6],
)
return None
def get_all_users(self) -> List[User]:
"""Get all users."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
SELECT id, username, email, full_name, role, active, created_at
FROM users ORDER BY created_at DESC
""")
users = []
for row in cursor.fetchall():
users.append(User(
id=row[0],
username=row[1],
email=row[2],
full_name=row[3],
role=row[4],
active=bool(row[5]),
created_at=row[6]
))
users.append(
User(
id=row[0],
username=row[1],
email=row[2],
full_name=row[3],
role=row[4],
active=bool(row[5]),
created_at=row[6],
)
)
conn.close()
return users
def update_user(self, user_id: int, update_data: UserUpdate) -> Optional[User]:
"""Update user data."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
try:
# Build update query dynamically
updates = []
params = []
if update_data.full_name is not None:
updates.append("full_name = ?")
params.append(update_data.full_name)
if update_data.email is not None:
updates.append("email = ?")
params.append(update_data.email)
if update_data.role is not None:
updates.append("role = ?")
params.append(update_data.role)
if update_data.active is not None:
updates.append("active = ?")
params.append(update_data.active)
if not updates:
return self.get_user_by_id(user_id)
params.append(user_id)
query = f"UPDATE users SET {', '.join(updates)} WHERE id = ?"
cursor.execute(query, params)
conn.commit()
return self.get_user_by_id(user_id)
except Exception as e:
conn.rollback()
raise e
finally:
conn.close()
def delete_user(self, user_id: int) -> bool:
"""Delete user (soft delete by setting active=False)."""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
try:
cursor.execute("UPDATE users SET active = 0 WHERE id = ?", (user_id,))
conn.commit()
@@ -347,35 +377,40 @@ class UserManager:
finally:
conn.close()
# Initialize user manager
user_manager = UserManager()
# FastAPI app for User Management
user_app = FastAPI(title="N8N User Management", version="1.0.0")
def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security)) -> User:
def get_current_user(
credentials: HTTPAuthorizationCredentials = Depends(security),
) -> User:
"""Get current authenticated user."""
token = credentials.credentials
user = user_manager.verify_token(token)
if user is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authentication credentials",
headers={"WWW-Authenticate": "Bearer"},
)
return user
def require_admin(current_user: User = Depends(get_current_user)) -> User:
"""Require admin role."""
if current_user.role != "admin":
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Admin access required"
status_code=status.HTTP_403_FORBIDDEN, detail="Admin access required"
)
return current_user
@user_app.post("/auth/register", response_model=User)
async def register_user(user_data: UserCreate):
"""Register a new user."""
@@ -387,76 +422,86 @@ async def register_user(user_data: UserCreate):
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@user_app.post("/auth/login", response_model=Token)
async def login_user(login_data: UserLogin):
"""Login user and return access token."""
user = user_manager.authenticate_user(login_data.username, login_data.password)
if user is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid username or password",
headers={"WWW-Authenticate": "Bearer"},
)
access_token = user_manager.create_access_token(user)
return Token(
access_token=access_token,
token_type="bearer",
expires_in=ACCESS_TOKEN_EXPIRE_MINUTES * 60
expires_in=ACCESS_TOKEN_EXPIRE_MINUTES * 60,
)
@user_app.get("/auth/me", response_model=User)
async def get_current_user_info(current_user: User = Depends(get_current_user)):
"""Get current user information."""
return current_user
@user_app.get("/users", response_model=List[User])
async def get_all_users(admin: User = Depends(require_admin)):
"""Get all users (admin only)."""
return user_manager.get_all_users()
@user_app.get("/users/{user_id}", response_model=User)
async def get_user(user_id: int, current_user: User = Depends(get_current_user)):
"""Get user by ID."""
# Users can only view their own profile unless they're admin
if current_user.id != user_id and current_user.role != "admin":
raise HTTPException(status_code=403, detail="Access denied")
user = user_manager.get_user_by_id(user_id)
if user is None:
raise HTTPException(status_code=404, detail="User not found")
return user
@user_app.put("/users/{user_id}", response_model=User)
async def update_user(user_id: int, update_data: UserUpdate,
current_user: User = Depends(get_current_user)):
async def update_user(
user_id: int,
update_data: UserUpdate,
current_user: User = Depends(get_current_user),
):
"""Update user data."""
# Users can only update their own profile unless they're admin
if current_user.id != user_id and current_user.role != "admin":
raise HTTPException(status_code=403, detail="Access denied")
# Non-admin users cannot change roles
if current_user.role != "admin" and update_data.role is not None:
raise HTTPException(status_code=403, detail="Cannot change role")
user = user_manager.update_user(user_id, update_data)
if user is None:
raise HTTPException(status_code=404, detail="User not found")
return user
@user_app.delete("/users/{user_id}")
async def delete_user(user_id: int, admin: User = Depends(require_admin)):
"""Delete user (admin only)."""
success = user_manager.delete_user(user_id)
if not success:
raise HTTPException(status_code=404, detail="User not found")
return {"message": "User deleted successfully"}
@user_app.get("/auth/dashboard")
async def get_auth_dashboard():
"""Get authentication dashboard HTML."""
@@ -841,6 +886,8 @@ async def get_auth_dashboard():
"""
return HTMLResponse(content=html_content)
if __name__ == "__main__":
import uvicorn
uvicorn.run(user_app, host="127.0.0.1", port=8004)
test_workflows.py
+64 -49
View File
@@ -6,85 +6,100 @@ Validate that our upgraded workflows are working properly
import json
from pathlib import Path
from typing import Dict, List, Any
def test_sample_workflows():
"""Test sample workflows to ensure they're working"""
print("🔍 Testing sample workflows...")
samples = []
categories = ['Manual', 'Webhook', 'Schedule', 'Http', 'Code']
categories = ["Manual", "Webhook", "Schedule", "Http", "Code"]
for category in categories:
category_path = Path('workflows') / category
category_path = Path("workflows") / category
if category_path.exists():
workflow_files = list(category_path.glob('*.json'))[:2] # Test first 2 from each category
workflow_files = list(category_path.glob("*.json"))[
:2
] # Test first 2 from each category
for workflow_file in workflow_files:
try:
with open(workflow_file, 'r', encoding='utf-8') as f:
with open(workflow_file, "r", encoding="utf-8") as f:
data = json.load(f)
# Validate basic structure
has_name = 'name' in data and data['name']
has_nodes = 'nodes' in data and isinstance(data['nodes'], list)
has_connections = 'connections' in data and isinstance(data['connections'], dict)
samples.append({
'file': str(workflow_file),
'name': data.get('name', 'Unnamed'),
'nodes': len(data.get('nodes', [])),
'connections': len(data.get('connections', {})),
'has_name': has_name,
'has_nodes': has_nodes,
'has_connections': has_connections,
'valid': has_name and has_nodes and has_connections,
'category': category
})
has_name = "name" in data and data["name"]
has_nodes = "nodes" in data and isinstance(data["nodes"], list)
has_connections = "connections" in data and isinstance(
data["connections"], dict
)
samples.append(
{
"file": str(workflow_file),
"name": data.get("name", "Unnamed"),
"nodes": len(data.get("nodes", [])),
"connections": len(data.get("connections", {})),
"has_name": has_name,
"has_nodes": has_nodes,
"has_connections": has_connections,
"valid": has_name and has_nodes and has_connections,
"category": category,
}
)
except Exception as e:
samples.append({
'file': str(workflow_file),
'error': str(e),
'valid': False,
'category': category
})
samples.append(
{
"file": str(workflow_file),
"error": str(e),
"valid": False,
"category": category,
}
)
print(f"\n📊 Tested {len(samples)} sample workflows:")
print("=" * 60)
valid_count = 0
for sample in samples:
if sample['valid']:
print(f"✅ {sample['name']} ({sample['category']}) - {sample['nodes']} nodes, {sample['connections']} connections")
if sample["valid"]:
print(
f"✅ {sample['name']} ({sample['category']}) - {sample['nodes']} nodes, {sample['connections']} connections"
)
valid_count += 1
else:
print(f"❌ {sample['file']} - Error: {sample.get('error', 'Invalid structure')}")
print(
f"❌ {sample['file']} - Error: {sample.get('error', 'Invalid structure')}"
)
print(f"\n🎯 Result: {valid_count}/{len(samples)} workflows are valid and ready!")
# Category breakdown
category_stats = {}
for sample in samples:
category = sample.get('category', 'unknown')
category = sample.get("category", "unknown")
if category not in category_stats:
category_stats[category] = {'valid': 0, 'total': 0}
category_stats[category]['total'] += 1
if sample['valid']:
category_stats[category]['valid'] += 1
print(f"\n📁 Category Breakdown:")
category_stats[category] = {"valid": 0, "total": 0}
category_stats[category]["total"] += 1
if sample["valid"]:
category_stats[category]["valid"] += 1
print("\n📁 Category Breakdown:")
for category, stats in category_stats.items():
success_rate = (stats['valid'] / stats['total']) * 100 if stats['total'] > 0 else 0
success_rate = (
(stats["valid"] / stats["total"]) * 100 if stats["total"] > 0 else 0
)
print(f" {category}: {stats['valid']}/{stats['total']} ({success_rate:.1f}%)")
return valid_count, len(samples)
if __name__ == "__main__":
valid_count, total_count = test_sample_workflows()
if valid_count == total_count:
print(f"\n🎉 ALL SAMPLE WORKFLOWS ARE VALID! 🎉")
print("\n🎉 ALL SAMPLE WORKFLOWS ARE VALID! 🎉")
elif valid_count > total_count * 0.8:
print(f"\n✅ Most workflows are valid ({valid_count}/{total_count})")
else:
workflow_db.py
+437 -365
View File
File diff suppressed because it is too large Load Diff