ScrapeGraph MCP Server

{
  "properties": {
    "url": {
      "type": "string",
      "description": "The target website URL where the agentic scraping workflow should start.\n- Must include protocol (http:// or https://)\n- Should be the starting page for your automation workflow\n- The agent will begin its actions from this URL\n- Examples:\n  * https://example.com/search (start at search page)\n  * https://shop.example.com/login (begin with login flow)\n  * https://app.example.com/dashboard (start at main interface)\n  * https://forms.example.com/contact (begin at form page)\n- Considerations:\n  * Choose a starting point that makes sense for your workflow\n  * Ensure the page is publicly accessible or handle authentication\n  * Consider the logical flow of actions from this starting point"
    },
    "user_prompt": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "High-level instructions for what the agent should accomplish.\n- Describes the overall goal and desired outcome of the automation\n- Should be clear and specific about what you want to achieve\n- Works in conjunction with the steps parameter for detailed guidance\n- Examples:\n  * \"Navigate to the search page, search for laptops, and extract the top 5 results with prices\"\n  * \"Fill out the contact form with sample data and submit it\"\n  * \"Login to the dashboard and extract all recent notifications\"\n  * \"Browse the product catalog and collect information about all items\"\n  * \"Navigate through the multi-step checkout process and capture each step\"\n- Tips for better results:\n  * Be specific about the end goal\n  * Mention what data you want extracted\n  * Include context about the expected workflow\n  * Specify any particular elements or sections to focus on"
    },
    "output_schema": {
      "anyOf": [
        {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "additionalProperties": true,
              "type": "object"
            }
          ],
          "default": null,
          "description": "Desired output structure as a JSON schema dict or JSON string",
          "oneOf": [
            {
              "type": "string"
            },
            {
              "type": "object"
            }
          ]
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Desired output structure for extracted data.\n- Can be provided as a dictionary or JSON string\n- Defines the format and structure of the final extracted data\n- Helps ensure consistent, predictable output format\n- Examples:\n  * Simple object: {'type': 'object', 'properties': {'title': {'type': 'string'}, 'price': {'type': 'number'}}}\n  * Array of objects: {'type': 'array', 'items': {'type': 'object', 'properties': {'name': {'type': 'string'}, 'value': {'type': 'string'}}}}\n  * Complex nested: {'type': 'object', 'properties': {'products': {'type': 'array', 'items': {...}}, 'total_count': {'type': 'number'}}}\n  * As JSON string: '{\"type\": \"object\", \"properties\": {\"results\": {\"type\": \"array\"}}}'\n- Default: None (agent will infer structure from prompt and steps)"
    },
    "steps": {
      "anyOf": [
        {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            }
          ],
          "default": null,
          "description": "Step-by-step instructions for the agent as a list of strings or JSON array string",
          "oneOf": [
            {
              "type": "string"
            },
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            }
          ]
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Step-by-step instructions for the agent.\n- Can be provided as a list of strings or JSON array string\n- Provides detailed, sequential instructions for the automation workflow\n- Each step should be a clear, actionable instruction\n- Examples as list:\n  * ['Click the search button', 'Enter \"laptops\" in the search box', 'Press Enter', 'Wait for results to load', 'Extract product information']\n  * ['Fill in email field with test@example.com', 'Fill in password field', 'Click login button', 'Navigate to profile page']\n- Examples as JSON string:\n  * '[\"Open navigation menu\", \"Click on Products\", \"Select category filters\", \"Extract all product data\"]'\n- Best practices:\n  * Break complex actions into simple steps\n  * Be specific about UI elements (button text, field names, etc.)\n  * Include waiting/loading steps when necessary\n  * Specify extraction points clearly\n  * Order steps logically for the workflow"
    },
    "ai_extraction": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Enable AI-powered extraction mode for intelligent data parsing.\n- Default: true (recommended for most use cases)\n- Options:\n  * true: Uses advanced AI to intelligently extract and structure data\n    - Better at handling complex page layouts\n    - Can adapt to different content structures\n    - Provides more accurate data extraction\n    - Recommended for most scenarios\n  * false: Uses simpler extraction methods\n    - Faster processing but less intelligent\n    - May miss complex or nested data\n    - Use when speed is more important than accuracy\n- Performance impact:\n  * true: Higher processing time but better results\n  * false: Faster execution but potentially less accurate extraction"
    },
    "persistent_session": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Maintain session state between steps.\n- Default: false (each step starts fresh)\n- Options:\n  * true: Keeps cookies, login state, and session data between steps\n    - Essential for authenticated workflows\n    - Maintains shopping cart contents, user preferences, etc.\n    - Required for multi-step processes that depend on previous actions\n    - Use for: Login flows, shopping processes, form wizards\n  * false: Each step starts with a clean session\n    - Faster and simpler for independent actions\n    - No state carried between steps\n    - Use for: Simple data extraction, public content scraping\n- Examples when to use true:\n  * Login → Navigate to protected area → Extract data\n  * Add items to cart → Proceed to checkout → Extract order details\n  * Multi-step form completion with session dependencies"
    },
    "timeout_seconds": {
      "anyOf": [
        {
          "type": "number"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Maximum time to wait for the entire workflow.\n- Default: 120 seconds (2 minutes)\n- Recommended ranges:\n  * 60-120: Simple workflows (2-5 steps)\n  * 180-300: Medium complexity (5-10 steps)\n  * 300-600: Complex workflows (10+ steps or slow sites)\n  * 600+: Very complex or slow-loading workflows\n- Considerations:\n  * Include time for page loads, form submissions, and processing\n  * Factor in network latency and site response times\n  * Allow extra time for AI processing and extraction\n  * Balance between thoroughness and efficiency\n- Examples:\n  * 60.0: Quick single-page data extraction\n  * 180.0: Multi-step form filling and submission\n  * 300.0: Complex navigation and comprehensive data extraction\n  * 600.0: Extensive workflows with multiple page interactions"
    }
  },
  "required": [
    "url"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": false,
  "readOnlyHint": false
}

{
  "properties": {
    "website_url": {
      "type": "string",
      "description": "The complete URL of the webpage to convert to markdown format.\n- Must include protocol (http:// or https://)\n- Supports most web content types (HTML, articles, documentation)\n- Works with both static and dynamic content\n- Examples:\n  * https://example.com/page\n  * https://docs.python.org/3/tutorial/\n  * https://github.com/user/repo/README.md\n- Invalid examples:\n  * example.com (missing protocol)\n  * ftp://example.com (unsupported protocol)\n  * localhost:3000 (missing protocol)"
    }
  },
  "required": [
    "website_url"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": true,
  "readOnlyHint": true
}

{
  "properties": {
    "website_url": {
      "type": "string",
      "description": "The complete URL of the webpage to scrape.\n- Must include protocol (http:// or https://)\n- Returns raw HTML content of the page\n- Works with both static and dynamic websites\n- Examples:\n  * https://example.com/page\n  * https://api.example.com/docs\n  * https://news.site.com/article/123\n  * https://app.example.com/dashboard (may need render_heavy_js=true)\n- Supported protocols: HTTP, HTTPS\n- Invalid examples:\n  * example.com (missing protocol)\n  * ftp://example.com (unsupported protocol)"
    },
    "render_heavy_js": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Enable full JavaScript rendering for dynamic content.\n- Default: false (faster, lower cost, works for most static sites)\n- Set to true for sites that require JavaScript execution to display content\n- When to use true:\n  * Single Page Applications (React, Angular, Vue.js)\n  * Sites with dynamic content loading via AJAX\n  * Content that appears only after JavaScript execution\n  * Interactive web applications\n  * Sites where initial HTML is mostly empty\n- When to use false (default):\n  * Static websites and blogs\n  * Server-side rendered content\n  * Traditional HTML pages\n  * News articles and documentation\n  * When you need faster processing\n- Performance impact:\n  * false: 2-5 seconds processing time\n  * true: 15-30 seconds processing time (waits for JS execution)\n- Cost: Same (1 credit) regardless of render_heavy_js setting"
    }
  },
  "required": [
    "website_url"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": true,
  "readOnlyHint": true
}

{
  "properties": {
    "user_prompt": {
      "type": "string",
      "description": "Search query or natural language instructions for information to find.\n- Can be a simple search query or detailed extraction instructions\n- The AI will search the web and extract relevant data from found pages\n- Be specific about what information you want extracted\n- Examples:\n  * \"Find latest AI research papers published in 2024 with author names and abstracts\"\n  * \"Search for Python web scraping tutorials with ratings and difficulty levels\"\n  * \"Get current cryptocurrency prices and market caps for top 10 coins\"\n  * \"Find contact information for tech startups in San Francisco\"\n  * \"Search for job openings for data scientists with salary information\"\n- Tips for better results:\n  * Include specific fields you want extracted\n  * Mention timeframes or filters (e.g., \"latest\", \"2024\", \"top 10\")\n  * Specify data types needed (prices, dates, ratings, etc.)"
    },
    "num_results": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Number of websites to search and extract data from.\n- Default: 3 websites (costs 30 credits total)\n- Range: 1-20 websites (recommended to stay under 10 for cost efficiency)\n- Each website costs 10 credits, so total cost = num_results × 10\n- Examples:\n  * 1: Quick single-source lookup (10 credits)\n  * 3: Standard research (30 credits) - good balance of coverage and cost\n  * 5: Comprehensive research (50 credits)\n  * 10: Extensive analysis (100 credits)\n- Note: More results provide broader coverage but increase costs and processing time"
    },
    "number_of_scrolls": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Number of infinite scrolls per searched webpage.\n- Default: 0 (no scrolling on search result pages)\n- Range: 0-10 scrolls per page\n- Useful when search results point to pages with dynamic content loading\n- Each scroll waits for content to load before continuing\n- Examples:\n  * 0: Static content pages, news articles, documentation\n  * 2: Social media pages, product listings with lazy loading\n  * 5: Extensive feeds, long-form content with infinite scroll\n- Note: Increases processing time significantly (adds 5-10 seconds per scroll per page)"
    }
  },
  "required": [
    "user_prompt"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": false,
  "readOnlyHint": true
}

{
  "properties": {
    "website_url": {
      "type": "string",
      "description": "The base URL of the website to extract sitemap from.\n- Must include protocol (http:// or https://)\n- Should be the root domain or main section you want to map\n- The tool will discover all accessible pages from this starting point\n- Examples:\n  * https://example.com (discover entire website structure)\n  * https://docs.example.com (map documentation site)\n  * https://blog.company.com (discover all blog pages)\n  * https://shop.example.com (map e-commerce structure)\n- Best practices:\n  * Use root domain (https://example.com) for complete site mapping\n  * Use subdomain (https://docs.example.com) for focused mapping\n  * Ensure the URL is accessible and doesn't require authentication\n- Discovery methods:\n  * Checks for robots.txt and sitemap.xml files\n  * Crawls navigation links and menus\n  * Discovers pages through internal link analysis\n  * Identifies common URL patterns and structures"
    }
  },
  "required": [
    "website_url"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": true,
  "readOnlyHint": true
}

{
  "properties": {
    "request_id": {
      "type": "string",
      "description": "The unique request ID returned by smartcrawler_initiate. Use this to retrieve the crawling results. Keep polling until status is 'completed'. Example: 'req_abc123xyz'"
    }
  },
  "required": [
    "request_id"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": true,
  "readOnlyHint": true
}

{
  "properties": {
    "url": {
      "type": "string",
      "description": "The starting URL to begin crawling from.\n- Must include protocol (http:// or https://)\n- The crawler will discover and process linked pages from this starting point\n- Should be a page with links to other pages you want to crawl\n- Examples:\n  * https://docs.example.com (documentation site root)\n  * https://blog.company.com (blog homepage)\n  * https://example.com/products (product category page)\n  * https://news.site.com/category/tech (news section)\n- Best practices:\n  * Use homepage or main category pages as starting points\n  * Ensure the starting page has links to content you want to crawl\n  * Consider site structure when choosing the starting URL"
    },
    "prompt": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "AI prompt for data extraction.\n- REQUIRED when extraction_mode is 'ai'\n- Ignored when extraction_mode is 'markdown'\n- Describes what data to extract from each crawled page\n- Applied consistently across all discovered pages\n- Examples:\n  * \"Extract API endpoint name, method, parameters, and description\"\n  * \"Get article title, author, publication date, and summary\"\n  * \"Find product name, price, description, and availability\"\n  * \"Extract job title, company, location, salary, and requirements\"\n- Tips for better results:\n  * Be specific about fields you want from each page\n  * Consider that different pages may have different content structures\n  * Use general terms that apply across multiple page types"
    },
    "extraction_mode": {
      "default": "ai",
      "type": "string",
      "description": "Extraction mode for processing crawled pages.\n- Default: \"ai\"\n- Options:\n  * \"ai\": AI-powered structured data extraction (10 credits per page)\n    - Uses the prompt to extract specific data from each page\n    - Returns structured JSON data\n    - More expensive but provides targeted information\n    - Best for: Data collection, research, structured analysis\n  * \"markdown\": Simple markdown conversion (2 credits per page)\n    - Converts each page to clean markdown format\n    - No AI processing, just content conversion\n    - More cost-effective for content archival\n    - Best for: Documentation backup, content migration, reading\n- Cost comparison:\n  * AI mode: 50 pages = 500 credits\n  * Markdown mode: 50 pages = 100 credits"
    },
    "depth": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Maximum depth of link traversal from the starting URL.\n- Default: unlimited (will follow links until max_pages or no more links)\n- Depth levels:\n  * 0: Only the starting URL (no link following)\n  * 1: Starting URL + pages directly linked from it\n  * 2: Starting URL + direct links + links from those pages\n  * 3+: Continues following links to specified depth\n- Examples:\n  * 1: Crawl blog homepage + all blog posts\n  * 2: Crawl docs homepage + category pages + individual doc pages\n  * 3: Deep crawling for comprehensive site coverage\n- Considerations:\n  * Higher depth can lead to exponential page growth\n  * Use with max_pages to control scope and cost\n  * Consider site structure when setting depth"
    },
    "max_pages": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Maximum number of pages to crawl in total.\n- Default: unlimited (will crawl until no more links or depth limit)\n- Recommended ranges:\n  * 10-20: Testing and small sites\n  * 50-100: Medium sites and focused crawling\n  * 200-500: Large sites and comprehensive analysis\n  * 1000+: Enterprise-level crawling (high cost)\n- Cost implications:\n  * AI mode: max_pages × 10 credits\n  * Markdown mode: max_pages × 2 credits\n- Examples:\n  * 10: Quick site sampling (20-100 credits)\n  * 50: Standard documentation crawl (100-500 credits)\n  * 200: Comprehensive site analysis (400-2000 credits)\n- Note: Crawler stops when this limit is reached, regardless of remaining links"
    },
    "same_domain_only": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Whether to crawl only within the same domain.\n- Default: true (recommended for most use cases)\n- Options:\n  * true: Only crawl pages within the same domain as starting URL\n    - Prevents following external links\n    - Keeps crawling focused on the target site\n    - Reduces risk of crawling unrelated content\n    - Example: Starting at docs.example.com only crawls docs.example.com pages\n  * false: Allow crawling external domains\n    - Follows links to other domains\n    - Can lead to very broad crawling scope\n    - May crawl unrelated or unwanted content\n    - Use with caution and appropriate max_pages limit\n- Recommendations:\n  * Use true for focused site crawling\n  * Use false only when you specifically need cross-domain data\n  * Always set max_pages when using false to prevent runaway crawling"
    }
  },
  "required": [
    "url"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": false,
  "readOnlyHint": false
}

{
  "properties": {
    "user_prompt": {
      "type": "string"
    },
    "website_url": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "website_html": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "website_markdown": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "output_schema": {
      "anyOf": [
        {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "additionalProperties": true,
              "type": "object"
            }
          ],
          "default": null,
          "description": "JSON schema dict or JSON string defining the expected output structure",
          "oneOf": [
            {
              "type": "string"
            },
            {
              "type": "object"
            }
          ]
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "number_of_scrolls": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "total_pages": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "render_heavy_js": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    },
    "stealth": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null
    }
  },
  "required": [
    "user_prompt"
  ],
  "type": "object",
  "additionalProperties": false
}

{
  "type": "object",
  "additionalProperties": true
}

{
  "destructiveHint": false,
  "idempotentHint": true,
  "readOnlyHint": true
}