ai.smithery/arjunkmrm-scrapermcp_el

Dependency "aiohttp@3.9.0" has known CVEs:

Dependency "mcp@null" has known CVEs:

Pattern "(?:delay|sleep|timeout|interval)\s*[:=]\s*(?:[^;]*(?:secret|token|password|credential|key|env))" matched in source_code: "timeout = ClientTimeout(total=120) # Create asynchronous HTTP client session async with aiohttp.ClientSession( headers=headers, timeout=timeout, connector=aiohttp.TCPConnector(), max_field_size=32768, ) as session: try: # Use session to initiate GET request async with session.get( url, # Target URL proxy=proxy, # Use proxy proxy_auth=proxy_auth, # Use proxy authentication ssl=False, # Disable SSL verification ) as response: # Check if response status code is 200 (success) if response.status == 200: # Return response text content return await response.text() else: # Construct error message containing status code and URL error_msg = f"Status code: {response.status}, URL: {url}" # Throw retry exception raise ScrapeRetryException(error_msg) except aiohttp.ClientError as e: error_msg = f"HTTP client error" raise ScrapeRetryException(error_msg) except asyncio.TimeoutError: error_msg = ( f"Request timeout: 60 seconds" ) raise ScrapeRetryException(error_msg) except Exception as e: error_msg = f"Unknown error:" raise ScrapeRetryException(error_msg) async def scrape(url: str, myProxyConfig: ProxyConfig) -> str: """ Web scraping method Parameters: url: URL address to scrape myProxyConfig: Proxy configuration object Returns: Returns web page content text on success, returns empty string on failure """ try: result = await scrape_with_retry(url, myProxyConfig) return result except ScrapeRetryException: return "" def clean_html(html: str) -> str: """Clean HTML string""" cleaner = Cleaner( scripts=True, kill_tags=["nav", "svg", "footer", "noscript", "script", "form"], style=True, remove_tags=[], safe_attrs=list(defs.safe_attrs) + ["idx"], inline_style=True, links=True, meta=False, embedded=True, frames=False, forms=False, annoying_tags=False, page_structure=False, javascript=True, comments=True, ) return cleaner.clean_html(html) def strip_html(thor_mcp_html: str) -> str: """Simplify HTML string, remove unnecessary elements, attributes and redundant content""" import re # Call clean_html function for initial cleaning (assuming the function is already defined externally) thor_mcp_cleaned_html = clean_html(thor_mcp_html) # Parse the cleaned HTML string into XML tree structure thor_mcp_html_tree = fromstring(thor_mcp_cleaned_html) # Traverse all elements in the HTML tree (including nested descendant elements) for thor_mcp_element in thor_mcp_html_tree.iter(): # Remove style attribute (inline styles) from all elements if "style" in thor_mcp_element.attrib: del thor_mcp_element.attrib["style"] # Use del statement to delete element attribute if ( ( not thor_mcp_element.attrib # No attributes or (len(thor_mcp_element.attrib) == 1 and "idx" in thor_mcp_element.attrib) # Or only contains idx attribute ) and not thor_mcp_element.getchildren() # No child elements and (not thor_mcp_element.text or not thor_mcp_element.text.strip()) # No text or blank text and (not thor_mcp_element.tail or not thor_mcp_element.tail.strip()) # No tail text or blank tail ): # Get parent element (may be None if it's the root element) thor_mcp_parent = thor_mcp_element.getparent() # Only remove if parent element exists if thor_mcp_parent is not None: # Remove current element from parent's tree structure thor_mcp_parent.remove(thor_mcp_element) # Convert processed XML tree back to HTML string return tostring(thor_mcp_html_tree, encoding='unicode') # Remove elements containing "footer" or "hidden" in class or id thor_mcp_xpath_query = ( ".//*[contains(@class, 'footer') or contains(@id, 'footer') or " "contains(@class, 'hidden') or contains(@id, 'hidden')]" ) # Use XPath query to find all elements that need to be removed thor_mcp_elements_to_remove = thor_mcp_html_tree.xpath(thor_mcp_xpath_query) # Traverse all elements that need to be removed for thor_mcp_element in thor_mcp_elements_to_remove: # Get the parent element of the current element thor_mcp_parent = thor_mcp_element.getparent() # Only perform removal if parent element exists if thor_mcp_parent is not None: # Remove current element from parent element thor_mcp_parent.remove(thor_mcp_element) # Reserialize HTML tree to string thor_mcp_stripped_html = tostring(thor_mcp_html_tree, encoding="unicode") # Replace multiple spaces with single space thor_mcp_stripped_html = re.sub(r"\s{2,}", " ", thor_mcp_stripped_html) # Replace consecutive newlines with empty string thor_mcp_stripped_html = re.sub(r"\n{2,}", "", thor_mcp_stripped_html) return thor_mcp_stripped_html def extract_links_with_text(thor_mcp_html: str, thor_mcp_base_url: str | None = None) -> list[str]: """ Extract links with display text from HTML Parameters: thor_mcp_html (str): Input HTML string thor_mcp_base_url (str | None): Base URL for converting relative URLs to absolute URLs If None, relative URLs remain unchanged Returns: list[str]: List of links in format [display text] URL """ # Use lxml's fromstring function to parse HTML string into XML tree structure thor_mcp_html_tree = fromstring(thor_mcp_html) # Initialize empty list to store formatted links thor_mcp_links = [] # Traverse all <a> tags containing href attribute (XPath selector) for thor_mcp_link in thor_mcp_html_tree.xpath("//a[@href]"): # Get value of href attribute (link target address) thor_mcp_href = thor_mcp_link.get("href") # Get all text content within the tag (including child tag text), and remove leading/trailing whitespace thor_mcp_text = thor_mcp_link.text_content().strip() # Only process when both href and text exist (filter empty links or empty text) if thor_mcp_href and thor_mcp_text: # Skip empty text or pure whitespace text (although strip() is used, prevent special whitespace characters) if not thor_mcp_text: continue # Skip in-page anchor links (starting with #) if thor_mcp_href.startswith("#"): continue # Skip JavaScript pseudo-links if thor_mcp_href.startswith("javascript:"): continue # Convert URL when base_url is provided and it's a relative path (starting with /) if thor_mcp_base_url and thor_mcp_href.startswith("/"): # Remove trailing slash from base_url to avoid double slash issue thor_mcp_base = thor_mcp_base_url.rstrip("/") # Concatenate into absolute URL thor_mcp_href = f"{thor_mcp_base}{thor_mcp_href}" # Add formatted link to result list: [text] URL thor_mcp_links.append(f"[{thor_mcp_text}] {thor_mcp_href}") # Return list of all qualified links return thor_mcp_links def get_content(thor_mcp_content: str, thor_mcp_output_format: str) -> str: """ Extract content from response and convert to appropriate format Parameters: thor_mcp_content: Response content string thor_mcp_output_format: Output format ("html", "links", or other formats converted to markdown) Returns: Formatted content string """ if thor_mcp_output_format == "html": return thor_mcp_content if thor_mcp_output_format == "links": thor_mcp_links = extract_links_with_text(thor_mcp_content) return "\n".join(thor_mcp_links) thor_mcp_stripped_html = strip_html(thor_mcp_content) # Simplify HTML content return markdownify(thor_mcp_stripped_html) # For other formats, return original content string # Main program entry point (when running this script directly) if __name__ == "__main__": # If current script is the main program entry # Get the Starlette app and add CORS middleware app = mcp.streamable_http_app() # Get streamable HTTP application instance # Add CORS middleware with proper header exposure for MCP session management app.add_middleware( # Add CORS middleware to application CORSMiddleware, # Use CORS middleware class allow_origins=["*"], # Allow cross-origin requests from all origins (should be more restrictive in production) allow_credentials=True, # Allow credentials (such as cookies) allow_methods=["GET", "POST", "OPTIONS"], # Allowed HTTP methods allow_headers=["*"], # Allow all request headers expose_headers=["mcp-session-id", "mcp-protocol-version"], # Allow client to read MCP session ID and protocol version headers max_age=86400, # Preflight request cache time (seconds) ) # Use PORT environment variable port = int(os.environ.get("PORT", 8081)) # Get port number from env" (at position 9603)

Pattern "(?:read|write|modify|delete).*(?:file|path|directory)(?!.*(?:lock|mutex|semaphore|flock|atomic))" matched in source_code: "Read HTML from local cache: {thor_mcp_file" (at position 5649)

Pattern "session\.(?:get|post|put|delete|patch|request)\s*\([^)]*(?:req|request|input|param|params|args|url|uri|href|target|endpoint|host)" matched in source_code: "session.get( url, # Target URL" (at position 9956)

Pattern "allow_origins\s*=\s*\[\s*['"]\*['"]\s*\]" matched in source_code: "allow_origins=["*"]" (at position 18757)