Willem Jiang
975b344ca7
* fix: resolve issue #651 - crawl error with None content handling Fixed issue #651 by adding comprehensive null-safety checks and error handling to the crawl system. The fix prevents the ‘TypeError: Incoming markup is of an invalid type: None’ crash by: 1. Validating HTTP responses from Jina API 2. Handling None/empty content at extraction stage 3. Adding fallback handling in Article markdown/message conversion 4. Improving error diagnostics with detailed logging 5. Adding 16 new tests with 100% coverage for critical paths * Update src/crawler/readability_extractor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update src/crawler/article.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
34 lines
1.1 KiB
Python
34 lines
1.1 KiB
Python
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
import logging
|
|
import os
|
|
|
|
import requests
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class JinaClient:
|
|
def crawl(self, url: str, return_format: str = "html") -> str:
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"X-Return-Format": return_format,
|
|
}
|
|
if os.getenv("JINA_API_KEY"):
|
|
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
|
|
else:
|
|
logger.warning(
|
|
"Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information."
|
|
)
|
|
data = {"url": url}
|
|
response = requests.post("https://r.jina.ai/", headers=headers, json=data)
|
|
|
|
if response.status_code != 200:
|
|
raise ValueError(f"Jina API returned status {response.status_code}: {response.text}")
|
|
|
|
if not response.text or not response.text.strip():
|
|
raise ValueError("Jina API returned empty response")
|
|
|
|
return response.text
|