574 行
20 KiB
Python
574 行
20 KiB
Python
# -*- coding: utf-8 -*-
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from html import unescape
|
||
from html.parser import HTMLParser
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
|
||
|
||
class _QzoneFeedHtmlParser(HTMLParser):
|
||
_VOID_TAGS = {
|
||
"area",
|
||
"base",
|
||
"br",
|
||
"col",
|
||
"embed",
|
||
"hr",
|
||
"img",
|
||
"input",
|
||
"link",
|
||
"meta",
|
||
"param",
|
||
"source",
|
||
"track",
|
||
"wbr",
|
||
}
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self.text_parts: list[str] = []
|
||
self.repost_parts: list[str] = []
|
||
self.image_urls: list[str] = []
|
||
self.image_items: list[dict[str, str]] = []
|
||
self._class_stack: list[set[str]] = []
|
||
self._tag_stack: list[str] = []
|
||
|
||
@staticmethod
|
||
def _classes(attrs: list[tuple[str, str | None]]) -> set[str]:
|
||
for key, value in attrs:
|
||
if str(key or "").lower() == "class":
|
||
return {item.strip() for item in str(value or "").split() if item.strip()}
|
||
return set()
|
||
|
||
@staticmethod
|
||
def _attr(attrs: list[tuple[str, str | None]], name: str) -> str:
|
||
lowered = str(name or "").lower()
|
||
for key, value in attrs:
|
||
if str(key or "").lower() == lowered:
|
||
return str(value or "")
|
||
return ""
|
||
|
||
@classmethod
|
||
def _first_attr(cls, attrs: list[tuple[str, str | None]], *names: str) -> str:
|
||
for name in names:
|
||
value = cls._attr(attrs, name).strip()
|
||
if value:
|
||
return value
|
||
return ""
|
||
|
||
@staticmethod
|
||
def _is_content_image_url(value: str) -> bool:
|
||
source = str(value or "").strip().lower()
|
||
return bool(source) and not source.startswith("http://qzonestyle.gtimg.cn")
|
||
|
||
def _record_image(self, attrs: list[tuple[str, str | None]]) -> None:
|
||
preview_url = self._first_attr(attrs, "src", "data-src", "data-lazy-src", "data-original-src")
|
||
full_url = self._first_attr(
|
||
attrs,
|
||
"data-original",
|
||
"data-origin",
|
||
"data-origin-url",
|
||
"data-original-url",
|
||
"data-originalurl",
|
||
"data-big-url",
|
||
"data-bigurl",
|
||
"data-full-url",
|
||
"data-fullurl",
|
||
"data-raw-url",
|
||
"data-rawurl",
|
||
"origsrc",
|
||
"original",
|
||
"data-url",
|
||
)
|
||
if not self._is_content_image_url(full_url):
|
||
full_url = ""
|
||
if not self._is_content_image_url(preview_url):
|
||
preview_url = ""
|
||
source = full_url or preview_url
|
||
if not source:
|
||
return
|
||
item = {"preview_url": preview_url or source, "full_url": source}
|
||
if item not in self.image_items:
|
||
self.image_items.append(item)
|
||
if source not in self.image_urls:
|
||
self.image_urls.append(source)
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
normalized_tag = str(tag or "").lower()
|
||
classes = self._classes(attrs)
|
||
if normalized_tag != "img":
|
||
if normalized_tag not in self._VOID_TAGS:
|
||
self._tag_stack.append(normalized_tag)
|
||
self._class_stack.append(classes)
|
||
elif classes:
|
||
self._class_stack.append(classes)
|
||
self._class_stack.pop()
|
||
return
|
||
active_classes = self._class_stack + [classes]
|
||
if any({"img-box", "video-img"} & item for item in active_classes):
|
||
self._record_image(attrs)
|
||
|
||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
normalized_tag = str(tag or "").lower()
|
||
if normalized_tag == "img":
|
||
self.handle_starttag(tag, attrs)
|
||
return
|
||
classes = self._classes(attrs)
|
||
if classes:
|
||
self._class_stack.append(classes)
|
||
self._class_stack.pop()
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
normalized_tag = str(tag or "").lower()
|
||
for index in range(len(self._tag_stack) - 1, -1, -1):
|
||
if self._tag_stack[index] == normalized_tag:
|
||
del self._tag_stack[index:]
|
||
del self._class_stack[index:]
|
||
return
|
||
if self._tag_stack and self._class_stack:
|
||
self._tag_stack.pop()
|
||
self._class_stack.pop()
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
text = unescape(str(data or "")).strip()
|
||
if not text:
|
||
return
|
||
if any("f-info" in item for item in self._class_stack):
|
||
self.text_parts.append(text)
|
||
if any("txt-box" in item for item in self._class_stack):
|
||
self.repost_parts.append(text)
|
||
|
||
|
||
def _parse_feed_html(html_content: str) -> tuple[str, str, list[str]]:
|
||
text, repost, image_urls, _items = _parse_feed_html_details(html_content)
|
||
return text, repost, image_urls
|
||
|
||
|
||
def _parse_feed_html_details(html_content: str) -> tuple[str, str, list[str], list[dict[str, str]]]:
|
||
parser = _QzoneFeedHtmlParser()
|
||
parser.feed(str(html_content or ""))
|
||
text = "".join(parser.text_parts).strip()
|
||
repost = "".join(parser.repost_parts).strip()
|
||
if ":" in repost:
|
||
repost = repost.split(":", 1)[1].strip()
|
||
return text, repost, parser.image_urls, parser.image_items
|
||
|
||
|
||
def _html_attr_value(html_content: str, *names: str) -> str:
|
||
source = str(html_content or "")
|
||
for name in names:
|
||
escaped = re.escape(str(name or ""))
|
||
match = re.search(rf"""{escaped}\s*=\s*["']([^"']+)["']""", source, flags=re.IGNORECASE)
|
||
if match:
|
||
return unescape(match.group(1)).strip()
|
||
return ""
|
||
|
||
|
||
def _normalized_dict(value: Any) -> dict[str, Any]:
|
||
if not isinstance(value, dict):
|
||
return {}
|
||
return {str(key or "").strip().lower().replace("-", "_"): item for key, item in value.items()}
|
||
|
||
|
||
def _first_value(source: Any, *keys: str) -> Any:
|
||
normalized = _normalized_dict(source)
|
||
for key in keys:
|
||
value = normalized.get(str(key or "").lower().replace("-", "_"))
|
||
if value not in (None, ""):
|
||
return value
|
||
return None
|
||
|
||
|
||
def _safe_int(value: Any) -> int:
|
||
text = str(value or "").strip().lstrip("oO")
|
||
try:
|
||
return int(float(text))
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
|
||
def _clean_plain_text(value: Any) -> str:
|
||
source = str(value or "").strip()
|
||
if not source:
|
||
return ""
|
||
source = re.sub(r"<br\s*/?>", "\n", source, flags=re.IGNORECASE)
|
||
source = re.sub(r"<[^>]+>", "", source)
|
||
source = unescape(source)
|
||
return re.sub(r"[ \t\r\f\v]+", " ", source).strip()
|
||
|
||
|
||
def _looks_like_feed(value: Any) -> bool:
|
||
if not isinstance(value, dict) or not value:
|
||
return False
|
||
normalized = _normalized_dict(value)
|
||
if isinstance(normalized.get("comm"), dict) and isinstance(normalized.get("userinfo"), dict):
|
||
return True
|
||
def present(item: Any) -> bool:
|
||
if isinstance(item, (dict, list, tuple, set)):
|
||
return bool(item)
|
||
return item not in (None, "")
|
||
|
||
has_content = any(
|
||
present(normalized.get(key))
|
||
for key in ("html", "feedhtml", "feed_html", "content", "text", "summary", "con")
|
||
)
|
||
has_identity = any(
|
||
present(normalized.get(key))
|
||
for key in ("uin", "hostuin", "opuin", "user_uin", "useruin", "owner_uin", "owneruin", "userinfo")
|
||
)
|
||
has_id = any(
|
||
present(normalized.get(key))
|
||
for key in ("key", "tid", "fid", "topicid", "topic_id", "feedid", "feed_id", "cellid", "id")
|
||
)
|
||
return has_content and has_identity and has_id
|
||
|
||
|
||
_SKIPPED_FEED_BRANCHES = {
|
||
"comment",
|
||
"comments",
|
||
"commentlist",
|
||
"comment_list",
|
||
"reply",
|
||
"replys",
|
||
"replies",
|
||
"likemans",
|
||
"like_list",
|
||
"userinfo",
|
||
"user",
|
||
}
|
||
|
||
|
||
def _collect_feed_candidates(data: Any) -> tuple[list[dict[str, Any]], list[str]]:
|
||
candidates: list[dict[str, Any]] = []
|
||
containers: list[str] = []
|
||
seen: set[int] = set()
|
||
|
||
def walk(value: Any, path: str, depth: int) -> None:
|
||
if depth > 10:
|
||
return
|
||
if isinstance(value, list):
|
||
if value and path:
|
||
containers.append(path)
|
||
for index, item in enumerate(value):
|
||
walk(item, f"{path}[{index}]" if path else f"[{index}]", depth + 1)
|
||
return
|
||
if not isinstance(value, dict):
|
||
return
|
||
object_id = id(value)
|
||
if object_id in seen:
|
||
return
|
||
seen.add(object_id)
|
||
if _looks_like_feed(value):
|
||
candidates.append(value)
|
||
return
|
||
for key, item in value.items():
|
||
normalized_key = str(key or "").strip().lower().replace("-", "_")
|
||
if normalized_key in _SKIPPED_FEED_BRANCHES:
|
||
continue
|
||
if isinstance(item, (dict, list)):
|
||
child_path = f"{path}.{normalized_key}" if path else normalized_key
|
||
walk(item, child_path, depth + 1)
|
||
|
||
walk(data, "", 0)
|
||
return candidates, list(dict.fromkeys(containers))
|
||
|
||
|
||
def _structured_feed_identity(feed: dict[str, Any], html_content: str) -> tuple[int, str, str]:
|
||
userinfo = _first_value(feed, "userinfo", "user", "author", "owner")
|
||
if not isinstance(userinfo, dict):
|
||
userinfo = {}
|
||
id_info = _first_value(feed, "id", "identity")
|
||
if not isinstance(id_info, dict):
|
||
id_info = {}
|
||
uin = _safe_int(
|
||
_first_value(feed, "uin", "hostuin", "opuin", "user_uin", "useruin", "owner_uin", "owneruin")
|
||
or _first_value(userinfo, "uin", "qq", "id")
|
||
or _html_attr_value(html_content, "data-uin", "data-hostuin", "data-opuin", "uin")
|
||
)
|
||
tid = str(
|
||
_first_value(feed, "key", "tid", "fid", "topicid", "topic_id", "feedid", "feed_id", "cellid")
|
||
or _first_value(id_info, "cellid", "tid", "fid", "key", "id")
|
||
or _html_attr_value(
|
||
html_content,
|
||
"data-key",
|
||
"data-tid",
|
||
"data-fid",
|
||
"data-topicid",
|
||
"data-feedid",
|
||
)
|
||
or ""
|
||
).strip()
|
||
name = str(
|
||
_first_value(feed, "nickname", "name", "nick")
|
||
or _first_value(userinfo, "nickname", "name", "nick")
|
||
or _html_attr_value(html_content, "data-nickname", "data-name", "data-nick")
|
||
or uin
|
||
or "QQ空间用户"
|
||
).strip()
|
||
return uin, tid, name
|
||
|
||
|
||
def _structured_feed_common(feed: dict[str, Any]) -> dict[str, Any]:
|
||
common = _first_value(feed, "comm", "common", "cell_comm")
|
||
return common if isinstance(common, dict) else {}
|
||
|
||
|
||
def _structured_feed_text(feed: dict[str, Any], html_content: str) -> tuple[str, str]:
|
||
summary = _first_value(feed, "summary")
|
||
if isinstance(summary, dict):
|
||
text = _clean_plain_text(_first_value(summary, "summary", "content", "text"))
|
||
else:
|
||
text = _clean_plain_text(summary)
|
||
text = text or _clean_plain_text(_first_value(feed, "content", "text", "con"))
|
||
|
||
original = _first_value(feed, "original", "forward", "repost", "rt_con")
|
||
rt_con = ""
|
||
if isinstance(original, dict):
|
||
original_summary = _first_value(original, "summary", "content", "text", "con")
|
||
if isinstance(original_summary, dict):
|
||
original_summary = _first_value(original_summary, "summary", "content", "text")
|
||
rt_con = _clean_plain_text(original_summary)
|
||
else:
|
||
rt_con = _clean_plain_text(original)
|
||
|
||
if html_content:
|
||
html_text, html_repost, _urls, _items = _parse_feed_html_details(html_content)
|
||
text = text or html_text or _clean_plain_text(html_content)
|
||
rt_con = rt_con or html_repost
|
||
if not text:
|
||
operation = _first_value(feed, "operation")
|
||
share_info = _first_value(operation, "share_info", "shareinfo") if isinstance(operation, dict) else None
|
||
if isinstance(share_info, dict):
|
||
text = _clean_plain_text(_first_value(share_info, "summary", "title"))
|
||
return text, rt_con
|
||
|
||
|
||
def _valid_image_url(value: Any) -> str:
|
||
source = unescape(str(value or "").strip())
|
||
return source if source.lower().startswith(("https://", "http://", "//")) else ""
|
||
|
||
|
||
def _photo_variant_item(value: Any) -> dict[str, str] | None:
|
||
variants: list[tuple[int, str]] = []
|
||
|
||
def collect(item: Any, depth: int = 0) -> None:
|
||
if depth > 5:
|
||
return
|
||
if isinstance(item, list):
|
||
for child in item:
|
||
collect(child, depth + 1)
|
||
return
|
||
if not isinstance(item, dict):
|
||
return
|
||
direct = _valid_image_url(
|
||
_first_value(
|
||
item,
|
||
"origin_url",
|
||
"original_url",
|
||
"raw_url",
|
||
"large_url",
|
||
"big_url",
|
||
"url3",
|
||
"url2",
|
||
"url1",
|
||
"smallurl",
|
||
"pic_url",
|
||
"coverurl",
|
||
"url",
|
||
"src",
|
||
)
|
||
)
|
||
if direct:
|
||
width = _safe_int(_first_value(item, "width", "w"))
|
||
height = _safe_int(_first_value(item, "height", "h"))
|
||
variants.append((max(1, width * height), direct))
|
||
for child in item.values():
|
||
if isinstance(child, (dict, list)):
|
||
collect(child, depth + 1)
|
||
|
||
collect(value)
|
||
if not variants:
|
||
return None
|
||
variants = list(dict.fromkeys(variants))
|
||
preview = min(variants, key=lambda item: item[0])[1]
|
||
full = max(variants, key=lambda item: item[0])[1]
|
||
return {"preview_url": preview, "full_url": full}
|
||
|
||
|
||
def _structured_feed_images(feed: dict[str, Any], html_content: str) -> tuple[list[str], list[dict[str, str]]]:
|
||
items: list[dict[str, str]] = []
|
||
for key in ("pic", "pics", "images", "photos", "video"):
|
||
branch = _first_value(feed, key)
|
||
if branch in (None, ""):
|
||
continue
|
||
if isinstance(branch, dict) and isinstance(_first_value(branch, "picdata", "pic_data"), list):
|
||
sources = list(_first_value(branch, "picdata", "pic_data") or [])
|
||
elif isinstance(branch, list):
|
||
sources = branch
|
||
else:
|
||
sources = [branch]
|
||
for source in sources:
|
||
item = _photo_variant_item(source)
|
||
if item and item not in items:
|
||
items.append(item)
|
||
if html_content:
|
||
_text, _repost, _urls, html_items = _parse_feed_html_details(html_content)
|
||
for item in html_items:
|
||
if item not in items:
|
||
items.append(item)
|
||
return [item["full_url"] for item in items], items
|
||
|
||
|
||
def _balanced_object(source: str, start: int) -> str:
|
||
if start < 0 or start >= len(source) or source[start] != "{":
|
||
return ""
|
||
depth = 0
|
||
quote = ""
|
||
escaped = False
|
||
for index in range(start, len(source)):
|
||
char = source[index]
|
||
if quote:
|
||
if escaped:
|
||
escaped = False
|
||
elif char == "\\":
|
||
escaped = True
|
||
elif char == quote:
|
||
quote = ""
|
||
continue
|
||
if char in {"'", '"'}:
|
||
quote = char
|
||
elif char == "{":
|
||
depth += 1
|
||
elif char == "}":
|
||
depth -= 1
|
||
if depth == 0:
|
||
return source[start : index + 1]
|
||
return ""
|
||
|
||
|
||
def parse_qzone_h5_index_html(html_content: str) -> dict[str, Any]:
|
||
source = str(html_content or "")
|
||
token_match = re.search(
|
||
r"window\.shine0callback.*?return\s+[\"']([0-9a-zA-Z_-]+?)[\"']\s*;",
|
||
source,
|
||
flags=re.DOTALL,
|
||
)
|
||
token = token_match.group(1).strip() if token_match else ""
|
||
data_match = re.search(r"var\s+FrontPage\s*=.*?\bdata\s*:\s*\{", source, flags=re.DOTALL)
|
||
object_text = _balanced_object(source, data_match.end() - 1) if data_match else ""
|
||
payload: dict[str, Any] = {}
|
||
if object_text:
|
||
normalized = object_text.replace("undefined", "null")
|
||
try:
|
||
parsed = json.loads(normalized)
|
||
except Exception:
|
||
try:
|
||
relaxed = re.sub(r",\s*([}\]])", r"\1", normalized)
|
||
relaxed = re.sub(r"([{,]\s*)([A-Za-z_$][\w$]*)\s*:", r'\1"\2":', relaxed)
|
||
parsed = json.loads(relaxed)
|
||
except Exception:
|
||
try:
|
||
import json5 # type: ignore
|
||
|
||
parsed = json5.loads(normalized)
|
||
except Exception:
|
||
parsed = {}
|
||
if isinstance(parsed, dict):
|
||
payload = parsed
|
||
return {"token": token, "payload": payload}
|
||
|
||
|
||
def parse_recent_feeds(data: dict[str, Any], diagnostics: dict[str, Any] | None = None) -> list[Any]:
|
||
feeds, containers = _collect_feed_candidates(data)
|
||
stats: dict[str, Any] = {
|
||
"response_keys": sorted(str(key) for key in data)[:24] if isinstance(data, dict) else [],
|
||
"containers": containers[:12],
|
||
"candidate_count": len(feeds),
|
||
"parsed_count": 0,
|
||
"skipped_missing_identity": 0,
|
||
"skipped_empty_content": 0,
|
||
"skipped_duplicate": 0,
|
||
"appids": [],
|
||
}
|
||
posts: list[Any] = []
|
||
seen_posts: set[tuple[int, str]] = set()
|
||
appids: set[str] = set()
|
||
for feed in feeds:
|
||
html_content = str(_first_value(feed, "html", "feedhtml", "feed_html", "html_content") or "")
|
||
common = _structured_feed_common(feed)
|
||
uin, tid, name = _structured_feed_identity(feed, html_content)
|
||
if not uin or not tid:
|
||
stats["skipped_missing_identity"] += 1
|
||
continue
|
||
post_key = (uin, tid)
|
||
if post_key in seen_posts:
|
||
stats["skipped_duplicate"] += 1
|
||
continue
|
||
text, rt_con = _structured_feed_text(feed, html_content)
|
||
image_urls, image_items = _structured_feed_images(feed, html_content)
|
||
if not (text or rt_con or image_urls):
|
||
stats["skipped_empty_content"] += 1
|
||
continue
|
||
seen_posts.add(post_key)
|
||
appid = str(_first_value(feed, "appid") or _first_value(common, "appid") or "311")
|
||
typeid = str(
|
||
_first_value(feed, "typeid", "type", "feedstype")
|
||
or _first_value(common, "typeid", "type", "feedstype")
|
||
or "0"
|
||
)
|
||
abstime = _safe_int(
|
||
_first_value(feed, "abstime", "created_time", "create_time", "timestamp", "pubtime", "time")
|
||
or _first_value(common, "abstime", "created_time", "create_time", "timestamp", "time")
|
||
)
|
||
fid = str(
|
||
_first_value(feed, "fid", "key", "tid")
|
||
or _first_value(_first_value(feed, "id"), "cellid", "fid", "tid")
|
||
or _first_value(common, "ugcrightkey")
|
||
or tid
|
||
)
|
||
unikey = str(
|
||
_first_value(feed, "unikey", "likekey", "like_key", "orglikekey")
|
||
or _first_value(common, "unikey", "likekey", "like_key", "orglikekey")
|
||
or _html_attr_value(html_content, "data-unikey", "unikey")
|
||
or ""
|
||
)
|
||
curkey = str(
|
||
_first_value(feed, "curkey", "curlikekey", "likekey", "like_key")
|
||
or _first_value(common, "curkey", "curlikekey", "likekey", "like_key")
|
||
or _html_attr_value(html_content, "data-curkey", "curkey")
|
||
or ""
|
||
)
|
||
like_info = _first_value(feed, "like")
|
||
liked = bool(_first_value(like_info, "isliked", "liked")) if isinstance(like_info, dict) else False
|
||
appids.add(appid)
|
||
posts.append(
|
||
SimpleNamespace(
|
||
tid=tid,
|
||
uin=uin,
|
||
name=name,
|
||
text=text,
|
||
rt_con=rt_con,
|
||
images=image_urls,
|
||
image_items=image_items,
|
||
comments=[],
|
||
create_time=abstime,
|
||
appid=appid,
|
||
typeid=typeid,
|
||
abstime=abstime,
|
||
fid=fid,
|
||
unikey=unikey,
|
||
curkey=curkey,
|
||
liked=liked,
|
||
raw=feed,
|
||
status="approved",
|
||
)
|
||
)
|
||
stats["parsed_count"] = len(posts)
|
||
stats["appids"] = sorted(appids)[:16]
|
||
if diagnostics is not None:
|
||
diagnostics.clear()
|
||
diagnostics.update(stats)
|
||
return posts
|