575 行
20 KiB
Python
575 行
20 KiB
Python
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import re
|
|||
|
|
from html import unescape
|
|||
|
|
from html.parser import HTMLParser
|
|||
|
|
from types import SimpleNamespace
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
|
|||
|
|
class _QzoneFeedHtmlParser(HTMLParser):
|
|||
|
|
_VOID_TAGS = {
|
|||
|
|
"area",
|
|||
|
|
"base",
|
|||
|
|
"br",
|
|||
|
|
"col",
|
|||
|
|
"embed",
|
|||
|
|
"hr",
|
|||
|
|
"img",
|
|||
|
|
"input",
|
|||
|
|
"link",
|
|||
|
|
"meta",
|
|||
|
|
"param",
|
|||
|
|
"source",
|
|||
|
|
"track",
|
|||
|
|
"wbr",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def __init__(self) -> None:
|
|||
|
|
super().__init__(convert_charrefs=True)
|
|||
|
|
self.text_parts: list[str] = []
|
|||
|
|
self.repost_parts: list[str] = []
|
|||
|
|
self.image_urls: list[str] = []
|
|||
|
|
self.image_items: list[dict[str, str]] = []
|
|||
|
|
self._class_stack: list[set[str]] = []
|
|||
|
|
self._tag_stack: list[str] = []
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _classes(attrs: list[tuple[str, str | None]]) -> set[str]:
|
|||
|
|
for key, value in attrs:
|
|||
|
|
if str(key or "").lower() == "class":
|
|||
|
|
return {item.strip() for item in str(value or "").split() if item.strip()}
|
|||
|
|
return set()
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _attr(attrs: list[tuple[str, str | None]], name: str) -> str:
|
|||
|
|
lowered = str(name or "").lower()
|
|||
|
|
for key, value in attrs:
|
|||
|
|
if str(key or "").lower() == lowered:
|
|||
|
|
return str(value or "")
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def _first_attr(cls, attrs: list[tuple[str, str | None]], *names: str) -> str:
|
|||
|
|
for name in names:
|
|||
|
|
value = cls._attr(attrs, name).strip()
|
|||
|
|
if value:
|
|||
|
|
return value
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _is_content_image_url(value: str) -> bool:
|
|||
|
|
source = str(value or "").strip().lower()
|
|||
|
|
return bool(source) and not source.startswith("http://qzonestyle.gtimg.cn")
|
|||
|
|
|
|||
|
|
def _record_image(self, attrs: list[tuple[str, str | None]]) -> None:
|
|||
|
|
preview_url = self._first_attr(attrs, "src", "data-src", "data-lazy-src", "data-original-src")
|
|||
|
|
full_url = self._first_attr(
|
|||
|
|
attrs,
|
|||
|
|
"data-original",
|
|||
|
|
"data-origin",
|
|||
|
|
"data-origin-url",
|
|||
|
|
"data-original-url",
|
|||
|
|
"data-originalurl",
|
|||
|
|
"data-big-url",
|
|||
|
|
"data-bigurl",
|
|||
|
|
"data-full-url",
|
|||
|
|
"data-fullurl",
|
|||
|
|
"data-raw-url",
|
|||
|
|
"data-rawurl",
|
|||
|
|
"origsrc",
|
|||
|
|
"original",
|
|||
|
|
"data-url",
|
|||
|
|
)
|
|||
|
|
if not self._is_content_image_url(full_url):
|
|||
|
|
full_url = ""
|
|||
|
|
if not self._is_content_image_url(preview_url):
|
|||
|
|
preview_url = ""
|
|||
|
|
source = full_url or preview_url
|
|||
|
|
if not source:
|
|||
|
|
return
|
|||
|
|
item = {"preview_url": preview_url or source, "full_url": source}
|
|||
|
|
if item not in self.image_items:
|
|||
|
|
self.image_items.append(item)
|
|||
|
|
if source not in self.image_urls:
|
|||
|
|
self.image_urls.append(source)
|
|||
|
|
|
|||
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|||
|
|
normalized_tag = str(tag or "").lower()
|
|||
|
|
classes = self._classes(attrs)
|
|||
|
|
if normalized_tag != "img":
|
|||
|
|
if normalized_tag not in self._VOID_TAGS:
|
|||
|
|
self._tag_stack.append(normalized_tag)
|
|||
|
|
self._class_stack.append(classes)
|
|||
|
|
elif classes:
|
|||
|
|
self._class_stack.append(classes)
|
|||
|
|
self._class_stack.pop()
|
|||
|
|
return
|
|||
|
|
active_classes = self._class_stack + [classes]
|
|||
|
|
if any({"img-box", "video-img"} & item for item in active_classes):
|
|||
|
|
self._record_image(attrs)
|
|||
|
|
|
|||
|
|
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|||
|
|
normalized_tag = str(tag or "").lower()
|
|||
|
|
if normalized_tag == "img":
|
|||
|
|
self.handle_starttag(tag, attrs)
|
|||
|
|
return
|
|||
|
|
classes = self._classes(attrs)
|
|||
|
|
if classes:
|
|||
|
|
self._class_stack.append(classes)
|
|||
|
|
self._class_stack.pop()
|
|||
|
|
|
|||
|
|
def handle_endtag(self, tag: str) -> None:
|
|||
|
|
normalized_tag = str(tag or "").lower()
|
|||
|
|
for index in range(len(self._tag_stack) - 1, -1, -1):
|
|||
|
|
if self._tag_stack[index] == normalized_tag:
|
|||
|
|
del self._tag_stack[index:]
|
|||
|
|
del self._class_stack[index:]
|
|||
|
|
return
|
|||
|
|
if self._tag_stack and self._class_stack:
|
|||
|
|
self._tag_stack.pop()
|
|||
|
|
self._class_stack.pop()
|
|||
|
|
|
|||
|
|
def handle_data(self, data: str) -> None:
|
|||
|
|
text = unescape(str(data or "")).strip()
|
|||
|
|
if not text:
|
|||
|
|
return
|
|||
|
|
if any("f-info" in item for item in self._class_stack):
|
|||
|
|
self.text_parts.append(text)
|
|||
|
|
if any("txt-box" in item for item in self._class_stack):
|
|||
|
|
self.repost_parts.append(text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _parse_feed_html(html_content: str) -> tuple[str, str, list[str]]:
|
|||
|
|
text, repost, image_urls, _items = _parse_feed_html_details(html_content)
|
|||
|
|
return text, repost, image_urls
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _parse_feed_html_details(html_content: str) -> tuple[str, str, list[str], list[dict[str, str]]]:
|
|||
|
|
parser = _QzoneFeedHtmlParser()
|
|||
|
|
parser.feed(str(html_content or ""))
|
|||
|
|
text = "".join(parser.text_parts).strip()
|
|||
|
|
repost = "".join(parser.repost_parts).strip()
|
|||
|
|
if ":" in repost:
|
|||
|
|
repost = repost.split(":", 1)[1].strip()
|
|||
|
|
return text, repost, parser.image_urls, parser.image_items
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _html_attr_value(html_content: str, *names: str) -> str:
|
|||
|
|
source = str(html_content or "")
|
|||
|
|
for name in names:
|
|||
|
|
escaped = re.escape(str(name or ""))
|
|||
|
|
match = re.search(rf"""{escaped}\s*=\s*["']([^"']+)["']""", source, flags=re.IGNORECASE)
|
|||
|
|
if match:
|
|||
|
|
return unescape(match.group(1)).strip()
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _normalized_dict(value: Any) -> dict[str, Any]:
|
|||
|
|
if not isinstance(value, dict):
|
|||
|
|
return {}
|
|||
|
|
return {str(key or "").strip().lower().replace("-", "_"): item for key, item in value.items()}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _first_value(source: Any, *keys: str) -> Any:
|
|||
|
|
normalized = _normalized_dict(source)
|
|||
|
|
for key in keys:
|
|||
|
|
value = normalized.get(str(key or "").lower().replace("-", "_"))
|
|||
|
|
if value not in (None, ""):
|
|||
|
|
return value
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _safe_int(value: Any) -> int:
|
|||
|
|
text = str(value or "").strip().lstrip("oO")
|
|||
|
|
try:
|
|||
|
|
return int(float(text))
|
|||
|
|
except (TypeError, ValueError):
|
|||
|
|
return 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _clean_plain_text(value: Any) -> str:
|
|||
|
|
source = str(value or "").strip()
|
|||
|
|
if not source:
|
|||
|
|
return ""
|
|||
|
|
source = re.sub(r"<br\s*/?>", "\n", source, flags=re.IGNORECASE)
|
|||
|
|
source = re.sub(r"<[^>]+>", "", source)
|
|||
|
|
source = unescape(source)
|
|||
|
|
return re.sub(r"[ \t\r\f\v]+", " ", source).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _looks_like_feed(value: Any) -> bool:
|
|||
|
|
if not isinstance(value, dict) or not value:
|
|||
|
|
return False
|
|||
|
|
normalized = _normalized_dict(value)
|
|||
|
|
if isinstance(normalized.get("comm"), dict) and isinstance(normalized.get("userinfo"), dict):
|
|||
|
|
return True
|
|||
|
|
def present(item: Any) -> bool:
|
|||
|
|
if isinstance(item, (dict, list, tuple, set)):
|
|||
|
|
return bool(item)
|
|||
|
|
return item not in (None, "")
|
|||
|
|
|
|||
|
|
has_content = any(
|
|||
|
|
present(normalized.get(key))
|
|||
|
|
for key in ("html", "feedhtml", "feed_html", "content", "text", "summary", "con")
|
|||
|
|
)
|
|||
|
|
has_identity = any(
|
|||
|
|
present(normalized.get(key))
|
|||
|
|
for key in ("uin", "hostuin", "opuin", "user_uin", "useruin", "owner_uin", "owneruin", "userinfo")
|
|||
|
|
)
|
|||
|
|
has_id = any(
|
|||
|
|
present(normalized.get(key))
|
|||
|
|
for key in ("key", "tid", "fid", "topicid", "topic_id", "feedid", "feed_id", "cellid", "id")
|
|||
|
|
)
|
|||
|
|
return has_content and has_identity and has_id
|
|||
|
|
|
|||
|
|
|
|||
|
|
_SKIPPED_FEED_BRANCHES = {
|
|||
|
|
"comment",
|
|||
|
|
"comments",
|
|||
|
|
"commentlist",
|
|||
|
|
"comment_list",
|
|||
|
|
"reply",
|
|||
|
|
"replys",
|
|||
|
|
"replies",
|
|||
|
|
"likemans",
|
|||
|
|
"like_list",
|
|||
|
|
"userinfo",
|
|||
|
|
"user",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _collect_feed_candidates(data: Any) -> tuple[list[dict[str, Any]], list[str]]:
|
|||
|
|
candidates: list[dict[str, Any]] = []
|
|||
|
|
containers: list[str] = []
|
|||
|
|
seen: set[int] = set()
|
|||
|
|
|
|||
|
|
def walk(value: Any, path: str, depth: int) -> None:
|
|||
|
|
if depth > 10:
|
|||
|
|
return
|
|||
|
|
if isinstance(value, list):
|
|||
|
|
if value and path:
|
|||
|
|
containers.append(path)
|
|||
|
|
for index, item in enumerate(value):
|
|||
|
|
walk(item, f"{path}[{index}]" if path else f"[{index}]", depth + 1)
|
|||
|
|
return
|
|||
|
|
if not isinstance(value, dict):
|
|||
|
|
return
|
|||
|
|
object_id = id(value)
|
|||
|
|
if object_id in seen:
|
|||
|
|
return
|
|||
|
|
seen.add(object_id)
|
|||
|
|
if _looks_like_feed(value):
|
|||
|
|
candidates.append(value)
|
|||
|
|
return
|
|||
|
|
for key, item in value.items():
|
|||
|
|
normalized_key = str(key or "").strip().lower().replace("-", "_")
|
|||
|
|
if normalized_key in _SKIPPED_FEED_BRANCHES:
|
|||
|
|
continue
|
|||
|
|
if isinstance(item, (dict, list)):
|
|||
|
|
child_path = f"{path}.{normalized_key}" if path else normalized_key
|
|||
|
|
walk(item, child_path, depth + 1)
|
|||
|
|
|
|||
|
|
walk(data, "", 0)
|
|||
|
|
return candidates, list(dict.fromkeys(containers))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _structured_feed_identity(feed: dict[str, Any], html_content: str) -> tuple[int, str, str]:
|
|||
|
|
userinfo = _first_value(feed, "userinfo", "user", "author", "owner")
|
|||
|
|
if not isinstance(userinfo, dict):
|
|||
|
|
userinfo = {}
|
|||
|
|
id_info = _first_value(feed, "id", "identity")
|
|||
|
|
if not isinstance(id_info, dict):
|
|||
|
|
id_info = {}
|
|||
|
|
uin = _safe_int(
|
|||
|
|
_first_value(feed, "uin", "hostuin", "opuin", "user_uin", "useruin", "owner_uin", "owneruin")
|
|||
|
|
or _first_value(userinfo, "uin", "qq", "id")
|
|||
|
|
or _html_attr_value(html_content, "data-uin", "data-hostuin", "data-opuin", "uin")
|
|||
|
|
)
|
|||
|
|
tid = str(
|
|||
|
|
_first_value(feed, "key", "tid", "fid", "topicid", "topic_id", "feedid", "feed_id", "cellid")
|
|||
|
|
or _first_value(id_info, "cellid", "tid", "fid", "key", "id")
|
|||
|
|
or _html_attr_value(
|
|||
|
|
html_content,
|
|||
|
|
"data-key",
|
|||
|
|
"data-tid",
|
|||
|
|
"data-fid",
|
|||
|
|
"data-topicid",
|
|||
|
|
"data-feedid",
|
|||
|
|
)
|
|||
|
|
or ""
|
|||
|
|
).strip()
|
|||
|
|
name = str(
|
|||
|
|
_first_value(feed, "nickname", "name", "nick")
|
|||
|
|
or _first_value(userinfo, "nickname", "name", "nick")
|
|||
|
|
or _html_attr_value(html_content, "data-nickname", "data-name", "data-nick")
|
|||
|
|
or uin
|
|||
|
|
or "QQ空间用户"
|
|||
|
|
).strip()
|
|||
|
|
return uin, tid, name
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _structured_feed_common(feed: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
common = _first_value(feed, "comm", "common", "cell_comm")
|
|||
|
|
return common if isinstance(common, dict) else {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _structured_feed_text(feed: dict[str, Any], html_content: str) -> tuple[str, str]:
|
|||
|
|
summary = _first_value(feed, "summary")
|
|||
|
|
if isinstance(summary, dict):
|
|||
|
|
text = _clean_plain_text(_first_value(summary, "summary", "content", "text"))
|
|||
|
|
else:
|
|||
|
|
text = _clean_plain_text(summary)
|
|||
|
|
text = text or _clean_plain_text(_first_value(feed, "content", "text", "con"))
|
|||
|
|
|
|||
|
|
original = _first_value(feed, "original", "forward", "repost", "rt_con")
|
|||
|
|
rt_con = ""
|
|||
|
|
if isinstance(original, dict):
|
|||
|
|
original_summary = _first_value(original, "summary", "content", "text", "con")
|
|||
|
|
if isinstance(original_summary, dict):
|
|||
|
|
original_summary = _first_value(original_summary, "summary", "content", "text")
|
|||
|
|
rt_con = _clean_plain_text(original_summary)
|
|||
|
|
else:
|
|||
|
|
rt_con = _clean_plain_text(original)
|
|||
|
|
|
|||
|
|
if html_content:
|
|||
|
|
html_text, html_repost, _urls, _items = _parse_feed_html_details(html_content)
|
|||
|
|
text = text or html_text or _clean_plain_text(html_content)
|
|||
|
|
rt_con = rt_con or html_repost
|
|||
|
|
if not text:
|
|||
|
|
operation = _first_value(feed, "operation")
|
|||
|
|
share_info = _first_value(operation, "share_info", "shareinfo") if isinstance(operation, dict) else None
|
|||
|
|
if isinstance(share_info, dict):
|
|||
|
|
text = _clean_plain_text(_first_value(share_info, "summary", "title"))
|
|||
|
|
return text, rt_con
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _valid_image_url(value: Any) -> str:
|
|||
|
|
source = unescape(str(value or "").strip())
|
|||
|
|
return source if source.lower().startswith(("https://", "http://", "//")) else ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _photo_variant_item(value: Any) -> dict[str, str] | None:
|
|||
|
|
variants: list[tuple[int, str]] = []
|
|||
|
|
|
|||
|
|
def collect(item: Any, depth: int = 0) -> None:
|
|||
|
|
if depth > 5:
|
|||
|
|
return
|
|||
|
|
if isinstance(item, list):
|
|||
|
|
for child in item:
|
|||
|
|
collect(child, depth + 1)
|
|||
|
|
return
|
|||
|
|
if not isinstance(item, dict):
|
|||
|
|
return
|
|||
|
|
direct = _valid_image_url(
|
|||
|
|
_first_value(
|
|||
|
|
item,
|
|||
|
|
"origin_url",
|
|||
|
|
"original_url",
|
|||
|
|
"raw_url",
|
|||
|
|
"large_url",
|
|||
|
|
"big_url",
|
|||
|
|
"url3",
|
|||
|
|
"url2",
|
|||
|
|
"url1",
|
|||
|
|
"smallurl",
|
|||
|
|
"pic_url",
|
|||
|
|
"coverurl",
|
|||
|
|
"url",
|
|||
|
|
"src",
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
if direct:
|
|||
|
|
width = _safe_int(_first_value(item, "width", "w"))
|
|||
|
|
height = _safe_int(_first_value(item, "height", "h"))
|
|||
|
|
variants.append((max(1, width * height), direct))
|
|||
|
|
for child in item.values():
|
|||
|
|
if isinstance(child, (dict, list)):
|
|||
|
|
collect(child, depth + 1)
|
|||
|
|
|
|||
|
|
collect(value)
|
|||
|
|
if not variants:
|
|||
|
|
return None
|
|||
|
|
variants = list(dict.fromkeys(variants))
|
|||
|
|
preview = min(variants, key=lambda item: item[0])[1]
|
|||
|
|
full = max(variants, key=lambda item: item[0])[1]
|
|||
|
|
return {"preview_url": preview, "full_url": full}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _structured_feed_images(feed: dict[str, Any], html_content: str) -> tuple[list[str], list[dict[str, str]]]:
|
|||
|
|
items: list[dict[str, str]] = []
|
|||
|
|
for key in ("pic", "pics", "images", "photos", "video"):
|
|||
|
|
branch = _first_value(feed, key)
|
|||
|
|
if branch in (None, ""):
|
|||
|
|
continue
|
|||
|
|
if isinstance(branch, dict) and isinstance(_first_value(branch, "picdata", "pic_data"), list):
|
|||
|
|
sources = list(_first_value(branch, "picdata", "pic_data") or [])
|
|||
|
|
elif isinstance(branch, list):
|
|||
|
|
sources = branch
|
|||
|
|
else:
|
|||
|
|
sources = [branch]
|
|||
|
|
for source in sources:
|
|||
|
|
item = _photo_variant_item(source)
|
|||
|
|
if item and item not in items:
|
|||
|
|
items.append(item)
|
|||
|
|
if html_content:
|
|||
|
|
_text, _repost, _urls, html_items = _parse_feed_html_details(html_content)
|
|||
|
|
for item in html_items:
|
|||
|
|
if item not in items:
|
|||
|
|
items.append(item)
|
|||
|
|
return [item["full_url"] for item in items], items
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _balanced_object(source: str, start: int) -> str:
|
|||
|
|
if start < 0 or start >= len(source) or source[start] != "{":
|
|||
|
|
return ""
|
|||
|
|
depth = 0
|
|||
|
|
quote = ""
|
|||
|
|
escaped = False
|
|||
|
|
for index in range(start, len(source)):
|
|||
|
|
char = source[index]
|
|||
|
|
if quote:
|
|||
|
|
if escaped:
|
|||
|
|
escaped = False
|
|||
|
|
elif char == "\\":
|
|||
|
|
escaped = True
|
|||
|
|
elif char == quote:
|
|||
|
|
quote = ""
|
|||
|
|
continue
|
|||
|
|
if char in {"'", '"'}:
|
|||
|
|
quote = char
|
|||
|
|
elif char == "{":
|
|||
|
|
depth += 1
|
|||
|
|
elif char == "}":
|
|||
|
|
depth -= 1
|
|||
|
|
if depth == 0:
|
|||
|
|
return source[start : index + 1]
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_qzone_h5_index_html(html_content: str) -> dict[str, Any]:
|
|||
|
|
source = str(html_content or "")
|
|||
|
|
token_match = re.search(
|
|||
|
|
r"window\.shine0callback.*?return\s+[\"']([0-9a-zA-Z_-]+?)[\"']\s*;",
|
|||
|
|
source,
|
|||
|
|
flags=re.DOTALL,
|
|||
|
|
)
|
|||
|
|
token = token_match.group(1).strip() if token_match else ""
|
|||
|
|
data_match = re.search(r"var\s+FrontPage\s*=.*?\bdata\s*:\s*\{", source, flags=re.DOTALL)
|
|||
|
|
object_text = _balanced_object(source, data_match.end() - 1) if data_match else ""
|
|||
|
|
payload: dict[str, Any] = {}
|
|||
|
|
if object_text:
|
|||
|
|
normalized = object_text.replace("undefined", "null")
|
|||
|
|
try:
|
|||
|
|
parsed = json.loads(normalized)
|
|||
|
|
except Exception:
|
|||
|
|
try:
|
|||
|
|
relaxed = re.sub(r",\s*([}\]])", r"\1", normalized)
|
|||
|
|
relaxed = re.sub(r"([{,]\s*)([A-Za-z_$][\w$]*)\s*:", r'\1"\2":', relaxed)
|
|||
|
|
parsed = json.loads(relaxed)
|
|||
|
|
except Exception:
|
|||
|
|
try:
|
|||
|
|
import json5 # type: ignore
|
|||
|
|
|
|||
|
|
parsed = json5.loads(normalized)
|
|||
|
|
except Exception:
|
|||
|
|
parsed = {}
|
|||
|
|
if isinstance(parsed, dict):
|
|||
|
|
payload = parsed
|
|||
|
|
return {"token": token, "payload": payload}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_recent_feeds(data: dict[str, Any], diagnostics: dict[str, Any] | None = None) -> list[Any]:
|
|||
|
|
feeds, containers = _collect_feed_candidates(data)
|
|||
|
|
stats: dict[str, Any] = {
|
|||
|
|
"response_keys": sorted(str(key) for key in data)[:24] if isinstance(data, dict) else [],
|
|||
|
|
"containers": containers[:12],
|
|||
|
|
"candidate_count": len(feeds),
|
|||
|
|
"parsed_count": 0,
|
|||
|
|
"skipped_missing_identity": 0,
|
|||
|
|
"skipped_empty_content": 0,
|
|||
|
|
"skipped_duplicate": 0,
|
|||
|
|
"appids": [],
|
|||
|
|
}
|
|||
|
|
posts: list[Any] = []
|
|||
|
|
seen_posts: set[tuple[int, str]] = set()
|
|||
|
|
appids: set[str] = set()
|
|||
|
|
for feed in feeds:
|
|||
|
|
html_content = str(_first_value(feed, "html", "feedhtml", "feed_html", "html_content") or "")
|
|||
|
|
common = _structured_feed_common(feed)
|
|||
|
|
uin, tid, name = _structured_feed_identity(feed, html_content)
|
|||
|
|
if not uin or not tid:
|
|||
|
|
stats["skipped_missing_identity"] += 1
|
|||
|
|
continue
|
|||
|
|
post_key = (uin, tid)
|
|||
|
|
if post_key in seen_posts:
|
|||
|
|
stats["skipped_duplicate"] += 1
|
|||
|
|
continue
|
|||
|
|
text, rt_con = _structured_feed_text(feed, html_content)
|
|||
|
|
image_urls, image_items = _structured_feed_images(feed, html_content)
|
|||
|
|
if not (text or rt_con or image_urls):
|
|||
|
|
stats["skipped_empty_content"] += 1
|
|||
|
|
continue
|
|||
|
|
seen_posts.add(post_key)
|
|||
|
|
appid = str(_first_value(feed, "appid") or _first_value(common, "appid") or "311")
|
|||
|
|
typeid = str(
|
|||
|
|
_first_value(feed, "typeid", "type", "feedstype")
|
|||
|
|
or _first_value(common, "typeid", "type", "feedstype")
|
|||
|
|
or "0"
|
|||
|
|
)
|
|||
|
|
abstime = _safe_int(
|
|||
|
|
_first_value(feed, "abstime", "created_time", "create_time", "timestamp", "pubtime", "time")
|
|||
|
|
or _first_value(common, "abstime", "created_time", "create_time", "timestamp", "time")
|
|||
|
|
)
|
|||
|
|
fid = str(
|
|||
|
|
_first_value(feed, "fid", "key", "tid")
|
|||
|
|
or _first_value(_first_value(feed, "id"), "cellid", "fid", "tid")
|
|||
|
|
or _first_value(common, "ugcrightkey")
|
|||
|
|
or tid
|
|||
|
|
)
|
|||
|
|
unikey = str(
|
|||
|
|
_first_value(feed, "unikey", "likekey", "like_key", "orglikekey")
|
|||
|
|
or _first_value(common, "unikey", "likekey", "like_key", "orglikekey")
|
|||
|
|
or _html_attr_value(html_content, "data-unikey", "unikey")
|
|||
|
|
or ""
|
|||
|
|
)
|
|||
|
|
curkey = str(
|
|||
|
|
_first_value(feed, "curkey", "curlikekey", "likekey", "like_key")
|
|||
|
|
or _first_value(common, "curkey", "curlikekey", "likekey", "like_key")
|
|||
|
|
or _html_attr_value(html_content, "data-curkey", "curkey")
|
|||
|
|
or ""
|
|||
|
|
)
|
|||
|
|
like_info = _first_value(feed, "like")
|
|||
|
|
liked = bool(_first_value(like_info, "isliked", "liked")) if isinstance(like_info, dict) else False
|
|||
|
|
appids.add(appid)
|
|||
|
|
posts.append(
|
|||
|
|
SimpleNamespace(
|
|||
|
|
tid=tid,
|
|||
|
|
uin=uin,
|
|||
|
|
name=name,
|
|||
|
|
text=text,
|
|||
|
|
rt_con=rt_con,
|
|||
|
|
images=image_urls,
|
|||
|
|
image_items=image_items,
|
|||
|
|
comments=[],
|
|||
|
|
create_time=abstime,
|
|||
|
|
appid=appid,
|
|||
|
|
typeid=typeid,
|
|||
|
|
abstime=abstime,
|
|||
|
|
fid=fid,
|
|||
|
|
unikey=unikey,
|
|||
|
|
curkey=curkey,
|
|||
|
|
liked=liked,
|
|||
|
|
raw=feed,
|
|||
|
|
status="approved",
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
stats["parsed_count"] = len(posts)
|
|||
|
|
stats["appids"] = sorted(appids)[:16]
|
|||
|
|
if diagnostics is not None:
|
|||
|
|
diagnostics.clear()
|
|||
|
|
diagnostics.update(stats)
|
|||
|
|
return posts
|