# -*- coding: utf-8 -*- from __future__ import annotations import json import re from html import unescape from html.parser import HTMLParser from types import SimpleNamespace from typing import Any class _QzoneFeedHtmlParser(HTMLParser): _VOID_TAGS = { "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr", } def __init__(self) -> None: super().__init__(convert_charrefs=True) self.text_parts: list[str] = [] self.repost_parts: list[str] = [] self.image_urls: list[str] = [] self.image_items: list[dict[str, str]] = [] self._class_stack: list[set[str]] = [] self._tag_stack: list[str] = [] @staticmethod def _classes(attrs: list[tuple[str, str | None]]) -> set[str]: for key, value in attrs: if str(key or "").lower() == "class": return {item.strip() for item in str(value or "").split() if item.strip()} return set() @staticmethod def _attr(attrs: list[tuple[str, str | None]], name: str) -> str: lowered = str(name or "").lower() for key, value in attrs: if str(key or "").lower() == lowered: return str(value or "") return "" @classmethod def _first_attr(cls, attrs: list[tuple[str, str | None]], *names: str) -> str: for name in names: value = cls._attr(attrs, name).strip() if value: return value return "" @staticmethod def _is_content_image_url(value: str) -> bool: source = str(value or "").strip().lower() return bool(source) and not source.startswith("http://qzonestyle.gtimg.cn") def _record_image(self, attrs: list[tuple[str, str | None]]) -> None: preview_url = self._first_attr(attrs, "src", "data-src", "data-lazy-src", "data-original-src") full_url = self._first_attr( attrs, "data-original", "data-origin", "data-origin-url", "data-original-url", "data-originalurl", "data-big-url", "data-bigurl", "data-full-url", "data-fullurl", "data-raw-url", "data-rawurl", "origsrc", "original", "data-url", ) if not self._is_content_image_url(full_url): full_url = "" if not self._is_content_image_url(preview_url): preview_url = "" source = full_url or preview_url if not source: return item = {"preview_url": preview_url or source, "full_url": source} if item not in self.image_items: self.image_items.append(item) if source not in self.image_urls: self.image_urls.append(source) def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: normalized_tag = str(tag or "").lower() classes = self._classes(attrs) if normalized_tag != "img": if normalized_tag not in self._VOID_TAGS: self._tag_stack.append(normalized_tag) self._class_stack.append(classes) elif classes: self._class_stack.append(classes) self._class_stack.pop() return active_classes = self._class_stack + [classes] if any({"img-box", "video-img"} & item for item in active_classes): self._record_image(attrs) def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: normalized_tag = str(tag or "").lower() if normalized_tag == "img": self.handle_starttag(tag, attrs) return classes = self._classes(attrs) if classes: self._class_stack.append(classes) self._class_stack.pop() def handle_endtag(self, tag: str) -> None: normalized_tag = str(tag or "").lower() for index in range(len(self._tag_stack) - 1, -1, -1): if self._tag_stack[index] == normalized_tag: del self._tag_stack[index:] del self._class_stack[index:] return if self._tag_stack and self._class_stack: self._tag_stack.pop() self._class_stack.pop() def handle_data(self, data: str) -> None: text = unescape(str(data or "")).strip() if not text: return if any("f-info" in item for item in self._class_stack): self.text_parts.append(text) if any("txt-box" in item for item in self._class_stack): self.repost_parts.append(text) def _parse_feed_html(html_content: str) -> tuple[str, str, list[str]]: text, repost, image_urls, _items = _parse_feed_html_details(html_content) return text, repost, image_urls def _parse_feed_html_details(html_content: str) -> tuple[str, str, list[str], list[dict[str, str]]]: parser = _QzoneFeedHtmlParser() parser.feed(str(html_content or "")) text = "".join(parser.text_parts).strip() repost = "".join(parser.repost_parts).strip() if ":" in repost: repost = repost.split(":", 1)[1].strip() return text, repost, parser.image_urls, parser.image_items def _html_attr_value(html_content: str, *names: str) -> str: source = str(html_content or "") for name in names: escaped = re.escape(str(name or "")) match = re.search(rf"""{escaped}\s*=\s*["']([^"']+)["']""", source, flags=re.IGNORECASE) if match: return unescape(match.group(1)).strip() return "" def _normalized_dict(value: Any) -> dict[str, Any]: if not isinstance(value, dict): return {} return {str(key or "").strip().lower().replace("-", "_"): item for key, item in value.items()} def _first_value(source: Any, *keys: str) -> Any: normalized = _normalized_dict(source) for key in keys: value = normalized.get(str(key or "").lower().replace("-", "_")) if value not in (None, ""): return value return None def _safe_int(value: Any) -> int: text = str(value or "").strip().lstrip("oO") try: return int(float(text)) except (TypeError, ValueError): return 0 def _clean_plain_text(value: Any) -> str: source = str(value or "").strip() if not source: return "" source = re.sub(r"", "\n", source, flags=re.IGNORECASE) source = re.sub(r"<[^>]+>", "", source) source = unescape(source) return re.sub(r"[ \t\r\f\v]+", " ", source).strip() def _looks_like_feed(value: Any) -> bool: if not isinstance(value, dict) or not value: return False normalized = _normalized_dict(value) if isinstance(normalized.get("comm"), dict) and isinstance(normalized.get("userinfo"), dict): return True def present(item: Any) -> bool: if isinstance(item, (dict, list, tuple, set)): return bool(item) return item not in (None, "") has_content = any( present(normalized.get(key)) for key in ("html", "feedhtml", "feed_html", "content", "text", "summary", "con") ) has_identity = any( present(normalized.get(key)) for key in ("uin", "hostuin", "opuin", "user_uin", "useruin", "owner_uin", "owneruin", "userinfo") ) has_id = any( present(normalized.get(key)) for key in ("key", "tid", "fid", "topicid", "topic_id", "feedid", "feed_id", "cellid", "id") ) return has_content and has_identity and has_id _SKIPPED_FEED_BRANCHES = { "comment", "comments", "commentlist", "comment_list", "reply", "replys", "replies", "likemans", "like_list", "userinfo", "user", } def _collect_feed_candidates(data: Any) -> tuple[list[dict[str, Any]], list[str]]: candidates: list[dict[str, Any]] = [] containers: list[str] = [] seen: set[int] = set() def walk(value: Any, path: str, depth: int) -> None: if depth > 10: return if isinstance(value, list): if value and path: containers.append(path) for index, item in enumerate(value): walk(item, f"{path}[{index}]" if path else f"[{index}]", depth + 1) return if not isinstance(value, dict): return object_id = id(value) if object_id in seen: return seen.add(object_id) if _looks_like_feed(value): candidates.append(value) return for key, item in value.items(): normalized_key = str(key or "").strip().lower().replace("-", "_") if normalized_key in _SKIPPED_FEED_BRANCHES: continue if isinstance(item, (dict, list)): child_path = f"{path}.{normalized_key}" if path else normalized_key walk(item, child_path, depth + 1) walk(data, "", 0) return candidates, list(dict.fromkeys(containers)) def _structured_feed_identity(feed: dict[str, Any], html_content: str) -> tuple[int, str, str]: userinfo = _first_value(feed, "userinfo", "user", "author", "owner") if not isinstance(userinfo, dict): userinfo = {} id_info = _first_value(feed, "id", "identity") if not isinstance(id_info, dict): id_info = {} uin = _safe_int( _first_value(feed, "uin", "hostuin", "opuin", "user_uin", "useruin", "owner_uin", "owneruin") or _first_value(userinfo, "uin", "qq", "id") or _html_attr_value(html_content, "data-uin", "data-hostuin", "data-opuin", "uin") ) tid = str( _first_value(feed, "key", "tid", "fid", "topicid", "topic_id", "feedid", "feed_id", "cellid") or _first_value(id_info, "cellid", "tid", "fid", "key", "id") or _html_attr_value( html_content, "data-key", "data-tid", "data-fid", "data-topicid", "data-feedid", ) or "" ).strip() name = str( _first_value(feed, "nickname", "name", "nick") or _first_value(userinfo, "nickname", "name", "nick") or _html_attr_value(html_content, "data-nickname", "data-name", "data-nick") or uin or "QQ空间用户" ).strip() return uin, tid, name def _structured_feed_common(feed: dict[str, Any]) -> dict[str, Any]: common = _first_value(feed, "comm", "common", "cell_comm") return common if isinstance(common, dict) else {} def _structured_feed_text(feed: dict[str, Any], html_content: str) -> tuple[str, str]: summary = _first_value(feed, "summary") if isinstance(summary, dict): text = _clean_plain_text(_first_value(summary, "summary", "content", "text")) else: text = _clean_plain_text(summary) text = text or _clean_plain_text(_first_value(feed, "content", "text", "con")) original = _first_value(feed, "original", "forward", "repost", "rt_con") rt_con = "" if isinstance(original, dict): original_summary = _first_value(original, "summary", "content", "text", "con") if isinstance(original_summary, dict): original_summary = _first_value(original_summary, "summary", "content", "text") rt_con = _clean_plain_text(original_summary) else: rt_con = _clean_plain_text(original) if html_content: html_text, html_repost, _urls, _items = _parse_feed_html_details(html_content) text = text or html_text or _clean_plain_text(html_content) rt_con = rt_con or html_repost if not text: operation = _first_value(feed, "operation") share_info = _first_value(operation, "share_info", "shareinfo") if isinstance(operation, dict) else None if isinstance(share_info, dict): text = _clean_plain_text(_first_value(share_info, "summary", "title")) return text, rt_con def _valid_image_url(value: Any) -> str: source = unescape(str(value or "").strip()) return source if source.lower().startswith(("https://", "http://", "//")) else "" def _photo_variant_item(value: Any) -> dict[str, str] | None: variants: list[tuple[int, str]] = [] def collect(item: Any, depth: int = 0) -> None: if depth > 5: return if isinstance(item, list): for child in item: collect(child, depth + 1) return if not isinstance(item, dict): return direct = _valid_image_url( _first_value( item, "origin_url", "original_url", "raw_url", "large_url", "big_url", "url3", "url2", "url1", "smallurl", "pic_url", "coverurl", "url", "src", ) ) if direct: width = _safe_int(_first_value(item, "width", "w")) height = _safe_int(_first_value(item, "height", "h")) variants.append((max(1, width * height), direct)) for child in item.values(): if isinstance(child, (dict, list)): collect(child, depth + 1) collect(value) if not variants: return None variants = list(dict.fromkeys(variants)) preview = min(variants, key=lambda item: item[0])[1] full = max(variants, key=lambda item: item[0])[1] return {"preview_url": preview, "full_url": full} def _structured_feed_images(feed: dict[str, Any], html_content: str) -> tuple[list[str], list[dict[str, str]]]: items: list[dict[str, str]] = [] for key in ("pic", "pics", "images", "photos", "video"): branch = _first_value(feed, key) if branch in (None, ""): continue if isinstance(branch, dict) and isinstance(_first_value(branch, "picdata", "pic_data"), list): sources = list(_first_value(branch, "picdata", "pic_data") or []) elif isinstance(branch, list): sources = branch else: sources = [branch] for source in sources: item = _photo_variant_item(source) if item and item not in items: items.append(item) if html_content: _text, _repost, _urls, html_items = _parse_feed_html_details(html_content) for item in html_items: if item not in items: items.append(item) return [item["full_url"] for item in items], items def _balanced_object(source: str, start: int) -> str: if start < 0 or start >= len(source) or source[start] != "{": return "" depth = 0 quote = "" escaped = False for index in range(start, len(source)): char = source[index] if quote: if escaped: escaped = False elif char == "\\": escaped = True elif char == quote: quote = "" continue if char in {"'", '"'}: quote = char elif char == "{": depth += 1 elif char == "}": depth -= 1 if depth == 0: return source[start : index + 1] return "" def parse_qzone_h5_index_html(html_content: str) -> dict[str, Any]: source = str(html_content or "") token_match = re.search( r"window\.shine0callback.*?return\s+[\"']([0-9a-zA-Z_-]+?)[\"']\s*;", source, flags=re.DOTALL, ) token = token_match.group(1).strip() if token_match else "" data_match = re.search(r"var\s+FrontPage\s*=.*?\bdata\s*:\s*\{", source, flags=re.DOTALL) object_text = _balanced_object(source, data_match.end() - 1) if data_match else "" payload: dict[str, Any] = {} if object_text: normalized = object_text.replace("undefined", "null") try: parsed = json.loads(normalized) except Exception: try: relaxed = re.sub(r",\s*([}\]])", r"\1", normalized) relaxed = re.sub(r"([{,]\s*)([A-Za-z_$][\w$]*)\s*:", r'\1"\2":', relaxed) parsed = json.loads(relaxed) except Exception: try: import json5 # type: ignore parsed = json5.loads(normalized) except Exception: parsed = {} if isinstance(parsed, dict): payload = parsed return {"token": token, "payload": payload} def parse_recent_feeds(data: dict[str, Any], diagnostics: dict[str, Any] | None = None) -> list[Any]: feeds, containers = _collect_feed_candidates(data) stats: dict[str, Any] = { "response_keys": sorted(str(key) for key in data)[:24] if isinstance(data, dict) else [], "containers": containers[:12], "candidate_count": len(feeds), "parsed_count": 0, "skipped_missing_identity": 0, "skipped_empty_content": 0, "skipped_duplicate": 0, "appids": [], } posts: list[Any] = [] seen_posts: set[tuple[int, str]] = set() appids: set[str] = set() for feed in feeds: html_content = str(_first_value(feed, "html", "feedhtml", "feed_html", "html_content") or "") common = _structured_feed_common(feed) uin, tid, name = _structured_feed_identity(feed, html_content) if not uin or not tid: stats["skipped_missing_identity"] += 1 continue post_key = (uin, tid) if post_key in seen_posts: stats["skipped_duplicate"] += 1 continue text, rt_con = _structured_feed_text(feed, html_content) image_urls, image_items = _structured_feed_images(feed, html_content) if not (text or rt_con or image_urls): stats["skipped_empty_content"] += 1 continue seen_posts.add(post_key) appid = str(_first_value(feed, "appid") or _first_value(common, "appid") or "311") typeid = str( _first_value(feed, "typeid", "type", "feedstype") or _first_value(common, "typeid", "type", "feedstype") or "0" ) abstime = _safe_int( _first_value(feed, "abstime", "created_time", "create_time", "timestamp", "pubtime", "time") or _first_value(common, "abstime", "created_time", "create_time", "timestamp", "time") ) fid = str( _first_value(feed, "fid", "key", "tid") or _first_value(_first_value(feed, "id"), "cellid", "fid", "tid") or _first_value(common, "ugcrightkey") or tid ) unikey = str( _first_value(feed, "unikey", "likekey", "like_key", "orglikekey") or _first_value(common, "unikey", "likekey", "like_key", "orglikekey") or _html_attr_value(html_content, "data-unikey", "unikey") or "" ) curkey = str( _first_value(feed, "curkey", "curlikekey", "likekey", "like_key") or _first_value(common, "curkey", "curlikekey", "likekey", "like_key") or _html_attr_value(html_content, "data-curkey", "curkey") or "" ) like_info = _first_value(feed, "like") liked = bool(_first_value(like_info, "isliked", "liked")) if isinstance(like_info, dict) else False appids.add(appid) posts.append( SimpleNamespace( tid=tid, uin=uin, name=name, text=text, rt_con=rt_con, images=image_urls, image_items=image_items, comments=[], create_time=abstime, appid=appid, typeid=typeid, abstime=abstime, fid=fid, unikey=unikey, curkey=curkey, liked=liked, raw=feed, status="approved", ) ) stats["parsed_count"] = len(posts) stats["appids"] = sorted(appids)[:16] if diagnostics is not None: diagnostics.clear() diagnostics.update(stats) return posts