From cd82d9e410c19e6d16a790b95e79a9ce7b69cd5b Mon Sep 17 00:00:00 2001 From: admin9webs Date: Fri, 10 Jul 2026 16:10:23 +0800 Subject: [PATCH] Add web/wechat-article-extraction --- skills/web/wechat-article-extraction/SKILL.md | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 skills/web/wechat-article-extraction/SKILL.md diff --git a/skills/web/wechat-article-extraction/SKILL.md b/skills/web/wechat-article-extraction/SKILL.md new file mode 100644 index 0000000..39d2415 --- /dev/null +++ b/skills/web/wechat-article-extraction/SKILL.md @@ -0,0 +1,108 @@ +--- +name: wechat-article-extraction +description: Extract full article content from WeChat public account (微信公众号) articles at mp.weixin.qq.com. Plain requests with desktop UA works; all "smarter" tools fail. +version: 1.0 +tags: [wechat, scraping, article, chinese] +--- + +# WeChat Article Extraction (mp.weixin.qq.com) + +Extract full article content from WeChat public account (微信公众号) articles. + +## What Works + +**Plain `requests` with desktop Chrome User-Agent.** The full article HTML is server-rendered and included in the initial response — no JS rendering needed. + +```python +import requests, re, html + +session = requests.Session() +headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', +} +resp = session.get(url, headers=headers, timeout=20) +content = resp.text + +# Title +title = re.search(r'var\s+msg_title\s*=\s*["\'](.+?)["\']', content) +title_text = html.unescape(title.group(1)) if title else None + +# Article body +body = re.search(r'id="js_content"[^>]*>(.*?)', content, re.DOTALL) +text = re.sub(r'<[^>]+>', '\n', body.group(1)) +text = html.unescape(text) +text = re.sub(r'\n{3,}', '\n\n', text).strip() +``` + +## What Does NOT Work + +| Tool | Result | Reason | +|------|--------|--------| +| Jina Reader (`r.jina.ai`) | ❌ CAPTCHA block | Detected as bot, returns "环境异常" | +| Crawl4AI (Playwright) | ❌ Timeout | Anti-bot blocks `js_content` selector from appearing | +| Browser automation | ❌ Sandbox/CAPTCHA | Chrome sandbox issues + verification wall | +| Google Cache | ❌ No cache | Google hasn't cached most WeChat articles | +| Mobile UA (MicroMessenger) | ❌ Empty content | Triggers anti-bot, content hidden behind JS | +| SerpAPI search | ❌ No results | WeChat articles rarely indexed | + +## Extraction Strategy (Two Patterns) + +WeChat articles come in two rendering variants. Try **Pattern 1** first, fall back to **Pattern 2**. + +### Pattern 1: Server-rendered HTML (`js_content` div) + +Most common. The full article HTML is in `
...
`. + +```python +body = re.search(r'id="js_content"[^>]*>(.*?)', content, re.DOTALL) +if body and body.group(1).strip(): + text = re.sub(r'', '\n', body.group(1)) + text = re.sub(r']*>', '\n', text) + text = re.sub(r'

', '', text) + text = re.sub(r'<[^>]+>', '', text) + text = html.unescape(text) +``` + +### Pattern 2: JS-string embedded (`window.msg_title`) + +Some shorter articles embed the ENTIRE content inside a JS string assignment. +The `
` is empty — content is populated dynamically by JS. +The article text lives in `window.msg_title = '...'` with `\\n` for newlines +and `\\'` for escaped quotes. + +```python +# Try Pattern 2 when js_content is empty +title_match = re.search(r"window\.msg_title\s*=\s*window\.title\s*=\s*'(.+?)'\s*\|\|\s*", content, re.S) +if not title_match: + title_match = re.search(r"window\.msg_title\s*=\s*window\.title\s*=\s*'(.+?)';(?:\s*$|\s*\n)", content, re.S) +if title_match: + article = title_match.group(1).replace('\\n', '\n').replace("\\'", "'").replace('\\\\', '\\') + # First line is the title, rest is body + lines = article.strip().split('\n') + title = lines[0] + body = '\n'.join(lines[1:]) +``` + +The `msg_title` variable is also the source of the `` tag — it's a single JS string +that contains the full article text. The regex MUST use `'` (single quotes) because WeChat +uses single-quoted JS strings for this value. + +## Key Patterns + +- **Title**: `var msg_title = "...";` — HTML-escaped, use `html.unescape()` +- **Description**: `var msg_desc = "...";` +- **Body (Pattern 1)**: `<div id="js_content">...</div>` — server-rendered HTML with inline tags +- **Body (Pattern 2)**: `window.msg_title = '...\\n...';` — JS-embedded, `\\n` for newlines +- **Images**: `data-src` attribute (not `src`) — WeChat lazy-loads images +- **Author**: `var nickname = "...";` in JS + +## Pitfalls + +- **Desktop UA is critical** — mobile/bot user agents trigger the verification wall even with `requests` +- **Pattern 2: `js_content` div exists but is empty** — don't assume failure; check `window.msg_title` as fallback +- **Pattern 2: use single quotes in regex** — WeChat uses `'...'` not `"..."` for `msg_title` JS strings +- **Pattern 2: `\\n` not `\n`** — the JS string literal has escaped newlines; use `.replace('\\n', '\n')` +- **Image URLs need transformation** — `data-src` values are relative (`//mmbiz.qpic.cn/...`), prepend `https:` +- **Don't over-engineer** — sophisticated tools (Crawl4AI, CamouFox) are overkill and actually worse for WeChat articles \ No newline at end of file