Add V6 stock strategy analysis and JD fapai scrape skill

- Add V6 strategy section to us-stock-trading-evaluation: fixes indicator dilution, SMA period hardcode, and stoploss tier issues; includes hyperopt results and backtest comparison - Add jd-fapai-scrape skill: Playwright-based scraper for JD judicial auction properties with incremental update support Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-09-09 08:05:57 +00:00
parent cf92ce902c
commit e332d346f6
5 changed files with 943 additions and 0 deletions
@@ -0,0 +1,551 @@
+#!/usr/bin/env python3
+"""
+京东法拍房源抓取脚本 v3
+- 项目链接 (detail_url): https://m.jd.com/product/{skuId}.html
+- 拍卖轮次 (auction_round): 优先从 getAuctionLabelConfigs API 获取，无标签时从折扣率推断
+- 去重 key = paimaiId（每次拍卖唯一，同一房产的一拍/二拍为不同 paimaiId，各自保留）
+- 增量更新：遇到已知 paimaiId 即停止滚动，不全量下载
+- 按发布时间降序排序（最新在前）
+
+用法:
+  python3 scrape.py --keyword 东莞市 --output-dir data         # 首次抓取
+  python3 scrape.py --keyword 东莞市 --output-dir data -i     # 增量更新
+"""
+
+import argparse
+import csv
+import datetime
+import json
+import os
+import sys
+import time
+from collections import Counter
+
+BASE_URL = (
+    "https://pro.m.jd.com/mall/active/3Rja7L2jmC72Ta8eoa31VFDvaAjf/index.html"
+    "?pageParamMiddle=judicature_search_middle"
+    "&pageParam=judicature_icon_search_after"
+    "&pageFrom=judicature_search_home"
+    "&tabParam=all"
+    "&fixSearchParams=false"
+    "&navh=49"
+    "&stath=33"
+    "&tttparams=W6cw9AU4eyJhZGRyZXNzSWQiOjEzODUwMzYwMCwiYXJlYUNvZGUiOjAsImNvcm5lciI6MSwiZF9icmFuZCI6IkhPTk9SIiwiZGwiOjIsImdMYXQiOiIyMi42MjM4MjkiLCJnTG5nIjoiMTE0LjAyNzk0OCIsImdwc19hcmVhIjoiMTlfMTYwN18zMTU1XzYyMTE5IiwibGF0IjoyMi41OTkwMzcsImxic0FyZWEiOiIxOV8xNjA3XzMxNTVfNjIxMTkiLCJsYnNMYXQiOiIyMi42MDA1MzciLCJsYnNMbmciOiIxMTQuMDE0Njk4IiwibG5nIjoxMTQuMDE2ODksIm1vZGVsIjoiVkVSLUFOMTAiLCJvcyI6IjE2IiwicG9zTGF0IjoiMjIuNjIzODI5IiwicG9zTG5nIjoiMTE0LjAyNzk0OCIsInByc3RhdGUiOiIwIiwic2NhbGUiOjMsInVlbXBzIjoiMC0yLTk5OSIsInVuX2FyZWEiOiIxOV8xNjA3XzQ3Mzg4XzYyMTM5Iiwid2lkdGgiOjEwNj8B9"
+    "&spo_lng=114.01689"
+    "&spo_lbsEffect=2"
+    "&spo_reqSource=1"
+    "&spo_multiThirdCateIds={cate_id}"
+    "&spo_lat=22.599037"
+    "&spo_keyword={keyword}"
+    "&spo_sortField={sort_field}"
+)
+
+STATUS_MAP = {0: "未开始", 1: "进行中", 2: "已结束", 3: "已撤回", 4: "已流拍"}
+AUCTION_TYPE_MAP = {1: "诉讼拍卖", 5: "司法拍卖", 7: "商业拍卖"}
+
+
+def find_chromium():
+    for path in [
+        "/usr/bin/chromium-browser", "/usr/bin/chromium",
+        "/usr/bin/google-chrome", "/usr/bin/google-chrome-stable",
+    ]:
+        if os.path.isfile(path):
+            return path
+    return None
+
+
+def ts_to_str(ts):
+    if ts:
+        try:
+            return datetime.datetime.fromtimestamp(ts / 1000).strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+        except Exception:
+            return str(ts)
+    return ""
+
+
+def infer_round_from_discount(discount_rate):
+    """从折扣率推断拍卖轮次（7折≈一拍, 5.6折≈二拍/变卖, 10折≈无折扣）"""
+    if not discount_rate:
+        return ""
+    if discount_rate >= 9.5:
+        return "无折扣"
+    if discount_rate >= 6.5:
+        return "一拍(推断)"
+    if discount_rate >= 5.0:
+        return "二拍/变卖(推断)"
+    return "低折扣"
+
+
+def load_existing(json_path):
+    if not os.path.exists(json_path):
+        return [], set()
+    with open(json_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    known_ids = {item["paimaiId"] for item in data if "paimaiId" in item}
+    return data, known_ids
+
+
+def fetch_listings(keyword, cate_id, sort_field, max_scrolls,
+                   known_paimai_ids=None, early_stop=True):
+    url = BASE_URL.format(cate_id=cate_id, keyword=keyword, sort_field=sort_field)
+    print(f"搜索 URL: {url}")
+    print(f"关键词: {keyword} | 类目ID: {cate_id} | 排序: {sort_field}")
+
+    if known_paimai_ids is not None and early_stop:
+        print(f"增量模式: 已知 {len(known_paimai_ids)} 个 paimaiId，遇到已知项即停止")
+
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        print("错误: 未安装 Playwright，请运行: pip install playwright")
+        sys.exit(1)
+
+    chromium_path = find_chromium()
+
+    with sync_playwright() as p:
+        launch_args = ["--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"]
+        if chromium_path:
+            browser = p.chromium.launch(
+                executable_path=chromium_path, headless=True, args=launch_args
+            )
+        else:
+            try:
+                browser = p.chromium.launch(headless=True, args=launch_args)
+            except Exception:
+                print("错误: 未找到 Chromium，请运行:")
+                print("  python3 -m playwright install chromium")
+                sys.exit(1)
+
+        context = browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (Linux; Android 13; VER-AN10) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/116.0.0.0 Mobile Safari/537.36"
+            ),
+            viewport={"width": 375, "height": 812},
+            is_mobile=True,
+        )
+        page = context.new_page()
+
+        # 用 dict 在闭包中累积数据
+        all_listings = {}
+        current_info = {}
+        label_configs = {}
+        state = {"search_call_count": 0, "stopped_early": False}
+
+        def handle_response(response):
+            resp_url = response.url
+            if "api.m.jd.com" not in resp_url or "functionId=" not in resp_url:
+                return
+
+            try:
+                body = response.text()
+                if not body or len(body) <= 50:
+                    return
+                data = json.loads(body)
+            except Exception:
+                return
+
+            if "getSearchData" in resp_url:
+                state["search_call_count"] += 1
+                result_data = data.get("data", {}).get("resultData", [])
+                for item in result_data:
+                    d = item.get("data", {})
+                    if d and "paimaiId" in d:
+                        all_listings[d["paimaiId"]] = d
+
+                # 增量模式：本批全部已知就停
+                if known_paimai_ids is not None and early_stop and result_data:
+                    new_in_batch = [
+                        item for item in result_data
+                        if item.get("data", {}).get("paimaiId") not in known_paimai_ids
+                    ]
+                    if len(new_in_batch) == 0:
+                        print(f"  第 {state['search_call_count']} 批数据全部已知，停止滚动")
+                        state["stopped_early"] = True
+
+            elif "getPaimaiCurrentInfoByIdsForApi" in resp_url:
+                if data.get("statusCode") == 200:
+                    for pid, info in data.get("data", {}).items():
+                        current_info[pid] = info
+
+            elif "getAuctionLabelConfigs" in resp_url:
+                if data.get("code") == 0:
+                    for pid, config in data.get("data", {}).items():
+                        labels = config.get("labelConfigs", [])
+                        label_configs[pid] = [
+                            l.get("labelName", "") for l in labels
+                        ]
+
+        page.on("response", handle_response)
+
+        print("正在打开页面...")
+        page.goto(url, wait_until="networkidle", timeout=60000)
+        print("页面加载完成。")
+
+        # 从 SSR 内嵌数据中提取 page 1（API 只返回 page 2+，page 1 在 HTML 里）
+        ssr_listings = page.evaluate("""() => {
+            const scripts = document.querySelectorAll('script');
+            for (const s of scripts) {
+                const text = s.textContent || '';
+                if (!text.includes('resultData') || !text.includes('paimaiId')) continue;
+                const idx = text.indexOf('"resultData"');
+                if (idx === -1) continue;
+                const arrStart = text.indexOf('[', idx);
+                if (arrStart === -1) continue;
+                let depth = 0, arrEnd = -1;
+                for (let i = arrStart; i < text.length; i++) {
+                    if (text[i] === '[') depth++;
+                    else if (text[i] === ']') { depth--; if (depth === 0) { arrEnd = i + 1; break; } }
+                }
+                if (arrEnd === -1) continue;
+                try {
+                    const arr = JSON.parse(text.substring(arrStart, arrEnd));
+                    return arr.map(item => item.data || item).filter(d => d && d.paimaiId);
+                } catch(e) { continue; }
+            }
+            return [];
+        }""")
+        if ssr_listings:
+            for d in ssr_listings:
+                if "paimaiId" in d:
+                    all_listings[d["paimaiId"]] = d
+            # 增量模式：检查 page 1 是否全部已知
+            if known_paimai_ids is not None and early_stop:
+                new_in_ssr = [d for d in ssr_listings
+                    if d.get("paimaiId") not in known_paimai_ids]
+                if len(new_in_ssr) == 0:
+                    print(f"  SSR page 1: {len(ssr_listings)} 条全部已知，停止")
+                    state["stopped_early"] = True
+            print(f"  SSR page 1: 提取 {len(ssr_listings)} 条")
+
+        time.sleep(3)
+
+        # 无限滚动
+        prev_count = 0
+        no_progress = 0
+
+        for i in range(max_scrolls):
+            if state["stopped_early"]:
+                break
+
+            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            time.sleep(2)
+
+            curr_count = len(all_listings)
+            if curr_count > prev_count:
+                label_count = len(label_configs)
+                print(f"  第 {i+1} 次滚动: 累计 {curr_count} 条 (标签 {label_count})")
+                prev_count = curr_count
+                no_progress = 0
+            else:
+                no_progress += 1
+
+            page_text = page.evaluate("document.body.innerText")
+            if "没有更多" in page_text or "到底了" in page_text:
+                print(f"  第 {i+1} 次滚动时到达底部")
+                break
+
+            if no_progress >= 5:
+                print(f"  连续 {no_progress} 次无新数据，停止")
+                break
+
+        if not state["stopped_early"]:
+            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            time.sleep(3)
+
+        # 滚动结束后等待标签 API 响应
+        time.sleep(2)
+
+        browser.close()
+
+    # 合并三个 API 的数据
+    results = []
+    for pid, listing in all_listings.items():
+        # 构造项目链接
+        sku_id = listing.get("skuId", "")
+        listing["detail_url"] = (
+            f"https://m.jd.com/product/{sku_id}.html" if sku_id else ""
+        )
+
+        # 合并实时信息
+        if str(pid) in current_info:
+            info = current_info[str(pid)]
+            listing["_currentPrice"] = info.get("currentPrice")
+            listing["_auctionStatus"] = info.get("auctionStatus")
+            listing["_bidCount"] = info.get("bidCount")
+            listing["_accessNumber"] = info.get("accessNumber")
+            listing["_startTime"] = info.get("startTime")
+            listing["_endTime"] = info.get("endTime")
+
+        # 拍卖轮次：优先用 API 标签，无标签时从折扣率推断
+        labels = label_configs.get(str(pid), [])
+        round_labels = [
+            l for l in labels if l in ("一拍", "二拍", "三拍", "变卖", "重新拍卖")
+        ]
+        if round_labels:
+            listing["auction_round"] = "、".join(round_labels)
+        else:
+            listing["auction_round"] = infer_round_from_discount(
+                listing.get("discountRate")
+            )
+        listing["all_labels"] = "、".join(labels) if labels else ""
+
+        results.append(listing)
+
+    return results, state["stopped_early"]
+
+
+def sort_by_publish_time(listings):
+    # API (spo_sortField=11) 已按发布时间降序返回，直接保持原顺序
+    return list(listings)
+
+
+def merge_incremental(new_listings, existing_path, now_str):
+    with open(existing_path, "r", encoding="utf-8") as f:
+        existing = json.load(f)
+
+    existing_map = {item["paimaiId"]: item for item in existing if "paimaiId" in item}
+    new_paimai_ids = set()
+    new_items = []
+    updated_items = []
+
+    for item in new_listings:
+        pid = item.get("paimaiId")
+        if pid is None:
+            continue
+        new_paimai_ids.add(pid)
+        if pid not in existing_map:
+            item["_first_seen"] = now_str
+            item["_last_seen"] = now_str
+            new_items.append(item)
+        else:
+            old = existing_map[pid]
+            old["_last_seen"] = now_str
+            for key in ["_currentPrice", "_auctionStatus", "_bidCount",
+                        "_accessNumber", "_startTime", "_endTime",
+                        "currentPrice", "currentPriceCN", "paimaiStatus",
+                        "displayStatus", "auction_round", "all_labels"]:
+                if key in item:
+                    old[key] = item[key]
+            updated_items.append(old)
+
+    merged = []
+    seen = set()
+    for item in new_listings:
+        pid = item.get("paimaiId")
+        if pid and pid not in seen:
+            if pid in existing_map:
+                merged.append(existing_map[pid])
+            else:
+                merged.append(item)
+            seen.add(pid)
+    for item in existing:
+        pid = item.get("paimaiId")
+        if pid and pid not in seen:
+            merged.append(item)
+            seen.add(pid)
+
+    return merged, new_items, updated_items
+
+
+def save_csv(listings, csv_path):
+    with open(csv_path, "w", newline="", encoding="utf-8-sig") as f:
+        writer = csv.writer(f)
+        writer.writerow([
+            "序号", "拍卖ID", "标题", "项目链接", "位置", "用途", "户型",
+            "面积(㎡)", "小区", "均价", "起拍价(元)", "当前价(元)",
+            "当前价(万)", "折扣率", "拍卖轮次", "标签", "状态",
+            "出价次数", "围观次数", "开始时间", "结束时间",
+            "SKU ID", "拍卖类型", "首次发现", "最后更新",
+        ])
+
+        for i, item in enumerate(listings):
+            house = item.get("houseAttributes", {})
+            writer.writerow([
+                i + 1,
+                item.get("paimaiId", ""),
+                item.get("title", ""),
+                item.get("detail_url", ""),
+                "、".join(item.get("location", [])),
+                house.get("housePurpose", ""),
+                house.get("houseTypeInfo", ""),
+                house.get("houseArea", ""),
+                house.get("plotName", ""),
+                house.get("averagePrice", ""),
+                item.get("startPrice", ""),
+                item.get("_currentPrice", item.get("currentPrice", "")),
+                item.get("currentPriceCN", ""),
+                item.get("discountRate", ""),
+                item.get("auction_round", ""),
+                item.get("all_labels", ""),
+                STATUS_MAP.get(
+                    item.get("_auctionStatus", item.get("paimaiStatus")), ""
+                ),
+                item.get("_bidCount", ""),
+                item.get("_accessNumber", ""),
+                ts_to_str(item.get("_startTime")),
+                ts_to_str(item.get("_endTime")),
+                item.get("skuId", ""),
+                AUCTION_TYPE_MAP.get(
+                    item.get("auctionType"),
+                    str(item.get("auctionType", "")),
+                ),
+                item.get("_first_seen", ""),
+                item.get("_last_seen", ""),
+            ])
+
+
+def print_stats(listings, new_items=None, updated_items=None, early_stopped=False):
+    print(f"\n{'=' * 60}")
+    print(f"房源总数: {len(listings)} 条")
+    if new_items is not None:
+        print(f"本次新增: {len(new_items)} 条")
+    if updated_items is not None:
+        print(f"本次更新: {len(updated_items)} 条")
+    if early_stopped:
+        print(f"(增量模式: 遇到已知数据即停止，未全量下载)")
+    print(f"{'=' * 60}")
+
+    prices = [
+        item.get("_currentPrice") or item.get("currentPrice", 0)
+        for item in listings
+        if item.get("_currentPrice") or item.get("currentPrice")
+    ]
+    if prices:
+        print(f"\n价格统计:")
+        print(f"  最低: ¥{min(prices):,.2f}")
+        print(f"  最高: ¥{max(prices):,.2f}")
+        print(f"  平均: ¥{sum(prices) / len(prices):,.2f}")
+        print(f"  中位: ¥{sorted(prices)[len(prices) // 2]:,.2f}")
+
+    statuses = Counter(
+        STATUS_MAP.get(
+            item.get("_auctionStatus", item.get("paimaiStatus")), "未知"
+        )
+        for item in listings
+    )
+    print(f"\n状态分布:")
+    for s, c in statuses.most_common():
+        print(f"  {s}: {c} 条")
+
+    rounds = Counter(item.get("auction_round", "") for item in listings)
+    print(f"\n拍卖轮次分布:")
+    for r, c in rounds.most_common():
+        if r:
+            print(f"  {r}: {c} 条")
+
+    if new_items:
+        print(f"\n新增房源 (前 10 条):")
+        for i, item in enumerate(new_items[:10]):
+            price = item.get("_currentPrice") or item.get("currentPrice", 0)
+            print(f"  [{i+1}] {item.get('title', 'N/A')[:50]}")
+            print(f"      ¥{price:,.0f} | {item.get('auction_round','')} | "
+                  f"{ts_to_str(item.get('_startTime'))}")
+            print(f"      {item.get('detail_url','')}")
+
+
+def scrape(keyword="塘厦", cate_id="15", sort_field="11",
+           max_scrolls=100, output_dir=".", incremental=False):
+    os.makedirs(output_dir, exist_ok=True)
+
+    now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    print(f"运行时间: {now_str}")
+    print(f"模式: {'增量更新' if incremental else '首次抓取'}")
+    print()
+
+    json_path = os.path.join(output_dir, f"{keyword}_法拍房源.json")
+    csv_path = os.path.join(output_dir, f"{keyword}_法拍房源.csv")
+
+    known_ids = None
+    if incremental:
+        existing, known_ids = load_existing(json_path)
+        if not known_ids:
+            print("未找到已有数据，转为首次抓取模式")
+            incremental = False
+        else:
+            print(f"已有数据: {len(existing)} 条, 已知 {len(known_ids)} 个 paimaiId")
+
+    new_listings, early_stopped = fetch_listings(
+        keyword, cate_id, sort_field, max_scrolls,
+        known_paimai_ids=known_ids if incremental else None,
+        early_stop=incremental,
+    )
+    print(f"\n本次抓取: {len(new_listings)} 条")
+
+    if incremental:
+        merged, new_items, updated_items = merge_incremental(
+            new_listings, json_path, now_str
+        )
+        print(f"合并后总数: {len(merged)} 条")
+        print(f"新增: {len(new_items)} 条 | 更新: {len(updated_items)} 条")
+
+        with open(json_path, "w", encoding="utf-8") as f:
+            json.dump(merged, f, indent=2, ensure_ascii=False)
+        save_csv(merged, csv_path)
+
+        log_path = os.path.join(output_dir, f"{keyword}_增量日志.log")
+        with open(log_path, "a", encoding="utf-8") as f:
+            f.write(f"\n{'=' * 60}\n")
+            f.write(f"时间: {now_str}\n")
+            f.write(f"新增: {len(new_items)} 条 | 更新: {len(updated_items)} 条\n")
+            f.write(f"提前停止: {'是' if early_stopped else '否'}\n")
+            for item in new_items:
+                price = item.get("_currentPrice") or item.get("currentPrice", 0)
+                f.write(f"  [新] {item.get('title', 'N/A')[:60]} | "
+                        f"{item.get('auction_round','')} | ¥{price:,.0f} | "
+                        f"{ts_to_str(item.get('_startTime'))}\n"
+                        f"       {item.get('detail_url','')}\n")
+            for item in updated_items:
+                price = item.get("_currentPrice") or item.get("currentPrice", 0)
+                f.write(f"  [更] {item.get('title', 'N/A')[:60]} | "
+                        f"{item.get('auction_round','')} | ¥{price:,.0f} | "
+                        f"{ts_to_str(item.get('_startTime'))}\n")
+
+        print(f"\n输出文件:")
+        print(f"  JSON: {json_path}")
+        print(f"  CSV:  {csv_path}")
+        print(f"  日志: {log_path}")
+
+        print_stats(merged, new_items, updated_items, early_stopped=early_stopped)
+        return merged
+    else:
+        sorted_listings = sort_by_publish_time(new_listings)
+        for item in sorted_listings:
+            item["_first_seen"] = now_str
+            item["_last_seen"] = now_str
+
+        with open(json_path, "w", encoding="utf-8") as f:
+            json.dump(sorted_listings, f, indent=2, ensure_ascii=False)
+        save_csv(sorted_listings, csv_path)
+
+        print(f"\n输出文件:")
+        print(f"  JSON: {json_path}")
+        print(f"  CSV:  {csv_path}")
+
+        print_stats(sorted_listings)
+        return sorted_listings
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="京东法拍房源抓取（支持增量更新）")
+    parser.add_argument("--keyword", default="塘厦", help="搜索关键词 (默认: 塘厦)")
+    parser.add_argument("--cate-id", default="15", help="类目ID (默认: 15=法拍房)")
+    parser.add_argument("--sort-field", default="11", help="排序 (默认: 11=最新发布)")
+    parser.add_argument("--max-scrolls", type=int, default=100, help="最大滚动次数")
+    parser.add_argument("--output-dir", default=".", help="输出目录")
+    parser.add_argument("--incremental", "-i", action="store_true",
+                        help="增量更新：只下载新房源，遇到已知项即停止")
+
+    args = parser.parse_args()
+
+    scrape(
+        keyword=args.keyword,
+        cate_id=args.cate_id,
+        sort_field=args.sort_field,
+        max_scrolls=args.max_scrolls,
+        output_dir=args.output_dir,
+        incremental=args.incremental,
+    )