#!/usr/bin/env python3
"""
京东法拍历史数据分批抓取脚本（一次性使用）

用 sortField=2（按结束时间升序，只返回已结束拍卖）滚动全量历史数据。
支持 date range 过滤：快速跳过不需要的早期数据，到达目标区间后开始收集。

用法:
  # Batch 1: 2023-2026
  python3 scrape_history.py --keyword 东莞市 --start-date 2023-01-01 --output-dir output/法拍

  # Batch 2: 2017-2022
  python3 scrape_history.py --keyword 东莞市 --end-date 2023-01-01 --output-dir output/法拍
"""

import argparse
import csv
import datetime
import json
import os
import sys
import time
from collections import Counter

BASE_URL = (
    "https://pro.m.jd.com/mall/active/3Rja7L2jmC72Ta8eoa31VFDvaAjf/index.html"
    "?pageParamMiddle=judicature_search_middle"
    "&pageParam=judicature_icon_search_after"
    "&pageFrom=judicature_search_home"
    "&tabParam=all"
    "&fixSearchParams=false"
    "&spo_multiThirdCateIds={cate_id}"
    "&spo_keyword={keyword}"
    "&spo_sortField=2"
)

STATUS_MAP = {0: "未开始", 1: "进行中", 2: "已结束", 3: "已撤回", 4: "已流拍"}
AUCTION_TYPE_MAP = {1: "诉讼拍卖", 5: "司法拍卖", 7: "商业拍卖"}


def find_chromium():
    for path in [
        "/usr/bin/chromium-browser", "/usr/bin/chromium",
        "/usr/bin/google-chrome", "/usr/bin/google-chrome-stable",
    ]:
        if os.path.isfile(path):
            return path
    return None


def ts_to_str(ts):
    if ts:
        try:
            return datetime.datetime.fromtimestamp(ts / 1000).strftime(
                "%Y-%m-%d %H:%M:%S"
            )
        except Exception:
            return str(ts)
    return ""


def ts_to_date(ts):
    if ts:
        try:
            return datetime.datetime.fromtimestamp(ts / 1000).date()
        except Exception:
            return None
    return None


def infer_round_from_discount(discount_rate):
    if not discount_rate:
        return ""
    if discount_rate >= 9.5:
        return "无折扣"
    if discount_rate >= 6.5:
        return "一拍(推断)"
    if discount_rate >= 5.0:
        return "二拍/变卖(推断)"
    return "低折扣"


def parse_date(s):
    if not s:
        return None
    try:
        return datetime.datetime.strptime(s, "%Y-%m-%d").date()
    except ValueError:
        return None


def fetch_history(keyword, cate_id, start_date, end_date, max_scrolls, output_dir):
    url = BASE_URL.format(cate_id=cate_id, keyword=keyword)
    print(f"搜索 URL: {url}")
    print(f"关键词: {keyword} | 类目ID: {cate_id} | 排序: sortField=2 (结束时间升序)")

    if start_date:
        print(f"起始日期: {start_date} (跳过此日期之前的拍卖)")
    if end_date:
        print(f"截止日期: {end_date} (到达此日期后停止)")

    try:
        from playwright.sync_api import sync_playwright
    except ImportError:
        print("错误: 未安装 Playwright")
        sys.exit(1)

    chromium_path = find_chromium()

    with sync_playwright() as p:
        launch_args = ["--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"]
        if chromium_path:
            browser = p.chromium.launch(
                executable_path=chromium_path, headless=True, args=launch_args
            )
        else:
            browser = p.chromium.launch(headless=True, args=launch_args)

        context = browser.new_context(
            user_agent=(
                "Mozilla/5.0 (Linux; Android 13; VER-AN10) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/116.0.0.0 Mobile Safari/537.36"
            ),
            viewport={"width": 375, "height": 812},
            is_mobile=True,
        )
        page = context.new_page()

        all_listings = {}
        current_info = {}
        label_configs = {}
        state = {
            "stopped_early": False,
            "skipped": 0,
            "collected": 0,
            "first_collected_date": None,
            "last_collected_date": None,
        }

        def handle_response(response):
            resp_url = response.url
            if "api.m.jd.com" not in resp_url or "functionId=" not in resp_url:
                return

            try:
                body = response.text()
                if not body or len(body) <= 50:
                    return
                data = json.loads(body)
            except Exception:
                return

            if "getSearchData" in resp_url and "Num" not in resp_url:
                result_data = data.get("data", {}).get("resultData", [])
                for item in result_data:
                    d = item.get("data", {})
                    if d and "paimaiId" in d:
                        all_listings[d["paimaiId"]] = d

            elif "getPaimaiCurrentInfoByIdsForApi" in resp_url:
                if data.get("statusCode") == 200:
                    for pid, info in data.get("data", {}).items():
                        current_info[pid] = info

            elif "getAuctionLabelConfigs" in resp_url:
                if data.get("code") == 0:
                    for pid, config in data.get("data", {}).items():
                        labels = config.get("labelConfigs", [])
                        label_configs[pid] = [
                            l.get("labelName", "") for l in labels
                        ]

        page.on("response", handle_response)

        print("正在打开页面...")
        page.goto(url, wait_until="networkidle", timeout=60000)
        print("页面加载完成。")

        # Extract SSR page 1
        ssr_listings = page.evaluate("""() => {
            const scripts = document.querySelectorAll('script');
            for (const s of scripts) {
                const text = s.textContent || '';
                if (!text.includes('resultData') || !text.includes('paimaiId')) continue;
                const idx = text.indexOf('"resultData"');
                if (idx === -1) continue;
                const arrStart = text.indexOf('[', idx);
                if (arrStart === -1) continue;
                let depth = 0, arrEnd = -1;
                for (let i = arrStart; i < text.length; i++) {
                    if (text[i] === '[') depth++;
                    else if (text[i] === ']') { depth--; if (depth === 0) { arrEnd = i + 1; break; } }
                }
                if (arrEnd === -1) continue;
                try {
                    const arr = JSON.parse(text.substring(arrStart, arrEnd));
                    return arr.map(item => item.data || item).filter(d => d && d.paimaiId);
                } catch(e) { continue; }
            }
            return [];
        }""")
        if ssr_listings:
            for d in ssr_listings:
                if "paimaiId" in d:
                    all_listings[d["paimaiId"]] = d
            print(f"  SSR page 1: 提取 {len(ssr_listings)} 条")

        time.sleep(5)

        # Infinite scroll with date filtering
        prev_count = 0
        no_progress = 0
        scroll_count = 0
        collected_pids = set()

        for i in range(max_scrolls):
            if state["stopped_early"]:
                break

            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            time.sleep(3)

            curr_count = len(all_listings)

            # Check end dates of newly seen items
            new_pids = set(all_listings.keys()) - collected_pids
            for pid in new_pids:
                collected_pids.add(pid)
                info = current_info.get(str(pid), {})
                end_ts = info.get("endTime")
                end_dt = ts_to_date(end_ts)

                if end_dt:
                    if end_date and end_dt >= end_date:
                        # Past our end date - stop
                        print(f"  到达截止日期 {end_date}，停止")
                        state["stopped_early"] = True
                        break

                    if start_date and end_dt < start_date:
                        state["skipped"] += 1
                    else:
                        state["collected"] += 1
                        if state["first_collected_date"] is None:
                            state["first_collected_date"] = end_dt
                            print(f"  开始收集: {end_dt} (paimaiId={pid})")
                        state["last_collected_date"] = end_dt
                else:
                    # No end time yet - might be upcoming or missing info
                    state["skipped"] += 1

            if curr_count > prev_count:
                print(f"  滚动 {i+1}: 累计 {curr_count} 条 | 收集 {state['collected']} | 跳过 {state['skipped']} | "
                      f"日期范围: {state['first_collected_date']} ~ {state['last_collected_date']}")
                prev_count = curr_count
                no_progress = 0
            else:
                no_progress += 1

            page_text = page.evaluate("document.body.innerText")
            if "没有更多" in page_text or "到底了" in page_text:
                print(f"  第 {i+1} 次滚动时到达底部")
                break

            if no_progress >= 10:
                print(f"  连续 {no_progress} 次无新数据，停止")
                break

            scroll_count = i + 1

        # Final scroll to catch remaining API responses
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        time.sleep(3)
        browser.close()

    # Build results: only items within date range
    results = []
    for pid, listing in all_listings.items():
        info = current_info.get(str(pid), {})
        end_ts = info.get("endTime")
        end_dt = ts_to_date(end_ts)

        if start_date and end_dt and end_dt < start_date:
            continue
        if end_date and end_dt and end_dt >= end_date:
            continue

        # Build listing with merged info
        sku_id = listing.get("skuId", "")
        listing["detail_url"] = (
            f"https://m.jd.com/product/{sku_id}.html" if sku_id else ""
        )

        if str(pid) in current_info:
            info = current_info[str(pid)]
            listing["_currentPrice"] = info.get("currentPrice")
            listing["_auctionStatus"] = info.get("auctionStatus")
            listing["_bidCount"] = info.get("bidCount")
            listing["_accessNumber"] = info.get("accessNumber")
            listing["_startTime"] = info.get("startTime")
            listing["_endTime"] = info.get("endTime")

        labels = label_configs.get(str(pid), [])
        round_labels = [
            l for l in labels if l in ("一拍", "二拍", "三拍", "变卖", "重新拍卖")
        ]
        if round_labels:
            listing["auction_round"] = "、".join(round_labels)
        else:
            listing["auction_round"] = infer_round_from_discount(
                listing.get("discountRate")
            )
        listing["all_labels"] = "、".join(labels) if labels else ""

        results.append(listing)

    # Sort by end time
    results.sort(key=lambda x: x.get("_endTime", 0) or 0)
    return results, state


def save_csv(listings, csv_path, batch_label):
    now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(csv_path, "w", newline="", encoding="utf-8-sig") as f:
        writer = csv.writer(f)
        writer.writerow([
            "序号", "拍卖ID", "标题", "项目链接", "位置", "用途", "户型",
            "面积(㎡)", "小区", "均价", "起拍价(元)", "当前价(元)",
            "当前价(万)", "折扣率", "拍卖轮次", "标签", "状态",
            "出价次数", "围观次数", "开始时间", "结束时间",
            "SKU ID", "拍卖类型", "首次发现", "最后更新",
        ])

        for i, item in enumerate(listings):
            house = item.get("houseAttributes", {})
            writer.writerow([
                i + 1,
                item.get("paimaiId", ""),
                item.get("title", ""),
                item.get("detail_url", ""),
                "、".join(item.get("location", [])),
                house.get("housePurpose", ""),
                house.get("houseTypeInfo", ""),
                house.get("houseArea", ""),
                house.get("plotName", ""),
                house.get("averagePrice", ""),
                item.get("startPrice", ""),
                item.get("_currentPrice", item.get("currentPrice", "")),
                item.get("currentPriceCN", ""),
                item.get("discountRate", ""),
                item.get("auction_round", ""),
                item.get("all_labels", ""),
                STATUS_MAP.get(
                    item.get("_auctionStatus", item.get("paimaiStatus")), ""
                ),
                item.get("_bidCount", ""),
                item.get("_accessNumber", ""),
                ts_to_str(item.get("_startTime")),
                ts_to_str(item.get("_endTime")),
                item.get("skuId", ""),
                AUCTION_TYPE_MAP.get(
                    item.get("auctionType"),
                    str(item.get("auctionType", "")),
                ),
                now_str,
                now_str,
            ])


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="京东法拍历史数据分批抓取")
    parser.add_argument("--keyword", default="东莞市", help="搜索关键词")
    parser.add_argument("--cate-id", default="15", help="类目ID (15=法拍房)")
    parser.add_argument("--start-date", default=None, help="起始日期 YYYY-MM-DD (含)")
    parser.add_argument("--end-date", default=None, help="截止日期 YYYY-MM-DD (不含)")
    parser.add_argument("--max-scrolls", type=int, default=800, help="最大滚动次数")
    parser.add_argument("--output-dir", default="output/法拍", help="输出目录")
    parser.add_argument("--batch-label", default="", help="批次标签 (用于文件名)")

    args = parser.parse_args()

    start_date = parse_date(args.start_date)
    end_date = parse_date(args.end_date)
    batch_label = args.batch_label or (
        f"{args.start_date or 'start'}_{args.end_date or 'end'}"
    )

    now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    print(f"运行时间: {now_str}")
    print(f"批次: {batch_label}")
    print()

    listings, state = fetch_history(
        keyword=args.keyword,
        cate_id=args.cate_id,
        start_date=start_date,
        end_date=end_date,
        max_scrolls=args.max_scrolls,
        output_dir=args.output_dir,
    )

    csv_path = os.path.join(
        args.output_dir, f"{args.keyword}_法拍房源_{batch_label}.csv"
    )
    save_csv(listings, csv_path, batch_label)

    print(f"\n{'=' * 60}")
    print(f"批次: {batch_label}")
    print(f"收集: {len(listings)} 条")
    print(f"跳过: {state['skipped']} 条")
    if state['first_collected_date']:
        print(f"日期范围: {state['first_collected_date']} ~ {state['last_collected_date']}")
    print(f"输出: {csv_path}")
    print(f"{'=' * 60}")

    # Stats
    statuses = Counter(
        STATUS_MAP.get(
            l.get("_auctionStatus", l.get("paimaiStatus")), "未知"
        )
        for l in listings
    )
    print(f"\n状态分布:")
    for s, c in statuses.most_common():
        print(f"  {s}: {c} 条")

    rounds = Counter(l.get("auction_round", "") for l in listings)
    print(f"\n拍卖轮次分布:")
    for r, c in rounds.most_common():
        if r:
            print(f"  {r}: {c} 条")