wiki-agentandClaude Opus 4.6
f06d84a003
Scraped all 31 Dongguan sub-regions using sortField=2 (end-time ascending) to bypass JD's ~4000-item API cap. Merged 34 CSV files by paimaiId into 24,949 unique records covering 2017-07 to 2026-11. Key findings: - 上架量 grew ~50x: 98 (2017) → 4,813 (2026) - 流拍率 peaked at 81.6% (2024), eased to 67.4% (2026) - 樟木头: 558 records, failure rate peaked 94.2% (2024) - 塘厦: 205 records, 2026 failure rate 51.0% Includes: scrape_history.py, batch_scrape_towns.sh, analyze_trends.py, yearly_stats.py, and updated SKILL.md + url_structure.md documenting the 4000-item cap and sub-region scraping strategy. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
433 lines
15 KiB
Python
433 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
京东法拍历史数据分批抓取脚本（一次性使用）
|
|
|
|
用 sortField=2（按结束时间升序，只返回已结束拍卖）滚动全量历史数据。
|
|
支持 date range 过滤：快速跳过不需要的早期数据，到达目标区间后开始收集。
|
|
|
|
用法:
|
|
# Batch 1: 2023-2026
|
|
python3 scrape_history.py --keyword 东莞市 --start-date 2023-01-01 --output-dir output/法拍
|
|
|
|
# Batch 2: 2017-2022
|
|
python3 scrape_history.py --keyword 东莞市 --end-date 2023-01-01 --output-dir output/法拍
|
|
"""
|
|
|
|
import argparse
|
|
import csv
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from collections import Counter
|
|
|
|
BASE_URL = (
|
|
"https://pro.m.jd.com/mall/active/3Rja7L2jmC72Ta8eoa31VFDvaAjf/index.html"
|
|
"?pageParamMiddle=judicature_search_middle"
|
|
"&pageParam=judicature_icon_search_after"
|
|
"&pageFrom=judicature_search_home"
|
|
"&tabParam=all"
|
|
"&fixSearchParams=false"
|
|
"&spo_multiThirdCateIds={cate_id}"
|
|
"&spo_keyword={keyword}"
|
|
"&spo_sortField=2"
|
|
)
|
|
|
|
STATUS_MAP = {0: "未开始", 1: "进行中", 2: "已结束", 3: "已撤回", 4: "已流拍"}
|
|
AUCTION_TYPE_MAP = {1: "诉讼拍卖", 5: "司法拍卖", 7: "商业拍卖"}
|
|
|
|
|
|
def find_chromium():
|
|
for path in [
|
|
"/usr/bin/chromium-browser", "/usr/bin/chromium",
|
|
"/usr/bin/google-chrome", "/usr/bin/google-chrome-stable",
|
|
]:
|
|
if os.path.isfile(path):
|
|
return path
|
|
return None
|
|
|
|
|
|
def ts_to_str(ts):
|
|
if ts:
|
|
try:
|
|
return datetime.datetime.fromtimestamp(ts / 1000).strftime(
|
|
"%Y-%m-%d %H:%M:%S"
|
|
)
|
|
except Exception:
|
|
return str(ts)
|
|
return ""
|
|
|
|
|
|
def ts_to_date(ts):
|
|
if ts:
|
|
try:
|
|
return datetime.datetime.fromtimestamp(ts / 1000).date()
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def infer_round_from_discount(discount_rate):
|
|
if not discount_rate:
|
|
return ""
|
|
if discount_rate >= 9.5:
|
|
return "无折扣"
|
|
if discount_rate >= 6.5:
|
|
return "一拍(推断)"
|
|
if discount_rate >= 5.0:
|
|
return "二拍/变卖(推断)"
|
|
return "低折扣"
|
|
|
|
|
|
def parse_date(s):
|
|
if not s:
|
|
return None
|
|
try:
|
|
return datetime.datetime.strptime(s, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def fetch_history(keyword, cate_id, start_date, end_date, max_scrolls, output_dir):
|
|
url = BASE_URL.format(cate_id=cate_id, keyword=keyword)
|
|
print(f"搜索 URL: {url}")
|
|
print(f"关键词: {keyword} | 类目ID: {cate_id} | 排序: sortField=2 (结束时间升序)")
|
|
|
|
if start_date:
|
|
print(f"起始日期: {start_date} (跳过此日期之前的拍卖)")
|
|
if end_date:
|
|
print(f"截止日期: {end_date} (到达此日期后停止)")
|
|
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
except ImportError:
|
|
print("错误: 未安装 Playwright")
|
|
sys.exit(1)
|
|
|
|
chromium_path = find_chromium()
|
|
|
|
with sync_playwright() as p:
|
|
launch_args = ["--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"]
|
|
if chromium_path:
|
|
browser = p.chromium.launch(
|
|
executable_path=chromium_path, headless=True, args=launch_args
|
|
)
|
|
else:
|
|
browser = p.chromium.launch(headless=True, args=launch_args)
|
|
|
|
context = browser.new_context(
|
|
user_agent=(
|
|
"Mozilla/5.0 (Linux; Android 13; VER-AN10) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/116.0.0.0 Mobile Safari/537.36"
|
|
),
|
|
viewport={"width": 375, "height": 812},
|
|
is_mobile=True,
|
|
)
|
|
page = context.new_page()
|
|
|
|
all_listings = {}
|
|
current_info = {}
|
|
label_configs = {}
|
|
state = {
|
|
"stopped_early": False,
|
|
"skipped": 0,
|
|
"collected": 0,
|
|
"first_collected_date": None,
|
|
"last_collected_date": None,
|
|
}
|
|
|
|
def handle_response(response):
|
|
resp_url = response.url
|
|
if "api.m.jd.com" not in resp_url or "functionId=" not in resp_url:
|
|
return
|
|
|
|
try:
|
|
body = response.text()
|
|
if not body or len(body) <= 50:
|
|
return
|
|
data = json.loads(body)
|
|
except Exception:
|
|
return
|
|
|
|
if "getSearchData" in resp_url and "Num" not in resp_url:
|
|
result_data = data.get("data", {}).get("resultData", [])
|
|
for item in result_data:
|
|
d = item.get("data", {})
|
|
if d and "paimaiId" in d:
|
|
all_listings[d["paimaiId"]] = d
|
|
|
|
elif "getPaimaiCurrentInfoByIdsForApi" in resp_url:
|
|
if data.get("statusCode") == 200:
|
|
for pid, info in data.get("data", {}).items():
|
|
current_info[pid] = info
|
|
|
|
elif "getAuctionLabelConfigs" in resp_url:
|
|
if data.get("code") == 0:
|
|
for pid, config in data.get("data", {}).items():
|
|
labels = config.get("labelConfigs", [])
|
|
label_configs[pid] = [
|
|
l.get("labelName", "") for l in labels
|
|
]
|
|
|
|
page.on("response", handle_response)
|
|
|
|
print("正在打开页面...")
|
|
page.goto(url, wait_until="networkidle", timeout=60000)
|
|
print("页面加载完成。")
|
|
|
|
# Extract SSR page 1
|
|
ssr_listings = page.evaluate("""() => {
|
|
const scripts = document.querySelectorAll('script');
|
|
for (const s of scripts) {
|
|
const text = s.textContent || '';
|
|
if (!text.includes('resultData') || !text.includes('paimaiId')) continue;
|
|
const idx = text.indexOf('"resultData"');
|
|
if (idx === -1) continue;
|
|
const arrStart = text.indexOf('[', idx);
|
|
if (arrStart === -1) continue;
|
|
let depth = 0, arrEnd = -1;
|
|
for (let i = arrStart; i < text.length; i++) {
|
|
if (text[i] === '[') depth++;
|
|
else if (text[i] === ']') { depth--; if (depth === 0) { arrEnd = i + 1; break; } }
|
|
}
|
|
if (arrEnd === -1) continue;
|
|
try {
|
|
const arr = JSON.parse(text.substring(arrStart, arrEnd));
|
|
return arr.map(item => item.data || item).filter(d => d && d.paimaiId);
|
|
} catch(e) { continue; }
|
|
}
|
|
return [];
|
|
}""")
|
|
if ssr_listings:
|
|
for d in ssr_listings:
|
|
if "paimaiId" in d:
|
|
all_listings[d["paimaiId"]] = d
|
|
print(f" SSR page 1: 提取 {len(ssr_listings)} 条")
|
|
|
|
time.sleep(5)
|
|
|
|
# Infinite scroll with date filtering
|
|
prev_count = 0
|
|
no_progress = 0
|
|
scroll_count = 0
|
|
collected_pids = set()
|
|
|
|
for i in range(max_scrolls):
|
|
if state["stopped_early"]:
|
|
break
|
|
|
|
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
|
time.sleep(3)
|
|
|
|
curr_count = len(all_listings)
|
|
|
|
# Check end dates of newly seen items
|
|
new_pids = set(all_listings.keys()) - collected_pids
|
|
for pid in new_pids:
|
|
collected_pids.add(pid)
|
|
info = current_info.get(str(pid), {})
|
|
end_ts = info.get("endTime")
|
|
end_dt = ts_to_date(end_ts)
|
|
|
|
if end_dt:
|
|
if end_date and end_dt >= end_date:
|
|
# Past our end date - stop
|
|
print(f" 到达截止日期 {end_date}，停止")
|
|
state["stopped_early"] = True
|
|
break
|
|
|
|
if start_date and end_dt < start_date:
|
|
state["skipped"] += 1
|
|
else:
|
|
state["collected"] += 1
|
|
if state["first_collected_date"] is None:
|
|
state["first_collected_date"] = end_dt
|
|
print(f" 开始收集: {end_dt} (paimaiId={pid})")
|
|
state["last_collected_date"] = end_dt
|
|
else:
|
|
# No end time yet - might be upcoming or missing info
|
|
state["skipped"] += 1
|
|
|
|
if curr_count > prev_count:
|
|
print(f" 滚动 {i+1}: 累计 {curr_count} 条 | 收集 {state['collected']} | 跳过 {state['skipped']} | "
|
|
f"日期范围: {state['first_collected_date']} ~ {state['last_collected_date']}")
|
|
prev_count = curr_count
|
|
no_progress = 0
|
|
else:
|
|
no_progress += 1
|
|
|
|
page_text = page.evaluate("document.body.innerText")
|
|
if "没有更多" in page_text or "到底了" in page_text:
|
|
print(f" 第 {i+1} 次滚动时到达底部")
|
|
break
|
|
|
|
if no_progress >= 10:
|
|
print(f" 连续 {no_progress} 次无新数据，停止")
|
|
break
|
|
|
|
scroll_count = i + 1
|
|
|
|
# Final scroll to catch remaining API responses
|
|
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
|
time.sleep(3)
|
|
browser.close()
|
|
|
|
# Build results: only items within date range
|
|
results = []
|
|
for pid, listing in all_listings.items():
|
|
info = current_info.get(str(pid), {})
|
|
end_ts = info.get("endTime")
|
|
end_dt = ts_to_date(end_ts)
|
|
|
|
if start_date and end_dt and end_dt < start_date:
|
|
continue
|
|
if end_date and end_dt and end_dt >= end_date:
|
|
continue
|
|
|
|
# Build listing with merged info
|
|
sku_id = listing.get("skuId", "")
|
|
listing["detail_url"] = (
|
|
f"https://m.jd.com/product/{sku_id}.html" if sku_id else ""
|
|
)
|
|
|
|
if str(pid) in current_info:
|
|
info = current_info[str(pid)]
|
|
listing["_currentPrice"] = info.get("currentPrice")
|
|
listing["_auctionStatus"] = info.get("auctionStatus")
|
|
listing["_bidCount"] = info.get("bidCount")
|
|
listing["_accessNumber"] = info.get("accessNumber")
|
|
listing["_startTime"] = info.get("startTime")
|
|
listing["_endTime"] = info.get("endTime")
|
|
|
|
labels = label_configs.get(str(pid), [])
|
|
round_labels = [
|
|
l for l in labels if l in ("一拍", "二拍", "三拍", "变卖", "重新拍卖")
|
|
]
|
|
if round_labels:
|
|
listing["auction_round"] = "、".join(round_labels)
|
|
else:
|
|
listing["auction_round"] = infer_round_from_discount(
|
|
listing.get("discountRate")
|
|
)
|
|
listing["all_labels"] = "、".join(labels) if labels else ""
|
|
|
|
results.append(listing)
|
|
|
|
# Sort by end time
|
|
results.sort(key=lambda x: x.get("_endTime", 0) or 0)
|
|
return results, state
|
|
|
|
|
|
def save_csv(listings, csv_path, batch_label):
|
|
now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
with open(csv_path, "w", newline="", encoding="utf-8-sig") as f:
|
|
writer = csv.writer(f)
|
|
writer.writerow([
|
|
"序号", "拍卖ID", "标题", "项目链接", "位置", "用途", "户型",
|
|
"面积(㎡)", "小区", "均价", "起拍价(元)", "当前价(元)",
|
|
"当前价(万)", "折扣率", "拍卖轮次", "标签", "状态",
|
|
"出价次数", "围观次数", "开始时间", "结束时间",
|
|
"SKU ID", "拍卖类型", "首次发现", "最后更新",
|
|
])
|
|
|
|
for i, item in enumerate(listings):
|
|
house = item.get("houseAttributes", {})
|
|
writer.writerow([
|
|
i + 1,
|
|
item.get("paimaiId", ""),
|
|
item.get("title", ""),
|
|
item.get("detail_url", ""),
|
|
"、".join(item.get("location", [])),
|
|
house.get("housePurpose", ""),
|
|
house.get("houseTypeInfo", ""),
|
|
house.get("houseArea", ""),
|
|
house.get("plotName", ""),
|
|
house.get("averagePrice", ""),
|
|
item.get("startPrice", ""),
|
|
item.get("_currentPrice", item.get("currentPrice", "")),
|
|
item.get("currentPriceCN", ""),
|
|
item.get("discountRate", ""),
|
|
item.get("auction_round", ""),
|
|
item.get("all_labels", ""),
|
|
STATUS_MAP.get(
|
|
item.get("_auctionStatus", item.get("paimaiStatus")), ""
|
|
),
|
|
item.get("_bidCount", ""),
|
|
item.get("_accessNumber", ""),
|
|
ts_to_str(item.get("_startTime")),
|
|
ts_to_str(item.get("_endTime")),
|
|
item.get("skuId", ""),
|
|
AUCTION_TYPE_MAP.get(
|
|
item.get("auctionType"),
|
|
str(item.get("auctionType", "")),
|
|
),
|
|
now_str,
|
|
now_str,
|
|
])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="京东法拍历史数据分批抓取")
|
|
parser.add_argument("--keyword", default="东莞市", help="搜索关键词")
|
|
parser.add_argument("--cate-id", default="15", help="类目ID (15=法拍房)")
|
|
parser.add_argument("--start-date", default=None, help="起始日期 YYYY-MM-DD (含)")
|
|
parser.add_argument("--end-date", default=None, help="截止日期 YYYY-MM-DD (不含)")
|
|
parser.add_argument("--max-scrolls", type=int, default=800, help="最大滚动次数")
|
|
parser.add_argument("--output-dir", default="output/法拍", help="输出目录")
|
|
parser.add_argument("--batch-label", default="", help="批次标签 (用于文件名)")
|
|
|
|
args = parser.parse_args()
|
|
|
|
start_date = parse_date(args.start_date)
|
|
end_date = parse_date(args.end_date)
|
|
batch_label = args.batch_label or (
|
|
f"{args.start_date or 'start'}_{args.end_date or 'end'}"
|
|
)
|
|
|
|
now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
print(f"运行时间: {now_str}")
|
|
print(f"批次: {batch_label}")
|
|
print()
|
|
|
|
listings, state = fetch_history(
|
|
keyword=args.keyword,
|
|
cate_id=args.cate_id,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
max_scrolls=args.max_scrolls,
|
|
output_dir=args.output_dir,
|
|
)
|
|
|
|
csv_path = os.path.join(
|
|
args.output_dir, f"{args.keyword}_法拍房源_{batch_label}.csv"
|
|
)
|
|
save_csv(listings, csv_path, batch_label)
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print(f"批次: {batch_label}")
|
|
print(f"收集: {len(listings)} 条")
|
|
print(f"跳过: {state['skipped']} 条")
|
|
if state['first_collected_date']:
|
|
print(f"日期范围: {state['first_collected_date']} ~ {state['last_collected_date']}")
|
|
print(f"输出: {csv_path}")
|
|
print(f"{'=' * 60}")
|
|
|
|
# Stats
|
|
statuses = Counter(
|
|
STATUS_MAP.get(
|
|
l.get("_auctionStatus", l.get("paimaiStatus")), "未知"
|
|
)
|
|
for l in listings
|
|
)
|
|
print(f"\n状态分布:")
|
|
for s, c in statuses.most_common():
|
|
print(f" {s}: {c} 条")
|
|
|
|
rounds = Counter(l.get("auction_round", "") for l in listings)
|
|
print(f"\n拍卖轮次分布:")
|
|
for r, c in rounds.most_common():
|
|
if r:
|
|
print(f" {r}: {c} 条")
|