wiki-agentandClaude Opus 4.6
f06d84a003
Scraped all 31 Dongguan sub-regions using sortField=2 (end-time ascending) to bypass JD's ~4000-item API cap. Merged 34 CSV files by paimaiId into 24,949 unique records covering 2017-07 to 2026-11. Key findings: - 上架量 grew ~50x: 98 (2017) → 4,813 (2026) - 流拍率 peaked at 81.6% (2024), eased to 67.4% (2026) - 樟木头: 558 records, failure rate peaked 94.2% (2024) - 塘厦: 205 records, 2026 failure rate 51.0% Includes: scrape_history.py, batch_scrape_towns.sh, analyze_trends.py, yearly_stats.py, and updated SKILL.md + url_structure.md documenting the 4000-item cap and sub-region scraping strategy. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
67 lines
2.5 KiB
Python
67 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
||
"""Compute yearly statistics from all JD法拍 CSV files."""
|
||
import csv, glob, os
|
||
from collections import Counter
|
||
|
||
all_items = dict()
|
||
for csv_path in sorted(glob.glob("output/法拍/*_法拍房源*.csv")):
|
||
try:
|
||
with open(csv_path, encoding="utf-8-sig") as f:
|
||
reader = csv.DictReader(f)
|
||
for row in reader:
|
||
pid = row.get("拍卖ID", "")
|
||
if not pid:
|
||
continue
|
||
if pid not in all_items:
|
||
all_items[pid] = row
|
||
elif row.get("结束时间") and not all_items[pid].get("结束时间"):
|
||
all_items[pid] = row
|
||
except Exception as e:
|
||
print(f"Warning: {csv_path}: {e}")
|
||
|
||
items = list(all_items.values())
|
||
|
||
def compute_yearly(rows, region_filter=None):
|
||
if region_filter:
|
||
rows = [r for r in rows if region_filter in r.get("标题", "")]
|
||
listed = Counter()
|
||
failed = Counter()
|
||
ended = Counter()
|
||
for r in rows:
|
||
start = r.get("开始时间", "")[:7]
|
||
end = r.get("结束时间", "")[:7]
|
||
status = r.get("状态", "")
|
||
bid = r.get("出价次数", "")
|
||
if start:
|
||
listed[start[:4]] += 1
|
||
if end and status == "已结束":
|
||
ended[end[:4]] += 1
|
||
if bid in ("0", ""):
|
||
failed[end[:4]] += 1
|
||
years = sorted(listed.keys() | ended.keys())
|
||
results = []
|
||
for y in years:
|
||
l = listed.get(y, 0)
|
||
e = ended.get(y, 0)
|
||
f = failed.get(y, 0)
|
||
rate = round(f / e * 100, 1) if e else None
|
||
results.append((y, l, e, f, rate))
|
||
return results
|
||
|
||
for region_name, region_filter in [("东莞全市", None), ("樟木头", "樟木头"), ("塘厦", "塘厦")]:
|
||
data = compute_yearly(items, region_filter)
|
||
total = sum(r[1] for r in data)
|
||
total_f = sum(r[3] for r in data)
|
||
total_e = sum(r[2] for r in data)
|
||
overall_rate = round(total_f / total_e * 100, 1) if total_e else 0
|
||
print(f"\n{'='*60}")
|
||
print(f"{region_name}（共 {total} 条，流拍 {total_f} 条，整体流拍率 {overall_rate}%）")
|
||
print(f"{'Year':>6} {'上架':>6} {'已结束':>6} {'流拍':>6} {'流拍率':>8}")
|
||
for y, l, e, f, rate in data:
|
||
rate_str = f"{rate}%" if rate is not None else "—"
|
||
print(f"{y:>6} {l:>6} {e:>6} {f:>6} {rate_str:>8}")
|
||
|
||
print(f"\n总记录数: {len(items)}")
|
||
csv_count = len(glob.glob("output/法拍/*_法拍房源*.csv"))
|
||
print(f"CSV文件数: {csv_count}")
|