Files
wiki-agentandClaude Opus 4.6 f06d84a003 Add 东莞法拍房 historical data: 31 sub-regions, 24,949 records (2017-2026)
Scraped all 31 Dongguan sub-regions using sortField=2 (end-time ascending)
to bypass JD's ~4000-item API cap. Merged 34 CSV files by paimaiId into
24,949 unique records covering 2017-07 to 2026-11.

Key findings:
- 上架量 grew ~50x: 98 (2017) → 4,813 (2026)
- 流拍率 peaked at 81.6% (2024), eased to 67.4% (2026)
- 樟木头: 558 records, failure rate peaked 94.2% (2024)
- 塘厦: 205 records, 2026 failure rate 51.0%

Includes: scrape_history.py, batch_scrape_towns.sh, analyze_trends.py,
yearly_stats.py, and updated SKILL.md + url_structure.md documenting
the 4000-item cap and sub-region scraping strategy.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-09-12 02:59:54 +00:00

521 lines
18 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
Generate trend analysis HTML from JD法拍 CSV files.
Reads all *_法拍房源*.csv files in output/法拍/, merges by paimaiId,
computes monthly trends (上架量/流拍量), and generates an interactive HTML chart.
"""
import csv
import json
import os
import glob
from collections import Counter
CSV_DIR = "output/法拍"
OUTPUT_HTML = os.path.join(CSV_DIR, "东莞法拍房趋势分析.html")
def read_all_csvs(csv_dir):
all_items = {}
csv_files = glob.glob(os.path.join(csv_dir, "*_法拍房源*.csv"))
for csv_path in csv_files:
try:
with open(csv_path, encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
for row in reader:
pid = row.get("拍卖ID", "")
if not pid:
continue
if pid not in all_items:
all_items[pid] = row
else:
existing = all_items[pid]
if row.get("结束时间") and not existing.get("结束时间"):
all_items[pid] = row
except Exception as e:
print(f"Warning: failed to read {csv_path}: {e}")
return list(all_items.values())
def compute_monthly_trends(items, region_filter=None):
if region_filter:
items = [r for r in items if region_filter in r.get("标题", "")]
listed_by_month = Counter()
failed_by_month = Counter()
ended_by_month = Counter()
for r in items:
start = r.get("开始时间", "")[:7]
end = r.get("结束时间", "")[:7]
status = r.get("状态", "")
bid_count = r.get("出价次数", "")
if start:
listed_by_month[start] += 1
if end and status == "已结束":
ended_by_month[end] += 1
if bid_count in ("0", ""):
failed_by_month[end] += 1
month_set = (
set(listed_by_month.keys())
| set(ended_by_month.keys())
| set(failed_by_month.keys())
)
if not month_set:
return []
all_months = sorted(month_set)
start_m = all_months[0]
end_m = all_months[-1]
full_months = []
y, m = int(start_m[:4]), int(start_m[5:7])
ey, em = int(end_m[:4]), int(end_m[5:7])
while (y, m) <= (ey, em):
full_months.append(f"{y:04d}-{m:02d}")
m += 1
if m > 12:
m = 1
y += 1
results = []
for month in full_months:
listed = listed_by_month.get(month, 0)
failed = failed_by_month.get(month, 0)
ended = ended_by_month.get(month, 0)
rate = round(failed / ended * 100, 1) if ended > 0 else None
results.append(
{
"month": month,
"listed": listed,
"failed": failed,
"ended": ended,
"rate": rate,
}
)
return results
def generate_svg(data, panel_idx):
n = len(data)
if n == 0:
return "<svg></svg>"
W, H = 1120, 240
padL, padR, padT, padB = 52, 24, 16, 40
plotW = W - padL - padR
plotH = H - padT - padB
max_val = max((max(d["listed"], d["failed"]) for d in data), default=1)
max_val = max(max_val, 5)
y_ticks = 5
y_step_val = max_val / y_ticks if max_val > 0 else 1
def y_pos(val):
if max_val == 0:
return padT + plotH
return padT + plotH - (val / max_val) * plotH
def x_pos(i):
if n == 1:
return padL + plotW / 2
return padL + (i / (n - 1)) * plotW
svg_parts = []
# Y-axis grid lines and labels
for t in range(y_ticks + 1):
val = t * y_step_val
y = y_pos(val)
svg_parts.append(
f'<line x1="{padL}" y1="{y:.1f}" x2="{W-padR}" y2="{y:.1f}" '
f'stroke="var(--gridline)" stroke-width="1"/>'
)
svg_parts.append(
f'<text x="{padL-8}" y="{y+4:.1f}" text-anchor="end" font-size="11" '
f'fill="var(--text-muted)" font-family="system-ui" '
f'font-variant-numeric="tabular-nums">{int(val)}</text>'
)
# X-axis labels (year markers)
prev_year = None
for i, d in enumerate(data):
year = d["month"][:4]
if year != prev_year:
x = x_pos(i)
svg_parts.append(
f'<text x="{x:.1f}" y="{H-12}" text-anchor="middle" font-size="11" '
f'fill="var(--text-muted)" font-family="system-ui">{year}</text>'
)
if prev_year is not None:
svg_parts.append(
f'<line x1="{x:.1f}" y1="{padT}" x2="{x:.1f}" y2="{padT+plotH}" '
f'stroke="var(--gridline)" stroke-width="0.5" stroke-dasharray="2 4"/>'
)
prev_year = year
# Axis line
svg_parts.append(
f'<line x1="{padL}" y1="{padT+plotH}" x2="{W-padR}" y2="{padT+plotH}" '
f'stroke="var(--axis-line)" stroke-width="1"/>'
)
# Area fill for listed (blue)
area_pts = [f"{x_pos(0):.1f},{y_pos(data[0]['listed']):.1f}"]
for i, d in enumerate(data):
area_pts.append(f"{x_pos(i):.1f},{y_pos(d['listed']):.1f}")
area_pts.append(f"{x_pos(n-1):.1f},{padT+plotH:.1f}")
area_pts.append(f"{x_pos(0):.1f},{padT+plotH:.1f}")
svg_parts.append(
f'<path d="M{" L".join(area_pts)} Z" fill="var(--series-1)" opacity="0.06"/>'
)
# Area fill for failed (orange)
area_pts2 = [f"{x_pos(0):.1f},{y_pos(data[0]['failed']):.1f}"]
for i, d in enumerate(data):
area_pts2.append(f"{x_pos(i):.1f},{y_pos(d['failed']):.1f}")
area_pts2.append(f"{x_pos(n-1):.1f},{padT+plotH:.1f}")
area_pts2.append(f"{x_pos(0):.1f},{padT+plotH:.1f}")
svg_parts.append(
f'<path d="M{" L".join(area_pts2)} Z" fill="var(--series-2)" opacity="0.06"/>'
)
# Line for listed
line_pts = [f"{x_pos(i):.1f},{y_pos(d['listed']):.1f}" for i, d in enumerate(data)]
svg_parts.append(
f'<path d="M{" L".join(line_pts)}" fill="none" stroke="var(--series-1)" '
f'stroke-width="1.5" stroke-linejoin="round" stroke-linecap="round"/>'
)
# Line for failed
line_pts2 = [f"{x_pos(i):.1f},{y_pos(d['failed']):.1f}" for i, d in enumerate(data)]
svg_parts.append(
f'<path d="M{" L".join(line_pts2)}" fill="none" stroke="var(--series-2)" '
f'stroke-width="1.5" stroke-linejoin="round" stroke-linecap="round"/>'
)
# Data point circles + hit areas
hit_w = plotW / n
for i, d in enumerate(data):
x = x_pos(i)
y1 = y_pos(d["listed"])
y2 = y_pos(d["failed"])
r = 2.5 if n > 50 else 3.5
svg_parts.append(
f'<circle cx="{x:.1f}" cy="{y1:.1f}" r="{r}" fill="var(--series-1)" '
f'stroke="var(--surface-1)" stroke-width="1.5"/>'
)
svg_parts.append(
f'<circle cx="{x:.1f}" cy="{y2:.1f}" r="{r}" fill="var(--series-2)" '
f'stroke="var(--surface-1)" stroke-width="1.5"/>'
)
hx = x - hit_w / 2
svg_parts.append(
f'<rect x="{hx:.1f}" y="{padT}" width="{hit_w:.1f}" height="{plotH}" '
f'fill="transparent" class="hit" data-idx="{i}"/>'
)
# Last point data labels
last = data[-1]
lx = x_pos(n - 1)
svg_parts.append(
f'<text x="{lx-6:.1f}" y="{y_pos(last["listed"])-8:.1f}" text-anchor="end" '
f'font-size="11" fill="var(--text-primary)" font-family="system-ui" '
f'font-variant-numeric="tabular-nums" font-weight="600">{last["listed"]}</text>'
)
if last["failed"] > 0:
svg_parts.append(
f'<text x="{lx-6:.1f}" y="{y_pos(last["failed"])+14:.1f}" text-anchor="end" '
f'font-size="11" fill="var(--text-primary)" font-family="system-ui" '
f'font-variant-numeric="tabular-nums">{last["failed"]}</text>'
)
# Crosshair
svg_parts.append(
f'<line class="crosshair" x1="0" y1="{padT}" x2="0" y2="{padT+plotH}" '
f'stroke="var(--text-muted)" stroke-width="1" stroke-dasharray="3 3" opacity="0"/>'
)
return (
f'<svg viewBox="0 0 {W} {H}" width="100%" height="{H}">'
+ "".join(svg_parts)
+ "</svg>"
)
def generate_table_row_html(region, data):
rows = [f'<tr class="group-header"><td colspan="6">{region}(共 {len(data)} 个月)</td></tr>']
for d in data:
rate_str = f'{d["rate"]:.1f}%' if d["rate"] is not None else "—"
rows.append(
f'<tr><td></td><td style="text-align:left">{d["month"]}</td>'
f'<td>{d["listed"]}</td><td>{d["failed"]}</td>'
f'<td>{d["ended"]}</td><td>{rate_str}</td></tr>'
)
return "\n".join(rows)
def generate_html(panels, items_count):
svgs = {}
for key, data in panels.items():
idx = list(panels.keys()).index(key)
svgs[key] = generate_svg(data, idx)
data_json = json.dumps(panels, ensure_ascii=False)
panel_keys = list(panels.keys())
# Date range
all_months = []
for data in panels.values():
all_months.extend([d["month"] for d in data])
date_range = f"{min(all_months)} {max(all_months)}" if all_months else ""
# Sample counts
sample_counts = {}
for key in panel_keys:
if key == "东莞全市":
sample_counts[key] = items_count
else:
sample_counts[key] = sum(
1 for d in panels[key] for _ in range(d["listed"])
)
table_html = "\n".join(
generate_table_row_html(key, data) for key, data in panels.items()
)
panel_html = ""
for i, key in enumerate(panel_keys):
subtitle = f"样本量 {sample_counts[key]} 套" if key != "东莞全市" else f"全市法拍住宅 · 共 {items_count} 条记录"
panel_html += f"""
<div class="panel">
<div class="panel-header">
<span class="panel-title">{key}</span>
<span class="panel-subtitle">{subtitle}</span>
</div>
<div class="chart-wrap" id="panel-{i}">
{svgs[key]}
<div class="tooltip" id="tooltip-{i}"></div>
</div>
</div>"""
html = f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>东莞法拍房上架量与流拍量趋势</title>
<style>
.viz-root {{
color-scheme: light;
--surface-1: #fcfcfb;
--page-plane: #f9f9f7;
--text-primary: #0b0b0b;
--text-secondary: #52514e;
--text-muted: #898781;
--gridline: #e1e0d9;
--axis-line: #c3c2b7;
--series-1: #2a78d6;
--series-2: #eb6834;
--border-ring: rgba(11,11,11,0.10);
}}
@media (prefers-color-scheme: dark) {{
:root:where(:not([data-theme="light"])) .viz-root {{
color-scheme: dark;
--surface-1: #1a1a19;
--page-plane: #0d0d0d;
--text-primary: #ffffff;
--text-secondary: #c3c2b7;
--text-muted: #898781;
--gridline: #2c2c2a;
--axis-line: #383835;
--series-1: #3987e5;
--series-2: #d95926;
--border-ring: rgba(255,255,255,0.10);
}}
}}
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
body {{
background: var(--page-plane);
font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
color: var(--text-primary);
padding: 32px 24px;
line-height: 1.5;
}}
.viz-root {{
max-width: 1200px;
margin: 0 auto;
background: var(--surface-1);
border-radius: 8px;
padding: 36px 40px 40px;
}}
h1 {{ font-size: 20px; font-weight: 600; margin-bottom: 6px; }}
.subtitle {{ font-size: 13px; color: var(--text-secondary); margin-bottom: 28px; }}
.legend {{ display: flex; gap: 24px; margin-bottom: 24px; font-size: 13px; color: var(--text-secondary); }}
.legend-item {{ display: flex; align-items: center; gap: 8px; }}
.legend-swatch {{ width: 20px; height: 3px; border-radius: 2px; }}
.legend-swatch.s1 {{ background: var(--series-1); }}
.legend-swatch.s2 {{ background: var(--series-2); }}
.panels {{ display: grid; grid-template-columns: 1fr; gap: 36px; }}
.panel {{ border-top: 1px solid var(--border-ring); padding-top: 20px; }}
.panel:first-child {{ border-top: none; padding-top: 0; }}
.panel-header {{ display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 14px; }}
.panel-title {{ font-size: 15px; font-weight: 600; }}
.panel-subtitle {{ font-size: 12px; color: var(--text-muted); }}
.chart-wrap {{ position: relative; overflow-x: auto; }}
svg {{ display: block; width: 100%; min-width: 600px; height: auto; overflow: visible; }}
.tooltip {{
position: absolute; pointer-events: none;
background: var(--surface-1);
border: 1px solid var(--border-ring);
border-radius: 6px;
padding: 10px 14px;
font-size: 12px;
color: var(--text-primary);
box-shadow: 0 4px 12px rgba(0,0,0,0.08);
opacity: 0;
transition: opacity 0.12s;
z-index: 10;
white-space: nowrap;
}}
.tt-month {{ font-weight: 600; margin-bottom: 6px; }}
.tt-row {{ display: flex; justify-content: space-between; gap: 16px; margin-bottom: 2px; }}
.tt-row span {{ display: flex; align-items: center; gap: 6px; }}
.tt-dot {{ width: 8px; height: 8px; border-radius: 50%; display: inline-block; }}
.tt-val {{ font-variant-numeric: tabular-nums; }}
.note {{ font-size: 12px; color: var(--text-muted); margin-top: 24px; line-height: 1.6; }}
.data-table-wrap {{ margin-top: 28px; }}
.data-table-toggle {{
font-size: 13px; color: var(--series-1); cursor: pointer;
background: none; border: none; padding: 4px 0;
font-family: inherit;
}}
.data-table {{
max-height: 0; overflow: hidden; transition: max-height 0.3s;
margin-top: 12px;
}}
.data-table.visible {{ max-height: 600px; overflow-y: auto; }}
.data-table table {{ width: 100%; border-collapse: collapse; font-size: 12px; }}
.data-table th, .data-table td {{ padding: 6px 10px; text-align: right; border-bottom: 1px solid var(--gridline); }}
.data-table th {{ color: var(--text-secondary); font-weight: 600; }}
.data-table td:first-child, .data-table th:first-child {{ text-align: left; }}
.data-table tr.group-header td {{ font-weight: 600; color: var(--text-primary); background: var(--page-plane); }}
</style>
</head>
<body>
<div class="viz-root">
<h1>东莞法拍房上架量与流拍量趋势</h1>
<p class="subtitle">{date_range} · 按拍卖开始时间月度汇总 · 流拍 = 已结束且出价次数为0</p>
<div class="legend">
<div class="legend-item"><span class="legend-swatch s1"></span>上架量</div>
<div class="legend-item"><span class="legend-swatch s2"></span>流拍量</div>
</div>
<div class="panels">
{panel_html}
</div>
<p class="note">
注:上架量按「开始时间」归入对应月份,流拍量按「结束时间」归入对应月份。流拍率 = 流拍量 / 已结束量。近期月份(最近2-3个月)的已结束量和流拍量可能不完整(部分拍卖尚未结束)。
</p>
<div class="data-table-wrap">
<button class="data-table-toggle" id="tableToggle">显示/隐藏数据表</button>
<div class="data-table" id="dataTable">
<table>
<thead><tr><th>区域</th><th>月份</th><th>上架量</th><th>流拍量</th><th>已结束</th><th>流拍率</th></tr></thead>
<tbody>
{table_html}
</tbody>
</table>
</div>
</div>
</div>
<script>
const DATA = {data_json};
const PANEL_KEYS = {json.dumps(panel_keys, ensure_ascii=False)};
function getCss(prop) {{
const root = document.querySelector('.viz-root');
return getComputedStyle(root).getPropertyValue(prop).trim();
}}
function setupPanel(key, idx) {{
const wrap = document.getElementById('panel-' + idx);
const svg = wrap.querySelector('svg');
const crosshair = wrap.querySelector('.crosshair');
const tt = document.getElementById('tooltip-' + idx);
const data = DATA[key];
if (!data || data.length === 0) return;
function showTooltip(i) {{
const d = data[i];
if (!d) return;
const s1 = getCss('--series-1') || '#2a78d6';
const s2 = getCss('--series-2') || '#eb6834';
tt.innerHTML =
'<div class="tt-month">' + d.month + '</div>' +
'<div class="tt-row"><span><span class="tt-dot" style="background:' + s1 + '"></span>上架量</span><span class="tt-val">' + d.listed + ' 套</span></div>' +
'<div class="tt-row"><span><span class="tt-dot" style="background:' + s2 + '"></span>流拍量</span><span class="tt-val">' + d.failed + ' 套</span></div>' +
'<div class="tt-row"><span>已结束</span><span class="tt-val">' + d.ended + ' 套</span></div>' +
'<div class="tt-row"><span>流拍率</span><span class="tt-val">' + (d.rate !== null ? d.rate + '%' : '—') + '</span></div>';
const rect = svg.getBoundingClientRect();
const viewBox = svg.viewBox.baseVal;
const scaleX = rect.width / viewBox.width;
const hitEl = wrap.querySelectorAll('.hit')[i];
const hitRect = hitEl.getBoundingClientRect();
tt.style.left = (hitRect.left - rect.left + hitRect.width / 2 + 12) + 'px';
tt.style.top = '10px';
tt.style.opacity = 1;
const cx = parseFloat(hitEl.getAttribute('x')) + parseFloat(hitEl.getAttribute('width')) / 2;
crosshair.setAttribute('x1', cx);
crosshair.setAttribute('x2', cx);
crosshair.style.opacity = 1;
}}
function hide() {{
tt.style.opacity = 0;
crosshair.style.opacity = 0;
}}
wrap.querySelectorAll('.hit').forEach(hit => {{
hit.addEventListener('mouseenter', () => showTooltip(parseInt(hit.dataset.idx)));
hit.addEventListener('mouseleave', hide);
}});
}}
PANEL_KEYS.forEach((k, i) => setupPanel(k, i));
document.getElementById('tableToggle').addEventListener('click', () => {{
document.getElementById('dataTable').classList.toggle('visible');
}});
</script>
</body>
</html>"""
with open(OUTPUT_HTML, "w", encoding="utf-8") as f:
f.write(html)
print(f"Generated: {OUTPUT_HTML}")
for key, data in panels.items():
total_listed = sum(d["listed"] for d in data)
total_failed = sum(d["failed"] for d in data)
print(f" {key}: {len(data)} months, {total_listed} listed, {total_failed} failed")
if __name__ == "__main__":
items = read_all_csvs(CSV_DIR)
print(f"Total unique items: {len(items)}")
panels = {
"东莞全市": compute_monthly_trends(items),
"樟木头": compute_monthly_trends(items, "樟木头"),
"塘厦": compute_monthly_trends(items, "塘厦"),
}
generate_html(panels, len(items))