Add 东莞法拍房 historical data: 31 sub-regions, 24,949 records (2017-2026)

Scraped all 31 Dongguan sub-regions using sortField=2 (end-time ascending)
to bypass JD's ~4000-item API cap. Merged 34 CSV files by paimaiId into
24,949 unique records covering 2017-07 to 2026-11.

Key findings:
- 上架量 grew ~50x: 98 (2017) → 4,813 (2026)
- 流拍率 peaked at 81.6% (2024), eased to 67.4% (2026)
- 樟木头: 558 records, failure rate peaked 94.2% (2024)
- 塘厦: 205 records, 2026 failure rate 51.0%

Includes: scrape_history.py, batch_scrape_towns.sh, analyze_trends.py,
yearly_stats.py, and updated SKILL.md + url_structure.md documenting
the 4000-item cap and sub-region scraping strategy.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
wiki-agent
2026-09-12 02:59:54 +00:00
co-authored by Claude Opus 4.6
parent bfbb4e6a26
commit f06d84a003
40 changed files with 25953 additions and 5 deletions
+520
View File
@@ -0,0 +1,520 @@
#!/usr/bin/env python3
"""
Generate trend analysis HTML from JD法拍 CSV files.
Reads all *_法拍房源*.csv files in output/法拍/, merges by paimaiId,
computes monthly trends (上架量/流拍量), and generates an interactive HTML chart.
"""
import csv
import json
import os
import glob
from collections import Counter
CSV_DIR = "output/法拍"
OUTPUT_HTML = os.path.join(CSV_DIR, "东莞法拍房趋势分析.html")
def read_all_csvs(csv_dir):
all_items = {}
csv_files = glob.glob(os.path.join(csv_dir, "*_法拍房源*.csv"))
for csv_path in csv_files:
try:
with open(csv_path, encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
for row in reader:
pid = row.get("拍卖ID", "")
if not pid:
continue
if pid not in all_items:
all_items[pid] = row
else:
existing = all_items[pid]
if row.get("结束时间") and not existing.get("结束时间"):
all_items[pid] = row
except Exception as e:
print(f"Warning: failed to read {csv_path}: {e}")
return list(all_items.values())
def compute_monthly_trends(items, region_filter=None):
if region_filter:
items = [r for r in items if region_filter in r.get("标题", "")]
listed_by_month = Counter()
failed_by_month = Counter()
ended_by_month = Counter()
for r in items:
start = r.get("开始时间", "")[:7]
end = r.get("结束时间", "")[:7]
status = r.get("状态", "")
bid_count = r.get("出价次数", "")
if start:
listed_by_month[start] += 1
if end and status == "已结束":
ended_by_month[end] += 1
if bid_count in ("0", ""):
failed_by_month[end] += 1
month_set = (
set(listed_by_month.keys())
| set(ended_by_month.keys())
| set(failed_by_month.keys())
)
if not month_set:
return []
all_months = sorted(month_set)
start_m = all_months[0]
end_m = all_months[-1]
full_months = []
y, m = int(start_m[:4]), int(start_m[5:7])
ey, em = int(end_m[:4]), int(end_m[5:7])
while (y, m) <= (ey, em):
full_months.append(f"{y:04d}-{m:02d}")
m += 1
if m > 12:
m = 1
y += 1
results = []
for month in full_months:
listed = listed_by_month.get(month, 0)
failed = failed_by_month.get(month, 0)
ended = ended_by_month.get(month, 0)
rate = round(failed / ended * 100, 1) if ended > 0 else None
results.append(
{
"month": month,
"listed": listed,
"failed": failed,
"ended": ended,
"rate": rate,
}
)
return results
def generate_svg(data, panel_idx):
n = len(data)
if n == 0:
return "<svg></svg>"
W, H = 1120, 240
padL, padR, padT, padB = 52, 24, 16, 40
plotW = W - padL - padR
plotH = H - padT - padB
max_val = max((max(d["listed"], d["failed"]) for d in data), default=1)
max_val = max(max_val, 5)
y_ticks = 5
y_step_val = max_val / y_ticks if max_val > 0 else 1
def y_pos(val):
if max_val == 0:
return padT + plotH
return padT + plotH - (val / max_val) * plotH
def x_pos(i):
if n == 1:
return padL + plotW / 2
return padL + (i / (n - 1)) * plotW
svg_parts = []
# Y-axis grid lines and labels
for t in range(y_ticks + 1):
val = t * y_step_val
y = y_pos(val)
svg_parts.append(
f'<line x1="{padL}" y1="{y:.1f}" x2="{W-padR}" y2="{y:.1f}" '
f'stroke="var(--gridline)" stroke-width="1"/>'
)
svg_parts.append(
f'<text x="{padL-8}" y="{y+4:.1f}" text-anchor="end" font-size="11" '
f'fill="var(--text-muted)" font-family="system-ui" '
f'font-variant-numeric="tabular-nums">{int(val)}</text>'
)
# X-axis labels (year markers)
prev_year = None
for i, d in enumerate(data):
year = d["month"][:4]
if year != prev_year:
x = x_pos(i)
svg_parts.append(
f'<text x="{x:.1f}" y="{H-12}" text-anchor="middle" font-size="11" '
f'fill="var(--text-muted)" font-family="system-ui">{year}</text>'
)
if prev_year is not None:
svg_parts.append(
f'<line x1="{x:.1f}" y1="{padT}" x2="{x:.1f}" y2="{padT+plotH}" '
f'stroke="var(--gridline)" stroke-width="0.5" stroke-dasharray="2 4"/>'
)
prev_year = year
# Axis line
svg_parts.append(
f'<line x1="{padL}" y1="{padT+plotH}" x2="{W-padR}" y2="{padT+plotH}" '
f'stroke="var(--axis-line)" stroke-width="1"/>'
)
# Area fill for listed (blue)
area_pts = [f"{x_pos(0):.1f},{y_pos(data[0]['listed']):.1f}"]
for i, d in enumerate(data):
area_pts.append(f"{x_pos(i):.1f},{y_pos(d['listed']):.1f}")
area_pts.append(f"{x_pos(n-1):.1f},{padT+plotH:.1f}")
area_pts.append(f"{x_pos(0):.1f},{padT+plotH:.1f}")
svg_parts.append(
f'<path d="M{" L".join(area_pts)} Z" fill="var(--series-1)" opacity="0.06"/>'
)
# Area fill for failed (orange)
area_pts2 = [f"{x_pos(0):.1f},{y_pos(data[0]['failed']):.1f}"]
for i, d in enumerate(data):
area_pts2.append(f"{x_pos(i):.1f},{y_pos(d['failed']):.1f}")
area_pts2.append(f"{x_pos(n-1):.1f},{padT+plotH:.1f}")
area_pts2.append(f"{x_pos(0):.1f},{padT+plotH:.1f}")
svg_parts.append(
f'<path d="M{" L".join(area_pts2)} Z" fill="var(--series-2)" opacity="0.06"/>'
)
# Line for listed
line_pts = [f"{x_pos(i):.1f},{y_pos(d['listed']):.1f}" for i, d in enumerate(data)]
svg_parts.append(
f'<path d="M{" L".join(line_pts)}" fill="none" stroke="var(--series-1)" '
f'stroke-width="1.5" stroke-linejoin="round" stroke-linecap="round"/>'
)
# Line for failed
line_pts2 = [f"{x_pos(i):.1f},{y_pos(d['failed']):.1f}" for i, d in enumerate(data)]
svg_parts.append(
f'<path d="M{" L".join(line_pts2)}" fill="none" stroke="var(--series-2)" '
f'stroke-width="1.5" stroke-linejoin="round" stroke-linecap="round"/>'
)
# Data point circles + hit areas
hit_w = plotW / n
for i, d in enumerate(data):
x = x_pos(i)
y1 = y_pos(d["listed"])
y2 = y_pos(d["failed"])
r = 2.5 if n > 50 else 3.5
svg_parts.append(
f'<circle cx="{x:.1f}" cy="{y1:.1f}" r="{r}" fill="var(--series-1)" '
f'stroke="var(--surface-1)" stroke-width="1.5"/>'
)
svg_parts.append(
f'<circle cx="{x:.1f}" cy="{y2:.1f}" r="{r}" fill="var(--series-2)" '
f'stroke="var(--surface-1)" stroke-width="1.5"/>'
)
hx = x - hit_w / 2
svg_parts.append(
f'<rect x="{hx:.1f}" y="{padT}" width="{hit_w:.1f}" height="{plotH}" '
f'fill="transparent" class="hit" data-idx="{i}"/>'
)
# Last point data labels
last = data[-1]
lx = x_pos(n - 1)
svg_parts.append(
f'<text x="{lx-6:.1f}" y="{y_pos(last["listed"])-8:.1f}" text-anchor="end" '
f'font-size="11" fill="var(--text-primary)" font-family="system-ui" '
f'font-variant-numeric="tabular-nums" font-weight="600">{last["listed"]}</text>'
)
if last["failed"] > 0:
svg_parts.append(
f'<text x="{lx-6:.1f}" y="{y_pos(last["failed"])+14:.1f}" text-anchor="end" '
f'font-size="11" fill="var(--text-primary)" font-family="system-ui" '
f'font-variant-numeric="tabular-nums">{last["failed"]}</text>'
)
# Crosshair
svg_parts.append(
f'<line class="crosshair" x1="0" y1="{padT}" x2="0" y2="{padT+plotH}" '
f'stroke="var(--text-muted)" stroke-width="1" stroke-dasharray="3 3" opacity="0"/>'
)
return (
f'<svg viewBox="0 0 {W} {H}" width="100%" height="{H}">'
+ "".join(svg_parts)
+ "</svg>"
)
def generate_table_row_html(region, data):
rows = [f'<tr class="group-header"><td colspan="6">{region}(共 {len(data)} 个月)</td></tr>']
for d in data:
rate_str = f'{d["rate"]:.1f}%' if d["rate"] is not None else ""
rows.append(
f'<tr><td></td><td style="text-align:left">{d["month"]}</td>'
f'<td>{d["listed"]}</td><td>{d["failed"]}</td>'
f'<td>{d["ended"]}</td><td>{rate_str}</td></tr>'
)
return "\n".join(rows)
def generate_html(panels, items_count):
svgs = {}
for key, data in panels.items():
idx = list(panels.keys()).index(key)
svgs[key] = generate_svg(data, idx)
data_json = json.dumps(panels, ensure_ascii=False)
panel_keys = list(panels.keys())
# Date range
all_months = []
for data in panels.values():
all_months.extend([d["month"] for d in data])
date_range = f"{min(all_months)} {max(all_months)}" if all_months else ""
# Sample counts
sample_counts = {}
for key in panel_keys:
if key == "东莞全市":
sample_counts[key] = items_count
else:
sample_counts[key] = sum(
1 for d in panels[key] for _ in range(d["listed"])
)
table_html = "\n".join(
generate_table_row_html(key, data) for key, data in panels.items()
)
panel_html = ""
for i, key in enumerate(panel_keys):
subtitle = f"样本量 {sample_counts[key]}" if key != "东莞全市" else f"全市法拍住宅 · 共 {items_count} 条记录"
panel_html += f"""
<div class="panel">
<div class="panel-header">
<span class="panel-title">{key}</span>
<span class="panel-subtitle">{subtitle}</span>
</div>
<div class="chart-wrap" id="panel-{i}">
{svgs[key]}
<div class="tooltip" id="tooltip-{i}"></div>
</div>
</div>"""
html = f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>东莞法拍房上架量与流拍量趋势</title>
<style>
.viz-root {{
color-scheme: light;
--surface-1: #fcfcfb;
--page-plane: #f9f9f7;
--text-primary: #0b0b0b;
--text-secondary: #52514e;
--text-muted: #898781;
--gridline: #e1e0d9;
--axis-line: #c3c2b7;
--series-1: #2a78d6;
--series-2: #eb6834;
--border-ring: rgba(11,11,11,0.10);
}}
@media (prefers-color-scheme: dark) {{
:root:where(:not([data-theme="light"])) .viz-root {{
color-scheme: dark;
--surface-1: #1a1a19;
--page-plane: #0d0d0d;
--text-primary: #ffffff;
--text-secondary: #c3c2b7;
--text-muted: #898781;
--gridline: #2c2c2a;
--axis-line: #383835;
--series-1: #3987e5;
--series-2: #d95926;
--border-ring: rgba(255,255,255,0.10);
}}
}}
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
body {{
background: var(--page-plane);
font-family: system-ui, -apple-system, "Segoe UI", sans-serif;
color: var(--text-primary);
padding: 32px 24px;
line-height: 1.5;
}}
.viz-root {{
max-width: 1200px;
margin: 0 auto;
background: var(--surface-1);
border-radius: 8px;
padding: 36px 40px 40px;
}}
h1 {{ font-size: 20px; font-weight: 600; margin-bottom: 6px; }}
.subtitle {{ font-size: 13px; color: var(--text-secondary); margin-bottom: 28px; }}
.legend {{ display: flex; gap: 24px; margin-bottom: 24px; font-size: 13px; color: var(--text-secondary); }}
.legend-item {{ display: flex; align-items: center; gap: 8px; }}
.legend-swatch {{ width: 20px; height: 3px; border-radius: 2px; }}
.legend-swatch.s1 {{ background: var(--series-1); }}
.legend-swatch.s2 {{ background: var(--series-2); }}
.panels {{ display: grid; grid-template-columns: 1fr; gap: 36px; }}
.panel {{ border-top: 1px solid var(--border-ring); padding-top: 20px; }}
.panel:first-child {{ border-top: none; padding-top: 0; }}
.panel-header {{ display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 14px; }}
.panel-title {{ font-size: 15px; font-weight: 600; }}
.panel-subtitle {{ font-size: 12px; color: var(--text-muted); }}
.chart-wrap {{ position: relative; overflow-x: auto; }}
svg {{ display: block; width: 100%; min-width: 600px; height: auto; overflow: visible; }}
.tooltip {{
position: absolute; pointer-events: none;
background: var(--surface-1);
border: 1px solid var(--border-ring);
border-radius: 6px;
padding: 10px 14px;
font-size: 12px;
color: var(--text-primary);
box-shadow: 0 4px 12px rgba(0,0,0,0.08);
opacity: 0;
transition: opacity 0.12s;
z-index: 10;
white-space: nowrap;
}}
.tt-month {{ font-weight: 600; margin-bottom: 6px; }}
.tt-row {{ display: flex; justify-content: space-between; gap: 16px; margin-bottom: 2px; }}
.tt-row span {{ display: flex; align-items: center; gap: 6px; }}
.tt-dot {{ width: 8px; height: 8px; border-radius: 50%; display: inline-block; }}
.tt-val {{ font-variant-numeric: tabular-nums; }}
.note {{ font-size: 12px; color: var(--text-muted); margin-top: 24px; line-height: 1.6; }}
.data-table-wrap {{ margin-top: 28px; }}
.data-table-toggle {{
font-size: 13px; color: var(--series-1); cursor: pointer;
background: none; border: none; padding: 4px 0;
font-family: inherit;
}}
.data-table {{
max-height: 0; overflow: hidden; transition: max-height 0.3s;
margin-top: 12px;
}}
.data-table.visible {{ max-height: 600px; overflow-y: auto; }}
.data-table table {{ width: 100%; border-collapse: collapse; font-size: 12px; }}
.data-table th, .data-table td {{ padding: 6px 10px; text-align: right; border-bottom: 1px solid var(--gridline); }}
.data-table th {{ color: var(--text-secondary); font-weight: 600; }}
.data-table td:first-child, .data-table th:first-child {{ text-align: left; }}
.data-table tr.group-header td {{ font-weight: 600; color: var(--text-primary); background: var(--page-plane); }}
</style>
</head>
<body>
<div class="viz-root">
<h1>东莞法拍房上架量与流拍量趋势</h1>
<p class="subtitle">{date_range} · 按拍卖开始时间月度汇总 · 流拍 = 已结束且出价次数为0</p>
<div class="legend">
<div class="legend-item"><span class="legend-swatch s1"></span>上架量</div>
<div class="legend-item"><span class="legend-swatch s2"></span>流拍量</div>
</div>
<div class="panels">
{panel_html}
</div>
<p class="note">
注:上架量按「开始时间」归入对应月份,流拍量按「结束时间」归入对应月份。流拍率 = 流拍量 / 已结束量。近期月份(最近2-3个月)的已结束量和流拍量可能不完整(部分拍卖尚未结束)。
</p>
<div class="data-table-wrap">
<button class="data-table-toggle" id="tableToggle">显示/隐藏数据表</button>
<div class="data-table" id="dataTable">
<table>
<thead><tr><th>区域</th><th>月份</th><th>上架量</th><th>流拍量</th><th>已结束</th><th>流拍率</th></tr></thead>
<tbody>
{table_html}
</tbody>
</table>
</div>
</div>
</div>
<script>
const DATA = {data_json};
const PANEL_KEYS = {json.dumps(panel_keys, ensure_ascii=False)};
function getCss(prop) {{
const root = document.querySelector('.viz-root');
return getComputedStyle(root).getPropertyValue(prop).trim();
}}
function setupPanel(key, idx) {{
const wrap = document.getElementById('panel-' + idx);
const svg = wrap.querySelector('svg');
const crosshair = wrap.querySelector('.crosshair');
const tt = document.getElementById('tooltip-' + idx);
const data = DATA[key];
if (!data || data.length === 0) return;
function showTooltip(i) {{
const d = data[i];
if (!d) return;
const s1 = getCss('--series-1') || '#2a78d6';
const s2 = getCss('--series-2') || '#eb6834';
tt.innerHTML =
'<div class="tt-month">' + d.month + '</div>' +
'<div class="tt-row"><span><span class="tt-dot" style="background:' + s1 + '"></span>上架量</span><span class="tt-val">' + d.listed + ' 套</span></div>' +
'<div class="tt-row"><span><span class="tt-dot" style="background:' + s2 + '"></span>流拍量</span><span class="tt-val">' + d.failed + ' 套</span></div>' +
'<div class="tt-row"><span>已结束</span><span class="tt-val">' + d.ended + ' 套</span></div>' +
'<div class="tt-row"><span>流拍率</span><span class="tt-val">' + (d.rate !== null ? d.rate + '%' : '') + '</span></div>';
const rect = svg.getBoundingClientRect();
const viewBox = svg.viewBox.baseVal;
const scaleX = rect.width / viewBox.width;
const hitEl = wrap.querySelectorAll('.hit')[i];
const hitRect = hitEl.getBoundingClientRect();
tt.style.left = (hitRect.left - rect.left + hitRect.width / 2 + 12) + 'px';
tt.style.top = '10px';
tt.style.opacity = 1;
const cx = parseFloat(hitEl.getAttribute('x')) + parseFloat(hitEl.getAttribute('width')) / 2;
crosshair.setAttribute('x1', cx);
crosshair.setAttribute('x2', cx);
crosshair.style.opacity = 1;
}}
function hide() {{
tt.style.opacity = 0;
crosshair.style.opacity = 0;
}}
wrap.querySelectorAll('.hit').forEach(hit => {{
hit.addEventListener('mouseenter', () => showTooltip(parseInt(hit.dataset.idx)));
hit.addEventListener('mouseleave', hide);
}});
}}
PANEL_KEYS.forEach((k, i) => setupPanel(k, i));
document.getElementById('tableToggle').addEventListener('click', () => {{
document.getElementById('dataTable').classList.toggle('visible');
}});
</script>
</body>
</html>"""
with open(OUTPUT_HTML, "w", encoding="utf-8") as f:
f.write(html)
print(f"Generated: {OUTPUT_HTML}")
for key, data in panels.items():
total_listed = sum(d["listed"] for d in data)
total_failed = sum(d["failed"] for d in data)
print(f" {key}: {len(data)} months, {total_listed} listed, {total_failed} failed")
if __name__ == "__main__":
items = read_all_csvs(CSV_DIR)
print(f"Total unique items: {len(items)}")
panels = {
"东莞全市": compute_monthly_trends(items),
"樟木头": compute_monthly_trends(items, "樟木头"),
"塘厦": compute_monthly_trends(items, "塘厦"),
}
generate_html(panels, len(items))