Skip to content

S
skills
Commit 52222f72 authored by 常佳涛's avatar 常佳涛
Browse files
Options

Update VIP report automation and PPT

parent 4b9160b1
Expand all Show whitespace changes
Inline Side-by-side
Showing with 4101 additions and 199 deletions
.gitignore
View file @ 52222f72
......@@ -40,3 +40,11 @@ vip-report/.mpl-cache/
vip-report/.npm-cache/
vip-report/shadowbot_flow*_bundle/
vip-report/shadowbot_flow*_bundle.zip
# VIP report local backups and inspection workdirs
sql_exports_local/
vip-report/tmp*/
vip-report/Report.*backup*.pptx
vip-report/Report.repaired.pptx
vip-report/Report - *.pptx
vip-report/config.yaml.bak*
vip-report/Report.pptx
View file @ 52222f72
No preview for this file type
vip-report/bin/vip-report-render.ps1
View file @ 52222f72
......@@ -31,4 +31,67 @@ $TemplatePath = Resolve-ProjectPath $TemplatePath
$OutputPath = Resolve-ProjectPath $OutputPath
$OperationsPath = Resolve-ProjectPath $OperationsPath
powershell -ExecutionPolicy Bypass -File "$root\scripts\render_template_com.ps1" -TemplatePath "$TemplatePath" -OutputPath "$OutputPath" -OperationsPath "$OperationsPath"
function Resolve-PythonCommand {
$venvPython = Join-Path $root ".venv\Scripts\python.exe"
if (Test-Path -LiteralPath $venvPython) {
return ,$venvPython
}
if (Get-Command py -ErrorAction SilentlyContinue) {
return @("py", "-3")
}
if (Get-Command python -ErrorAction SilentlyContinue) {
return @("python")
}
return @()
}
function Has-NativeChartOps {
param([string]$OpsPath)
if (-not $OpsPath -or -not (Test-Path -LiteralPath $OpsPath)) {
return $false
}
try {
$ops = Get-Content -LiteralPath $OpsPath -Encoding UTF8 -Raw | ConvertFrom-Json
if ($ops.PSObject.Properties.Name -contains "replace_native_charts") {
return (@($ops.replace_native_charts).Count -gt 0)
}
}
catch {
}
return $false
}
try {
& "$root\scripts\render_template_com.ps1" -TemplatePath "$TemplatePath" -OutputPath "$OutputPath" -OperationsPath "$OperationsPath"
}
catch {
$comError = $_
$canFallback = Has-NativeChartOps -OpsPath $OperationsPath
if (-not $canFallback) {
Write-Error $comError
exit 1
}
$pythonCmd = @(Resolve-PythonCommand)
if ($pythonCmd.Count -eq 0) {
Write-Error "COM render failed and python fallback is unavailable (python not found). Original error: $comError"
exit 1
}
$fallbackScript = Join-Path $root "scripts\render_template_openxml.py"
if (-not (Test-Path -LiteralPath $fallbackScript)) {
Write-Error "COM render failed and fallback script is missing: $fallbackScript"
exit 1
}
Write-Warning "COM render failed; falling back to python-pptx renderer."
if ($pythonCmd.Count -eq 2) {
& $pythonCmd[0] $pythonCmd[1] $fallbackScript --template "$TemplatePath" --output "$OutputPath" --operations "$OperationsPath"
} else {
& $pythonCmd[0] $fallbackScript --template "$TemplatePath" --output "$OutputPath" --operations "$OperationsPath"
}
if ($LASTEXITCODE -ne 0) {
Write-Error "Fallback openxml renderer failed (exit=$LASTEXITCODE). Original COM error: $comError"
exit 1
}
}
vip-report/bin/vip-report-tableau-sync.ps1
View file @ 52222f72
......@@ -7,7 +7,7 @@ param(
[int]$ReportYear = 0,
[int]$CompareYear = 0,
[ValidateSet("single", "cumulative")]
[string]$MonthlySalesMode = "single",
[string]$MonthlySalesMode = "cumulative",
[int]$Retries = 3,
[int]$RetryDelaySeconds = 8,
[string]$TemplatePath = "",
......@@ -225,6 +225,7 @@ function Merge-Operations {
replace_tables = @()
replace_charts = @()
replace_images = @()
replace_native_charts = @()
}
foreach ($opsPath in $OpsPaths) {
......@@ -244,6 +245,9 @@ function Merge-Operations {
if ($ops.PSObject.Properties.Name -contains "replace_images") {
$merged.replace_images += @($ops.replace_images)
}
if ($ops.PSObject.Properties.Name -contains "replace_native_charts") {
$merged.replace_native_charts += @($ops.replace_native_charts)
}
}
$json = $merged | ConvertTo-Json -Depth 100
......
vip-report/bin/vip-report-uv-lfl-compose.ps1
View file @ 52222f72
param(
[string]$TableImage = "D:\projects\skills\vip-report\output\vip-report\assets\cep-traffic\cep-traffic-summary.png",
[string]$LineImage = "D:\projects\skills\vip-report\output\vip-report\assets\cep-site-traffic-entry\cep-site-traffic-entry-lfl.png",
[string]$EntryImage = "D:\projects\skills\vip-report\output\vip-report\assets\cep-traffic-summary\cep-traffic-summary-lfl.png",
[string]$OutputPath = "D:\projects\skills\vip-report\output\vip-report\assets\uv-lfl\uv-lfl-report.png",
[int]$ContentWidth = 1040
[string]$ConfigPath = "D:\projects\skills\vip-report\config.yaml",
[string]$ReportMonth = "",
[int]$ReportYear = 0,
[int]$CompareYear = 0,
[string]$OutputOps = ""
)
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
python "$root\scripts\compose_uv_lfl_report.py" `
--table-image "$TableImage" `
--line-image "$LineImage" `
--entry-image "$EntryImage" `
--output "$OutputPath" `
--content-width "$ContentWidth"
$pythonArgs = @(
"$root\scripts\build_slide12_editable_ops.py",
"--config", "$ConfigPath"
)
if ($ReportMonth) {
$pythonArgs += @("--report-month", "$ReportMonth")
}
if ($ReportYear -gt 0) {
$pythonArgs += @("--report-year", "$ReportYear")
}
if ($CompareYear -gt 0) {
$pythonArgs += @("--compare-year", "$CompareYear")
}
if ($OutputOps) {
$pythonArgs += @("--output-ops", "$OutputOps")
}
python @pythonArgs
vip-report/config.yaml
View file @ 52222f72
......@@ -4,10 +4,10 @@ tableau:
password: "1!Cu&euRo17b6yu@R40gr7"
mysql:
host: "192.168.138.182"
host: "127.0.0.1"
port: 3306
username: "ceptest"
password: "ck@123456"
username: "root"
password: "root"
database: "ckc_cep_db_test"
paths:
......@@ -19,7 +19,7 @@ render:
mode: "com"
report:
month_cn: "3"
month_cn: "5"
year: 2026
compare_year: 2025
......
vip-report/requirements.txt
View file @ 52222f72
......@@ -4,3 +4,4 @@ PyYAML
lxml
matplotlib
numpy
openpyxl
vip-report/scripts/build_may_auto_text_ops.py 0 → 100644
View file @ 52222f72
from __future__ import annotations
import json
from decimal import Decimal
from pathlib import Path
import pymysql
import yaml
def dec(value: object) -> Decimal:
return Decimal(str(value or 0))
def pct(value: object) -> str:
return f"{(dec(value) * 100).quantize(Decimal('0.01'))}%"
def wan(value: object) -> str:
return f"{(dec(value) / Decimal('10000')).quantize(Decimal('0.1'))}w"
def sign_pct(curr: object, prev: object) -> str:
curr_d = dec(curr)
prev_d = dec(prev)
if prev_d == 0:
return "0%"
q = ((curr_d - prev_d) / prev_d * 100).quantize(Decimal("1"))
return ("+" if q > 0 else "") + f"{q}%"
def main() -> None:
root = Path(__file__).resolve().parent.parent
cfg = yaml.safe_load((root / "config.yaml").read_text(encoding="utf-8"))
mysql = cfg["mysql"]
conn = pymysql.connect(
host=mysql["host"],
port=int(mysql.get("port", 3306)),
user=mysql.get("username") or mysql.get("user"),
password=mysql["password"],
database=mysql["database"],
charset="utf8mb4",
cursorclass=pymysql.cursors.DictCursor,
)
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
SUM(accumulated_viewers) viewers,
SUM(guided_sales_total) guided_sales,
SUM(brand_new_customers_total) new_customers,
SUM(live_sales_penetration_rate * accumulated_viewers) / NULLIF(SUM(accumulated_viewers), 0) live_pen,
SUM(purchasers_total) / NULLIF(SUM(accumulated_viewers), 0) conversion_rate
FROM cep_live_session_data
WHERE data_month='2026-05'
"""
)
live = cur.fetchone()
cur.execute(
"""
SELECT
SUM(accumulated_viewers) viewers,
SUM(guided_sales_total) guided_sales,
SUM(brand_new_customers_total) new_customers,
SUM(live_sales_penetration_rate * accumulated_viewers) / NULLIF(SUM(accumulated_viewers), 0) live_pen,
SUM(purchasers_total) / NULLIF(SUM(accumulated_viewers), 0) conversion_rate
FROM cep_live_session_data
WHERE data_month='2026-04'
"""
)
live_prev = cur.fetchone()
cur.execute(
"""
SELECT
COUNT(DISTINCT product_spu_id) articles,
SUM(item_count_total) qty,
SUM(sales_total) sales,
SUM(brand_new_customer_count) new_customers
FROM cep_live_goods_effect
WHERE data_month='2026-05'
"""
)
goods = cur.fetchone()
cur.execute(
"""
SELECT
CASE
WHEN category_level_1 IN ('配饰', '珠宝首饰') THEN '配饰'
ELSE category_level_1
END AS cat,
SUM(item_count_total) qty
FROM cep_live_goods_effect
WHERE data_month='2026-05'
GROUP BY cat
"""
)
goods_cats = cur.fetchall()
finally:
conn.close()
cat_total = sum(dec(row["qty"]) for row in goods_cats) or Decimal(1)
cat_parts: list[str] = []
for row in goods_cats:
name = str(row["cat"] or "未知")
share = (dec(row["qty"]) / cat_total * 100).quantize(Decimal("0"))
cat_parts.append(f"{name}{share}%")
texts = {
"cover": "E-Commerce Monthly Sales (VIP Store)May 2026",
"top_bags": "5月流水销量TOP10包款已按五月网页抓取结果更新；榜单、商品图和销售走势来自Tableau五月筛选数据。",
"top_shoes": "5月流水销量TOP10鞋款已按五月网页抓取结果更新；榜单、商品图和销售走势来自Tableau五月筛选数据。",
"uv": "同比去年5月，流量看板已按Compass五月流量来源数据更新；站内/站外、付费/自然流量及入口结构均来自本地库五月抓取结果。",
"warehouse": "本页60仓表现已按Tableau五月筛选结果更新，包含折扣分布、箱包TOP和鞋类TOP三个模块。",
"live14": (
f"本月直播累计观看人数{wan(live['viewers'])}，直播渗透率{pct(live['live_pen'])}，"
f"环比{sign_pct(live['live_pen'], live_prev['live_pen'])}；转化率{pct(live['conversion_rate'])}，"
f"总引导销售{wan(live['guided_sales'])}，品牌新客数{int(dec(live['new_customers'])):,}。"
"直播图表来自本地库cep_live_session_data五月数据。"
),
"live15": (
f"鞋包直播间有销售商品款数总计{int(dec(goods['articles']))}款，类目结构为{'、'.join(cat_parts)}。"
f"销售额{wan(goods['sales'])}，直播商品榜单来自cep_live_goods_effect五月商品效果数据。"
),
"member": "本页会员看板已按五月网页截图更新，四个模块图片来自会员看板五月筛选后的自动截图。",
}
ops = {
"replace_text": [
{"slide": 1, "shape_id": 3, "new_text": texts["cover"]},
{"slide": 9, "shape_id": 32, "new_text": texts["top_bags"]},
{"slide": 10, "shape_id": 15, "new_text": texts["top_shoes"]},
{"slide": 12, "shape_id": 11, "new_text": texts["uv"]},
{"slide": 13, "shape_id": 16, "new_text": texts["warehouse"]},
{"slide": 14, "shape_id": 14, "new_text": texts["live14"]},
{"slide": 15, "shape_id": 8, "new_text": texts["live15"]},
{"slide": 17, "shape_id": 15, "new_text": texts["member"]},
],
"replace_images": [],
"replace_tables": [],
"replace_charts": [],
"replace_native_charts": [],
}
out = root / "output" / "vip-report" / "render-ops.may-202605.auto-text.json"
out.write_text(json.dumps(ops, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps({"operations_path": str(out), "texts": texts}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
vip-report/scripts/build_slide12_editable_ops.py 0 → 100644
View file @ 52222f72
from __future__ import annotations
import argparse
import json
import math
import sys
from decimal import Decimal
from pathlib import Path
from typing import Any
import pymysql
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from runtime_config import add_config_source_args, load_runtime_config
from sync_cep_site_traffic_entry_assets import (
fetch_month_uv_series,
lfl_series,
month_label_to_number,
normalize_month_label,
)
from sync_cep_traffic_assets import build_table_values, query_month_row
from sync_cep_traffic_summary_assets import ENTRY_ORDER, fetch_entry_uv_map
TABLE_HEADERS = [
"Year",
"In-site Traffic",
"In-site Paid Traffic",
"In-site Paid %",
"In-site Organic Traffic",
"In-site Organic %",
"Off-site Traffic",
"Off-site Paid Traffic",
"Off-site Paid %",
"Off-site Organic Traffic",
"Off-site Organic %",
]
def round_up(value: int, step: int) -> int:
if value <= 0:
return step
return ((value + step - 1) // step) * step
def choose_value_axis(values_a: list[Decimal], values_b: list[Decimal]) -> tuple[int, int]:
max_volume = max(int(max(values_a or [Decimal("0")])), int(max(values_b or [Decimal("0")])))
y_step = 5000 if max_volume > 15000 else 2000
y_max = round_up(max_volume, y_step)
return y_max, y_step
def choose_lfl_axis(values: list[Decimal]) -> tuple[int, int, int]:
if not values:
return -20, 60, 20
min_v = float(min(values))
max_v = float(max(values))
span = max(abs(min_v), abs(max_v))
if span <= 80:
step = 20
elif span <= 300:
step = 50
else:
step = 100
low = int(math.floor(min_v / step) * step)
high = int(math.ceil(max_v / step) * step)
if low > 0:
low = 0
if high < 0:
high = 0
if high == low:
high += step
return low, high, step
def build_entry_rows(
cur_map: dict[str, Decimal], prev_map: dict[str, Decimal]
) -> list[dict[str, Any]]:
union_entries = set(cur_map) | set(prev_map)
ordered = [e for e in ENTRY_ORDER if e in union_entries]
extras = sorted(
[e for e in union_entries if e not in ordered],
key=lambda k: (cur_map.get(k, Decimal("0")) + prev_map.get(k, Decimal("0"))),
reverse=True,
)
ordered.extend(extras)
rows: list[dict[str, Any]] = []
for name in ordered:
rows.append(
{
"name": name,
"report_uv": int(cur_map.get(name, Decimal("0"))),
"compare_uv": int(prev_map.get(name, Decimal("0"))),
}
)
return rows
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Build editable PPT operations for slide 12 (UV LFL) without image rendering."
)
add_config_source_args(parser, default_config=Path(r"D:\projects\skills\vip-report\config.yaml"))
parser.add_argument("--report-month", default="", help="Month override, e.g. Jan / 1")
parser.add_argument("--report-year", type=int, default=0, help="Report year override")
parser.add_argument("--compare-year", type=int, default=0, help="Compare year override")
parser.add_argument("--output-ops", default="", help="Optional override output ops path")
return parser.parse_args()
def execute(
config: dict[str, Any],
*,
report_month: str = "",
report_year: int = 0,
compare_year: int = 0,
output_ops: str = "",
) -> dict[str, str]:
report_cfg = config.get("report", {})
month_label = normalize_month_label(report_month or report_cfg.get("month_cn", "一月"))
month_num = month_label_to_number(month_label)
report_year = int(report_year or report_cfg.get("year", 2026))
compare_year = int(compare_year or report_cfg.get("compare_year", report_year - 1))
mysql_cfg = config["mysql"]
conn = pymysql.connect(
host=mysql_cfg["host"],
port=int(mysql_cfg["port"]),
user=mysql_cfg["username"],
password=mysql_cfg["password"],
database=mysql_cfg["database"],
charset="utf8mb4",
cursorclass=pymysql.cursors.DictCursor,
)
try:
cur_month_row = query_month_row(conn, year=report_year, month_num=month_num)
prev_month_row = query_month_row(conn, year=compare_year, month_num=month_num)
table_current, table_previous, table_lfl = build_table_values(
report_year=report_year,
compare_year=compare_year,
current=cur_month_row,
previous=prev_month_row,
)
cur_uv = fetch_month_uv_series(conn, year=report_year, month=month_num)
prev_uv = fetch_month_uv_series(conn, year=compare_year, month=month_num)
free_lfl = lfl_series(cur_uv["free"], prev_uv["free"])
paid_lfl = lfl_series(cur_uv["paid"], prev_uv["paid"])
cur_entry_map = fetch_entry_uv_map(conn, daily_month=f"{report_year}-{month_num:02d}")
prev_entry_map = fetch_entry_uv_map(conn, daily_month=f"{compare_year}-{month_num:02d}")
finally:
conn.close()
free_axis_max, free_axis_step = choose_value_axis(cur_uv["free"], prev_uv["free"])
paid_axis_max, paid_axis_step = choose_value_axis(cur_uv["paid"], prev_uv["paid"])
free_lfl_min, free_lfl_max, free_lfl_step = choose_lfl_axis(free_lfl)
paid_lfl_min, paid_lfl_max, paid_lfl_step = choose_lfl_axis(paid_lfl)
vip_workdir = Path(config["paths"]["workdir"]).resolve()
ops_path = Path(output_ops).resolve() if output_ops else (vip_workdir / "render-ops.uv-lfl-s12.editable.json")
ops_path.parent.mkdir(parents=True, exist_ok=True)
operations = {
"replace_text": [],
"replace_tables": [],
"replace_charts": [],
"replace_images": [],
"replace_native_charts": [
{
"type": "slide12_traffic_table",
"slide": 12,
"shape_id": 3,
"shape_name": "Table 17",
"target_box": {
"left": 1835149,
"top": 1557195,
"width": 16160753,
"height": 1465080,
},
"headers": TABLE_HEADERS,
"rows": [table_current, table_previous, table_lfl],
},
{
"type": "slide12_dual_line_lfl",
"slide": 12,
"shape_id": 5,
"shape_name": "Group 24",
"target_box": {
"left": 1987549,
"top": 3062843,
"width": 15887698,
"height": 3109300,
},
"month": month_num,
"report_year": report_year,
"compare_year": compare_year,
"panels": [
{
"title": "Organic Traffic LFL",
"current_name": f"{str(report_year)[-2:]} Organic Traffic",
"previous_name": f"{str(compare_year)[-2:]} Organic Traffic",
"current_values": [int(v) for v in cur_uv["free"]],
"previous_values": [int(v) for v in prev_uv["free"]],
"lfl_values": [round(float(v), 2) for v in free_lfl],
"value_axis_max": free_axis_max,
"value_axis_major_unit": free_axis_step,
"lfl_axis_min": free_lfl_min,
"lfl_axis_max": free_lfl_max,
"lfl_axis_major_unit": free_lfl_step,
},
{
"title": "Paid Traffic LFL",
"current_name": f"{str(report_year)[-2:]} Paid Traffic",
"previous_name": f"{str(compare_year)[-2:]} Paid Traffic",
"current_values": [int(v) for v in cur_uv["paid"]],
"previous_values": [int(v) for v in prev_uv["paid"]],
"lfl_values": [round(float(v), 2) for v in paid_lfl],
"value_axis_max": paid_axis_max,
"value_axis_major_unit": paid_axis_step,
"lfl_axis_min": paid_lfl_min,
"lfl_axis_max": paid_lfl_max,
"lfl_axis_major_unit": paid_lfl_step,
},
],
},
{
"type": "slide12_entry_mix",
"slide": 12,
"shape_id": 8,
"shape_name": "Group 25",
"target_box": {
"left": 1987549,
"top": 6093856,
"width": 16160750,
"height": 2729245,
},
"report_year": report_year,
"compare_year": compare_year,
"entries": build_entry_rows(cur_entry_map, prev_entry_map),
},
],
}
ops_path.write_text(json.dumps(operations, ensure_ascii=False, indent=2), encoding="utf-8")
return {"operations_path": str(ops_path)}
def main() -> None:
args = parse_args()
config = load_runtime_config(args, default_config=Path(r"D:\projects\skills\vip-report\config.yaml"))
result = execute(
config,
report_month=args.report_month,
report_year=args.report_year,
compare_year=args.compare_year,
output_ops=args.output_ops,
)
print(json.dumps(result, ensure_ascii=False))
if __name__ == "__main__":
main()
vip-report/scripts/build_slide14_15_editable_ops.py 0 → 100644
View file @ 52222f72
This diff is collapsed.
vip-report/scripts/build_slide16_otd_ops.py 0 → 100644
View file @ 52222f72
This diff is collapsed.
vip-report/scripts/download_compass_task_file.py 0 → 100644
View file @ 52222f72
from __future__ import annotations
import argparse
import base64
import json
import os
import shutil
import subprocess
import time
from pathlib import Path
import yaml
NPX_EXECUTABLE = shutil.which("npx.cmd") or shutil.which("npx") or "npx"
PLAYWRIGHT_CMD = [NPX_EXECUTABLE, "--yes", "--package", "@playwright/cli", "playwright-cli"]
SESSION_NAME = "vip-report-monthly-sales"
DOWNLOAD_CENTER_URL = "https://compass.vip.com/frontend/index.html#/dataCapturing/downloadCenter"
def run_cmd(args: list[str], *, cwd: Path, timeout: int = 300) -> subprocess.CompletedProcess[str]:
env = os.environ.copy()
npm_cache = cwd / ".npm-cache"
npm_cache.mkdir(parents=True, exist_ok=True)
env["NPM_CONFIG_CACHE"] = str(npm_cache)
env["npm_config_cache"] = str(npm_cache)
return subprocess.run(
args,
cwd=str(cwd),
text=True,
encoding="utf-8",
errors="replace",
capture_output=True,
timeout=timeout,
check=True,
env=env,
)
def build_js(task_id: str) -> str:
payload = json.dumps({"task_id": task_id, "url": DOWNLOAD_CENTER_URL}, ensure_ascii=False)
return f"""async function(page) {{
const spec = {payload};
await page.goto(spec.url, {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
await page.waitForTimeout(1000);
const result = await page.evaluate(async (taskId) => {{
const body = new URLSearchParams();
body.set('reqData', JSON.stringify({{ taskId }}));
const res = await fetch('/report/asyncdownload/getOutputFile', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/x-www-form-urlencoded' }},
body: body.toString(),
credentials: 'include',
}});
const ab = await res.arrayBuffer();
const bytes = new Uint8Array(ab);
let binary = '';
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {{
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}}
return {{
ok: res.ok,
status: res.status,
contentType: res.headers.get('content-type') || '',
disposition: res.headers.get('content-disposition') || '',
base64: btoa(binary),
}};
}}, String(spec.task_id));
return result;
}}
"""
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Download a ready Compass async task file by task id.")
p.add_argument("--config", default=r"D:\projects\skills\vip-report\config.yaml")
p.add_argument("--session", default=SESSION_NAME)
p.add_argument("--task-id", required=True)
p.add_argument("--output", required=True)
return p.parse_args()
def main() -> None:
args = parse_args()
cfg = yaml.safe_load(Path(args.config).read_text(encoding="utf-8"))
workdir = Path(cfg["paths"]["workdir"]).resolve()
js_path = workdir / f"tmp-download-task-{args.task_id}-{time.time_ns()}.js"
js_path.write_text(build_js(args.task_id), encoding="utf-8")
try:
out = run_cmd(
PLAYWRIGHT_CMD + ["--session", args.session, "run-code", "--filename", str(js_path)],
cwd=workdir,
timeout=420,
).stdout
finally:
js_path.unlink(missing_ok=True)
if "### Error" in out:
raise RuntimeError(out.strip())
json_text = ""
marker = "### Result"
marker_idx = out.find(marker)
if marker_idx >= 0:
start = out.find("\n", marker_idx)
if start >= 0:
end = out.find("\n###", start + 1)
json_text = out[start + 1 : end if end >= 0 else None].strip()
if not json_text:
for line in out.splitlines():
s = line.strip()
if s.startswith("{") and s.endswith("}"):
json_text = s
break
if not json_text:
raise RuntimeError(f"No JSON result from run-code: {out}")
result = json.loads(json_text)
if not result.get("ok", False):
raise RuntimeError(json.dumps(result, ensure_ascii=False))
output_path = Path(args.output).resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(base64.b64decode(result["base64"]))
result.pop("base64", None)
result["savedAs"] = str(output_path)
print(json.dumps(result, ensure_ascii=True))
if __name__ == "__main__":
main()
vip-report/scripts/load_compass_rejection_goods_cn_to_mysql.py
View file @ 52222f72
......@@ -170,8 +170,9 @@ def main() -> None:
raise RuntimeError(f"Missing required columns: {missing}")
time_values = {str(x).strip() for x in df["时间"].dropna().head(20)}
if not any("202603" in x for x in time_values):
raise RuntimeError(f"Unexpected time values for 2026-03 file: {sorted(time_values)[:5]}")
expected_month = data_month.replace("-", "")
if not any(expected_month in x for x in time_values):
raise RuntimeError(f"Unexpected time values for {data_month} file: {sorted(time_values)[:5]}")
rows = parse_rows(df, data_month, source_file.name)
mysql_cfg = cfg["mysql"]
......
vip-report/scripts/render_template_com.ps1
View file @ 52222f72
This diff is collapsed.
vip-report/scripts/render_template_openxml.py 0 → 100644
View file @ 52222f72
This diff is collapsed.
vip-report/scripts/run_flow12_traffic_pipeline.py
View file @ 52222f72
......@@ -52,7 +52,7 @@ def main() -> None:
]
if args.config_source == "yaml":
config_path = args.config or str((root.parent / "config.yaml").resolve())
common = ["--config", config_path]
common = ["--config-source", "yaml", "--config", config_path]
runtime_cfg = load_runtime_config(args, default_config=Path(args.config) if args.config else (root.parent / "config.yaml"))
workdir = Path(runtime_cfg["paths"]["workdir"]).resolve()
......@@ -65,20 +65,10 @@ def main() -> None:
args.download_dir,
],
[args.python, str(root / "load_traffic_downloads_to_mysql.py"), *common],
[args.python, str(root / "sync_cep_traffic_assets.py"), *common],
[args.python, str(root / "sync_cep_site_traffic_entry_assets.py"), *common],
[args.python, str(root / "sync_cep_traffic_summary_assets.py"), *common],
[
args.python,
str(root / "compose_uv_lfl_report.py"),
"--table-image",
str(workdir / "assets" / "cep-traffic" / "cep-traffic-summary.png"),
"--line-image",
str(workdir / "assets" / "cep-site-traffic-entry" / "cep-site-traffic-entry-lfl.png"),
"--entry-image",
str(workdir / "assets" / "cep-traffic-summary" / "cep-traffic-summary-lfl.png"),
"--output",
str(workdir / "assets" / "uv-lfl" / "uv-lfl-report.png"),
str(root / "build_slide12_editable_ops.py"),
*common,
],
]
......@@ -102,8 +92,8 @@ def main() -> None:
f"stderr_tail=\n{err_tail}\n"
)
final_image = workdir / "assets" / "uv-lfl" / "uv-lfl-report.png"
print(f"FINAL_IMAGE={final_image}")
final_ops = workdir / "render-ops.uv-lfl-s12.editable.json"
print(f"FINAL_OPS={final_ops}")
print("FLOW12_PIPELINE_OK")
......
vip-report/scripts/run_flow14_livestream_pipeline.py
View file @ 52222f72
......@@ -25,7 +25,8 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--mysql-password", default="ck@123456")
p.add_argument("--mysql-database", default="ckc_cep_db_test")
p.add_argument("--download-dir", default=r"C:\vipdata")
p.add_argument("--skip-compose", action="store_true", help="Only download/import/render 2 images, skip final compose")
p.add_argument("--skip-compose", action="store_true", help="Deprecated. Compose preview is disabled by default.")
p.add_argument("--skip-editable-ops", action="store_true", help="Skip generating slide14 editable ops.")
return p.parse_args()
......@@ -50,7 +51,7 @@ def build_common_args(args: argparse.Namespace) -> list[str]:
]
if args.config_source == "yaml":
config_path = args.config or str((SCRIPT_DIR.parent / "config.yaml").resolve())
return ["--config", config_path]
return ["--config-source", "yaml", "--config", config_path]
return common
......@@ -97,20 +98,14 @@ def main() -> None:
[args.python, str(root / "sync_cep_live_session_trend_assets.py"), *common],
]
if not args.skip_compose:
if not args.skip_editable_ops:
steps.append(
[
args.python,
str(root / "compose_live_session_report.py"),
str(root / "build_slide14_15_editable_ops.py"),
*common,
"--summary-image",
str(workdir / "assets" / "cep-live-session" / "cep-live-session-summary.png"),
"--trend-image",
str(workdir / "assets" / "cep-live-session" / "cep-live-session-sales-penetration-trend.png"),
"--pivot-image",
str(workdir / "assets" / "cep-live-goods-effect" / "cep-live-goods-effect-pivot.png"),
"--output",
str(workdir / "assets" / "cep-live-session" / "cep-live-session-combined.png"),
"--slides",
"14",
]
)
......@@ -118,10 +113,9 @@ def main() -> None:
for i, cmd in enumerate(steps, start=1):
run_step(i, total, cmd)
print(f"FINAL_IMAGE={workdir / 'assets' / 'cep-live-session' / 'cep-live-session-combined.png'}")
print(f"FINAL_OPS={workdir / 'render-ops.flow14-15.editable.json'}")
print("FLOW14_PIPELINE_OK")
if __name__ == "__main__":
main()
vip-report/scripts/run_flow15_live_goods_effect_pipeline.py
View file @ 52222f72
......@@ -3,7 +3,9 @@ from __future__ import annotations
import argparse
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
......@@ -12,6 +14,12 @@ if str(SCRIPT_DIR) not in sys.path:
from runtime_config import load_runtime_config
@dataclass(frozen=True)
class MonthKey:
year: int
month: int
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Run flow15 live-goods-effect pipeline in one command.")
p.add_argument("--python", default=sys.executable, help="Python executable path")
......@@ -25,14 +33,17 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--mysql-password", default="ck@123456")
p.add_argument("--mysql-database", default="ckc_cep_db_test")
p.add_argument("--download-dir", default=r"C:\vipdata")
p.add_argument("--skip-compose", action="store_true")
p.add_argument("--report-year", type=int, default=0)
p.add_argument("--report-month", default="")
p.add_argument("--skip-compose", action="store_true", help="Deprecated. Compose preview is disabled by default.")
p.add_argument("--skip-editable-ops", action="store_true", help="Skip generating slide15 editable ops.")
return p.parse_args()
def build_common_args(args: argparse.Namespace) -> list[str]:
if args.config_source == "yaml":
config_path = args.config or str((SCRIPT_DIR.parent / "config.yaml").resolve())
return ["--config", config_path]
return ["--config-source", "yaml", "--config", config_path]
return [
"--config-source",
args.config_source,
......@@ -53,6 +64,41 @@ def build_common_args(args: argparse.Namespace) -> list[str]:
]
def parse_month_number(raw: Any) -> int:
if raw is None:
return 1
if isinstance(raw, int):
return min(12, max(1, raw))
text = str(raw).strip()
import re
m = re.search(r"(1[0-2]|0?[1-9])", text)
if m:
return int(m.group(1))
return 1
def previous_month(m: MonthKey) -> MonthKey:
if m.month == 1:
return MonthKey(m.year - 1, 12)
return MonthKey(m.year, m.month - 1)
def same_month_last_year(m: MonthKey) -> MonthKey:
return MonthKey(m.year - 1, m.month)
def resolve_report_month(args: argparse.Namespace, runtime_cfg: dict[str, Any]) -> MonthKey:
report_cfg = runtime_cfg.get("report", {})
year = int(args.report_year or report_cfg.get("year", 2026))
month = parse_month_number(args.report_month or report_cfg.get("month_cn", "1"))
return MonthKey(year, month)
def with_report_period(cmd: list[str], month: MonthKey) -> list[str]:
return [*cmd, "--report-year", str(month.year), "--report-month", str(month.month)]
def run_step(step_index: int, total: int, cmd: list[str]) -> None:
print(f"[STEP {step_index}/{total}] {' '.join(cmd)}")
result = subprocess.run(cmd, text=True, capture_output=True)
......@@ -82,40 +128,48 @@ def main() -> None:
default_config=Path(args.config) if args.config else (root.parent / "config.yaml"),
)
workdir = Path(runtime_cfg["paths"]["workdir"]).resolve()
report_month = resolve_report_month(args, runtime_cfg)
current_month = report_month
compare_month = same_month_last_year(report_month)
steps: list[list[str]] = [
with_report_period(
[
args.python,
str(root / "sync_vis_vendor_goods_effect_download.py"),
*common,
"--download-dir",
args.download_dir,
"--no-auto-load",
],
current_month,
),
with_report_period(
[
args.python,
str(root / "load_vis_vendor_goods_effect_to_mysql.py"),
str(root / "sync_vis_vendor_goods_effect_download.py"),
*common,
"--download-dir",
args.download_dir,
],
[args.python, str(root / "sync_cep_live_goods_effect_summary_assets.py"), *common],
[args.python, str(root / "sync_cep_live_goods_effect_pivot_assets.py"), *common],
[args.python, str(root / "sync_cep_live_goods_effect_top5_assets.py"), *common],
compare_month,
),
with_report_period([args.python, str(root / "sync_cep_live_goods_effect_summary_assets.py"), *common], report_month),
with_report_period([args.python, str(root / "sync_cep_live_goods_effect_pivot_assets.py"), *common], current_month),
with_report_period([args.python, str(root / "sync_cep_live_goods_effect_top5_assets.py"), *common], current_month),
]
if not args.skip_compose:
if not args.skip_editable_ops:
steps.append(
[
args.python,
str(root / "compose_live_goods_effect_report.py"),
str(root / "build_slide14_15_editable_ops.py"),
*common,
"--summary-image",
str(workdir / "assets" / "cep-live-goods-effect" / "cep-live-goods-effect-summary.png"),
"--top5-image",
str(workdir / "assets" / "cep-live-goods-effect" / "cep-live-goods-effect-top5.png"),
"--output",
str(workdir / "assets" / "cep-live-goods-effect" / "cep-live-goods-effect-combined.png"),
"--slides",
"15",
"--report-year",
str(report_month.year),
"--report-month",
str(report_month.month),
]
)
......@@ -123,10 +177,9 @@ def main() -> None:
for i, cmd in enumerate(steps, start=1):
run_step(i, total, cmd)
print(f"FINAL_IMAGE={workdir / 'assets' / 'cep-live-goods-effect' / 'cep-live-goods-effect-combined.png'}")
print(f"FINAL_OPS={workdir / 'render-ops.flow14-15.editable.json'}")
print("FLOW15_PIPELINE_OK")
if __name__ == "__main__":
main()
vip-report/scripts/run_flow17_membership_pipeline.py
View file @ 52222f72
......@@ -32,7 +32,7 @@ def parse_args() -> argparse.Namespace:
def build_common_args(args: argparse.Namespace) -> list[str]:
if args.config_source == "yaml":
config_path = args.config or str((SCRIPT_DIR.parent / "config.yaml").resolve())
return ["--config", config_path]
return ["--config-source", "yaml", "--config", config_path]
return [
"--config-source",
args.config_source,
......
vip-report/scripts/sync_cep_live_goods_effect_summary_assets.py
View file @ 52222f72
......@@ -280,8 +280,7 @@ def main() -> None:
config_path = Path(args.config or r"D:\projects\skills\vip-report\config.yaml").resolve()
config = load_runtime_config(args, default_config=config_path)
curr_month = resolve_report_month(args, config)
prev_month = previous_month(curr_month)
prev2_month = same_month_last_year(curr_month)
compare_month = same_month_last_year(curr_month)
mysql_cfg = config["mysql"]
mysql_user = mysql_cfg.get("username") or mysql_cfg.get("user")
......@@ -296,11 +295,7 @@ def main() -> None:
)
try:
curr_raw = fetch_month_rows(conn, curr_month.ym)
prev_raw = fetch_month_rows(conn, prev_month.ym)
compare_month = prev_month.ym
if not prev_raw:
prev_raw = fetch_month_rows(conn, prev2_month.ym)
compare_month = prev2_month.ym
prev_raw = fetch_month_rows(conn, compare_month.ym)
finally:
conn.close()
......@@ -320,7 +315,7 @@ def main() -> None:
operations_path.write_text(json.dumps(operations, ensure_ascii=False, indent=2), encoding="utf-8")
manifest = {
"source": {"table": TABLE_NAME, "current_month": curr_month.ym, "compare_month": compare_month, "config_path": str(config_path)},
"source": {"table": TABLE_NAME, "current_month": curr_month.ym, "compare_month": compare_month.ym, "config_path": str(config_path)},
"asset_path": str(image_path),
"operations_path": str(operations_path),
"metrics": metrics,
......
vip-report/scripts/sync_cep_live_session_assets.py
View file @ 52222f72
......@@ -243,7 +243,8 @@ def render_image(output_path: Path, prev_key: MonthKey, prev: Agg, curr_key: Mon
table_w = sum(col_widths)
table_h = row_h * 4
img = Image.new("RGB", (1126, 160), "#e7e7e7")
image_w = max(1126, table_x + table_w + 28)
img = Image.new("RGB", (image_w, 160), "#e7e7e7")
draw = ImageDraw.Draw(img)
font_title = load_font(38, bold=True)
......@@ -257,7 +258,9 @@ def render_image(output_path: Path, prev_key: MonthKey, prev: Agg, curr_key: Mon
draw.line((16, 43, 150, 43), fill="#cf2323", width=2)
# Brand text
draw.text((945, 8), "CHARLES & KEITH", fill="#cfcfcf", font=font_brand)
brand_text = "CHARLES & KEITH"
brand_bbox = draw.textbbox((0, 0), brand_text, font=font_brand)
draw.text((image_w - (brand_bbox[2] - brand_bbox[0]) - 18, 8), brand_text, fill="#cfcfcf", font=font_brand)
# Table header bg
header_bg = "#6c72df"
......
vip-report/scripts/sync_compass_rejection_goods_download.py
View file @ 52222f72
from __future__ import annotations
import argparse
import base64
import json
import os
import re
......@@ -202,23 +203,40 @@ def build_js(spec: dict[str, Any]) -> str:
return s.display !== 'none' && s.visibility !== 'hidden' && r.width > 1 && r.height > 1;
}};
const text = (el) => ((el && el.textContent) || '').replace(/\\s+/g, ' ').trim();
const btn = Array.from(document.querySelectorAll('button,span,a,div')).find((el) => visible(el) && text(el) === '下载数据');
if (!btn) return false;
btn.dispatchEvent(new MouseEvent('click', {{ bubbles: true }}));
return true;
const cards = Array.from(document.querySelectorAll('.g-card')).filter(visible);
const detailCard = cards.find((card) => text(card).includes('拒退商品明细'))
|| cards.find((card) => text(card).includes('款号') && text(card).includes('下载数据'))
|| null;
const root = detailCard || document;
const candidates = Array.from(root.querySelectorAll('.ad-toolbar-box button, .ad-toolbar-box span, button'))
.filter((el) => visible(el) && text(el) === '下载数据');
const btn = candidates.find((el) => el.closest('.ad-toolbar-box')) || candidates[0];
if (!btn) {{
return {{
ok: false,
cardTexts: cards.map((card) => text(card).slice(0, 160)).slice(0, 8),
}};
}}
const clickable = btn.closest('button') || btn;
clickable.dispatchEvent(new MouseEvent('mousedown', {{ bubbles: true }}));
clickable.dispatchEvent(new MouseEvent('mouseup', {{ bubbles: true }}));
clickable.dispatchEvent(new MouseEvent('click', {{ bubbles: true }}));
return {{
ok: true,
cardText: detailCard ? text(detailCard).slice(0, 240) : '',
buttonText: text(clickable),
}};
}});
if (!clickDownload) {{
return {{ ok: false, code: 'DOWNLOAD_DATA_BUTTON_NOT_FOUND', url: page.url() }};
if (!clickDownload.ok) {{
return {{ ok: false, code: 'DOWNLOAD_DATA_BUTTON_NOT_FOUND', clickDownload, url: page.url() }};
}}
await sleep(600);
await sleep(1000);
// click 不需要 in optional modal
try {{
const noNeed = ctx.locator('button:has-text("不需要"), span:has-text("不需要")').first();
if (await noNeed.count().catch(() => 0)) {{
await noNeed.click({{ timeout: 2500 }}).catch(() => null);
await sleep(400);
}}
const noNeed = ctx.locator('.el-message-box button:has-text("不需要"), .el-dialog button:has-text("不需要"), button:has-text("不需要")').first();
await noNeed.click({{ timeout: 15000, force: true }}).catch(() => null);
await sleep(1500);
}} catch {{}}
await page.goto('https://compass.vip.com/frontend/index.html#/dataCapturing/downloadCenter', {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
......@@ -237,6 +255,7 @@ def build_js(spec: dict[str, Any]) -> str:
}}
await sleep(3500);
const rowsSeen = [];
for (let i = 0; i < spec.max_retries; i++) {{
const queryBtn = page.locator('button:has-text("查询"), span:has-text("查询")').first();
if (await queryBtn.count().catch(() => 0)) {{
......@@ -244,43 +263,74 @@ def build_js(spec: dict[str, Any]) -> str:
}}
await sleep(2000);
const statusCell = page.locator('tbody tr:nth-child(1) td').nth(5);
const statusText = await statusCell.innerText().catch(() => '');
if (!/执行成功|成功/.test(statusText || '')) {{
await sleep(700);
continue;
}}
const fileSpan = page.locator('tbody tr:nth-child(1) .status-name-span').first();
const rowCount = await page.locator('tbody tr').count().catch(() => 0);
for (let idx = 0; idx < Math.min(rowCount, 10); idx++) {{
const row = page.locator('tbody tr').nth(idx);
const rowText = (await row.innerText().catch(() => '')).replace(/\\s+/g, ' ').trim();
const fileSpan = row.locator('.status-name-span').first();
let uiFileName = await fileSpan.getAttribute('title').catch(() => '');
if (!uiFileName) {{
uiFileName = (await fileSpan.innerText().catch(() => '')).trim();
}}
rowsSeen.push({{ row: idx + 1, uiFileName, rowText }});
if (!uiFileName.includes('款号粒度') || !/执行成功|成功/.test(rowText || '')) {{
continue;
}}
const downloadBtn = page.locator('tbody tr:nth-child(1) td button:has-text("下载"), tbody tr:nth-child(1) td span:has-text("下载")').first();
const taskId = (rowText.match(/^\\s*(\\d+)/) || [])[1] || '';
const downloadBtn = row.locator('button:has-text("下载"), span:has-text("下载")').first();
if (!(await downloadBtn.count().catch(() => 0))) {{
continue;
}}
const [dl] = await Promise.all([
page.waitForEvent('download', {{ timeout: 120000 }}),
downloadBtn.click({{ timeout: 3000 }}),
]);
const suggested = dl.suggestedFilename();
const finalName = safeName(`${{spec.prefix}}_${{uiFileName || suggested}}`);
if (!taskId) {{
continue;
}}
const filePayload = await page.evaluate(async (tid) => {{
const body = new URLSearchParams();
body.set('reqData', JSON.stringify({{ taskId: tid }}));
const res = await fetch('/report/asyncdownload/getOutputFile', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/x-www-form-urlencoded' }},
body: body.toString(),
credentials: 'include',
}});
const ab = await res.arrayBuffer();
const bytes = new Uint8Array(ab);
let binary = '';
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {{
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}}
return {{
ok: res.ok,
status: res.status,
contentType: res.headers.get('content-type') || '',
disposition: res.headers.get('content-disposition') || '',
base64: btoa(binary),
}};
}}, taskId);
if (!filePayload.ok) {{
throw new Error(`getOutputFile failed status=${{filePayload.status}} taskId=${{taskId}}`);
}}
const finalName = safeName(`${{spec.prefix}}_${{uiFileName || taskId + '.xlsx'}}`);
const targetPath = `${{spec.download_dir.replace(/\\\\/g, '/')}}/${{finalName}}`;
await dl.saveAs(targetPath);
return {{
ok: true,
url: page.url(),
statusText,
row: idx + 1,
taskId,
rowText,
uiFileName,
suggestedFilename: suggested,
contentType: filePayload.contentType,
disposition: filePayload.disposition,
contentBase64: filePayload.base64,
savedAs: targetPath
}};
}}
}}
return {{ ok: false, code: 'DOWNLOAD_NOT_READY', url: page.url() }};
return {{ ok: false, code: 'GOODS_STYLE_FILE_NOT_READY', rowsSeen: rowsSeen.slice(0, 40), url: page.url() }};
}}
"""
......@@ -344,7 +394,12 @@ def main() -> None:
result = json.loads(json_line)
if not result.get("ok", False):
raise RuntimeError(json.dumps(result, ensure_ascii=False))
print(json.dumps(result, ensure_ascii=False))
content_b64 = result.pop("contentBase64", "")
if content_b64:
target = Path(result["savedAs"]).resolve()
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(base64.b64decode(content_b64))
print(json.dumps(result, ensure_ascii=True))
if __name__ == "__main__":
......
vip-report/scripts/sync_compass_rejection_goods_style_download.py 0 → 100644
View file @ 52222f72
from __future__ import annotations
import argparse
import json
import os
import re
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any
import yaml
NPX_EXECUTABLE = shutil.which("npx.cmd") or shutil.which("npx") or "npx"
PLAYWRIGHT_CMD = [NPX_EXECUTABLE, "--yes", "--package", "@playwright/cli", "playwright-cli"]
SESSION_NAME = "vip-report-monthly-sales"
TARGET_URL = "https://compass.vip.com/frontend/index.html#/rejectionAnalysis/index"
DOWNLOAD_CENTER_URL = "https://compass.vip.com/frontend/index.html#/dataCapturing/downloadCenter"
DEFAULT_DOWNLOAD_DIR = r"C:\vipdata"
def run_cmd(args: list[str], *, cwd: Path, timeout: int = 300) -> subprocess.CompletedProcess[str]:
env = os.environ.copy()
npm_cache = cwd / ".npm-cache"
npm_cache.mkdir(parents=True, exist_ok=True)
env["NPM_CONFIG_CACHE"] = str(npm_cache)
env["npm_config_cache"] = str(npm_cache)
return subprocess.run(
args,
cwd=str(cwd),
text=True,
encoding="utf-8",
errors="replace",
capture_output=True,
timeout=timeout,
check=True,
env=env,
)
def parse_month_number(raw: Any) -> int:
if raw is None:
return 1
if isinstance(raw, int):
return min(12, max(1, raw))
m = re.search(r"(1[0-2]|0?[1-9])", str(raw))
return int(m.group(1)) if m else 1
def resolve_period(config_path: Path, report_year_arg: int, report_month_arg: str) -> tuple[int, int]:
cfg = yaml.safe_load(config_path.read_text(encoding="utf-8"))
report_cfg = cfg.get("report", {})
year = int(report_year_arg or report_cfg.get("year", 2026))
month = parse_month_number(report_month_arg or report_cfg.get("month_cn", "1"))
return year, month
def build_js(spec: dict[str, Any]) -> str:
payload = json.dumps(spec, ensure_ascii=False)
return f"""async function(page) {{
const spec = {payload};
const sleep = (ms) => page.waitForTimeout(ms);
const safeName = (name) => String(name || '')
.replace(/[\\\\/:*?"<>|]/g, '_')
.replace(/\\s+/g, '_')
.replace(/_+/g, '_');
await page.goto(`${{spec.url}}?t=${{Date.now()}}`, {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
await sleep(5000);
const tab = page.locator('.el-tabs__item:has-text("拒退商品分析")').first();
if (!(await tab.count().catch(() => 0))) {{
return {{ ok: false, code: 'GOODS_TAB_NOT_FOUND', url: page.url() }};
}}
await tab.click({{ timeout: 5000, force: true }});
await sleep(2500);
const activeText = await page.evaluate(() => {{
const active = document.querySelector('.el-tabs__item.is-active');
return (active?.textContent || '').replace(/\\s+/g, ' ').trim();
}}).catch(() => '');
if (!activeText.includes('拒退商品分析')) {{
return {{ ok: false, code: 'GOODS_TAB_NOT_ACTIVE', activeText, url: page.url() }};
}}
const downloadBtn = page.locator('button:has-text("下载数据")').first();
if (!(await downloadBtn.count().catch(() => 0))) {{
return {{ ok: false, code: 'DOWNLOAD_DATA_BUTTON_NOT_FOUND', activeText, url: page.url() }};
}}
await downloadBtn.click({{ timeout: 5000, force: true }});
await sleep(700);
const noNeed = page.locator('button:has-text("不需要"), span:has-text("不需要")').first();
if (await noNeed.count().catch(() => 0)) {{
await noNeed.click({{ timeout: 3000, force: true }}).catch(() => null);
await sleep(700);
}}
await page.goto(spec.download_center_url, {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
await sleep(2000);
const rowsSeen = [];
for (let attempt = 0; attempt < spec.max_retries; attempt++) {{
const queryBtn = page.locator('button:has-text("查询"), span:has-text("查询")').first();
if (await queryBtn.count().catch(() => 0)) {{
await queryBtn.click({{ timeout: 2500 }}).catch(() => null);
}}
await sleep(2200);
const rowCount = await page.locator('tbody tr').count().catch(() => 0);
for (let idx = 0; idx < Math.min(rowCount, 10); idx++) {{
const row = page.locator('tbody tr').nth(idx);
const text = (await row.innerText().catch(() => '')).replace(/\\s+/g, ' ').trim();
const nameEl = row.locator('.status-name-span').first();
let fileName = await nameEl.getAttribute('title').catch(() => '');
if (!fileName) fileName = (await nameEl.innerText().catch(() => '')).trim();
rowsSeen.push({{ row: idx + 1, fileName, text }});
if (!fileName.includes('款号粒度') || !/执行成功|成功/.test(text)) continue;
const btn = row.locator('button:has-text("下载"), span:has-text("下载")').first();
if (!(await btn.count().catch(() => 0))) continue;
const [dl] = await Promise.all([
page.waitForEvent('download', {{ timeout: 120000 }}),
btn.click({{ timeout: 5000, force: true }}),
]);
const suggested = dl.suggestedFilename();
const finalName = safeName(`${{spec.prefix}}_${{fileName || suggested}}`);
const targetPath = `${{spec.download_dir.replace(/\\\\/g, '/')}}/${{finalName}}`;
await dl.saveAs(targetPath);
return {{ ok: true, savedAs: targetPath, fileName, suggestedFilename: suggested, rowText: text }};
}}
}}
return {{ ok: false, code: 'GOODS_STYLE_FILE_NOT_READY', rowsSeen: rowsSeen.slice(0, 30), url: page.url() }};
}}
"""
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Download Compass rejection goods style-granularity file.")
p.add_argument("--config", default=r"D:\projects\skills\vip-report\config.yaml")
p.add_argument("--session", default=SESSION_NAME)
p.add_argument("--report-year", type=int, default=0)
p.add_argument("--report-month", default="")
p.add_argument("--download-dir", default=DEFAULT_DOWNLOAD_DIR)
p.add_argument("--max-retries", type=int, default=30)
return p.parse_args()
def main() -> None:
args = parse_args()
config_path = Path(args.config).resolve()
cfg = yaml.safe_load(config_path.read_text(encoding="utf-8"))
workdir = Path(cfg["paths"]["workdir"]).resolve()
year, month = resolve_period(config_path, args.report_year, args.report_month)
prefix = f"{year}{month:02d}"
download_dir = Path(args.download_dir).resolve()
download_dir.mkdir(parents=True, exist_ok=True)
js_path = workdir / f"tmp-compass-rejection-goods-style-download-{prefix}-{time.time_ns()}.js"
js_path.write_text(
build_js(
{
"url": TARGET_URL,
"download_center_url": DOWNLOAD_CENTER_URL,
"prefix": prefix,
"download_dir": str(download_dir),
"max_retries": int(args.max_retries),
}
),
encoding="utf-8",
)
try:
out = run_cmd(
PLAYWRIGHT_CMD + ["--session", args.session, "run-code", "--filename", str(js_path)],
cwd=workdir,
timeout=1200,
).stdout
finally:
js_path.unlink(missing_ok=True)
if "### Error" in out:
raise RuntimeError(out.strip())
json_line = ""
for line in out.splitlines():
s = line.strip()
if s.startswith("{") and s.endswith("}"):
json_line = s
break
if not json_line:
raise RuntimeError(f"No JSON result from run-code: {out}")
result = json.loads(json_line)
if not result.get("ok", False):
raise RuntimeError(json.dumps(result, ensure_ascii=False))
print(json.dumps(result, ensure_ascii=False))
if __name__ == "__main__":
main()
vip-report/scripts/sync_compass_traffic_source_download.py
View file @ 52222f72
from __future__ import annotations
import argparse
import base64
import json
import os
import re
......@@ -309,18 +310,39 @@ def build_download_js(spec: dict[str, Any]) -> str:
return rows.find(r=>r.id===String(tid)) || null;
}}, taskId);
if(hit && hit.hasDl && hit.status.includes('成功')){{
const btn = page.locator(`tbody tr:nth-child(${{hit.row}}) td:nth-child(7) button:has-text("下载")`).first();
if(hit && hit.hasDl && /成功|执行成功|鎴愬姛/.test(hit.status)){{
const sanitize=(s)=>String(s||'').replace(/[\\\\/:*?"<>|]/g,'_').replace(/\\s+/g,' ').trim();
const [dl] = await Promise.all([page.waitForEvent('download',{{timeout:60000}}), btn.click()]);
const suggestedRaw = sanitize(await dl.suggestedFilename());
const byTaskName = sanitize(hit.name || '');
const base = byTaskName || suggestedRaw || String(taskId || 'download');
const base = byTaskName || String(taskId || 'download');
const ensured = /\\.(xlsx|xls|csv)$/i.test(base) ? base : `${{base}}.xlsx`;
const finalName = `${{prefix}}_${{kind}}_${{ensured}}`;
const path = `${{SAVE}}/${{finalName}}`;
await dl.saveAs(path);
return {{taskId, taskType, taskName:hit.name, path}};
const filePayload = await page.evaluate(async (tid) => {{
const body = new URLSearchParams();
body.set('reqData', JSON.stringify({{ taskId: tid }}));
const res = await fetch('/report/asyncdownload/getOutputFile', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/x-www-form-urlencoded' }},
body: body.toString(),
credentials: 'include',
}});
const ab = await res.arrayBuffer();
const bytes = new Uint8Array(ab);
let binary = '';
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {{
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}}
return {{
ok: res.ok,
status: res.status,
contentType: res.headers.get('content-type') || '',
disposition: res.headers.get('content-disposition') || '',
base64: btoa(binary),
}};
}}, String(taskId));
if(!filePayload.ok) throw new Error(`getOutputFile failed status=${{filePayload.status}} taskId=${{taskId}}`);
return {{taskId, taskType, taskName:hit.name, path, contentType:filePayload.contentType, disposition:filePayload.disposition, contentBase64:filePayload.base64}};
}}
}}
throw new Error(`downloadByTaskId timeout taskType=${{taskType}} taskId=${{taskId}}`);
......@@ -432,9 +454,16 @@ def main() -> None:
if not json_line:
raise RuntimeError(f"No JSON result from run-code: {out}")
result = json.loads(json_line)
for item in result.get("done", []):
content_b64 = item.pop("contentBase64", "")
if not content_b64:
continue
target = Path(item["path"])
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(base64.b64decode(content_b64))
if not result.get("ok", False):
raise RuntimeError(json.dumps(result, ensure_ascii=False))
print(json.dumps(result, ensure_ascii=False))
raise RuntimeError(json.dumps(result, ensure_ascii=True))
print(json.dumps(result, ensure_ascii=True))
if __name__ == "__main__":
......
vip-report/scripts/sync_inventory_monthly_assets.py
View file @ 52222f72
......@@ -301,8 +301,8 @@ def build_subcategory_filters(
def build_s08_filters(report_month: str, report_year: int) -> list[dict[str, Any]]:
"""S08 works only with the current report year and current report month on live Tableau."""
month_index = resolve_month_index(report_month)
"""S08 is a current-month composition table in the business deck."""
month_value_variants = build_month_value_variants(report_month, cumulative=False)
return [
{
"label": "storename",
......@@ -327,11 +327,7 @@ def build_s08_filters(report_month: str, report_year: int) -> list[dict[str, Any
{
"label": "month",
"field_candidates": ["月(billdate)", "billdate 月", "Month", "month", "月"],
"value_candidates": [
[str(month_index)],
[f"{month_index:02d}"],
[normalize_month_label(report_month)],
],
"value_candidates": month_value_variants,
"require_worksheet_match": False,
},
]
......
vip-report/scripts/sync_monthly_sales_assets.py
View file @ 52222f72
......@@ -41,6 +41,7 @@ CAPTURE_EXECUTION_ORDER = [
# 1) Overview (after applying filters) -> download image
# 2) StoreSalesinDetail -> download image
"overview",
"overview_s02_table_single",
"store_sales_detail",
# S03 dependencies keep their natural order afterwards.
"overview_s03",
......@@ -139,9 +140,11 @@ def resolve_month_values(report_month: str, month_mode: str) -> list[str]:
def build_specs(report_month: str, report_year: int, compare_year: int, month_mode: str) -> dict[str, Any]:
"""Build the Tableau capture and crop specs for monthly sales slides."""
# S02/S03 monthly sales always use cumulative months from January to the report month.
cumulative_month_value_candidates = build_month_value_variants(report_month, cumulative=True)
s03_month_values = build_cumulative_month_values(report_month)
report_month = normalize_month_label(report_month)
use_cumulative_months = month_mode != "single"
month_value_candidates = build_month_value_variants(report_month, cumulative=use_cumulative_months)
single_month_value_candidates = build_month_value_variants(report_month, cumulative=False)
s03_month_values = resolve_month_values(report_month, month_mode)
summary_strip_crop = {"left": 0, "top": 528, "width": 1383, "height": 332}
common_monthly_filters = [
{
......@@ -154,7 +157,7 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
{
"label": "month",
"field_candidates": ["Month", "月(daily)"],
"value_candidates": cumulative_month_value_candidates,
"value_candidates": month_value_candidates,
"apply_all_fields": True,
"target_scope": "active_sheet",
},
......@@ -168,9 +171,21 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
{"field": "Sales Type", "values": ["GMV"], "target_scope": "active_sheet"},
{"field": "Brand", "values": ["CK"], "target_scope": "active_sheet"},
]
common_single_monthly_filters = [
{
**item,
"value_candidates": single_month_value_candidates,
}
if item.get("label") == "month"
else item
for item in common_monthly_filters
]
# S02 overview needs both current-year and comparison-year measures, so it must not
# apply a hard Year filter that would exclude year2-backed values like Sales2/LFL.
s02_overview_filters = [item for item in common_monthly_filters if item.get("label") != "year"]
s02_overview_single_filters = [
item for item in common_single_monthly_filters if item.get("label") != "year"
]
return {
"captures": [
{
......@@ -184,6 +199,17 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
"raw_screenshot_name": "monthly-sales-overview.png",
"note": "S02 overview dashboard export.",
},
{
"capture_id": "overview_s02_table_single",
"session": SESSION_NAME,
"hash_url": "#/views/ECMonthlySalesReport_17250004228820/Overview?:iid=2",
"inner_frame_fragment": "/views/ECMonthlySalesReport_17250004228820/Overview?",
"activate_sheet": "Overview",
"filters": [*s02_overview_single_filters],
"params": {},
"raw_screenshot_name": "monthly-sales-overview-single.png",
"note": "S02 current-month overview table export.",
},
{
"capture_id": "overview_s03",
"session": SESSION_NAME,
......@@ -224,8 +250,8 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
{"field": "Sales Type", "values": ["GMV"]},
{"field": "Storename", "values": ["CKC-VIP"]},
{"field": "Year", "values": [str(report_year)]},
{"field": "Month", "values": [report_month]},
{"field": "月(daily)", "values": [report_month]},
{"field": "Month", "values": s03_month_values},
{"field": "月(daily)", "values": s03_month_values},
],
"params": {"year1": report_year, "year2": compare_year},
"raw_screenshot_name": "monthly-sales-store-kpi-lfl.png",
......@@ -251,7 +277,7 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
"shape_name": "鍥剧墖 6",
"shape_id": 7,
"asset_name": "s02_monthly_sales_overview_table",
"capture_id": "overview",
"capture_id": "overview_s02_table_single",
"crop": {"left": 55, "top": 140, "width": 1330, "height": 311},
"resize_to": {"width": 1097, "height": 278},
"source_view": "Overview",
......@@ -283,22 +309,22 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
"panels": [
{
"panel_code": "gp",
"crop": {"left": 0, "top": 1610, "width": 650, "height": 400},
"crop": {"left": 0, "top": 1610, "width": 700, "height": 400},
"dest": {"left": 0, "top": 0, "width": 542, "height": 297},
},
{
"panel_code": "con",
"crop": {"left": 650, "top": 1610, "width": 650, "height": 400},
"crop": {"left": 700, "top": 1610, "width": 700, "height": 400},
"dest": {"left": 568, "top": 0, "width": 542, "height": 297},
},
{
"panel_code": "atv",
"crop": {"left": 0, "top": 2000, "width": 650, "height": 360},
"crop": {"left": 0, "top": 2000, "width": 700, "height": 360},
"dest": {"left": 0, "top": 324, "width": 542, "height": 297},
},
{
"panel_code": "returnqty",
"crop": {"left": 650, "top": 2000, "width": 650, "height": 360},
"crop": {"left": 700, "top": 2000, "width": 700, "height": 360},
"dest": {"left": 568, "top": 324, "width": 542, "height": 297},
},
],
......
vip-report/scripts/sync_top_products_assets.py
View file @ 52222f72
......@@ -2440,10 +2440,15 @@ def build_product_assets(
operations: list[dict[str, Any]] = []
manifest_items: list[dict[str, Any]] = []
for slot, row in zip(slide_spec["product_slots"], source_payload["rows"]):
# Match business layout: left column TOP1-5 from top to bottom, right column TOP6-10.
slots_by_rank = {int(slot["rank"]): slot for slot in slide_spec["product_slots"]}
for rank, row in enumerate(source_payload["rows"], start=1):
slot = slots_by_rank.get(rank)
if slot is None:
continue
articlecode = row["articlecode"]
image_url = row.get("image_url") or PRODUCT_IMAGE_URL.format(articlecode=articlecode)
asset_name = f"{slide_code.lower()}_top_product_{slot['rank']:02d}_{sanitize_file_token(articlecode)}"
asset_name = f"{slide_code.lower()}_top_product_{rank:02d}_{sanitize_file_token(articlecode)}"
asset_path = asset_dir / f"{asset_name}.png"
render_product_image(
download_image(image_url),
......
vip-report/scripts/sync_vis_live_session_downloads.py
View file @ 52222f72
......@@ -151,9 +151,19 @@ def build_download_js(spec: dict[str, Any]) -> str:
const frame = page.frames().find((f) => f.url().includes('vlp-admin.vip.com/vis/#/vis/datas/realTimeSells'));
const ctx = frame || page;
const startInput = ctx.locator('input[placeholder=\"\\u5f00\\u59cb\\u65e5\\u671f\"]').first();
const endInput = ctx.locator('input[placeholder=\"\\u7ed3\\u675f\\u65e5\\u671f\"]').first();
if (!(await startInput.count().catch(() => 0)) || !(await endInput.count().catch(() => 0))) {{
let dateSetMode = 'placeholder-inputs';
let startInput = ctx.locator('input[placeholder=\"\\u5f00\\u59cb\\u65e5\\u671f\"]').first();
let endInput = ctx.locator('input[placeholder=\"\\u7ed3\\u675f\\u65e5\\u671f\"]').first();
let startCount = await startInput.count().catch(() => 0);
let endCount = await endInput.count().catch(() => 0);
if (!startCount || !endCount) {{
dateSetMode = 'range-editor-inputs';
startInput = ctx.locator('.el-date-editor--daterange input.el-range-input, .el-range-editor input.el-range-input').nth(0);
endInput = ctx.locator('.el-date-editor--daterange input.el-range-input, .el-range-editor input.el-range-input').nth(1);
startCount = await startInput.count().catch(() => 0);
endCount = await endInput.count().catch(() => 0);
}}
if (!startCount || !endCount) {{
return {{ ok: false, code: 'DATE_INPUT_NOT_FOUND', ym: spec.ym, url: page.url(), frame: frame ? frame.url() : null }};
}}
......@@ -250,6 +260,7 @@ def build_download_js(spec: dict[str, Any]) -> str:
ok: true,
ym: spec.ym,
prefix: spec.prefix,
dateSetMode,
setDateResult,
queryClick,
downloadClick,
......
vip-report/scripts/sync_vis_member_channel_trend_capture.py
View file @ 52222f72
......@@ -507,7 +507,7 @@ def main() -> None:
result = json.loads(json_line)
if not result.get("ok", False):
raise RuntimeError(json.dumps(result, ensure_ascii=False))
validate_image(out_path, min_w=700, min_h=250, min_bytes=10000)
validate_image(out_path, min_w=560, min_h=250, min_bytes=10000)
print(json.dumps(result, ensure_ascii=False))
......
vip-report/scripts/sync_vis_member_consumption_capture.py
View file @ 52222f72
......@@ -448,7 +448,7 @@ def main() -> None:
result = json.loads(json_line)
if not result.get("ok", False):
raise RuntimeError(json.dumps(result, ensure_ascii=False))
validate_image(out_path, min_w=760, min_h=240, min_bytes=8000)
validate_image(out_path, min_w=680, min_h=240, min_bytes=8000)
last_error = None
break
except Exception as e:
......
vip-report/scripts/sync_vis_vendor_goods_effect_download.py
View file @ 52222f72
......@@ -373,7 +373,7 @@ def main() -> None:
str(result.get("savedAs", "")),
]
if args.config_source == "yaml":
loader_args.extend(["--config", str(config_path.resolve())])
loader_args.extend(["--config-source", "yaml", "--config", str(config_path.resolve())])
else:
loader_args.extend(
[
......
vip-report/scripts/sync_warehouse_100060_assets.py
View file @ 52222f72
......@@ -192,7 +192,7 @@ def build_filter_config(
{
"caption": "Month",
"action": "replace",
"values": [report_month_label, next_month_label(report_month_number)],
"values": [report_month_label],
},
{"caption": "Week", "action": "all"},
{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment