Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
S
skills
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
allen.wang
skills
Commits
52222f72
Commit
52222f72
authored
Jun 12, 2026
by
常佳涛
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Update VIP report automation and PPT
parent
4b9160b1
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
32 changed files
with
4101 additions
and
199 deletions
+4101
-199
.gitignore
.gitignore
+8
-0
Report.pptx
vip-report/Report.pptx
+0
-0
vip-report-render.ps1
vip-report/bin/vip-report-render.ps1
+64
-1
vip-report-tableau-sync.ps1
vip-report/bin/vip-report-tableau-sync.ps1
+5
-1
vip-report-uv-lfl-compose.ps1
vip-report/bin/vip-report-uv-lfl-compose.ps1
+23
-11
config.yaml
vip-report/config.yaml
+4
-4
requirements.txt
vip-report/requirements.txt
+1
-0
build_may_auto_text_ops.py
vip-report/scripts/build_may_auto_text_ops.py
+150
-0
build_slide12_editable_ops.py
vip-report/scripts/build_slide12_editable_ops.py
+266
-0
build_slide14_15_editable_ops.py
vip-report/scripts/build_slide14_15_editable_ops.py
+1020
-0
build_slide16_otd_ops.py
vip-report/scripts/build_slide16_otd_ops.py
+316
-0
download_compass_task_file.py
vip-report/scripts/download_compass_task_file.py
+130
-0
load_compass_rejection_goods_cn_to_mysql.py
...eport/scripts/load_compass_rejection_goods_cn_to_mysql.py
+3
-2
render_template_com.ps1
vip-report/scripts/render_template_com.ps1
+673
-29
render_template_openxml.py
vip-report/scripts/render_template_openxml.py
+930
-0
run_flow12_traffic_pipeline.py
vip-report/scripts/run_flow12_traffic_pipeline.py
+5
-15
run_flow14_livestream_pipeline.py
vip-report/scripts/run_flow14_livestream_pipeline.py
+8
-14
run_flow15_live_goods_effect_pipeline.py
vip-report/scripts/run_flow15_live_goods_effect_pipeline.py
+83
-30
run_flow17_membership_pipeline.py
vip-report/scripts/run_flow17_membership_pipeline.py
+1
-1
sync_cep_live_goods_effect_summary_assets.py
...port/scripts/sync_cep_live_goods_effect_summary_assets.py
+3
-8
sync_cep_live_session_assets.py
vip-report/scripts/sync_cep_live_session_assets.py
+5
-2
sync_compass_rejection_goods_download.py
vip-report/scripts/sync_compass_rejection_goods_download.py
+100
-45
sync_compass_rejection_goods_style_download.py
...rt/scripts/sync_compass_rejection_goods_style_download.py
+200
-0
sync_compass_traffic_source_download.py
vip-report/scripts/sync_compass_traffic_source_download.py
+38
-9
sync_inventory_monthly_assets.py
vip-report/scripts/sync_inventory_monthly_assets.py
+3
-7
sync_monthly_sales_assets.py
vip-report/scripts/sync_monthly_sales_assets.py
+37
-11
sync_top_products_assets.py
vip-report/scripts/sync_top_products_assets.py
+7
-2
sync_vis_live_session_downloads.py
vip-report/scripts/sync_vis_live_session_downloads.py
+14
-3
sync_vis_member_channel_trend_capture.py
vip-report/scripts/sync_vis_member_channel_trend_capture.py
+1
-1
sync_vis_member_consumption_capture.py
vip-report/scripts/sync_vis_member_consumption_capture.py
+1
-1
sync_vis_vendor_goods_effect_download.py
vip-report/scripts/sync_vis_vendor_goods_effect_download.py
+1
-1
sync_warehouse_100060_assets.py
vip-report/scripts/sync_warehouse_100060_assets.py
+1
-1
No files found.
.gitignore
View file @
52222f72
...
...
@@ -40,3 +40,11 @@ vip-report/.mpl-cache/
vip-report/.npm-cache/
vip-report/shadowbot_flow*_bundle/
vip-report/shadowbot_flow*_bundle.zip
# VIP report local backups and inspection workdirs
sql_exports_local/
vip-report/tmp*/
vip-report/Report.*backup*.pptx
vip-report/Report.repaired.pptx
vip-report/Report - *.pptx
vip-report/config.yaml.bak*
vip-report/Report.pptx
View file @
52222f72
No preview for this file type
vip-report/bin/vip-report-render.ps1
View file @
52222f72
...
...
@@ -31,4 +31,67 @@ $TemplatePath = Resolve-ProjectPath $TemplatePath
$OutputPath
=
Resolve-ProjectPath
$OutputPath
$OperationsPath
=
Resolve-ProjectPath
$OperationsPath
powershell -ExecutionPolicy Bypass -File
"
$root
\scripts\render_template_com.ps1"
-TemplatePath
"
$TemplatePath
"
-OutputPath
"
$OutputPath
"
-OperationsPath
"
$OperationsPath
"
function
Resolve-PythonCommand
{
$venvPython
=
Join-Path
$root
".venv\Scripts\python.exe"
if
(
Test-Path
-LiteralPath
$venvPython
)
{
return
,
$venvPython
}
if
(
Get-Command py -ErrorAction SilentlyContinue
)
{
return
@
(
"py"
,
"-3"
)
}
if
(
Get-Command python -ErrorAction SilentlyContinue
)
{
return
@
(
"python"
)
}
return
@
()
}
function
Has-NativeChartOps
{
param
([
string
]
$OpsPath
)
if
(
-not
$OpsPath
-or -not
(
Test-Path
-LiteralPath
$OpsPath
))
{
return
$false
}
try
{
$ops
=
Get-Content
-LiteralPath
$OpsPath
-Encoding UTF8 -Raw |
ConvertFrom-Json
if
(
$ops
.PSObject.Properties.Name -contains
"replace_native_charts"
)
{
return
(
@
(
$ops
.replace_native_charts
)
.Count -gt 0
)
}
}
catch
{
}
return
$false
}
try
{
&
"
$root
\scripts\render_template_com.ps1"
-TemplatePath
"
$TemplatePath
"
-OutputPath
"
$OutputPath
"
-OperationsPath
"
$OperationsPath
"
}
catch
{
$comError
=
$_
$canFallback
=
Has-NativeChartOps -OpsPath
$OperationsPath
if
(
-not
$canFallback
)
{
Write-Error
$comError
exit
1
}
$pythonCmd
=
@
(
Resolve-PythonCommand
)
if
(
$pythonCmd
.Count -eq 0
)
{
Write-Error
"COM render failed and python fallback is unavailable (python not found). Original error:
$comError
"
exit
1
}
$fallbackScript
=
Join-Path
$root
"scripts\render_template_openxml.py"
if
(
-not
(
Test-Path
-LiteralPath
$fallbackScript
))
{
Write-Error
"COM render failed and fallback script is missing:
$fallbackScript
"
exit
1
}
Write-Warning
"COM render failed; falling back to python-pptx renderer."
if
(
$pythonCmd
.Count -eq 2
)
{
&
$pythonCmd
[
0]
$pythonCmd
[
1]
$fallbackScript
--template
"
$TemplatePath
"
--output
"
$OutputPath
"
--operations
"
$OperationsPath
"
}
else
{
&
$pythonCmd
[
0]
$fallbackScript
--template
"
$TemplatePath
"
--output
"
$OutputPath
"
--operations
"
$OperationsPath
"
}
if
(
$LASTEXITCODE
-ne 0
)
{
Write-Error
"Fallback openxml renderer failed (exit=
$LASTEXITCODE
). Original COM error:
$comError
"
exit
1
}
}
vip-report/bin/vip-report-tableau-sync.ps1
View file @
52222f72
...
...
@@ -7,7 +7,7 @@ param(
[
int
]
$ReportYear
=
0,
[
int
]
$CompareYear
=
0,
[
ValidateSet
(
"single"
,
"cumulative"
)]
[
string
]
$MonthlySalesMode
=
"
singl
e"
,
[
string
]
$MonthlySalesMode
=
"
cumulativ
e"
,
[
int
]
$Retries
=
3,
[
int
]
$RetryDelaySeconds
=
8,
[
string
]
$TemplatePath
=
""
,
...
...
@@ -225,6 +225,7 @@ function Merge-Operations {
replace_tables
=
@
()
replace_charts
=
@
()
replace_images
=
@
()
replace_native_charts
=
@
()
}
foreach
(
$opsPath
in
$OpsPaths
)
{
...
...
@@ -244,6 +245,9 @@ function Merge-Operations {
if
(
$ops
.PSObject.Properties.Name -contains
"replace_images"
)
{
$merged
.replace_images +
=
@
(
$ops
.replace_images
)
}
if
(
$ops
.PSObject.Properties.Name -contains
"replace_native_charts"
)
{
$merged
.replace_native_charts +
=
@
(
$ops
.replace_native_charts
)
}
}
$json
=
$merged
|
ConvertTo-Json
-Depth 100
...
...
vip-report/bin/vip-report-uv-lfl-compose.ps1
View file @
52222f72
param
(
[
string
]
$
TableImage
=
"D:\projects\skills\vip-report\output\vip-report\assets\cep-traffic\cep-traffic-summary.png
"
,
[
string
]
$
LineImage
=
"D:\projects\skills\vip-report\output\vip-report\assets\cep-site-traffic-entry\cep-site-traffic-entry-lfl.png
"
,
[
string
]
$EntryImage
=
"D:\projects\skills\vip-report\output\vip-report\assets\cep-traffic-summary\cep-traffic-summary-lfl.png"
,
[
string
]
$OutputPath
=
"D:\projects\skills\vip-report\output\vip-report\assets\uv-lfl\uv-lfl-report.png"
,
[
int
]
$ContentWidth
=
1040
[
string
]
$
ConfigPath
=
"D:\projects\skills\vip-report\config.yaml
"
,
[
string
]
$
ReportMonth
=
"
"
,
[
int
]
$ReportYear
=
0
,
[
int
]
$CompareYear
=
0
,
[
string
]
$OutputOps
=
""
)
$ErrorActionPreference
=
"Stop"
$root
=
Split-Path
-Parent
$PSScriptRoot
python
"
$root
\scripts\compose_uv_lfl_report.py"
`
--table-image
"
$TableImage
"
`
--line-image
"
$LineImage
"
`
--entry-image
"
$EntryImage
"
`
--output
"
$OutputPath
"
`
--content-width
"
$ContentWidth
"
$pythonArgs
=
@
(
"
$root
\scripts\build_slide12_editable_ops.py"
,
"--config"
,
"
$ConfigPath
"
)
if
(
$ReportMonth
)
{
$pythonArgs
+
=
@
(
"--report-month"
,
"
$ReportMonth
"
)
}
if
(
$ReportYear
-gt 0
)
{
$pythonArgs
+
=
@
(
"--report-year"
,
"
$ReportYear
"
)
}
if
(
$CompareYear
-gt 0
)
{
$pythonArgs
+
=
@
(
"--compare-year"
,
"
$CompareYear
"
)
}
if
(
$OutputOps
)
{
$pythonArgs
+
=
@
(
"--output-ops"
,
"
$OutputOps
"
)
}
python @pythonArgs
vip-report/config.yaml
View file @
52222f72
...
...
@@ -4,10 +4,10 @@ tableau:
password
:
"
1!Cu&euRo17b6yu@R40gr7"
mysql
:
host
:
"
1
92.168.138.182
"
host
:
"
1
27.0.0.1
"
port
:
3306
username
:
"
ceptes
t"
password
:
"
ck@123456
"
username
:
"
roo
t"
password
:
"
root
"
database
:
"
ckc_cep_db_test"
paths
:
...
...
@@ -19,7 +19,7 @@ render:
mode
:
"
com"
report
:
month_cn
:
"
3
"
month_cn
:
"
5
"
year
:
2026
compare_year
:
2025
...
...
vip-report/requirements.txt
View file @
52222f72
...
...
@@ -4,3 +4,4 @@ PyYAML
lxml
matplotlib
numpy
openpyxl
vip-report/scripts/build_may_auto_text_ops.py
0 → 100644
View file @
52222f72
from
__future__
import
annotations
import
json
from
decimal
import
Decimal
from
pathlib
import
Path
import
pymysql
import
yaml
def
dec
(
value
:
object
)
->
Decimal
:
return
Decimal
(
str
(
value
or
0
))
def
pct
(
value
:
object
)
->
str
:
return
f
"{(dec(value) * 100).quantize(Decimal('0.01'))}
%
"
def
wan
(
value
:
object
)
->
str
:
return
f
"{(dec(value) / Decimal('10000')).quantize(Decimal('0.1'))}w"
def
sign_pct
(
curr
:
object
,
prev
:
object
)
->
str
:
curr_d
=
dec
(
curr
)
prev_d
=
dec
(
prev
)
if
prev_d
==
0
:
return
"0
%
"
q
=
((
curr_d
-
prev_d
)
/
prev_d
*
100
)
.
quantize
(
Decimal
(
"1"
))
return
(
"+"
if
q
>
0
else
""
)
+
f
"{q}
%
"
def
main
()
->
None
:
root
=
Path
(
__file__
)
.
resolve
()
.
parent
.
parent
cfg
=
yaml
.
safe_load
((
root
/
"config.yaml"
)
.
read_text
(
encoding
=
"utf-8"
))
mysql
=
cfg
[
"mysql"
]
conn
=
pymysql
.
connect
(
host
=
mysql
[
"host"
],
port
=
int
(
mysql
.
get
(
"port"
,
3306
)),
user
=
mysql
.
get
(
"username"
)
or
mysql
.
get
(
"user"
),
password
=
mysql
[
"password"
],
database
=
mysql
[
"database"
],
charset
=
"utf8mb4"
,
cursorclass
=
pymysql
.
cursors
.
DictCursor
,
)
try
:
with
conn
.
cursor
()
as
cur
:
cur
.
execute
(
"""
SELECT
SUM(accumulated_viewers) viewers,
SUM(guided_sales_total) guided_sales,
SUM(brand_new_customers_total) new_customers,
SUM(live_sales_penetration_rate * accumulated_viewers) / NULLIF(SUM(accumulated_viewers), 0) live_pen,
SUM(purchasers_total) / NULLIF(SUM(accumulated_viewers), 0) conversion_rate
FROM cep_live_session_data
WHERE data_month='2026-05'
"""
)
live
=
cur
.
fetchone
()
cur
.
execute
(
"""
SELECT
SUM(accumulated_viewers) viewers,
SUM(guided_sales_total) guided_sales,
SUM(brand_new_customers_total) new_customers,
SUM(live_sales_penetration_rate * accumulated_viewers) / NULLIF(SUM(accumulated_viewers), 0) live_pen,
SUM(purchasers_total) / NULLIF(SUM(accumulated_viewers), 0) conversion_rate
FROM cep_live_session_data
WHERE data_month='2026-04'
"""
)
live_prev
=
cur
.
fetchone
()
cur
.
execute
(
"""
SELECT
COUNT(DISTINCT product_spu_id) articles,
SUM(item_count_total) qty,
SUM(sales_total) sales,
SUM(brand_new_customer_count) new_customers
FROM cep_live_goods_effect
WHERE data_month='2026-05'
"""
)
goods
=
cur
.
fetchone
()
cur
.
execute
(
"""
SELECT
CASE
WHEN category_level_1 IN ('配饰', '珠宝首饰') THEN '配饰'
ELSE category_level_1
END AS cat,
SUM(item_count_total) qty
FROM cep_live_goods_effect
WHERE data_month='2026-05'
GROUP BY cat
"""
)
goods_cats
=
cur
.
fetchall
()
finally
:
conn
.
close
()
cat_total
=
sum
(
dec
(
row
[
"qty"
])
for
row
in
goods_cats
)
or
Decimal
(
1
)
cat_parts
:
list
[
str
]
=
[]
for
row
in
goods_cats
:
name
=
str
(
row
[
"cat"
]
or
"未知"
)
share
=
(
dec
(
row
[
"qty"
])
/
cat_total
*
100
)
.
quantize
(
Decimal
(
"0"
))
cat_parts
.
append
(
f
"{name}{share}
%
"
)
texts
=
{
"cover"
:
"E-Commerce Monthly Sales (VIP Store)May 2026"
,
"top_bags"
:
"5月流水销量TOP10包款已按五月网页抓取结果更新;榜单、商品图和销售走势来自Tableau五月筛选数据。"
,
"top_shoes"
:
"5月流水销量TOP10鞋款已按五月网页抓取结果更新;榜单、商品图和销售走势来自Tableau五月筛选数据。"
,
"uv"
:
"同比去年5月,流量看板已按Compass五月流量来源数据更新;站内/站外、付费/自然流量及入口结构均来自本地库五月抓取结果。"
,
"warehouse"
:
"本页60仓表现已按Tableau五月筛选结果更新,包含折扣分布、箱包TOP和鞋类TOP三个模块。"
,
"live14"
:
(
f
"本月直播累计观看人数{wan(live['viewers'])},直播渗透率{pct(live['live_pen'])},"
f
"环比{sign_pct(live['live_pen'], live_prev['live_pen'])};转化率{pct(live['conversion_rate'])},"
f
"总引导销售{wan(live['guided_sales'])},品牌新客数{int(dec(live['new_customers'])):,}。"
"直播图表来自本地库cep_live_session_data五月数据。"
),
"live15"
:
(
f
"鞋包直播间有销售商品款数总计{int(dec(goods['articles']))}款,类目结构为{'、'.join(cat_parts)}。"
f
"销售额{wan(goods['sales'])},直播商品榜单来自cep_live_goods_effect五月商品效果数据。"
),
"member"
:
"本页会员看板已按五月网页截图更新,四个模块图片来自会员看板五月筛选后的自动截图。"
,
}
ops
=
{
"replace_text"
:
[
{
"slide"
:
1
,
"shape_id"
:
3
,
"new_text"
:
texts
[
"cover"
]},
{
"slide"
:
9
,
"shape_id"
:
32
,
"new_text"
:
texts
[
"top_bags"
]},
{
"slide"
:
10
,
"shape_id"
:
15
,
"new_text"
:
texts
[
"top_shoes"
]},
{
"slide"
:
12
,
"shape_id"
:
11
,
"new_text"
:
texts
[
"uv"
]},
{
"slide"
:
13
,
"shape_id"
:
16
,
"new_text"
:
texts
[
"warehouse"
]},
{
"slide"
:
14
,
"shape_id"
:
14
,
"new_text"
:
texts
[
"live14"
]},
{
"slide"
:
15
,
"shape_id"
:
8
,
"new_text"
:
texts
[
"live15"
]},
{
"slide"
:
17
,
"shape_id"
:
15
,
"new_text"
:
texts
[
"member"
]},
],
"replace_images"
:
[],
"replace_tables"
:
[],
"replace_charts"
:
[],
"replace_native_charts"
:
[],
}
out
=
root
/
"output"
/
"vip-report"
/
"render-ops.may-202605.auto-text.json"
out
.
write_text
(
json
.
dumps
(
ops
,
ensure_ascii
=
False
,
indent
=
2
),
encoding
=
"utf-8"
)
print
(
json
.
dumps
({
"operations_path"
:
str
(
out
),
"texts"
:
texts
},
ensure_ascii
=
False
,
indent
=
2
))
if
__name__
==
"__main__"
:
main
()
vip-report/scripts/build_slide12_editable_ops.py
0 → 100644
View file @
52222f72
from
__future__
import
annotations
import
argparse
import
json
import
math
import
sys
from
decimal
import
Decimal
from
pathlib
import
Path
from
typing
import
Any
import
pymysql
SCRIPT_DIR
=
Path
(
__file__
)
.
resolve
()
.
parent
if
str
(
SCRIPT_DIR
)
not
in
sys
.
path
:
sys
.
path
.
insert
(
0
,
str
(
SCRIPT_DIR
))
from
runtime_config
import
add_config_source_args
,
load_runtime_config
from
sync_cep_site_traffic_entry_assets
import
(
fetch_month_uv_series
,
lfl_series
,
month_label_to_number
,
normalize_month_label
,
)
from
sync_cep_traffic_assets
import
build_table_values
,
query_month_row
from
sync_cep_traffic_summary_assets
import
ENTRY_ORDER
,
fetch_entry_uv_map
TABLE_HEADERS
=
[
"Year"
,
"In-site Traffic"
,
"In-site Paid Traffic"
,
"In-site Paid
%
"
,
"In-site Organic Traffic"
,
"In-site Organic
%
"
,
"Off-site Traffic"
,
"Off-site Paid Traffic"
,
"Off-site Paid
%
"
,
"Off-site Organic Traffic"
,
"Off-site Organic
%
"
,
]
def
round_up
(
value
:
int
,
step
:
int
)
->
int
:
if
value
<=
0
:
return
step
return
((
value
+
step
-
1
)
//
step
)
*
step
def
choose_value_axis
(
values_a
:
list
[
Decimal
],
values_b
:
list
[
Decimal
])
->
tuple
[
int
,
int
]:
max_volume
=
max
(
int
(
max
(
values_a
or
[
Decimal
(
"0"
)])),
int
(
max
(
values_b
or
[
Decimal
(
"0"
)])))
y_step
=
5000
if
max_volume
>
15000
else
2000
y_max
=
round_up
(
max_volume
,
y_step
)
return
y_max
,
y_step
def
choose_lfl_axis
(
values
:
list
[
Decimal
])
->
tuple
[
int
,
int
,
int
]:
if
not
values
:
return
-
20
,
60
,
20
min_v
=
float
(
min
(
values
))
max_v
=
float
(
max
(
values
))
span
=
max
(
abs
(
min_v
),
abs
(
max_v
))
if
span
<=
80
:
step
=
20
elif
span
<=
300
:
step
=
50
else
:
step
=
100
low
=
int
(
math
.
floor
(
min_v
/
step
)
*
step
)
high
=
int
(
math
.
ceil
(
max_v
/
step
)
*
step
)
if
low
>
0
:
low
=
0
if
high
<
0
:
high
=
0
if
high
==
low
:
high
+=
step
return
low
,
high
,
step
def
build_entry_rows
(
cur_map
:
dict
[
str
,
Decimal
],
prev_map
:
dict
[
str
,
Decimal
]
)
->
list
[
dict
[
str
,
Any
]]:
union_entries
=
set
(
cur_map
)
|
set
(
prev_map
)
ordered
=
[
e
for
e
in
ENTRY_ORDER
if
e
in
union_entries
]
extras
=
sorted
(
[
e
for
e
in
union_entries
if
e
not
in
ordered
],
key
=
lambda
k
:
(
cur_map
.
get
(
k
,
Decimal
(
"0"
))
+
prev_map
.
get
(
k
,
Decimal
(
"0"
))),
reverse
=
True
,
)
ordered
.
extend
(
extras
)
rows
:
list
[
dict
[
str
,
Any
]]
=
[]
for
name
in
ordered
:
rows
.
append
(
{
"name"
:
name
,
"report_uv"
:
int
(
cur_map
.
get
(
name
,
Decimal
(
"0"
))),
"compare_uv"
:
int
(
prev_map
.
get
(
name
,
Decimal
(
"0"
))),
}
)
return
rows
def
parse_args
()
->
argparse
.
Namespace
:
parser
=
argparse
.
ArgumentParser
(
description
=
"Build editable PPT operations for slide 12 (UV LFL) without image rendering."
)
add_config_source_args
(
parser
,
default_config
=
Path
(
r"D:\projects\skills\vip-report\config.yaml"
))
parser
.
add_argument
(
"--report-month"
,
default
=
""
,
help
=
"Month override, e.g. Jan / 1"
)
parser
.
add_argument
(
"--report-year"
,
type
=
int
,
default
=
0
,
help
=
"Report year override"
)
parser
.
add_argument
(
"--compare-year"
,
type
=
int
,
default
=
0
,
help
=
"Compare year override"
)
parser
.
add_argument
(
"--output-ops"
,
default
=
""
,
help
=
"Optional override output ops path"
)
return
parser
.
parse_args
()
def
execute
(
config
:
dict
[
str
,
Any
],
*
,
report_month
:
str
=
""
,
report_year
:
int
=
0
,
compare_year
:
int
=
0
,
output_ops
:
str
=
""
,
)
->
dict
[
str
,
str
]:
report_cfg
=
config
.
get
(
"report"
,
{})
month_label
=
normalize_month_label
(
report_month
or
report_cfg
.
get
(
"month_cn"
,
"一月"
))
month_num
=
month_label_to_number
(
month_label
)
report_year
=
int
(
report_year
or
report_cfg
.
get
(
"year"
,
2026
))
compare_year
=
int
(
compare_year
or
report_cfg
.
get
(
"compare_year"
,
report_year
-
1
))
mysql_cfg
=
config
[
"mysql"
]
conn
=
pymysql
.
connect
(
host
=
mysql_cfg
[
"host"
],
port
=
int
(
mysql_cfg
[
"port"
]),
user
=
mysql_cfg
[
"username"
],
password
=
mysql_cfg
[
"password"
],
database
=
mysql_cfg
[
"database"
],
charset
=
"utf8mb4"
,
cursorclass
=
pymysql
.
cursors
.
DictCursor
,
)
try
:
cur_month_row
=
query_month_row
(
conn
,
year
=
report_year
,
month_num
=
month_num
)
prev_month_row
=
query_month_row
(
conn
,
year
=
compare_year
,
month_num
=
month_num
)
table_current
,
table_previous
,
table_lfl
=
build_table_values
(
report_year
=
report_year
,
compare_year
=
compare_year
,
current
=
cur_month_row
,
previous
=
prev_month_row
,
)
cur_uv
=
fetch_month_uv_series
(
conn
,
year
=
report_year
,
month
=
month_num
)
prev_uv
=
fetch_month_uv_series
(
conn
,
year
=
compare_year
,
month
=
month_num
)
free_lfl
=
lfl_series
(
cur_uv
[
"free"
],
prev_uv
[
"free"
])
paid_lfl
=
lfl_series
(
cur_uv
[
"paid"
],
prev_uv
[
"paid"
])
cur_entry_map
=
fetch_entry_uv_map
(
conn
,
daily_month
=
f
"{report_year}-{month_num:02d}"
)
prev_entry_map
=
fetch_entry_uv_map
(
conn
,
daily_month
=
f
"{compare_year}-{month_num:02d}"
)
finally
:
conn
.
close
()
free_axis_max
,
free_axis_step
=
choose_value_axis
(
cur_uv
[
"free"
],
prev_uv
[
"free"
])
paid_axis_max
,
paid_axis_step
=
choose_value_axis
(
cur_uv
[
"paid"
],
prev_uv
[
"paid"
])
free_lfl_min
,
free_lfl_max
,
free_lfl_step
=
choose_lfl_axis
(
free_lfl
)
paid_lfl_min
,
paid_lfl_max
,
paid_lfl_step
=
choose_lfl_axis
(
paid_lfl
)
vip_workdir
=
Path
(
config
[
"paths"
][
"workdir"
])
.
resolve
()
ops_path
=
Path
(
output_ops
)
.
resolve
()
if
output_ops
else
(
vip_workdir
/
"render-ops.uv-lfl-s12.editable.json"
)
ops_path
.
parent
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
operations
=
{
"replace_text"
:
[],
"replace_tables"
:
[],
"replace_charts"
:
[],
"replace_images"
:
[],
"replace_native_charts"
:
[
{
"type"
:
"slide12_traffic_table"
,
"slide"
:
12
,
"shape_id"
:
3
,
"shape_name"
:
"Table 17"
,
"target_box"
:
{
"left"
:
1835149
,
"top"
:
1557195
,
"width"
:
16160753
,
"height"
:
1465080
,
},
"headers"
:
TABLE_HEADERS
,
"rows"
:
[
table_current
,
table_previous
,
table_lfl
],
},
{
"type"
:
"slide12_dual_line_lfl"
,
"slide"
:
12
,
"shape_id"
:
5
,
"shape_name"
:
"Group 24"
,
"target_box"
:
{
"left"
:
1987549
,
"top"
:
3062843
,
"width"
:
15887698
,
"height"
:
3109300
,
},
"month"
:
month_num
,
"report_year"
:
report_year
,
"compare_year"
:
compare_year
,
"panels"
:
[
{
"title"
:
"Organic Traffic LFL"
,
"current_name"
:
f
"{str(report_year)[-2:]} Organic Traffic"
,
"previous_name"
:
f
"{str(compare_year)[-2:]} Organic Traffic"
,
"current_values"
:
[
int
(
v
)
for
v
in
cur_uv
[
"free"
]],
"previous_values"
:
[
int
(
v
)
for
v
in
prev_uv
[
"free"
]],
"lfl_values"
:
[
round
(
float
(
v
),
2
)
for
v
in
free_lfl
],
"value_axis_max"
:
free_axis_max
,
"value_axis_major_unit"
:
free_axis_step
,
"lfl_axis_min"
:
free_lfl_min
,
"lfl_axis_max"
:
free_lfl_max
,
"lfl_axis_major_unit"
:
free_lfl_step
,
},
{
"title"
:
"Paid Traffic LFL"
,
"current_name"
:
f
"{str(report_year)[-2:]} Paid Traffic"
,
"previous_name"
:
f
"{str(compare_year)[-2:]} Paid Traffic"
,
"current_values"
:
[
int
(
v
)
for
v
in
cur_uv
[
"paid"
]],
"previous_values"
:
[
int
(
v
)
for
v
in
prev_uv
[
"paid"
]],
"lfl_values"
:
[
round
(
float
(
v
),
2
)
for
v
in
paid_lfl
],
"value_axis_max"
:
paid_axis_max
,
"value_axis_major_unit"
:
paid_axis_step
,
"lfl_axis_min"
:
paid_lfl_min
,
"lfl_axis_max"
:
paid_lfl_max
,
"lfl_axis_major_unit"
:
paid_lfl_step
,
},
],
},
{
"type"
:
"slide12_entry_mix"
,
"slide"
:
12
,
"shape_id"
:
8
,
"shape_name"
:
"Group 25"
,
"target_box"
:
{
"left"
:
1987549
,
"top"
:
6093856
,
"width"
:
16160750
,
"height"
:
2729245
,
},
"report_year"
:
report_year
,
"compare_year"
:
compare_year
,
"entries"
:
build_entry_rows
(
cur_entry_map
,
prev_entry_map
),
},
],
}
ops_path
.
write_text
(
json
.
dumps
(
operations
,
ensure_ascii
=
False
,
indent
=
2
),
encoding
=
"utf-8"
)
return
{
"operations_path"
:
str
(
ops_path
)}
def
main
()
->
None
:
args
=
parse_args
()
config
=
load_runtime_config
(
args
,
default_config
=
Path
(
r"D:\projects\skills\vip-report\config.yaml"
))
result
=
execute
(
config
,
report_month
=
args
.
report_month
,
report_year
=
args
.
report_year
,
compare_year
=
args
.
compare_year
,
output_ops
=
args
.
output_ops
,
)
print
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
if
__name__
==
"__main__"
:
main
()
vip-report/scripts/build_slide14_15_editable_ops.py
0 → 100644
View file @
52222f72
This diff is collapsed.
Click to expand it.
vip-report/scripts/build_slide16_otd_ops.py
0 → 100644
View file @
52222f72
This diff is collapsed.
Click to expand it.
vip-report/scripts/download_compass_task_file.py
0 → 100644
View file @
52222f72
from
__future__
import
annotations
import
argparse
import
base64
import
json
import
os
import
shutil
import
subprocess
import
time
from
pathlib
import
Path
import
yaml
NPX_EXECUTABLE
=
shutil
.
which
(
"npx.cmd"
)
or
shutil
.
which
(
"npx"
)
or
"npx"
PLAYWRIGHT_CMD
=
[
NPX_EXECUTABLE
,
"--yes"
,
"--package"
,
"@playwright/cli"
,
"playwright-cli"
]
SESSION_NAME
=
"vip-report-monthly-sales"
DOWNLOAD_CENTER_URL
=
"https://compass.vip.com/frontend/index.html#/dataCapturing/downloadCenter"
def
run_cmd
(
args
:
list
[
str
],
*
,
cwd
:
Path
,
timeout
:
int
=
300
)
->
subprocess
.
CompletedProcess
[
str
]:
env
=
os
.
environ
.
copy
()
npm_cache
=
cwd
/
".npm-cache"
npm_cache
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
env
[
"NPM_CONFIG_CACHE"
]
=
str
(
npm_cache
)
env
[
"npm_config_cache"
]
=
str
(
npm_cache
)
return
subprocess
.
run
(
args
,
cwd
=
str
(
cwd
),
text
=
True
,
encoding
=
"utf-8"
,
errors
=
"replace"
,
capture_output
=
True
,
timeout
=
timeout
,
check
=
True
,
env
=
env
,
)
def
build_js
(
task_id
:
str
)
->
str
:
payload
=
json
.
dumps
({
"task_id"
:
task_id
,
"url"
:
DOWNLOAD_CENTER_URL
},
ensure_ascii
=
False
)
return
f
"""async function(page) {{
const spec = {payload};
await page.goto(spec.url, {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
await page.waitForTimeout(1000);
const result = await page.evaluate(async (taskId) => {{
const body = new URLSearchParams();
body.set('reqData', JSON.stringify({{ taskId }}));
const res = await fetch('/report/asyncdownload/getOutputFile', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/x-www-form-urlencoded' }},
body: body.toString(),
credentials: 'include',
}});
const ab = await res.arrayBuffer();
const bytes = new Uint8Array(ab);
let binary = '';
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {{
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}}
return {{
ok: res.ok,
status: res.status,
contentType: res.headers.get('content-type') || '',
disposition: res.headers.get('content-disposition') || '',
base64: btoa(binary),
}};
}}, String(spec.task_id));
return result;
}}
"""
def
parse_args
()
->
argparse
.
Namespace
:
p
=
argparse
.
ArgumentParser
(
description
=
"Download a ready Compass async task file by task id."
)
p
.
add_argument
(
"--config"
,
default
=
r"D:\projects\skills\vip-report\config.yaml"
)
p
.
add_argument
(
"--session"
,
default
=
SESSION_NAME
)
p
.
add_argument
(
"--task-id"
,
required
=
True
)
p
.
add_argument
(
"--output"
,
required
=
True
)
return
p
.
parse_args
()
def
main
()
->
None
:
args
=
parse_args
()
cfg
=
yaml
.
safe_load
(
Path
(
args
.
config
)
.
read_text
(
encoding
=
"utf-8"
))
workdir
=
Path
(
cfg
[
"paths"
][
"workdir"
])
.
resolve
()
js_path
=
workdir
/
f
"tmp-download-task-{args.task_id}-{time.time_ns()}.js"
js_path
.
write_text
(
build_js
(
args
.
task_id
),
encoding
=
"utf-8"
)
try
:
out
=
run_cmd
(
PLAYWRIGHT_CMD
+
[
"--session"
,
args
.
session
,
"run-code"
,
"--filename"
,
str
(
js_path
)],
cwd
=
workdir
,
timeout
=
420
,
)
.
stdout
finally
:
js_path
.
unlink
(
missing_ok
=
True
)
if
"### Error"
in
out
:
raise
RuntimeError
(
out
.
strip
())
json_text
=
""
marker
=
"### Result"
marker_idx
=
out
.
find
(
marker
)
if
marker_idx
>=
0
:
start
=
out
.
find
(
"
\n
"
,
marker_idx
)
if
start
>=
0
:
end
=
out
.
find
(
"
\n
###"
,
start
+
1
)
json_text
=
out
[
start
+
1
:
end
if
end
>=
0
else
None
]
.
strip
()
if
not
json_text
:
for
line
in
out
.
splitlines
():
s
=
line
.
strip
()
if
s
.
startswith
(
"{"
)
and
s
.
endswith
(
"}"
):
json_text
=
s
break
if
not
json_text
:
raise
RuntimeError
(
f
"No JSON result from run-code: {out}"
)
result
=
json
.
loads
(
json_text
)
if
not
result
.
get
(
"ok"
,
False
):
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
output_path
=
Path
(
args
.
output
)
.
resolve
()
output_path
.
parent
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
output_path
.
write_bytes
(
base64
.
b64decode
(
result
[
"base64"
]))
result
.
pop
(
"base64"
,
None
)
result
[
"savedAs"
]
=
str
(
output_path
)
print
(
json
.
dumps
(
result
,
ensure_ascii
=
True
))
if
__name__
==
"__main__"
:
main
()
vip-report/scripts/load_compass_rejection_goods_cn_to_mysql.py
View file @
52222f72
...
...
@@ -170,8 +170,9 @@ def main() -> None:
raise
RuntimeError
(
f
"Missing required columns: {missing}"
)
time_values
=
{
str
(
x
)
.
strip
()
for
x
in
df
[
"时间"
]
.
dropna
()
.
head
(
20
)}
if
not
any
(
"202603"
in
x
for
x
in
time_values
):
raise
RuntimeError
(
f
"Unexpected time values for 2026-03 file: {sorted(time_values)[:5]}"
)
expected_month
=
data_month
.
replace
(
"-"
,
""
)
if
not
any
(
expected_month
in
x
for
x
in
time_values
):
raise
RuntimeError
(
f
"Unexpected time values for {data_month} file: {sorted(time_values)[:5]}"
)
rows
=
parse_rows
(
df
,
data_month
,
source_file
.
name
)
mysql_cfg
=
cfg
[
"mysql"
]
...
...
vip-report/scripts/render_template_com.ps1
View file @
52222f72
This diff is collapsed.
Click to expand it.
vip-report/scripts/render_template_openxml.py
0 → 100644
View file @
52222f72
This diff is collapsed.
Click to expand it.
vip-report/scripts/run_flow12_traffic_pipeline.py
View file @
52222f72
...
...
@@ -52,7 +52,7 @@ def main() -> None:
]
if
args
.
config_source
==
"yaml"
:
config_path
=
args
.
config
or
str
((
root
.
parent
/
"config.yaml"
)
.
resolve
())
common
=
[
"--config"
,
config_path
]
common
=
[
"--config
-source"
,
"yaml"
,
"--config
"
,
config_path
]
runtime_cfg
=
load_runtime_config
(
args
,
default_config
=
Path
(
args
.
config
)
if
args
.
config
else
(
root
.
parent
/
"config.yaml"
))
workdir
=
Path
(
runtime_cfg
[
"paths"
][
"workdir"
])
.
resolve
()
...
...
@@ -65,20 +65,10 @@ def main() -> None:
args
.
download_dir
,
],
[
args
.
python
,
str
(
root
/
"load_traffic_downloads_to_mysql.py"
),
*
common
],
[
args
.
python
,
str
(
root
/
"sync_cep_traffic_assets.py"
),
*
common
],
[
args
.
python
,
str
(
root
/
"sync_cep_site_traffic_entry_assets.py"
),
*
common
],
[
args
.
python
,
str
(
root
/
"sync_cep_traffic_summary_assets.py"
),
*
common
],
[
args
.
python
,
str
(
root
/
"compose_uv_lfl_report.py"
),
"--table-image"
,
str
(
workdir
/
"assets"
/
"cep-traffic"
/
"cep-traffic-summary.png"
),
"--line-image"
,
str
(
workdir
/
"assets"
/
"cep-site-traffic-entry"
/
"cep-site-traffic-entry-lfl.png"
),
"--entry-image"
,
str
(
workdir
/
"assets"
/
"cep-traffic-summary"
/
"cep-traffic-summary-lfl.png"
),
"--output"
,
str
(
workdir
/
"assets"
/
"uv-lfl"
/
"uv-lfl-report.png"
),
str
(
root
/
"build_slide12_editable_ops.py"
),
*
common
,
],
]
...
...
@@ -102,8 +92,8 @@ def main() -> None:
f
"stderr_tail=
\n
{err_tail}
\n
"
)
final_
image
=
workdir
/
"assets"
/
"uv-lfl"
/
"uv-lfl-report.png
"
print
(
f
"FINAL_
IMAGE={final_image
}"
)
final_
ops
=
workdir
/
"render-ops.uv-lfl-s12.editable.json
"
print
(
f
"FINAL_
OPS={final_ops
}"
)
print
(
"FLOW12_PIPELINE_OK"
)
...
...
vip-report/scripts/run_flow14_livestream_pipeline.py
View file @
52222f72
...
...
@@ -25,7 +25,8 @@ def parse_args() -> argparse.Namespace:
p
.
add_argument
(
"--mysql-password"
,
default
=
"ck@123456"
)
p
.
add_argument
(
"--mysql-database"
,
default
=
"ckc_cep_db_test"
)
p
.
add_argument
(
"--download-dir"
,
default
=
r"C:\vipdata"
)
p
.
add_argument
(
"--skip-compose"
,
action
=
"store_true"
,
help
=
"Only download/import/render 2 images, skip final compose"
)
p
.
add_argument
(
"--skip-compose"
,
action
=
"store_true"
,
help
=
"Deprecated. Compose preview is disabled by default."
)
p
.
add_argument
(
"--skip-editable-ops"
,
action
=
"store_true"
,
help
=
"Skip generating slide14 editable ops."
)
return
p
.
parse_args
()
...
...
@@ -50,7 +51,7 @@ def build_common_args(args: argparse.Namespace) -> list[str]:
]
if
args
.
config_source
==
"yaml"
:
config_path
=
args
.
config
or
str
((
SCRIPT_DIR
.
parent
/
"config.yaml"
)
.
resolve
())
return
[
"--config"
,
config_path
]
return
[
"--config
-source"
,
"yaml"
,
"--config
"
,
config_path
]
return
common
...
...
@@ -97,20 +98,14 @@ def main() -> None:
[
args
.
python
,
str
(
root
/
"sync_cep_live_session_trend_assets.py"
),
*
common
],
]
if
not
args
.
skip_
compose
:
if
not
args
.
skip_
editable_ops
:
steps
.
append
(
[
args
.
python
,
str
(
root
/
"
compose_live_session_report
.py"
),
str
(
root
/
"
build_slide14_15_editable_ops
.py"
),
*
common
,
"--summary-image"
,
str
(
workdir
/
"assets"
/
"cep-live-session"
/
"cep-live-session-summary.png"
),
"--trend-image"
,
str
(
workdir
/
"assets"
/
"cep-live-session"
/
"cep-live-session-sales-penetration-trend.png"
),
"--pivot-image"
,
str
(
workdir
/
"assets"
/
"cep-live-goods-effect"
/
"cep-live-goods-effect-pivot.png"
),
"--output"
,
str
(
workdir
/
"assets"
/
"cep-live-session"
/
"cep-live-session-combined.png"
),
"--slides"
,
"14"
,
]
)
...
...
@@ -118,10 +113,9 @@ def main() -> None:
for
i
,
cmd
in
enumerate
(
steps
,
start
=
1
):
run_step
(
i
,
total
,
cmd
)
print
(
f
"FINAL_
IMAGE={workdir / 'assets' / 'cep-live-session' / 'cep-live-session-combined.png
'}"
)
print
(
f
"FINAL_
OPS={workdir / 'render-ops.flow14-15.editable.json
'}"
)
print
(
"FLOW14_PIPELINE_OK"
)
if
__name__
==
"__main__"
:
main
()
vip-report/scripts/run_flow15_live_goods_effect_pipeline.py
View file @
52222f72
...
...
@@ -3,7 +3,9 @@ from __future__ import annotations
import
argparse
import
subprocess
import
sys
from
dataclasses
import
dataclass
from
pathlib
import
Path
from
typing
import
Any
SCRIPT_DIR
=
Path
(
__file__
)
.
resolve
()
.
parent
if
str
(
SCRIPT_DIR
)
not
in
sys
.
path
:
...
...
@@ -12,6 +14,12 @@ if str(SCRIPT_DIR) not in sys.path:
from
runtime_config
import
load_runtime_config
@
dataclass
(
frozen
=
True
)
class
MonthKey
:
year
:
int
month
:
int
def
parse_args
()
->
argparse
.
Namespace
:
p
=
argparse
.
ArgumentParser
(
description
=
"Run flow15 live-goods-effect pipeline in one command."
)
p
.
add_argument
(
"--python"
,
default
=
sys
.
executable
,
help
=
"Python executable path"
)
...
...
@@ -25,14 +33,17 @@ def parse_args() -> argparse.Namespace:
p
.
add_argument
(
"--mysql-password"
,
default
=
"ck@123456"
)
p
.
add_argument
(
"--mysql-database"
,
default
=
"ckc_cep_db_test"
)
p
.
add_argument
(
"--download-dir"
,
default
=
r"C:\vipdata"
)
p
.
add_argument
(
"--skip-compose"
,
action
=
"store_true"
)
p
.
add_argument
(
"--report-year"
,
type
=
int
,
default
=
0
)
p
.
add_argument
(
"--report-month"
,
default
=
""
)
p
.
add_argument
(
"--skip-compose"
,
action
=
"store_true"
,
help
=
"Deprecated. Compose preview is disabled by default."
)
p
.
add_argument
(
"--skip-editable-ops"
,
action
=
"store_true"
,
help
=
"Skip generating slide15 editable ops."
)
return
p
.
parse_args
()
def
build_common_args
(
args
:
argparse
.
Namespace
)
->
list
[
str
]:
if
args
.
config_source
==
"yaml"
:
config_path
=
args
.
config
or
str
((
SCRIPT_DIR
.
parent
/
"config.yaml"
)
.
resolve
())
return
[
"--config"
,
config_path
]
return
[
"--config
-source"
,
"yaml"
,
"--config
"
,
config_path
]
return
[
"--config-source"
,
args
.
config_source
,
...
...
@@ -53,6 +64,41 @@ def build_common_args(args: argparse.Namespace) -> list[str]:
]
def
parse_month_number
(
raw
:
Any
)
->
int
:
if
raw
is
None
:
return
1
if
isinstance
(
raw
,
int
):
return
min
(
12
,
max
(
1
,
raw
))
text
=
str
(
raw
)
.
strip
()
import
re
m
=
re
.
search
(
r"(1[0-2]|0?[1-9])"
,
text
)
if
m
:
return
int
(
m
.
group
(
1
))
return
1
def
previous_month
(
m
:
MonthKey
)
->
MonthKey
:
if
m
.
month
==
1
:
return
MonthKey
(
m
.
year
-
1
,
12
)
return
MonthKey
(
m
.
year
,
m
.
month
-
1
)
def
same_month_last_year
(
m
:
MonthKey
)
->
MonthKey
:
return
MonthKey
(
m
.
year
-
1
,
m
.
month
)
def
resolve_report_month
(
args
:
argparse
.
Namespace
,
runtime_cfg
:
dict
[
str
,
Any
])
->
MonthKey
:
report_cfg
=
runtime_cfg
.
get
(
"report"
,
{})
year
=
int
(
args
.
report_year
or
report_cfg
.
get
(
"year"
,
2026
))
month
=
parse_month_number
(
args
.
report_month
or
report_cfg
.
get
(
"month_cn"
,
"1"
))
return
MonthKey
(
year
,
month
)
def
with_report_period
(
cmd
:
list
[
str
],
month
:
MonthKey
)
->
list
[
str
]:
return
[
*
cmd
,
"--report-year"
,
str
(
month
.
year
),
"--report-month"
,
str
(
month
.
month
)]
def
run_step
(
step_index
:
int
,
total
:
int
,
cmd
:
list
[
str
])
->
None
:
print
(
f
"[STEP {step_index}/{total}] {' '.join(cmd)}"
)
result
=
subprocess
.
run
(
cmd
,
text
=
True
,
capture_output
=
True
)
...
...
@@ -82,40 +128,48 @@ def main() -> None:
default_config
=
Path
(
args
.
config
)
if
args
.
config
else
(
root
.
parent
/
"config.yaml"
),
)
workdir
=
Path
(
runtime_cfg
[
"paths"
][
"workdir"
])
.
resolve
()
report_month
=
resolve_report_month
(
args
,
runtime_cfg
)
current_month
=
report_month
compare_month
=
same_month_last_year
(
report_month
)
steps
:
list
[
list
[
str
]]
=
[
[
args
.
python
,
str
(
root
/
"sync_vis_vendor_goods_effect_download.py"
),
*
common
,
"--download-dir"
,
args
.
download_dir
,
"--no-auto-load"
,
],
[
args
.
python
,
str
(
root
/
"load_vis_vendor_goods_effect_to_mysql.py"
),
*
common
,
"--download-dir"
,
args
.
download_dir
,
],
[
args
.
python
,
str
(
root
/
"sync_cep_live_goods_effect_summary_assets.py"
),
*
common
],
[
args
.
python
,
str
(
root
/
"sync_cep_live_goods_effect_pivot_assets.py"
),
*
common
],
[
args
.
python
,
str
(
root
/
"sync_cep_live_goods_effect_top5_assets.py"
),
*
common
],
with_report_period
(
[
args
.
python
,
str
(
root
/
"sync_vis_vendor_goods_effect_download.py"
),
*
common
,
"--download-dir"
,
args
.
download_dir
,
],
current_month
,
),
with_report_period
(
[
args
.
python
,
str
(
root
/
"sync_vis_vendor_goods_effect_download.py"
),
*
common
,
"--download-dir"
,
args
.
download_dir
,
],
compare_month
,
),
with_report_period
([
args
.
python
,
str
(
root
/
"sync_cep_live_goods_effect_summary_assets.py"
),
*
common
],
report_month
),
with_report_period
([
args
.
python
,
str
(
root
/
"sync_cep_live_goods_effect_pivot_assets.py"
),
*
common
],
current_month
),
with_report_period
([
args
.
python
,
str
(
root
/
"sync_cep_live_goods_effect_top5_assets.py"
),
*
common
],
current_month
),
]
if
not
args
.
skip_
compose
:
if
not
args
.
skip_
editable_ops
:
steps
.
append
(
[
args
.
python
,
str
(
root
/
"
compose_live_goods_effect_report
.py"
),
str
(
root
/
"
build_slide14_15_editable_ops
.py"
),
*
common
,
"--s
ummary-image
"
,
str
(
workdir
/
"assets"
/
"cep-live-goods-effect"
/
"cep-live-goods-effect-summary.png"
)
,
"--
top5-image
"
,
str
(
workdir
/
"assets"
/
"cep-live-goods-effect"
/
"cep-live-goods-effect-top5.png"
),
"--
output
"
,
str
(
workdir
/
"assets"
/
"cep-live-goods-effect"
/
"cep-live-goods-effect-combined.png"
),
"--s
lides
"
,
"15"
,
"--
report-year
"
,
str
(
report_month
.
year
),
"--
report-month
"
,
str
(
report_month
.
month
),
]
)
...
...
@@ -123,10 +177,9 @@ def main() -> None:
for
i
,
cmd
in
enumerate
(
steps
,
start
=
1
):
run_step
(
i
,
total
,
cmd
)
print
(
f
"FINAL_
IMAGE={workdir / 'assets' / 'cep-live-goods-effect' / 'cep-live-goods-effect-combined.png
'}"
)
print
(
f
"FINAL_
OPS={workdir / 'render-ops.flow14-15.editable.json
'}"
)
print
(
"FLOW15_PIPELINE_OK"
)
if
__name__
==
"__main__"
:
main
()
vip-report/scripts/run_flow17_membership_pipeline.py
View file @
52222f72
...
...
@@ -32,7 +32,7 @@ def parse_args() -> argparse.Namespace:
def
build_common_args
(
args
:
argparse
.
Namespace
)
->
list
[
str
]:
if
args
.
config_source
==
"yaml"
:
config_path
=
args
.
config
or
str
((
SCRIPT_DIR
.
parent
/
"config.yaml"
)
.
resolve
())
return
[
"--config"
,
config_path
]
return
[
"--config
-source"
,
"yaml"
,
"--config
"
,
config_path
]
return
[
"--config-source"
,
args
.
config_source
,
...
...
vip-report/scripts/sync_cep_live_goods_effect_summary_assets.py
View file @
52222f72
...
...
@@ -280,8 +280,7 @@ def main() -> None:
config_path
=
Path
(
args
.
config
or
r"D:\projects\skills\vip-report\config.yaml"
)
.
resolve
()
config
=
load_runtime_config
(
args
,
default_config
=
config_path
)
curr_month
=
resolve_report_month
(
args
,
config
)
prev_month
=
previous_month
(
curr_month
)
prev2_month
=
same_month_last_year
(
curr_month
)
compare_month
=
same_month_last_year
(
curr_month
)
mysql_cfg
=
config
[
"mysql"
]
mysql_user
=
mysql_cfg
.
get
(
"username"
)
or
mysql_cfg
.
get
(
"user"
)
...
...
@@ -296,11 +295,7 @@ def main() -> None:
)
try
:
curr_raw
=
fetch_month_rows
(
conn
,
curr_month
.
ym
)
prev_raw
=
fetch_month_rows
(
conn
,
prev_month
.
ym
)
compare_month
=
prev_month
.
ym
if
not
prev_raw
:
prev_raw
=
fetch_month_rows
(
conn
,
prev2_month
.
ym
)
compare_month
=
prev2_month
.
ym
prev_raw
=
fetch_month_rows
(
conn
,
compare_month
.
ym
)
finally
:
conn
.
close
()
...
...
@@ -320,7 +315,7 @@ def main() -> None:
operations_path
.
write_text
(
json
.
dumps
(
operations
,
ensure_ascii
=
False
,
indent
=
2
),
encoding
=
"utf-8"
)
manifest
=
{
"source"
:
{
"table"
:
TABLE_NAME
,
"current_month"
:
curr_month
.
ym
,
"compare_month"
:
compare_month
,
"config_path"
:
str
(
config_path
)},
"source"
:
{
"table"
:
TABLE_NAME
,
"current_month"
:
curr_month
.
ym
,
"compare_month"
:
compare_month
.
ym
,
"config_path"
:
str
(
config_path
)},
"asset_path"
:
str
(
image_path
),
"operations_path"
:
str
(
operations_path
),
"metrics"
:
metrics
,
...
...
vip-report/scripts/sync_cep_live_session_assets.py
View file @
52222f72
...
...
@@ -243,7 +243,8 @@ def render_image(output_path: Path, prev_key: MonthKey, prev: Agg, curr_key: Mon
table_w
=
sum
(
col_widths
)
table_h
=
row_h
*
4
img
=
Image
.
new
(
"RGB"
,
(
1126
,
160
),
"#e7e7e7"
)
image_w
=
max
(
1126
,
table_x
+
table_w
+
28
)
img
=
Image
.
new
(
"RGB"
,
(
image_w
,
160
),
"#e7e7e7"
)
draw
=
ImageDraw
.
Draw
(
img
)
font_title
=
load_font
(
38
,
bold
=
True
)
...
...
@@ -257,7 +258,9 @@ def render_image(output_path: Path, prev_key: MonthKey, prev: Agg, curr_key: Mon
draw
.
line
((
16
,
43
,
150
,
43
),
fill
=
"#cf2323"
,
width
=
2
)
# Brand text
draw
.
text
((
945
,
8
),
"CHARLES & KEITH"
,
fill
=
"#cfcfcf"
,
font
=
font_brand
)
brand_text
=
"CHARLES & KEITH"
brand_bbox
=
draw
.
textbbox
((
0
,
0
),
brand_text
,
font
=
font_brand
)
draw
.
text
((
image_w
-
(
brand_bbox
[
2
]
-
brand_bbox
[
0
])
-
18
,
8
),
brand_text
,
fill
=
"#cfcfcf"
,
font
=
font_brand
)
# Table header bg
header_bg
=
"#6c72df"
...
...
vip-report/scripts/sync_compass_rejection_goods_download.py
View file @
52222f72
from
__future__
import
annotations
import
argparse
import
base64
import
json
import
os
import
re
...
...
@@ -202,23 +203,40 @@ def build_js(spec: dict[str, Any]) -> str:
return s.display !== 'none' && s.visibility !== 'hidden' && r.width > 1 && r.height > 1;
}};
const text = (el) => ((el && el.textContent) || '').replace(/
\\
s+/g, ' ').trim();
const btn = Array.from(document.querySelectorAll('button,span,a,div')).find((el) => visible(el) && text(el) === '下载数据');
if (!btn) return false;
btn.dispatchEvent(new MouseEvent('click', {{ bubbles: true }}));
return true;
const cards = Array.from(document.querySelectorAll('.g-card')).filter(visible);
const detailCard = cards.find((card) => text(card).includes('拒退商品明细'))
|| cards.find((card) => text(card).includes('款号') && text(card).includes('下载数据'))
|| null;
const root = detailCard || document;
const candidates = Array.from(root.querySelectorAll('.ad-toolbar-box button, .ad-toolbar-box span, button'))
.filter((el) => visible(el) && text(el) === '下载数据');
const btn = candidates.find((el) => el.closest('.ad-toolbar-box')) || candidates[0];
if (!btn) {{
return {{
ok: false,
cardTexts: cards.map((card) => text(card).slice(0, 160)).slice(0, 8),
}};
}}
const clickable = btn.closest('button') || btn;
clickable.dispatchEvent(new MouseEvent('mousedown', {{ bubbles: true }}));
clickable.dispatchEvent(new MouseEvent('mouseup', {{ bubbles: true }}));
clickable.dispatchEvent(new MouseEvent('click', {{ bubbles: true }}));
return {{
ok: true,
cardText: detailCard ? text(detailCard).slice(0, 240) : '',
buttonText: text(clickable),
}};
}});
if (!clickDownload) {{
return {{ ok: false, code: 'DOWNLOAD_DATA_BUTTON_NOT_FOUND', url: page.url() }};
if (!clickDownload
.ok
) {{
return {{ ok: false, code: 'DOWNLOAD_DATA_BUTTON_NOT_FOUND',
clickDownload,
url: page.url() }};
}}
await sleep(
6
00);
await sleep(
10
00);
// click 不需要 in optional modal
try {{
const noNeed = ctx.locator('button:has-text("不需要"), span:has-text("不需要")').first();
if (await noNeed.count().catch(() => 0)) {{
await noNeed.click({{ timeout: 2500 }}).catch(() => null);
await sleep(400);
}}
const noNeed = ctx.locator('.el-message-box button:has-text("不需要"), .el-dialog button:has-text("不需要"), button:has-text("不需要")').first();
await noNeed.click({{ timeout: 15000, force: true }}).catch(() => null);
await sleep(1500);
}} catch {{}}
await page.goto('https://compass.vip.com/frontend/index.html#/dataCapturing/downloadCenter', {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
...
...
@@ -237,6 +255,7 @@ def build_js(spec: dict[str, Any]) -> str:
}}
await sleep(3500);
const rowsSeen = [];
for (let i = 0; i < spec.max_retries; i++) {{
const queryBtn = page.locator('button:has-text("查询"), span:has-text("查询")').first();
if (await queryBtn.count().catch(() => 0)) {{
...
...
@@ -244,43 +263,74 @@ def build_js(spec: dict[str, Any]) -> str:
}}
await sleep(2000);
const statusCell = page.locator('tbody tr:nth-child(1) td').nth(5);
const statusText = await statusCell.innerText().catch(() => '');
if (!/执行成功|成功/.test(statusText || '')) {{
await sleep(700);
continue;
}}
const rowCount = await page.locator('tbody tr').count().catch(() => 0);
for (let idx = 0; idx < Math.min(rowCount, 10); idx++) {{
const row = page.locator('tbody tr').nth(idx);
const rowText = (await row.innerText().catch(() => '')).replace(/
\\
s+/g, ' ').trim();
const fileSpan = row.locator('.status-name-span').first();
let uiFileName = await fileSpan.getAttribute('title').catch(() => '');
if (!uiFileName) {{
uiFileName = (await fileSpan.innerText().catch(() => '')).trim();
}}
rowsSeen.push({{ row: idx + 1, uiFileName, rowText }});
if (!uiFileName.includes('款号粒度') || !/执行成功|成功/.test(rowText || '')) {{
continue;
}}
const fileSpan = page.locator('tbody tr:nth-child(1) .status-name-span').first()
;
let uiFileName = await fileSpan.getAttribute('title').catch(() => ''
);
if (!uiFileName
) {{
uiFileName = (await fileSpan.innerText().catch(() => '')).trim()
;
}}
const taskId = (rowText.match(/^
\\
s*(
\\
d+)/) || [])[1] || ''
;
const downloadBtn = row.locator('button:has-text("下载"), span:has-text("下载")').first(
);
if (!(await downloadBtn.count().catch(() => 0))
) {{
continue
;
}}
const downloadBtn = page.locator('tbody tr:nth-child(1) td button:has-text("下载"), tbody tr:nth-child(1) td span:has-text("下载")').first();
if (!(await downloadBtn.count().catch(() => 0))) {{
continue;
if (!taskId) {{
continue;
}}
const filePayload = await page.evaluate(async (tid) => {{
const body = new URLSearchParams();
body.set('reqData', JSON.stringify({{ taskId: tid }}));
const res = await fetch('/report/asyncdownload/getOutputFile', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/x-www-form-urlencoded' }},
body: body.toString(),
credentials: 'include',
}});
const ab = await res.arrayBuffer();
const bytes = new Uint8Array(ab);
let binary = '';
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {{
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}}
return {{
ok: res.ok,
status: res.status,
contentType: res.headers.get('content-type') || '',
disposition: res.headers.get('content-disposition') || '',
base64: btoa(binary),
}};
}}, taskId);
if (!filePayload.ok) {{
throw new Error(`getOutputFile failed status=${{filePayload.status}} taskId=${{taskId}}`);
}}
const finalName = safeName(`${{spec.prefix}}_${{uiFileName || taskId + '.xlsx'}}`);
const targetPath = `${{spec.download_dir.replace(/
\\\\
/g, '/')}}/${{finalName}}`;
return {{
ok: true,
url: page.url(),
row: idx + 1,
taskId,
rowText,
uiFileName,
contentType: filePayload.contentType,
disposition: filePayload.disposition,
contentBase64: filePayload.base64,
savedAs: targetPath
}};
}}
const [dl] = await Promise.all([
page.waitForEvent('download', {{ timeout: 120000 }}),
downloadBtn.click({{ timeout: 3000 }}),
]);
const suggested = dl.suggestedFilename();
const finalName = safeName(`${{spec.prefix}}_${{uiFileName || suggested}}`);
const targetPath = `${{spec.download_dir.replace(/
\\\\
/g, '/')}}/${{finalName}}`;
await dl.saveAs(targetPath);
return {{
ok: true,
url: page.url(),
statusText,
uiFileName,
suggestedFilename: suggested,
savedAs: targetPath
}};
}}
return {{ ok: false, code: '
DOWNLOAD_NOT_READY'
, url: page.url() }};
return {{ ok: false, code: '
GOODS_STYLE_FILE_NOT_READY', rowsSeen: rowsSeen.slice(0, 40)
, url: page.url() }};
}}
"""
...
...
@@ -344,7 +394,12 @@ def main() -> None:
result
=
json
.
loads
(
json_line
)
if
not
result
.
get
(
"ok"
,
False
):
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
print
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
content_b64
=
result
.
pop
(
"contentBase64"
,
""
)
if
content_b64
:
target
=
Path
(
result
[
"savedAs"
])
.
resolve
()
target
.
parent
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
target
.
write_bytes
(
base64
.
b64decode
(
content_b64
))
print
(
json
.
dumps
(
result
,
ensure_ascii
=
True
))
if
__name__
==
"__main__"
:
...
...
vip-report/scripts/sync_compass_rejection_goods_style_download.py
0 → 100644
View file @
52222f72
from
__future__
import
annotations
import
argparse
import
json
import
os
import
re
import
shutil
import
subprocess
import
time
from
pathlib
import
Path
from
typing
import
Any
import
yaml
NPX_EXECUTABLE
=
shutil
.
which
(
"npx.cmd"
)
or
shutil
.
which
(
"npx"
)
or
"npx"
PLAYWRIGHT_CMD
=
[
NPX_EXECUTABLE
,
"--yes"
,
"--package"
,
"@playwright/cli"
,
"playwright-cli"
]
SESSION_NAME
=
"vip-report-monthly-sales"
TARGET_URL
=
"https://compass.vip.com/frontend/index.html#/rejectionAnalysis/index"
DOWNLOAD_CENTER_URL
=
"https://compass.vip.com/frontend/index.html#/dataCapturing/downloadCenter"
DEFAULT_DOWNLOAD_DIR
=
r"C:\vipdata"
def
run_cmd
(
args
:
list
[
str
],
*
,
cwd
:
Path
,
timeout
:
int
=
300
)
->
subprocess
.
CompletedProcess
[
str
]:
env
=
os
.
environ
.
copy
()
npm_cache
=
cwd
/
".npm-cache"
npm_cache
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
env
[
"NPM_CONFIG_CACHE"
]
=
str
(
npm_cache
)
env
[
"npm_config_cache"
]
=
str
(
npm_cache
)
return
subprocess
.
run
(
args
,
cwd
=
str
(
cwd
),
text
=
True
,
encoding
=
"utf-8"
,
errors
=
"replace"
,
capture_output
=
True
,
timeout
=
timeout
,
check
=
True
,
env
=
env
,
)
def
parse_month_number
(
raw
:
Any
)
->
int
:
if
raw
is
None
:
return
1
if
isinstance
(
raw
,
int
):
return
min
(
12
,
max
(
1
,
raw
))
m
=
re
.
search
(
r"(1[0-2]|0?[1-9])"
,
str
(
raw
))
return
int
(
m
.
group
(
1
))
if
m
else
1
def
resolve_period
(
config_path
:
Path
,
report_year_arg
:
int
,
report_month_arg
:
str
)
->
tuple
[
int
,
int
]:
cfg
=
yaml
.
safe_load
(
config_path
.
read_text
(
encoding
=
"utf-8"
))
report_cfg
=
cfg
.
get
(
"report"
,
{})
year
=
int
(
report_year_arg
or
report_cfg
.
get
(
"year"
,
2026
))
month
=
parse_month_number
(
report_month_arg
or
report_cfg
.
get
(
"month_cn"
,
"1"
))
return
year
,
month
def
build_js
(
spec
:
dict
[
str
,
Any
])
->
str
:
payload
=
json
.
dumps
(
spec
,
ensure_ascii
=
False
)
return
f
"""async function(page) {{
const spec = {payload};
const sleep = (ms) => page.waitForTimeout(ms);
const safeName = (name) => String(name || '')
.replace(/[
\\\\
/:*?"<>|]/g, '_')
.replace(/
\\
s+/g, '_')
.replace(/_+/g, '_');
await page.goto(`${{spec.url}}?t=${{Date.now()}}`, {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
await sleep(5000);
const tab = page.locator('.el-tabs__item:has-text("拒退商品分析")').first();
if (!(await tab.count().catch(() => 0))) {{
return {{ ok: false, code: 'GOODS_TAB_NOT_FOUND', url: page.url() }};
}}
await tab.click({{ timeout: 5000, force: true }});
await sleep(2500);
const activeText = await page.evaluate(() => {{
const active = document.querySelector('.el-tabs__item.is-active');
return (active?.textContent || '').replace(/
\\
s+/g, ' ').trim();
}}).catch(() => '');
if (!activeText.includes('拒退商品分析')) {{
return {{ ok: false, code: 'GOODS_TAB_NOT_ACTIVE', activeText, url: page.url() }};
}}
const downloadBtn = page.locator('button:has-text("下载数据")').first();
if (!(await downloadBtn.count().catch(() => 0))) {{
return {{ ok: false, code: 'DOWNLOAD_DATA_BUTTON_NOT_FOUND', activeText, url: page.url() }};
}}
await downloadBtn.click({{ timeout: 5000, force: true }});
await sleep(700);
const noNeed = page.locator('button:has-text("不需要"), span:has-text("不需要")').first();
if (await noNeed.count().catch(() => 0)) {{
await noNeed.click({{ timeout: 3000, force: true }}).catch(() => null);
await sleep(700);
}}
await page.goto(spec.download_center_url, {{ waitUntil: 'domcontentloaded', timeout: 120000 }});
await sleep(2000);
const rowsSeen = [];
for (let attempt = 0; attempt < spec.max_retries; attempt++) {{
const queryBtn = page.locator('button:has-text("查询"), span:has-text("查询")').first();
if (await queryBtn.count().catch(() => 0)) {{
await queryBtn.click({{ timeout: 2500 }}).catch(() => null);
}}
await sleep(2200);
const rowCount = await page.locator('tbody tr').count().catch(() => 0);
for (let idx = 0; idx < Math.min(rowCount, 10); idx++) {{
const row = page.locator('tbody tr').nth(idx);
const text = (await row.innerText().catch(() => '')).replace(/
\\
s+/g, ' ').trim();
const nameEl = row.locator('.status-name-span').first();
let fileName = await nameEl.getAttribute('title').catch(() => '');
if (!fileName) fileName = (await nameEl.innerText().catch(() => '')).trim();
rowsSeen.push({{ row: idx + 1, fileName, text }});
if (!fileName.includes('款号粒度') || !/执行成功|成功/.test(text)) continue;
const btn = row.locator('button:has-text("下载"), span:has-text("下载")').first();
if (!(await btn.count().catch(() => 0))) continue;
const [dl] = await Promise.all([
page.waitForEvent('download', {{ timeout: 120000 }}),
btn.click({{ timeout: 5000, force: true }}),
]);
const suggested = dl.suggestedFilename();
const finalName = safeName(`${{spec.prefix}}_${{fileName || suggested}}`);
const targetPath = `${{spec.download_dir.replace(/
\\\\
/g, '/')}}/${{finalName}}`;
await dl.saveAs(targetPath);
return {{ ok: true, savedAs: targetPath, fileName, suggestedFilename: suggested, rowText: text }};
}}
}}
return {{ ok: false, code: 'GOODS_STYLE_FILE_NOT_READY', rowsSeen: rowsSeen.slice(0, 30), url: page.url() }};
}}
"""
def
parse_args
()
->
argparse
.
Namespace
:
p
=
argparse
.
ArgumentParser
(
description
=
"Download Compass rejection goods style-granularity file."
)
p
.
add_argument
(
"--config"
,
default
=
r"D:\projects\skills\vip-report\config.yaml"
)
p
.
add_argument
(
"--session"
,
default
=
SESSION_NAME
)
p
.
add_argument
(
"--report-year"
,
type
=
int
,
default
=
0
)
p
.
add_argument
(
"--report-month"
,
default
=
""
)
p
.
add_argument
(
"--download-dir"
,
default
=
DEFAULT_DOWNLOAD_DIR
)
p
.
add_argument
(
"--max-retries"
,
type
=
int
,
default
=
30
)
return
p
.
parse_args
()
def
main
()
->
None
:
args
=
parse_args
()
config_path
=
Path
(
args
.
config
)
.
resolve
()
cfg
=
yaml
.
safe_load
(
config_path
.
read_text
(
encoding
=
"utf-8"
))
workdir
=
Path
(
cfg
[
"paths"
][
"workdir"
])
.
resolve
()
year
,
month
=
resolve_period
(
config_path
,
args
.
report_year
,
args
.
report_month
)
prefix
=
f
"{year}{month:02d}"
download_dir
=
Path
(
args
.
download_dir
)
.
resolve
()
download_dir
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
js_path
=
workdir
/
f
"tmp-compass-rejection-goods-style-download-{prefix}-{time.time_ns()}.js"
js_path
.
write_text
(
build_js
(
{
"url"
:
TARGET_URL
,
"download_center_url"
:
DOWNLOAD_CENTER_URL
,
"prefix"
:
prefix
,
"download_dir"
:
str
(
download_dir
),
"max_retries"
:
int
(
args
.
max_retries
),
}
),
encoding
=
"utf-8"
,
)
try
:
out
=
run_cmd
(
PLAYWRIGHT_CMD
+
[
"--session"
,
args
.
session
,
"run-code"
,
"--filename"
,
str
(
js_path
)],
cwd
=
workdir
,
timeout
=
1200
,
)
.
stdout
finally
:
js_path
.
unlink
(
missing_ok
=
True
)
if
"### Error"
in
out
:
raise
RuntimeError
(
out
.
strip
())
json_line
=
""
for
line
in
out
.
splitlines
():
s
=
line
.
strip
()
if
s
.
startswith
(
"{"
)
and
s
.
endswith
(
"}"
):
json_line
=
s
break
if
not
json_line
:
raise
RuntimeError
(
f
"No JSON result from run-code: {out}"
)
result
=
json
.
loads
(
json_line
)
if
not
result
.
get
(
"ok"
,
False
):
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
print
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
if
__name__
==
"__main__"
:
main
()
vip-report/scripts/sync_compass_traffic_source_download.py
View file @
52222f72
from
__future__
import
annotations
import
argparse
import
base64
import
json
import
os
import
re
...
...
@@ -309,18 +310,39 @@ def build_download_js(spec: dict[str, Any]) -> str:
return rows.find(r=>r.id===String(tid)) || null;
}}, taskId);
if(hit && hit.hasDl && hit.status.includes('成功')){{
const btn = page.locator(`tbody tr:nth-child(${{hit.row}}) td:nth-child(7) button:has-text("下载")`).first();
if(hit && hit.hasDl && /成功|执行成功|鎴愬姛/.test(hit.status)){{
const sanitize=(s)=>String(s||'').replace(/[
\\\\
/:*?"<>|]/g,'_').replace(/
\\
s+/g,' ').trim();
const [dl] = await Promise.all([page.waitForEvent('download',{{timeout:60000}}), btn.click()]);
const suggestedRaw = sanitize(await dl.suggestedFilename());
const byTaskName = sanitize(hit.name || '');
const base = byTaskName ||
suggestedRaw ||
String(taskId || 'download');
const base = byTaskName || String(taskId || 'download');
const ensured = /
\\
.(xlsx|xls|csv)$/i.test(base) ? base : `${{base}}.xlsx`;
const finalName = `${{prefix}}_${{kind}}_${{ensured}}`;
const path = `${{SAVE}}/${{finalName}}`;
await dl.saveAs(path);
return {{taskId, taskType, taskName:hit.name, path}};
const filePayload = await page.evaluate(async (tid) => {{
const body = new URLSearchParams();
body.set('reqData', JSON.stringify({{ taskId: tid }}));
const res = await fetch('/report/asyncdownload/getOutputFile', {{
method: 'POST',
headers: {{ 'Content-Type': 'application/x-www-form-urlencoded' }},
body: body.toString(),
credentials: 'include',
}});
const ab = await res.arrayBuffer();
const bytes = new Uint8Array(ab);
let binary = '';
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {{
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}}
return {{
ok: res.ok,
status: res.status,
contentType: res.headers.get('content-type') || '',
disposition: res.headers.get('content-disposition') || '',
base64: btoa(binary),
}};
}}, String(taskId));
if(!filePayload.ok) throw new Error(`getOutputFile failed status=${{filePayload.status}} taskId=${{taskId}}`);
return {{taskId, taskType, taskName:hit.name, path, contentType:filePayload.contentType, disposition:filePayload.disposition, contentBase64:filePayload.base64}};
}}
}}
throw new Error(`downloadByTaskId timeout taskType=${{taskType}} taskId=${{taskId}}`);
...
...
@@ -432,9 +454,16 @@ def main() -> None:
if
not
json_line
:
raise
RuntimeError
(
f
"No JSON result from run-code: {out}"
)
result
=
json
.
loads
(
json_line
)
for
item
in
result
.
get
(
"done"
,
[]):
content_b64
=
item
.
pop
(
"contentBase64"
,
""
)
if
not
content_b64
:
continue
target
=
Path
(
item
[
"path"
])
target
.
parent
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
target
.
write_bytes
(
base64
.
b64decode
(
content_b64
))
if
not
result
.
get
(
"ok"
,
False
):
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
Fals
e
))
print
(
json
.
dumps
(
result
,
ensure_ascii
=
Fals
e
))
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
Tru
e
))
print
(
json
.
dumps
(
result
,
ensure_ascii
=
Tru
e
))
if
__name__
==
"__main__"
:
...
...
vip-report/scripts/sync_inventory_monthly_assets.py
View file @
52222f72
...
...
@@ -301,8 +301,8 @@ def build_subcategory_filters(
def
build_s08_filters
(
report_month
:
str
,
report_year
:
int
)
->
list
[
dict
[
str
,
Any
]]:
"""S08
works only with the current report year and current report month on live Tableau
."""
month_
index
=
resolve_month_index
(
report_month
)
"""S08
is a current-month composition table in the business deck
."""
month_
value_variants
=
build_month_value_variants
(
report_month
,
cumulative
=
False
)
return
[
{
"label"
:
"storename"
,
...
...
@@ -327,11 +327,7 @@ def build_s08_filters(report_month: str, report_year: int) -> list[dict[str, Any
{
"label"
:
"month"
,
"field_candidates"
:
[
"月(billdate)"
,
"billdate 月"
,
"Month"
,
"month"
,
"月"
],
"value_candidates"
:
[
[
str
(
month_index
)],
[
f
"{month_index:02d}"
],
[
normalize_month_label
(
report_month
)],
],
"value_candidates"
:
month_value_variants
,
"require_worksheet_match"
:
False
,
},
]
...
...
vip-report/scripts/sync_monthly_sales_assets.py
View file @
52222f72
...
...
@@ -41,6 +41,7 @@ CAPTURE_EXECUTION_ORDER = [
# 1) Overview (after applying filters) -> download image
# 2) StoreSalesinDetail -> download image
"overview"
,
"overview_s02_table_single"
,
"store_sales_detail"
,
# S03 dependencies keep their natural order afterwards.
"overview_s03"
,
...
...
@@ -139,9 +140,11 @@ def resolve_month_values(report_month: str, month_mode: str) -> list[str]:
def
build_specs
(
report_month
:
str
,
report_year
:
int
,
compare_year
:
int
,
month_mode
:
str
)
->
dict
[
str
,
Any
]:
"""Build the Tableau capture and crop specs for monthly sales slides."""
# S02/S03 monthly sales always use cumulative months from January to the report month.
cumulative_month_value_candidates
=
build_month_value_variants
(
report_month
,
cumulative
=
True
)
s03_month_values
=
build_cumulative_month_values
(
report_month
)
report_month
=
normalize_month_label
(
report_month
)
use_cumulative_months
=
month_mode
!=
"single"
month_value_candidates
=
build_month_value_variants
(
report_month
,
cumulative
=
use_cumulative_months
)
single_month_value_candidates
=
build_month_value_variants
(
report_month
,
cumulative
=
False
)
s03_month_values
=
resolve_month_values
(
report_month
,
month_mode
)
summary_strip_crop
=
{
"left"
:
0
,
"top"
:
528
,
"width"
:
1383
,
"height"
:
332
}
common_monthly_filters
=
[
{
...
...
@@ -154,7 +157,7 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
{
"label"
:
"month"
,
"field_candidates"
:
[
"Month"
,
"月(daily)"
],
"value_candidates"
:
cumulative_
month_value_candidates
,
"value_candidates"
:
month_value_candidates
,
"apply_all_fields"
:
True
,
"target_scope"
:
"active_sheet"
,
},
...
...
@@ -168,9 +171,21 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
{
"field"
:
"Sales Type"
,
"values"
:
[
"GMV"
],
"target_scope"
:
"active_sheet"
},
{
"field"
:
"Brand"
,
"values"
:
[
"CK"
],
"target_scope"
:
"active_sheet"
},
]
common_single_monthly_filters
=
[
{
**
item
,
"value_candidates"
:
single_month_value_candidates
,
}
if
item
.
get
(
"label"
)
==
"month"
else
item
for
item
in
common_monthly_filters
]
# S02 overview needs both current-year and comparison-year measures, so it must not
# apply a hard Year filter that would exclude year2-backed values like Sales2/LFL.
s02_overview_filters
=
[
item
for
item
in
common_monthly_filters
if
item
.
get
(
"label"
)
!=
"year"
]
s02_overview_single_filters
=
[
item
for
item
in
common_single_monthly_filters
if
item
.
get
(
"label"
)
!=
"year"
]
return
{
"captures"
:
[
{
...
...
@@ -184,6 +199,17 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
"raw_screenshot_name"
:
"monthly-sales-overview.png"
,
"note"
:
"S02 overview dashboard export."
,
},
{
"capture_id"
:
"overview_s02_table_single"
,
"session"
:
SESSION_NAME
,
"hash_url"
:
"#/views/ECMonthlySalesReport_17250004228820/Overview?:iid=2"
,
"inner_frame_fragment"
:
"/views/ECMonthlySalesReport_17250004228820/Overview?"
,
"activate_sheet"
:
"Overview"
,
"filters"
:
[
*
s02_overview_single_filters
],
"params"
:
{},
"raw_screenshot_name"
:
"monthly-sales-overview-single.png"
,
"note"
:
"S02 current-month overview table export."
,
},
{
"capture_id"
:
"overview_s03"
,
"session"
:
SESSION_NAME
,
...
...
@@ -224,8 +250,8 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
{
"field"
:
"Sales Type"
,
"values"
:
[
"GMV"
]},
{
"field"
:
"Storename"
,
"values"
:
[
"CKC-VIP"
]},
{
"field"
:
"Year"
,
"values"
:
[
str
(
report_year
)]},
{
"field"
:
"Month"
,
"values"
:
[
report_month
]
},
{
"field"
:
"月(daily)"
,
"values"
:
[
report_month
]
},
{
"field"
:
"Month"
,
"values"
:
s03_month_values
},
{
"field"
:
"月(daily)"
,
"values"
:
s03_month_values
},
],
"params"
:
{
"year1"
:
report_year
,
"year2"
:
compare_year
},
"raw_screenshot_name"
:
"monthly-sales-store-kpi-lfl.png"
,
...
...
@@ -251,7 +277,7 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
"shape_name"
:
"鍥剧墖 6"
,
"shape_id"
:
7
,
"asset_name"
:
"s02_monthly_sales_overview_table"
,
"capture_id"
:
"overview"
,
"capture_id"
:
"overview
_s02_table_single
"
,
"crop"
:
{
"left"
:
55
,
"top"
:
140
,
"width"
:
1330
,
"height"
:
311
},
"resize_to"
:
{
"width"
:
1097
,
"height"
:
278
},
"source_view"
:
"Overview"
,
...
...
@@ -283,22 +309,22 @@ def build_specs(report_month: str, report_year: int, compare_year: int, month_mo
"panels"
:
[
{
"panel_code"
:
"gp"
,
"crop"
:
{
"left"
:
0
,
"top"
:
1610
,
"width"
:
65
0
,
"height"
:
400
},
"crop"
:
{
"left"
:
0
,
"top"
:
1610
,
"width"
:
70
0
,
"height"
:
400
},
"dest"
:
{
"left"
:
0
,
"top"
:
0
,
"width"
:
542
,
"height"
:
297
},
},
{
"panel_code"
:
"con"
,
"crop"
:
{
"left"
:
650
,
"top"
:
1610
,
"width"
:
65
0
,
"height"
:
400
},
"crop"
:
{
"left"
:
700
,
"top"
:
1610
,
"width"
:
70
0
,
"height"
:
400
},
"dest"
:
{
"left"
:
568
,
"top"
:
0
,
"width"
:
542
,
"height"
:
297
},
},
{
"panel_code"
:
"atv"
,
"crop"
:
{
"left"
:
0
,
"top"
:
2000
,
"width"
:
65
0
,
"height"
:
360
},
"crop"
:
{
"left"
:
0
,
"top"
:
2000
,
"width"
:
70
0
,
"height"
:
360
},
"dest"
:
{
"left"
:
0
,
"top"
:
324
,
"width"
:
542
,
"height"
:
297
},
},
{
"panel_code"
:
"returnqty"
,
"crop"
:
{
"left"
:
650
,
"top"
:
2000
,
"width"
:
65
0
,
"height"
:
360
},
"crop"
:
{
"left"
:
700
,
"top"
:
2000
,
"width"
:
70
0
,
"height"
:
360
},
"dest"
:
{
"left"
:
568
,
"top"
:
324
,
"width"
:
542
,
"height"
:
297
},
},
],
...
...
vip-report/scripts/sync_top_products_assets.py
View file @
52222f72
...
...
@@ -2440,10 +2440,15 @@ def build_product_assets(
operations
:
list
[
dict
[
str
,
Any
]]
=
[]
manifest_items
:
list
[
dict
[
str
,
Any
]]
=
[]
for
slot
,
row
in
zip
(
slide_spec
[
"product_slots"
],
source_payload
[
"rows"
]):
# Match business layout: left column TOP1-5 from top to bottom, right column TOP6-10.
slots_by_rank
=
{
int
(
slot
[
"rank"
]):
slot
for
slot
in
slide_spec
[
"product_slots"
]}
for
rank
,
row
in
enumerate
(
source_payload
[
"rows"
],
start
=
1
):
slot
=
slots_by_rank
.
get
(
rank
)
if
slot
is
None
:
continue
articlecode
=
row
[
"articlecode"
]
image_url
=
row
.
get
(
"image_url"
)
or
PRODUCT_IMAGE_URL
.
format
(
articlecode
=
articlecode
)
asset_name
=
f
"{slide_code.lower()}_top_product_{
slot['rank']
:02d}_{sanitize_file_token(articlecode)}"
asset_name
=
f
"{slide_code.lower()}_top_product_{
rank
:02d}_{sanitize_file_token(articlecode)}"
asset_path
=
asset_dir
/
f
"{asset_name}.png"
render_product_image
(
download_image
(
image_url
),
...
...
vip-report/scripts/sync_vis_live_session_downloads.py
View file @
52222f72
...
...
@@ -151,9 +151,19 @@ def build_download_js(spec: dict[str, Any]) -> str:
const frame = page.frames().find((f) => f.url().includes('vlp-admin.vip.com/vis/#/vis/datas/realTimeSells'));
const ctx = frame || page;
const startInput = ctx.locator('input[placeholder=
\"\\
u5f00
\\
u59cb
\\
u65e5
\\
u671f
\"
]').first();
const endInput = ctx.locator('input[placeholder=
\"\\
u7ed3
\\
u675f
\\
u65e5
\\
u671f
\"
]').first();
if (!(await startInput.count().catch(() => 0)) || !(await endInput.count().catch(() => 0))) {{
let dateSetMode = 'placeholder-inputs';
let startInput = ctx.locator('input[placeholder=
\"\\
u5f00
\\
u59cb
\\
u65e5
\\
u671f
\"
]').first();
let endInput = ctx.locator('input[placeholder=
\"\\
u7ed3
\\
u675f
\\
u65e5
\\
u671f
\"
]').first();
let startCount = await startInput.count().catch(() => 0);
let endCount = await endInput.count().catch(() => 0);
if (!startCount || !endCount) {{
dateSetMode = 'range-editor-inputs';
startInput = ctx.locator('.el-date-editor--daterange input.el-range-input, .el-range-editor input.el-range-input').nth(0);
endInput = ctx.locator('.el-date-editor--daterange input.el-range-input, .el-range-editor input.el-range-input').nth(1);
startCount = await startInput.count().catch(() => 0);
endCount = await endInput.count().catch(() => 0);
}}
if (!startCount || !endCount) {{
return {{ ok: false, code: 'DATE_INPUT_NOT_FOUND', ym: spec.ym, url: page.url(), frame: frame ? frame.url() : null }};
}}
...
...
@@ -250,6 +260,7 @@ def build_download_js(spec: dict[str, Any]) -> str:
ok: true,
ym: spec.ym,
prefix: spec.prefix,
dateSetMode,
setDateResult,
queryClick,
downloadClick,
...
...
vip-report/scripts/sync_vis_member_channel_trend_capture.py
View file @
52222f72
...
...
@@ -507,7 +507,7 @@ def main() -> None:
result
=
json
.
loads
(
json_line
)
if
not
result
.
get
(
"ok"
,
False
):
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
validate_image
(
out_path
,
min_w
=
70
0
,
min_h
=
250
,
min_bytes
=
10000
)
validate_image
(
out_path
,
min_w
=
56
0
,
min_h
=
250
,
min_bytes
=
10000
)
print
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
...
...
vip-report/scripts/sync_vis_member_consumption_capture.py
View file @
52222f72
...
...
@@ -448,7 +448,7 @@ def main() -> None:
result
=
json
.
loads
(
json_line
)
if
not
result
.
get
(
"ok"
,
False
):
raise
RuntimeError
(
json
.
dumps
(
result
,
ensure_ascii
=
False
))
validate_image
(
out_path
,
min_w
=
76
0
,
min_h
=
240
,
min_bytes
=
8000
)
validate_image
(
out_path
,
min_w
=
68
0
,
min_h
=
240
,
min_bytes
=
8000
)
last_error
=
None
break
except
Exception
as
e
:
...
...
vip-report/scripts/sync_vis_vendor_goods_effect_download.py
View file @
52222f72
...
...
@@ -373,7 +373,7 @@ def main() -> None:
str
(
result
.
get
(
"savedAs"
,
""
)),
]
if
args
.
config_source
==
"yaml"
:
loader_args
.
extend
([
"--config"
,
str
(
config_path
.
resolve
())])
loader_args
.
extend
([
"--config
-source"
,
"yaml"
,
"--config
"
,
str
(
config_path
.
resolve
())])
else
:
loader_args
.
extend
(
[
...
...
vip-report/scripts/sync_warehouse_100060_assets.py
View file @
52222f72
...
...
@@ -192,7 +192,7 @@ def build_filter_config(
{
"caption"
:
"Month"
,
"action"
:
"replace"
,
"values"
:
[
report_month_label
,
next_month_label
(
report_month_number
)
],
"values"
:
[
report_month_label
],
},
{
"caption"
:
"Week"
,
"action"
:
"all"
},
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment