Commit aec72fb1 authored by allen.wang's avatar allen.wang

fix:优化报表效果-合并session

parent 141c5999
......@@ -70,6 +70,20 @@ function Write-StageLog {
Write-Host ("[{0}] [{1}] {2}" -f $timestamp, $Scope, $Message)
}
function Get-SafeSessionToken {
param([string]$Value)
$token = [System.IO.Path]::GetFileName($Value)
if (-not $token) {
$token = "output"
}
$token = ($token -replace "[^A-Za-z0-9_-]", "-").Trim("-")
if (-not $token) {
$token = "output"
}
return $token
}
function Invoke-PythonScriptWithRetry {
param(
[string]$ScriptPath,
......@@ -248,6 +262,17 @@ $inventorySlides = Resolve-GroupSlides -RequestedSlides $normalizedSlides -Group
$topSlides = Resolve-GroupSlides -RequestedSlides $normalizedSlides -GroupSlides @("S09", "S10")
$campaignSlides = Resolve-GroupSlides -RequestedSlides $normalizedSlides -GroupSlides @("S11")
$warehouseSlides = Resolve-GroupSlides -RequestedSlides $normalizedSlides -GroupSlides @("S13")
$sharedSessionToken = Get-SafeSessionToken -Value $outputDir
$sharedPlaywrightSession = "vip-report-tableau-$sharedSessionToken"
$sharedTableauStatePath = Join-Path $outputDir ".playwright-cli\tableau-shared-state.json"
$sharedBrowserConfigPath = Join-Path $root ".playwright\cli.config.json"
$sharedTableauArgs = @(
"--playwright-session", $sharedPlaywrightSession,
"--tableau-state-path", $sharedTableauStatePath
)
if (Test-Path -LiteralPath $sharedBrowserConfigPath) {
$sharedTableauArgs += @("--browser-config-path", $sharedBrowserConfigPath)
}
$opsPaths = @()
Push-Location $root
......@@ -268,6 +293,7 @@ try {
$args += "--single-month"
}
$args += @("--output-dir", $outputDir)
$args += $sharedTableauArgs
Invoke-PythonScriptWithRetry -ScriptPath (Join-Path $root "scripts\sync_monthly_sales_assets.py") -ScriptArgs $args -StageName "monthly-sales" -SlideCodes $monthlySlides -MaxAttempts $Retries -SleepSeconds $RetryDelaySeconds
$opsPaths += Join-Path $outputDir "render-ops.monthly-sales.live.json"
}
......@@ -285,6 +311,7 @@ try {
$args += @("--compare-year", "$CompareYear")
}
$args += @("--output-dir", $outputDir)
$args += $sharedTableauArgs
$inventorySlideList = @($inventorySlides)
$inventoryRetries = if ($inventorySlideList.Count -eq 1) { 1 } else { $Retries }
Invoke-PythonScriptWithRetry -ScriptPath (Join-Path $root "scripts\sync_inventory_monthly_assets.py") -ScriptArgs $args -StageName "inventory-monthly" -SlideCodes $inventorySlides -MaxAttempts $inventoryRetries -SleepSeconds $RetryDelaySeconds
......@@ -304,6 +331,7 @@ try {
$args += @("--compare-year", "$CompareYear")
}
$args += @("--output-dir", $outputDir)
$args += $sharedTableauArgs
Invoke-PythonScriptWithRetry -ScriptPath (Join-Path $root "scripts\sync_top_products_assets.py") -ScriptArgs $args -StageName "top-products" -SlideCodes $topSlides -MaxAttempts $Retries -SleepSeconds $RetryDelaySeconds
$opsPaths += Join-Path $outputDir "render-ops.top-products.live.json"
}
......@@ -338,6 +366,7 @@ try {
$args += @("--compare-year", "$CompareYear")
}
$args += @("--output-dir", $outputDir)
$args += $sharedTableauArgs
Invoke-PythonScriptWithRetry -ScriptPath (Join-Path $root "scripts\sync_warehouse_100060_assets.py") -ScriptArgs $args -StageName "warehouse-100060" -SlideCodes $warehouseSlides -MaxAttempts $Retries -SleepSeconds $RetryDelaySeconds
$opsPaths += Join-Path $outputDir "render-ops.warehouse-100060.live.json"
}
......
......@@ -783,10 +783,24 @@ def run_cmd(
check: bool = True,
) -> subprocess.CompletedProcess[str]:
"""Helper."""
env = os.environ.copy()
local_tmp = cwd / ".tmp"
npm_cache = cwd / ".npm-cache"
browsers_path = cwd / ".ms-playwright"
daemon_dir = cwd / ".playwright-daemon"
for path in (local_tmp, npm_cache, browsers_path, daemon_dir):
path.mkdir(parents=True, exist_ok=True)
env["TEMP"] = str(local_tmp)
env["TMP"] = str(local_tmp)
env["npm_config_cache"] = str(npm_cache)
env["NPM_CONFIG_CACHE"] = str(npm_cache)
env["PLAYWRIGHT_BROWSERS_PATH"] = str(browsers_path)
env["PLAYWRIGHT_DAEMON_SESSION_DIR"] = str(daemon_dir)
creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0
process = subprocess.Popen(
args,
cwd=str(cwd),
env=env,
text=True,
encoding="utf-8",
errors="replace",
......@@ -2758,10 +2772,32 @@ def apply_snapshot_quick_filters(
}
def ensure_browser_session(session: str, *, cwd: Path, deadline: float | None = None) -> None:
def ensure_browser_session(
session: str,
*,
cwd: Path,
deadline: float | None = None,
browser_config_path: Path | None = None,
) -> None:
"""Helper."""
try:
run_playwright(
["--session", session, "tab-list"],
cwd=cwd,
timeout=remaining_timeout(15, deadline=deadline, action="checking existing browser session"),
)
return
except subprocess.CalledProcessError as error:
combined = f"{error.stdout}\n{error.stderr}"
if "is not open" not in combined:
raise
args = ["--session", session, "open"]
resolved_browser_config_path = browser_config_path or PLAYWRIGHT_CLI_CONFIG_PATH
if resolved_browser_config_path is not None:
args.extend(["--config", str(resolved_browser_config_path)])
args.append("about:blank")
run_playwright(
["--session", session, "open", "--config", str(PLAYWRIGHT_CLI_CONFIG_PATH), "about:blank"],
args,
cwd=cwd,
timeout=remaining_timeout(60, deadline=deadline, action="opening the browser session"),
)
......@@ -2914,6 +2950,21 @@ def parse_args() -> argparse.Namespace:
default="",
help="Override the output/work directory used for generated assets and ops.",
)
parser.add_argument(
"--playwright-session",
default="",
help="Override the shared Playwright session name used for Tableau capture.",
)
parser.add_argument(
"--tableau-state-path",
default="",
help="Override the shared Playwright storage-state path used for Tableau login reuse.",
)
parser.add_argument(
"--browser-config-path",
default="",
help="Optional Playwright CLI config path used when opening a new shared browser session.",
)
return parser.parse_args()
def collect_required_capture_ids(filtered_assets: list[dict[str, Any]]) -> set[str]:
"""Helper."""
......@@ -3413,19 +3464,33 @@ def main() -> None:
run_deadline = resolve_deadline(
SINGLE_SLIDE_TIMEOUT_SECONDS if should_use_single_slide_timeout(requested) else None
)
session = (
session = args.playwright_session.strip() if args.playwright_session.strip() else (
f"{SESSION_NAME}-{vip_workdir.name}-{int(time.time())}"
if args.output_dir
else f"{SESSION_NAME}-{int(time.time())}"
)
state_path = (
vip_workdir / ".playwright-cli" / f"{SESSION_NAME}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / SESSION_NAME / "state.json"
Path(args.tableau_state_path).expanduser().resolve()
if args.tableau_state_path.strip()
else (
vip_workdir / ".playwright-cli" / f"{SESSION_NAME}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / SESSION_NAME / "state.json"
)
)
browser_config_path = (
Path(args.browser_config_path).expanduser().resolve()
if args.browser_config_path.strip()
else PLAYWRIGHT_CLI_CONFIG_PATH
)
browser_started_at = time.monotonic()
ensure_browser_session(session, cwd=vip_workdir, deadline=run_deadline)
ensure_browser_session(
session,
cwd=vip_workdir,
deadline=run_deadline,
browser_config_path=browser_config_path,
)
log_timing("open browser session", browser_started_at, deadline=run_deadline)
load_state_started_at = time.monotonic()
......
......@@ -2,6 +2,7 @@
import argparse
import json
import os
import re
import shutil
import subprocess
......@@ -313,9 +314,23 @@ def run_cmd(
check: bool = True,
) -> subprocess.CompletedProcess[str]:
"""Helper."""
env = os.environ.copy()
local_tmp = cwd / ".tmp"
npm_cache = cwd / ".npm-cache"
browsers_path = cwd / ".ms-playwright"
daemon_dir = cwd / ".playwright-daemon"
for path in (local_tmp, npm_cache, browsers_path, daemon_dir):
path.mkdir(parents=True, exist_ok=True)
env["TEMP"] = str(local_tmp)
env["TMP"] = str(local_tmp)
env["npm_config_cache"] = str(npm_cache)
env["NPM_CONFIG_CACHE"] = str(npm_cache)
env["PLAYWRIGHT_BROWSERS_PATH"] = str(browsers_path)
env["PLAYWRIGHT_DAEMON_SESSION_DIR"] = str(daemon_dir)
return subprocess.run(
args,
cwd=str(cwd),
env=env,
text=True,
encoding="utf-8",
errors="replace",
......@@ -748,9 +763,32 @@ def run_code(session: str, script_path: Path, *, cwd: Path, timeout: int = 120)
)
def ensure_browser_session(session: str, *, cwd: Path) -> None:
def session_is_open(session: str, *, cwd: Path) -> bool:
"""Helper."""
run_playwright(["--session", session, "open", "about:blank"], cwd=cwd, timeout=60)
try:
run_playwright(["--session", session, "tab-list"], cwd=cwd, timeout=15)
return True
except subprocess.CalledProcessError as error:
combined = f"{error.stdout}\n{error.stderr}"
if "is not open" in combined:
return False
raise
def ensure_browser_session(
session: str,
*,
cwd: Path,
browser_config_path: Path | None = None,
) -> None:
"""Helper."""
if session_is_open(session, cwd=cwd):
return
args = ["--session", session, "open"]
if browser_config_path is not None:
args.extend(["--config", str(browser_config_path)])
args.append("about:blank")
run_playwright(args, cwd=cwd, timeout=60)
def save_state(session: str, state_path: Path, *, cwd: Path) -> None:
......@@ -901,6 +939,21 @@ def parse_args() -> argparse.Namespace:
default="",
help="Override the output/work directory used for generated assets and ops.",
)
parser.add_argument(
"--playwright-session",
default="",
help="Override the shared Playwright session name used for Tableau capture.",
)
parser.add_argument(
"--tableau-state-path",
default="",
help="Override the shared Playwright storage-state path used for Tableau login reuse.",
)
parser.add_argument(
"--browser-config-path",
default="",
help="Optional Playwright CLI config path used when opening a new shared browser session.",
)
return parser.parse_args()
......@@ -1056,14 +1109,25 @@ def main() -> None:
log_progress("stage", f"start monthly-sales slides={','.join(sorted(requested))}")
session = SESSION_NAME if not args.output_dir else f"{SESSION_NAME}-{vip_workdir.name}"
session = args.playwright_session.strip() if args.playwright_session.strip() else (
SESSION_NAME if not args.output_dir else f"{SESSION_NAME}-{vip_workdir.name}"
)
state_path = (
vip_workdir / ".playwright-cli" / f"{session}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / session / "state.json"
Path(args.tableau_state_path).expanduser().resolve()
if args.tableau_state_path.strip()
else (
vip_workdir / ".playwright-cli" / f"{session}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / session / "state.json"
)
)
browser_config_path = (
Path(args.browser_config_path).expanduser().resolve()
if args.browser_config_path.strip()
else None
)
ensure_browser_session(session, cwd=vip_workdir)
ensure_browser_session(session, cwd=vip_workdir, browser_config_path=browser_config_path)
load_state_if_present(session, state_path, cwd=vip_workdir)
base_url = config["tableau"]["base_url"].rstrip("/")
......
......@@ -303,6 +303,21 @@ def parse_args() -> argparse.Namespace:
default="",
help="Override the output/work directory used for generated assets and ops.",
)
parser.add_argument(
"--playwright-session",
default="",
help="Override the shared Playwright session name used for Tableau capture.",
)
parser.add_argument(
"--tableau-state-path",
default="",
help="Override the shared Playwright storage-state path used for Tableau login reuse.",
)
parser.add_argument(
"--browser-config-path",
default="",
help="Optional Playwright CLI config path used when opening a new shared browser session.",
)
return parser.parse_args()
......@@ -1623,9 +1638,20 @@ def normalize_top_products_quick_filters(
}
def ensure_browser_session(session: str, *, cwd: Path) -> None:
def ensure_browser_session(session: str, *, cwd: Path, browser_config_path: Path | None = None) -> None:
"""Helper."""
run_playwright(["--session", session, "open", "about:blank"], cwd=cwd, timeout=60)
try:
run_playwright(["--session", session, "tab-list"], cwd=cwd, timeout=15)
return
except subprocess.CalledProcessError as error:
combined = f"{error.stdout}\n{error.stderr}"
if "is not open" not in combined:
raise
args = ["--session", session, "open"]
if browser_config_path is not None:
args.extend(["--config", str(browser_config_path)])
args.append("about:blank")
run_playwright(args, cwd=cwd, timeout=60)
def save_state(session: str, state_path: Path, *, cwd: Path) -> None:
......@@ -2467,13 +2493,24 @@ def main() -> None:
asset_dir.mkdir(parents=True, exist_ok=True)
data_dir.mkdir(parents=True, exist_ok=True)
session = SESSION_NAME if not args.output_dir else f"{SESSION_NAME}-{vip_workdir.name}"
session = args.playwright_session.strip() if args.playwright_session.strip() else (
SESSION_NAME if not args.output_dir else f"{SESSION_NAME}-{vip_workdir.name}"
)
state_path = (
vip_workdir / ".playwright-cli" / f"{session}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / session / "state.json"
Path(args.tableau_state_path).expanduser().resolve()
if args.tableau_state_path.strip()
else (
vip_workdir / ".playwright-cli" / f"{session}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / session / "state.json"
)
)
browser_config_path = (
Path(args.browser_config_path).expanduser().resolve()
if args.browser_config_path.strip()
else None
)
ensure_browser_session(session, cwd=vip_workdir)
ensure_browser_session(session, cwd=vip_workdir, browser_config_path=browser_config_path)
load_state_if_present(session, state_path, cwd=vip_workdir)
base_url = config["tableau"]["base_url"].rstrip("/")
......
......@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import calendar
import json
import os
import shutil
import subprocess
import time
......@@ -316,16 +317,58 @@ def run_cmd(
check: bool = True,
) -> subprocess.CompletedProcess[str]:
"""Helper."""
return subprocess.run(
env = os.environ.copy()
local_tmp = cwd / ".tmp"
npm_cache = cwd / ".npm-cache"
browsers_path = cwd / ".ms-playwright"
daemon_dir = cwd / ".playwright-daemon"
for path in (local_tmp, npm_cache, browsers_path, daemon_dir):
path.mkdir(parents=True, exist_ok=True)
env["TEMP"] = str(local_tmp)
env["TMP"] = str(local_tmp)
env["npm_config_cache"] = str(npm_cache)
env["NPM_CONFIG_CACHE"] = str(npm_cache)
env["PLAYWRIGHT_BROWSERS_PATH"] = str(browsers_path)
env["PLAYWRIGHT_DAEMON_SESSION_DIR"] = str(daemon_dir)
creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0
process = subprocess.Popen(
args,
cwd=str(cwd),
env=env,
text=True,
encoding="utf-8",
errors="replace",
capture_output=True,
timeout=timeout,
check=check,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
creationflags=creationflags,
)
try:
stdout, stderr = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired as exc:
if process.poll() is None:
if os.name == "nt":
subprocess.run(
["taskkill", "/F", "/T", "/PID", str(process.pid)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
else:
process.kill()
stdout, stderr = process.communicate()
exc.stdout = stdout
exc.stderr = stderr
raise
result = subprocess.CompletedProcess(args, process.returncode, stdout, stderr)
if check and process.returncode != 0:
raise subprocess.CalledProcessError(
process.returncode,
args,
output=stdout,
stderr=stderr,
)
return result
def run_playwright(args: list[str], *, cwd: Path, timeout: int = 120) -> str:
......@@ -573,9 +616,20 @@ def run_code(session: str, script_path: Path, *, cwd: Path, timeout: int = 120)
)
def ensure_browser_session(session: str, *, cwd: Path) -> None:
def ensure_browser_session(session: str, *, cwd: Path, browser_config_path: Path | None = None) -> None:
"""Helper."""
run_playwright(["--session", session, "open", "about:blank"], cwd=cwd, timeout=60)
try:
run_playwright(["--session", session, "tab-list"], cwd=cwd, timeout=15)
return
except subprocess.CalledProcessError as error:
combined = f"{error.stdout}\n{error.stderr}"
if "is not open" not in combined:
raise
args = ["--session", session, "open"]
if browser_config_path is not None:
args.extend(["--config", str(browser_config_path)])
args.append("about:blank")
run_playwright(args, cwd=cwd, timeout=60)
def save_state(session: str, state_path: Path, *, cwd: Path) -> None:
......@@ -701,6 +755,21 @@ def parse_args() -> argparse.Namespace:
default="",
help="Override the output/work directory used for generated assets and ops.",
)
parser.add_argument(
"--playwright-session",
default="",
help="Override the shared Playwright session name used for Tableau capture.",
)
parser.add_argument(
"--tableau-state-path",
default="",
help="Override the shared Playwright storage-state path used for Tableau login reuse.",
)
parser.add_argument(
"--browser-config-path",
default="",
help="Optional Playwright CLI config path used when opening a new shared browser session.",
)
return parser.parse_args()
......@@ -843,14 +912,25 @@ def main() -> None:
raw_screenshots: dict[str, Path] = {}
if not use_template_lock:
session = SESSION_NAME if not args.output_dir else f"{SESSION_NAME}-{vip_workdir.name}"
session = args.playwright_session.strip() if args.playwright_session.strip() else (
SESSION_NAME if not args.output_dir else f"{SESSION_NAME}-{vip_workdir.name}"
)
state_path = (
vip_workdir / ".playwright-cli" / f"{session}-state.json"
if args.output_dir
else vip_workdir / ".playwright-cli" / f"{session}-state.json"
Path(args.tableau_state_path).expanduser().resolve()
if args.tableau_state_path.strip()
else (
vip_workdir / ".playwright-cli" / f"{session}-state.json"
if args.output_dir
else workspace_root / "output" / "playwright" / session / "state.json"
)
)
browser_config_path = (
Path(args.browser_config_path).expanduser().resolve()
if args.browser_config_path.strip()
else None
)
ensure_browser_session(session, cwd=vip_workdir)
ensure_browser_session(session, cwd=vip_workdir, browser_config_path=browser_config_path)
load_state_if_present(session, state_path, cwd=vip_workdir)
base_url = config["tableau"]["base_url"].rstrip("/")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment