wangyunlong/gyxx-flow
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: [workflow] 提交工作区代码

Browse Source
This commit is contained in:
wangyunlong committed 2026-09-30 10:46:15 +08:00
1 parent 2ff588a65d
commit 01f3d7aaf7
131 files changed
+24306 -1320
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/gyxx_flow/console.py
+312 -35
View File
@@ -115,6 +115,7 @@ _CONTENT_POSTGRES_WORKFLOW_IDS = frozenset(
{
"content.notes_master.daily",
"content.metrics.daily",
"content.metrics.self_operated.daily",
"content.metrics.backfill",
"content.marketing_report.daily",
"content.creator_report.monthly",
@@ -167,7 +168,8 @@ _MODULES = (
_WORKFLOW_NAMES = {
"content.notes_master.daily": "笔记清单与合作同步",
"content.metrics.daily": "内容指标日报采集",
"content.metrics.daily": "合作达人内容指标日报采集",
"content.metrics.self_operated.daily": "自营内容指标日报采集",
"content.marketing_report.daily": "营销日报生成与发送",
"content.relogin.weekly": "内容平台登录态刷新",
"content.creator_report.monthly": "达人月报生成",
@@ -193,6 +195,14 @@ _WORKFLOW_NAMES = {
"shop.metrics.weekly": "店铺经营指标周采集",
"shop.competitor.weekly": "竞店指标周采集",
"shop.jd_self_operated.daily": "京东自营业绩日采集",
"shop.jd_customer_service.daily": "京东客服消息采集",
"shop.jd_negative_reviews.daily": "京东差评采集",
"shop.dy_negative_reviews.daily": "抖音差评采集",
"shop.jd_after_sales_issues.daily": "京东品退问题采集",
"shop.douyin_customer_service.daily": "抖音客服消息采集",
"shop.tmall_customer_service.daily": "天猫客服消息采集",
"shop.tmall_negative_reviews.daily": "天猫差评采集",
"shop.douyin_after_sales_quality.daily": "抖音品退采集",
"shop.douyin_price_appeal": "抖音待改价申诉",
"shop.jd_self_operated.history": "京东自营业绩单日回采",
"supply.purchase_confirmation.daily": "采购确认日处理",
@@ -201,6 +211,69 @@ _WORKFLOW_NAMES = {
"supply.purchase_order_update": "采购单更新",
}
JD_CUSTOMER_SERVICE_WORKFLOW_ID = "shop.jd_customer_service.daily"
JD_NEGATIVE_REVIEWS_WORKFLOW_ID = "shop.jd_negative_reviews.daily"
JD_AFTER_SALES_ISSUES_WORKFLOW_ID = "shop.jd_after_sales_issues.daily"
DOUYIN_CUSTOMER_SERVICE_WORKFLOW_ID = "shop.douyin_customer_service.daily"
TMALL_CUSTOMER_SERVICE_WORKFLOW_ID = "shop.tmall_customer_service.daily"
TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID = "shop.tmall_negative_reviews.daily"
DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID = "shop.dy_negative_reviews.daily"
DOUYIN_AFTER_SALES_QUALITY_WORKFLOW_ID = "shop.douyin_after_sales_quality.daily"
JD_DATE_RANGE_WORKFLOW_IDS = frozenset(
{
JD_CUSTOMER_SERVICE_WORKFLOW_ID,
JD_NEGATIVE_REVIEWS_WORKFLOW_ID,
JD_AFTER_SALES_ISSUES_WORKFLOW_ID,
}
)
# 支持控制台自定义开始/结束日期的工作流；京东系在此之上额外提供店铺选择。
DATE_RANGE_WORKFLOW_IDS = JD_DATE_RANGE_WORKFLOW_IDS | {
DOUYIN_CUSTOMER_SERVICE_WORKFLOW_ID,
TMALL_CUSTOMER_SERVICE_WORKFLOW_ID,
TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID,
DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID,
DOUYIN_AFTER_SALES_QUALITY_WORKFLOW_ID,
}
JD_CUSTOMER_SERVICE_STORE_OPTIONS = (
{
"value": "flagship",
"label": "光影行星旗舰店",
"account_id": "jd-shop",
},
{
"value": "self-operated",
"label": "京东自营店（光影自营研发）",
"account_id": "jd-self-operated",
},
)
JD_CUSTOMER_SERVICE_STORE_VALUES = frozenset(
item["value"] for item in JD_CUSTOMER_SERVICE_STORE_OPTIONS
)
JD_AFTER_SALES_STORE_OPTIONS = (
{
"value": "flagship",
"label": "光影行星旗舰店（研发账号）",
"account_id": "jd-shop",
},
)
def _jd_store_options(workflow_id: str) -> tuple[dict[str, str], ...]:
if workflow_id == JD_AFTER_SALES_ISSUES_WORKFLOW_ID:
return JD_AFTER_SALES_STORE_OPTIONS
return JD_CUSTOMER_SERVICE_STORE_OPTIONS
def _tmall_customer_service_date_range_constraints(
today: date,
) -> dict[str, str]:
month_index = today.year * 12 + today.month - 1 - 3
year, zero_based_month = divmod(month_index, 12)
month = zero_based_month + 1
oldest_day = min(today.day, calendar.monthrange(year, month)[1])
oldest = date(year, month, oldest_day)
return {"min": oldest.isoformat(), "max": today.isoformat()}
_SECRET_VALUE = re.compile(
r"(?i)([\"']?(?:api[_-]?key|access[_-]?token|token|password|secret)"
r"[\"']?\s*[=:]\s*)"
@@ -250,6 +323,8 @@ class RunLauncher(Protocol):
tmall_topic_keyword: str | None = None,
force_refresh: bool = False,
tmall_baibu_import_mode: str | None = None,
jd_customer_service_store: str | None = None,
business_date_end: date | None = None,
) -> dict[str, object]: ...
def active(self) -> tuple[dict[str, object], ...]: ...
@@ -318,6 +393,7 @@ class _ActiveConsoleRun:
operation_id: str
workflow_id: str
business_date: str
business_date_end: str | None
mode: str
shadow: bool
started_at: str
@@ -375,6 +451,8 @@ class SubprocessConsoleRunLauncher:
"status": "completed",
"exit_code": exit_code,
}
if active.business_date_end is not None:
result["business_date_end"] = active.business_date_end
self._remember_finished_locked(result)
return result
@@ -399,6 +477,7 @@ class SubprocessConsoleRunLauncher:
"started_at": item.started_at,
"status": "running",
}
| ({"business_date_end": item.business_date_end} if item.business_date_end else {})
for item in self._runs.values()
)
@@ -413,6 +492,8 @@ class SubprocessConsoleRunLauncher:
tmall_topic_keyword: str | None = None,
force_refresh: bool = False,
tmall_baibu_import_mode: str | None = None,
jd_customer_service_store: str | None = None,
business_date_end: date | None = None,
) -> dict[str, object]:
with self._lock:
self._reap_locked()
@@ -423,19 +504,52 @@ class SubprocessConsoleRunLauncher:
operation_id = f"op-{uuid.uuid4().hex[:16]}"
started_at = datetime.now(timezone.utc).isoformat(timespec="microseconds")
known_run_ids = self._journal_run_ids(workflow_id, business_date)
range_end = business_date_end or business_date
if range_end < business_date:
raise ConsoleRequestError("结束日期不能早于开始日期")
known_run_ids = self._journal_run_ids_range(
workflow_id, business_date, range_end
)
self.log_root.mkdir(parents=True, exist_ok=True)
log_path = self.log_root / f"{workflow_id}-{operation_id}.log"
stream = log_path.open("a", encoding="utf-8", newline="\n")
argv = [
str(self.python_executable),
"-m",
"gyxx_flow",
"run",
workflow_id,
"--date",
business_date.isoformat(),
]
if range_end != business_date and workflow_id in {
DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID,
TMALL_CUSTOMER_SERVICE_WORKFLOW_ID,
TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID,
}:
# 这些后台都支持按日期区间直接查询，无需逐日启动。
argv = [
str(self.python_executable),
"-m",
"gyxx_flow",
"run",
workflow_id,
"--date",
business_date.isoformat(),
]
elif range_end != business_date:
argv = [
str(self.python_executable),
"-m",
"gyxx_flow",
"backfill",
workflow_id,
"--from",
business_date.isoformat(),
"--to",
range_end.isoformat(),
]
else:
argv = [
str(self.python_executable),
"-m",
"gyxx_flow",
"run",
workflow_id,
"--date",
business_date.isoformat(),
]
if execute:
argv.append("--execute")
if shadow:
@@ -455,10 +569,34 @@ class SubprocessConsoleRunLauncher:
},
workflow_id=workflow_id,
)
if (
workflow_id == DOUYIN_NEGATIVE_REVIEWS_WORKFLOW_ID
and range_end != business_date
):
environment["GYXX_NEG_REVIEW_FROM"] = business_date.isoformat()
environment["GYXX_NEG_REVIEW_TO"] = range_end.isoformat()
if workflow_id == TMALL_CUSTOMER_SERVICE_WORKFLOW_ID:
environment["GYXX_TMALL_CUSTOMER_SERVICE_START_DATE"] = (
business_date.isoformat()
)
environment["GYXX_TMALL_CUSTOMER_SERVICE_END_DATE"] = (
range_end.isoformat()
)
if workflow_id == TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID:
environment["GYXX_TMALL_NEGATIVE_REVIEWS_START_DATE"] = (
business_date.isoformat()
)
environment["GYXX_TMALL_NEGATIVE_REVIEWS_END_DATE"] = (
range_end.isoformat()
)
if video_upload_store is not None:
environment["GYXX_VIDEO_UPLOAD_STORE"] = video_upload_store
if tmall_topic_keyword is not None:
environment["GYXX_TMALL_TOPIC_KEYWORD"] = tmall_topic_keyword
if jd_customer_service_store is not None:
environment["GYXX_JD_CUSTOMER_SERVICE_STORE"] = (
jd_customer_service_store
)
if force_refresh:
environment["GYXX_FORCE_REFRESH"] = "true"
else:
@@ -496,6 +634,9 @@ class SubprocessConsoleRunLauncher:
operation_id=operation_id,
workflow_id=workflow_id,
business_date=business_date.isoformat(),
business_date_end=(
range_end.isoformat() if range_end != business_date else None
),
mode="execute" if execute else "dry_run",
shadow=shadow,
started_at=started_at,
@@ -515,6 +656,8 @@ class SubprocessConsoleRunLauncher:
"started_at": started_at,
"status": "accepted",
}
if active.business_date_end is not None:
result["business_date_end"] = active.business_date_end
if video_upload_store is not None:
result["store"] = video_upload_store
if tmall_topic_keyword is not None:
@@ -523,6 +666,8 @@ class SubprocessConsoleRunLauncher:
result["force_refresh"] = True
if tmall_baibu_import_mode is not None:
result["tmall_baibu_import_mode"] = tmall_baibu_import_mode
if jd_customer_service_store is not None:
result["store"] = jd_customer_service_store
return result
def cancel(
@@ -572,6 +717,8 @@ class SubprocessConsoleRunLauncher:
"already_cancelled": False,
"journal_status": journal_status,
}
if active.business_date_end is not None:
result["business_date_end"] = active.business_date_end
if run_id is not None:
result["run_id"] = run_id
self._write_cancellation_record(result)
@@ -616,6 +763,8 @@ class SubprocessConsoleRunLauncher:
"log_path": log_path.relative_to(self.settings.data_root).as_posix(),
"known_run_ids": sorted(active.known_run_ids),
}
if active.business_date_end is not None:
payload["business_date_end"] = active.business_date_end
try:
atomic_write_json(self._active_path(active.operation_id), payload)
except OSError:
@@ -645,6 +794,7 @@ class SubprocessConsoleRunLauncher:
operation_id = payload["operation_id"]
workflow_id = payload["workflow_id"]
business_date = payload["business_date"]
business_date_end = payload.get("business_date_end")
mode = payload["mode"]
shadow = payload["shadow"]
started_at = payload["started_at"]
@@ -663,6 +813,10 @@ class SubprocessConsoleRunLauncher:
parsed_date = date.fromisoformat(business_date)
if parsed_date.isoformat() != business_date:
return None
if business_date_end is not None:
parsed_end = date.fromisoformat(business_date_end)
if parsed_end.isoformat() != business_date_end or parsed_end < parsed_date:
return None
if mode not in {"execute", "dry_run"} or not isinstance(shadow, bool):
return None
if not isinstance(started_at, str):
@@ -695,6 +849,7 @@ class SubprocessConsoleRunLauncher:
operation_id=operation_id,
workflow_id=workflow_id,
business_date=business_date,
business_date_end=business_date_end,
mode=mode,
shadow=shadow,
started_at=started_at,
@@ -734,6 +889,19 @@ class SubprocessConsoleRunLauncher:
root = self._journal_run_root(workflow_id, business_date.isoformat())
return frozenset(path.parent.name for path in root.glob("*/run.json"))
def _journal_run_ids_range(
self,
workflow_id: str,
start_date: date,
end_date: date,
) -> frozenset[str]:
run_ids: set[str] = set()
current = start_date
while current <= end_date:
run_ids.update(self._journal_run_ids(workflow_id, current))
current += timedelta(days=1)
return frozenset(run_ids)
def _locked_run_id(self, active: _ActiveConsoleRun) -> str | None:
lock_root = self.settings.data_root / "state" / "locks"
for path in lock_root.glob("*.lock"):
@@ -762,24 +930,29 @@ class SubprocessConsoleRunLauncher:
return None
def _candidate_journals(self, active: _ActiveConsoleRun) -> list[Path]:
root = self._journal_run_root(active.workflow_id, active.business_date)
candidates: list[Path] = []
for path in root.glob("*/run.json"):
if path.parent.name in active.known_run_ids:
continue
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError):
continue
if not isinstance(payload, dict):
continue
if (
payload.get("workflow_id") == active.workflow_id
and payload.get("business_date") == active.business_date
and payload.get("shadow") is active.shadow
and payload.get("status") == "running"
):
candidates.append(path)
start_date = date.fromisoformat(active.business_date)
end_date = date.fromisoformat(active.business_date_end or active.business_date)
current = start_date
while current <= end_date:
root = self._journal_run_root(active.workflow_id, current.isoformat())
for path in root.glob("*/run.json"):
if path.parent.name in active.known_run_ids:
continue
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError):
continue
if not isinstance(payload, dict):
continue
if (
payload.get("workflow_id") == active.workflow_id
and payload.get("business_date") == current.isoformat()
and payload.get("shadow") is active.shadow
and payload.get("status") == "running"
):
candidates.append(path)
current += timedelta(days=1)
return candidates
def _finalize_cancelled_journal(
@@ -790,13 +963,19 @@ class SubprocessConsoleRunLauncher:
) -> tuple[str, str | None]:
path: Path | None = None
if run_id is not None:
candidate = (
self._journal_run_root(active.workflow_id, active.business_date)
/ run_id
/ "run.json"
)
if candidate.exists():
path = candidate
start_date = date.fromisoformat(active.business_date)
end_date = date.fromisoformat(active.business_date_end or active.business_date)
current = start_date
while current <= end_date:
candidate = (
self._journal_run_root(active.workflow_id, current.isoformat())
/ run_id
/ "run.json"
)
if candidate.exists():
path = candidate
break
current += timedelta(days=1)
if path is None:
candidates = self._candidate_journals(active)
if len(candidates) == 1:
@@ -1389,6 +1568,36 @@ class WorkflowConsoleService:
if entry.workflow_id == "product.sales_sheet.daily"
else "date"
),
"date_range_mode": (
"date_range"
if entry.workflow_id in DATE_RANGE_WORKFLOW_IDS
else "single"
),
**(
{
"date_range_constraints": (
_tmall_customer_service_date_range_constraints(
now.date()
)
),
"date_range_hint": (
"仅支持最近三个月；系统按每次最多 31 天拆分查询，"
"聊天记录可能延迟一天。"
),
}
if entry.workflow_id == TMALL_CUSTOMER_SERVICE_WORKFLOW_ID
else {}
),
**(
{
"date_range_hint": (
"请指定评价时间范围；页面不选时间会查询所有日期，"
"工作流会固定选择负面评价后再搜索。"
)
}
if entry.workflow_id == TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID
else {}
),
"registered": entry.workflow_id in registered_ids,
"note": entry.note,
"source_task_name": entry.source_task_name,
@@ -1405,6 +1614,23 @@ class WorkflowConsoleService:
if entry.topic_config is not None
else {}
),
**(
{
"store_config": {
"label": "京东店铺/账号",
"hint": (
"选择账号对应的店铺；支持直接提交自定义日期范围，"
"相同店铺、业务日和数据身份重复执行会幂等更新。"
),
"options": [
dict(item)
for item in _jd_store_options(entry.workflow_id)
],
}
}
if entry.workflow_id in JD_DATE_RANGE_WORKFLOW_IDS
else {}
),
"schedule": _schedule_payload(schedule),
"next_run_at": _next_run_at(schedule, now),
"last_run": _run_payload(
@@ -1959,6 +2185,10 @@ class WorkflowConsoleService:
if workflow_id == "product.video_upload":
allowed.add("store")
allowed.add("tmall_topic_keyword")
if workflow_id in JD_DATE_RANGE_WORKFLOW_IDS:
allowed.add("store")
if workflow_id in DATE_RANGE_WORKFLOW_IDS:
allowed.add("business_date_end")
if workflow_id == TMALL_BAIBU_WORKFLOW_ID:
allowed.add("tmall_baibu_import_mode")
unknown = set(payload) - allowed
@@ -1971,6 +2201,35 @@ class WorkflowConsoleService:
raise ConsoleRequestError("业务日期必须使用 YYYY-MM-DD 格式") from exc
if business_date.isoformat() != raw_date:
raise ConsoleRequestError("业务日期必须使用 YYYY-MM-DD 格式")
business_date_end: date | None = None
if workflow_id in DATE_RANGE_WORKFLOW_IDS:
raw_end = payload.get("business_date_end", raw_date)
try:
business_date_end = date.fromisoformat(raw_end)
except (TypeError, ValueError) as exc:
raise ConsoleRequestError("结束日期必须使用 YYYY-MM-DD 格式") from exc
if business_date_end.isoformat() != raw_end:
raise ConsoleRequestError("结束日期必须使用 YYYY-MM-DD 格式")
if business_date_end < business_date:
raise ConsoleRequestError("结束日期不能早于开始日期")
if workflow_id == TMALL_CUSTOMER_SERVICE_WORKFLOW_ID:
constraints = _tmall_customer_service_date_range_constraints(
datetime.now(ZoneInfo("Asia/Shanghai")).date()
)
oldest = date.fromisoformat(constraints["min"])
latest = date.fromisoformat(constraints["max"])
if business_date < oldest:
raise ConsoleRequestError(
f"天猫客服最早仅支持 {oldest.isoformat()} 起的数据"
)
if business_date_end > latest:
raise ConsoleRequestError(
f"天猫客服结束日期不能晚于 {latest.isoformat()}"
)
if workflow_id == TMALL_NEGATIVE_REVIEWS_WORKFLOW_ID:
latest = datetime.now(ZoneInfo("Asia/Shanghai")).date()
if business_date_end is not None and business_date_end > latest:
raise ConsoleRequestError("天猫差评结束日期不能晚于今天")
execute = payload.get("execute", False)
shadow = payload.get("shadow", False)
if not isinstance(execute, bool) or not isinstance(shadow, bool):
@@ -1984,6 +2243,7 @@ class WorkflowConsoleService:
raise ConsoleRequestError("正式执行前必须确认外部影响")
video_upload_store: str | None = None
tmall_topic_keyword: str | None = None
jd_customer_service_store: str | None = None
if workflow_id == "product.video_upload":
raw_store = payload.get("store", "flagship")
if raw_store not in {"flagship", "luggage"}:
@@ -1993,6 +2253,19 @@ class WorkflowConsoleService:
payload.get("tmall_topic_keyword"),
entry,
)
if workflow_id in JD_DATE_RANGE_WORKFLOW_IDS:
raw_store = payload.get("store", "flagship")
allowed_store_values = {
item["value"] for item in _jd_store_options(workflow_id)
}
if raw_store not in allowed_store_values:
message = (
"京东品退店铺参数无效"
if workflow_id == JD_AFTER_SALES_ISSUES_WORKFLOW_ID
else "京东客服店铺参数无效"
)
raise ConsoleRequestError(message)
jd_customer_service_store = str(raw_store)
tmall_baibu_import_mode: str | None = None
if workflow_id == TMALL_BAIBU_WORKFLOW_ID:
raw_import_mode = payload.get(
@@ -2030,6 +2303,10 @@ class WorkflowConsoleService:
}
if tmall_baibu_import_mode is not None:
launch_kwargs["tmall_baibu_import_mode"] = tmall_baibu_import_mode
if jd_customer_service_store is not None:
launch_kwargs["jd_customer_service_store"] = jd_customer_service_store
if business_date_end is not None and business_date_end != business_date:
launch_kwargs["business_date_end"] = business_date_end
return self.launcher.launch(
workflow_id,
business_date,