wangyunlong/gyxx-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete production workflow migration

Browse Source
This commit is contained in:
wangyunlong
2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/history/content-windows-launchers/weekly_comment_scrape.bat
+80
View File
@@ -0,0 +1,80 @@
@echo off
if not defined GYXX_PYTHON set "GYXX_PYTHON=python"
REM Weekly three-platform comment scraper wrapper.
REM Runs bilibili, xiaohongshu, douyin in parallel via PowerShell.
REM Register: schtasks /Create /SC WEEKLY /D SUN /TN YingxiaoYunying_WeeklyCommentScrape /TR %PROJECT_DIR%\data\tools\weekly_comment_scrape.bat /ST 12:00 /F
setlocal
for %%I in ("%~dp0..\..") do set "PROJECT_DIR=%%~fI"
if not defined GYXX_DATA_ROOT (
if not defined GYXX_PROJECT_ROOT (
echo ERROR: GYXX_DATA_ROOT or GYXX_PROJECT_ROOT must be defined. 1>&2
exit /b 3
)
set "GYXX_DATA_ROOT=%GYXX_PROJECT_ROOT%\var"
)
set "PYTHON_EXE=%GYXX_PYTHON%"
set "LOG_DIR=%GYXX_DATA_ROOT%\logs\content_marketing"
set "TMP_DIR=%GYXX_DATA_ROOT%\tmp\content_marketing"
if not exist "%LOG_DIR%" mkdir "%LOG_DIR%"
if not exist "%TMP_DIR%" mkdir "%TMP_DIR%"
for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /value') do set "NOW=%%I"
set "STAMP=%NOW:~0,4%-%NOW:~4,2%-%NOW:~6,2%_%NOW:~8,2%-%NOW:~10,2%-%NOW:~12,2%"
set "MASTER_LOG=%LOG_DIR%\weekly_%STAMP%.log"
echo === weekly scrape started %STAMP% > "%MASTER_LOG%"
set "DOUYIN_LOG=%LOG_DIR%\weekly_%STAMP%_douyin.log"
set "XHS_LOG=%LOG_DIR%\weekly_%STAMP%_xiaohongshu.log"
set "BILI_LOG=%LOG_DIR%\weekly_%STAMP%_bilibili.log"
echo [%STAMP%] launching douyin, xiaohongshu, bilibili in parallel >> "%MASTER_LOG%"
set "PS_SCRIPT=%GYXX_DATA_ROOT%\tmp\content_marketing\weekly_scrape_%STAMP%.ps1"
> "%PS_SCRIPT%" echo $ErrorActionPreference = 'Continue'
>> "%PS_SCRIPT%" echo $env:PYTHONIOENCODING = 'utf-8'
>> "%PS_SCRIPT%" echo $env:PYTHONUTF8 = '1'
>> "%PS_SCRIPT%" echo Set-Location -LiteralPath '%PROJECT_DIR%'
>> "%PS_SCRIPT%" echo $py = '%GYXX_PYTHON%'
>> "%PS_SCRIPT%" echo $stamp = '%STAMP%'
>> "%PS_SCRIPT%" echo $logDir = '%GYXX_DATA_ROOT%\logs\content_marketing'
>> "%PS_SCRIPT%" echo $jobs = @(
>> "%PS_SCRIPT%" echo [pscustomobject]@{ Name='bilibili'; Script='%PROJECT_DIR%\data\tools\batch_rescrape_bilibili.py'; Log=Join-Path $logDir ("weekly_${stamp}_bilibili.log") },
>> "%PS_SCRIPT%" echo [pscustomobject]@{ Name='xiaohongshu'; Script='%PROJECT_DIR%\data\tools\batch_rescrape_xiaohongshu.py'; Log=Join-Path $logDir ("weekly_${stamp}_xiaohongshu.log") },
>> "%PS_SCRIPT%" echo [pscustomobject]@{ Name='douyin'; Script='%PROJECT_DIR%\data\tools\batch_rescrape_douyin.py'; Log=Join-Path $logDir ("weekly_${stamp}_douyin.log") }
>> "%PS_SCRIPT%" echo )
>> "%PS_SCRIPT%" echo $started = @{}
>> "%PS_SCRIPT%" echo foreach ($job in $jobs) {
>> "%PS_SCRIPT%" echo $logPath = $job.Log
>> "%PS_SCRIPT%" echo $header = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === $($job.Name) launch ==="
>> "%PS_SCRIPT%" echo $cmd = "`"$py`" -X utf8 `"$($job.Script)`" >> `"$logPath`" 2>&1"
>> "%PS_SCRIPT%" echo Add-Content -LiteralPath $logPath -Value $header -Encoding utf8
>> "%PS_SCRIPT%" echo $proc = Start-Process -FilePath cmd.exe -ArgumentList '/c', "`"$cmd`"" -PassThru -WindowStyle Hidden
>> "%PS_SCRIPT%" echo if ($null -eq $proc) {
>> "%PS_SCRIPT%" echo Add-Content -LiteralPath $logPath -Value "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] !!! Start-Process returned null for $($job.Name) !!!" -Encoding utf8
>> "%PS_SCRIPT%" echo continue
>> "%PS_SCRIPT%" echo }
>> "%PS_SCRIPT%" echo $started[$job.Name] = @{ Proc=$proc; Log=$logPath }
>> "%PS_SCRIPT%" echo }
>> "%PS_SCRIPT%" echo $exitCodes = @{}
>> "%PS_SCRIPT%" echo foreach ($job in $jobs) {
>> "%PS_SCRIPT%" echo $entry = $started[$job.Name]
>> "%PS_SCRIPT%" echo $entry.Proc.WaitForExit()
>> "%PS_SCRIPT%" echo $rc = $entry.Proc.ExitCode
>> "%PS_SCRIPT%" echo $exitCodes[$job.Name] = $rc
>> "%PS_SCRIPT%" echo Add-Content -LiteralPath $entry.Log -Value "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] === $($job.Name) done, exit=$rc ===" -Encoding utf8
>> "%PS_SCRIPT%" echo }
>> "%PS_SCRIPT%" echo foreach ($k in $exitCodes.Keys) { Write-Output ("{0}={1}" -f $k, $exitCodes[$k]) }
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%PS_SCRIPT%" >> "%MASTER_LOG%" 2>&1
set "RC=%ERRORLEVEL%"
del "%PS_SCRIPT%" 2>nul
echo [%STAMP%] all three scrapers finished, ps exit=%RC% >> "%MASTER_LOG%"
echo === weekly scrape done %STAMP% >> "%MASTER_LOG%"
endlocal & exit /b %RC%