param(
|
[string]$StartDate = "2026-04-23",
|
[string]$EndDate = "2026-06-17",
|
[string]$LogDir = "D:\manage_system\data\result\stat\ds_minute_backfill_20260619"
|
)
|
|
$ErrorActionPreference = "Stop"
|
$root = Split-Path -Parent $PSScriptRoot
|
$script = Join-Path $root "scripts\kline_front_incremental_sync.py"
|
$localDoc = Get-ChildItem -LiteralPath (Join-Path $root "data") -Filter "*.local.md" |
|
Select-String -Pattern "mysql://root:" -List |
|
Select-Object -First 1 -ExpandProperty Path
|
$sourceDoc = Get-ChildItem -LiteralPath (Join-Path $root "data") -Filter "*.local.md" |
|
Select-String -Pattern "trading_xuntou" -List |
|
Select-Object -First 1 -ExpandProperty Path
|
|
New-Item -ItemType Directory -Force -Path $LogDir | Out-Null
|
|
$env:TIANXIA_MYSQL_PASSWORD = (Select-String -LiteralPath $localDoc -Pattern 'mysql://root:([^@]+)@' | Select-Object -First 1).Matches[0].Groups[1].Value
|
$env:KLINE_SOURCE_MYSQL_PASSWORD = (Select-String -LiteralPath $sourceDoc -Pattern 'Password.*`([^`]+)`' | Select-Object -First 1).Matches[0].Groups[1].Value
|
|
$datesPath = Join-Path $LogDir "dates.txt"
|
@"
|
import os, pymysql
|
source = pymysql.connect(host='317w7246e5.vicp.fun', port=50176, user='root', password=os.environ['KLINE_SOURCE_MYSQL_PASSWORD'], database='trading_xuntou', charset='utf8mb4', connect_timeout=10, read_timeout=120)
|
local = pymysql.connect(host='127.0.0.1', port=3306, user='root', password=os.environ['TIANXIA_MYSQL_PASSWORD'], database='tianxia', charset='utf8mb4', connect_timeout=10, read_timeout=120)
|
with source.cursor() as cur:
|
cur.execute("SELECT trade_date FROM cn_stock_kline_front_coverage_daily WHERE period='1m' AND trade_date BETWEEN %s AND %s AND row_count > 0 ORDER BY trade_date", ('$StartDate', '$EndDate'))
|
source_dates = [str(row[0]) for row in cur.fetchall()]
|
with local.cursor() as cur:
|
cur.execute("SELECT DISTINCT trade_date FROM a_share_minute_price WHERE trade_date BETWEEN %s AND %s", ('$StartDate', '$EndDate'))
|
done_dates = {str(row[0]) for row in cur.fetchall()}
|
for d in source_dates:
|
if d not in done_dates:
|
print(d)
|
source.close()
|
local.close()
|
"@ | python - | Set-Content -LiteralPath $datesPath -Encoding ASCII
|
|
$dates = Get-Content -LiteralPath $datesPath | Where-Object { $_ -match '^\d{4}-\d{2}-\d{2}$' }
|
$summaryPath = Join-Path $LogDir "summary.tsv"
|
"date`tstatus`trun_id`tstarted_at`tfinished_at" | Set-Content -LiteralPath $summaryPath -Encoding UTF8
|
|
foreach ($date in $dates) {
|
$runId = "ds_minute_sync_" + ($date -replace "-", "") + "_" + (Get-Date -Format "HHmmss")
|
$started = Get-Date -Format "s"
|
$out = Join-Path $LogDir "$runId.out.json"
|
$err = Join-Path $LogDir "$runId.err.txt"
|
try {
|
python $script --run-id $runId --periods 1m --start-date $date --end-date $date --minute-max-minutes-per-batch 120 --minute-symbols-per-batch 200 --append-to-main > $out 2> $err
|
$status = if ($LASTEXITCODE -eq 0) { "SUCCESS" } else { "FAILED_$LASTEXITCODE" }
|
} catch {
|
$_ | Out-File -LiteralPath $err -Append -Encoding UTF8
|
$status = "FAILED_EXCEPTION"
|
}
|
$finished = Get-Date -Format "s"
|
"$date`t$status`t$runId`t$started`t$finished" | Add-Content -LiteralPath $summaryPath -Encoding UTF8
|
if ($status -notlike "SUCCESS") {
|
break
|
}
|
}
|
