|
@@ -1,8 +1,14 @@
|
|
|
# -*- coding: utf-8 -*-
|
|
|
# @Time : 2024/6/11
|
|
|
# @Author : 魏志亮
|
|
|
+import datetime
|
|
|
import os
|
|
|
import sys
|
|
|
+import traceback
|
|
|
+
|
|
|
+import pandas as pd
|
|
|
+
|
|
|
+from utils.conf.read_conf import yaml_conf, read_conf
|
|
|
|
|
|
|
|
|
def get_exec_data(batch_no=None, read_type=None, run_count=1):
|
|
@@ -37,39 +43,52 @@ def run(data: dict = dict(), save_db=False):
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- env = None
|
|
|
+ conf_path = r'conf/etl_config.yaml'
|
|
|
if len(sys.argv) >= 2:
|
|
|
- env = sys.argv[1]
|
|
|
- else:
|
|
|
- env = 'prod'
|
|
|
- print(sys.argv)
|
|
|
- if env is None:
|
|
|
- raise Exception("请配置运行环境")
|
|
|
-
|
|
|
- os.environ['env'] = env
|
|
|
-
|
|
|
- run_count = 1
|
|
|
- if len(sys.argv) >= 3:
|
|
|
- run_count = int(sys.argv[2])
|
|
|
-
|
|
|
- conf_path = '/data/config/etl_config.yaml'
|
|
|
- if len(sys.argv) >= 4:
|
|
|
- conf_path = sys.argv[3]
|
|
|
+ conf_path = sys.argv[1]
|
|
|
|
|
|
+ begin = datetime.datetime.now()
|
|
|
+ print("开始执行:", begin)
|
|
|
os.environ['ETL_CONF'] = conf_path
|
|
|
+ yaml_config = yaml_conf(conf_path)
|
|
|
+ env = read_conf(yaml_config, "env", "dev")
|
|
|
+ os.environ['env'] = env
|
|
|
+ run_count = int(read_conf(yaml_config, "run_batch_count", 1))
|
|
|
|
|
|
from utils.log.trans_log import trans_print
|
|
|
from service.plt_service import get_batch_exec_data, get_data_by_batch_no_and_type
|
|
|
from etl.wind_power.fault_warn.FaultWarnTrans import FaultWarnTrans
|
|
|
from etl.wind_power.min_sec.MinSecTrans import MinSecTrans
|
|
|
+ from utils.file.trans_methods import read_file_to_df
|
|
|
|
|
|
+ df = read_file_to_df("tmp_file/rebuild_data.csv")
|
|
|
+ results = list()
|
|
|
data = dict()
|
|
|
- data['batch_code'] = "test"
|
|
|
- data['batch_name'] = "test"
|
|
|
- data['transfer_type'] = "fault"
|
|
|
- # data['transfer_addr'] = r"D:\报警\唐珍风电2023年报警信息.xlsx"
|
|
|
- data['transfer_addr'] = r"D:\故障\故障数据\故障记录_20230101_20240101.csv"
|
|
|
- data['field_code'] = "test"
|
|
|
- data['field_name'] = "唐珍风电故障"
|
|
|
-
|
|
|
- run(data=data, save_db=False)
|
|
|
+ for batch_code, batch_name, transfer_type, transfer_addr, field_code, field_name \
|
|
|
+ in zip(df['batch_code'], df['batch_name'], df['transfer_type'], df['transfer_addr'], df['field_code'],
|
|
|
+ df['field_name']):
|
|
|
+ batch_begin = datetime.datetime.now()
|
|
|
+ transfer_addr = transfer_addr.replace(r"/data/download/collection_data",
|
|
|
+ r"/data/download/datang_shangxian")
|
|
|
+ trans_print("开始执行批次:", batch_code, batch_name, transfer_type, field_code, field_name)
|
|
|
+ trans_print("批次路径:", transfer_addr)
|
|
|
+
|
|
|
+ data['batch_code'] = batch_code
|
|
|
+ data['batch_name'] = batch_name
|
|
|
+ data['transfer_type'] = transfer_type
|
|
|
+ data['transfer_addr'] = transfer_addr
|
|
|
+ data['field_code'] = field_code
|
|
|
+ data['field_name'] = field_name
|
|
|
+ try:
|
|
|
+ run(data=data, save_db=True)
|
|
|
+ results.append((batch_code, batch_name, transfer_type, field_code, field_name, 'success'))
|
|
|
+ except Exception as e:
|
|
|
+ results.append((batch_code, batch_name, transfer_type, field_code, field_name, 'error'))
|
|
|
+ trans_print(traceback.format_exc())
|
|
|
+ finally:
|
|
|
+ trans_print("执行结束,耗时:", datetime.datetime.now() - batch_begin, "总耗时:", datetime.datetime.now() - begin)
|
|
|
+
|
|
|
+ for data in results:
|
|
|
+ trans_print(data)
|
|
|
+
|
|
|
+ trans_print("执行结束,总耗时:", datetime.datetime.now() - begin)
|