去掉配置文件,从数据库读取

添加定时任务

app.py

@@ -1,107 +1,39 @@
 # -*- coding: utf-8 -*-
-# @Time    : 2024/5/17
+# @Time    : 2024/6/6
 # @Author  : 魏志亮
+from apscheduler.executors.pool import ThreadPoolExecutor
+from flask import Flask
 import datetime
-
-import pandas as pd
-
-from base.TranseParam import TranseParam
-from base.WindFarms import WindFarms
-from utils.conf.read_conf import read_yaml_file, read_param_from_config
-from utils.db.trans_mysql import get_exec_data, get_trans_conf
-from utils.log.trans_log import trans_print, logger, init_log
-
-
-def init_params() -> list[WindFarms]:
-    df = get_exec_data()
-    print(df)
-    exec_subjects = list()
-
-    path = r'D:\transdata\test\虹梯官风电场-山西-大唐\收资数据'
-    changshang_name = '虹梯官风电场'
-    # read_type = 'minute'
-    read_type = 'second'
-    batch_no = 'hongtiguan_' + str(datetime.datetime.now().strftime('%Y%m%d'))
-
-    df = pd.DataFrame(columns=['batch_no', 'transfer_type', 'transfer_file_addr', 'field_code', 'field_name'],
-                      data=[
-                          [batch_no, read_type, path, 'A', changshang_name]])
-
-    if df is None:
-        trans_print("当前任务正在执行")
-
-    if df.empty:
-        trans_print("当前没有需要执行的任务")
-    else:
-        for batch_no, transfer_type, transfer_file_addr, field_code, field_name in zip(df['batch_no'],
-                                                                                       df['transfer_type'],
-                                                                                       df['transfer_file_addr'],
-                                                                                       df['field_code'],
-                                                                                       df['field_name']):
-            init_log(batch_no, field_name, transfer_type)
-
-            conf_df = get_trans_conf(field_name, transfer_type)
-            if conf_df is None:
-                trans_print(f"未找到{field_name}的{transfer_type}配置")
-                continue
-
-            conf = conf_df.iloc[0].to_dict()
-
-            resolve_col_prefix = read_param_from_config(conf, 'resolve_col_prefix')
-            wind_name_exec = read_param_from_config(conf, 'wind_name_exec', None)
-            wind_full_name = read_param_from_config(conf, 'wind_full_name')
-            is_vertical_table = read_param_from_config(conf, 'is_vertical_table', False)
-            merge_columns = read_param_from_config(conf, 'merge_columns', False)
-
-            vertical_cols = read_param_from_config(conf, 'vertical_read_cols', '').split(',')
-            index_cols = read_param_from_config(conf, 'vertical_index_cols', '').split(',')
-            vertical_key = read_param_from_config(conf, 'vertical_col_key')
-            vertical_value = read_param_from_config(conf, 'vertical_col_value')
-            need_valid_cols = not merge_columns
-
-            cols_trans_all = dict()
-            trans_cols = ['wind_turbine_number', 'time_stamp', 'active_power', 'rotor_speed', 'generator_speed',
-                          'wind_velocity', 'pitch_angle_blade_1', 'pitch_angle_blade_2', 'pitch_angle_blade_3',
-                          'cabin_position', 'true_wind_direction', 'yaw_error1', 'set_value_of_active_power',
-                          'gearbox_oil_temperature', 'generatordrive_end_bearing_temperature',
-                          'generatornon_drive_end_bearing_temperature', 'wind_turbine_status',
-                          'wind_turbine_status2',
-                          'cabin_temperature', 'twisted_cable_angle', 'front_back_vibration_of_the_cabin',
-                          'side_to_side_vibration_of_the_cabin', 'actual_torque', 'given_torque',
-                          'clockwise_yaw_count',
-                          'counterclockwise_yaw_count', 'unusable', 'power_curve_available',
-                          'required_gearbox_speed',
-                          'inverter_speed_master_control', 'outside_cabin_temperature', 'main_bearing_temperature',
-                          'gearbox_high_speed_shaft_bearing_temperature',
-                          'gearboxmedium_speed_shaftbearing_temperature',
-                          'gearbox_low_speed_shaft_bearing_temperature', 'generator_winding1_temperature',
-                          'generator_winding2_temperature', 'generator_winding3_temperature',
-                          'turbulence_intensity', 'param1',
-                          'param2', 'param3', 'param4', 'param5', 'param6', 'param7', 'param8', 'param9', 'param10']
-
-            for col in trans_cols:
-                cols_trans_all[col] = read_param_from_config(conf, col, '')
-
-            trans_subject = WindFarms(field_name, batch_no=batch_no, field_code=field_code,
-                                      wind_full_name=wind_full_name)
-
-            params = TranseParam(read_type=transfer_type, read_path=transfer_file_addr,
-                                 cols_tran=cols_trans_all,
-                                 wind_name_exec=wind_name_exec, is_vertical_table=is_vertical_table,
-                                 vertical_cols=vertical_cols, vertical_key=vertical_key,
-                                 vertical_value=vertical_value, index_cols=index_cols, merge_columns=merge_columns,
-                                 resolve_col_prefix=resolve_col_prefix, need_valid_cols=need_valid_cols)
-
-            trans_subject.set_trans_param(params)
-            exec_subjects.append(trans_subject)
-    return exec_subjects
+from flask_apscheduler import APScheduler
+
+
+class Config(object):
+    JOBS = [
+        {
+            'id': 'job1',
+            'func': 'schedule_service:run_schedule',
+            'args': (0, 3),
+            'trigger': 'interval',
+            'seconds': 60
+        }
+    ]
+    SCHEDULER_EXECUTORS = {'default': ThreadPoolExecutor(6)}
+    # 调度器开关开启
+    SCHEDULER_API_ENABLED = True
+    # 设置容错时间为 2min
+    # coalesce积攒得任务跑几次，在时间允许得范围内 True:默认最后一次，False:在时间允许范围内全部提交
+    # max_instances 同时允许并发的最大并发量
+    # misfire_grace_time 如果重启任务在这个时间范围内，就能继续重启
+    SCHEDULER_JOB_DEFAULTS = {'coalesce': False, 'max_instances': 2, 'misfire_grace_time': 60}
+    # 配置时区
+    SCHEDULER_TIMEZONE = 'Asia/Shanghai'
 
 
 if __name__ == '__main__':
-    exec_subjects = init_params()
-    for exec_subject in exec_subjects:
-        try:
-            exec_subject.run()
-        except Exception as e:
-            print(e)
-            logger.exception(e)
+    app = Flask(__name__)
+    app.config.from_object(Config())
+
+    scheduler = APScheduler()
+    scheduler.init_app(app)
+    scheduler.start()
+    app.run()

conf/db.yaml

@@ -0,0 +1,34 @@
+plt_connect_pool_config:
+  charset: "utf8mb4"
+  mincached: 2 # 初始化时，连接池中至少创建空闲的链接，0表示不创建
+  maxcached: 5 # 连接池允许最大的连接数， 0和None表示不限制连接数
+  maxshared: 0 # 连接池最多可共享的连接数量，0和None表示全部共享。PS：pymysql不支持事务
+  maxconnections: 10 # 连接池最大并发连接数量
+  blocking: True # 连接池中没有可用连接后，是否阻塞等待
+  setsession: [ ] # 开始会话前执行的命令列表
+
+plt:
+  host: 192.168.50.233
+  port: 3306
+  user: admin
+  password: admin123456
+  database: energy
+
+
+trans_connect_pool_config:
+  charset: "utf8"
+  mincached: 1 # 初始化时，连接池中至少创建空闲的链接，0表示不创建
+  maxcached: 20 # 连接池允许最大的连接数， 0和None表示不限制连接数
+  maxshared: 0 # 连接池最多可共享的连接数量，0和None表示全部共享。PS：pymysql不支持事务
+  maxconnections: 10 # 连接池最大并发连接数量
+  blocking: True # 连接池中没有可用连接后，是否阻塞等待
+  setsession: [ ] # 开始会话前执行的命令列表
+
+
+trans:
+  host: 192.168.50.233
+  port: 3306
+  user: admin
+  password: admin123456
+  database: energy_data
+

config_files/吉山风电场/second.yaml

@@ -1,112 +0,0 @@
-# 没有验证 需要和 实施一起验证
-
-wind_full_name: 吉山风电场-江西-大唐
-#  是否是竖表
-is_vertical_table: False
-# 是否需要合并列,吉山风电场-重庆海装处理
-merge_columns: True
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 转化的类名是否需要处理 比如 重庆还装表头自带 风机名称
-trans_col_exec: column[column.find('_')+1:]
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号 文件夹中是风机编号,压缩包的话会多一层目录
-  # eg: /data/download/xx/风机001/cc.csv $folder[1]
-  # eg: /data/download/xx/风机001/unzip_files/cc.csv $folder[2]
-  # 如果不规则,则配置下方的 wind_name_exec
-  wind_turbine_number: $folder[1]
-  # 时间戳
-  time_stamp: 采样时间
-  # 有功功率
-  active_power: 发电机有功功率
-  # 风轮转速
-  rotor_speed: 风轮转速
-  # 发电机转速
-  generator_speed: 发电机转速
-  # 风速
-  wind_velocity: 瞬时风速
-  # 桨距角1
-  pitch_angle_blade_1: 1#叶片变桨角度
-  # 桨距角2
-  pitch_angle_blade_2: 2#叶片变桨角度
-  # 桨距角3
-  pitch_angle_blade_3: 3#叶片变桨角度
-  # 绝对风向
-  true_wind_direction: null
-  # 对风角度
-  yaw_error1: null
-  # 机舱位置
-  cabin_position: null
-  # 环境温度
-  outside_cabin_temperature: 环境温度
-  # 有功功率设定值
-  set_value_of_active_power: null
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油池温度
-  # 主轴承轴承温度
-  main_bearing_temperature: 主轴前轴承温度
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴前端温度
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: null
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机前轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机后轴承温度
-  # 发电机绕组1温度
-  generator_winding1_temperature: 发电机绕组v2温度
-  # 发电机绕组2温度
-  generator_winding2_temperature: 发电机绕组w1温度
-  # 发电机绕组3温度
-  generator_winding3_temperature: 发电机绕组u2温度
-  # 风机状态1
-  wind_turbine_status: null
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 机舱温度
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: null
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: null
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: null
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-#wind_name_exec:
-

config_files/和风元宝山/minute.yaml

@@ -1,105 +0,0 @@
-wind_full_name: 和风元宝山-河南-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  #风机编号
-  wind_turbine_number: 风机
-  #时间戳
-  time_stamp: 时间
-  #有功功率
-  active_power: 有功功率
-  #风轮转速
-  rotor_speed: 叶轮转速
-  #发电机转速
-  generator_speed: 发电机转速
-  #风速
-  wind_velocity: 环境风速
-  #桨距角1
-  pitch_angle_blade_1: 叶片1变桨角度
-  #桨距角2
-  pitch_angle_blade_2: 叶片2变桨角度
-  #桨距角3
-  pitch_angle_blade_3: 叶片3变桨角度
-  #机舱位置
-  cabin_position: 机舱位置
-  #绝对风向
-  true_wind_direction: 环境风向
-  #对风角度
-  yaw_error1: 对风角度
-  #有功功率设定值
-  set_value_of_active_power: 有功设定反馈值
-  #齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油温
-  #发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机驱动侧轴承温度
-  #发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机非驱动侧轴承温度
-  #风机状态1
-  wind_turbine_status: null
-  #风机状态2
-  wind_turbine_status2: null
-  #机舱内温度
-  cabin_temperature: 机舱温度
-  #扭缆角度
-  twisted_cable_angle: 扭缆角度
-  #机舱前后振动
-  front_back_vibration_of_the_cabin: X方向振动幅度
-  #机舱左右振动
-  side_to_side_vibration_of_the_cabin: Y方向振动幅度
-  #实际力矩
-  actual_torque: null
-  #给定力矩
-  given_torque: null
-  #顺时针偏航次数
-  clockwise_yaw_count: null
-  #逆时针偏航次数
-  counterclockwise_yaw_count: null
-  #不可利用
-  unusable: null
-  #功率曲线可用
-  power_curve_available: null
-  #齿轮箱转速
-  required_gearbox_speed: null
-  #变频器转速(主控)
-  inverter_speed_master_control: null
-  #环境温度
-  outside_cabin_temperature: 环境温度
-  #主轴承轴承温度
-  main_bearing_temperature: 主轴前轴承温度
-  #齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴承温度
-  #齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: 齿轮箱中速轴承温度
-  #齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 齿轮箱低速轴承温度
-  #发电机绕组1温度
-  generator_winding1_temperature: 发电机定子绕组温度U1
-  #发电机绕组2温度
-  generator_winding2_temperature: 发电机定子绕组温度V1
-  #发电机绕组3温度
-  generator_winding3_temperature: 发电机定子绕组温度W1
-  #湍流强度
-  turbulence_intensity: 湍流强度
-  #齿轮箱油压
-  param1: 齿轮箱油压
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "wind_name.replace('元宝山','').replace('号风机','')"
-
-

config_files/唐龙三期风电场/second.yaml

@@ -1,104 +0,0 @@
-wind_full_name: 唐龙三期风电场-安徽-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号
-  wind_turbine_number: 设备名
-  # 时间戳
-  time_stamp: 时间
-  # 有功功率
-  active_power: 有功功率
-  # 风轮转速
-  rotor_speed: 风轮转速
-  # 发电机转速
-  generator_speed: 发电机转速
-  # 风速
-  wind_velocity: 风速
-  # 桨距角1
-  pitch_angle_blade_1: 桨距角1
-  # 桨距角2
-  pitch_angle_blade_2: 桨距角2
-  # 桨距角3
-  pitch_angle_blade_3: 桨距角3
-  # 绝对风向
-  true_wind_direction: 风向
-  # 对风角度
-  yaw_error1: 60秒平均风向角
-  # 机舱位置
-  cabin_position: 机舱位置
-  # 环境温度
-  outside_cabin_temperature: 舱外温度
-  # 有功功率设定值
-  set_value_of_active_power: null
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油温
-  # 主轴承轴承温度
-  main_bearing_temperature: null
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴承温度
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 齿轮箱低速轴承温度
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机驱动端轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: null
-  # 发电机绕组1温度
-  generator_winding1_temperature: 发电机定子U温度
-  # 发电机绕组2温度
-  generator_winding2_temperature: 发电机定子V温度
-  # 发电机绕组3温度
-  generator_winding3_temperature: 发电机定子W温度
-  # 风机状态1
-  wind_turbine_status: 风机标准状态
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 舱内温度
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: 扭缆位置
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: 机舱振动传感器Y
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: 机舱振动传感器X
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "wind_name.replace('唐龙风场',').replace('号风机','')"
-
-

config_files/密马风电场/minute.yaml

@@ -1,103 +0,0 @@
-wind_full_name: 密马风电场-山西-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  #风机编号
-  wind_turbine_number: 风机
-  #时间戳
-  time_stamp: 时间
-  #有功功率
-  active_power: 平均发电机功率实时值(kW)
-  #风轮转速
-  rotor_speed: 平均主轴转速
-  #发电机转速
-  generator_speed: 平均发电机转速实时值
-  #风速
-  wind_velocity: 平均风速实时值(m/s)
-  #桨距角1
-  pitch_angle_blade_1: null
-  #桨距角2
-  pitch_angle_blade_2: 平均变桨角度
-  #桨距角3
-  pitch_angle_blade_3: null
-  #机舱位置
-  cabin_position: 平均机舱方向绝对角度
-  #绝对风向
-  true_wind_direction: 平均风向1秒平均值
-  #对风角度
-  yaw_error1: null
-  #有功功率设定值
-  set_value_of_active_power: null
-  #齿轮箱油温
-  gearbox_oil_temperature: 平均齿轮油温
-  #发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 平均发电机de端轴承温度
-  #发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 平均发电机nde端轴承温度
-  #风机状态1
-  wind_turbine_status: 平均机组运行模式
-  #风机状态2
-  wind_turbine_status2: 平均机组刹车程序
-  #机舱内温度
-  cabin_temperature: 平均机舱温度
-  #扭缆角度
-  twisted_cable_angle: null
-  #机舱前后振动
-  front_back_vibration_of_the_cabin: null
-  #机舱左右振动
-  side_to_side_vibration_of_the_cabin: null
-  #实际力矩
-  actual_torque: null
-  #给定力矩
-  given_torque: null
-  #顺时针偏航次数
-  clockwise_yaw_count: null
-  #逆时针偏航次数
-  counterclockwise_yaw_count: null
-  #不可利用
-  unusable: null
-  #功率曲线可用
-  power_curve_available: null
-  #齿轮箱转速
-  required_gearbox_speed: null
-  #变频器转速(主控)
-  inverter_speed_master_control: null
-  #环境温度
-  outside_cabin_temperature: 平均机舱外温度
-  #主轴承轴承温度
-  main_bearing_temperature: 平均转子轴承a温度
-  #齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 平均齿轮箱nde端轴承温度
-  #齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  #齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 平均齿轮箱de端轴承温度
-  #发电机绕组1温度
-  generator_winding1_temperature: 平均发电机定子绕组u温度
-  #发电机绕组2温度
-  generator_winding2_temperature: null
-  #发电机绕组3温度
-  generator_winding3_temperature: 平均发电机定子绕组v温度
-  #湍流强度
-  turbulence_intensity: 平均发电机定子绕组w温度
-
-# 自定义风机编号映射 必须是 wind_name
-#wind_name_exec: "wind_name.replace('号风机','')"
-

config_files/密马风电场/second.yaml

@@ -1,103 +0,0 @@
-wind_full_name: 密马风电场-山西-大唐
-#  是否是竖表
-is_vertical_table: True
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号
-  wind_turbine_number: 资产名称
-  # 时间戳
-  time_stamp: 数据时间
-  # 有功功率
-  active_power: 发电机有功功率
-  # 风轮转速
-  rotor_speed: null
-  # 发电机转速
-  generator_speed: null
-  # 风速
-  wind_velocity: 风速
-  # 桨距角1
-  pitch_angle_blade_1: 1号桨叶片角度(桨距角)-1分钟采样值
-  # 桨距角2
-  pitch_angle_blade_2: null
-  # 桨距角3
-  pitch_angle_blade_3: null
-  # 绝对风向
-  true_wind_direction: null
-  # 对风角度
-  yaw_error1: null
-  # 机舱位置
-  cabin_position: null
-  # 环境温度
-  outside_cabin_temperature: 舱外温度
-  # 有功功率设定值
-  set_value_of_active_power: null
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油池温度
-  # 主轴承轴承温度
-  main_bearing_temperature: null
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: null
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 齿轮箱高速轴非驱动端轴承温度
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机驱动端轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机非驱动端轴承温度
-  # 发电机绕组1温度
-  generator_winding1_temperature: null
-  # 发电机绕组2温度
-  generator_winding2_temperature: null
-  # 发电机绕组3温度
-  generator_winding3_temperature: null
-  # 风机状态1
-  wind_turbine_status: 厂家风机状态
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 舱内温度
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: null
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: null
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: null
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "wind_name.replace('号风机','')"
-

config_files/招远风电场/minute.yaml

@@ -1,120 +0,0 @@
-wind_full_name: 招远风电场-山东-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-
-trans_col_exec: column.replace('有功功率-平均','当前理论可发最大功率-平均')
-# 转化的类名是否需要处理 比如 重庆还装表头自带 风机名称
-resolve_col_prefix: column.replace('有功功率-平均','当前理论可发最大功率-平均')
-
-# 是否只读取需要的字段,默认True ,如果多文件夹列名不一致,则需要设置为False,不然验证通不过
-need_valid_cols: False
-
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号  $file[0:3]获取文件名的前3位
-  wind_turbine_number: 风机
-  # 时间戳
-  time_stamp: 时间
-  # 有功功率
-  active_power: 当前理论可发最大功率-平均
-  # 风轮转速
-  rotor_speed: 叶轮转速-平均
-  # 发电机转速
-  generator_speed: 发电机转速-平均
-  # 风速
-  wind_velocity: 实际风速-平均
-  # 桨距角1
-  pitch_angle_blade_1: 桨叶角度A-平均
-  # 桨距角2
-  pitch_angle_blade_2: 桨叶角度B-平均
-  # 桨距角3
-  pitch_angle_blade_3: 桨叶角度C-平均
-  # 绝对风向
-  true_wind_direction: 绝对风向-平均
-  # 对风角度
-  yaw_error1: null
-  # 机舱位置
-  cabin_position: 机舱位置-平均
-  # 环境温度
-  outside_cabin_temperature: 机舱外温度-平均
-  # 有功功率设定值
-  set_value_of_active_power: null
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油温度-平均
-  # 主轴承轴承温度   出现了俩  主轴轴承温度-平均、主轴轴承2(后轴承)温度-平均
-  main_bearing_temperature: 主轴轴承温度-平均
-  # 齿轮箱高速轴轴承温度   出现了俩  齿轮箱高速轴轴承(风轮侧)温度-平均、齿轮箱高速轴轴承(电机侧)温度-平均
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴轴承(风轮侧)温度-平均
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: null
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机前轴承温度-平均
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机后轴承温度-平均
-  # 发电机绕组1温度
-  generator_winding1_temperature: 发电机定子绕组温度-平均
-  # 发电机绕组2温度
-  generator_winding2_temperature: null
-  # 发电机绕组3温度
-  generator_winding3_temperature: null
-  # 风机状态1
-  wind_turbine_status: null
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 机舱内温度-平均
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: null
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: 塔筒前后振动-平均
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: 塔筒左右振动-平均
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 eg: excel名称: 系统编号
-#wind_name_exec:
-#  风机A001: A001
-#  风机A002: A002
-#  风机A003: A003
-#  风机A004: A004
-#  风机A005: A005
-#  风机A006: A006
-#  风机A007: A007
-#  风机A008: A008
-
-

config_files/招远风电场/second.yaml

@@ -1,100 +0,0 @@
-wind_full_name: 招远风电场-山东-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号  $file[0:3]获取文件名的前3位
-  wind_turbine_number: $file[0:3]
-  # 时间戳
-  time_stamp: Time
-  # 有功功率
-  active_power: 有功功率
-  # 风轮转速
-  rotor_speed: 叶轮转速
-  # 发电机转速
-  generator_speed: 发电机转速
-  # 风速
-  wind_velocity: 实际风速
-  # 桨距角1
-  pitch_angle_blade_1: 桨叶角度A
-  # 桨距角2
-  pitch_angle_blade_2: 桨叶角度B
-  # 桨距角3
-  pitch_angle_blade_3: 桨叶角度C
-  # 绝对风向
-  true_wind_direction: 绝对风向
-  # 对风角度
-  yaw_error1: 风向1s
-  # 机舱位置
-  cabin_position: 机舱位置
-  # 环境温度
-  outside_cabin_temperature: 机舱外温度
-  # 有功功率设定值
-  set_value_of_active_power: 有功设定反馈
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油温度
-  # 主轴承轴承温度
-  main_bearing_temperature: 主轴轴承温度
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴轴承(风轮侧)温度
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: null
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机前轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机后轴承温度
-  # 发电机绕组1温度
-  generator_winding1_temperature: 发电机定子绕组温度
-  # 发电机绕组2温度
-  generator_winding2_temperature: null
-  # 发电机绕组3温度
-  generator_winding3_temperature: null
-  # 风机状态1
-  wind_turbine_status: 设备主要状态
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 机舱内温度
-  # 湍流强度
-  turbulence_intensity: 湍流强度
-  # 扭缆角度
-  twisted_cable_angle: 电缆扭角
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: 塔筒前后振动
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: 塔筒左右振动
-  # 实际力矩
-  actual_torque: 实际力矩
-  # 给定力矩
-  given_torque: 给定力矩
-  # 顺时针偏航次数
-  clockwise_yaw_count: 顺时针偏航次数
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: 逆时针偏航次数
-  # 不可利用
-  unusable: 不可利用
-  # 功率曲线可用
-  power_curve_available: 功率曲线可用
-  # 齿轮箱转速
-  required_gearbox_speed: 齿轮箱转速
-  # 变频器转速(主控)
-  inverter_speed_master_control: 变频器转速(主控)
-

config_files/昌平坳风场/minute.yaml

@@ -1,109 +0,0 @@
-wind_full_name: 昌平坳风场-贵州-大唐
-#  是否是竖表
-is_vertical_table: False
-# 是否需要合并列,吉山风电场-重庆海装处理
-merge_columns: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 转化的类名是否需要处理 比如 重庆还装表头自带 风机名称
-#trans_col_exec: column[column.find('_')+1:]
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号 文件夹中是风机编号,压缩包的话会多一层目录
-  # eg: /data/download/xx/风机001/cc.csv $folder[1]
-  # eg: /data/download/xx/风机001/unzip_files/cc.csv $folder[2]
-  # 如果不规则,则配置下方的 wind_name_exec
-  wind_turbine_number: 风机
-  # 时间戳
-  time_stamp: 时间
-  # 有功功率
-  active_power: 平均功率
-  # 风轮转速
-  rotor_speed: null
-  # 发电机转速
-  generator_speed: 平均发电机转速
-  # 风速
-  wind_velocity: 平均风速(m/s)
-  # 桨距角1
-  pitch_angle_blade_1: 平均桨叶1角度
-  # 桨距角2
-  pitch_angle_blade_2: 平均桨叶2角度
-  # 桨距角3
-  pitch_angle_blade_3: 平均桨叶3角度
-  # 绝对风向
-  true_wind_direction: 平均风角度（对北）
-  # 对风角度
-  yaw_error1: 平均风向（机舱）
-  # 机舱位置
-  cabin_position: 平均机舱角度
-  # 环境温度
-  outside_cabin_temperature: null
-  # 有功功率设定值
-  set_value_of_active_power: null
-  # 齿轮箱油温
-  gearbox_oil_temperature: 平均齿轮箱油温
-  # 主轴承轴承温度
-  main_bearing_temperature: 平均主轴轴承a温度
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 平均齿轮箱非驱动端轴承温度
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 平均齿轮箱驱动端轴承温度
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 平均发电机驱动端轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 平均发电机非驱动端轴承温度
-  # 发电机绕组1温度
-  generator_winding1_temperature: 平均发电机u相温度
-  # 发电机绕组2温度
-  generator_winding2_temperature: 平均发电机v相温度
-  # 发电机绕组3温度
-  generator_winding3_temperature: 平均发电机w相温度
-  # 风机状态1
-  wind_turbine_status: null
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 平均机舱温度
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: 平均扭缆角度
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: 最大x方向振动值
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: 最大y方向振动值
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: 平均转矩设定值
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "win_name.replace('#','')"

config_files/昌平坳风场/second.yaml

@@ -1,110 +0,0 @@
-wind_full_name: 昌平坳风场-贵州-大唐
-#  是否是竖表
-is_vertical_table: False
-# 是否需要合并列,吉山风电场-重庆海装处理
-merge_columns: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 转化的类名是否需要处理 比如 重庆还装表头自带 风机名称
-#trans_col_exec: column[column.find('_')+1:]
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号 文件夹中是风机编号,压缩包的话会多一层目录
-  # eg: /data/download/xx/风机001/cc.csv $folder[1]
-  # eg: /data/download/xx/风机001/unzip_files/cc.csv $folder[2]
-  # 如果不规则,则配置下方的 wind_name_exec
-  wind_turbine_number: 风机编号
-  # 时间戳
-  time_stamp: 时间
-  # 有功功率
-  active_power: iGenPower
-  # 风轮转速
-  rotor_speed: iRotorSpeedPDM
-  # 发电机转速
-  generator_speed: iGenSpeed
-  # 风速
-  wind_velocity: iWindSpeed_real
-  # 桨距角1
-  pitch_angle_blade_1: iPitchAngle1
-  # 桨距角2
-  pitch_angle_blade_2: iPitchAngle2
-  # 桨距角3
-  pitch_angle_blade_3: iPitchAngle3
-  # 绝对风向
-  true_wind_direction: iwindDirection
-  # 对风角度
-  yaw_error1: iVaneDiiection
-  # 机舱位置
-  cabin_position: iNacellePositionLtd
-  # 环境温度
-  outside_cabin_temperature: iTempOutdoor_1sec
-  # 有功功率设定值
-  set_value_of_active_power: iActivePoweiSetPointValue
-  # 齿轮箱油温
-  gearbox_oil_temperature: iTemp1GearOil_1sec
-  # 主轴承轴承温度
-  main_bearing_temperature: iTempRotorBearA_1sec
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: iTempGearBearNDE_1sec
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: iTempGearBearDE_1sec
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: iTempGenBearDE_1sec
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: iTempGenBearNDE_1sec
-  # 发电机绕组1温度
-  generator_winding1_temperature: iTempGenStatorU_1sec
-  # 发电机绕组2温度
-  generator_winding2_temperature: iTempGenStatorV_1sec
-  # 发电机绕组3温度
-  generator_winding3_temperature: iTempGenStatorW_1sec
-  # 风机状态1
-  wind_turbine_status: iTurbineOperationMode
-  # 风机状态2
-  wind_turbine_status2: iPowerLimit_Flag
-  # 机舱内温度
-  cabin_temperature: iTempNacelle_1sec
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: null
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: iVibrationZ
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: iVibrationY
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: iCAN_GeneratorTorqueSetPointValue
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-#wind_name_exec: "win_name.replace('20049','')"
-

config_files/昌西一风电场/minute.yaml

@@ -1,102 +0,0 @@
-wind_full_name: 昌西一风电场-甘肃-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  #风机编号
-  wind_turbine_number: $file
-  #时间戳
-  time_stamp: 系统时间
-  #有功功率
-  active_power: 有功功率[kw]
-  #风轮转速
-  rotor_speed: null
-  #发电机转速
-  generator_speed: 发电机平均转速[rpm]
-  #风速
-  wind_velocity: 1秒平均风速[m/s]
-  #桨距角1
-  pitch_angle_blade_1: 叶片1角度[°]
-  #桨距角2
-  pitch_angle_blade_2: 叶片2角度[°]
-  #桨距角3
-  pitch_angle_blade_3: 叶片3角度[°]
-  #机舱位置
-  cabin_position: 平均机舱位置[°]
-  #绝对风向
-  true_wind_direction: 风向[°]
-  #对风角度
-  yaw_error1: null
-  #有功功率设定值
-  set_value_of_active_power: null
-  #齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油温[℃]
-  #发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 驱动方向发电机轴承温度[℃]
-  #发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 非驱动方向发电机轴承温度[℃]
-  #风机状态1
-  wind_turbine_status: null
-  #风机状态2
-  wind_turbine_status2: null
-  #机舱内温度
-  cabin_temperature: 机舱内温度[℃]
-  #扭缆角度
-  twisted_cable_angle: null
-  #机舱前后振动
-  front_back_vibration_of_the_cabin: null
-  #机舱左右振动
-  side_to_side_vibration_of_the_cabin: null
-  #实际力矩
-  actual_torque: null
-  #给定力矩
-  given_torque: null
-  #顺时针偏航次数
-  clockwise_yaw_count: null
-  #逆时针偏航次数
-  counterclockwise_yaw_count: null
-  #不可利用
-  unusable: null
-  #功率曲线可用
-  power_curve_available: null
-  #齿轮箱转速
-  required_gearbox_speed: null
-  #变频器转速(主控)
-  inverter_speed_master_control: null
-  #环境温度
-  outside_cabin_temperature: 环境温度[℃]
-  #主轴承轴承温度
-  main_bearing_temperature: null
-  #齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱轴承温度[℃]
-  #齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  #齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: null
-  #发电机绕组1温度
-  generator_winding1_temperature: null
-  #发电机绕组2温度
-  generator_winding2_temperature: null
-  #发电机绕组3温度
-  generator_winding3_temperature: null
-  #湍流强度
-  turbulence_intensity: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "wind_name.replace('昌马','').replace('风机','')"

config_files/昌西一风电场/second.yaml

@@ -1,103 +0,0 @@
-wind_full_name: 昌西一风电场-甘肃-大唐
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号
-  wind_turbine_number: 风机
-  # 时间戳
-  time_stamp: 时间
-  # 有功功率
-  active_power: 有功功率
-  # 风轮转速
-  rotor_speed: 叶轮转速
-  # 发电机转速
-  generator_speed: 发电机转速
-  # 风速
-  wind_velocity: 环境风速
-  # 桨距角1
-  pitch_angle_blade_1: 叶片1变桨角度
-  # 桨距角2
-  pitch_angle_blade_2: 叶片2变桨角度
-  # 桨距角3
-  pitch_angle_blade_3: 叶片3变桨角度
-  # 绝对风向
-  true_wind_direction: 环境风向
-  # 对风角度
-  yaw_error1: 对风角度
-  # 机舱位置
-  cabin_position: 机舱位置
-  # 环境温度
-  outside_cabin_temperature: 环境温度
-  # 有功功率设定值
-  set_value_of_active_power: 有功设定反馈值
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油温
-  # 主轴承轴承温度
-  main_bearing_temperature: null
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴承温度
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: null
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机驱动侧轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机非驱动侧轴承温度
-  # 发电机绕组1温度
-  generator_winding1_temperature: 发电机1相温度
-  # 发电机绕组2温度
-  generator_winding2_temperature: 发电机2相温度
-  # 发电机绕组3温度
-  generator_winding3_temperature: 发电机3相温度
-  # 风机状态1
-  wind_turbine_status: 风机子状态
-  # 风机状态2
-  wind_turbine_status2: 转集团标准状态
-  # 机舱内温度
-  cabin_temperature: 机舱温度
-  # 湍流强度
-  turbulence_intensity: 湍流强度
-  # 扭缆角度
-  twisted_cable_angle: null
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: Y方向振动幅度
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: X方向振动幅度
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "wind_name.replace('昌马','').replace('风机','')"
-

config_files/虹梯官风电场/minute.yaml

@@ -1,103 +0,0 @@
-wind_full_name: 虹梯官风电场-山西-大唐
-#  是否是竖表
-is_vertical_table: True
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  #风机编号
-  wind_turbine_number: 资产名称
-  #时间戳
-  time_stamp: 时间
-  #有功功率
-  active_power: 电网有功功率
-  #风轮转速
-  rotor_speed: 主轴转速
-  #发电机转速
-  generator_speed: 发电机转速
-  #风速
-  wind_velocity: 风速
-  #桨距角1
-  pitch_angle_blade_1: 桨叶1电机位置
-  #桨距角2
-  pitch_angle_blade_2: 桨叶2电机位置
-  #桨距角3
-  pitch_angle_blade_3: 桨叶3电机位置
-  #机舱位置
-  cabin_position: 机舱偏北位置
-  #绝对风向
-  true_wind_direction: 30s绝对风向
-  #对风角度
-  yaw_error1: 30s机舱风向夹角
-  #有功功率设定值
-  set_value_of_active_power: 当前有功设定值
-  #齿轮箱油温
-  gearbox_oil_temperature: 齿轮油油温
-  #发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机驱动端温度
-  #发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机非驱动端温度
-  #风机状态1
-  wind_turbine_status: 风机主状态
-  #风机状态2
-  wind_turbine_status2: null
-  #机舱内温度
-  cabin_temperature: 机舱温度
-  #扭缆角度
-  twisted_cable_angle: null
-  #机舱前后振动
-  front_back_vibration_of_the_cabin: null
-  #机舱左右振动
-  side_to_side_vibration_of_the_cabin: null
-  #实际力矩
-  actual_torque: 变流器实际转矩(计算)
-  #给定力矩
-  given_torque: 主控转矩设定值
-  #顺时针偏航次数
-  clockwise_yaw_count: null
-  #逆时针偏航次数
-  counterclockwise_yaw_count: null
-  #不可利用
-  unusable: null
-  #功率曲线可用
-  power_curve_available: null
-  #齿轮箱转速
-  required_gearbox_speed: null
-  #变频器转速(主控)
-  inverter_speed_master_control: null
-  #环境温度
-  outside_cabin_temperature: 环境温度
-  #主轴承轴承温度
-  main_bearing_temperature: 主轴温度
-  #齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴温度
-  #齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  #齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 齿轮箱低速轴温度
-  #发电机绕组1温度
-  generator_winding1_temperature: 发电机线圈1温度
-  #发电机绕组2温度
-  generator_winding2_temperature: 发电机线圈2温度
-  #发电机绕组3温度
-  generator_winding3_temperature: 发电机线圈3温度
-  #湍流强度
-  turbulence_intensity: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "'C'+wind_name.replace('号风机','').zfill(3)"
-

config_files/虹梯官风电场/second.yaml

@@ -1,102 +0,0 @@
-wind_full_name: 虹梯官风电场-山西-大唐
-#  是否是竖表
-is_vertical_table: True
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号
-  wind_turbine_number: 资产名称
-  # 时间戳
-  time_stamp: 数据时间
-  # 有功功率
-  active_power: 发电机有功功率
-  # 风轮转速
-  rotor_speed: 风轮转速
-  # 发电机转速
-  generator_speed: 发电机转速
-  # 风速
-  wind_velocity: 风速
-  # 桨距角1
-  pitch_angle_blade_1: 桨叶片角度1
-  # 桨距角2
-  pitch_angle_blade_2: 桨叶片角度2
-  # 桨距角3
-  pitch_angle_blade_3: 桨叶片角度3
-  # 绝对风向
-  true_wind_direction: 风向
-  # 对风角度
-  yaw_error1: 机舱与风向夹角
-  # 机舱位置
-  cabin_position: 机舱角度(位置)
-  # 环境温度
-  outside_cabin_temperature: 舱外温度
-  # 有功功率设定值
-  set_value_of_active_power: null
-  # 齿轮箱油温
-  gearbox_oil_temperature: 齿轮箱油池温度
-  # 主轴承轴承温度
-  main_bearing_temperature: null
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 齿轮箱高速轴非驱动端轴承温度
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: null
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 发电机驱动端轴承温度
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 发电机非驱动端轴承温度
-  # 发电机绕组1温度
-  generator_winding1_temperature: null
-  # 发电机绕组2温度
-  generator_winding2_temperature: null
-  # 发电机绕组3温度
-  generator_winding3_temperature: null
-  # 风机状态1
-  wind_turbine_status: 厂家风机状态
-  # 风机状态2
-  wind_turbine_status2: null
-  # 机舱内温度
-  cabin_temperature: 舱内温度
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: null
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: null
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: null
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "'C'+wind_name.replace('号风机','').zfill(3)"

config_files/长清风电场/second.yaml

@@ -1,106 +0,0 @@
-wind_full_name: 长清风电场-山东-国电
-#  是否是竖表
-is_vertical_table: False
-#  如果是怎配置竖表需要查询的字段
-vertical_table_conf:
-  read_cols:
-    - 数据时间
-    - 资产名称
-    - 采集点名称
-    - 数值
-  index_cols:
-    - 数据时间
-    - 资产名称
-  col_key: 采集点名称
-  col_value: 数值
-#时间字段,转化年月日 以及格式化为 yyyy-MM-dd HH:mm:ss使用
-time_col: time_stamp
-#  风机编号字段
-wind_col: wind_turbine_number
-# 表数据和数据文件的映射关系
-trans_col:
-  # 风机编号
-  wind_turbine_number: 风机名称
-  # 时间戳
-  time_stamp: 时间
-  # 有功功率
-  active_power: 有功功率(kW)
-  # 风轮转速
-  rotor_speed: 风轮转速(rpm)
-  # 发电机转速
-  generator_speed: 发电机转速(rpm)
-  # 风速
-  wind_velocity: 30秒平均风速(m/s)
-  # 桨距角1
-  pitch_angle_blade_1: 桨距角1(°)
-  # 桨距角2
-  pitch_angle_blade_2: 桨距角2(°)
-  # 桨距角3
-  pitch_angle_blade_3: 桨距角3(°)
-  # 绝对风向
-  true_wind_direction: null
-  # 对风角度
-  yaw_error1: 风向角(°)
-  # 机舱位置
-  cabin_position: 机舱位置(°)
-  # 环境温度
-  outside_cabin_temperature: 舱外温度(℃)
-  # 有功功率设定值
-  set_value_of_active_power: 有功设定值
-  # 齿轮箱油温
-  gearbox_oil_temperature: null
-  # 主轴承轴承温度
-  main_bearing_temperature: null
-  # 齿轮箱高速轴轴承温度
-  gearbox_high_speed_shaft_bearing_temperature: 高速轴承温度(℃)
-  # 齿轮箱中速轴轴承温度
-  gearboxmedium_speed_shaftbearing_temperature: null
-  # 齿轮箱低速轴轴承温度
-  gearbox_low_speed_shaft_bearing_temperature: 低速轴承温度(℃)
-  # 发电机驱动端轴承温度
-  generatordrive_end_bearing_temperature: 驱动前轴承温度(℃)
-  # 发电机非驱动端轴承温度
-  generatornon_drive_end_bearing_temperature: 自由后轴承温度(℃)
-  # 发电机绕组1温度
-  generator_winding1_temperature: 发电机定子U温度(℃)
-  # 发电机绕组2温度
-  generator_winding2_temperature: 发电机定子V温度(℃)
-  # 发电机绕组3温度
-  generator_winding3_temperature: 发电机定子W温度(℃)
-  # 风机状态1
-  wind_turbine_status: null
-  # 风机状态2
-  wind_turbine_status2: 限功率运行
-  # 机舱内温度
-  cabin_temperature: null
-  # 湍流强度
-  turbulence_intensity: null
-  # 扭缆角度
-  twisted_cable_angle: 扭缆位置(°)
-  # 机舱前后振动
-  front_back_vibration_of_the_cabin: 机舱振动传感器Y(g)
-  # 机舱左右振动
-  side_to_side_vibration_of_the_cabin: 机舱振动传感器X(g)
-  # 实际力矩
-  actual_torque: null
-  # 给定力矩
-  given_torque: null
-  # 顺时针偏航次数
-  clockwise_yaw_count: null
-  # 逆时针偏航次数
-  counterclockwise_yaw_count: null
-  # 不可利用
-  unusable: null
-  # 功率曲线可用
-  power_curve_available: null
-  # 齿轮箱转速
-  required_gearbox_speed: null
-  # 变频器转速(主控)
-  inverter_speed_master_control: null
-  # 偏航激活状态 (param1-param10 预留字段,每个电厂自己定于对应的含义)
-  param1: 偏航激活状态
-
-# 自定义风机编号映射 必须是 wind_name
-wind_name_exec: "'A'+wind_name.replace('号风机','').zfill(3)"
-
-

etl/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/6/11
+# @Author  : 魏志亮

base/WindFarms.py → etl/base/WindFarms.py

@@ -1,34 +1,37 @@
 # -*- coding: utf-8 -*-
 # @Time    : 2024/5/15
 # @Author  : 魏志亮
-import copy
 import datetime
 import multiprocessing
-import sys
 import tempfile
 
-from base.TranseParam import TranseParam
-from utils.db.trans_mysql import creat_table_and_add_partition, update_trans_status, get_all_wind, \
-    rename_table, read_excel_and_save_to_db
+from etl.base.TranseParam import TranseParam
+from service.plt_service import get_all_wind, update_trans_status_error, update_trans_status_running, \
+    update_trans_status_success
+from service.trans_service import creat_table_and_add_partition, rename_table, save_df_to_db, save_file_to_db
+from utils.file.trans_methods import *
 from utils.log.trans_log import logger
-from utils.trans_methods import *
 from utils.zip.unzip import unzip, unrar
 
 
 class WindFarms(object):
 
-    def __init__(self, name, batch_no=None, field_code=None, params: TranseParam = None, wind_full_name=None):
+    def __init__(self, name, batch_no=None, field_code=None, params: TranseParam = None, wind_full_name=None,
+                 schedule_exec=True):
         self.name = name
         self.batch_no = batch_no
         self.field_code = field_code
         self.wind_full_name = wind_full_name
-        self.begin = datetime.datetime.now()
         self.save_zip = False
         self.trans_param = params
         self.__exist_wind_names = set()
         self.wind_col_trans = get_all_wind(self.field_code)
         self.batch_count = 50000
         self.save_path = None
+        self.schedule_exec = schedule_exec
+        self.min_date = None
+        self.max_date = None
+        self.total_count = 0
 
     def set_trans_param(self, params: TranseParam):
         self.trans_param = params
@@ -36,7 +39,7 @@ class WindFarms(object):
 
         if read_path.find(self.wind_full_name) == -1:
             message = "读取路径与配置路径不匹配:" + self.trans_param.read_path + ",配置文件为:" + self.wind_full_name
-            update_trans_status(self.batch_no, self.trans_param.read_type, "error", message)
+            update_trans_status_error(self.batch_no, self.trans_param.read_type, message, self.schedule_exec)
             raise ValueError(message)
 
         self.save_path = os.path.join(read_path[0:read_path.find(self.wind_full_name)], self.wind_full_name, "清理数据")
@@ -184,9 +187,32 @@ class WindFarms(object):
                 self.__exist_wind_names.add(name)
                 df[df['wind_turbine_number'] == name].to_csv(save_path, index=False, encoding='utf8')
 
+        self.__set_tongji_data(df)
         del df
         trans_print("保存", str(names), "到临时文件成功, 风机数量", len(names))
 
+    def __set_tongji_data(self, df):
+
+        min_date = df['time_stamp'].min()
+        max_date = df['time_stamp'].max()
+        if self.min_date is None or self.min_date > min_date:
+            self.min_date = min_date
+
+        if self.max_date is None or self.max_date < max_date:
+            self.max_date = max_date
+
+        self.total_count = self.total_count + df.shape[0]
+
+    def save_statistics_file(self):
+        save_path = os.path.join(os.path.dirname(self.__get_save_path()),
+                                 self.trans_param.read_type + '_statistics.txt')
+        create_file_path(save_path, is_file_path=True)
+        with open(save_path, 'w', encoding='utf8') as f:
+            f.write("总数据量:" + str(self.total_count) + "\n")
+            f.write("最小时间:" + str(self.min_date) + "\n")
+            f.write("最大时间:" + str(self.max_date) + "\n")
+            f.write("风机数量:" + str(len(self.__exist_wind_names)) + "\n")
+
     def save_to_csv(self, filename):
         df = read_file_to_df(filename)
 
@@ -248,7 +274,7 @@ class WindFarms(object):
         except Exception as e:
             logger.exception(e)
             message = "读取文件列表错误:" + self.trans_param.read_path + ",系统返回错误:" + str(e)
-            update_trans_status(self.batch_no, self.trans_param.read_type, "error", message)
+            update_trans_status_error(self.batch_no, self.trans_param.read_type, message, self.schedule_exec)
             raise e
         return all_files
 
@@ -289,7 +315,7 @@ class WindFarms(object):
             except Exception as e:
                 logger.exception(e)
                 message = "合并列出现错误:" + str(e)
-                update_trans_status(self.batch_no, self.trans_param.read_type, "error", message)
+                update_trans_status_error(self.batch_no, self.trans_param.read_type, message, self.schedule_exec)
                 raise e
 
         else:
@@ -299,7 +325,7 @@ class WindFarms(object):
                 except Exception as e:
                     logger.exception(e)
                     message = "读取文件错误:" + file + ",系统返回错误:" + str(e)
-                    update_trans_status(self.batch_no, self.trans_param.read_type, "error", message)
+                    update_trans_status_error(self.batch_no, self.trans_param.read_type, message, self.schedule_exec)
                     raise e
 
     def mutiprocessing_to_save_file(self):
@@ -313,7 +339,7 @@ class WindFarms(object):
         except Exception as e:
             logger.exception(e)
             message = "保存文件错误,系统返回错误:" + str(e)
-            update_trans_status(self.batch_no, self.trans_param.read_type, "error", message)
+            update_trans_status_error(self.batch_no, self.trans_param.read_type, message, self.schedule_exec)
             raise e
 
         trans_print("结束保存到excel文件")
@@ -327,13 +353,13 @@ class WindFarms(object):
         try:
 
             with multiprocessing.Pool(6) as pool:
-                pool.starmap(read_excel_and_save_to_db,
+                pool.starmap(save_file_to_db,
                              [(table_name, file, self.batch_count) for file in all_saved_files])
 
         except Exception as e:
             logger.exception(e)
             message = "保存到数据库错误,系统返回错误:" + str(e)
-            update_trans_status(self.batch_no, self.trans_param.read_type, "error", message)
+            update_trans_status_error(self.batch_no, self.trans_param.read_type, message, self.schedule_exec)
             raise e
         trans_print("结束保存到数据库文件")
 
@@ -368,27 +394,62 @@ class WindFarms(object):
         renamed_table_name = "del_" + table_name + "_" + datetime.datetime.now().strftime('%Y%m%d%H%M%S')
         rename_table(table_name, renamed_table_name)
 
-    def run(self):
+    def run(self, step=0, end=3):
+        begin = datetime.datetime.now()
         trans_print("开始执行", self.name, self.trans_param.read_type)
 
-        self.delete_batch_files()
-        self.delete_tmp_files()
-        self.delete_batch_db()
-
-        self.__params_valid([self.name, self.batch_no, self.field_code, self.save_path, self.trans_param.read_type,
-                             self.trans_param.read_path, 'wind_turbine_number', self.wind_full_name])
-
-        # 更新运行状态到运行中
-        update_trans_status(self.batch_no, self.trans_param.read_type, "running", "")
+        if step <= 0 and end >= 0:
+            tmp_begin = datetime.datetime.now()
+            trans_print("开始初始化字段")
+            self.delete_batch_files()
+            self.delete_tmp_files()
+            self.delete_batch_db()
 
-        # 开始读取数据并分类保存临时文件
-        self.__read_file_and_save_tmp()
+            self.__params_valid([self.name, self.batch_no, self.field_code, self.save_path, self.trans_param.read_type,
+                                 self.trans_param.read_path, self.wind_full_name])
 
-        self.mutiprocessing_to_save_file()
-
-        # self.mutiprocessing_to_save_db()
-
-        update_trans_status(self.batch_no, self.trans_param.read_type, "success", "",
-                            wind_count=len(read_excel_files(self.__get_read_tmp_path())))
+            if self.trans_param.resolve_col_prefix:
+                column = "测试"
+                eval(self.trans_param.resolve_col_prefix)
+
+            if self.trans_param.wind_name_exec:
+                wind_name = "测试"
+                eval(self.trans_param.wind_name_exec)
+
+            trans_print("初始化字段结束,耗时:", str(datetime.datetime.now() - tmp_begin), ",总耗时:",
+                        str(datetime.datetime.now() - begin))
+
+        if step <= 1 and end >= 1:
+            # 更新运行状态到运行中
+            tmp_begin = datetime.datetime.now()
+            trans_print("开始保存到临时文件")
+            update_trans_status_running(self.batch_no, self.trans_param.read_type, self.schedule_exec)
+
+            # 开始读取数据并分类保存临时文件
+            self.__read_file_and_save_tmp()
+            trans_print("保存到临时文件结束,耗时:", str(datetime.datetime.now() - tmp_begin), ",总耗时:",
+                        str(datetime.datetime.now() - begin))
+
+        if step <= 2 and end >= 2:
+            tmp_begin = datetime.datetime.now()
+            trans_print("开始保存到文件")
+            self.mutiprocessing_to_save_file()
+            self.save_statistics_file()
+            trans_print("保存到文件结束,耗时:", str(datetime.datetime.now() - tmp_begin), ",总耗时:",
+                        str(datetime.datetime.now() - begin))
+
+        if step <= 3 and end >= 3:
+            tmp_begin = datetime.datetime.now()
+            trans_print("开始保存到数据库")
+            self.mutiprocessing_to_save_db()
+            trans_print("保存到数据库结束,耗时:", str(datetime.datetime.now() - tmp_begin), ",总耗时:",
+                        str(datetime.datetime.now() - begin))
+        # 如果end==0 则说明只是进行了验证
+        if end != 0:
+            update_trans_status_success(self.batch_no, self.trans_param.read_type,
+                                        len(read_excel_files(self.__get_read_tmp_path())), self.schedule_exec)
+
+        trans_print("开始执行", self.name, self.trans_param.read_type, ",,总耗时:",
+                    str(datetime.datetime.now() - begin))
 
         self.delete_tmp_files()

gradio_web.py

@@ -1,199 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/6/3
-# @Author  : 魏志亮
-import copy
-
-import gradio as gr
-import yaml
-
-from utils.db.trans_mysql import *
-
-
-def test_click(wind_name, wind_full_name, type, is_vertical_table, merge_columns, vertical_read_cols,
-               vertical_index_cols, vertical_col_key, vertical_col_value, resolve_col_prefix, wind_name_exec,
-               wind_turbine_number, time_stamp, active_power, rotor_speed, generator_speed, wind_velocity,
-               pitch_angle_blade_1, pitch_angle_blade_2, pitch_angle_blade_3, cabin_position, true_wind_direction,
-               yaw_error1, set_value_of_active_power, gearbox_oil_temperature, generatordrive_end_bearing_temperature,
-               generatornon_drive_end_bearing_temperature, wind_turbine_status, wind_turbine_status2, cabin_temperature,
-               twisted_cable_angle, front_back_vibration_of_the_cabin, side_to_side_vibration_of_the_cabin,
-               actual_torque, given_torque, clockwise_yaw_count, counterclockwise_yaw_count, unusable,
-               power_curve_available, required_gearbox_speed, inverter_speed_master_control, outside_cabin_temperature,
-               main_bearing_temperature, gearbox_high_speed_shaft_bearing_temperature,
-               gearboxmedium_speed_shaftbearing_temperature, gearbox_low_speed_shaft_bearing_temperature,
-               generator_winding1_temperature, generator_winding2_temperature, generator_winding3_temperature,
-               turbulence_intensity, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10
-               ):
-    params = copy.deepcopy(vars())
-
-    error_message = ""
-    if wind_name is None or wind_name.strip() == '':
-        error_message += "风机名称必选"
-        gr.Warning(error_message)
-        return error_message
-
-    if wind_full_name is None or wind_full_name.strip() == '':
-        error_message += "风机全称必选"
-        gr.Warning(error_message)
-        return error_message
-
-    save_to_trans_conf(params)
-    return yaml.dump(vars(), allow_unicode=True, sort_keys=False)
-
-
-def fill_data(wind_name, type):
-    select_cols = ['wind_full_name', 'is_vertical_table', 'merge_columns', 'vertical_read_cols',
-                   'vertical_index_cols', 'vertical_col_key', 'vertical_col_value', 'resolve_col_prefix',
-                   'wind_name_exec',
-                   'wind_turbine_number', 'time_stamp', 'active_power', 'rotor_speed', 'generator_speed',
-                   'wind_velocity', 'pitch_angle_blade_1', 'pitch_angle_blade_2', 'pitch_angle_blade_3',
-                   'cabin_position', 'true_wind_direction', 'yaw_error1', 'set_value_of_active_power',
-                   'gearbox_oil_temperature', 'generatordrive_end_bearing_temperature',
-                   'generatornon_drive_end_bearing_temperature', 'wind_turbine_status', 'wind_turbine_status2',
-                   'cabin_temperature', 'twisted_cable_angle', 'front_back_vibration_of_the_cabin',
-                   'side_to_side_vibration_of_the_cabin', 'actual_torque', 'given_torque', 'clockwise_yaw_count',
-                   'counterclockwise_yaw_count', 'unusable', 'power_curve_available', 'required_gearbox_speed',
-                   'inverter_speed_master_control', 'outside_cabin_temperature', 'main_bearing_temperature',
-                   'gearbox_high_speed_shaft_bearing_temperature', 'gearboxmedium_speed_shaftbearing_temperature',
-                   'gearbox_low_speed_shaft_bearing_temperature', 'generator_winding1_temperature',
-                   'generator_winding2_temperature', 'generator_winding3_temperature', 'turbulence_intensity', 'param1',
-                   'param2', 'param3', 'param4', 'param5', 'param6', 'param7', 'param8', 'param9', 'param10']
-    print(wind_name, type)
-    df = get_trans_conf(wind_name, type)
-    print(df)
-    if df.empty:
-        return [''] * len(select_cols)
-    result = df[select_cols].iloc[0].values
-    return result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], \
-           result[8], result[9], \
-           result[10], result[11], result[12], result[13], result[14], result[15], result[16], result[17], result[18], \
-           result[19], result[20], result[21], result[22], result[23], result[24], result[25], result[26], result[27], \
-           result[28], result[29], result[30], result[31], result[32], result[33], result[34], result[35], result[36], \
-           result[37], result[38], result[39], result[40], result[41], result[42], result[43], result[44], result[45], \
-           result[46], result[47], result[48], result[49], result[50], result[51], result[52], result[53], result[54], \
-           result[55], result[56], result[57]
-
-
-with gr.Blocks(css=".container.svelte-1sk0pyu.svelte-1sk0pyu {width: 300px}") as demo:
-    wind_name = gr.Dropdown(label="电场名称", choices=get_all_wind_company())
-
-    types = {
-        '分钟映射': 'minute', '秒映射': 'second'
-    }
-
-    for name in types.keys():
-        with gr.Tab(label=name):
-            type = gr.Text(label="映射类型", value=types[name], visible=False)
-            wind_full_name = gr.Textbox(label="完整的电场名称")
-            merge_columns = gr.Checkbox(label="是否需合并(多个excel列合并成一个才需要选择)", value=False)
-            is_vertical_table = gr.Checkbox(label="是否是竖表", value=False)
-            vertical_read_cols = gr.Textbox(label="竖表--读取的字段", placeholder="逗号分隔")
-            vertical_index_cols = gr.Textbox(label="竖表--分组的字段", placeholder="逗号分隔,一般都是时间,机组")
-            vertical_col_key = gr.Textbox(label="竖表--数据点字段")
-            vertical_col_value = gr.Textbox(label="竖表--数据点数值")
-            resolve_col_prefix = gr.Textbox(label="处理列名",
-                                            placeholder="比如重庆海装 25_#桨距角,只需要 桨距角 可以用 column[column.find('#')+1:]")
-
-            wind_name_exec = gr.Textbox(label="风机编号代码处理",
-                                        placeholder="比如 昌平001号风机,可以配置 wind_name.replace('昌平','').replace('号风机','')")
-
-            wind_turbine_number = gr.Textbox(label="风机编号(wind_turbine_number)")
-            time_stamp = gr.Textbox(label="时间戳(time_stamp)")
-            active_power = gr.Textbox(label="有功功率(active_power)")
-            rotor_speed = gr.Textbox(label="风轮转速(rotor_speed)")
-            generator_speed = gr.Textbox(label="发电机转速(generator_speed)")
-            wind_velocity = gr.Textbox(label="风速(wind_velocity)")
-            pitch_angle_blade_1 = gr.Textbox(label="桨距角1(pitch_angle_blade_1)")
-            pitch_angle_blade_2 = gr.Textbox(label="桨距角2(pitch_angle_blade_2)")
-            pitch_angle_blade_3 = gr.Textbox(label="桨距角3(pitch_angle_blade_3)")
-            cabin_position = gr.Textbox(label="机舱位置(cabin_position)")
-            true_wind_direction = gr.Textbox(label="绝对风向(true_wind_direction)")
-            yaw_error1 = gr.Textbox(label="对风角度(yaw_error1)")
-            set_value_of_active_power = gr.Textbox(label="有功功率设定值(set_value_of_active_power)")
-            gearbox_oil_temperature = gr.Textbox(label="齿轮箱油温(gearbox_oil_temperature)")
-            generatordrive_end_bearing_temperature = gr.Textbox(
-                label="发电机驱动端轴承温度(generatordrive_end_bearing_temperature)")
-            generatornon_drive_end_bearing_temperature = gr.Textbox(
-                label="发电机非驱动端轴承温度(generatornon_drive_end_bearing_temperature)")
-            wind_turbine_status = gr.Textbox(label="风机状态1(wind_turbine_status)")
-            wind_turbine_status2 = gr.Textbox(label="风机状态2(wind_turbine_status2)")
-            cabin_temperature = gr.Textbox(label="机舱内温度(cabin_temperature)")
-            twisted_cable_angle = gr.Textbox(label="扭缆角度(twisted_cable_angle)")
-            front_back_vibration_of_the_cabin = gr.Textbox(label="机舱前后振动(front_back_vibration_of_the_cabin)")
-            side_to_side_vibration_of_the_cabin = gr.Textbox(label="机舱左右振动(side_to_side_vibration_of_the_cabin)")
-            actual_torque = gr.Textbox(label="实际力矩(actual_torque)")
-            given_torque = gr.Textbox(label="给定力矩(given_torque)")
-            clockwise_yaw_count = gr.Textbox(label="顺时针偏航次数(clockwise_yaw_count)")
-            counterclockwise_yaw_count = gr.Textbox(label="逆时针偏航次数(counterclockwise_yaw_count)")
-            unusable = gr.Textbox(label="不可利用(unusable)")
-            power_curve_available = gr.Textbox(label="功率曲线可用(power_curve_available)")
-            required_gearbox_speed = gr.Textbox(label="齿轮箱转速(required_gearbox_speed)")
-            inverter_speed_master_control = gr.Textbox(label="变频器转速(主控)(inverter_speed_master_control)")
-            outside_cabin_temperature = gr.Textbox(label="环境温度(outside_cabin_temperature)")
-            main_bearing_temperature = gr.Textbox(label="主轴承轴承温度(main_bearing_temperature)")
-            gearbox_high_speed_shaft_bearing_temperature = gr.Textbox(
-                label="齿轮箱高速轴轴承温度(gearbox_high_speed_shaft_bearing_temperature)")
-            gearboxmedium_speed_shaftbearing_temperature = gr.Textbox(
-                label="齿轮箱中速轴轴承温度(gearboxmedium_speed_shaftbearing_temperature)")
-            gearbox_low_speed_shaft_bearing_temperature = gr.Textbox(
-                label="齿轮箱低速轴轴承温度(gearbox_low_speed_shaft_bearing_temperature)")
-            generator_winding1_temperature = gr.Textbox(label="发电机绕组1温度(generator_winding1_temperature)")
-            generator_winding2_temperature = gr.Textbox(label="发电机绕组2温度(generator_winding2_temperature)")
-            generator_winding3_temperature = gr.Textbox(label="发电机绕组3温度(generator_winding3_temperature)")
-            turbulence_intensity = gr.Textbox(label="湍流强度(turbulence_intensity)")
-            param1 = gr.Textbox(label="齿轮箱油压(param1)")
-            param2 = gr.Textbox(label="预留字段2(param2)")
-            param3 = gr.Textbox(label="预留字段3(param3)")
-            param4 = gr.Textbox(label="预留字段4(param4)")
-            param5 = gr.Textbox(label="预留字段5(param5)")
-            param6 = gr.Textbox(label="预留字段6(param6)")
-            param7 = gr.Textbox(label="预留字段7(param7)")
-            param8 = gr.Textbox(label="预留字段8(param8)")
-            param9 = gr.Textbox(label="预留字段9(param9)")
-            param10 = gr.Textbox(label="预留字段10(param10)")
-
-            button = gr.Button(value="提交")
-            result = gr.Textbox(label="结果")
-
-            button.click(fn=test_click,
-                         inputs=[wind_name, wind_full_name, type, is_vertical_table, merge_columns, vertical_read_cols,
-                                 vertical_index_cols, vertical_col_key, vertical_col_value, resolve_col_prefix,
-                                 wind_name_exec, wind_turbine_number, time_stamp, active_power, rotor_speed,
-                                 generator_speed, wind_velocity, pitch_angle_blade_1, pitch_angle_blade_2,
-                                 pitch_angle_blade_3, cabin_position, true_wind_direction, yaw_error1,
-                                 set_value_of_active_power, gearbox_oil_temperature,
-                                 generatordrive_end_bearing_temperature, generatornon_drive_end_bearing_temperature,
-                                 wind_turbine_status, wind_turbine_status2, cabin_temperature, twisted_cable_angle,
-                                 front_back_vibration_of_the_cabin, side_to_side_vibration_of_the_cabin, actual_torque,
-                                 given_torque, clockwise_yaw_count, counterclockwise_yaw_count, unusable,
-                                 power_curve_available, required_gearbox_speed, inverter_speed_master_control,
-                                 outside_cabin_temperature, main_bearing_temperature,
-                                 gearbox_high_speed_shaft_bearing_temperature,
-                                 gearboxmedium_speed_shaftbearing_temperature,
-                                 gearbox_low_speed_shaft_bearing_temperature, generator_winding1_temperature,
-                                 generator_winding2_temperature, generator_winding3_temperature, turbulence_intensity,
-                                 param1, param2, param3, param4, param5, param6, param7, param8, param9, param10
-                                 ], outputs=[result])
-            wind_name.change(fill_data, inputs=[wind_name, type],
-                             outputs=[wind_full_name, is_vertical_table, merge_columns, vertical_read_cols,
-                                      vertical_index_cols, vertical_col_key, vertical_col_value, resolve_col_prefix,
-                                      wind_name_exec, wind_turbine_number, time_stamp, active_power, rotor_speed,
-                                      generator_speed, wind_velocity, pitch_angle_blade_1, pitch_angle_blade_2,
-                                      pitch_angle_blade_3, cabin_position, true_wind_direction, yaw_error1,
-                                      set_value_of_active_power, gearbox_oil_temperature,
-                                      generatordrive_end_bearing_temperature,
-                                      generatornon_drive_end_bearing_temperature,
-                                      wind_turbine_status, wind_turbine_status2, cabin_temperature, twisted_cable_angle,
-                                      front_back_vibration_of_the_cabin, side_to_side_vibration_of_the_cabin,
-                                      actual_torque,
-                                      given_torque, clockwise_yaw_count, counterclockwise_yaw_count, unusable,
-                                      power_curve_available, required_gearbox_speed, inverter_speed_master_control,
-                                      outside_cabin_temperature, main_bearing_temperature,
-                                      gearbox_high_speed_shaft_bearing_temperature,
-                                      gearboxmedium_speed_shaftbearing_temperature,
-                                      gearbox_low_speed_shaft_bearing_temperature, generator_winding1_temperature,
-                                      generator_winding2_temperature, generator_winding3_temperature,
-                                      turbulence_intensity,
-                                      param1, param2, param3, param4, param5, param6, param7, param8, param9, param10])
-
-if __name__ == "__main__":
-    demo.launch(server_name='0.0.0.0', server_port=7860, auth=('znzn', "znzn123"))

requirements.txt

@@ -1,5 +0,0 @@
-chardet==5.2.0
-PyMySQL==1.1.0
-PyYAML==6.0.1
-rarfile==4.2
-SQLAlchemy==2.0.30

schedule_service.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/6/11
+# @Author  : 魏志亮
+from etl.base.TranseParam import TranseParam
+from etl.base.WindFarms import WindFarms
+from service.plt_service import get_exec_data, update_trans_status_error
+from service.trans_service import get_trans_conf
+from utils.conf.read_conf import read_conf
+from utils.log.trans_log import init_log, trans_print, logger
+
+
+def run_schedule(step=0, end=3):
+    data = get_exec_data()
+    if data is None:
+        trans_print("当前有任务在执行")
+    elif len(data.keys()) == 0:
+        trans_print("当前无任务")
+    else:
+        batch_no = data['batch_code']
+        transfer_type = data['transfer_type']
+        transfer_file_addr = data['transfer_addr']
+        field_code = data['field_code']
+        field_name = data['field_name']
+
+        __exec_trans(step, end, batch_no, transfer_type, transfer_file_addr, field_name, field_code, schedule_exec=True)
+
+
+def run_local(step=0, end=3, batch_no=None, transfer_type=None, transfer_file_addr=None, field_name=None,
+              field_code="测试"):
+    if batch_no is None or str(batch_no).strip() == '':
+        return "批次编号不能为空"
+
+    if transfer_type not in ['second', 'minute']:
+        return "查询类型错误"
+
+    if transfer_file_addr is None or str(transfer_file_addr).strip() == '':
+        return "文件路径不能为空"
+
+    __exec_trans(step, end, batch_no, transfer_type, transfer_file_addr, field_name, field_code, schedule_exec=False)
+
+
+def __exec_trans(step, end, batch_no, transfer_type, transfer_file_addr=None, field_name=None, field_code="测试",
+                 schedule_exec=False):
+    init_log(batch_no, field_name, transfer_type)
+    conf_map = get_trans_conf(field_name, transfer_type)
+    if conf_map is None or type(conf_map) == tuple or len(conf_map.keys()) == 0:
+        message = f"未找到{field_name}的{transfer_type}配置"
+        trans_print(message)
+        update_trans_status_error(batch_no, transfer_type, message, schedule_exec)
+    else:
+
+        resolve_col_prefix = read_conf(conf_map, 'resolve_col_prefix')
+        wind_name_exec = read_conf(conf_map, 'wind_name_exec', None)
+        wind_full_name = read_conf(conf_map, 'wind_full_name')
+        is_vertical_table = read_conf(conf_map, 'is_vertical_table', False)
+        merge_columns = read_conf(conf_map, 'merge_columns', False)
+
+        vertical_cols = read_conf(conf_map, 'vertical_read_cols', '').split(',')
+        index_cols = read_conf(conf_map, 'vertical_index_cols', '').split(',')
+        vertical_key = read_conf(conf_map, 'vertical_col_key')
+        vertical_value = read_conf(conf_map, 'vertical_col_value')
+        need_valid_cols = not merge_columns
+
+        cols_trans_all = dict()
+        trans_cols = ['wind_turbine_number', 'time_stamp', 'active_power', 'rotor_speed', 'generator_speed',
+                      'wind_velocity', 'pitch_angle_blade_1', 'pitch_angle_blade_2', 'pitch_angle_blade_3',
+                      'cabin_position', 'true_wind_direction', 'yaw_error1', 'set_value_of_active_power',
+                      'gearbox_oil_temperature', 'generatordrive_end_bearing_temperature',
+                      'generatornon_drive_end_bearing_temperature', 'wind_turbine_status',
+                      'wind_turbine_status2',
+                      'cabin_temperature', 'twisted_cable_angle', 'front_back_vibration_of_the_cabin',
+                      'side_to_side_vibration_of_the_cabin', 'actual_torque', 'given_torque',
+                      'clockwise_yaw_count',
+                      'counterclockwise_yaw_count', 'unusable', 'power_curve_available',
+                      'required_gearbox_speed',
+                      'inverter_speed_master_control', 'outside_cabin_temperature', 'main_bearing_temperature',
+                      'gearbox_high_speed_shaft_bearing_temperature',
+                      'gearboxmedium_speed_shaftbearing_temperature',
+                      'gearbox_low_speed_shaft_bearing_temperature', 'generator_winding1_temperature',
+                      'generator_winding2_temperature', 'generator_winding3_temperature',
+                      'turbulence_intensity', 'param1',
+                      'param2', 'param3', 'param4', 'param5', 'param6', 'param7', 'param8', 'param9', 'param10']
+
+        for col in trans_cols:
+            cols_trans_all[col] = read_conf(conf_map, col, '')
+
+        trans_subject = WindFarms(field_name, batch_no=batch_no, field_code=field_code,
+                                  wind_full_name=wind_full_name, schedule_exec=schedule_exec)
+
+        params = TranseParam(read_type=transfer_type, read_path=transfer_file_addr,
+                             cols_tran=cols_trans_all,
+                             wind_name_exec=wind_name_exec, is_vertical_table=is_vertical_table,
+                             vertical_cols=vertical_cols, vertical_key=vertical_key,
+                             vertical_value=vertical_value, index_cols=index_cols, merge_columns=merge_columns,
+                             resolve_col_prefix=resolve_col_prefix, need_valid_cols=need_valid_cols)
+
+        trans_subject.set_trans_param(params)
+        try:
+            trans_subject.run(step=step, end=end)
+        except Exception as e:
+            logger.exception(e)
+            message = "系统返回错误:" + str(e)
+            update_trans_status_error(batch_no, transfer_type, message, schedule_exec)
+
+
+if __name__ == '__main__':
+    step = 0
+    end = 2
+    batch_no = '测试-111'
+    transfer_type = 'second'
+    transfer_file_addr = r'D:\transdata\test\唐龙三期风电场-安徽-大唐\收资数据\second'
+    field_name = '唐龙三期风电场'
+    field_code = "测试"
+    run_local(step, end, batch_no, transfer_type, transfer_file_addr, field_name, field_code)

service/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/6/7
+# @Author  : 魏志亮

service/plt_service.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/6/7
+# @Author  : 魏志亮
+from utils.db.ConnectMysqlPool import ConnectMysqlPool
+
+plt = ConnectMysqlPool("plt")
+
+
+def update_trans_status_running(batch_no, trans_type, schedule_exec=True):
+    if schedule_exec:
+        exec_sql = """
+        update data_transfer set trans_sys_status = 0,transfer_finish_time=now()  
+        where batch_code = %s  and transfer_type = %s
+        """
+        plt.execute(exec_sql, (batch_no, trans_type))
+
+
+def update_trans_status_error(batch_no, trans_type, message="", schedule_exec=True):
+    if schedule_exec:
+        exec_sql = """
+        update data_transfer set transfer_state = 2,trans_sys_status=2 ,err_info= %s,transfer_finish_time=now() 
+        where batch_code = %s  and  transfer_type = %s
+        """
+        plt.execute(exec_sql, (message, batch_no, trans_type))
+
+
+def update_trans_status_success(batch_no, trans_type, wind_count=0, schedule_exec=True):
+    if schedule_exec:
+        exec_sql = """
+        update data_transfer set transfer_state = 1,trans_sys_status = 1,err_info = '',engine_count =%s,transfer_finish_time=now()  
+        where batch_code = %s  and transfer_type = %s
+        """
+        plt.execute(exec_sql, (wind_count, batch_no, trans_type))
+
+
+# 获取执行的数据
+def get_exec_data() -> dict:
+    query_running_sql = "select 1 from data_transfer where trans_sys_status = 0"
+    query_next_exdc_sql = """
+    SELECT
+        t.*,a.field_name
+    FROM
+        data_transfer t INNER JOIN wind_field a on t.field_code = a.field_code
+    WHERE
+        (t.trans_sys_status = -1 or ( t.trans_sys_status = 2 and t.transfer_state = 0))
+    AND t.transfer_addr != ''
+    ORDER BY
+        t.update_time
+    LIMIT 1
+    """
+    data = plt.execute(query_running_sql)
+    if data:
+        return None
+    else:
+        data = plt.execute(query_next_exdc_sql)
+        if type(data) == tuple:
+            return {}
+        return data[0]
+
+
+def get_all_wind(field_code):
+    query_sql = "select engine_code,engine_name from wind_engine_group where field_code = %s and del_state = 0"
+    dict_datas = plt.execute(query_sql, (field_code,))
+    result = dict()
+    for data in dict_datas:
+        result[data['engine_name']] = data['engine_code']
+    return result
+
+
+def get_all_wind_company():
+    query_sql = "SELECT t.field_name FROM wind_field t where t.del_state = 0"
+    datas = plt.execute(query_sql)
+    datas = []
+    if datas:
+        return [v for data in datas for k, v in data.items()]
+    else:
+        return ['吉山风电场', '和风元宝山', '唐龙三期风电场', '密马风电场', '招远风电场', '昌平坳风场', '昌西一风电场', '虹梯官风电场', '长清风电场']
+
+
+if __name__ == '__main__':
+    print(get_all_wind('WOF01000002'))
+
+    print(get_all_wind_company())
+
+    print(get_exec_data())
+
+    # print(update_trans_status_success("test_唐龙-定时任务测试", "second", 10))

service/trans_service.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/6/7
+# @Author  : 魏志亮
+from pandas import DataFrame
+
+from utils.db.ConnectMysqlPool import ConnectMysqlPool
+from utils.file.trans_methods import read_file_to_df
+
+trans = ConnectMysqlPool("trans")
+
+
+def get_trans_conf(wind_name, trans_type) -> dict:
+    query_sql = "SELECT * FROM trans_conf where wind_name = %s and type = %s"
+    res = trans.execute(query_sql, (wind_name, trans_type))
+    print(res)
+    if type(res) == tuple:
+        return None
+    return res[0]
+
+
+def save_to_trans_conf(data_dict=dict()):
+    trans.save_dict(data_dict)
+
+
+def creat_table_and_add_partition(table_name, count, read_type):
+    create_sql = f"""
+    CREATE TABLE
+    IF NOT EXISTS `{table_name}` (
+        `wind_turbine_number` VARCHAR (20) DEFAULT NULL COMMENT '风机编号',
+        `time_stamp` datetime NOT NULL COMMENT '时间戳',
+        `active_power` DOUBLE DEFAULT NULL COMMENT '有功功率',
+        `rotor_speed` DOUBLE DEFAULT NULL COMMENT '风轮转速',
+        `generator_speed` DOUBLE DEFAULT NULL COMMENT '发电机转速',
+        `wind_velocity` DOUBLE DEFAULT NULL COMMENT '风速',
+        `pitch_angle_blade_1` DOUBLE DEFAULT NULL COMMENT '桨距角1',
+        `pitch_angle_blade_2` DOUBLE DEFAULT NULL COMMENT '桨距角2',
+        `pitch_angle_blade_3` DOUBLE DEFAULT NULL COMMENT '桨距角3',
+        `cabin_position` DOUBLE DEFAULT NULL COMMENT '机舱位置',
+        `true_wind_direction` DOUBLE DEFAULT NULL COMMENT '绝对风向',
+        `yaw_error1` DOUBLE DEFAULT NULL COMMENT '对风角度',
+        `set_value_of_active_power` DOUBLE DEFAULT NULL COMMENT '有功功率设定值',
+        `gearbox_oil_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱油温',
+        `generatordrive_end_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '发电机驱动端轴承温度',
+        `generatornon_drive_end_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '发电机非驱动端轴承温度',
+        `cabin_temperature` DOUBLE DEFAULT NULL COMMENT '机舱内温度',
+        `twisted_cable_angle` DOUBLE DEFAULT NULL COMMENT '扭缆角度',
+        `front_back_vibration_of_the_cabin` DOUBLE DEFAULT NULL COMMENT '机舱前后振动',
+        `side_to_side_vibration_of_the_cabin` DOUBLE DEFAULT NULL COMMENT '机舱左右振动',
+        `actual_torque` DOUBLE DEFAULT NULL COMMENT '实际力矩',
+        `given_torque` DOUBLE DEFAULT NULL COMMENT '给定力矩',
+        `clockwise_yaw_count` DOUBLE DEFAULT NULL COMMENT '顺时针偏航次数',
+        `counterclockwise_yaw_count` DOUBLE DEFAULT NULL COMMENT '逆时针偏航次数',
+        `unusable` DOUBLE DEFAULT NULL COMMENT '不可利用',
+        `power_curve_available` DOUBLE DEFAULT NULL COMMENT '功率曲线可用',
+        `required_gearbox_speed` DOUBLE DEFAULT NULL COMMENT '齿轮箱转速',
+        `inverter_speed_master_control` DOUBLE DEFAULT NULL COMMENT '变频器转速(主控)',
+        `outside_cabin_temperature` DOUBLE DEFAULT NULL COMMENT '环境温度',
+        `main_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '主轴承轴承温度',
+        `gearbox_high_speed_shaft_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱高速轴轴承温度',
+        `gearboxmedium_speed_shaftbearing_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱中速轴轴承温度',
+        `gearbox_low_speed_shaft_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱低速轴轴承温度',
+        `generator_winding1_temperature` DOUBLE DEFAULT NULL COMMENT '发电机绕组1温度',
+        `generator_winding2_temperature` DOUBLE DEFAULT NULL COMMENT '发电机绕组2温度',
+        `generator_winding3_temperature` DOUBLE DEFAULT NULL COMMENT '发电机绕组3温度',
+        `wind_turbine_status` DOUBLE DEFAULT NULL COMMENT '风机状态1',
+        `wind_turbine_status2` DOUBLE DEFAULT NULL COMMENT '风机状态2',
+        `turbulence_intensity` DOUBLE DEFAULT NULL COMMENT '湍流强度',
+        `year` INT (4) DEFAULT NULL COMMENT '年',
+        `month` INT (2) DEFAULT NULL COMMENT '月',
+        `day` INT (2) DEFAULT NULL COMMENT '日',
+        `param1` DOUBLE DEFAULT NULL COMMENT '预留1',
+        `param2` DOUBLE DEFAULT NULL COMMENT '预留2',
+        `param3` DOUBLE DEFAULT NULL COMMENT '预留3',
+        `param4` DOUBLE DEFAULT NULL COMMENT '预留4',
+        `param5` DOUBLE DEFAULT NULL COMMENT '预留5',
+        `param6` DOUBLE DEFAULT NULL COMMENT '预留6',
+        `param7` DOUBLE DEFAULT NULL COMMENT '预留7',
+        `param8` DOUBLE DEFAULT NULL COMMENT '预留8',
+        `param9` DOUBLE DEFAULT NULL COMMENT '预留9',
+        `param10` DOUBLE DEFAULT NULL COMMENT '预留10',
+         KEY `time_stamp` (`time_stamp`),
+         KEY `wind_turbine_number` (`wind_turbine_number`)
+    ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4
+    """
+
+    if read_type == 'second':
+        create_sql = create_sql + f" PARTITION BY KEY (`wind_turbine_number`) PARTITIONS {count}"
+
+    trans.execute(create_sql)
+
+
+def rename_table(table_name, renamed_table_name):
+    rename_sql = f"RENAME TABLE {table_name} TO {renamed_table_name}"
+    try:
+        trans.execute(rename_sql)
+    except Exception as e:
+        print(e)
+
+
+def save_file_to_db(table_name: str, file: str, batch_count=20000):
+    save_df_to_db(table_name, read_file_to_df(file), batch_count)
+
+
+def save_df_to_db(table_name: str, df: DataFrame, batch_count=20000):
+    trans.df_batch_save(table_name, df, batch_count)
+
+
+if __name__ == '__main__':
+    get_trans_conf('唐龙三期风电场', 'second')
+
+    save_file_to_db("test_唐龙-定时任务测试_second", r"D:\transdata\test\唐龙三期风电场-安徽-大唐\清理数据\test_唐龙-定时任务测试\second\C02.csv")

utils/__init__.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
-# @Time    : 2024/5/17
+# @Time    : 2024/6/6
 # @Author  : 魏志亮

utils/conf/__init__.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
-# @Time    : 2024/5/17
+# @Time    : 2024/6/7
 # @Author  : 魏志亮

utils/conf/generate_trans_cols.py

@@ -1,17 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/5/31
-# @Author  : 魏志亮
-import numpy as np
-import pandas as pd
-
-df = pd.read_excel(r"C:\Users\Administrator\Downloads\点检表配置.xls", sheet_name="和风元宝山-河南-大唐")
-
-df.replace(np.nan, '', inplace=True)
-# print("trans_col:")
-for cn, en, sec, min in zip(df['中文名'], df['英文名'], df['秒级映射'], df['分钟级映射']):
-    # print(f"\t#{cn}\n\t{en}: {min}")
-    print(f'{en} = gr.Textbox(label="{cn}({en})")')
-
-
-for i in range(2,11):
-    print(f'param{i} = gr.Textbox(label="预留字段{i}(param{i})")')

utils/conf/read_conf.py

@@ -1,29 +1,20 @@
 # -*- coding: utf-8 -*-
-# @Time    : 2024/5/17
+# @Time    : 2024/6/7
 # @Author  : 魏志亮
-import os
-
 import yaml
 
 
-def read_yaml_file(filename, type):
-    path = os.path.abspath(__file__)
-    for i in range(3):
-        path = os.path.dirname(path)
-    with open(path + os.sep + "config_files" + os.sep + filename + os.sep + type + ".yaml", 'r', encoding='utf-8') as f:
-        return yaml.safe_load(f)
+def yaml_conf(path, encoding='utf-8'):
+    with open(path, 'r', encoding=encoding) as f:
+        data = yaml.safe_load(f)
+    return data
 
 
-def read_param_from_config(conf, type, default_value=None):
-    if type in conf:
-        return conf[type]
+def read_conf(dict_conf, col, default_value=None):
+    if col in dict_conf:
+        res = dict_conf[col]
+        if res is None and default_value is not None:
+            return default_value
+        return res
     else:
         return default_value
-
-
-if __name__ == '__main__':
-    yaml_files = os.listdir("../../config_files")
-    types = ['second', 'minute']
-
-    print(yaml_files)
-

utils/db/ConnectMysqlPool.py

@@ -0,0 +1,155 @@
+import numpy as np
+import pandas as pd
+import pymysql
+from dbutils.pooled_db import PooledDB
+import os
+
+from pandas import DataFrame
+from pymysql.cursors import DictCursor
+
+from utils.conf.read_conf import yaml_conf
+from utils.log.trans_log import trans_print
+
+
+class ConnectMysqlPool:
+    """
+    连接MySQL数据库的连接池类。
+
+    属性:
+    db_account (dict): 数据库账号信息，包括用户名和密码等。
+    db (str): 数据库名称。
+    pool (PooledDB): MySQL连接池对象。
+
+    方法:
+    __init__: 初始化连接池类实例。
+    _obtaining_data: 从配置文件中获取测试数据。
+    create_mysql_pool: 创建MySQL连接池。
+    get_conn: 从连接池中获取一个连接。
+    close: 关闭数据库连接和游标。
+    execute: 使用连接执行SQL语句。
+    """
+
+    def __init__(self, connet_name):
+        """
+        初始化连接池类实例。
+
+        参数:
+        db (str): 测试库名称。
+        db_account (dict): 包含数据库账号信息的字典。
+        """
+        file_path = os.path.join(
+            os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
+            "conf",
+            "db.yaml"
+        )
+        self.yaml_data = yaml_conf(file_path)
+        self.connet_name = connet_name
+
+        # 创建连接池
+        self.pool = self.create_mysql_pool()
+
+    # 创建MySQL连接池
+    def create_mysql_pool(self):
+        """
+        根据配置信息创建MySQL连接池。
+
+        返回:
+        PooledDB: MySQL连接池对象。
+        """
+        pool = PooledDB(
+            **self.yaml_data[self.connet_name + '_connect_pool_config'],
+            **self.yaml_data[self.connet_name],
+            creator=pymysql
+        )
+        return pool
+
+    # 从连接池中获取一个连接
+    def get_conn(self):
+        """
+        从连接池中获取一个数据库连接。
+
+        返回:
+        connection: 数据库连接对象。
+        """
+        return self.pool.connection()
+
+    # 使用连接执行sql
+    def execute(self, sql, params=tuple()):
+        """
+        使用获取的连接执行SQL语句。
+
+        参数:
+        sql (str): SQL语句。
+        params (tuple): SQL参数。
+
+        返回:
+        list: 执行SQL语句后的结果集，若执行出错则返回None。
+        """
+
+        with self.get_conn() as conn:
+            with conn.cursor(cursor=DictCursor) as cursor:
+                try:
+                    cursor.execute(sql, params)
+                    trans_print("开始执行SQL:", cursor._executed)
+                    conn.commit()
+                    result = cursor.fetchall()
+                    return result
+                except Exception as e:
+                    print(f"执行sql：{sql}，报错：{e}")
+                    conn.rollback()
+                    raise e
+
+    def save_dict(self, table_name: str, params: dict):
+        keys = params.keys()
+        col_str = ",".join(keys)
+        data_s_str = ",".join(["%s"] * len(keys))
+
+        insert_sql = f"replace into {table_name} ({col_str}) values ({data_s_str})"
+        with self.get_conn() as conn:
+            with conn.cursor() as cursor:
+                try:
+                    cursor.execute(insert_sql, tuple(params.values()))
+                    conn.commit()
+                except Exception as e:
+                    print(f"执行sql：{insert_sql}，报错：{e}")
+                    conn.rollback()
+                    raise e
+
+    # 使用连接执行sql
+    def df_batch_save(self, table_name: str, df: DataFrame, batch_count=20000):
+        col_str = ",".join(df.columns)
+        data_s_str = ",".join(["%s"] * len(df.columns))
+
+        insert_sql = f"INSERT INTO `{table_name}` ({col_str}) values ({data_s_str})"
+
+        # 转化nan到null
+        df.replace(np.nan, None, inplace=True)
+
+        total_count = df.shape[0]
+        for i in range(0, total_count + 1, batch_count):
+            with self.get_conn() as conn:
+                with conn.cursor() as cursor:
+                    try:
+                        query_df = df.iloc[i:i + batch_count]
+                        values = [tuple(data) for data in query_df.values]
+                        cursor.executemany(insert_sql, values)
+                        conn.commit()
+                        result = cursor.fetchall()
+                        print(
+                            "总条数" + str(df.shape[0]) + ",已保存:" + str(i + batch_count))
+                    except Exception as e:
+                        conn.rollback()
+                        raise e
+
+
+if __name__ == '__main__':
+    plt = ConnectMysqlPool("plt")
+
+    print(plt.execute("select * from data_transfer limit 2"))
+
+    trans = ConnectMysqlPool("trans")
+
+    df = pd.DataFrame()
+    df['name'] = ['name' + str(i) for i in range(1000)]
+
+    print(trans.df_batch_save('test', df, 33))

utils/db/__init__.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
-# @Time    : 2024/5/15
+# @Time    : 2024/6/7
 # @Author  : 魏志亮

utils/db/trans_mysql.py

@@ -1,259 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/5/15
-# @Author  : 魏志亮
-import datetime
-
-import numpy as np
-import pandas as pd
-# 建立数据库连接
-import pymysql
-from pandas import DataFrame
-from sqlalchemy import create_engine, Engine, text
-
-from utils.log.trans_log import trans_print
-from utils.trans_methods import read_file_to_df
-
-plt_user = 'admin'
-plt_password = 'admin123456'
-plt_host = '192.168.50.233'
-plt_port = 3306
-plt_database = 'energy'
-
-plt_connect = pymysql.connect(host=plt_host,
-                              user=plt_user,
-                              password=plt_password,
-                              db=plt_database,
-                              charset='utf8mb4',
-                              cursorclass=pymysql.cursors.DictCursor)
-
-trans_user = 'admin'
-trans_password = 'admin123456'
-trans_host = '192.168.50.233'
-trans_port = 3306
-trans_database = 'energy_data'
-
-trans_connect = pymysql.connect(host=trans_host,
-                                user=trans_user,
-                                password=trans_password,
-                                db=trans_database,
-                                charset='utf8mb4',
-                                cursorclass=pymysql.cursors.DictCursor)
-
-susscess_sql = "update data_transfer set transfer_state = 1 where batch_code = '{batch_no}' and transfer_type = '{" \
-               "trans_type}'";
-
-error_sql = "update data_transfer set transfer_state = 2 ,err_info='{message}' where batch_code = '{batch_no}'  and " \
-            "transfer_type = '{trans_type}' "
-
-running_sql = "update data_transfer set transfer_state = 0,engine_count = {wind_count}  where batch_code = '{" \
-              "batch_no}'  and transfer_type = '{trans_type}' "
-
-
-def __query(connect: pymysql.Connection, sql):
-    trans_print('开始执行SQL:', sql)
-    connect.ping(reconnect=True)
-    with connect.cursor() as cursor:
-        cursor.execute(sql)
-        datas = cursor.fetchall()
-        df = pd.DataFrame(datas)
-    connect.close()
-    return df
-
-
-def __ddl_sql(connect: pymysql.Connection, sql):
-    trans_print('开始执行SQL:', sql)
-    connect.ping(reconnect=True)
-    with connect.cursor() as cursor:
-        cursor.execute(sql)
-    connect.commit()
-    connect.close()
-
-
-def update_trans_status(batch_no, trans_type, status, message="", wind_count=0):
-    if status == 'success':
-        exec_sql = susscess_sql
-    elif status == 'running':
-        exec_sql = running_sql
-    else:
-        exec_sql = error_sql
-
-    exec_sql = exec_sql.format(batch_no=batch_no, status=status, message=message, trans_type=trans_type,
-                               wind_count=wind_count)
-    __ddl_sql(plt_connect, exec_sql)
-
-
-# 获取执行的数据
-def get_exec_data():
-    query_running_sql = "select 1 from data_transfer where transfer_state = 0"
-    query_next_exdc_sql = """
-    SELECT
-        t.*,a.field_name
-    FROM
-        data_transfer t INNER JOIN wind_field a on t.field_code = a.field_code
-    WHERE
-        t.transfer_state = -1
-    AND t.transfer_addr != ''
-    ORDER BY
-        t.update_time
-    LIMIT 1
-    """
-    df = __query(plt_connect, query_running_sql)
-    if df.empty:
-        df = __query(plt_connect, query_next_exdc_sql)
-        return df
-    else:
-        return None
-
-
-def get_all_wind(field_code):
-    query_sql = f"select engine_code,engine_name from wind_engine_group where field_code = '{field_code}' and del_state = 0"
-    df = __query(plt_connect, query_sql)
-    dict_datas = dict()
-    if df.empty:
-        return dict_datas
-    else:
-        for engine_code, engine_name in zip(df['engine_code'], df['engine_name']):
-            dict_datas[engine_name] = engine_code
-        return dict_datas
-
-
-def get_all_wind_company():
-    query_sql = f"SELECT t.field_name FROM wind_field t where t.del_state = 0"
-    df = __query(plt_connect, query_sql)
-    df = pd.DataFrame()
-    df['field_name'] = ['吉山风电场', '和风元宝山', '唐龙三期风电场', '密马风电场', '招远风电场', '昌平坳风场', '昌西一风电场', '虹梯官风电场', '长清风电场']
-    if df.empty:
-        return []
-    else:
-        return list(df['field_name'].values)
-
-
-def get_trans_conf(wind_name, type):
-    query_sql = f"SELECT * FROM trans_conf where wind_name = '{wind_name}' and type = '{type}'"
-    return __query(trans_connect, query_sql)
-
-
-def save_to_trans_conf(data_dict=dict()):
-    keys = data_dict.keys()
-    col_str = ",".join(keys)
-    data_s_str = ",".join(["%s"] * len(keys))
-
-    insert_sql = f"replace into trans_conf ({col_str}) values ({data_s_str})"
-    trans_connect.ping(reconnect=True)
-    with trans_connect.cursor() as cursor:
-        cursor.execute(insert_sql, tuple(data_dict.values()))
-    trans_connect.commit()
-    trans_connect.close()
-
-
-def creat_table_and_add_partition(table_name, count, read_type):
-    query_table = f"SELECT t.TABLE_NAME FROM information_schema.`TABLES` t where t.TABLE_NAME = '{table_name}'"
-    df = __query(trans_connect, query_table)
-    if df.empty:
-        create_sql = f"""
-        CREATE TABLE
-        IF NOT EXISTS `{table_name}` (
-            `wind_turbine_number` VARCHAR (20) DEFAULT NULL COMMENT '风机编号',
-            `time_stamp` datetime NOT NULL COMMENT '时间戳',
-            `active_power` DOUBLE DEFAULT NULL COMMENT '有功功率',
-            `rotor_speed` DOUBLE DEFAULT NULL COMMENT '风轮转速',
-            `generator_speed` DOUBLE DEFAULT NULL COMMENT '发电机转速',
-            `wind_velocity` DOUBLE DEFAULT NULL COMMENT '风速',
-            `pitch_angle_blade_1` DOUBLE DEFAULT NULL COMMENT '桨距角1',
-            `pitch_angle_blade_2` DOUBLE DEFAULT NULL COMMENT '桨距角2',
-            `pitch_angle_blade_3` DOUBLE DEFAULT NULL COMMENT '桨距角3',
-            `cabin_position` DOUBLE DEFAULT NULL COMMENT '机舱位置',
-            `true_wind_direction` DOUBLE DEFAULT NULL COMMENT '绝对风向',
-            `yaw_error1` DOUBLE DEFAULT NULL COMMENT '对风角度',
-            `set_value_of_active_power` DOUBLE DEFAULT NULL COMMENT '有功功率设定值',
-            `gearbox_oil_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱油温',
-            `generatordrive_end_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '发电机驱动端轴承温度',
-            `generatornon_drive_end_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '发电机非驱动端轴承温度',
-            `cabin_temperature` DOUBLE DEFAULT NULL COMMENT '机舱内温度',
-            `twisted_cable_angle` DOUBLE DEFAULT NULL COMMENT '扭缆角度',
-            `front_back_vibration_of_the_cabin` DOUBLE DEFAULT NULL COMMENT '机舱前后振动',
-            `side_to_side_vibration_of_the_cabin` DOUBLE DEFAULT NULL COMMENT '机舱左右振动',
-            `actual_torque` DOUBLE DEFAULT NULL COMMENT '实际力矩',
-            `given_torque` DOUBLE DEFAULT NULL COMMENT '给定力矩',
-            `clockwise_yaw_count` DOUBLE DEFAULT NULL COMMENT '顺时针偏航次数',
-            `counterclockwise_yaw_count` DOUBLE DEFAULT NULL COMMENT '逆时针偏航次数',
-            `unusable` BIGINT (20) DEFAULT NULL COMMENT '不可利用',
-            `power_curve_available` BIGINT (20) DEFAULT NULL COMMENT '功率曲线可用',
-            `required_gearbox_speed` DOUBLE DEFAULT NULL COMMENT '齿轮箱转速',
-            `inverter_speed_master_control` DOUBLE DEFAULT NULL COMMENT '变频器转速(主控)',
-            `outside_cabin_temperature` DOUBLE DEFAULT NULL COMMENT '环境温度',
-            `main_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '主轴承轴承温度',
-            `gearbox_high_speed_shaft_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱高速轴轴承温度',
-            `gearboxmedium_speed_shaftbearing_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱中速轴轴承温度',
-            `gearbox_low_speed_shaft_bearing_temperature` DOUBLE DEFAULT NULL COMMENT '齿轮箱低速轴轴承温度',
-            `generator_winding1_temperature` DOUBLE DEFAULT NULL COMMENT '发电机绕组1温度',
-            `generator_winding2_temperature` DOUBLE DEFAULT NULL COMMENT '发电机绕组2温度',
-            `generator_winding3_temperature` DOUBLE DEFAULT NULL COMMENT '发电机绕组3温度',
-            `wind_turbine_status` BIGINT (20) DEFAULT NULL COMMENT '风机状态1',
-            `wind_turbine_status2` DOUBLE DEFAULT NULL COMMENT '风机状态2',
-            `turbulence_intensity` DOUBLE DEFAULT NULL COMMENT '湍流强度',
-            `year` INT (4) DEFAULT NULL COMMENT '年',
-            `month` INT (2) DEFAULT NULL COMMENT '月',
-            `day` INT (2) DEFAULT NULL COMMENT '日',
-            `param1` DOUBLE DEFAULT NULL COMMENT '预留1',
-            `param2` DOUBLE DEFAULT NULL COMMENT '预留2',
-            `param3` DOUBLE DEFAULT NULL COMMENT '预留3',
-            `param4` DOUBLE DEFAULT NULL COMMENT '预留4',
-            `param5` DOUBLE DEFAULT NULL COMMENT '预留5',
-            `param6` DOUBLE DEFAULT NULL COMMENT '预留6',
-            `param7` DOUBLE DEFAULT NULL COMMENT '预留7',
-            `param8` DOUBLE DEFAULT NULL COMMENT '预留8',
-            `param9` DOUBLE DEFAULT NULL COMMENT '预留9',
-            `param10` DOUBLE DEFAULT NULL COMMENT '预留10',
-             KEY `time_stamp` (`time_stamp`),
-             KEY `wind_turbine_number` (`wind_turbine_number`)
-        ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4
-        """
-
-        if read_type == 'second':
-            create_sql = create_sql + f" PARTITION BY KEY (`wind_turbine_number`) PARTITIONS {count}"
-
-        __ddl_sql(trans_connect, create_sql)
-
-
-def rename_table(table_name, renamed_table_name):
-    rename_sql = f"RENAME TABLE {table_name} TO {renamed_table_name}"
-    try:
-        __ddl_sql(trans_connect, rename_sql)
-    except Exception as e:
-        trans_print(e)
-
-
-def read_excel_and_save_to_db(table_name: str, file_path, batch_count=20000):
-    df = read_file_to_df(file_path)
-    trans_print("开始保存文件", file_path, "到数据库")
-    begin = datetime.datetime.now()
-    save_df_to_db(trans_connect, table_name, df, batch_count)
-    trans_print("保存文件", file_path, "到数据库成功,耗时:", str(datetime.datetime.now() - begin))
-
-
-def save_df_to_db(connection: pymysql.Connection, table_name: str, df: DataFrame, batch_count=20000):
-    col_str = ",".join(df.columns)
-    data_s_str = ",".join(["%s"] * len(df.columns))
-
-    insert_sql = f"INSERT INTO {table_name} ({col_str}) values ({data_s_str})"
-
-    # 转化nan到null
-    df.replace(np.nan, None, inplace=True)
-
-    total_count = df.shape[0]
-    for i in range(0, total_count + 1, batch_count):
-        connection.ping(reconnect=True)
-        with connection.cursor() as cursor:
-            query_df = df.iloc[i:i + batch_count]
-            values = [tuple(data) for data in query_df.values]
-            cursor.executemany(insert_sql, values)
-        connection.commit()
-        connection.close()
-        trans_print("保存条数成功,总条数", str(total_count), "已完成条数:",
-                    str(total_count if (i + batch_count) > total_count else (i + batch_count)))
-
-
-if __name__ == '__main__':
-    df = get_exec_data()
-    print(df)

utils/db/trans_pg.py

@@ -1,65 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2024/5/15
-# @Author  : 魏志亮
-
-import pandas as pd
-# 建立数据库连接
-from sqlalchemy import create_engine, Engine, text
-
-from utils.log.trans_log import trans_print
-
-trans_user = 'postgres'
-trans_password = 'admin123456'
-trans_host = '192.168.50.235'
-trans_port = 5432
-trans_database = 'postgres'
-
-trans_engine = create_engine(
-    f'postgresql://{trans_user}:{trans_password}@{trans_host}:{trans_port}/{trans_database}', echo=True)
-
-
-def __query(engine: Engine, sql):
-    trans_print('开始执行SQL:', sql)
-    with engine.connect() as conn:
-        df = pd.read_sql(text(sql), conn)
-    return df
-
-
-def __ddl_sql(engine: Engine, sql):
-    trans_print('开始执行SQL:', sql)
-    with engine.connect() as conn:
-        conn.execute(text(sql))
-        conn.commit()
-
-
-def creat_table_and_add_partition(table_name, partition):
-    lower_partition = partition.lower()
-    query_table = f"""
-    select  c.relname
-    from pg_class c
-             join pg_inherits pi on pi.inhrelid = c. oid
-             join pg_class c2 on c2.oid = pi.inhparent
-    where
-            c2.relname = '{table_name}' and c.relname = '{table_name}_{lower_partition}'
-    """
-    df = __query(trans_engine, query_table)
-
-    if df.empty:
-        add_partition_sql = f"""
-            create table {table_name}_{partition} PARTITION OF {table_name} FOR VALUES  IN ('{partition}');
-        """
-        __ddl_sql(trans_engine, add_partition_sql)
-
-
-if __name__ == '__main__':
-    # creat_table_and_add_partition("test_11", "123_002")
-    df = __query(trans_engine, """
-    select  c.relname
-from pg_class c
-         join pg_inherits pi on pi.inhrelid = c. oid
-         join pg_class c2 on c2.oid = pi.inhparent
-where
-        c2.relname = 'energy_data_second' and c.relname = 'energy_data_second_202405201453_a006';
-    """)
-    print(df)
-    print(df.empty)

utils/file/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/6/11
+# @Author  : 魏志亮