添加合并表业务逻辑,加快执行速度

etl/step/ReadAndSaveTmp.py

@@ -43,6 +43,11 @@ class ReadAndSaveTmp(object):
 
     def save_merge_data(self, file_path):
         df = self.read_excel_to_df(file_path)
+
+        if self.trans_param.wind_name_exec:
+            exec_str = f"df['wind_turbine_number'].apply(lambda wind_name: {self.trans_param.wind_name_exec} )"
+            df['wind_turbine_number'] = eval(exec_str)
+
         names = set(df['wind_turbine_number'].values)
         cols = list(df.columns)
         cols.sort()
@@ -340,10 +345,6 @@ class ReadAndSaveTmp(object):
 
                 df.rename(columns=cols_dict, inplace=True)
 
-            if self.trans_param.wind_name_exec:
-                exec_str = f"df['wind_turbine_number'].apply(lambda wind_name: {self.trans_param.wind_name_exec} )"
-                df['wind_turbine_number'] = eval(exec_str)
-
             return df
 
     def run(self):

test_app_run.py

@@ -138,10 +138,14 @@ if __name__ == '__main__':
 
     # run_schedule(run_count=run_count)
 
-    run_local(0, 4, batch_no='WOF043600007-WOB000001', batch_name='XALFDC0814', transfer_type='second',
-              transfer_file_addr=r'/data/download/collection_data/1进行中/新艾里风电场-吉林-大唐/收资数据/sec', field_name='新艾里风电场',
-              field_code="WOF043600007", save_db=True)
+    # run_local(0, 3, batch_no='test_11', batch_name='test', transfer_type='minute',
+    #           transfer_file_addr=r'D:\trans_data\密马风电场\收资数据\minute', field_name='密马风电场',
+    #           field_code="WOF035200003", save_db=False)
 
-    # run_local(3, 3, batch_no='WOF043600007-WOB000001', batch_name='XALFDC0814', transfer_type='second',
+    run_local(0, 3, batch_no='test_11', batch_name='test', transfer_type='minute',
+              transfer_file_addr=r'D:\trans_data\和风元宝山\收资数据\min', field_name='和风元宝山',
+              field_code="WOF039800012", save_db=False)
+
+    # run_local(0, 3, batch_no='WOF043600007-WOB000001', batch_name='XALFDC0814', transfer_type='second',
     #           transfer_file_addr=r'D:\trans_data\新艾里风电场\收资数据\1号风机', field_name='新艾里风电场',
     #           field_code="WOF043600007", save_db=False)

tmp_file/read_and_draw_png.py

@@ -22,9 +22,9 @@ def draw(file, fengchang='测试'):
 
 
 if __name__ == '__main__':
-    read_dir = r"Z:\collection_data\1进行中\新艾里风电场-吉林-大唐\清理数据\WOF043600007-WOB000001_XALFDC0814\minute"
+    read_dir = r"D:\data\清理数据\和风元宝山\test_11_test\minute"
 
     files = [read_dir + os.sep + i for i in os.listdir(read_dir)]
 
     with multiprocessing.Pool(6) as pool:
-        pool.starmap(draw, [(file, "新艾里风电场") for file in files])
+        pool.starmap(draw, [(file, "和风元宝山") for file in files])