import datetime import multiprocessing import os import traceback import pandas as pd import numpy as np from etl.wind_power.min_sec import TransParam from etl.common.PathsAndTable import PathsAndTable from etl.wind_power.min_sec.ClassIdentifier import ClassIdentifier from service.plt_service import update_trans_transfer_progress from utils.conf.read_conf import read_conf from utils.df_utils.util import get_time_space from utils.file.trans_methods import create_file_path, read_excel_files, read_file_to_df, split_array from utils.log.trans_log import trans_print from utils.systeminfo.sysinfo import use_files_get_max_cpu_count class StatisticsAndSaveFile(object): def __init__(self, paths_and_table: PathsAndTable, trans_param: TransParam, statistics_map, rated_power_and_cutout_speed_map): self.paths_and_table = paths_and_table self.trans_param = trans_param self.statistics_map = statistics_map self.lock = multiprocessing.Manager().Lock() self.rated_power_and_cutout_speed_map = rated_power_and_cutout_speed_map def set_statistics_data(self, df): if not df.empty: df['time_stamp'] = pd.to_datetime(df['time_stamp']) min_date = df['time_stamp'].min() max_date = df['time_stamp'].max() with self.lock: if 'min_date' in self.statistics_map.keys(): if self.statistics_map['min_date'] > min_date: self.statistics_map['min_date'] = min_date else: self.statistics_map['min_date'] = min_date if 'max_date' in self.statistics_map.keys(): if self.statistics_map['max_date'] < max_date: self.statistics_map['max_date'] = max_date else: self.statistics_map['max_date'] = max_date if 'total_count' in self.statistics_map.keys(): self.statistics_map['total_count'] = self.statistics_map['total_count'] + df.shape[0] else: self.statistics_map['total_count'] = df.shape[0] if 'time_granularity' not in self.statistics_map.keys(): self.statistics_map['time_granularity'] = get_time_space(df, 'time_stamp') def save_to_csv(self, filename): df = read_file_to_df(filename) if self.trans_param.is_vertical_table: df = df.pivot_table(index=['time_stamp', 'wind_turbine_number'], columns=self.trans_param.vertical_key, values=self.trans_param.vertical_value, aggfunc='max') # 重置索引以得到普通的列 df.reset_index(inplace=True) # 转化风机名称 origin_wind_name = str(df['wind_turbine_number'].values[0]) df['wind_turbine_number'] = df['wind_turbine_number'].astype('str') # df['wind_turbine_name'] = df['wind_turbine_number'] df['wind_turbine_number'] = df['wind_turbine_number'].map( self.trans_param.wind_col_trans).fillna(df['wind_turbine_number']) wind_col_name = str(df['wind_turbine_number'].values[0]) not_double_cols = ['wind_turbine_number', 'wind_turbine_name', 'time_stamp', 'param6', 'param7', 'param8', 'param9', 'param10'] # 删除 有功功率 和 风速均为空的情况 df.dropna(subset=['active_power', 'wind_velocity'], how='all', inplace=True) df.replace(np.nan, -999999999, inplace=True) number_cols = df.select_dtypes(include=['number']).columns.tolist() for col in df.columns: if col not in not_double_cols and col not in number_cols: if not df[col].isnull().all(): df[col] = pd.to_numeric(df[col], errors='coerce') # 删除包含NaN的行（即那些列A转换失败的行） df = df.dropna(subset=[col]) df.replace(-999999999, np.nan, inplace=True) df.drop_duplicates(['wind_turbine_number', 'time_stamp'], keep='first', inplace=True) # 添加年月日 solve_time_begin = datetime.datetime.now() # df = df[(df['time_stamp'].str.find('-') > 0) & (df['time_stamp'].str.find(':') > 0)] # trans_print(wind_col_name, "去掉非法时间后大小:", df.shape[0]) # df['time_stamp'] = pd.to_datetime(df['time_stamp'], errors="coerce", format='%d-%m-%Y %H:%M:%S') df['time_stamp'] = pd.to_datetime(df['time_stamp'], errors="coerce") df.dropna(subset=['time_stamp'], inplace=True) df.sort_values(by='time_stamp', inplace=True) df = df[[i for i in self.trans_param.cols_tran.keys() if i in df.columns]] power = df.sample(int(df.shape[0] / 100))['active_power'].median() if power > 10000: df['active_power'] = df['active_power'] / 1000 ## 做数据检测前,羡强行处理有功功率 df = df[df['active_power'] < 5000] rated_power_and_cutout_speed_tuple = read_conf(self.rated_power_and_cutout_speed_map, str(wind_col_name)) if rated_power_and_cutout_speed_tuple is None: rated_power_and_cutout_speed_tuple = (None, None) class_identifiler = ClassIdentifier(wind_turbine_number=wind_col_name, origin_df=df, rated_power=rated_power_and_cutout_speed_tuple[0], cut_out_speed=rated_power_and_cutout_speed_tuple[1]) df = class_identifiler.run() df['year'] = df['time_stamp'].dt.year df['month'] = df['time_stamp'].dt.month df['day'] = df['time_stamp'].dt.day df['time_stamp'] = df['time_stamp'].apply( lambda x: x.strftime('%Y-%m-%d %H:%M:%S')) df['wind_turbine_name'] = str(origin_wind_name) if self.paths_and_table.save_zip: save_path = os.path.join(self.paths_and_table.get_save_path(), str(wind_col_name) + '.csv.gz') else: save_path = os.path.join(self.paths_and_table.get_save_path(), str(wind_col_name) + '.csv') create_file_path(save_path, is_file_path=True) if self.paths_and_table.save_zip: df.to_csv(save_path, compression='gzip', index=False, encoding='utf-8') else: df.to_csv(save_path, index=False, encoding='utf-8') self.set_statistics_data(df) del df trans_print("保存" + str(wind_col_name) + "成功") def mutiprocessing_to_save_file(self): # 开始保存到正式文件 all_tmp_files = read_excel_files(self.paths_and_table.get_read_tmp_path()) # split_count = self.pathsAndTable.multi_pool_count split_count = use_files_get_max_cpu_count(all_tmp_files) all_arrays = split_array(all_tmp_files, split_count) try: for index, arr in enumerate(all_arrays): with multiprocessing.Pool(split_count) as pool: pool.starmap(self.save_to_csv, [(i,) for i in arr]) update_trans_transfer_progress(self.paths_and_table.batch_no, self.paths_and_table.read_type, round(50 + 20 * (index + 1) / len(all_arrays), 2), self.paths_and_table.save_db) except Exception as e: trans_print(traceback.format_exc()) message = "保存文件错误,系统返回错误:" + str(e) raise ValueError(message) def run(self): self.mutiprocessing_to_save_file() update_trans_transfer_progress(self.paths_and_table.batch_no, self.paths_and_table.read_type, 70, self.paths_and_table.save_db)