fengxiang_fengdianchang.py 2.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657
  1. import sys
  2. from multiprocessing import Pool
  3. from os import path
  4. path = path.dirname(path.dirname(path.abspath(__file__)))
  5. print(path)
  6. sys.path.insert(0, path)
  7. print(sys.path)
  8. from utils.file.trans_methods import *
  9. from utils.systeminfo.sysinfo import use_files_get_max_cpu_count
  10. def read_and_save_file(filename):
  11. try:
  12. basename = path.basename(filename)
  13. wind_number = basename.split("_")[0]
  14. df = read_file_to_df(filename, header=1)
  15. df['风机号'] = wind_number
  16. df['描述'] = pd.to_datetime(df['描述'], format='%d-%m-%Y %H:%M:%S')
  17. df.set_index(keys=['描述', '风机号'], inplace=True)
  18. return wind_number, df
  19. except Exception as e:
  20. print(basename, 'error')
  21. raise e
  22. if __name__ == '__main__':
  23. read_path = r'/data/download/collection_data/1进行中/枫香风电场-贵州-大唐/收资数据/枫香风电场收资表/1.10分钟SCADA数据'
  24. save_path = r'/data/download/collection_data/1进行中/枫香风电场-贵州-大唐/清理数据/枫香风电场收资表/1.10分钟SCADA数据'
  25. # read_path = r'D:\trans_data\枫香\收资数据\min'
  26. # save_path = r'D:\trans_data\枫香\清理数据\min'
  27. create_file_path(save_path, False)
  28. all_fils = read_excel_files(read_path)
  29. process_count = use_files_get_max_cpu_count(all_fils)
  30. with Pool(process_count) as pool:
  31. results = pool.starmap(read_and_save_file, [(i,) for i in all_fils])
  32. df_dict = dict()
  33. for result in results:
  34. wind_number, df = result
  35. cols = list(df.columns)
  36. cols.sort()
  37. cols_str = '-'.join(cols)
  38. if wind_number in df_dict.keys():
  39. if cols_str in df_dict[wind_number].keys():
  40. df_dict[wind_number][cols_str] = pd.concat([df_dict[wind_number][cols_str], df], axis=0)
  41. else:
  42. df_dict[wind_number][cols_str] = df
  43. else:
  44. df_dict[wind_number] = {cols_str: df}
  45. for wind_number, cols_dict in df_dict.items():
  46. df = pd.concat(cols_dict.values(), axis=1)
  47. df.sort_index(inplace=True)
  48. df.reset_index(inplace=True)
  49. df.to_csv(path.join(save_path, f"{wind_number}.csv"), encoding="utf-8", index=False)