energy-data-trans/tmp_file/张崾先统计缺失率-分.py

import multiprocessing
import os
import sys

sys.path.insert(0, os.path.abspath(__file__).split("tmp_file")[0])

import pandas as pd

from utils.file.trans_methods import read_file_to_df


def save_percent(value, save_decimal=7):
    return round(value, save_decimal) * 100


def read_and_select(file_path, read_cols):
    result_df = pd.DataFrame()
    df = read_file_to_df(file_path, read_cols=read_cols)
    wind_name = os.path.basename(file_path).split('.')[0]
    df['风机号'] = wind_name
    df = df.query("(startTime>='2023-10-01 00:00:00') & (startTime<'2024-10-01 00:00:00')")
    count = 366 * 24 * 6  # 十分钟数据  2024年366天
    repeat_time_count = df.shape[0] - len(df['startTime'].unique())
    print(wind_name, count, repeat_time_count)
    result_df['风机号'] = [wind_name]
    result_df['重复率'] = [save_percent(repeat_time_count / count)]
    result_df['重复次数'] = [repeat_time_count]
    result_df['总记录数'] = [count]

    for read_col in read_cols:

        if read_col != 'startTime':
            df[read_col] = pd.to_numeric(df[read_col], errors='coerce')
        else:
            df[read_col] = pd.to_datetime(df[read_col], errors='coerce')

    group_df = df.groupby(by=['风机号']).count()
    group_df.reset_index(inplace=True)
    count_df = pd.DataFrame(group_df)
    total_count = count_df[read_cols].values[0].sum()
    print(wind_name, total_count, count * len(read_cols))
    result_df['平均缺失率,单位%'] = [save_percent(1 - total_count / (count * len(read_cols)))]
    result_df['缺失数值'] = ['-'.join([str(count - i) for i in count_df[read_cols].values[0]])]
    del group_df

    error_fengsu_count = df.query("(风速10min < 0) | (风速10min > 80)").shape[0]
    error_yougong_gonglv = df.query("(有功功率 < -200) | (有功功率 > 4800)").shape[0]

    result_df['平均异常率'] = [save_percent((error_fengsu_count + error_yougong_gonglv) / (2 * count))]

    return result_df


if __name__ == '__main__':
    read_cols_str = 'startTime,有功功率,叶轮转速,发电机转速,风速10min,桨叶1角度,桨叶2角度,桨叶3角度,机舱位置,偏航误差,发电机轴承温度,机舱内温度,环境温度,发电机U相温度,发电机V相温度,发电机W相温度'
    read_cols = [i for i in read_cols_str.split(",") if i]
    read_dir = r'/data/download/collection_data/1进行中/张崾先风电场-陕西-华电/收资数据/导出数据2'

    files = os.listdir(read_dir)

    with multiprocessing.Pool(16) as pool:
        dfs = pool.starmap(read_and_select, [(os.path.join(read_dir, i), read_cols) for i in files])

    df = pd.concat(dfs, ignore_index=True)
    df.sort_values(by=['风机号'], inplace=True)

    df.to_csv("张崾先统计-分钟.csv", encoding='utf8', index=False)