修改,不直接引入os使用

app_run.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # @Time    : 2024/6/11
 # @Author  : 魏志亮
-import os
 import sys
+from os import *
 
 from utils.conf.read_conf import yaml_conf, read_conf
 
@@ -46,10 +46,10 @@ if __name__ == '__main__':
     if len(sys.argv) >= 2:
         env = sys.argv[1]
 
-    conf_path = os.path.abspath(f"./conf/etl_config_{env}.yaml")
-    os.environ['ETL_CONF'] = conf_path
+    conf_path = path.abspath(f"./conf/etl_config_{env}.yaml")
+    environ['ETL_CONF'] = conf_path
     yaml_config = yaml_conf(conf_path)
-    os.environ['env'] = env
+    environ['env'] = env
     run_count = int(read_conf(yaml_config, "run_batch_count", 1))
 
     from utils.log.trans_log import trans_print
@@ -58,6 +58,6 @@ if __name__ == '__main__':
     from etl.wind_power.min_sec.MinSecTrans import MinSecTrans
 
     trans_print("所有请求参数:", sys.argv, "env:", env, "最大可执行个数:", run_count)
-    trans_print("配置文件路径:", os.environ.get("ETL_CONF"))
+    trans_print("配置文件路径:", environ.get("ETL_CONF"))
 
     run(run_count=run_count)

etl/common/PathsAndTable.py

@@ -1,5 +1,5 @@
-import os
 import shutil
+from os import path, environ, sep
 
 from service.trans_service import drop_table, creat_min_sec_table, create_warn_fault_table
 from utils.conf.read_conf import *
@@ -19,52 +19,52 @@ class PathsAndTable(object):
         self.multi_pool_count = 6
         self.is_delete_db = False
 
-        yaml_config = yaml_conf(os.environ.get('ETL_CONF'))
+        yaml_config = yaml_conf(environ.get('ETL_CONF'))
 
         save_path_conf = read_conf(yaml_config, "save_path")
         self.tmp_base_path = read_conf(yaml_config, "tmp_base_path", "/tmp")
         if save_path_conf:
-            self.save_path = save_path_conf + os.sep + self.field_name
+            self.save_path = save_path_conf + sep + self.field_name
         else:
             find_index = read_path.find(read_conf(yaml_config, 'etl_origin_path_contain', "etl_origin_path_contain"))
             if find_index == -1:
                 raise Exception("路径未包含原始数据特定字符:" + read_path)
-            self.save_path = read_path[0:find_index] + os.sep + "清理数据"
+            self.save_path = read_path[0:find_index] + sep + "清理数据"
 
         if self.save_path is None:
             raise Exception("未配置保存路径:" + read_path)
 
     def get_save_path(self):
-        return os.path.join(self.save_path, self.batch_no + "_" + self.batch_name, self.read_type)
+        return path.join(self.save_path, self.batch_no + "_" + self.batch_name, self.read_type)
 
     def get_tmp_path(self):
-        return os.path.join(self.tmp_base_path, self.field_name, self.batch_no + "_" + self.batch_name,
-                            self.read_type)
+        return path.join(self.tmp_base_path, self.field_name, self.batch_no + "_" + self.batch_name,
+                         self.read_type)
 
     def get_excel_tmp_path(self):
-        return os.path.join(self.get_tmp_path(), 'excel_tmp' + os.sep)
+        return path.join(self.get_tmp_path(), 'excel_tmp' + sep)
 
     def get_read_tmp_path(self):
-        return os.path.join(self.get_tmp_path(), 'read_tmp')
+        return path.join(self.get_tmp_path(), 'read_tmp')
 
     def get_merge_tmp_path(self, wind_turbine_number=None):
         if wind_turbine_number is None:
-            return os.path.join(self.get_tmp_path(), 'merge_tmp')
+            return path.join(self.get_tmp_path(), 'merge_tmp')
         else:
-            return os.path.join(self.get_tmp_path(), 'merge_tmp', str(wind_turbine_number))
+            return path.join(self.get_tmp_path(), 'merge_tmp', str(wind_turbine_number))
 
     def get_table_name(self):
         return "_".join([self.batch_no, self.read_type])
 
     def delete_batch_files(self):
         trans_print("开始删除已存在的批次文件夹")
-        if os.path.exists(self.get_save_path()):
+        if path.exists(self.get_save_path()):
             shutil.rmtree(self.get_save_path())
         trans_print("删除已存在的批次文件夹")
 
     def delete_tmp_files(self):
         trans_print("开始删除临时文件夹")
-        if os.path.exists(self.get_tmp_path()):
+        if path.exists(self.get_tmp_path()):
             shutil.rmtree(self.get_tmp_path())
         trans_print("删除临时文件夹删除成功")
 

etl/common/SaveToDb.py

@@ -1,6 +1,7 @@
 import multiprocessing
 import traceback
 from os import path
+
 from etl.common.PathsAndTable import PathsAndTable
 from service.plt_service import update_trans_transfer_progress
 from service.trans_service import save_file_to_db

etl/common/UnzipAndRemove.py

@@ -1,6 +1,6 @@
 import multiprocessing
-import os
 import traceback
+from os import *
 
 from etl.common.PathsAndTable import PathsAndTable
 from service.plt_service import update_trans_transfer_progress
@@ -41,7 +41,7 @@ class UnzipAndRemove(object):
     def remove_file_to_tmp_path(self):
         # 读取文件
         try:
-            if os.path.isfile(self.pathsAndTable.read_path):
+            if path.isfile(self.pathsAndTable.read_path):
                 all_files = [self.pathsAndTable.read_path]
             else:
                 all_files = read_files(self.pathsAndTable.read_path)

etl/wind_power/fault_warn/FaultWarnTrans.py

@@ -1,11 +1,11 @@
-import os
+from os import *
 
 import numpy as np
 import pandas as pd
 
 from etl.common.BaseDataTrans import BaseDataTrans
 from service.plt_service import update_trans_status_error
-from service.trans_service import get_fault_warn_conf
+from service.trans_service import get_fault_warn_conf, get_trans_exec_code
 from utils.conf.read_conf import read_conf
 from utils.file.trans_methods import read_excel_files, read_file_to_df, create_file_path, valid_eval
 from utils.log.trans_log import trans_print
@@ -112,13 +112,18 @@ class FaultWarnTrans(BaseDataTrans):
                 df['time_diff'] = (df['end_time'] - df['begin_time']).dt.total_seconds()
                 df.loc[df['time_diff'] < 0, 'time_diff'] = np.nan
 
+            # 如果有需要处理的,先进行代码处理,在保存文件
+            exec_code = get_trans_exec_code(self.batch_no, self.read_type)
+            if exec_code:
+                exec(exec_code)
+
             # 根绝开始时间进行排序
             df.sort_values(by=['wind_turbine_number', 'begin_time'], inplace=True)
 
             if self.save_zip:
-                save_path = os.path.join(self.pathsAndTable.get_save_path(), str(self.batch_name) + '.csv.gz')
+                save_path = path.join(self.pathsAndTable.get_save_path(), str(self.batch_name) + '.csv.gz')
             else:
-                save_path = os.path.join(self.pathsAndTable.get_save_path(), str(self.batch_name) + '.csv')
+                save_path = path.join(self.pathsAndTable.get_save_path(), str(self.batch_name) + '.csv')
 
             create_file_path(save_path, is_file_path=True)
             if self.save_zip:

etl/wind_power/min_sec/MinSecTrans.py

@@ -4,9 +4,9 @@
 import multiprocessing
 
 from etl.common.BaseDataTrans import BaseDataTrans
-from etl.wind_power.min_sec.TransParam import TransParam
 from etl.wind_power.min_sec.ReadAndSaveTmp import ReadAndSaveTmp
 from etl.wind_power.min_sec.StatisticsAndSaveFile import StatisticsAndSaveFile
+from etl.wind_power.min_sec.TransParam import TransParam
 from service.plt_service import update_trans_status_success, update_trans_status_error
 from service.trans_service import batch_statistics, get_min_sec_conf
 from utils.conf.read_conf import read_conf
@@ -119,7 +119,3 @@ class MinSecTrans(BaseDataTrans):
                                                 None, None,
                                                 None, self.save_db)
 
-
-if __name__ == '__main__':
-    test = MinSecTrans(batch_no="WOF053600062-WOB000009", read_type="minute", save_db=False)
-    test.run()

etl/wind_power/min_sec/ReadAndSaveTmp.py

@@ -1,13 +1,13 @@
 import base64
 import datetime
 import multiprocessing
-import os
 import traceback
+from os import *
 
 import pandas as pd
 
-from etl.wind_power.min_sec import TransParam
 from etl.common.PathsAndTable import PathsAndTable
+from etl.wind_power.min_sec import TransParam
 from service.plt_service import update_trans_transfer_progress
 from utils.file.trans_methods import read_excel_files, split_array, del_blank, \
     create_file_path, read_file_to_df, valid_eval
@@ -26,7 +26,7 @@ class ReadAndSaveTmp(object):
 
     def _save_to_tmp_csv_by_name(self, df, name):
         save_name = str(name) + '.csv'
-        save_path = os.path.join(self.pathsAndTable.get_read_tmp_path(), save_name)
+        save_path = path.join(self.pathsAndTable.get_read_tmp_path(), save_name)
         create_file_path(save_path, is_file_path=True)
 
         with self.lock:
@@ -65,7 +65,7 @@ class ReadAndSaveTmp(object):
                 else:
                     contains_name = False
                     self.exist_wind_names.append(exist_name)
-                save_path = os.path.join(merge_path, csv_name)
+                save_path = path.join(merge_path, csv_name)
                 if contains_name:
                     df.to_csv(save_path, index=False, encoding='utf-8', mode='a',
                               header=False)
@@ -158,8 +158,8 @@ class ReadAndSaveTmp(object):
                                                round(20 + 20 * (index + 1) / len(all_arrays), 2),
                                                self.pathsAndTable.save_db)
 
-            dirs = [os.path.join(self.pathsAndTable.get_merge_tmp_path(), dir_name) for dir_name in
-                    os.listdir(self.pathsAndTable.get_merge_tmp_path())]
+            dirs = [path.join(self.pathsAndTable.get_merge_tmp_path(), dir_name) for dir_name in
+                    listdir(self.pathsAndTable.get_merge_tmp_path())]
             dir_total_size = get_dir_size(dirs[0])
             split_count = max_file_size_get_max_cpu_count(dir_total_size)
             all_arrays = split_array(dirs, split_count)
@@ -271,7 +271,7 @@ class ReadAndSaveTmp(object):
 
             for k, v in trans_dict.items():
                 if k.startswith("$file"):
-                    file = ".".join(os.path.basename(file_path).split(".")[0:-1])
+                    file = ".".join(path.basename(file_path).split(".")[0:-1])
                     if k == "$file":
                         ks = k.split("|")
                         bool_contains = False
@@ -308,7 +308,7 @@ class ReadAndSaveTmp(object):
                     datas = str(k.split(",")[1].replace("$file_date", "").replace("[", "").replace("]", "")).split(":")
                     if len(datas) != 2:
                         raise Exception("字段映射出现错误 :" + str(trans_dict))
-                    file = ".".join(os.path.basename(file_path).split(".")[0:-1])
+                    file = ".".join(path.basename(file_path).split(".")[0:-1])
                     date_str = str(file[int(datas[0]):int(datas[1])]).strip()
                     df[v] = df[k.split(",")[0]].apply(lambda x: date_str + " " + str(x))
 
@@ -322,8 +322,8 @@ class ReadAndSaveTmp(object):
                     if not bool_contains:
                         cengshu = int(str(ks[0].replace("$folder", "").replace("[", "").replace("]", "")))
                         for i in range(cengshu):
-                            folder = os.path.dirname(folder)
-                        df[v] = str(str(folder).split(os.sep)[-1]).strip()
+                            folder = path.dirname(folder)
+                        df[v] = str(str(folder).split(sep)[-1]).strip()
                 elif k.startswith("$sheet_name"):
                     df[v] = df['sheet_name']
 

etl/wind_power/min_sec/StatisticsAndSaveFile.py

@@ -1,15 +1,16 @@
 import datetime
 import multiprocessing
-import os
 import traceback
+from os import path
 
-import pandas as pd
 import numpy as np
+import pandas as pd
 
-from etl.wind_power.min_sec import TransParam
 from etl.common.PathsAndTable import PathsAndTable
+from etl.wind_power.min_sec import TransParam
 from etl.wind_power.min_sec.ClassIdentifier import ClassIdentifier
 from service.plt_service import update_trans_transfer_progress
+from service.trans_service import get_trans_exec_code
 from utils.conf.read_conf import read_conf
 from utils.df_utils.util import get_time_space
 from utils.file.trans_methods import create_file_path, read_excel_files, read_file_to_df, split_array
@@ -117,6 +118,13 @@ class StatisticsAndSaveFile(object):
         if rated_power_and_cutout_speed_tuple is None:
             rated_power_and_cutout_speed_tuple = (None, None)
 
+        # 如果有需要处理的,先进行代码处理,在进行打标签
+        exec_code = get_trans_exec_code(self.paths_and_table.batch_no, self.paths_and_table.read_type)
+        if exec_code:
+            if 'import ' in exec_code:
+                raise Exception("执行代码不支持导入包")
+            exec(exec_code)
+
         class_identifiler = ClassIdentifier(wind_turbine_number=wind_col_name, origin_df=df,
                                             rated_power=rated_power_and_cutout_speed_tuple[0],
                                             cut_out_speed=rated_power_and_cutout_speed_tuple[1])
@@ -131,9 +139,9 @@ class StatisticsAndSaveFile(object):
         df['wind_turbine_name'] = str(origin_wind_name)
 
         if self.paths_and_table.save_zip:
-            save_path = os.path.join(self.paths_and_table.get_save_path(), str(wind_col_name) + '.csv.gz')
+            save_path = path.join(self.paths_and_table.get_save_path(), str(wind_col_name) + '.csv.gz')
         else:
-            save_path = os.path.join(self.paths_and_table.get_save_path(), str(wind_col_name) + '.csv')
+            save_path = path.join(self.paths_and_table.get_save_path(), str(wind_col_name) + '.csv')
         create_file_path(save_path, is_file_path=True)
         if self.paths_and_table.save_zip:
             df.to_csv(save_path, compression='gzip', index=False, encoding='utf-8')

etl/wind_power/wave/WaveTrans.py

@@ -1,8 +1,5 @@
-import datetime
 import json
 import multiprocessing
-from os.path import basename, dirname
-import pandas as pd
 
 from service.plt_service import get_all_wind
 from service.trans_service import get_wave_conf, save_df_to_db, get_or_create_wave_table, \

service/trans_service.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # @Time    : 2024/6/7
 # @Author  : 魏志亮
-import os
 import traceback
+from os import *
 
 import pandas as pd
 
@@ -163,7 +163,7 @@ def clear_table(table_name, save_db=True):
 
 
 def save_file_to_db(table_name: str, file: str, batch_count=100000):
-    base_name = os.path.basename(file)
+    base_name = path.basename(file)
     try:
         for i, df in enumerate(pd.read_csv(file, chunksize=batch_count)):
             # df.to_sql(table_name, engine, if_exists='append', index=False)
@@ -264,11 +264,21 @@ def delete_exist_wave_data(table_name, ids):
         trans.execute(delete_sql, array)
 
 
+def get_trans_exec_code(batch_no, type):
+    query_sql = f"SELECT * from batch_exec_code t where t.batch_no = '{batch_no}' and type='{type}' and t.`status` = 1 limit 1"
+    res = trans.execute(query_sql)
+    if type(res) == tuple:
+        return None
+    exec_code = res[0]['exec_code']
+    trans_print("批次", batch_no, '类型', type, '获取到执行代码:', exec_code)
+    return exec_code
+
+
 if __name__ == '__main__':
     # path_prix = r"/data/download/collection_data/2完成/招远风电场-山东-大唐/清理数据/WOF063100040-WOB00013/second"
     # files = ["WOG00030.csv", "WOG00034.csv"]
     # for path in files:
-    #     save_file_to_db("WOF063100040-WOB00013_second", path_prix + os.sep + path, batch_count=100000)
+    #     save_file_to_db("WOF063100040-WOB00013_second", path_prix + sep + path, batch_count=100000)
 
     # sql = """
     # SELECT wind_turbine_number, time_stamp, wind_velocity, active_power

test_run_local.py

@@ -2,9 +2,9 @@
 # @Time    : 2024/6/11
 # @Author  : 魏志亮
 import datetime
-import os
 import sys
 import traceback
+from os import *
 
 from utils.conf.read_conf import yaml_conf, read_conf
 
@@ -45,10 +45,10 @@ if __name__ == '__main__':
     if len(sys.argv) >= 2:
         env = sys.argv[1]
 
-    conf_path = os.path.abspath(f"./conf/etl_config_{env}.yaml")
-    os.environ['ETL_CONF'] = conf_path
+    conf_path = path.abspath(f"./conf/etl_config_{env}.yaml")
+    environ['ETL_CONF'] = conf_path
     yaml_config = yaml_conf(conf_path)
-    os.environ['env'] = env
+    environ['env'] = env
     run_count = int(read_conf(yaml_config, "run_batch_count", 1))
 
     from utils.log.trans_log import trans_print

test_run_local_piliang.py

@@ -2,11 +2,9 @@
 # @Time    : 2024/6/11
 # @Author  : 魏志亮
 import datetime
-import os
 import sys
 import traceback
-
-import pandas as pd
+from os import *
 
 from utils.conf.read_conf import yaml_conf, read_conf
 
@@ -47,10 +45,10 @@ if __name__ == '__main__':
     if len(sys.argv) >= 2:
         env = sys.argv[1]
 
-    conf_path = os.path.abspath(f"./conf/etl_config_{env}.yaml")
-    os.environ['ETL_CONF'] = conf_path
+    conf_path = path.abspath(f"./conf/etl_config_{env}.yaml")
+    environ['ETL_CONF'] = conf_path
     yaml_config = yaml_conf(conf_path)
-    os.environ['env'] = env
+    environ['env'] = env
     run_count = int(read_conf(yaml_config, "run_batch_count", 1))
 
     from utils.log.trans_log import trans_print

tmp_file/baiyushan_20240906.py

@@ -1,6 +1,5 @@
-import datetime
-import os
 from multiprocessing import Pool
+from os import *
 
 import chardet
 import pandas as pd
@@ -55,11 +54,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -83,10 +82,10 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def read_status(status_path):
@@ -155,20 +154,20 @@ def select_data_and_save(name, fault_df, origin_df):
 
     name = name.replace('#', 'F')
     df.drop_duplicates(inplace=True)
-    df.to_csv(save_path + os.sep + name + '.csv', index=False, encoding='utf8')
+    df.to_csv(save_path + sep + name + '.csv', index=False, encoding='utf8')
 
 
 if __name__ == '__main__':
     base_path = r'/data/download/白玉山/需要整理的数据'
-    save_path = base_path + os.sep + 'sele_data_202409261135'
+    save_path = base_path + sep + 'sele_data_202409261135'
     create_file_path(save_path)
-    status_df = read_status(base_path + os.sep + '设备状态')
-    fault_df = read_fault_data(base_path + os.sep + '故障')
-    data_df = read_10min_data(base_path + os.sep + '十分钟')
+    status_df = read_status(base_path + sep + '设备状态')
+    fault_df = read_fault_data(base_path + sep + '故障')
+    data_df = read_10min_data(base_path + sep + '十分钟')
 
-    status_df.to_csv(base_path + os.sep + '设备状态' + '.csv', index=False, encoding='utf8')
-    fault_df.to_csv(base_path + os.sep + '故障' + '.csv', index=False, encoding='utf8')
-    data_df.to_csv(base_path + os.sep + '十分钟' + '.csv', index=False, encoding='utf8')
+    status_df.to_csv(base_path + sep + '设备状态' + '.csv', index=False, encoding='utf8')
+    fault_df.to_csv(base_path + sep + '故障' + '.csv', index=False, encoding='utf8')
+    data_df.to_csv(base_path + sep + '十分钟' + '.csv', index=False, encoding='utf8')
 
     print(status_df.shape)
     print(fault_df.shape)
@@ -182,9 +181,9 @@ if __name__ == '__main__':
         fault_list.append(status_df[con1 & con2]['处理后时间'].min())
     fault_df['结束时间'] = fault_list
 
-    status_df.to_csv(base_path + os.sep + '设备状态' + '.csv', index=False, encoding='utf8')
-    fault_df.to_csv(base_path + os.sep + '故障' + '.csv', index=False, encoding='utf8')
-    data_df.to_csv(base_path + os.sep + '十分钟' + '.csv', index=False, encoding='utf8')
+    status_df.to_csv(base_path + sep + '设备状态' + '.csv', index=False, encoding='utf8')
+    fault_df.to_csv(base_path + sep + '故障' + '.csv', index=False, encoding='utf8')
+    data_df.to_csv(base_path + sep + '十分钟' + '.csv', index=False, encoding='utf8')
 
     names = set(fault_df['设备名称'])
     fault_map = dict()

tmp_file/cp_online_data_to_other.py

@@ -1,7 +1,7 @@
 import datetime
 import multiprocessing
-import os
 import shutil
+from os import *
 
 not_move_dir = ["乌梅山风电场-江西-大唐",
                 "诺木洪风电场-甘肃-华电",
@@ -18,12 +18,12 @@ save_base_dir = r"/data/download/datang_shangxian"
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
+    for item in listdir(path):
         if item not in not_move_dir:
-            item_path = os.path.join(path, item)
-            if os.path.isdir(item_path):
+            item_path = path.join(path, item)
+            if path.isdir(item_path):
                 __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-            elif os.path.isfile(item_path):
+            elif path.isfile(item_path):
                 if path not in directory_dict:
                     directory_dict[path] = []
 
@@ -36,7 +36,7 @@ def __build_directory_dict(directory_dict, path, filter_types=None):
 
 # 读取路径下所有的excel文件
 def read_excel_files(read_path):
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
 
     directory_dict = {}
@@ -47,7 +47,7 @@ def read_excel_files(read_path):
 
 # 读取路径下所有的文件
 def read_files(read_path):
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
     directory_dict = {}
     __build_directory_dict(directory_dict, read_path, filter_types=['xls', 'xlsx', 'csv', 'gz', 'zip', 'rar'])
@@ -63,10 +63,10 @@ def create_file_path(path, is_file_path=False):
     :param is_file_path: 传入的path是否包含具体的文件名
     """
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def copy_to_new(from_path):

tmp_file/fengxiang_fengdianchang.py

@@ -1,8 +1,7 @@
+import sys
 from multiprocessing import Pool
-
-import sys, os
-
-path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+from os import path
+path = path.dirname(path.dirname(path.abspath(__file__)))
 print(path)
 sys.path.insert(0, path)
 print(sys.path)
@@ -13,7 +12,7 @@ from utils.systeminfo.sysinfo import use_files_get_max_cpu_count
 
 def read_and_save_file(filename):
     try:
-        basename = os.path.basename(filename)
+        basename = path.basename(filename)
         wind_number = basename.split("_")[0]
         df = read_file_to_df(filename, header=1)
         df['风机号'] = wind_number
@@ -55,4 +54,4 @@ if __name__ == '__main__':
         df = pd.concat(cols_dict.values(), axis=1)
         df.sort_index(inplace=True)
         df.reset_index(inplace=True)
-        df.to_csv(os.path.join(save_path, f"{wind_number}.csv"), encoding="utf-8", index=False)
+        df.to_csv(path.join(save_path, f"{wind_number}.csv"), encoding="utf-8", index=False)

tmp_file/gradio_web.py

@@ -7,7 +7,7 @@ import gradio as gr
 import yaml
 
 from service.plt_service import get_all_wind_company
-from service.trans_service import get_min_sec_conf, get_min_sec_conf_test
+from service.trans_service import get_min_sec_conf_test
 
 
 # from utils.db.trans_mysql import *

tmp_file/hebing_matlib_result.py

@@ -1,4 +1,5 @@
-import os
+from os import *
+
 import pandas as pd
 
 read_path = r"D:\data\电量损失及散点图"
@@ -7,16 +8,16 @@ df = pd.DataFrame()
 cols = ['风机', '应发电量', '实发电量', '停机损失电量', '坏点+限电损失电量', '性能损失电量', '坏点损失电量', '限电损失电量', '超发电量', '应发电量百分比', '实发电量百分比',
         '停机损失电量百分比', '坏点+限电损失电量百分比', '性能损失电量百分比', '坏点损失电量百分比', '限电损失电量百分比', '超发电量百分比', '平均风速', '可利用率']
 
-for root, dir, files in os.walk(read_path):
+for root, dir, files in walk(read_path):
     if files:
-        base_name = os.path.basename(root)
+        base_name = path.basename(root)
         wind_df = pd.DataFrame()
         print(root)
-        df1 = pd.read_excel(os.path.join(root, "EPPer.xls"), usecols=['应发电量百分比', '实发电量百分比',
+        df1 = pd.read_excel(path.join(root, "EPPer.xls"), usecols=['应发电量百分比', '实发电量百分比',
                                                                      '停机损失电量百分比', '坏点+限电损失电量百分比', '性能损失电量百分比',
                                                                      '坏点损失电量百分比',
                                                                      '限电损失电量百分比', '超发电量百分比', '平均风速', '可利用率'])
-        df2 = pd.read_excel(os.path.join(root, "EPKW.xls"),
+        df2 = pd.read_excel(path.join(root, "EPKW.xls"),
                             usecols=['应发电量', '实发电量', '停机损失电量', '坏点+限电损失电量', '性能损失电量', '坏点损失电量', '限电损失电量', '超发电量'])
         wind_df = pd.concat([df1, df2], axis=1)
         wind_df['风机'] = base_name

tmp_file/hebing_muti_batch.py

@@ -1,10 +1,10 @@
 import multiprocessing
+import sys
+from os import *
 
-import os
 import pandas as pd
-import sys
 
-sys.path.insert(0, os.path.abspath(__file__).split("tmp_file")[0])
+sys.path.insert(0, path.abspath(__file__).split("tmp_file")[0])
 
 
 def hebing_and_save(new_batch_save_path, name, paths):
@@ -16,7 +16,7 @@ def hebing_and_save(new_batch_save_path, name, paths):
     df.sort_values(by=['time_stamp'], inplace=True)
 
     create_file_path(new_batch_save_path)
-    df.to_csv(os.path.join(new_batch_save_path, name), index=False, encoding='utf8')
+    df.to_csv(path.join(new_batch_save_path, name), index=False, encoding='utf8')
 
 
 if __name__ == '__main__':
@@ -27,10 +27,10 @@ if __name__ == '__main__':
 
     from utils.conf.read_conf import yaml_conf
 
-    conf_path = os.path.abspath(__file__).split("tmp_file")[0] + f"/conf/etl_config_{env}.yaml"
-    os.environ['ETL_CONF'] = conf_path
+    conf_path = path.abspath(__file__).split("tmp_file")[0] + f"/conf/etl_config_{env}.yaml"
+    environ['ETL_CONF'] = conf_path
     yaml_config = yaml_conf(conf_path)
-    os.environ['env'] = env
+    environ['env'] = env
 
     from utils.file.trans_methods import read_file_to_df, create_file_path
 
@@ -61,17 +61,17 @@ if __name__ == '__main__':
             raise Exception("No exec process")
 
         read_paths.append(exec_process.pathsAndTable.get_save_path())
-        new_batch_save_path = os.path.join(exec_process.pathsAndTable.save_path, save_batch + "_" + save_batch_name,
+        new_batch_save_path = path.join(exec_process.pathsAndTable.save_path, save_batch + "_" + save_batch_name,
                                            trans_type)
 
     file_dict = dict()
 
     for read_path in read_paths:
-        for file in os.listdir(read_path):
+        for file in listdir(read_path):
             if file in file_dict:
-                file_dict[file].append(os.path.join(read_path, file))
+                file_dict[file].append(path.join(read_path, file))
             else:
-                file_dict[file] = [os.path.join(read_path, file)]
+                file_dict[file] = [path.join(read_path, file)]
 
     with multiprocessing.Pool(len(file_dict.keys())) as pool:
         pool.starmap(hebing_and_save, [(new_batch_save_path, name, paths) for name, paths in file_dict.items()])

tmp_file/organize_xinhua_files.py

@@ -1,7 +1,7 @@
-import multiprocessing
 import datetime
-import os
+import multiprocessing
 import warnings
+from os import *
 
 import numpy as np
 import pandas as pd
@@ -11,11 +11,11 @@ warnings.filterwarnings("ignore")
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -28,7 +28,7 @@ def __build_directory_dict(directory_dict, path, filter_types=None):
 
 # 读取路径下所有的excel文件
 def read_excel_files(read_path):
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
 
     directory_dict = {}
@@ -45,10 +45,10 @@ def create_file_path(path, is_file_path=False):
     :param is_file_path: 传入的path是否包含具体的文件名
     """
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def boolean_is_check_data(df_cols):
@@ -96,7 +96,7 @@ def save_to_file(dfs, wind_name, save_path='', param='', is_check=False, all_col
     loss_cols.sort()
 
     loss_cols.insert(0, wind_name)
-    loss_cols.insert(0, os.path.basename(save_path) + '-' + param)
+    loss_cols.insert(0, path.basename(save_path) + '-' + param)
 
     result_data_list.append(loss_cols)
 
@@ -104,7 +104,7 @@ def save_to_file(dfs, wind_name, save_path='', param='', is_check=False, all_col
         if col not in df.columns:
             df[col] = np.nan
 
-    df.to_csv(os.path.join(save_path, param, wind_name + '.csv'), encoding='utf8', index=False)
+    df.to_csv(path.join(save_path, param, wind_name + '.csv'), encoding='utf8', index=False)
 
 
 if __name__ == '__main__':

tmp_file/organize_xinhua_files_data.py

@@ -1,9 +1,8 @@
-import multiprocessing
 import datetime
-import os
+import multiprocessing
 import warnings
+from os import *
 
-import numpy as np
 import pandas as pd
 
 warnings.filterwarnings("ignore")
@@ -11,11 +10,11 @@ warnings.filterwarnings("ignore")
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
             if filter_types is None or len(filter_types) == 0:
@@ -27,7 +26,7 @@ def __build_directory_dict(directory_dict, path, filter_types=None):
 
 # 读取路径下所有的excel文件
 def read_excel_files(read_path):
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
     directory_dict = {}
     __build_directory_dict(directory_dict, read_path, filter_types=['xls', 'xlsx', 'csv', 'gz'])
@@ -42,9 +41,9 @@ def create_file_path(path, is_file_path=False):
     :param is_file_path: 传入的path是否包含具体的文件名
     """
     if is_file_path:
-        path = os.path.dirname(path)
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+        path = path.dirname(path)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def boolean_is_check_data(df_cols, need_valid=True):
@@ -134,7 +133,7 @@ def sae_to_csv(wind_name, df):
                 if col != origin:
                     del df[col]
 
-        df.to_csv(os.path.join(save_path, wind_name + '.csv'), encoding='utf8')
+        df.to_csv(path.join(save_path, wind_name + '.csv'), encoding='utf8')
 
     except Exception as e:
         print(wind_name, df.columns)

tmp_file/orgranize_hongyang.py

@@ -1,7 +1,7 @@
 import copy
 import multiprocessing
-import os
 import warnings
+from os import *
 
 import chardet
 import pandas as pd
@@ -16,11 +16,11 @@ save_dir = r'D:\data\红阳\红阳秒级分测点\整理'
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -33,7 +33,7 @@ def __build_directory_dict(directory_dict, path, filter_types=None):
 
 # 读取路径下所有的excel文件
 def read_excel_files(read_path):
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
 
     directory_dict = {}
@@ -94,4 +94,4 @@ if __name__ == '__main__':
                 wind_name_df[wind_name] = query_df
 
     for wind_name, df in wind_name_df.items():
-        df.to_csv(os.path.join(save_dir, wind_name + '#.csv'), index=False, encoding='utf8')
+        df.to_csv(path.join(save_dir, wind_name + '#.csv'), index=False, encoding='utf8')

tmp_file/power_derating.py

@@ -1,7 +1,8 @@
 import multiprocessing
-import os
+from os import *
 
 import matplotlib
+
 matplotlib.use('Agg')
 matplotlib.rcParams['font.family'] = 'SimHei'
 matplotlib.rcParams['font.sans-serif'] = ['SimHei']
@@ -15,7 +16,7 @@ import pandas as pd
 
 
 def select_data(file, curve_wv, curve_ap, save_path):
-    name = os.path.basename(file).split("@")[0]
+    name = path.basename(file).split("@")[0]
     try:
         df = read_file_to_df(file)
         df.dropna(subset=['有功功率 kW均值', '风速 m/s均值', '有功功率设定 kW均值'], inplace=True)
@@ -49,7 +50,7 @@ def select_data(file, curve_wv, curve_ap, save_path):
         ap_gt_0_df['marker'] = ap_gt_0_in
         df = pd.concat([ap_gt_0_df, ap_le_0_df])
 
-        df.to_csv(os.path.join(save_path, name + '.csv'), index=False, encoding='utf-8')
+        df.to_csv(path.join(save_path, name + '.csv'), index=False, encoding='utf-8')
 
         df = df[['时间', '风速 m/s均值', '有功功率 kW均值', '有功功率设定 kW均值', 'marker']]
 
@@ -67,10 +68,10 @@ def select_data(file, curve_wv, curve_ap, save_path):
             plt.ylabel('有功功率均值')
 
             # 保存
-            plt.savefig(os.path.join(save_path, name + '均值.png'))
+            plt.savefig(path.join(save_path, name + '均值.png'))
 
     except Exception as e:
-        print(os.path.basename(file), "出错", str(e))
+        print(path.basename(file), "出错", str(e))
         raise e
 
 

tmp_file/power_derating_biaozhun.py

@@ -1,8 +1,7 @@
-import os
+from os import *
 
 import matplotlib
 import numpy as np
-from matplotlib import pyplot as plt
 
 from utils.draw.draw_file import scatter
 
@@ -73,19 +72,19 @@ if __name__ == '__main__':
     wind_power_df_class = ContractPowerCurve(wind_power_df)
 
     for file in all_files:
-        name = os.path.basename(file).split("@")[0]
+        name = path.basename(file).split("@")[0]
         try:
             df = read_file_to_df(file)
             df = marker_active_power(wind_power_df_class, df)
             df = df[df['marker'] == 1]
-            df.to_csv(os.path.join(save_path, name + '.csv'), index=False, encoding='utf-8')
+            df.to_csv(path.join(save_path, name + '.csv'), index=False, encoding='utf-8')
 
             # 使用scatter函数绘制散点图
             if not df.empty:
                 scatter(name, x_label='风速均值', y_label='有功功率均值', x_values=df['风速 m/s均值'].values,
                         y_values=df['有功功率 kW均值'].values, color='green',
-                        save_file_path=os.path.join(save_path, name + '均值.png'))
+                        save_file_path=path.join(save_path, name + '均值.png'))
 
         except Exception as e:
-            print(os.path.basename(file), "出错", str(e))
+            print(path.basename(file), "出错", str(e))
             raise e

tmp_file/power_derating_for_chunlin.py

@@ -1,4 +1,4 @@
-import os
+from os import *
 
 import matplotlib
 import numpy as np
@@ -78,7 +78,7 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
         print('文件读取成功', file_path, '文件数量', df.shape)
     except Exception as e:
         print('读取文件出错', file_path, str(e))
-        message = '文件:' + os.path.basename(file_path) + ',' + str(e)
+        message = '文件:' + path.basename(file_path) + ',' + str(e)
         raise ValueError(message)
 
     return df
@@ -86,11 +86,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -122,10 +122,10 @@ class ContractPowerCurve(object):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def scatter(title, x_label, y_label, x_values, y_values, color='blue', size=10, save_file_path=''):
@@ -193,21 +193,21 @@ if __name__ == '__main__':
     wind_power_df_class = ContractPowerCurve(wind_power_df)
 
     for file in all_files:
-        name = os.path.basename(file).split("@")[0]
+        name = path.basename(file).split("@")[0]
         try:
             df = read_file_to_df(file)
             df = marker_active_power(wind_power_df_class, df)
             df = df[df['marker'] == 1]
             # 保存筛选后数据
             name = name.replace('HD', 'HD2')
-            df.to_csv(os.path.join(save_path, name + '.csv'), index=False, encoding='utf-8')
+            df.to_csv(path.join(save_path, name + '.csv'), index=False, encoding='utf-8')
 
             # 使用scatter函数绘制散点图
             if not df.empty:
                 scatter(name, x_label='风速均值', y_label='有功功率均值', x_values=df['风速 m/s均值'].values,
                         y_values=df['有功功率 kW均值'].values, color='green',
-                        save_file_path=os.path.join(save_path, name + '均值.png'))
+                        save_file_path=path.join(save_path, name + '均值.png'))
 
         except Exception as e:
-            print(os.path.basename(file), "出错", str(e))
+            print(path.basename(file), "出错", str(e))
             raise e

tmp_file/pv_youxiaoxing.py

@@ -1,9 +1,7 @@
 import multiprocessing
-import os
+from os import *
 
 import matplotlib
-import numpy as np
-from matplotlib import pyplot as plt
 
 matplotlib.use('Agg')
 matplotlib.rcParams['font.family'] = 'SimHei'  # 或者 'Microsoft YaHei'
@@ -108,7 +106,7 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
         print('文件读取成功', file_path, '文件数量', df.shape)
     except Exception as e:
         print('读取文件出错', file_path, str(e))
-        message = '文件:' + os.path.basename(file_path) + ',' + str(e)
+        message = '文件:' + path.basename(file_path) + ',' + str(e)
         raise ValueError(message)
 
     return df
@@ -116,11 +114,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -144,10 +142,10 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def time_biaozhun(df):
@@ -235,7 +233,7 @@ def calc(df, file_name):
         error_df = error_df[error_df_cols]
         lose_df = lose_df[lose_df_cols]
     except Exception as e:
-        print("异常文件", os.path.basename(file_name))
+        print("异常文件", path.basename(file_name))
         raise e
 
     return error_df, lose_df
@@ -243,7 +241,7 @@ def calc(df, file_name):
 
 def run(file_path):
     df = read_file_to_df(file_path)
-    return calc(df, os.path.basename(file_path))
+    return calc(df, path.basename(file_path))
 
 
 if __name__ == '__main__':
@@ -259,6 +257,6 @@ if __name__ == '__main__':
 
     error_df = pd.concat([df[0] for df in df_arrys])
     lose_df = pd.concat([df[1] for df in df_arrys])
-    with pd.ExcelWriter(os.path.join(save_path, "玉湖光伏数据统计.xlsx")) as writer:
+    with pd.ExcelWriter(path.join(save_path, "玉湖光伏数据统计.xlsx")) as writer:
         error_df.to_excel(writer, sheet_name='error_percent', index=False)
         lose_df.to_excel(writer, sheet_name='lose_percent', index=False)

tmp_file/qinghai-nuomuhong-guifan.py

@@ -4,13 +4,12 @@ Spyder 编辑器
 
 这是一个临时脚本文件。
 """
-import copy
 import datetime
 import multiprocessing
-import os
+from os import *
 
-import pandas as pd
 import numpy as np
+import pandas as pd
 
 dianjian_str = """
 wind_turbine_number		
@@ -108,7 +107,7 @@ def read_df(file_path):
 
 
 def get_wind_name_files(path):
-    files = os.listdir(path)
+    files = listdir(path)
     return files
 
 
@@ -117,7 +116,7 @@ def combine_df(save_path, file):
     df = read_df(file)
     print("读取", file, df.shape)
     df.replace("-", np.nan,inplace=True)
-    df.to_csv(os.path.join(save_path, os.path.basename(file)), encoding='utf-8', index=False)
+    df.to_csv(path.join(save_path, path.basename(file)), encoding='utf-8', index=False)
 
     print('整理完成', '耗时:', (datetime.datetime.now() - begin).seconds)
 
@@ -128,8 +127,8 @@ if __name__ == '__main__':
 
     # read_path = r'D:\trans_data\诺木洪\收资数据\min-666'
     # save_path = r'D:\trans_data\诺木洪\清理数据\min-666'
-    if not os.path.exists(save_path):
-        os.makedirs(save_path, exist_ok=True)
+    if not path.exists(save_path):
+        makedirs(save_path, exist_ok=True)
 
     with multiprocessing.Pool(20) as pool:
-        pool.starmap(combine_df, [(save_path, read_path + os.sep + file) for file in os.listdir(read_path)])
+        pool.starmap(combine_df, [(save_path, read_path + sep + file) for file in listdir(read_path)])

tmp_file/qinghai-nuomuhong.py

@@ -7,10 +7,10 @@ Spyder 编辑器
 import copy
 import datetime
 import multiprocessing
-import os
+from os import *
 
-import pandas as pd
 import numpy as np
+import pandas as pd
 
 dianjian_str = """
 wind_turbine_number		
@@ -108,11 +108,11 @@ def read_df(file_path):
 
 
 def get_wind_name_files(path):
-    files = os.listdir(path)
+    files = listdir(path)
 
     wind_files_map = dict()
     for file in files:
-        full_file = os.path.join(path, file)
+        full_file = path.join(path, file)
         file_datas = str(file).split("@")
         key = file_datas[0].replace("HD", "HD2")
         if key in wind_files_map.keys():
@@ -143,7 +143,7 @@ def combine_df(save_path, wind_name, files):
             df[col] = np.nan
 
     df = df[dianjian_dict.keys()]
-    df.to_csv(os.path.join(save_path, wind_name + ".csv"), encoding='utf-8', index=False)
+    df.to_csv(path.join(save_path, wind_name + ".csv"), encoding='utf-8', index=False)
 
     print(wind_name, '整理完成', '耗时:', (datetime.datetime.now() - begin).seconds)
 
@@ -154,8 +154,8 @@ if __name__ == '__main__':
 
     # read_path = r'D:\trans_data\诺木洪\收资数据\min'
     # save_path = r'D:\trans_data\诺木洪\清理数据\min'
-    if not os.path.exists(save_path):
-        os.makedirs(save_path, exist_ok=True)
+    if not path.exists(save_path):
+        makedirs(save_path, exist_ok=True)
     wind_files_map = get_wind_name_files(read_path)
 
     with multiprocessing.Pool(20) as pool:

tmp_file/read_and_draw_png.py

@@ -1,5 +1,5 @@
 import multiprocessing
-import os
+from os import *
 
 from etl.wind_power.min_sec.ClassIdentifier import ClassIdentifier
 from utils.draw.draw_file import scatter
@@ -7,7 +7,7 @@ from utils.file.trans_methods import read_file_to_df
 
 
 def draw(file, fengchang='测试'):
-    name = os.path.basename(file).split('.')[0]
+    name = path.basename(file).split('.')[0]
     df = read_file_to_df(file)
 
     identifier = ClassIdentifier(wind_turbine_number='test', origin_df=df, rated_power=1500, cut_out_speed=25,
@@ -24,14 +24,14 @@ def draw(file, fengchang='测试'):
     legend_map = {"停机": 'red', "好点": 'green', "欠发": 'blue', "超发": 'black', "额定风速以上的超发": 'orange', "限电": 'magenta'}
     scatter(name, x_label='风速', y_label='有功功率', x_values=df['wind_velocity'].values,
             y_values=df['active_power'].values, color=c, col_map=legend_map,
-            save_file_path=os.path.dirname(
-                os.path.dirname(__file__)) + os.sep + "tmp" + os.sep + str(fengchang) + os.sep + name + '结果.png')
+            save_file_path=path.dirname(
+                path.dirname(__file__)) + sep + "tmp" + sep + str(fengchang) + sep + name + '结果.png')
 
 
 if __name__ == '__main__':
     read_dir = r"D:\data\清理数据\和风元宝山\test_11_test\test"
 
-    files = [read_dir + os.sep + i for i in os.listdir(read_dir)]
+    files = [read_dir + sep + i for i in listdir(read_dir)]
 
     if len(files) == 1:
         draw(files[0], "和风元宝山4")

tmp_file/select_part_cols.py

@@ -1,6 +1,7 @@
 import datetime
 import multiprocessing
-import os
+from os import *
+
 import pandas as pd
 
 read_dir = r'/data/download/collection_data/2完成/招远风电场-山东-大唐/清理数据/WOF01000010-WOB000002_ZY24年7-9月秒级/second'
@@ -8,19 +9,19 @@ save_dir = r'/data/download/collection_data/2完成/招远风电场-山东-大
 
 
 def read_and_select_and_save(file):
-    df = pd.read_csv(read_dir + os.sep + file,
+    df = pd.read_csv(read_dir + sep + file,
                      usecols=['active_power', 'wind_velocity', 'pitch_angle_blade_1', 'yaw_error1', 'lab'])
     df = df[df['yaw_error1'] <= 360]
     df['yaw_error1'] = df['yaw_error1'].apply(lambda x: x - 360 if 180 <= x <= 360 else x)
     condition = (df['active_power'] > 0) & (df['wind_velocity'] > 0)
     df = df[condition]
 
-    df.to_csv(os.path.join(save_dir, file), index=False, encoding='utf8')
+    df.to_csv(path.join(save_dir, file), index=False, encoding='utf8')
     print(f'{file}处理完成')
 
 
 if __name__ == '__main__':
     begin = datetime.datetime.now()
     with multiprocessing.Pool(32) as pool:
-        pool.starmap(read_and_select_and_save, [(file,) for file in os.listdir(read_dir)])
+        pool.starmap(read_and_select_and_save, [(file,) for file in listdir(read_dir)])
     print(f'总耗时:{datetime.datetime.now() - begin}')

tmp_file/test_wave.py

@@ -1,7 +1,7 @@
-import os
 import sys
+from os import *
 
-sys.path.insert(0, os.path.abspath(__file__).split("tmp_file")[0])
+sys.path.insert(0, path.abspath(__file__).split("tmp_file")[0])
 
 from etl.wind_power.wave.WaveTrans import WaveTrans
 

tmp_file/zibo_guzhang_select_time.py

@@ -35,7 +35,7 @@ all_files = read_excel_files(r"D:\data\淄博\淄博风场buffer文件(1)")
 dfs = pd.DataFrame()
 
 for file in all_files:
-    base_name = os.path.basename(file)
+    base_name = path.basename(file)
     if base_name.startswith("b"):
         try:
             turbnine_no = int(base_name.split("_")[0].replace("b", ""))

tmp_file/光伏箱体.py

@@ -5,11 +5,11 @@ Created on Tue Jul  9 16:28:48 2024
 @author: Administrator
 """
 
-import os
-from datetime import datetime, timedelta
-import pandas as pd
+from datetime import datetime
+from os import *
+
 import chardet
-from multiprocessing import Pool
+import pandas as pd
 
 
 # 获取文件编码
@@ -61,11 +61,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -89,15 +89,15 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def read_and_save_csv(file_path):
     begin = datetime.now()
-    base_name = os.path.basename(file_path)
+    base_name = path.basename(file_path)
     print('开始', base_name)
 
     df1 = read_file_to_df(file_path + "箱变（1-8号逆变器）数据1.xls")
@@ -130,7 +130,7 @@ def read_and_save_csv(file_path):
     df = df[columns]
     df.sort_values(by='时间', inplace=True)
 
-    df.to_csv(os.path.join(r'D:\trans_data\大唐玉湖性能分析离线分析', '05整理数据', base_name + '_箱变.csv'), encoding='utf-8',
+    df.to_csv(path.join(r'D:\trans_data\大唐玉湖性能分析离线分析', '05整理数据', base_name + '_箱变.csv'), encoding='utf-8',
               index=False)
     print('结束', base_name, '耗时:' + str(datetime.now() - begin))
 
@@ -142,10 +142,10 @@ if __name__ == '__main__':
 
     all_paths = set()
     for file in all_files:
-        base_name = os.path.basename(file).split("箱变")[0]
-        base_path = os.path.dirname(file)
+        base_name = path.basename(file).split("箱变")[0]
+        base_path = path.dirname(file)
         if base_name not in all_paths:
-            all_paths.add(os.path.join(base_path, base_name))
+            all_paths.add(path.join(base_path, base_name))
 
     all_datas = list(all_paths)
     all_datas.sort()

tmp_file/对比文件夹列名差值.py

@@ -1,8 +1,3 @@
-import multiprocessing
-import os
-
-import pandas as pd
-
 from utils.file.trans_methods import *
 
 
@@ -83,8 +78,8 @@ if __name__ == '__main__':
     other_dfs = dict()
     list1 = list()
     for file in files2:
-        month = os.path.basename(os.path.dirname(os.path.dirname(file)))[0:2]
-        wind_name = month + os.path.basename(os.path.dirname(file)).split('#')[0] + '号风机'
+        month = path.basename(path.dirname(path.dirname(file)))[0:2]
+        wind_name = month + path.basename(path.dirname(file)).split('#')[0] + '号风机'
         df = read_file_to_df(file, nrows=1)
         if boolean_is_check_data(df.columns):
             list1.extend([str(i).split('_')[-1] for i in list(df.columns) if i != 'sheet_name'])

tmp_file/玉湖光伏-标准化.py

@@ -5,10 +5,11 @@ Created on Tue Jul  9 16:28:48 2024
 @author: Administrator
 """
 import multiprocessing
-import os
-from datetime import datetime, timedelta
-import pandas as pd
+from datetime import datetime
+from os import *
+
 import chardet
+import pandas as pd
 
 pd.options.mode.copy_on_write = True
 
@@ -62,11 +63,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -90,10 +91,10 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def generate_df(pv_df, col):
@@ -116,7 +117,7 @@ def generate_df(pv_df, col):
 
 def read_and_save_csv(file_path, save_path):
     begin = datetime.now()
-    base_name = os.path.basename(file_path)
+    base_name = path.basename(file_path)
     print('开始', base_name)
 
     df = read_file_to_df(file_path)
@@ -133,7 +134,7 @@ def read_and_save_csv(file_path, save_path):
 
     saved_pv_df = pd.concat(dfs)
     saved_pv_df.sort_values(by=['箱变', '逆变器', 'PV', '时间'], inplace=True)
-    save_file = os.path.join(save_path, os.path.basename(file_path).split(".")[0], 'PV.csv')
+    save_file = path.join(save_path, path.basename(file_path).split(".")[0], 'PV.csv')
     create_file_path(save_file, True)
 
     saved_pv_df.to_csv(save_file, encoding='utf-8', index=False)

tmp_file/玉湖光伏-标准化_1.py

@@ -5,10 +5,11 @@ Created on Tue Jul  9 16:28:48 2024
 @author: Administrator
 """
 import multiprocessing
-import os
-from datetime import datetime, timedelta
-import pandas as pd
+from datetime import datetime
+from os import *
+
 import chardet
+import pandas as pd
 
 pd.options.mode.copy_on_write = True
 
@@ -62,11 +63,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -90,10 +91,10 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def generate_df(pv_df, col):
@@ -133,7 +134,7 @@ def replece_col_to_biaozhun(col):
 
 def read_and_save_csv(file_path, save_path):
     begin = datetime.now()
-    base_name = os.path.basename(file_path)
+    base_name = path.basename(file_path)
     print('开始', base_name)
 
     df = read_file_to_df(file_path)
@@ -157,7 +158,7 @@ def read_and_save_csv(file_path, save_path):
 
     # saved_pv_df = pd.concat(dfs)
     df.sort_values(by=['时间'], inplace=True)
-    save_file = os.path.join(save_path, os.path.basename(file_path))
+    save_file = path.join(save_path, path.basename(file_path))
     create_file_path(save_file, True)
 
     df.to_csv(save_file, encoding='utf-8', index=False)

tmp_file/玉湖光伏-标准化_2.py

@@ -5,12 +5,12 @@ Created on Tue Jul  9 16:28:48 2024
 @author: Administrator
 """
 import multiprocessing
-import os
-from datetime import datetime, timedelta
+from datetime import datetime
+from os import *
 
+import chardet
 import numpy as np
 import pandas as pd
-import chardet
 
 pd.options.mode.copy_on_write = True
 
@@ -64,11 +64,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -92,10 +92,10 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def split_index(split_data: str, split_str: str):
@@ -199,7 +199,7 @@ def df_to_biaozhun(df):
 
 def read_and_save_csv(file_path, save_path):
     begin = datetime.now()
-    base_name = os.path.basename(file_path)
+    base_name = path.basename(file_path)
     print('开始', base_name)
 
     df = read_file_to_df(file_path)
@@ -223,7 +223,7 @@ def read_and_save_csv(file_path, save_path):
     df = df_to_biaozhun(df)
 
     # df.sort_values(by=['时间', "箱变", "逆变器", "支路"], inplace=True)
-    # save_file = os.path.join(save_path, os.path.basename(file_path))
+    # save_file = path.join(save_path, path.basename(file_path))
     # create_file_path(save_file, True)
 
     # df.to_csv(save_file, encoding='utf-8', index=False)
@@ -277,7 +277,7 @@ if __name__ == '__main__':
 
     saved_pv_df = pd.concat(dfs)
     saved_pv_df.sort_values(by=['时间', "箱变", "逆变器", "支路"], inplace=True)
-    save_file = os.path.join(save_path, "合并.csv")
+    save_file = path.join(save_path, "合并.csv")
     create_file_path(save_file, True)
     saved_pv_df.sort_values(by=['时间', "箱变", "逆变器", "支路"], inplace=True)
     saved_pv_df.to_csv(save_file, encoding='utf-8', index=False)

tmp_file/玉湖光伏-气象标准化.py

@@ -4,13 +4,10 @@ Created on Tue Jul  9 16:28:48 2024
 
 @author: Administrator
 """
-import multiprocessing
-import os
-from datetime import datetime, timedelta
+from os import *
 
-import numpy as np
-import pandas as pd
 import chardet
+import pandas as pd
 
 pd.options.mode.copy_on_write = True
 
@@ -64,11 +61,11 @@ def read_file_to_df(file_path, read_cols=list(), header=0):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -92,10 +89,10 @@ def read_excel_files(read_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 if __name__ == '__main__':
@@ -104,12 +101,12 @@ if __name__ == '__main__':
     path = r'Z:\大唐玉湖性能分析离线分析\05整理数据\气象站数据'
     save_path = r'Z:\大唐玉湖性能分析离线分析\06整理数据\气象站数据'
 
-    fengsu_df = read_file_to_df(os.path.join(path, '风速.csv'), read_cols=['当前时间', '实际风速'])
-    fengxiang_df = read_file_to_df(os.path.join(path, '风向.csv'), read_cols=['当前时间', '实际风向'])
-    fuzhaodu_df = read_file_to_df(os.path.join(path, '辐照度.csv'), read_cols=['时间', '水平总辐照度', '倾斜总辐照度', '散射辐照度'])
-    shidu_df = read_file_to_df(os.path.join(path, '湿度.csv'), read_cols=['时间', '实际湿度'])
-    wendu_df = read_file_to_df(os.path.join(path, '温度.csv'), read_cols=['时间', '实际温度'])
-    yali_df = read_file_to_df(os.path.join(path, '压力.csv'), read_cols=['时间', '实际气压'])
+    fengsu_df = read_file_to_df(path.join(path, '风速.csv'), read_cols=['当前时间', '实际风速'])
+    fengxiang_df = read_file_to_df(path.join(path, '风向.csv'), read_cols=['当前时间', '实际风向'])
+    fuzhaodu_df = read_file_to_df(path.join(path, '辐照度.csv'), read_cols=['时间', '水平总辐照度', '倾斜总辐照度', '散射辐照度'])
+    shidu_df = read_file_to_df(path.join(path, '湿度.csv'), read_cols=['时间', '实际湿度'])
+    wendu_df = read_file_to_df(path.join(path, '温度.csv'), read_cols=['时间', '实际温度'])
+    yali_df = read_file_to_df(path.join(path, '压力.csv'), read_cols=['时间', '实际气压'])
 
     fengsu_df.rename(columns={'当前时间': '时间'}, inplace=True)
     fengxiang_df.rename(columns={'当前时间': '时间'}, inplace=True)
@@ -122,4 +119,4 @@ if __name__ == '__main__':
 
     df = pd.concat(dfs, axis=1)
     create_file_path(save_path, is_file_path=False)
-    df.to_csv(os.path.join(save_path, '气象合并.csv'), encoding='utf-8')
+    df.to_csv(path.join(save_path, '气象合并.csv'), encoding='utf-8')

tmp_file/白玉山每月限电损失.py

@@ -1,16 +1,16 @@
-import os
+from os import *
 
 import pandas as pd
 
 read_path = r'D:\data\白玉山后评估数据资料\需要整理的数据\每月发电量和限电量、限电率'
 
 all_paths = list()
-for root, dirs, files in os.walk(read_path):
+for root, dirs, files in walk(read_path):
     if files:
         for file in files:
             year_mont = int(file.split("(")[1].split("_")[0])
             if year_mont >= 20230901 and year_mont < 20240901:
-                all_paths.append(os.path.join(root, file))
+                all_paths.append(path.join(root, file))
 
 df = pd.DataFrame()
 

utils/db/ConnectMysql.py

@@ -1,5 +1,5 @@
-import os
 import traceback
+from os import *
 
 import pandas as pd
 import pymysql
@@ -13,8 +13,8 @@ from utils.log.trans_log import trans_print
 class ConnectMysql:
 
     def __init__(self, connet_name):
-        config_path = os.path.abspath(__file__).split("utils")[0] + 'conf' + os.sep + 'etl_config_dev.yaml'
-        self.yaml_data = yaml_conf(os.environ.get('ETL_CONF', config_path))
+        config_path = path.abspath(__file__).split("utils")[0] + 'conf' + sep + 'etl_config_prod.yaml'
+        self.yaml_data = yaml_conf(environ.get('ETL_CONF', config_path))
         self.connet_name = connet_name
         self.config = self.yaml_data[self.connet_name]
 

utils/file/trans_methods.py

@@ -3,9 +3,9 @@
 # @Author  : 魏志亮
 import ast
 import datetime
-import os
 import shutil
 import warnings
+from os import *
 
 import chardet
 import pandas as pd
@@ -89,7 +89,7 @@ def read_file_to_df(file_path, read_cols=list(), trans_cols=None, nrows=None, no
     find_cols = list()
     if trans_cols:
         header = find_read_header(file_path, trans_cols, resolve_col_prefix)
-        trans_print(os.path.basename(file_path), "读取第", header, "行")
+        trans_print(path.basename(file_path), "读取第", header, "行")
         if header is None:
             if not_find_header == 'raise':
                 message = '未匹配到开始行，请检查并重新指定'
@@ -137,7 +137,7 @@ def read_file_to_df(file_path, read_cols=list(), trans_cols=None, nrows=None, no
             trans_print('文件读取成功:', file_path, '数据数量:', df.shape, '耗时:', datetime.datetime.now() - begin)
         except Exception as e:
             trans_print('读取文件出错', file_path, str(e))
-            message = '文件:' + os.path.basename(file_path) + ',' + str(e)
+            message = '文件:' + path.basename(file_path) + ',' + str(e)
             raise ValueError(message)
 
     return df
@@ -145,11 +145,11 @@ def read_file_to_df(file_path, read_cols=list(), trans_cols=None, nrows=None, no
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -164,7 +164,7 @@ def __build_directory_dict(directory_dict, path, filter_types=None):
 def read_excel_files(read_path, filter_types=None):
     if filter_types is None:
         filter_types = ['xls', 'xlsx', 'csv', 'gz']
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
 
     directory_dict = {}
@@ -177,7 +177,7 @@ def read_excel_files(read_path, filter_types=None):
 def read_files(read_path, filter_types=None):
     if filter_types is None:
         filter_types = ['xls', 'xlsx', 'csv', 'gz', 'zip', 'rar']
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
     directory_dict = {}
     __build_directory_dict(directory_dict, read_path, filter_types=filter_types)
@@ -203,10 +203,10 @@ def create_file_path(path, is_file_path=False):
     :param is_file_path: 传入的path是否包含具体的文件名
     """
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 def valid_eval(eval_str):
@@ -228,9 +228,10 @@ if __name__ == '__main__':
     # aa = valid_eval("df['123'].apply(lambda wind_name: wind_name.replace('元宝山','').replace('号风机',''))")
     # print(aa)
     #
-    # aa = valid_eval("'记录时间' if column == '时间' else column;import os; os.path")
+    # aa = valid_eval("'记录时间' if column == '时间' else column;from os import *; path")
     # print(aa)
 
-    df = read_file_to_df(r"D:\data\11-12月.xls", trans_cols=['风机', '时间', '有功功率', '无功功率', '功率因数', '频率'], nrows=30)
+    df = read_file_to_df(r"D:\data\11-12月.xls",
+                         trans_cols=['风机', '时间', '有功功率', '无功功率', '功率因数', '频率'], nrows=30)
 
     print(df.columns)

utils/file/trans_methods.py_1

@@ -2,7 +2,7 @@
 # @Time    : 2024/5/16
 # @Author  : 魏志亮
 import datetime
-import os
+from os import *
 import shutil
 import warnings
 
@@ -86,7 +86,7 @@ def read_file_to_df(file_path, read_cols=list(), trans_cols=None, nrows=None):
     find_cols = list()
     if trans_cols:
         header, find_cols = find_read_header(file_path, trans_cols)
-        trans_print(os.path.basename(file_path), "读取第", header, "行")
+        trans_print(path.basename(file_path), "读取第", header, "行")
         if header is None:
             message = '未匹配到开始行，请检查并重新指定'
             trans_print(message)
@@ -129,7 +129,7 @@ def read_file_to_df(file_path, read_cols=list(), trans_cols=None, nrows=None):
         trans_print('文件读取成功:', file_path, '数据数量:', df.shape, '耗时:', datetime.datetime.now() - begin)
     except Exception as e:
         trans_print('读取文件出错', file_path, str(e))
-        message = '文件:' + os.path.basename(file_path) + ',' + str(e)
+        message = '文件:' + path.basename(file_path) + ',' + str(e)
         raise ValueError(message)
 
     return df
@@ -137,11 +137,11 @@ def read_file_to_df(file_path, read_cols=list(), trans_cols=None, nrows=None):
 
 def __build_directory_dict(directory_dict, path, filter_types=None):
     # 遍历目录下的所有项
-    for item in os.listdir(path):
-        item_path = os.path.join(path, item)
-        if os.path.isdir(item_path):
+    for item in listdir(path):
+        item_path = path.join(path, item)
+        if path.isdir(item_path):
             __build_directory_dict(directory_dict, item_path, filter_types=filter_types)
-        elif os.path.isfile(item_path):
+        elif path.isfile(item_path):
             if path not in directory_dict:
                 directory_dict[path] = []
 
@@ -154,7 +154,7 @@ def __build_directory_dict(directory_dict, path, filter_types=None):
 
 # 读取路径下所有的excel文件
 def read_excel_files(read_path):
-    if os.path.isfile(read_path):
+    if path.isfile(read_path):
         return [read_path]
 
     directory_dict = {}
@@ -184,10 +184,10 @@ def copy_to_new(from_path, to_path):
 # 创建路径
 def create_file_path(path, is_file_path=False):
     if is_file_path:
-        path = os.path.dirname(path)
+        path = path.dirname(path)
 
-    if not os.path.exists(path):
-        os.makedirs(path, exist_ok=True)
+    if not path.exists(path):
+        makedirs(path, exist_ok=True)
 
 
 if __name__ == '__main__':

utils/log/trans_log.py

@@ -4,15 +4,15 @@
 
 import datetime
 import logging
-import os
 import sys
+from os import *
 
 from utils.conf.read_conf import read_conf, yaml_conf
 
 
 def set_trance_id(trace_id):
     """设置当前线程的链路ID"""
-    os.environ['trace_id'] = trace_id
+    environ['trace_id'] = trace_id
 
 
 class ContextFilter(logging.Filter):
@@ -20,8 +20,8 @@ class ContextFilter(logging.Filter):
 
     def filter(self, record):
         record.trace_id = ''
-        if 'trace_id' in os.environ.keys():
-            record.trace_id = os.environ['trace_id']
+        if 'trace_id' in environ.keys():
+            record.trace_id = environ['trace_id']
 
         return True
 
@@ -35,16 +35,16 @@ stout_handle.setLevel(logging.INFO)
 stout_handle.addFilter(ContextFilter())
 logger.addHandler(stout_handle)
 
-config_path = os.path.abspath(__file__).split("utils")[0] + 'conf' + os.sep + 'etl_config_dev.yaml'
-config = yaml_conf(os.environ.get('ETL_CONF', config_path))
+config_path = path.abspath(__file__).split("utils")[0] + 'conf' + sep + 'etl_config_dev.yaml'
+config = yaml_conf(environ.get('ETL_CONF', config_path))
 log_path_dir = read_conf(config, 'log_path_dir', "/data/logs")
 
-log_path = log_path_dir + os.sep + r'etl_tools_' + (os.environ['env'] if 'env' in os.environ else 'dev')
-file_path = os.path.join(log_path)
+log_path = log_path_dir + sep + r'etl_tools_' + (environ['env'] if 'env' in environ else 'dev')
+file_path = path.join(log_path)
 
-if not os.path.exists(file_path):
-    os.makedirs(file_path, exist_ok=True)
-file_name = file_path + os.sep + str(datetime.date.today()) + '.log'
+if not path.exists(file_path):
+    makedirs(file_path, exist_ok=True)
+file_name = file_path + sep + str(datetime.date.today()) + '.log'
 
 file_handler = logging.FileHandler(file_name, encoding='utf-8')
 file_handler.setFormatter(

utils/systeminfo/sysinfo.py

@@ -1,4 +1,5 @@
-import os
+from os import *
+
 import psutil
 
 from utils.log.trans_log import trans_print
@@ -6,7 +7,7 @@ from utils.log.trans_log import trans_print
 
 def print_memory_usage(detail=""):
     # 获取当前进程ID
-    pid = os.getpid()
+    pid = getpid()
     # 获取进程信息
     py = psutil.Process(pid)
     # 获取内存信息
@@ -34,12 +35,12 @@ def get_available_cpu_count_with_percent(percent: float = 1):
 
 
 def get_file_size(file_path):
-    return os.path.getsize(file_path)
+    return path.getsize(file_path)
 
 
 def get_dir_size(dir_path):
-    return sum(get_file_size(os.path.join(dir_path, file)) for file in os.listdir(dir_path) if
-               os.path.isfile(os.path.join(dir_path, file)))
+    return sum(get_file_size(path.join(dir_path, file)) for file in listdir(dir_path) if
+               path.isfile(path.join(dir_path, file)))
 
 
 def get_available_memory_with_percent(percent: float = 1):

utils/zip/unzip.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 # @Time    : 2024/5/17
 # @Author  : 魏志亮
-import os
 import traceback
 import zipfile
+from os import *
 
 import rarfile
 
@@ -51,24 +51,24 @@ def unzip(zip_filepath, dest_path):
     dest_path = dest_path
     trans_print('解压再次读取', dest_path)
     if is_success:
-        for root, dirs, files in os.walk(dest_path):
+        for root, dirs, files in walk(dest_path):
             for file in files:
-                file_path = os.path.join(root, file)
+                file_path = path.join(root, file)
                 # 检查文件是否是zip文件
                 if file_path.endswith('.zip'):
                     if file_path.endswith('.csv.zip'):
-                        os.rename(file_path, file_path.replace(".csv.zip", ".csv.gz"))
+                        rename(file_path, file_path.replace(".csv.zip", ".csv.gz"))
                     else:
                         # 如果是，递归解压
-                        unzip(file_path, dest_path + os.sep + get_desc_path(str(file)))
+                        unzip(file_path, dest_path + sep + get_desc_path(str(file)))
                         # 删除已解压的zip文件（可选）
-                        os.remove(file_path)
+                        remove(file_path)
                     # 检查文件是否是zip文件
                 if file_path.endswith('.rar'):
                     # 如果是，递归解压
-                    unrar(file_path, dest_path + os.sep + get_desc_path(str(file)))
+                    unrar(file_path, dest_path + sep + get_desc_path(str(file)))
                     # 删除已解压的zip文件（可选）
-                    os.remove(file_path)
+                    remove(file_path)
 
     return is_success, ''
 
@@ -80,8 +80,8 @@ def unrar(rar_file_path, dest_dir):
     trans_print('开始读取文件:', rar_file_path)
     dest_path = dest_dir
     trans_print("解压到:", dest_path)
-    if not os.path.exists(dest_path):
-        os.makedirs(dest_path)
+    if not path.exists(dest_path):
+        makedirs(dest_path)
 
     try:
         # 打开RAR文件
@@ -100,21 +100,21 @@ def unrar(rar_file_path, dest_dir):
     # 遍历解压后的文件
     print('解压再次读取', dest_path)
     if is_success:
-        for root, dirs, files in os.walk(dest_path):
+        for root, dirs, files in walk(dest_path):
             for file in files:
-                file_path = os.path.join(root, file)
+                file_path = path.join(root, file)
                 # 检查文件是否是zip文件
                 if file_path.endswith('.rar'):
                     # 如果是，递归解压
                     unrar(file_path, get_desc_path(file_path))
                     # 删除已解压的zip文件（可选）
-                    os.remove(file_path)
+                    remove(file_path)
 
                 if file_path.endswith('.zip'):
                     # 如果是，递归解压
                     unzip(file_path, get_desc_path(file_path))
                     # 删除已解压的zip文件（可选）
-                    os.remove(file_path)
+                    remove(file_path)
 
     return is_success, ''