开始执行批次,移除数据库字段无关信息

service/plt_service.py

@@ -26,7 +26,9 @@ def update_timeout_trans_data():
 def update_trans_status_running(batch_no, trans_type, schedule_exec=True):
     if schedule_exec:
         exec_sql = """
-        update data_transfer set transfer_state = 0,trans_sys_status = 0 ,transfer_start_time = now()
+        update data_transfer set transfer_state = 0,trans_sys_status = 0 ,transfer_start_time = now(),err_info='',
+        engine_count =0,time_granularity=0,transfer_finish_time=null,
+        data_min_time= null,data_max_time= null,transfer_data_count=null
         where batch_code = %s  and transfer_type = %s
         """
         plt.execute(exec_sql, (batch_no, trans_type))
@@ -155,5 +157,5 @@ if __name__ == '__main__':
     # print(get_batch_exec_data(run_count=2))
     # print("**********************")
     print(get_data_by_batch_no_and_type("test_", "second"))
-# print(update_trans_status_success("test_唐龙-定时任务测试", "second", 10))
-    begin = datetime.datetime.now()
+    # print(update_trans_status_success("test_唐龙-定时任务测试", "second", 10))
+    begin = datetime.datetime.now()

tmp_file/error_ms_data.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+import pandas as pd
+
+
+def convert_date(date_str):
+    cut_index = str(date_str).rfind("_")
+    date = date_str[0:cut_index].replace("_", "-")
+    time = date_str[cut_index + 1:].replace(":", ".")
+
+    return datetime.strptime(f"{date} {time}", '%Y-%m-%d %H.%M.%S.%f')
+
+
+df = pd.read_csv(r"d:/data/b2_240828_2324_Err 1.csv", header=1)
+df.dropna(subset='TimeStamp', inplace=True)
+df.drop_duplicates(subset='TimeStamp', keep="first", inplace=True)
+
+origin_columns = list(df.columns)
+
+df['TimeStamp1'] = df['TimeStamp'].apply(convert_date)
+df.sort_values(by='TimeStamp1', inplace=True)
+
+# df['DateTime'] = pd.to_datetime(df['TimeStamp'], format="%Y-%m-%d %H:%M:%S")
+df['DateTime'] = df['TimeStamp1'].apply(lambda x: x.strftime("%Y-%m-%d %H:%M:%S"))
+
+print(df.shape)
+
+dateTime_count = df['DateTime'].value_counts()
+
+dateTime_count_1 = dateTime_count[dateTime_count == 1]
+dateTime_count_gt1 = dateTime_count[dateTime_count > 1]
+
+df1 = df[df['DateTime'].isin(dateTime_count_1.index.values)]
+df2 = df[df['DateTime'].isin(dateTime_count_gt1.index.values)]
+
+print(df1.shape)
+print(df2.shape)
+origin_columns.insert(0, 'DateTime')
+df1.to_csv("1秒数据.csv", encoding='utf-8', index=False, columns=origin_columns, date_format="%Y-%m-%d %H:%M:%S.%f")
+df2.to_csv("毫秒数据.csv", encoding='utf-8', index=False, columns=origin_columns, date_format="%Y-%m-%d %H:%M:%S.%f")