|
@@ -27,6 +27,9 @@ class BaseDataTrans(object):
|
|
|
self.batch_count = 100000
|
|
|
self.save_db = save_db
|
|
|
self.filed_conf = self.get_filed_conf()
|
|
|
+
|
|
|
+ # trans_print("是否是秒转分钟:", self.boolean_sec_to_min)
|
|
|
+
|
|
|
try:
|
|
|
self.pathsAndTable = PathsAndTable(self.batch_no, self.batch_name, self.read_path, self.field_name,
|
|
|
self.read_type, save_db, self.save_zip)
|
|
@@ -79,31 +82,36 @@ class BaseDataTrans(object):
|
|
|
begin = datetime.datetime.now()
|
|
|
trans_print("开始清理数据,临时文件夹:", self.pathsAndTable.get_tmp_path())
|
|
|
self.clean_file_and_db()
|
|
|
- trans_print("清理数据结束,耗时:", datetime.datetime.now() - begin, "总耗时:", datetime.datetime.now() - begin)
|
|
|
+ trans_print("清理数据结束,耗时:", datetime.datetime.now() - begin, "总耗时:",
|
|
|
+ datetime.datetime.now() - total_begin)
|
|
|
|
|
|
if self.step <= 1 and self.end >= 1:
|
|
|
begin = datetime.datetime.now()
|
|
|
trans_print("开始解压移动文件")
|
|
|
self.unzip_or_remove_to_tmp_dir()
|
|
|
- trans_print("解压移动文件结束:耗时:", datetime.datetime.now() - begin, "总耗时:", datetime.datetime.now() - begin)
|
|
|
+ trans_print("解压移动文件结束:耗时:", datetime.datetime.now() - begin, "总耗时:",
|
|
|
+ datetime.datetime.now() - total_begin)
|
|
|
|
|
|
if self.step <= 2 and self.end >= 2:
|
|
|
begin = datetime.datetime.now()
|
|
|
trans_print("开始保存数据到临时文件")
|
|
|
self.read_and_save_tmp_file()
|
|
|
- trans_print("保存数据到临时文件结束,耗时:", datetime.datetime.now() - begin, "总耗时:", datetime.datetime.now() - begin)
|
|
|
+ trans_print("保存数据到临时文件结束,耗时:", datetime.datetime.now() - begin, "总耗时:",
|
|
|
+ datetime.datetime.now() - total_begin)
|
|
|
|
|
|
if self.step <= 3 and self.end >= 3:
|
|
|
begin = datetime.datetime.now()
|
|
|
trans_print("开始保存数据到正式文件")
|
|
|
self.statistics_and_save_to_file()
|
|
|
- trans_print("保存数据到正式文件结束,耗时:", datetime.datetime.now() - begin, "总耗时:", datetime.datetime.now() - begin)
|
|
|
+ trans_print("保存数据到正式文件结束,耗时:", datetime.datetime.now() - begin, "总耗时:",
|
|
|
+ datetime.datetime.now() - total_begin)
|
|
|
|
|
|
if self.step <= 4 and self.end >= 4:
|
|
|
begin = datetime.datetime.now()
|
|
|
trans_print("开始保存到数据库,是否存库:", self.pathsAndTable.save_db)
|
|
|
self.save_to_db()
|
|
|
- trans_print("保存到数据结束,耗时:", datetime.datetime.now() - begin, "总耗时:", datetime.datetime.now() - begin)
|
|
|
+ trans_print("保存到数据结束,耗时:", datetime.datetime.now() - begin, "总耗时:",
|
|
|
+ datetime.datetime.now() - total_begin)
|
|
|
|
|
|
self.update_exec_progress()
|
|
|
except Exception as e:
|
|
@@ -111,7 +119,7 @@ class BaseDataTrans(object):
|
|
|
update_trans_status_error(self.batch_no, self.read_type, str(e), self.save_db)
|
|
|
raise e
|
|
|
finally:
|
|
|
- # self.pathsAndTable.delete_tmp_files()
|
|
|
+ self.pathsAndTable.delete_tmp_files()
|
|
|
trans_print("执行结束,总耗时:", str(datetime.datetime.now() - total_begin))
|
|
|
|
|
|
|