diff --git a/AutoCoverTool/script/multi_trainer.py b/AutoCoverTool/script/multi_trainer.py index fa4b194..7a67ea4 100644 --- a/AutoCoverTool/script/multi_trainer.py +++ b/AutoCoverTool/script/multi_trainer.py @@ -1,87 +1,92 @@ """ 分布式训练 """ import os import json import glob +from script.upload_mode2db import upload_one2db def put_data(): dir_list = glob.glob("/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/*") already_user_ids = [ "10133099162839896", "10133099162997509", "10133099163727028", "10133099163890661", "10133099163991355", "10133099164311744", "10133099164313669", "10133099165386135", "10133099166041782", "10133099166050735", "10133099166238022", "10133099166605472", "10133099166892845", "10133099166898301", "10133099167125366", "10133099167394822", "10133099167940583", "10133099168376799", "10133099168924385", "8725724286358130", "finish" ] from online.beanstalk_helper import BeanstalkHelper config = {"addr": "sg-test-common-box-1:11300", "consumer": "auto_user_svc_trainer"} bean_helper = BeanstalkHelper(config) for idx, line in enumerate(dir_list): cur_dir = str(line.split("/")[-1]) if cur_dir not in already_user_ids: message = json.dumps({"user_id": str(cur_dir)}) bean_helper.put_payload_to_beanstalk(config["consumer"], message, ttr=2 * 86400) print(len("tot_num={}".format(len(dir_list)))) def process(): from online.beanstalk_helper import BeanstalkHelper bb_dir = "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/" config = {"addr": "sg-test-common-box-1:11300", "consumer": "auto_user_svc_trainer"} bean_helper = BeanstalkHelper(config) bean = bean_helper.get_beanstalkd() bean.watch(config["consumer"]) while True: payload = bean.reserve(5) if not payload: print("bean sleep...") continue in_data = json.loads(payload.body) user_id = in_data["user_id"] try: user_id_int = int(float(user_id)) g_2000_path = os.path.join(bb_dir, os.path.join(str(user_id_int), "logs/32k/G_2000.pth")) if os.path.exists(g_2000_path): print("log, {} already exists!....".format(user_id_int)) payload.delete() continue src_dir = os.path.join(bb_dir, os.path.join(str(user_id_int), "src")) if not os.path.exists(src_dir): print("log, {} src not exists!....".format(user_id_int)) payload.delete() continue cmd = "bash script/train.sh {}".format(user_id_int) print(cmd) os.system(cmd) + + # 上传到cos并更新数据库 + if os.path.exists(g_2000_path): + upload_one2db(g_2000_path, user_id_int) print("log, train {} ok....".format(user_id_int)) except Exception as ex: pass payload.delete() if __name__ == '__main__': process() diff --git a/AutoCoverTool/script/upload_mode2db.py b/AutoCoverTool/script/upload_mode2db.py index be3773d..2c8af9b 100644 --- a/AutoCoverTool/script/upload_mode2db.py +++ b/AutoCoverTool/script/upload_mode2db.py @@ -1,280 +1,280 @@ """ 将训练好的模型写入到数据库 """ from online.common import update_db, get_data_by_mysql import os GS_COSCMD = "/data/gpu_env_common/env/anaconda3/envs/auto_song_cover/bin/coscmd" GS_CONFIG_PATH = "/home/normal/.cos.conf" def exec_cmd(cmd): ret = os.system(cmd) if ret != 0: return False return True def exec_cmd_and_result(cmd): r = os.popen(cmd) text = r.read() r.close() return text def upload_file2cos(key, file_path, region, bucket_name): """ 将文件上传到cos :param key: 桶上的具体地址 :param file_path: 本地文件地址 :param region: 区域 :param bucket_name: 桶地址 :return: """ cmd = "{} -c {} -r {} -b {} upload {} {}".format(GS_COSCMD, GS_CONFIG_PATH, region, bucket_name, file_path, key) print(cmd) if exec_cmd(cmd): cmd = "{} -c {} -r {} -b {} info {}".format(GS_COSCMD, GS_CONFIG_PATH, region, bucket_name, key) \ + "| grep Content-Length |awk \'{print $2}\'" res_str = exec_cmd_and_result(cmd) size = float(res_str) if size > 0: return True return False return False def upload_one2db(db_path, db_name): # 判断已经在数据库中，则不上传 sql = "select * from av_db.av_svc_model where user_id=\"{}\"".format(db_name) data = get_data_by_mysql(sql) if len(data) == 0: print('start upload ....') - key = "av_res/so_vits_models/3.0/male/{}.pth".format(db_name) + key = "av_res/so_vits_models/3.0/unknown/{}.pth".format(db_name) if upload_file2cos(key, db_path, region="ap-beijing", bucket_name="av-audit-sync-bj-1256122840"): # 写入到数据库 - sql = "insert into av_db.av_svc_model (model_version, user_id, model_url, gender) values (\"sovits3.0\", \"{}\", \"{}\", 1)".format( + sql = "insert into av_db.av_svc_model (model_version, user_id, model_url, gender) values (\"sovits3.0\", \"{}\", \"{}\", 3)".format( db_name, key) update_db(sql) if __name__ == '__main__': # arr = [ # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099162839896/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099162997509/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099163727028/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099163890661/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099163991355/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099164311744/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099164313669/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099165386135/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099166041782/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099166050735/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099166238022/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099166605472/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099166892845/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099166898301/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099167125366/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099167394822/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099167940583/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099168376799/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099168924385/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099169324630/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099169381678/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099169957610/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099169975944/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099170492806/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099170892510/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099171013390/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099171081854/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099171087756/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099171192036/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099171607206/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099171754668/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099172086640/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099172138002/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099172207062/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099172269180/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099172339368/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574138359736/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574138369704/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574138530136/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574139102564/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574139888146/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574139967984/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574140258122/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574140405046/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574140676612/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574140780266/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574142812606/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574143134746/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574143604234/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574143906306/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574144526110/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574144707118/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574145823464/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574145904464/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574146080322/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574146420792/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574146884926/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147339012/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147372254/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147425002/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147597736/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147647706/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147658166/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574147828554/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574148014424/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574148247626/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574148624370/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574148669184/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574148692388/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574148859406/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574149000590/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574149067094/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574149143568/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574149221618/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10414574149303702/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049115833380/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049115944594/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049115987498/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049116130908/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049116285936/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049117044138/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049117276112/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049117685892/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049119207544/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049119659788/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049120426324/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049120919532/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049121183928/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049121338248/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049121406512/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049121502826/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049123071172/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049123219186/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049123447868/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049123506368/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049123660154/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049123805538/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049124073344/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049124110520/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049124182084/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049124450100/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049124595430/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049124833978/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125084058/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125481092/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125584584/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125798928/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125820940/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125864268/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125885128/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125972416/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125997808/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10696049125999636/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524091895906/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524092611108/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524092703694/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524092737576/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524092926748/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524093350560/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524093613618/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524094859474/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524096635844/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524096695280/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524096819198/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524096995342/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524098416100/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524098804908/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524099612646/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524100174518/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524100978492/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524101050108/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524101220516/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524101243434/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524101575638/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524101593280/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524101680844/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102313334/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102348346/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102432628/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102444474/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102525738/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102533320/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102598012/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102674590/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102678972/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10977524102679572/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899906849269/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899908853925/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899908854526/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899908904395/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899909347935/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899909790502/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910057693/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910105120/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910461551/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910516883/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910808376/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910826302/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899910943438/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899911011477/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899911821662/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899911962207/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912327206/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912442110/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912511535/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912520616/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912538184/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912584668/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912859360/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912895306/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912929958/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/1125899912987231/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/3635049378/logs/32k/G_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/8725724286358130/logs/32k/G_2000.pth" # ] arr = [ # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/10133099162058848.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/10133099162258844.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/10133099162560644.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/10133099162797809.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/10414574138721494.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/10414574140317353.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/1688849864840588.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/3634463651.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/5629499489839033.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/5910973794723621.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/6755399374234747.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/8162774327817435.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/8162774329368194.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/denglijun_g2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/female/taylor_g2000.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/10133099161752147.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/10133099161920114.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/10133099162292113.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/10133099162767013.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/1125899914308640.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/12384898975368914.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/12947848931397021.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/3096224748076687.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/3096224751151928.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/5066549357604730.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/5348024335101054.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/6755399442719465.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/7036874421386111.pth", "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/male/jianli_g_2000.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/unknown/10133099171607206.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/unknown/10133099172269180.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/unknown/10414574138369704.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/unknown/10414574143604234.pth", # "/data/rsync/jianli.yang/AutoCoverTool/data/online_models/models/unknown/10414574144707118.pth" ] for d_file in arr: # d_file = "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/multi_users/10133099162839896/logs/32k/G_2000.pth" # d_name = "10133099162839896" d_name = d_file.split("/")[-1].split(".")[0] print(d_file, d_name) upload_one2db(d_file, d_name) # exit(-1) diff --git a/AutoCoverTool/svc_inference/svc_inference_one.py b/AutoCoverTool/svc_inference/svc_inference_one.py index 5218e1e..ecb8e8c 100644 --- a/AutoCoverTool/svc_inference/svc_inference_one.py +++ b/AutoCoverTool/svc_inference/svc_inference_one.py @@ -1,210 +1,215 @@ """ SVC推理逻辑 conda activate auto_song_cover_t4 export PYTHONPATH=$PWD:$PWD/ref/music_remover/demucs:$PWD/ref/so_vits_svc:$PWD/ref/split_dirty_frame:$PWD/ref/online """ import os +import json import shutil from ref.so_vits_svc.inference_main import * from ref.speaker_feature_extractor.sf_extractor_interface import SFExtractorInterface gs_draw_volume_exe = "/data/gpu_env_common/bin/draw_volume" gs_simple_mixer_path = "/data/gpu_env_common/bin/simple_mixer" gs_svci_success = 0 gs_svci_data_params_check_model_path = 1 gs_svci_data_params_check_vocal_path = 2 gs_svci_data_params_check_acc_path = 3 gs_svci_data_params_check_video_path = 4 gs_svci_data_prepare_transcode_media = 5 gs_svci_data_inference = 6 gs_svci_svc_trans_442 = 7 gs_svci_svc_volume = 8 gs_svci_svc_mix = 9 gs_svci_svc_mix_gen = 10 gs_svci_svc_mix_audio_video = 11 class SVCInferenceOne: def __init__(self): self.vocal_32_wav_path = None self.vocal_wav_path = None self.acc_wav_path = None self.config = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.json") self.spk_emb_inst = SFExtractorInterface() def mix(self, work_dir, svc_file, vocal_file, acc_file, mix_path): """ :param work_dir: :param svc_file: :param vocal_file: :param acc_file: :param mix_path: :return: """ cache_dir = os.path.join(work_dir, "cache") if os.path.exists(cache_dir): shutil.rmtree(cache_dir) os.makedirs(cache_dir) # svc转码到442 svc_442_file = os.path.join(cache_dir, "442.wav") st = time.time() cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {} -loglevel fatal".format(svc_file, svc_442_file) os.system(cmd) if not os.path.exists(svc_442_file): return gs_svci_svc_trans_442 logging.info("transcode,{},sp={}".format(svc_file, time.time() - st)) # 合并转码后再做一次拉伸，保证响度 st = time.time() volume_path = os.path.join(cache_dir, "volume.wav") cmd = "{} {} {} {}".format(gs_draw_volume_exe, svc_442_file, vocal_file, volume_path) os.system(cmd) if not os.path.exists(volume_path): print("{} ERROR draw volume".format(volume_path)) return gs_svci_svc_volume logging.info("draw_volume2,{},sp={}".format(svc_file, time.time() - st)) # 混合 st = time.time() mix_wav_path = os.path.join(cache_dir, "mix.wav") cmd = "{} {} {} {}".format(gs_simple_mixer_path, volume_path, acc_file, mix_wav_path) os.system(cmd) if not os.path.exists(mix_wav_path): return gs_svci_svc_mix logging.info("mixer,{},sp={}".format(svc_file, time.time() - st)) # 编码为m4a st = time.time() cmd = "ffmpeg -i {} -ab 128k -y {} -loglevel fatal".format(mix_wav_path, mix_path) print(cmd) os.system(cmd) if not os.path.exists(mix_path): return gs_svci_svc_mix logging.info("encode,{},sp={}".format(svc_file, time.time() - st)) return gs_svci_success def params_check(self, model_path, vocal_path, acc_path, video_path): if not os.path.exists(model_path): print("model_path={} is null".format(model_path)) return gs_svci_data_params_check_model_path if not os.path.exists(vocal_path): print("vocal_path={} is null".format(vocal_path)) return gs_svci_data_params_check_vocal_path if not os.path.exists(acc_path): print("acc_path={} is null".format(acc_path)) return gs_svci_data_params_check_acc_path if not os.path.exists(video_path): print("video_path={} is null".format(video_path)) return gs_svci_data_params_check_video_path return gs_svci_success def data_prepare(self, work_dir, vocal_path, acc_path): self.vocal_32_wav_path = os.path.join(work_dir, "vocal_32.wav") self.vocal_wav_path = os.path.join(work_dir, "vocal.wav") self.acc_wav_path = os.path.join(work_dir, "acc.wav") cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {}".format(vocal_path, self.vocal_wav_path) os.system(cmd) cmd = "ffmpeg -i {} -ar 32000 -ac 1 -y {}".format(vocal_path, self.vocal_32_wav_path) os.system(cmd) cmd = "ffmpeg -i {} -ar 44100 -ac 2 -y {}".format(acc_path, self.acc_wav_path) os.system(cmd) return os.path.exists(self.vocal_32_wav_path) and os.path.exists(self.acc_wav_path) def process_logic(self, work_dir, model_path, vocal_path, acc_path, video_path, out_path): # 1. 先转码人声和伴奏 st = time.time() if not self.data_prepare(work_dir, vocal_path, acc_path): print("transcode vocal={} or acc={} err!\n".format(vocal_path, acc_path)) return gs_svci_data_prepare_transcode_media print("transcode vocal and acc sp={}".format(time.time() - st)) # 2. 进行推理 # raw_audio_path, dst_path st = time.time() svc_file = os.path.join(work_dir, "trans_vocal.wav") # try: inf(model_path, self.config, self.vocal_32_wav_path, svc_file, 'prod') # except Exception as ex: # print(ex) if not os.path.exists(svc_file): print("inference err vocal_path={}, model_path={}".format(vocal_path, model_path)) return gs_svci_data_inference, [] print("inf sp={}".format(time.time() - st)) # 3. 生成作品 st = time.time() mix_tmp_path = os.path.join(work_dir, "mix.wav") err = self.mix(work_dir, svc_file, self.vocal_wav_path, self.acc_wav_path, mix_tmp_path) if err != gs_svci_success: return err, [] if not os.path.exists(mix_tmp_path): return gs_svci_svc_mix_gen, [] print("mix sp={}".format(time.time() - st)) st = time.time() # 4. 音频编码，并且和视频合并 cmd = "ffmpeg -i {} -i {} -acodec aac -strict -2 -b:a 128k -vcodec copy -shortest -af apad -y {}".format( video_path, mix_tmp_path, out_path) os.system(cmd) if not os.path.exists(out_path): print("mix audio_video err={}".format(video_path, mix_tmp_path)) return gs_svci_svc_mix_audio_video, [] print("mix audio and video sp={}".format(time.time() - st)) # 5. 提取emb st = time.time() emb = self.spk_emb_inst.process(svc_file) print("get emb sp={}".format(time.time() - st)) return gs_svci_success, emb def process(self, work_dir, model_path, vocal_path, acc_path, video_path, out_path): err = self.params_check(model_path, vocal_path, acc_path, video_path) if err != gs_svci_success: return err, [] if os.path.exists(work_dir): shutil.rmtree(work_dir) os.makedirs(work_dir) st = time.time() err, emb = self.process_logic(work_dir, model_path, vocal_path, acc_path, video_path, out_path) print("process_logic sp={}".format(time.time() - st)) shutil.rmtree(work_dir) return err, emb def test(): svc_inst = SVCInferenceOne() b_dir = "/data/rsync/jianli.yang/AutoCoverTool/data/test_svc_inference_one/" w_dir = os.path.join(b_dir, "rg_input") in_m4a = os.path.join(b_dir, "rg_input.m4a") in_acc_m4a = os.path.join(b_dir, "acc.m4a") in_video = os.path.join(b_dir, "rg.mp4") out_video = os.path.join(b_dir, "rg_input_out.mp4") m_path = "/data/rsync/jianli.yang/AutoCoverTool/data/train_users/jianli/logs/32k/G_2000.pth" err, emb = svc_inst.process(w_dir, m_path, in_m4a, in_acc_m4a, in_video, out_video) print(err) print(emb) if __name__ == '__main__': - if len(sys.argv) != 7: + if len(sys.argv) != 8: print("input err!") print( - "example: work_dir[临时工作目录，内部会自动清除数据] model_path in_media_path in_acc_media_path in_video_path out_video_path") + "example: work_dir[临时工作目录，内部会自动清除数据] model_path in_media_path in_acc_media_path in_video_path out_video_path emb_path") exit(-1) w_dir = sys.argv[1] m_path = sys.argv[2] in_m4a = sys.argv[3] in_acc_m4a = sys.argv[4] in_video = sys.argv[5] out_video = sys.argv[6] + emb_path = sys.argv[7] svc_inst = SVCInferenceOne() err, emb = svc_inst.process(w_dir, m_path, in_m4a, in_acc_m4a, in_video, out_video) - print(err) - print(emb) + with open(emb_path, "w") as f: + f.write(json.dumps({"emb": list(emb.tolist())})) + + # print(err) + # print(emb) diff --git a/AutoCoverTool/svc_inference/webui.py b/AutoCoverTool/svc_inference/webui.py index 42d40fc..9b5de2a 100644 --- a/AutoCoverTool/svc_inference/webui.py +++ b/AutoCoverTool/svc_inference/webui.py @@ -1,101 +1,92 @@ """ 构建唱歌音色转换网页(基于3.0) 要求: 1. 音频上传 2. 推理 3. 下载 """ import os import time import glob import shutil import librosa import soundfile import gradio as gr from online.common import update_db from ref.so_vits_svc.inference_main import * gs_tmp_dir = "/tmp/svc_inference" gs_model_dir = "/data/prod/so_vits_models/3.0" gs_test_wav_dir = "/data/prod/so_vits_models/test_svc_file/3.0" gs_config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.json") def generate_svc_file(): """ :return: """ if not os.path.exists(gs_test_wav_dir): os.makedirs(gs_test_wav_dir) test_wav_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../res/syz_test.wav") model_path_list = glob.glob(os.path.join(gs_model_dir, "*/*pth")) st = time.time() for idx, model_path in enumerate(model_path_list): - model_name = model_path.strip().split("/")[-1].split(".pth", "") + model_name = model_path.strip().split("/")[-1].replace(".pth", "") dst_path = os.path.join(gs_test_wav_dir, "{}.wav".format(model_name)) if not os.path.exists(dst_path): inf(model_path, gs_config_path, test_wav_path, dst_path, "prod") print("now_per={}/{}".format(idx, len(model_path_list), time.time() - st)) def update_state(gender, user_id): sql = "update av_db.av_svc_model set gender={} where user_id=\"{}\"".format(gender, user_id) update_db(sql) # 按钮控制 def click_male(user_id): + print("click_male={}".format(user_id)) pass def click_female(user_id): - pass + print("click_female={}".format(user_id)) def click_delete(user_id): - pass + print("click_delete={}".format(user_id)) def main(): # header st = time.time() generate_svc_file() print("generate svc sp={}".format(time.time() - st)) app = gr.Blocks() with app: # 头部介绍 gr.Markdown(value=""" ### 人声质量评价 作者：starmaker音视频 """) # 列表展示 # 1. 每一行有音频,性别,删除等按钮 svc_files = glob.glob(os.path.join(gs_test_wav_dir, "*wav")) for svc_file in svc_files: user_id = svc_file.split("/")[-1].replace(".wav", "") gr.Audio(source=svc_file) male_gender_btn = gr.Button("male") female_gender_btn = gr.Button("female") del_btn = gr.Button("female") male_gender_btn.click(click_male, inputs=[user_id]) female_gender_btn.click(click_female, inputs=[user_id]) del_btn.click(click_delete, inputs=[user_id]) - - refresh_btn = gr.Button("refresh_model_list") - refresh_btn.click(fn=model_select, inputs=[], outputs=gs_model_list_dropdown) - - # 音频输入框 - input_audio = gr.inputs.Audio(label="input") - vc_transform = gr.Number(label="变调（整数，可以正负，半音数量，升高八度就是12）", value=0) - gen_btn = gr.Button("generate", variant="primary") - output_audio = gr.outputs.Audio(label="output", type='filepath') - gen_btn.click(fn=svc, inputs=[input_audio, gs_model_list_dropdown, vc_transform], outputs=output_audio) # 本方法实现同一时刻只有一个程序在服务器端运行 app.queue(concurrency_count=1, max_size=2044).launch(server_name="0.0.0.0", inbrowser=True, quiet=True, server_port=7860) if __name__ == '__main__': main()